diff --git a/.gitignore b/.gitignore index 41e3565d..b85d1921 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,8 @@ __debug_* test/e2e/inventory* test/e2e/sa-key* test/e2e/kubeconfig* + +# kubetest2 +_artifacts +_artifacts-parallel +_rundir diff --git a/Makefile b/Makefile index 18f94467..a31d9908 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ REPO ?= stackitcloud/cloud-provider-stackit PLATFORMS ?= amd64 arm64 IS_DEV ?= true +E2E_RUN_ID ?= $(or $(PROW_JOB_ID),local-test-e2e) +E2E_PARALLEL_NODES ?= 4 + .PHONY: all all: verify @@ -48,6 +51,8 @@ image-%: $(APKO) $(KO) .PHONY: image-stackit-csi-plugin-test image-stackit-csi-plugin-test: export KO_CONFIG_PATH = test/e2e/.ko-kubetest2.yaml +image-stackit-csi-plugin-test: export IS_DEV = true +image-stackit-csi-plugin-test: export VERSION = $(VERSION)-kubetest2 image-stackit-csi-plugin-test: image-stackit-csi-plugin .PHONY: clean-tools-bin @@ -65,7 +70,7 @@ modules: ## Runs go mod to ensure modules are up to date. .PHONY: test test: ## Run tests. - ./hack/test.sh ./cmd/... ./pkg/... + ./hack/test.sh ./cmd/... ./pkg/... ./test/... .PHONY: test-cover test-cover: ## Run tests with coverage. @@ -119,7 +124,7 @@ verify-e2e-csi: $(KUBERNETES_TEST) -focus=$(FOCUS) \ -skip=$(SKIP) \ $(KUBERNETES_TEST) -- \ - -storage.testdriver=$(PWD)/test/e2e/csi-plugin/block-storage.yaml + -storage.testdriver=$(PWD)/test/csi-plugin/block-storage.yaml verify-image-stackit-csi-plugin: LOCAL = true verify-image-stackit-csi-plugin: APKO_EXTRA_PACKAGES = busybox @@ -127,6 +132,33 @@ verify-image-stackit-csi-plugin: image-stackit-csi-plugin @echo "verifying binaries in image" @docker run -v ./tools/csi-deps-check.sh:/tools/csi-deps-check.sh --entrypoint=/tools/csi-deps-check.sh $(REGISTRY)/$(REPO)/stackit-csi-plugin-dev:$(VERSION) +.PHONY: test-e2e +test-e2e: image-stackit-csi-plugin-test + go run ./test \ + --up \ + --test=ginkgo \ + --run-id "$(E2E_RUN_ID)" \ + --rundir "_rundir" \ + --region "eu01" \ + --kubernetes-version "$(E2E_K8S_VERSION)" \ + --availability-zone "eu01-1" \ + --machine-type "$(E2E_MACHINE_TYPE)" \ + --node-image-name "$(E2E_IMAGE_NAME)" \ + --node-image-version "$(E2E_IMAGE_VERSION)" \ + --nodepool-name "default" \ + --volume-size 100 \ + --kubeconfig-expiration-seconds 3600 \ + --csi-image-name "$(REGISTRY)/$(REPO)/stackit-csi-plugin-dev" \ + --csi-image-tag "$(VERSION)-kubetest2" \ + "$@" \ + -- \ + --test-package-version="v$(E2E_K8S_VERSION)" \ + --focus-regex="External.Storage" \ + --skip-regex="\[Feature:|\[Disruptive\]|\[Serial\]" \ + --ginkgo-args="-v" \ + --parallel="$(E2E_PARALLEL_NODES)" \ + --test-args="--storage.testdriver=$(PWD)/_rundir/$(E2E_RUN_ID)/csi-testdriver.yaml" + # generate mock types for the following services (space-separated list) .PHONY: mocks mocks: $(MOCKGEN) diff --git a/docs/testing.md b/docs/testing.md index 97000ba2..550e7ade 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -236,7 +236,7 @@ You can apply this manifest using `kubectl apply -f ` or ensuring The test configuration file allows for granular control over which specific tests are executed. This is useful for debugging or targeting a subset of features. -- **Configuration File:** `test/e2e/csi/block-storage.yaml` +- **Configuration File:** `test/csi-plugin/block-storage.yaml` - **Action:** Modify this YAML file to include or exclude specific test cases, adjust parameters, or change timeout settings for the E2E runs. This allows for focused testing without running the entire suite. ### Full Example diff --git a/go.mod b/go.mod index 90348018..29b29be8 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,13 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stackitcloud/stackit-sdk-go/core v0.26.0 + github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1 github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 + github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 + github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 + github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 + github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -31,6 +36,9 @@ require ( k8s.io/klog/v2 v2.140.0 k8s.io/mount-utils v0.36.3 k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 + sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 + sigs.k8s.io/kustomize/api v0.19.0 + sigs.k8s.io/kustomize/kyaml v0.19.0 ) replace k8s.io/cloud-provider => github.com/stackitcloud/cloud-provider v0.36.0-ske-2 @@ -52,6 +60,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.22.1 // indirect @@ -76,24 +85,27 @@ require ( github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect go.etcd.io/etcd/api/v3 v3.6.8 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.8 // indirect go.etcd.io/etcd/client/v3 v3.6.8 // indirect diff --git a/go.sum b/go.sum index 85b2cbc1..fc8fd7b9 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -105,6 +107,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= @@ -123,6 +127,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= @@ -151,6 +157,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= @@ -171,6 +179,8 @@ github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIj github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= @@ -184,12 +194,20 @@ github.com/stackitcloud/cloud-provider v0.36.0-ske-2 h1:OCDooD0twsMDdcGbcEEofRa6 github.com/stackitcloud/cloud-provider v0.36.0-ske-2/go.mod h1:y/3sksoC0taJZR0PcAAYUqVyD6Jzu2X0lD4yCEPXPuI= github.com/stackitcloud/stackit-sdk-go/core v0.26.0 h1:jQEb9gkehfp6VCP6TcYk7BI10cz4l0KM2L6hqYBH2QA= github.com/stackitcloud/stackit-sdk-go/core v0.26.0/go.mod h1:WU1hhxnjXw2EV7CYa1nlEvNpMiRY6CvmIOaHuL3pOaA= +github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1 h1:TtpVXxgCJ17AqSOIzooQru/H3QnC4Z2VIF7HbaeKeJs= +github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1/go.mod h1:c9JimEdshtcIHAI2j2N388ybW+InO1fvYPvL6rxb2uY= github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 h1:P8XhOi8TLA7NbLHhxU8dCJjgQpha/65agos3pI59+vw= github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1/go.mod h1:/QpOsaCOQjwGgQqG8y9xXqryp25AfkorPjInX9xIeOU= github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 h1:+NT2okD80TlFNDTKSgaEvV/o7LevTrATqcDL0r4nuhU= github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1/go.mod h1:+Ld3dn648I+YKcBV3fEkYpDSr3fel421+LurJGywSBs= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 h1:4UxxJmCSCwV8q4bT4G+D1JH8F9Gm6BKaLGixX5DVcvI= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2/go.mod h1:NEz3f+GV5G++BE9/MmZCsXJyCih7jtg0pZuSyG2sLEs= +github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 h1:zVVA/Puge2PK3G8lrGGolnQzxYwwsNUgfMyyRUUQc44= +github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1/go.mod h1:xJNa86o89czK2o60+s46vz8+NSKDTbQduFGFaztJnsA= +github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 h1:TNZHrunhsXRbuqZcucLs2Gqy1sEyvabufM7pB5Tscmo= +github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0/go.mod h1:fXq3TmVLb4JMSve989NFFViMFoYa83s7M3hJWgN6mdQ= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 h1:Dab1jzN0u9c67lvELoWf1RuagjO3eUBRytoX8SYL8Zs= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0/go.mod h1:NzcTU5GGlUF6Lys3Ra7ylRj4ZKxJr3f/29/yoE5tjPI= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -198,6 +216,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -217,6 +236,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= @@ -369,6 +390,12 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 h1:ttox7lK6ZgN/goexDpOxStOl1Q6j4CwprvK+rkS0c1M= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1/go.mod h1:1IMaKFT+Htrbn6ixEZq6GTCk13CP/S0vwujFC5qs55U= +sigs.k8s.io/kustomize/api v0.19.0 h1:F+2HB2mU1MSiR9Hp1NEgoU2q9ItNOaBJl0I4Dlus5SQ= +sigs.k8s.io/kustomize/api v0.19.0/go.mod h1:/BbwnivGVcBh1r+8m3tH1VNxJmHSk1PzP5fkP6lbL1o= +sigs.k8s.io/kustomize/kyaml v0.19.0 h1:RFge5qsO1uHhwJsu3ipV7RNolC7Uozc0jUBC/61XSlA= +sigs.k8s.io/kustomize/kyaml v0.19.0/go.mod h1:FeKD5jEOH+FbZPpqUghBP8mrLjJ3+zD3/rf9NNu1cwY= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= diff --git a/test/kubetest2.go b/test/kubetest2.go new file mode 100644 index 00000000..a8e9be55 --- /dev/null +++ b/test/kubetest2.go @@ -0,0 +1,11 @@ +package main + +import ( + "sigs.k8s.io/kubetest2/pkg/app" + + kubetest2stackit "github.com/stackitcloud/cloud-provider-stackit/test/kubetest2" +) + +func main() { + app.Main(kubetest2stackit.Name, kubetest2stackit.New) +} diff --git a/test/kubetest2/bootstrap_csi.go b/test/kubetest2/bootstrap_csi.go new file mode 100644 index 00000000..82da08d3 --- /dev/null +++ b/test/kubetest2/bootstrap_csi.go @@ -0,0 +1,545 @@ +package kubetest2 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "gopkg.in/yaml.v3" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/discovery" + memory "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + "sigs.k8s.io/kustomize/api/krusty" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +type csiApplier func(ctx context.Context, kubeconfigPath, stagedKustomizeDir string) error + +const ( + csiFieldManager = "kubetest2-stackit" + csiPollInterval = 2 * time.Second + csiRolloutTimeout = 5 * time.Minute + + csiManifestsPath = "test/kustomize" + baseManifestsPath = "deploy/csi-plugin" + csiTestDriverTemplatePath = "test/csi-plugin/block-storage.yaml" + csiTestDriverFileName = "csi-testdriver.yaml" + + kubetest2CSIStorageClassName = "premium-perf4-stackit-kubetest2" + kubetest2CSIDriverName = "kubetest2.csi.stackit.cloud" + + csiConfigMapName = "stackit-cloud-config" + csiConfigMapKey = "cloud.yaml" + csiConfigNamespace = "kube-system" +) + +// ensureCSI deploys the STACKIT CSI driver into the freshly provisioned +// cluster as the last bootstrap step. It stages the Kustomize overlay and base +// manifests into the run directory, applies the necessary image, project, region, +// and credential substitutions, and applies the manifests to the cluster using +// native Go Kubernetes clients. +func (d *Deployer) ensureCSI(ctx context.Context) error { + klog.Infof("Starting CSI deployment into cluster=%q", d.clusterName()) + + serviceAccountKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if !ok || strings.TrimSpace(serviceAccountKey) == "" { + return fmt.Errorf("cached child service-account key is missing or empty at %q", d.serviceAccountKeyPath) + } + + stagedKustomizeDir, err := d.stageCSIManifests(serviceAccountKey) + if err != nil { + return fmt.Errorf("stage CSI manifests: %w", err) + } + if _, err := d.stageCSITestDriver(); err != nil { + return fmt.Errorf("stage CSI testdriver: %w", err) + } + + applier := d.csiApplier + if applier == nil { + applier = applyCSIManifestsNative + } + + if err := applier(ctx, d.kubeconfigPath, stagedKustomizeDir); err != nil { + return fmt.Errorf("apply CSI manifests: %w", err) + } + + klog.Infof("CSI deployment completed successfully for cluster=%q", d.clusterName()) + return nil +} + +func applyCSIManifestsNative(ctx context.Context, kubeconfigPath, stagedKustomizeDir string) error { + objects, err := renderKustomize(stagedKustomizeDir) + if err != nil { + return fmt.Errorf("render Kustomize overlay %q: %w", stagedKustomizeDir, err) + } + + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return fmt.Errorf("build rest config from %q: %w", kubeconfigPath, err) + } + + dynClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create dynamic client: %w", err) + } + + discoClient, err := discovery.NewDiscoveryClientForConfig(restConfig) + if err != nil { + return fmt.Errorf("create discovery client: %w", err) + } + + cachedDisco := memory.NewMemCacheClient(discoClient) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDisco) + + klog.Infof("Applying %d rendered Kubernetes objects via server-side apply", len(objects)) + if err := applyUnstructuredObjects(ctx, dynClient, mapper, objects); err != nil { + return fmt.Errorf("apply Kubernetes objects: %w", err) + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create kubernetes clientset: %w", err) + } + + klog.Infof("Waiting for CSI controller deployment rollout") + if err := waitForDeploymentReady(ctx, clientset, "kube-system", "csi-stackit-controllerplugin", csiRolloutTimeout); err != nil { + return fmt.Errorf("wait for CSI controller deployment rollout: %w", err) + } + + klog.Infof("Waiting for CSI node daemonset rollout") + if err := waitForDaemonSetReady(ctx, clientset, "kube-system", "csi-stackit-nodeplugin", csiRolloutTimeout); err != nil { + return fmt.Errorf("wait for CSI node daemonset rollout: %w", err) + } + + return nil +} + +func renderKustomize(dir string) ([]*unstructured.Unstructured, error) { + k := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) + fSys := filesys.MakeFsOnDisk() + + resMap, err := k.Run(fSys, dir) + if err != nil { + return nil, fmt.Errorf("kustomize run: %w", err) + } + + yamlBytes, err := resMap.AsYaml() + if err != nil { + return nil, fmt.Errorf("kustomize as yaml: %w", err) + } + + return decodeYAMLToUnstructured(yamlBytes) +} + +func decodeYAMLToUnstructured(yamlBytes []byte) ([]*unstructured.Unstructured, error) { + decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(yamlBytes), 4096) + var objects []*unstructured.Unstructured + + for { + var rawObj map[string]interface{} + err := decoder.Decode(&rawObj) + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("decode YAML object: %w", err) + } + if len(rawObj) == 0 { + continue + } + + u := &unstructured.Unstructured{Object: rawObj} + if u.GetKind() == "" || u.GetName() == "" { + continue + } + objects = append(objects, u) + } + + return objects, nil +} + +func applyUnstructuredObjects(ctx context.Context, dynClient dynamic.Interface, mapper meta.RESTMapper, objects []*unstructured.Unstructured) error { + for _, obj := range objects { + gvk := obj.GroupVersionKind() + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return fmt.Errorf("get REST mapping for %s: %w", gvk.String(), err) + } + + var ri dynamic.ResourceInterface + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + ns := obj.GetNamespace() + if ns == "" { + ns = metav1.NamespaceDefault + } + ri = dynClient.Resource(mapping.Resource).Namespace(ns) + } else { + ri = dynClient.Resource(mapping.Resource) + } + + objJSON, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("marshal object %s/%s to JSON: %w", gvk.Kind, obj.GetName(), err) + } + + force := true + applyOpts := metav1.PatchOptions{ + FieldManager: csiFieldManager, + Force: &force, + } + + klog.Infof("Server-side applying %s %s/%s", gvk.Kind, obj.GetNamespace(), obj.GetName()) + if _, err := ri.Patch(ctx, obj.GetName(), types.ApplyPatchType, objJSON, applyOpts); err != nil { + return fmt.Errorf("server-side apply %s %s/%s: %w", gvk.Kind, obj.GetNamespace(), obj.GetName(), err) + } + } + return nil +} + +func waitForDeploymentReady(ctx context.Context, clientset kubernetes.Interface, namespace, name string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, csiPollInterval, timeout, true, func(ctx context.Context) (bool, error) { + deploy, err := clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + specReplicas := int32(1) + if deploy.Spec.Replicas != nil { + specReplicas = *deploy.Spec.Replicas + } + + if deploy.Status.ObservedGeneration >= deploy.Generation && + deploy.Status.UpdatedReplicas == specReplicas && + deploy.Status.AvailableReplicas == specReplicas && + deploy.Status.UnavailableReplicas == 0 { + return true, nil + } + return false, nil + }) +} + +func waitForDaemonSetReady(ctx context.Context, clientset kubernetes.Interface, namespace, name string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, csiPollInterval, timeout, true, func(ctx context.Context) (bool, error) { + ds, err := clientset.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + if ds.Status.ObservedGeneration >= ds.Generation && + ds.Status.DesiredNumberScheduled > 0 && + ds.Status.UpdatedNumberScheduled == ds.Status.DesiredNumberScheduled && + ds.Status.NumberAvailable == ds.Status.DesiredNumberScheduled && + ds.Status.NumberUnavailable == 0 { + return true, nil + } + return false, nil + }) +} + +func resolveManifestPath(relPath string) (string, error) { + candidates := []string{ + relPath, + filepath.Join("..", "..", relPath), + } + + for _, candidate := range candidates { + if _, err := os.Stat(candidate); err == nil { + return filepath.Abs(candidate) + } + } + + return "", fmt.Errorf("manifest path %q not found (tried %v)", relPath, candidates) +} + +func (d *Deployer) stageCSIManifests(serviceAccountKey string) (string, error) { + stagingRoot := filepath.Join(d.options.RunDir(), "manifests") + + stagedBaseDir := filepath.Join(stagingRoot, "deploy", "csi-plugin") + stagedKustomizeDir := filepath.Join(stagingRoot, "test", "kustomize") + + baseSrc, err := resolveManifestPath(baseManifestsPath) + if err != nil { + return "", fmt.Errorf("resolve base manifests path: %w", err) + } + + csiSrc, err := resolveManifestPath(csiManifestsPath) + if err != nil { + return "", fmt.Errorf("resolve CSI overlay manifests path: %w", err) + } + + if err := os.RemoveAll(stagingRoot); err != nil { + return "", fmt.Errorf("clean existing manifests staging dir %q: %w", stagingRoot, err) + } + + if err := copyDir(baseSrc, stagedBaseDir); err != nil { + return "", fmt.Errorf("copy base CSI manifests from %q to %q: %w", baseSrc, stagedBaseDir, err) + } + + if err := copyDir(csiSrc, stagedKustomizeDir); err != nil { + return "", fmt.Errorf("copy CSI overlay manifests from %q to %q: %w", csiSrc, stagedKustomizeDir, err) + } + + // 1. Replace image and tag in kustomization.yaml + kustomizationPath := filepath.Join(stagedKustomizeDir, "kustomization.yaml") + if err := replaceInFile(kustomizationPath, map[string]string{ + "REPLACE_WITH_IMAGE_NAME": d.csiImageName, + "REPLACE_WITH_TAG": d.csiImageTag, + }); err != nil { + return "", fmt.Errorf("substitute values in %q: %w", kustomizationPath, err) + } + + cloudConfigPath := filepath.Join(stagedKustomizeDir, "cloud-config.yaml") + cloudConfigContent, err := renderCSICloudConfigManifest(d.projectID, d.region, d.iaasEndpoint) + if err != nil { + return "", fmt.Errorf("render CSI cloud config manifest: %w", err) + } + if err := os.WriteFile(cloudConfigPath, cloudConfigContent, 0o600); err != nil { + return "", fmt.Errorf("write CSI cloud config manifest %q: %w", cloudConfigPath, err) + } + + cloudSecretPath := filepath.Join(stagedKustomizeDir, "cloud-secret.yaml") + if err := replaceInFile(cloudSecretPath, map[string]string{ + "REPLACE_WITH_SERVICEACCOUNT_JSON": serviceAccountKey, + }); err != nil { + return "", fmt.Errorf("substitute values in %q: %w", cloudSecretPath, err) + } + + return stagedKustomizeDir, nil +} + +func (d *Deployer) stageCSITestDriver() (string, error) { + templatePath, err := resolveManifestPath(csiTestDriverTemplatePath) + if err != nil { + return "", fmt.Errorf("resolve CSI testdriver template path: %w", err) + } + + content, err := renderCSITestDriverConfig(templatePath) + if err != nil { + return "", err + } + + outputPath := filepath.Join(d.options.RunDir(), csiTestDriverFileName) + if err := os.WriteFile(outputPath, content, 0o600); err != nil { + return "", fmt.Errorf("write CSI testdriver config %q: %w", outputPath, err) + } + + return outputPath, nil +} + +func renderCSITestDriverConfig(templatePath string) ([]byte, error) { + content, err := os.ReadFile(templatePath) + if err != nil { + return nil, fmt.Errorf("read CSI testdriver template %q: %w", templatePath, err) + } + + var config csiTestDriverConfig + if err := yaml.Unmarshal(content, &config); err != nil { + return nil, fmt.Errorf("decode CSI testdriver template %q: %w", templatePath, err) + } + + config.StorageClass.FromExistingClassName = kubetest2CSIStorageClassName + config.DriverInfo.Name = kubetest2CSIDriverName + + rendered, err := yaml.Marshal(&config) + if err != nil { + return nil, fmt.Errorf("encode CSI testdriver config: %w", err) + } + + return rendered, nil +} + +type csiTestDriverConfig struct { + StorageClass csiTestClassConfig `yaml:"StorageClass"` + SnapshotClass csiTestClassConfig `yaml:"SnapshotClass"` + DriverInfo csiTestDriverInfo `yaml:"DriverInfo"` +} + +type csiTestClassConfig struct { + FromExistingClassName string `yaml:"FromExistingClassName"` +} + +type csiTestDriverInfo struct { + Name string `yaml:"Name"` + SupportedFsType map[string]map[string]any `yaml:"SupportedFsType,omitempty"` + Capabilities map[string]bool `yaml:"Capabilities,omitempty"` +} + +type csiCloudConfigManifest struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata csiCloudConfigMetadata `yaml:"metadata"` + Data map[string]string `yaml:"data"` +} + +type csiCloudConfigMetadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` +} + +type csiCloudConfig struct { + Global csiCloudConfigGlobal `yaml:"global"` + BlockStorage csiCloudConfigBlockStorage `yaml:"blockStorage"` +} + +type csiCloudConfigGlobal struct { + ProjectID string `yaml:"projectId"` + Region string `yaml:"region"` + APIEndpoints *csiCloudConfigAPIEndpoints `yaml:"apiEndpoints,omitempty"` +} + +type csiCloudConfigAPIEndpoints struct { + IaaSAPI string `yaml:"iaasApi"` +} + +type csiCloudConfigBlockStorage struct { + RescanOnResize bool `yaml:"rescanOnResize"` +} + +func renderCSICloudConfigManifest(projectID, region, iaasEndpoint string) ([]byte, error) { + cloudConfig := csiCloudConfig{ + Global: csiCloudConfigGlobal{ + ProjectID: projectID, + Region: region, + }, + BlockStorage: csiCloudConfigBlockStorage{ + RescanOnResize: true, + }, + } + if iaasEndpoint != "" { + cloudConfig.Global.APIEndpoints = &csiCloudConfigAPIEndpoints{ + IaaSAPI: iaasEndpoint, + } + } + + cloudConfigBytes, err := yaml.Marshal(&cloudConfig) + if err != nil { + return nil, fmt.Errorf("encode CSI cloud config: %w", err) + } + + manifest := csiCloudConfigManifest{ + APIVersion: "v1", + Kind: "ConfigMap", + Metadata: csiCloudConfigMetadata{ + Name: csiConfigMapName, + Namespace: csiConfigNamespace, + }, + Data: map[string]string{ + csiConfigMapKey: string(cloudConfigBytes), + }, + } + + rendered, err := yaml.Marshal(&manifest) + if err != nil { + return nil, fmt.Errorf("encode CSI cloud config manifest: %w", err) + } + + return rendered, nil +} + +func replaceInFile(path string, replacements map[string]string) error { + contentBytes, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read file %q: %w", path, err) + } + + content := string(contentBytes) + for oldVal, newVal := range replacements { + content = strings.ReplaceAll(content, oldVal, newVal) + } + + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + return fmt.Errorf("write file %q: %w", path, err) + } + return nil +} + +func copyDir(src, dst string) error { + srcInfo, err := os.Stat(src) + if err != nil { + return err + } + + if err := os.MkdirAll(dst, srcInfo.Mode()); err != nil { + return err + } + + return filepath.WalkDir(src, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + if relPath == "." { + return nil + } + + targetPath := filepath.Join(dst, relPath) + if d.IsDir() { + dirInfo, err := d.Info() + if err != nil { + return err + } + return os.MkdirAll(targetPath, dirInfo.Mode()) + } + + return copyFile(path, targetPath) + }) +} + +func copyFile(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + srcInfo, err := srcFile.Stat() + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + + dstFile, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, srcInfo.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + if _, err := io.Copy(dstFile, srcFile); err != nil { + return err + } + return nil +} diff --git a/test/kubetest2/bootstrap_csi_test.go b/test/kubetest2/bootstrap_csi_test.go new file mode 100644 index 00000000..82d9b82e --- /dev/null +++ b/test/kubetest2/bootstrap_csi_test.go @@ -0,0 +1,235 @@ +package kubetest2 + +import ( + "context" + "os" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +var _ = Describe("ensureCSI", func() { + It("stages manifests with rendered cloud config and calls csiApplier", func() { + d := newTestDeployer() + configureValidUpInputs(d) + d.projectID = "project-123" + d.iaasEndpoint = "https://iaas.api.qa.stackit.cloud" + + serviceAccountKey := `{"credentials":{"privateKey":"dummy-private-key"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600)).To(Succeed()) + + var recordedKubeconfig string + var recordedStagedDir string + d.csiApplier = func(_ context.Context, kubeconfigPath, stagedKustomizeDir string) error { + recordedKubeconfig = kubeconfigPath + recordedStagedDir = stagedKustomizeDir + return nil + } + + Expect(d.ensureCSI(context.Background())).To(Succeed()) + + Expect(recordedKubeconfig).To(Equal(d.kubeconfigPath)) + Expect(recordedStagedDir).To(Equal(filepath.Join(d.options.RunDir(), "manifests", "test", "kustomize"))) + + // Verify substitutions in staged files + stagedKustomizeDir := filepath.Join(d.options.RunDir(), "manifests", "test", "kustomize") + + kustomizationBytes, err := os.ReadFile(filepath.Join(stagedKustomizeDir, "kustomization.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(kustomizationBytes)).To(ContainSubstring("newName: " + d.csiImageName)) + Expect(string(kustomizationBytes)).To(ContainSubstring("newTag: " + d.csiImageTag)) + Expect(string(kustomizationBytes)).NotTo(ContainSubstring("REPLACE_WITH_IMAGE_NAME")) + Expect(string(kustomizationBytes)).NotTo(ContainSubstring("REPLACE_WITH_TAG")) + + cloudConfigBytes, err := os.ReadFile(filepath.Join(stagedKustomizeDir, "cloud-config.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(cloudConfigBytes)).To(ContainSubstring("projectId: project-123")) + Expect(string(cloudConfigBytes)).To(ContainSubstring("region: " + d.region)) + Expect(string(cloudConfigBytes)).To(ContainSubstring("apiEndpoints:")) + Expect(string(cloudConfigBytes)).To(ContainSubstring("iaasApi: " + d.iaasEndpoint)) + Expect(string(cloudConfigBytes)).To(ContainSubstring("name: " + csiConfigMapName)) + Expect(string(cloudConfigBytes)).To(ContainSubstring("namespace: " + csiConfigNamespace)) + Expect(string(cloudConfigBytes)).NotTo(ContainSubstring("REPLACE_WITH_PROJECTID")) + Expect(string(cloudConfigBytes)).NotTo(ContainSubstring("REPLACE_WITH_OPTIONAL_IAAS_API")) + + cloudSecretBytes, err := os.ReadFile(filepath.Join(stagedKustomizeDir, "cloud-secret.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(cloudSecretBytes)).To(ContainSubstring(serviceAccountKey)) + Expect(string(cloudSecretBytes)).NotTo(ContainSubstring("REPLACE_WITH_SERVICEACCOUNT_JSON")) + + testDriverBytes, err := os.ReadFile(filepath.Join(d.options.RunDir(), csiTestDriverFileName)) + Expect(err).NotTo(HaveOccurred()) + Expect(string(testDriverBytes)).To(ContainSubstring("FromExistingClassName: " + kubetest2CSIStorageClassName)) + Expect(string(testDriverBytes)).To(ContainSubstring("Name: " + kubetest2CSIDriverName)) + Expect(string(testDriverBytes)).NotTo(ContainSubstring("FromExistingClassName: premium-perf4-stackit\n")) + Expect(string(testDriverBytes)).NotTo(ContainSubstring("Name: block-storage.csi.stackit.cloud")) + + // Verify that in-process renderKustomize parses the staged overlay correctly into unstructured objects + objects, err := renderKustomize(stagedKustomizeDir) + Expect(err).NotTo(HaveOccurred()) + Expect(objects).NotTo(BeEmpty()) + + var foundControllerPlugin, foundNodePlugin bool + for _, obj := range objects { + if obj.GetKind() == "Deployment" && obj.GetName() == "csi-stackit-controllerplugin" { + foundControllerPlugin = true + } + if obj.GetKind() == "DaemonSet" && obj.GetName() == "csi-stackit-nodeplugin" { + foundNodePlugin = true + } + } + Expect(foundControllerPlugin).To(BeTrue(), "expected csi-stackit-controllerplugin in rendered objects") + Expect(foundNodePlugin).To(BeTrue(), "expected csi-stackit-nodeplugin in rendered objects") + }) + + It("does not write iaas endpoint override when not set", func() { + d := newTestDeployer() + configureValidUpInputs(d) + d.projectID = "project-123" + + serviceAccountKey := `{"credentials":{"privateKey":"dummy-private-key"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600)).To(Succeed()) + + d.csiApplier = func(_ context.Context, _, _ string) error { + return nil + } + + Expect(d.ensureCSI(context.Background())).To(Succeed()) + + cloudConfigBytes, err := os.ReadFile(filepath.Join(d.options.RunDir(), "manifests", "test", "kustomize", "cloud-config.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(cloudConfigBytes)).NotTo(ContainSubstring("apiEndpoints:")) + Expect(string(cloudConfigBytes)).NotTo(ContainSubstring("iaasApi:")) + cloudSecretBytes, err := os.ReadFile(filepath.Join(d.options.RunDir(), "manifests", "test", "kustomize", "cloud-secret.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(cloudSecretBytes)).To(ContainSubstring(serviceAccountKey)) + }) + + It("fails when cached service account key is missing", func() { + d := newTestDeployer() + configureValidUpInputs(d) + d.projectID = "project-123" + + _ = os.Remove(d.serviceAccountKeyPath) + + err := d.ensureCSI(context.Background()) + Expect(err).To(MatchError(ContainSubstring("cached child service-account key is missing or empty"))) + }) + + It("fails when csiApplier returns an error", func() { + d := newTestDeployer() + configureValidUpInputs(d) + d.projectID = "project-123" + + serviceAccountKey := `{"credentials":{"privateKey":"dummy"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600)).To(Succeed()) + + d.csiApplier = func(_ context.Context, _, _ string) error { + return os.ErrInvalid + } + + err := d.ensureCSI(context.Background()) + Expect(err).To(MatchError(ContainSubstring("apply CSI manifests"))) + }) +}) + +var _ = Describe("renderCSICloudConfigManifest", func() { + It("renders the optional iaas endpoint only when configured", func() { + rendered, err := renderCSICloudConfigManifest("project-123", "eu01", "https://iaas.example.com") + Expect(err).NotTo(HaveOccurred()) + + text := string(rendered) + Expect(text).To(ContainSubstring("name: " + csiConfigMapName)) + Expect(text).To(ContainSubstring("namespace: " + csiConfigNamespace)) + Expect(text).To(ContainSubstring("projectId: project-123")) + Expect(text).To(ContainSubstring("region: eu01")) + Expect(text).To(ContainSubstring("apiEndpoints:")) + Expect(text).To(ContainSubstring("iaasApi: https://iaas.example.com")) + Expect(text).To(ContainSubstring("rescanOnResize: true")) + }) + + It("omits apiEndpoints when no iaas endpoint override is configured", func() { + rendered, err := renderCSICloudConfigManifest("project-123", "eu01", "") + Expect(err).NotTo(HaveOccurred()) + + text := string(rendered) + Expect(text).To(ContainSubstring("projectId: project-123")) + Expect(text).To(ContainSubstring("region: eu01")) + Expect(text).NotTo(ContainSubstring("apiEndpoints:")) + Expect(text).NotTo(ContainSubstring("iaasApi:")) + }) +}) + +var _ = Describe("renderCSITestDriverConfig", func() { + It("rewrites the checked-in testdriver for the kubetest2 CSI overlay", func() { + templatePath, err := resolveManifestPath(csiTestDriverTemplatePath) + Expect(err).NotTo(HaveOccurred()) + + rendered, err := renderCSITestDriverConfig(templatePath) + Expect(err).NotTo(HaveOccurred()) + + text := string(rendered) + Expect(text).To(ContainSubstring("FromExistingClassName: " + kubetest2CSIStorageClassName)) + Expect(text).To(ContainSubstring("Name: " + kubetest2CSIDriverName)) + Expect(text).To(ContainSubstring("SnapshotClass:")) + Expect(text).To(ContainSubstring("SupportedFsType:")) + Expect(text).To(ContainSubstring("controllerExpansion: true")) + Expect(strings.Count(text, kubetest2CSIStorageClassName)).To(Equal(1)) + }) +}) + +var _ = Describe("waitForDeploymentReady", func() { + It("succeeds when deployment is ready", func() { + replicas := int32(1) + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csi-stackit-controllerplugin", + Namespace: "kube-system", + Generation: 1, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + }, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + UnavailableReplicas: 0, + }, + } + + fakeClientset := fake.NewClientset(deploy) + err := waitForDeploymentReady(context.Background(), fakeClientset, "kube-system", "csi-stackit-controllerplugin", timeoutDuration) + Expect(err).NotTo(HaveOccurred()) + }) +}) + +var _ = Describe("waitForDaemonSetReady", func() { + It("succeeds when daemonset is ready", func() { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csi-stackit-nodeplugin", + Namespace: "kube-system", + Generation: 1, + }, + Status: appsv1.DaemonSetStatus{ + ObservedGeneration: 1, + DesiredNumberScheduled: 2, + UpdatedNumberScheduled: 2, + NumberAvailable: 2, + NumberUnavailable: 0, + }, + } + + fakeClientset := fake.NewClientset(ds) + err := waitForDaemonSetReady(context.Background(), fakeClientset, "kube-system", "csi-stackit-nodeplugin", timeoutDuration) + Expect(err).NotTo(HaveOccurred()) + }) +}) + +const timeoutDuration = 2 * csiPollInterval diff --git a/test/kubetest2/bootstrap_project.go b/test/kubetest2/bootstrap_project.go new file mode 100644 index 00000000..cf3a568e --- /dev/null +++ b/test/kubetest2/bootstrap_project.go @@ -0,0 +1,146 @@ +package kubetest2 + +import ( + "context" + "fmt" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "k8s.io/klog/v2" +) + +type managedProject struct { + ContainerID string + ProjectID string + Name string + Labels map[string]string +} + +// ensureProject idempotently resolves (or creates) the managed STACKIT project +// and enables the SKE service for it. Sets d.projectID on success. +func (d *Deployer) ensureProject(ctx context.Context) error { + project, err := d.resolveManagedProject(ctx) + if err != nil { + return err + } + d.projectID = project.ProjectID + + return d.ensureSKEServiceEnabled(ctx, project.ProjectID) +} + +func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { + projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) + if err != nil { + return nil, fmt.Errorf("list STACKIT projects under parent container %q: %w", d.parentContainerID, err) + } + + matches := make([]managedProject, 0, 1) + for i := range projects { + project := &projects[i] + if !d.matchesManagedProject(project) { + continue + } + matches = append(matches, managedProject{ + ContainerID: project.GetContainerId(), + ProjectID: project.GetProjectId(), + Name: project.GetName(), + Labels: project.GetLabels(), + }) + } + + switch len(matches) { + case 0: + return nil, nil + case 1: + return &matches[0], nil + default: + return nil, fmt.Errorf( + "found %d managed STACKIT projects for run token %q under parent container %q", + len(matches), + d.runToken(), + d.parentContainerID, + ) + } +} + +func (d *Deployer) resolveManagedProject(ctx context.Context) (*managedProject, error) { + project, err := d.findManagedProject(ctx) + if err != nil { + return nil, err + } + if project != nil { + klog.Infof("Reusing managed project=%q project_id=%q", project.Name, project.ProjectID) + return project, nil + } + + klog.Infof("Creating managed project=%q under parent_container_id=%q", d.projectName(), d.parentContainerID) + createdProject, err := d.projectClient.CreateProject( + ctx, + d.parentContainerID, + d.projectName(), + d.projectMemberEmail, + d.managedProjectLabels(), + ) + if err != nil { + return nil, fmt.Errorf("create STACKIT project %q: %w", d.projectName(), err) + } + + activeProject, err := d.projectClient.WaitForProjectActive(ctx, createdProject.GetContainerId()) + if err != nil { + return nil, fmt.Errorf("wait for STACKIT project %q to become active: %w", createdProject.GetProjectId(), err) + } + + return &managedProject{ + ContainerID: activeProject.GetContainerId(), + ProjectID: activeProject.GetProjectId(), + Name: activeProject.GetName(), + Labels: activeProject.GetLabels(), + }, nil +} + +func (d *Deployer) managedProjectLabels() map[string]string { + return map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: d.runToken(), + } +} + +func (d *Deployer) matchesManagedProject(project *resourcemanager.Project) bool { + if project.GetName() != d.projectName() { + return false + } + labels := project.GetLabels() + if labels == nil { + return false + } + return labels[projectLabelScopeKey] == projectLabelScopeValue && + labels[projectLabelManagedKey] == projectLabelManagedValue && + labels[projectLabelRunIDKey] == d.runToken() +} + +// ensureSKEServiceEnabled idempotently enables the SKE (Kubernetes Engine) +// service for the managed project and waits until it is enabled. +func (d *Deployer) ensureSKEServiceEnabled(ctx context.Context, projectID string) error { + status, err := d.serviceEnablementClient.GetServiceStatus(ctx, d.region, projectID, skeServiceID) + if err != nil { + if !stackiterrors.IsNotFound(err) { + return fmt.Errorf("get SKE service status for project %q: %w", projectID, err) + } + klog.Infof("SKE service not yet enabled for project_id=%q", projectID) + } else if status.GetState() == serviceenablement.SERVICESTATUSSTATE_ENABLED { + klog.Infof("SKE service already enabled for project_id=%q", projectID) + return nil + } else { + klog.Infof("SKE service in state %q for project_id=%q, enabling", status.GetState(), projectID) + } + + if err := d.serviceEnablementClient.EnableService(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("enable SKE service for project %q: %w", projectID, err) + } + if err := d.serviceEnablementClient.WaitForServiceEnabled(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("wait for SKE service enablement for project %q: %w", projectID, err) + } + return nil +} diff --git a/test/kubetest2/bootstrap_project_test.go b/test/kubetest2/bootstrap_project_test.go new file mode 100644 index 00000000..901a6aa9 --- /dev/null +++ b/test/kubetest2/bootstrap_project_test.go @@ -0,0 +1,157 @@ +package kubetest2 + +import ( + "context" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" +) + +var _ = Describe("matchesManagedProject", func() { + It("matches a project with the expected name and labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeTrue()) + }) + + It("rejects a project with a different name", func() { + d := newTestDeployer() + project := projectFixture("other-name", "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + It("rejects a project without labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", nil) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + DescribeTable("rejects a project with mismatched labels", + func(labels map[string]string) { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", labels) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }, + Entry("wrong scope", map[string]string{ + projectLabelScopeKey: "PRIVATE", + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("missing managed label", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("wrong run id", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: "deadbeef", + }), + ) +}) + +var _ = Describe("resolveManagedProject", func() { + It("creates a project when missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{ + createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + } + d.projectClient = projectClient + + project, err := d.resolveManagedProject(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(project.ProjectID).To(Equal("project-123")) + Expect(projectClient.createCalls).To(Equal(1)) + Expect(projectClient.waitActiveCalls).To(Equal(1)) + Expect(projectClient.lastCreateParentContainerID).To(Equal(d.parentContainerID)) + Expect(projectClient.lastCreateName).To(Equal(d.projectName())) + Expect(projectClient.lastCreateOwnerEmail).To(Equal(d.projectMemberEmail)) + Expect(projectClient.lastCreateLabels).To(Equal(d.managedProjectLabels())) + }) + + It("errors on multiple matches", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), + *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), + }, + } + + _, err := d.findManagedProject(context.Background()) + Expect(err).To(MatchError(ContainSubstring("found 2 managed STACKIT projects"))) + }) +}) + +var _ = Describe("ensureProject", func() { + It("resolves the managed project and enables the SKE service", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceEnablementClient := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = serviceEnablementClient + + Expect(d.ensureProject(context.Background())).To(Succeed()) + Expect(d.projectID).To(Equal("project-123")) + Expect(serviceEnablementClient.lastGetStatusProjectID).To(Equal("project-123")) + Expect(serviceEnablementClient.enableCalls).To(Equal(0)) + }) +}) + +var _ = Describe("ensureSKEServiceEnabled", func() { + It("skips when already enabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(0)) + Expect(client.waitCalls).To(Equal(0)) + Expect(client.lastGetStatusServiceID).To(Equal(skeServiceID)) + }) + + It("enables when not found", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + Expect(client.lastEnableProjectID).To(Equal("project-123")) + Expect(client.lastEnableServiceID).To(Equal(skeServiceID)) + }) + + It("enables when disabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusResult: disabledServiceStatusFixture(), + } + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + }) + + It("fails on get status error", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, + } + d.serviceEnablementClient = client + + err := d.ensureSKEServiceEnabled(context.Background(), "project-123") + Expect(err).To(HaveOccurred()) + Expect(client.enableCalls).To(Equal(0)) + }) +}) diff --git a/test/kubetest2/bootstrap_serviceaccount.go b/test/kubetest2/bootstrap_serviceaccount.go new file mode 100644 index 00000000..2d197ab0 --- /dev/null +++ b/test/kubetest2/bootstrap_serviceaccount.go @@ -0,0 +1,260 @@ +package kubetest2 + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" +) + +var childKeyCreationRetryBackoff = wait.Backoff{ + Duration: 3 * time.Second, + Factor: 2.0, + Steps: 5, +} + +type managedServiceAccount struct { + Email string + ProjectID string +} + +type serviceAccountKeyFile struct { + Active bool `json:"active"` + CreatedAt time.Time `json:"createdAt"` + Credentials serviceAccountKeyCredentialsFile `json:"credentials"` + ID string `json:"id"` + KeyAlgorithm string `json:"keyAlgorithm"` + KeyOrigin string `json:"keyOrigin"` + KeyType string `json:"keyType"` + PublicKey string `json:"publicKey"` + ValidUntil *time.Time `json:"validUntil,omitempty"` +} + +type serviceAccountKeyCredentialsFile struct { + Aud string `json:"aud"` + Iss string `json:"iss"` + Kid string `json:"kid"` + PrivateKey *string `json:"privateKey,omitempty"` + Sub string `json:"sub"` + TokenEndpoint string `json:"tokenEndpoint"` +} + +var childProjectRoles = []string{ + childProjectSKERole, + childProjectStorageRole, +} + +// ensureServiceAccount idempotently resolves (or creates) the managed child +// service account, grants it the required project roles, caches a +// service-account key and initializes the SKE client from it. Depends on +// d.projectID being set. +func (d *Deployer) ensureServiceAccount(ctx context.Context) error { + childServiceAccount, err := d.resolveManagedServiceAccount(ctx, d.projectID) + if err != nil { + return err + } + + if err := d.ensureProjectServiceAccountRoles(ctx, d.projectID, childServiceAccount.Email); err != nil { + return err + } + + serviceAccountKey, err := d.ensureCachedChildServiceAccountKey(ctx, d.projectID, childServiceAccount.Email) + if err != nil { + return err + } + return d.initializeSKEClient(serviceAccountKey) +} + +func (d *Deployer) resolveManagedServiceAccount(ctx context.Context, projectID string) (*managedServiceAccount, error) { + serviceAccounts, err := d.serviceAccountClient.ListServiceAccounts(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("list service accounts in STACKIT project %q: %w", projectID, err) + } + + for _, serviceAccount := range serviceAccounts { + if !d.matchesManagedServiceAccountEmail(serviceAccount.GetEmail()) { + continue + } + klog.Infof("Reusing managed service account=%q in project_id=%q", serviceAccount.GetEmail(), projectID) + return &managedServiceAccount{ + Email: serviceAccount.GetEmail(), + ProjectID: serviceAccount.GetProjectId(), + }, nil + } + + klog.Infof("Creating managed service account=%q in project_id=%q", d.serviceAccountName(), projectID) + createdServiceAccount, err := d.serviceAccountClient.CreateServiceAccount(ctx, projectID, d.serviceAccountName()) + if err != nil { + return nil, fmt.Errorf("create service account %q in STACKIT project %q: %w", d.serviceAccountName(), projectID, err) + } + return &managedServiceAccount{ + Email: createdServiceAccount.GetEmail(), + ProjectID: createdServiceAccount.GetProjectId(), + }, nil +} + +func (d *Deployer) ensureProjectServiceAccountRoles(ctx context.Context, projectID, serviceAccountEmail string) error { + members, err := d.authorizationClient.ListMembers(ctx, projectResourceType, projectID) + if err != nil { + return fmt.Errorf("list members for STACKIT project %q: %w", projectID, err) + } + + assignedRoles := make(map[string]struct{}, len(childProjectRoles)) + for _, member := range members { + if member.GetSubject() != serviceAccountEmail { + continue + } + assignedRoles[member.GetRole()] = struct{}{} + } + + missingMembers := make([]authorization.Member, 0, len(childProjectRoles)) + missingRoles := make([]string, 0, len(childProjectRoles)) + for _, role := range childProjectRoles { + if _, ok := assignedRoles[role]; ok { + klog.Infof("Managed service account=%q already has role=%q in project_id=%q", serviceAccountEmail, role, projectID) + continue + } + missingRoles = append(missingRoles, role) + missingMembers = append(missingMembers, *authorization.NewMember(role, serviceAccountEmail)) + } + + if len(missingMembers) == 0 { + return nil + } + + klog.Infof("Adding roles=%q for managed service account=%q in project_id=%q", strings.Join(missingRoles, ","), serviceAccountEmail, projectID) + if err := d.authorizationClient.AddMembers( + ctx, + projectID, + projectResourceType, + missingMembers, + ); err != nil { + return fmt.Errorf("add roles %q for service account %q in STACKIT project %q: %w", strings.Join(missingRoles, ","), serviceAccountEmail, projectID, err) + } + return nil +} + +func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (string, error) { + cachedKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return "", fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if ok { + klog.Infof("Reusing cached child service-account key %q", d.serviceAccountKeyPath) + return cachedKey, nil + } + + klog.Infof("Creating child service-account key for service_account=%q in project_id=%q", serviceAccountEmail, projectID) + createdKey, err := retryWithBackoff(ctx, childKeyCreationRetryBackoff, func() (*serviceaccount.CreateServiceAccountKeyResponse, error) { + return d.serviceAccountClient.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail) + }) + if err != nil { + return "", fmt.Errorf("create service-account key for %q in STACKIT project %q: %w", serviceAccountEmail, projectID, err) + } + keyJSON, err := serviceAccountKeyJSON(createdKey) + if err != nil { + return "", fmt.Errorf("serialize service-account key for %q: %w", serviceAccountEmail, err) + } + if err := d.writeCachedChildServiceAccountKey(keyJSON); err != nil { + return "", fmt.Errorf("write cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + return keyJSON, nil +} + +// retryWithBackoff retries fn until it succeeds or the backoff is exhausted. +// It is used for operations that may fail transiently, e.g. while a freshly +// created service account is still propagating through the STACKIT IAM and is +// not yet ready to have a key created for it. +func retryWithBackoff[T any](ctx context.Context, backoff wait.Backoff, fn func() (T, error)) (T, error) { + var result T + var lastErr error + + waitErr := wait.ExponentialBackoffWithContext(ctx, backoff, func(_ context.Context) (bool, error) { + val, err := fn() + if err != nil { + lastErr = err + return false, nil + } + result = val + return true, nil + }) + if waitErr != nil { + return result, fmt.Errorf("backoff failed: %w, last error: %v", waitErr, lastErr) + } + return result, nil +} + +func (d *Deployer) readCachedChildServiceAccountKey() (key string, ok bool, err error) { + if strings.TrimSpace(d.serviceAccountKeyPath) == "" { + return "", false, nil + } + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + if err != nil { + if os.IsNotExist(err) { + return "", false, nil + } + return "", false, err + } + return string(keyBytes), true, nil +} + +func (d *Deployer) writeCachedChildServiceAccountKey(serviceAccountKey string) error { + if strings.TrimSpace(d.serviceAccountKeyPath) == "" { + return nil + } + return os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600) +} + +func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { + if email == "" { + return false + } + localPart, _, found := strings.Cut(email, "@") + if !found { + return false + } + name := d.serviceAccountName() + return localPart == name || strings.HasPrefix(localPart, name+"-") +} + +func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyResponse) (string, error) { + credentials := createdKey.GetCredentials() + privateKey, ok := credentials.GetPrivateKeyOk() + if !ok || strings.TrimSpace(*privateKey) == "" { + return "", fmt.Errorf("service-account key response did not include a private key") + } + + serviceAccountKey := serviceAccountKeyFile{ + Active: createdKey.GetActive(), + CreatedAt: createdKey.GetCreatedAt(), + Credentials: serviceAccountKeyCredentialsFile{ + Aud: credentials.GetAud(), + Iss: credentials.GetIss(), + Kid: credentials.GetKid(), + PrivateKey: privateKey, + Sub: credentials.GetSub(), + TokenEndpoint: credentials.GetTokenEndpoint(), + }, + ID: createdKey.GetId(), + KeyAlgorithm: string(createdKey.GetKeyAlgorithm()), + KeyOrigin: string(createdKey.GetKeyOrigin()), + KeyType: string(createdKey.GetKeyType()), + PublicKey: createdKey.GetPublicKey(), + } + if validUntil, ok := createdKey.GetValidUntilOk(); ok { + serviceAccountKey.ValidUntil = validUntil + } + + keyJSON, err := json.Marshal(serviceAccountKey) + if err != nil { + return "", err + } + return string(keyJSON), nil +} diff --git a/test/kubetest2/bootstrap_serviceaccount_test.go b/test/kubetest2/bootstrap_serviceaccount_test.go new file mode 100644 index 00000000..ce1ed2b8 --- /dev/null +++ b/test/kubetest2/bootstrap_serviceaccount_test.go @@ -0,0 +1,175 @@ +package kubetest2 + +import ( + "context" + "errors" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + "k8s.io/apimachinery/pkg/util/wait" +) + +var _ = Describe("matchesManagedServiceAccountEmail", func() { + It("matches the managed service account prefix", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "@sa.stackit.cloud")).To(BeTrue()) + }) + + It("matches emails with generated suffixes", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "-aBc2defg@sa.stackit.cloud")).To(BeTrue()) + }) + + It("rejects longer prefixes without the generated suffix separator", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "extra@sa.stackit.cloud")).To(BeFalse()) + }) + + DescribeTable("rejects non-matching emails", + func(email string) { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(email)).To(BeFalse()) + }, + Entry("empty email", ""), + Entry("missing at sign", "kt2-no-separator"), + Entry("different local prefix", "other-account@sa.stackit.cloud"), + ) +}) + +var _ = Describe("ensureServiceAccount", func() { + It("reuses cached key and skips membership write", func() { + d := newTestDeployer() + d.projectID = "project-123" + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + serviceAccountEmail := d.serviceAccountName() + "-aBc2defg@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + authorizationClient := &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectSKERole, serviceAccountEmail), + *authorization.NewMember(childProjectStorageRole, serviceAccountEmail), + }, + } + fakeSKE := &fakeSKEClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return fakeSKE, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(receivedKey).To(Equal(cachedKey)) + Expect(authorizationClient.addCalls).To(Equal(0)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(0)) + }) + + It("creates key and adds membership", func() { + d := newTestDeployer() + d.projectID = "project-123" + + serviceAccountEmail := d.serviceAccountName() + "-aBc2defg@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(authorizationClient.addCalls).To(Equal(1)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) + Expect(authorizationClient.lastAddedType).To(Equal(projectResourceType)) + Expect(authorizationClient.lastAddedID).To(Equal("project-123")) + Expect(authorizationClient.lastAddedMembers).To(ConsistOf( + *authorization.NewMember(childProjectSKERole, serviceAccountEmail), + *authorization.NewMember(childProjectStorageRole, serviceAccountEmail), + )) + + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(receivedKey).To(Equal(string(keyBytes))) + Expect(string(keyBytes)).To(ContainSubstring(`"privateKey":"PRIVATE"`)) + + info, err := os.Stat(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + }) + + It("creates the managed service account when none exists", func() { + d := newTestDeployer() + d.projectID = "project-123" + + serviceAccountEmail := d.serviceAccountName() + "-aBc2defg@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + createResult: serviceAccountFixture(serviceAccountEmail, "project-123"), + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(serviceAccountClient.createCalls).To(Equal(1)) + Expect(serviceAccountClient.listCalls).To(Equal(1)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) + Expect(authorizationClient.addCalls).To(Equal(1)) + Expect(authorizationClient.lastAddedMembers).To(ConsistOf( + *authorization.NewMember(childProjectSKERole, serviceAccountEmail), + *authorization.NewMember(childProjectStorageRole, serviceAccountEmail), + )) + Expect(serviceAccountClient.lastCreatedName).To(Equal(d.serviceAccountName())) + Expect(receivedKey).To(ContainSubstring(`"privateKey":"PRIVATE"`)) + }) +}) + +var _ = Describe("retryWithBackoff", func() { + It("retries until success", func() { + calls := 0 + result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { + calls++ + if calls < 2 { + return "", errors.New("transient") + } + return "done", nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("done")) + Expect(calls).To(Equal(2)) + }) + + It("returns last error when exhausted", func() { + calls := 0 + _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { + calls++ + return 0, errors.New("always fails") + }) + Expect(err).To(HaveOccurred()) + Expect(calls).To(Equal(3)) + }) +}) diff --git a/test/kubetest2/bootstrap_ske.go b/test/kubetest2/bootstrap_ske.go new file mode 100644 index 00000000..e0d96e2c --- /dev/null +++ b/test/kubetest2/bootstrap_ske.go @@ -0,0 +1,168 @@ +package kubetest2 + +import ( + "context" + "fmt" + "os" + + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/klog/v2" +) + +// ensureCluster idempotently validates provider options, then creates or +// updates the SKE cluster, waits until it is ready and writes its kubeconfig. +// Depends on d.projectID and d.skeClient being set. +func (d *Deployer) ensureCluster(ctx context.Context) error { + if err := d.validateProviderOptions(ctx); err != nil { + return err + } + + clusterName := d.clusterName() + payload := d.clusterPayload() + + klog.Infof("Submitting SKE create/update request for cluster=%q", clusterName) + if _, err := d.skeClient.CreateOrUpdateCluster(ctx, d.projectID, d.region, clusterName, payload); err != nil { + return fmt.Errorf("create or update SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Submitted SKE create/update request for cluster=%q", clusterName) + + if _, err := d.skeClient.WaitForClusterReady(ctx, d.projectID, d.region, clusterName); err != nil { + return fmt.Errorf("wait for SKE cluster %q to become ready: %w", clusterName, err) + } + + return d.writeKubeconfig(ctx, clusterName) +} + +func (d *Deployer) writeKubeconfig(ctx context.Context, clusterName string) error { + klog.Infof("Creating kubeconfig for cluster=%q", clusterName) + kubeconfig, err := d.skeClient.CreateKubeconfig(ctx, d.projectID, d.region, clusterName, d.kubeconfigExpiresIn) + if err != nil { + return fmt.Errorf("create kubeconfig for SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Writing kubeconfig for cluster=%q to %q", clusterName, d.kubeconfigPath) + if err := os.WriteFile(d.kubeconfigPath, []byte(kubeconfig.GetKubeconfig()), 0o600); err != nil { + return fmt.Errorf("write kubeconfig %q: %w", d.kubeconfigPath, err) + } + return nil +} + +func (d *Deployer) validateProviderOptions(ctx context.Context) error { + klog.Infof("Validating SKE provider options for region=%q", d.region) + + options, err := d.skeClient.ListProviderOptions(ctx, d.region) + if err != nil { + return fmt.Errorf("list SKE provider options: %w", err) + } + + if !containsKubernetesVersion(options, d.kubernetesVersion) { + return incorrectUsagef("unsupported --kubernetes-version %q for region %q", d.kubernetesVersion, d.region) + } + if !containsAvailabilityZone(options, d.availabilityZone) { + return incorrectUsagef("unsupported --availability-zone %q for region %q", d.availabilityZone, d.region) + } + if !containsMachineType(options, d.machineType) { + return incorrectUsagef("unsupported --machine-type %q for region %q", d.machineType, d.region) + } + if !containsMachineImage(options, d.nodeImageName, d.nodeImageVersion) { + return incorrectUsagef( + "unsupported node image %q version %q for region %q", + d.nodeImageName, + d.nodeImageVersion, + d.region, + ) + } + if d.volumeType != "" && !containsVolumeType(options, d.volumeType) { + return incorrectUsagef("unsupported --volume-type %q for region %q", d.volumeType, d.region) + } + + klog.Infof("SKE provider option validation succeeded for region=%q", d.region) + + return nil +} + +func (d *Deployer) clusterPayload() ske.CreateOrUpdateClusterPayload { + klog.Infof( + "Building SKE cluster payload: cluster=%q kubernetes_version=%q availability_zone=%q machine_type=%q image=%q/%q node_count=%d nodepool=%q volume_size=%d volume_type=%q", + d.clusterName(), + d.kubernetesVersion, + d.availabilityZone, + d.machineType, + d.nodeImageName, + d.nodeImageVersion, + d.nodeCount, + d.nodepoolName, + d.volumeSizeGiB, + d.volumeType, + ) + + clusterKubernetes := ske.NewKubernetes(d.kubernetesVersion) + nodeImage := ske.NewImage(d.nodeImageName, d.nodeImageVersion) + nodeMachine := ske.NewMachine(*nodeImage, d.machineType) + nodeVolume := ske.NewVolume(d.volumeSizeGiB) + if d.volumeType != "" { + nodeVolume.SetType(d.volumeType) + } + + nodepool := ske.NewNodepool( + []string{d.availabilityZone}, + *nodeMachine, + d.nodeCount, + d.nodeCount, + d.nodepoolName, + *nodeVolume, + ) + nodepool.SetAllowSystemComponents(true) + + payload := ske.NewCreateOrUpdateClusterPayload(*clusterKubernetes, []ske.Nodepool{*nodepool}) + return *payload +} + +func containsKubernetesVersion(options *ske.ProviderOptions, version string) bool { + for _, item := range options.GetKubernetesVersions() { + if item.GetVersion() == version { + return true + } + } + return false +} + +func containsAvailabilityZone(options *ske.ProviderOptions, zone string) bool { + for _, item := range options.GetAvailabilityZones() { + if item.GetName() == zone { + return true + } + } + return false +} + +func containsMachineType(options *ske.ProviderOptions, machineType string) bool { + for _, item := range options.GetMachineTypes() { + if item.GetName() == machineType { + return true + } + } + return false +} + +func containsMachineImage(options *ske.ProviderOptions, imageName, imageVersion string) bool { + for _, image := range options.GetMachineImages() { + if image.GetName() != imageName { + continue + } + for _, version := range image.GetVersions() { + if version.GetVersion() == imageVersion { + return true + } + } + } + return false +} + +func containsVolumeType(options *ske.ProviderOptions, volumeType string) bool { + for _, item := range options.GetVolumeTypes() { + if item.GetName() == volumeType { + return true + } + } + return false +} diff --git a/test/kubetest2/bootstrap_ske_test.go b/test/kubetest2/bootstrap_ske_test.go new file mode 100644 index 00000000..32b84a68 --- /dev/null +++ b/test/kubetest2/bootstrap_ske_test.go @@ -0,0 +1,87 @@ +package kubetest2 + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("validateProviderOptions", func() { + It("accepts supported options", func() { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + Expect(d.validateProviderOptions(context.Background())).To(Succeed()) + }) + + DescribeTable("rejects unsupported options", + func(mutate func(*Deployer), wantErrContains string) { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + mutate(d) + Expect(d.validateProviderOptions(context.Background())).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("unsupported kubernetes version", func(d *Deployer) { d.kubernetesVersion = "9.9.9" }, "unsupported --kubernetes-version"), + Entry("unsupported availability zone", func(d *Deployer) { d.availabilityZone = "eu99-9" }, "unsupported --availability-zone"), + Entry("unsupported machine type", func(d *Deployer) { d.machineType = "x9.9" }, "unsupported --machine-type"), + Entry("unsupported node image name", func(d *Deployer) { d.nodeImageName = "fedora" }, "unsupported node image"), + Entry("unsupported node image version", func(d *Deployer) { d.nodeImageVersion = "v99" }, "unsupported node image"), + Entry("unsupported volume type", func(d *Deployer) { d.volumeType = "nvme" }, "unsupported --volume-type"), + ) + + It("fails when listing provider options errors", func() { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptionsErr: errors.New("boom")} + Expect(d.validateProviderOptions(context.Background())).To(MatchError(ContainSubstring("list SKE provider options"))) + }) + + It("allows an empty volume type", func() { + d := newValidTestDeployer() + d.volumeType = "" + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + Expect(d.validateProviderOptions(context.Background())).To(Succeed()) + }) +}) + +var _ = Describe("clusterPayload", func() { + It("builds the expected payload", func() { + d := newValidTestDeployer() + payload := d.clusterPayload() + + kubernetes := payload.GetKubernetes() + Expect(kubernetes.GetVersion()).To(Equal("1.31.0")) + + nodepools := payload.GetNodepools() + Expect(nodepools).To(HaveLen(1)) + + nodepool := nodepools[0] + Expect(nodepool.GetName()).To(Equal(defaultNodepoolName)) + Expect(nodepool.GetAvailabilityZones()).To(ConsistOf(defaultAvailabilityZone)) + + machine := nodepool.GetMachine() + Expect(machine.GetType()).To(Equal("g1.2")) + image := machine.GetImage() + Expect(image.GetName()).To(Equal("ubuntu")) + Expect(image.GetVersion()).To(Equal("v1")) + + Expect(nodepool.GetMinimum()).To(Equal(d.nodeCount)) + Expect(nodepool.GetMaximum()).To(Equal(d.nodeCount)) + + volume := nodepool.GetVolume() + Expect(volume.GetSize()).To(Equal(d.volumeSizeGiB)) + Expect(volume.GetType()).To(Equal("storage")) + + Expect(nodepool.GetAllowSystemComponents()).To(BeTrue()) + }) + + It("omits the volume type when unset", func() { + d := newValidTestDeployer() + d.volumeType = "" + payload := d.clusterPayload() + + Expect(payload.GetNodepools()).To(HaveLen(1)) + volume := payload.GetNodepools()[0].GetVolume() + Expect(volume.GetType()).To(BeEmpty()) + }) +}) diff --git a/test/kubetest2/clients.go b/test/kubetest2/clients.go new file mode 100644 index 00000000..b749bdd9 --- /dev/null +++ b/test/kubetest2/clients.go @@ -0,0 +1,257 @@ +package kubetest2 + +import ( + "context" + "fmt" + + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api/wait" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + serviceenablementwait "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api/wait" +) + +const ( + projectLabelScopeKey = "scope" + projectLabelScopeValue = "PUBLIC" + projectLabelManagedKey = "kt2_managed" + projectLabelManagedValue = "true" + projectLabelRunIDKey = "kt2_run_id" + projectOwnerRole = "owner" + projectResourceType = "project" + childProjectSKERole = "ske.admin" + childProjectStorageRole = "blockstorage.admin" + projectListPageSize = 100 + skeServiceID = "cloud.stackit.ske" +) + +type projectClient interface { + ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) + CreateProject(ctx context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) + WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) + DeleteProject(ctx context.Context, projectID string) error + WaitForProjectDeleted(ctx context.Context, projectID string) error +} + +type serviceAccountClient interface { + ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) + CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) + CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) +} + +type authorizationClient interface { + ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) + AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error +} + +type serviceEnablementClient interface { + GetServiceStatus(ctx context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) + EnableService(ctx context.Context, region, projectID, serviceID string) error + WaitForServiceEnabled(ctx context.Context, region, projectID, serviceID string) error +} + +type sdkProjectClient struct { + api *resourcemanager.APIClient +} + +type sdkServiceAccountClient struct { + api *serviceaccount.APIClient +} + +type sdkAuthorizationClient struct { + api *authorization.APIClient +} + +type sdkServiceEnablementClient struct { + api *serviceenablement.APIClient +} + +func apiClientOptions(serviceAccountKey, endpoint string) []sdkconfig.ConfigurationOption { + var opts []sdkconfig.ConfigurationOption + if serviceAccountKey != "" { + opts = append(opts, sdkconfig.WithServiceAccountKey(serviceAccountKey)) + } + if endpoint != "" { + opts = append(opts, sdkconfig.WithEndpoint(endpoint)) + } + return opts +} + +func apiEndpointURL(cfg *sdkconfig.Configuration) string { + if cfg == nil || len(cfg.Servers) == 0 { + return "unknown" + } + return cfg.Servers[0].URL +} + +func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) { + apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create Resource Manager client: %w", err) + } + return &sdkProjectClient{api: apiClient}, nil +} + +func newServiceAccountClient(serviceAccountKey, endpoint string) (serviceAccountClient, error) { + apiClient, err := serviceaccount.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create Service Account client: %w", err) + } + return &sdkServiceAccountClient{api: apiClient}, nil +} + +func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationClient, error) { + apiClient, err := authorization.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create Authorization client: %w", err) + } + return &sdkAuthorizationClient{api: apiClient}, nil +} + +func newServiceEnablementClient(serviceAccountKey, endpoint string) (serviceEnablementClient, error) { + apiClient, err := serviceenablement.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create Service Enablement client: %w", err) + } + return &sdkServiceEnablementClient{api: apiClient}, nil +} + +func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + projects := make([]resourcemanager.Project, 0, projectListPageSize) + offset := 0 + for { + resp, err := c.api.DefaultAPI.ListProjects(ctx). + ContainerParentId(parentContainerID). + Offset(float32(offset)). + Limit(float32(projectListPageSize)). + Execute() + if err != nil { + return nil, err + } + items := resp.GetItems() + projects = append(projects, items...) + if len(items) < projectListPageSize { + return projects, nil + } + offset += len(items) + } +} + +func (c *sdkProjectClient) CreateProject(ctx context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + payload := resourcemanager.NewCreateProjectPayload( + parentContainerID, + []resourcemanager.Member{*resourcemanager.NewMember(projectOwnerRole, ownerEmail)}, + name, + ) + payload.SetLabels(labels) + return c.api.DefaultAPI.CreateProject(ctx).CreateProjectPayload(*payload).Execute() +} + +func (c *sdkProjectClient) WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + return resourcemanagerwait.CreateProjectWaitHandler(ctx, c.api.DefaultAPI, containerID).WaitWithContext(ctx) +} + +func (c *sdkProjectClient) DeleteProject(ctx context.Context, projectID string) error { + return c.api.DefaultAPI.DeleteProject(ctx, projectID).Execute() +} + +func (c *sdkProjectClient) WaitForProjectDeleted(ctx context.Context, projectID string) error { + _, err := resourcemanagerwait.DeleteProjectWaitHandler(ctx, c.api.DefaultAPI, projectID).WaitWithContext(ctx) + return err +} + +func (c *sdkServiceAccountClient) ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + resp, err := c.api.DefaultAPI.ListServiceAccounts(ctx, projectID).Execute() + if err != nil { + return nil, err + } + return resp.GetItems(), nil +} + +func (c *sdkServiceAccountClient) CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + payload := serviceaccount.NewCreateServiceAccountPayload(name) + return c.api.DefaultAPI.CreateServiceAccount(ctx, projectID).CreateServiceAccountPayload(*payload).Execute() +} + +func (c *sdkServiceAccountClient) CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + payload := serviceaccount.NewCreateServiceAccountKeyPayloadWithDefaults() + resp, err := c.api.DefaultAPI.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() + if err != nil { + return nil, fmt.Errorf("%w (endpoint: %s)", err, apiEndpointURL(c.api.GetConfig())) + } + return resp, nil +} + +func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + resp, err := c.api.DefaultAPI.ListMembers(ctx, resourceType, resourceID).Execute() + if err != nil { + return nil, err + } + return resp.GetMembers(), nil +} + +func (c *sdkAuthorizationClient) AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error { + payload := authorization.NewAddMembersPayload(members, resourceType) + _, err := c.api.DefaultAPI.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() + return err +} + +func (c *sdkServiceEnablementClient) GetServiceStatus(ctx context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + return c.api.DefaultAPI.GetServiceStatusRegional(ctx, region, projectID, serviceID).Execute() +} + +func (c *sdkServiceEnablementClient) EnableService(ctx context.Context, region, projectID, serviceID string) error { + return c.api.DefaultAPI.EnableServiceRegional(ctx, region, projectID, serviceID).Execute() +} + +func (c *sdkServiceEnablementClient) WaitForServiceEnabled(ctx context.Context, region, projectID, serviceID string) error { + _, err := serviceenablementwait.EnableServiceWaitHandler(ctx, c.api.DefaultAPI, region, projectID, serviceID).WaitWithContext(ctx) + return err +} + +func (d *Deployer) initializeBootstrapClients() error { + if err := initializeBootstrapClient(d.projectClient != nil, &d.projectClient, func() (projectClient, error) { + return newProjectClient(d.serviceAccount, d.resourceManagerEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.serviceAccountClient != nil, &d.serviceAccountClient, func() (serviceAccountClient, error) { + return newServiceAccountClient(d.serviceAccount, d.serviceAccountEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.authorizationClient != nil, &d.authorizationClient, func() (authorizationClient, error) { + return newAuthorizationClient(d.serviceAccount, d.authorizationEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.serviceEnablementClient != nil, &d.serviceEnablementClient, func() (serviceEnablementClient, error) { + return newServiceEnablementClient(d.serviceAccount, d.serviceEnablementEndpoint) + }); err != nil { + return err + } + return nil +} + +func initializeBootstrapClient[T any](initialized bool, dst *T, build func() (T, error)) error { + if initialized { + return nil + } + client, err := build() + if err != nil { + return err + } + *dst = client + return nil +} + +func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { + client, err := d.skeClientFactory(d.region, serviceAccountKey, d.skeEndpoint) + if err != nil { + return err + } + d.skeClient = client + return nil +} diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go new file mode 100644 index 00000000..434da5a7 --- /dev/null +++ b/test/kubetest2/deployer.go @@ -0,0 +1,109 @@ +package kubetest2 + +import ( + "flag" + + "github.com/spf13/pflag" + "github.com/stackitcloud/cloud-provider-stackit/pkg/version" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/types" +) + +const ( + Name = "stackit" + + defaultRegion = "eu01" + defaultAvailabilityZone = "eu01-1" + defaultNodeCount int64 = 2 + defaultNodepoolName = "default" + defaultVolumeSizeGiB int64 = 100 + defaultKubeconfigExpiration int64 = 6 * 60 * 60 // 6 hours + minKubeconfigExpiration int64 = 10 * 60 // 10 minutes + maxKubeconfigExpiration int64 = 180 * 24 * 60 * 60 // 180 days +) + +type Deployer struct { + options types.Options + + region string + kubernetesVersion string + availabilityZone string + machineType string + nodeImageName string + nodeImageVersion string + nodeCount int64 + nodepoolName string + volumeSizeGiB int64 + volumeType string + kubeconfigExpiresIn int64 + + csiImageName string + csiImageTag string + + projectID string + serviceAccount string + parentContainerID string + projectMemberEmail string + kubeconfigPath string + serviceAccountKeyPath string + + resourceManagerEndpoint string + serviceAccountEndpoint string + authorizationEndpoint string + serviceEnablementEndpoint string + skeEndpoint string + iaasEndpoint string + + projectClient projectClient + serviceAccountClient serviceAccountClient + authorizationClient authorizationClient + serviceEnablementClient serviceEnablementClient + skeClient skeClient + skeClientFactory func(region, serviceAccount, endpoint string) (skeClient, error) + csiApplier csiApplier +} + +var _ types.NewDeployer = New +var _ types.Deployer = &Deployer{} +var _ types.DeployerWithInit = &Deployer{} +var _ types.DeployerWithKubeconfig = &Deployer{} +var _ types.DeployerWithProvider = &Deployer{} +var _ types.DeployerWithVersion = &Deployer{} + +func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { + d := &Deployer{ + options: opts, + region: defaultRegion, + availabilityZone: defaultAvailabilityZone, + nodeCount: defaultNodeCount, + nodepoolName: defaultNodepoolName, + volumeSizeGiB: defaultVolumeSizeGiB, + kubeconfigExpiresIn: defaultKubeconfigExpiration, + skeClientFactory: newSKEClient, + csiApplier: applyCSIManifestsNative, + } + + fs := pflag.NewFlagSet(Name, pflag.ContinueOnError) + bindFlags(fs, d) + + klog.InitFlags(nil) + fs.AddGoFlagSet(flag.CommandLine) + + return d, fs +} + +func (d *Deployer) Provider() string { + return Name +} + +func (d *Deployer) Version() string { + return version.Version +} + +func (d *Deployer) Build() error { + return nil +} + +func (d *Deployer) DumpClusterLogs() error { + return nil +} diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go new file mode 100644 index 00000000..4de69cea --- /dev/null +++ b/test/kubetest2/env.go @@ -0,0 +1,81 @@ +package kubetest2 + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "k8s.io/klog/v2" +) + +func (d *Deployer) loadEnvironment() error { + klog.Infof("Loading STACKIT environment variables") + + parentContainerID, ok := os.LookupEnv("STACKIT_PARENT_CONTAINER_ID") + if !ok || strings.TrimSpace(parentContainerID) == "" { + return incorrectUsagef("STACKIT_PARENT_CONTAINER_ID environment variable is required") + } + + serviceAccount := strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT")) + serviceAccountEmail := strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT_EMAIL")) + + var projectMemberEmail string + if serviceAccount != "" { + var err error + projectMemberEmail, err = extractServiceAccountEmail(serviceAccount) + if err != nil { + return incorrectUsagef("invalid STACKIT_SERVICE_ACCOUNT: %v", err) + } + } else if serviceAccountEmail != "" { + projectMemberEmail = serviceAccountEmail + } else { + return incorrectUsagef("either STACKIT_SERVICE_ACCOUNT or STACKIT_SERVICE_ACCOUNT_EMAIL must be provided") + } + + d.serviceAccount = serviceAccount + d.parentContainerID = parentContainerID + d.projectMemberEmail = projectMemberEmail + d.resourceManagerEndpoint = strings.TrimSpace(os.Getenv("STACKIT_RESOURCE_MANAGER_ENDPOINT")) + d.serviceAccountEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT")) + d.authorizationEndpoint = strings.TrimSpace(os.Getenv("STACKIT_AUTHORIZATION_ENDPOINT")) + d.serviceEnablementEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ENABLEMENT_ENDPOINT")) + d.skeEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SKE_ENDPOINT")) + d.iaasEndpoint = strings.TrimSpace(os.Getenv("STACKIT_IAAS_ENDPOINT")) + d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") + d.serviceAccountKeyPath = filepath.Join(d.options.RunDir(), "service-account-key.json") + + klog.Infof( + "Loaded STACKIT environment: parent_container_id=%q project_member_email=%q service_account_bytes=%d kubeconfig_path=%q service_account_key_path=%q", + d.parentContainerID, + d.projectMemberEmail, + len(d.serviceAccount), + d.kubeconfigPath, + d.serviceAccountKeyPath, + ) + + klog.Infof( + "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q service_enablement=%q ske=%q iaas=%q", + d.resourceManagerEndpoint, + d.serviceAccountEndpoint, + d.authorizationEndpoint, + d.serviceEnablementEndpoint, + d.skeEndpoint, + d.iaasEndpoint, + ) + + return nil +} + +func extractServiceAccountEmail(serviceAccountKey string) (string, error) { + var key serviceAccountKeyFile + if err := json.Unmarshal([]byte(serviceAccountKey), &key); err != nil { + return "", fmt.Errorf("parse service account key: %w", err) + } + email := strings.TrimSpace(key.Credentials.Iss) + if email == "" { + return "", fmt.Errorf("service account key has no email in credentials.iss") + } + return email, nil +} diff --git a/test/kubetest2/env_test.go b/test/kubetest2/env_test.go new file mode 100644 index 00000000..0de66940 --- /dev/null +++ b/test/kubetest2/env_test.go @@ -0,0 +1,134 @@ +package kubetest2 + +import ( + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("extractServiceAccountEmail", func() { + It("extracts the email from the credentials.iss field", func() { + email, err := extractServiceAccountEmail(validServiceAccountKey) + Expect(err).NotTo(HaveOccurred()) + Expect(email).To(Equal("owner@example.com")) + }) + + DescribeTable("rejects invalid service account keys", + func(key string, wantErrContains string) { + _, err := extractServiceAccountEmail(key) + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("malformed JSON", "not-json", "parse service account key"), + Entry("missing credentials.iss", `{"credentials":{"aud":"x"}}`, "no email in credentials.iss"), + Entry("blank credentials.iss", `{"credentials":{"iss":" "}}`, "no email in credentials.iss"), + ) +}) + +var _ = Describe("loadEnvironment", func() { + envVarKeys := []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_SERVICE_ACCOUNT_EMAIL", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SERVICE_ENABLEMENT_ENDPOINT", + "STACKIT_SKE_ENDPOINT", + "STACKIT_IAAS_ENDPOINT", + } + + DescribeTable("validates required environment variables", + func(env map[string]string, wantErrContains string) { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + for key, value := range env { + setEnvVar(key, value) + } + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + err := d.loadEnvironment() + if wantErrContains != "" { + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) + return + } + Expect(err).NotTo(HaveOccurred()) + Expect(d.parentContainerID).To(Equal("parent-1")) + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.projectID).To(BeEmpty()) + Expect(d.kubeconfigPath).To(Equal(filepath.Join(runDir, "kubeconfig"))) + Expect(d.serviceAccountKeyPath).To(Equal(filepath.Join(runDir, "service-account-key.json"))) + }, + Entry("missing all auth", map[string]string{ + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "either STACKIT_SERVICE_ACCOUNT or STACKIT_SERVICE_ACCOUNT_EMAIL"), + Entry("missing parent container with service account", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + }, "STACKIT_PARENT_CONTAINER_ID"), + Entry("missing parent container with workload identity", map[string]string{ + "STACKIT_SERVICE_ACCOUNT_EMAIL": "owner@example.com", + }, "STACKIT_PARENT_CONTAINER_ID"), + Entry("invalid service account key", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "invalid STACKIT_SERVICE_ACCOUNT"), + Entry("valid with service account key", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, ""), + Entry("valid with workload identity", map[string]string{ + "STACKIT_SERVICE_ACCOUNT_EMAIL": "owner@example.com", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, ""), + ) + + It("reads optional endpoints with service account", func() { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + setEnvVar("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) + setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") + setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + setEnvVar("STACKIT_IAAS_ENDPOINT", "https://iaas.example.com") + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + Expect(d.loadEnvironment()).To(Succeed()) + + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) + Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) + Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) + Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) + Expect(d.iaasEndpoint).To(Equal("https://iaas.example.com")) + }) + + It("reads optional endpoints with workload identity", func() { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + setEnvVar("STACKIT_SERVICE_ACCOUNT_EMAIL", "owner@example.com") + setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") + setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + setEnvVar("STACKIT_IAAS_ENDPOINT", "https://iaas.example.com") + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + Expect(d.loadEnvironment()).To(Succeed()) + + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) + Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) + Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) + Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) + Expect(d.iaasEndpoint).To(Equal("https://iaas.example.com")) + }) +}) diff --git a/test/kubetest2/flags.go b/test/kubetest2/flags.go new file mode 100644 index 00000000..bc40c812 --- /dev/null +++ b/test/kubetest2/flags.go @@ -0,0 +1,94 @@ +package kubetest2 + +import ( + "fmt" + "strings" + + "github.com/spf13/pflag" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/types" +) + +func bindFlags(fs *pflag.FlagSet, d *Deployer) { + fs.StringVar(&d.region, "region", defaultRegion, "STACKIT region for the SKE cluster") + fs.StringVar(&d.kubernetesVersion, "kubernetes-version", "", "Kubernetes version for the SKE cluster") + fs.StringVar(&d.availabilityZone, "availability-zone", defaultAvailabilityZone, "Availability zone for the SKE nodepool") + fs.StringVar(&d.machineType, "machine-type", "", "Machine type for the SKE nodepool") + fs.StringVar(&d.nodeImageName, "node-image-name", "", "Node image name for the SKE nodepool") + fs.StringVar(&d.nodeImageVersion, "node-image-version", "", "Node image version for the SKE nodepool") + fs.Int64Var(&d.nodeCount, "node-count", defaultNodeCount, "Node count for the SKE nodepool") + fs.StringVar(&d.nodepoolName, "nodepool-name", defaultNodepoolName, "Nodepool name for the SKE cluster") + fs.Int64Var(&d.volumeSizeGiB, "volume-size", defaultVolumeSizeGiB, "Root volume size in GiB for the SKE nodepool") + fs.StringVar(&d.volumeType, "volume-type", "", "Root volume type for the SKE nodepool") + fs.Int64Var(&d.kubeconfigExpiresIn, "kubeconfig-expiration-seconds", defaultKubeconfigExpiration, "Admin kubeconfig expiration in seconds") + fs.StringVar(&d.csiImageName, "csi-image-name", "", "Image name (repository) for the STACKIT CSI plugin") + fs.StringVar(&d.csiImageTag, "csi-image-tag", "", "Image tag for the STACKIT CSI plugin") +} + +func (d *Deployer) validate() error { + klog.Infof( + "Validating deployer configuration: run_id=%q region=%q kubernetes_version=%q availability_zone=%q machine_type=%q node_image_name=%q node_image_version=%q node_count=%d nodepool_name=%q volume_size=%d volume_type=%q kubeconfig_expiration_seconds=%d csi_image_name=%q csi_image_tag=%q", + d.options.RunID(), + d.region, + d.kubernetesVersion, + d.availabilityZone, + d.machineType, + d.nodeImageName, + d.nodeImageVersion, + d.nodeCount, + d.nodepoolName, + d.volumeSizeGiB, + d.volumeType, + d.kubeconfigExpiresIn, + d.csiImageName, + d.csiImageTag, + ) + + requiredFlags := map[string]string{ + "--region": d.region, + "--kubernetes-version": d.kubernetesVersion, + "--availability-zone": d.availabilityZone, + "--machine-type": d.machineType, + "--node-image-name": d.nodeImageName, + "--node-image-version": d.nodeImageVersion, + "--csi-image-name": d.csiImageName, + "--csi-image-tag": d.csiImageTag, + } + + for flagName, value := range requiredFlags { + if strings.TrimSpace(value) == "" { + return incorrectUsagef("%s is required", flagName) + } + } + + if strings.TrimSpace(d.nodepoolName) == "" { + return incorrectUsagef("--nodepool-name must not be empty") + } + if len(d.nodepoolName) > 15 { + return incorrectUsagef("--nodepool-name must be 15 characters or fewer") + } + if d.nodeCount < 1 { + return incorrectUsagef("--node-count must be greater than 0") + } + if d.volumeSizeGiB < 1 { + return incorrectUsagef("--volume-size must be greater than 0") + } + if d.kubeconfigExpiresIn < minKubeconfigExpiration || d.kubeconfigExpiresIn > maxKubeconfigExpiration { + return incorrectUsagef( + "--kubeconfig-expiration-seconds must be between %d and %d", + minKubeconfigExpiration, + maxKubeconfigExpiration, + ) + } + if strings.TrimSpace(d.options.RunID()) == "" { + return incorrectUsagef("kubetest2 run-id must not be empty") + } + + klog.Infof("Deployer configuration validation succeeded") + + return nil +} + +func incorrectUsagef(format string, args ...any) error { + return types.NewIncorrectUsage(fmt.Sprintf(format, args...)) +} diff --git a/test/kubetest2/flags_test.go b/test/kubetest2/flags_test.go new file mode 100644 index 00000000..ee66676b --- /dev/null +++ b/test/kubetest2/flags_test.go @@ -0,0 +1,44 @@ +package kubetest2 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func newValidTestDeployer() *Deployer { + d := newTestDeployer() + configureValidUpInputs(d) + d.nodeCount = defaultNodeCount + d.volumeSizeGiB = defaultVolumeSizeGiB + d.kubeconfigExpiresIn = defaultKubeconfigExpiration + return d +} + +var _ = Describe("validate", func() { + It("accepts a valid configuration", func() { + Expect(newValidTestDeployer().validate()).To(Succeed()) + }) + + DescribeTable("rejects invalid configuration", + func(mutate func(*Deployer), wantErrContains string) { + d := newValidTestDeployer() + mutate(d) + Expect(d.validate()).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("missing region", func(d *Deployer) { d.region = "" }, "--region is required"), + Entry("missing kubernetes version", func(d *Deployer) { d.kubernetesVersion = "" }, "--kubernetes-version is required"), + Entry("missing availability zone", func(d *Deployer) { d.availabilityZone = "" }, "--availability-zone is required"), + Entry("missing machine type", func(d *Deployer) { d.machineType = "" }, "--machine-type is required"), + Entry("missing node image name", func(d *Deployer) { d.nodeImageName = "" }, "--node-image-name is required"), + Entry("missing node image version", func(d *Deployer) { d.nodeImageVersion = "" }, "--node-image-version is required"), + Entry("empty nodepool name", func(d *Deployer) { d.nodepoolName = "" }, "--nodepool-name must not be empty"), + Entry("nodepool name too long", func(d *Deployer) { d.nodepoolName = "this-name-is-way-too-long" }, "--nodepool-name must be 15 characters or fewer"), + Entry("zero node count", func(d *Deployer) { d.nodeCount = 0 }, "--node-count must be greater than 0"), + Entry("zero volume size", func(d *Deployer) { d.volumeSizeGiB = 0 }, "--volume-size must be greater than 0"), + Entry("kubeconfig expiration too small", func(d *Deployer) { d.kubeconfigExpiresIn = minKubeconfigExpiration - 1 }, "--kubeconfig-expiration-seconds must be between"), + Entry("kubeconfig expiration too large", func(d *Deployer) { d.kubeconfigExpiresIn = maxKubeconfigExpiration + 1 }, "--kubeconfig-expiration-seconds must be between"), + Entry("missing csi image name", func(d *Deployer) { d.csiImageName = "" }, "--csi-image-name is required"), + Entry("missing csi image tag", func(d *Deployer) { d.csiImageTag = "" }, "--csi-image-tag is required"), + Entry("empty run id", func(d *Deployer) { d.options = fakeOptions{runID: "", runDir: d.options.RunDir()} }, "run-id must not be empty"), + ) +}) diff --git a/test/kubetest2/lifecycle.go b/test/kubetest2/lifecycle.go new file mode 100644 index 00000000..8a35723d --- /dev/null +++ b/test/kubetest2/lifecycle.go @@ -0,0 +1,144 @@ +package kubetest2 + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/klog/v2" +) + +type bootstrapStep struct { + name string + fn func(context.Context) error +} + +func (d *Deployer) Init() error { + klog.Infof("Initializing STACKIT kubetest2 deployer") + + if err := d.validate(); err != nil { + return err + } + if err := d.loadEnvironment(); err != nil { + return err + } + + if err := d.initializeBootstrapClients(); err != nil { + return err + } + + klog.Infof("STACKIT kubetest2 deployer initialized successfully") + + return nil +} + +func (d *Deployer) Up() error { + klog.Infof("Starting cluster up flow for cluster=%q", d.clusterName()) + + ctx := context.Background() + for _, step := range []bootstrapStep{ + {"project", d.ensureProject}, + {"service account", d.ensureServiceAccount}, + {"cluster", d.ensureCluster}, + {"csi", d.ensureCSI}, + } { + klog.Infof("Bootstrap step %q starting", step.name) + if err := step.fn(ctx); err != nil { + return fmt.Errorf("bootstrap step %q: %w", step.name, err) + } + klog.Infof("Bootstrap step %q completed", step.name) + } + + klog.Infof("Cluster up flow completed successfully for cluster=%q", d.clusterName()) + + return nil +} + +func (d *Deployer) Down() error { + ctx := context.Background() + + klog.Infof("Starting cluster down flow for cluster=%q", d.clusterName()) + + project, err := d.findManagedProject(ctx) + if err != nil { + return err + } + if project == nil { + klog.Infof("Managed project for run_id=%q is already absent", d.options.RunID()) + return nil + } + + d.projectID = project.ProjectID + if err := d.projectClient.DeleteProject(ctx, project.ProjectID); err != nil { + if !stackiterrors.IsNotFound(err) { + return fmt.Errorf("delete STACKIT project %q: %w", project.ProjectID, err) + } + klog.Infof("Project=%q already absent, treating delete as success", project.ProjectID) + return nil + } + + if err := d.projectClient.WaitForProjectDeleted(ctx, project.ProjectID); err != nil { + return fmt.Errorf("wait for STACKIT project %q deletion: %w", project.ProjectID, err) + } + + klog.Infof("Cluster down flow completed successfully for project=%q", project.ProjectID) + + return nil +} + +func (d *Deployer) IsUp() (bool, error) { + ctx := context.Background() + project, err := d.findManagedProject(ctx) + if err != nil { + return false, err + } + if project == nil { + klog.Infof("Managed project for run_id=%q not found during IsUp check", d.options.RunID()) + return false, nil + } + + d.projectID = project.ProjectID + serviceAccountKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return false, fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if !ok { + return false, fmt.Errorf( + "managed project %q exists but child service-account key cache %q is missing", + project.ProjectID, + d.serviceAccountKeyPath, + ) + } + if err := d.initializeSKEClient(serviceAccountKey); err != nil { + return false, err + } + + klog.Infof("Checking cluster state for cluster=%q", d.clusterName()) + cluster, err := d.skeClient.GetCluster(ctx, d.projectID, d.region, d.clusterName()) + if err != nil { + if stackiterrors.IsNotFound(err) { + klog.Infof("Cluster=%q not found during IsUp check", d.clusterName()) + return false, nil + } + return false, fmt.Errorf("get SKE cluster %q: %w", d.clusterName(), err) + } + + if cluster.Status == nil || cluster.Status.Aggregated == nil { + klog.Infof("Cluster=%q has no aggregated status yet", d.clusterName()) + return false, nil + } + + state := cluster.Status.GetAggregated() + klog.Infof("Cluster=%q current aggregated state=%q", d.clusterName(), state) + return state == ske.CLUSTERSTATUSSTATE_HEALTHY || state == ske.CLUSTERSTATUSSTATE_HIBERNATED, nil +} + +func (d *Deployer) Kubeconfig() (string, error) { + if d.kubeconfigPath == "" { + d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") + } + klog.Infof("Returning kubeconfig path %q", d.kubeconfigPath) + return d.kubeconfigPath, nil +} diff --git a/test/kubetest2/lifecycle_test.go b/test/kubetest2/lifecycle_test.go new file mode 100644 index 00000000..7438f147 --- /dev/null +++ b/test/kubetest2/lifecycle_test.go @@ -0,0 +1,161 @@ +package kubetest2 + +import ( + "net/http" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "github.com/stackitcloud/stackit-sdk-go/services/ske" +) + +var _ = Describe("Up", func() { + It("uses discovered project and writes kubeconfig", func() { + d := newTestDeployer() + configureValidUpInputs(d) + + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + d.serviceAccountClient = &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + d.authorizationClient = &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectSKERole, serviceAccountEmail), + *authorization.NewMember(childProjectStorageRole, serviceAccountEmail), + }, + } + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + + fakeSKE := &fakeSKEClient{ + providerOptions: providerOptionsFixture(), + createOrUpdateResult: ske.NewClusterWithDefaults(), + waitReadyResult: ske.NewClusterWithDefaults(), + kubeconfigResult: func() *ske.Kubeconfig { + cfg := ske.NewKubeconfig() + cfg.SetKubeconfig("apiVersion: v1\n") + return cfg + }(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + Expect(d.Up()).To(Succeed()) + Expect(fakeSKE.lastCreateProjectID).To(Equal("project-123")) + Expect(fakeSKE.lastCreateClusterName).To(Equal(d.clusterName())) + + kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(kubeconfigBytes)).To(Equal("apiVersion: v1\n")) + }) +}) + +var _ = Describe("Down", func() { + It("succeeds when project is missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{} + d.projectClient = projectClient + + Expect(d.Down()).To(Succeed()) + Expect(projectClient.deleteCalls).To(Equal(0)) + }) + + It("deletes project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + Expect(d.Down()).To(Succeed()) + projectClient := d.projectClient.(*fakeProjectClient) + Expect(projectClient.deleteCalls).To(Equal(1)) + Expect(projectClient.waitDeletedCalls).To(Equal(1)) + }) +}) + +var _ = Describe("IsUp", func() { + It("returns false when no project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{} + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) + + It("returns error when project without cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + _, err := d.IsUp() + Expect(err).To(MatchError(ContainSubstring("child service-account key cache"))) + }) + + It("queries cluster when project with cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + fakeSKE := &fakeSKEClient{ + getClusterResult: healthyClusterFixture(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeTrue()) + Expect(fakeSKE.lastGetProjectID).To(Equal("project-123")) + }) + + It("returns false when cluster missing", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + fakeSKE := &fakeSKEClient{ + getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.skeClientFactory = func(_, _, _ string) (skeClient, error) { + return fakeSKE, nil + } + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) +}) diff --git a/test/kubetest2/naming.go b/test/kubetest2/naming.go new file mode 100644 index 00000000..9a8c1c6b --- /dev/null +++ b/test/kubetest2/naming.go @@ -0,0 +1,27 @@ +package kubetest2 + +import ( + "crypto/sha256" + "encoding/hex" +) + +func runTokenForRun(runID string) string { + sum := sha256.Sum256([]byte(runID)) + return hex.EncodeToString(sum[:4]) +} + +func (d *Deployer) runToken() string { + return runTokenForRun(d.options.RunID()) +} + +func (d *Deployer) clusterName() string { + return "kt2" + d.runToken() +} + +func (d *Deployer) projectName() string { + return "kt2-" + d.runToken() +} + +func (d *Deployer) serviceAccountName() string { + return "kt2-" + d.runToken() +} diff --git a/test/kubetest2/ske_client.go b/test/kubetest2/ske_client.go new file mode 100644 index 00000000..ae9d6a45 --- /dev/null +++ b/test/kubetest2/ske_client.go @@ -0,0 +1,76 @@ +package kubetest2 + +import ( + "context" + "fmt" + "strconv" + + "github.com/stackitcloud/stackit-sdk-go/services/ske" + skewait "github.com/stackitcloud/stackit-sdk-go/services/ske/wait" + "k8s.io/klog/v2" +) + +type skeClient interface { + GetCluster(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) + ListProviderOptions(ctx context.Context, region string) (*ske.ProviderOptions, error) + CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) + WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) + CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) +} + +type sdkSKEClient struct { + api ske.DefaultApi +} + +func newSKEClient(region, serviceAccount, endpoint string) (skeClient, error) { + klog.Infof("Creating SKE API client for region=%q with service_account_bytes=%d", region, len(serviceAccount)) + + apiClient, err := ske.NewAPIClient(apiClientOptions(serviceAccount, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create SKE client: %w", err) + } + + klog.Infof("Created SKE API client successfully") + + return &sdkSKEClient{api: apiClient}, nil +} + +func (c *sdkSKEClient) GetCluster(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) { + klog.Infof("SKE GetCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + return c.api.GetCluster(ctx, projectID, region, name).Execute() +} + +func (c *sdkSKEClient) ListProviderOptions(ctx context.Context, region string) (*ske.ProviderOptions, error) { + klog.Infof("SKE ListProviderOptions: region=%q", region) + return c.api.ListProviderOptions(ctx, region).Execute() +} + +func (c *sdkSKEClient) CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + klog.Infof("SKE CreateOrUpdateCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + return c.api.CreateOrUpdateCluster(ctx, projectID, region, name).CreateOrUpdateClusterPayload(payload).Execute() +} + +func (c *sdkSKEClient) WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) { + klog.Infof("Waiting for SKE cluster to become ready: project_id=%q region=%q cluster=%q", projectID, region, name) + cluster, err := skewait.CreateOrUpdateClusterWaitHandler(ctx, c.api, projectID, region, name).WaitWithContext(ctx) + if err != nil { + return nil, err + } + if cluster != nil && cluster.Status != nil && cluster.Status.Aggregated != nil { + klog.Infof("SKE cluster is ready: cluster=%q state=%q", name, cluster.Status.GetAggregated()) + } + return cluster, nil +} + +func (c *sdkSKEClient) CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) { + klog.Infof( + "SKE CreateKubeconfig: project_id=%q region=%q cluster=%q expiration_seconds=%d", + projectID, + region, + name, + expirationSeconds, + ) + payload := ske.NewCreateKubeconfigPayload() + payload.SetExpirationSeconds(strconv.FormatInt(expirationSeconds, 10)) + return c.api.CreateKubeconfig(ctx, projectID, region, name).CreateKubeconfigPayload(*payload).Execute() +} diff --git a/test/kubetest2/suite_test.go b/test/kubetest2/suite_test.go new file mode 100644 index 00000000..3bef6c0b --- /dev/null +++ b/test/kubetest2/suite_test.go @@ -0,0 +1,396 @@ +package kubetest2 + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "sigs.k8s.io/kubetest2/pkg/types" +) + +func TestKubetest2(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Kubetest2 Suite") +} + +const validServiceAccountKey = `{"credentials":{"iss":"owner@example.com"}}` + +type fakeOptions struct { + runID string + runDir string +} + +func (o fakeOptions) HelpRequested() bool { return false } +func (o fakeOptions) ShouldBuild() bool { return false } +func (o fakeOptions) ShouldUp() bool { return false } +func (o fakeOptions) ShouldDown() bool { return false } +func (o fakeOptions) ShouldTest() bool { return false } +func (o fakeOptions) SkipTestJUnitReport() bool { return false } +func (o fakeOptions) RunID() string { return o.runID } +func (o fakeOptions) RunDir() string { return o.runDir } +func (o fakeOptions) RundirInArtifacts() bool { return false } +func (o fakeOptions) PostTestCmd() []string { return nil } +func (o fakeOptions) PreTestCmd() []string { return nil } + +type fakeProjectClient struct { + listProjectsResult []resourcemanager.Project + listProjectsErr error + + createProjectResult *resourcemanager.Project + createProjectErr error + waitActiveResult *resourcemanager.GetProjectResponse + waitActiveErr error + + deleteErr error + waitDeletedErr error + + createCalls int + waitActiveCalls int + deleteCalls int + waitDeletedCalls int + + lastListParentContainerID string + lastCreateParentContainerID string + lastCreateName string + lastCreateOwnerEmail string + lastCreateLabels map[string]string + lastWaitActiveContainerID string + lastDeletedProjectID string + lastWaitDeletedProjectID string +} + +func (c *fakeProjectClient) ListProjects(_ context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + c.lastListParentContainerID = parentContainerID + return c.listProjectsResult, c.listProjectsErr +} + +func (c *fakeProjectClient) CreateProject(_ context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + c.createCalls++ + c.lastCreateParentContainerID = parentContainerID + c.lastCreateName = name + c.lastCreateOwnerEmail = ownerEmail + c.lastCreateLabels = labels + return c.createProjectResult, c.createProjectErr +} + +func (c *fakeProjectClient) WaitForProjectActive(_ context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + c.waitActiveCalls++ + c.lastWaitActiveContainerID = containerID + return c.waitActiveResult, c.waitActiveErr +} + +func (c *fakeProjectClient) DeleteProject(_ context.Context, projectID string) error { + c.deleteCalls++ + c.lastDeletedProjectID = projectID + return c.deleteErr +} + +func (c *fakeProjectClient) WaitForProjectDeleted(_ context.Context, projectID string) error { + c.waitDeletedCalls++ + c.lastWaitDeletedProjectID = projectID + return c.waitDeletedErr +} + +type fakeServiceAccountClient struct { + listResult []serviceaccount.ServiceAccount + listResults [][]serviceaccount.ServiceAccount + listErr error + + createResult *serviceaccount.ServiceAccount + createErr error + + createKeyResult *serviceaccount.CreateServiceAccountKeyResponse + createKeyErr error + + createCalls int + createKeyCalls int + + lastProjectIDForList string + lastProjectIDForCreate string + lastCreatedName string + lastProjectIDForCreateKey string + lastCreateKeyEmail string + listCalls int +} + +func (c *fakeServiceAccountClient) ListServiceAccounts(_ context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + c.lastProjectIDForList = projectID + c.listCalls++ + if len(c.listResults) > 0 { + result := c.listResults[0] + if len(c.listResults) > 1 { + c.listResults = c.listResults[1:] + } + return result, c.listErr + } + return c.listResult, c.listErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccount(_ context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + c.createCalls++ + c.lastProjectIDForCreate = projectID + c.lastCreatedName = name + return c.createResult, c.createErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccountKey(_ context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + c.createKeyCalls++ + c.lastProjectIDForCreateKey = projectID + c.lastCreateKeyEmail = serviceAccountEmail + return c.createKeyResult, c.createKeyErr +} + +type fakeAuthorizationClient struct { + listMembersResult []authorization.Member + listMembersErr error + addMembersErr error + + addCalls int + + lastResourceType string + lastResourceID string + lastAddedType string + lastAddedID string + lastAddedMembers []authorization.Member +} + +func (c *fakeAuthorizationClient) ListMembers(_ context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + c.lastResourceType = resourceType + c.lastResourceID = resourceID + return c.listMembersResult, c.listMembersErr +} + +func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, resourceType string, members []authorization.Member) error { + c.addCalls++ + c.lastAddedID = resourceID + c.lastAddedType = resourceType + c.lastAddedMembers = members + return c.addMembersErr +} + +type fakeServiceEnablementClient struct { + getStatusResult *serviceenablement.ServiceStatus + getStatusErr error + enableErr error + waitErr error + + enableCalls int + waitCalls int + + lastGetStatusRegion string + lastGetStatusProjectID string + lastGetStatusServiceID string + lastEnableRegion string + lastEnableProjectID string + lastEnableServiceID string + lastWaitRegion string + lastWaitProjectID string + lastWaitServiceID string +} + +func (c *fakeServiceEnablementClient) GetServiceStatus(_ context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + c.lastGetStatusRegion = region + c.lastGetStatusProjectID = projectID + c.lastGetStatusServiceID = serviceID + return c.getStatusResult, c.getStatusErr +} + +func (c *fakeServiceEnablementClient) EnableService(_ context.Context, region, projectID, serviceID string) error { + c.enableCalls++ + c.lastEnableRegion = region + c.lastEnableProjectID = projectID + c.lastEnableServiceID = serviceID + return c.enableErr +} + +func (c *fakeServiceEnablementClient) WaitForServiceEnabled(_ context.Context, region, projectID, serviceID string) error { + c.waitCalls++ + c.lastWaitRegion = region + c.lastWaitProjectID = projectID + c.lastWaitServiceID = serviceID + return c.waitErr +} + +type fakeSKEClient struct { + providerOptions *ske.ProviderOptions + providerOptionsErr error + getClusterResult *ske.Cluster + getClusterErr error + + createOrUpdateResult *ske.Cluster + createOrUpdateErr error + waitReadyResult *ske.Cluster + waitReadyErr error + kubeconfigResult *ske.Kubeconfig + kubeconfigErr error + + lastGetProjectID string + lastCreateProjectID string + lastCreateRegion string + lastCreateClusterName string + lastKubeconfigProjectID string + lastExpirationSeconds int64 +} + +func (c *fakeSKEClient) GetCluster(_ context.Context, projectID, _, _ string) (*ske.Cluster, error) { + c.lastGetProjectID = projectID + return c.getClusterResult, c.getClusterErr +} + +func (c *fakeSKEClient) ListProviderOptions(_ context.Context, _ string) (*ske.ProviderOptions, error) { + return c.providerOptions, c.providerOptionsErr +} + +func (c *fakeSKEClient) CreateOrUpdateCluster(_ context.Context, projectID, region, name string, _ ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + c.lastCreateProjectID = projectID + c.lastCreateRegion = region + c.lastCreateClusterName = name + return c.createOrUpdateResult, c.createOrUpdateErr +} + +func (c *fakeSKEClient) WaitForClusterReady(_ context.Context, _, _, _ string) (*ske.Cluster, error) { + return c.waitReadyResult, c.waitReadyErr +} + +func (c *fakeSKEClient) CreateKubeconfig(_ context.Context, projectID, _, _ string, expirationSeconds int64) (*ske.Kubeconfig, error) { + c.lastKubeconfigProjectID = projectID + c.lastExpirationSeconds = expirationSeconds + return c.kubeconfigResult, c.kubeconfigErr +} + +func newTestDeployer() *Deployer { + runDir := GinkgoT().TempDir() + return &Deployer{ + options: fakeOptions{runID: "run-123", runDir: runDir}, + region: defaultRegion, + parentContainerID: "parent-123", + projectMemberEmail: "owner@example.com", + kubeconfigPath: filepath.Join(runDir, "kubeconfig"), + serviceAccountKeyPath: filepath.Join(runDir, "service-account-key.json"), + skeClientFactory: newSKEClient, + csiApplier: func(_ context.Context, _, _ string) error { + return nil + }, + } +} + +func configureValidUpInputs(d *Deployer) { + d.kubernetesVersion = "1.31.0" + d.availabilityZone = defaultAvailabilityZone + d.machineType = "g1.2" + d.nodeImageName = "ubuntu" + d.nodeImageVersion = "v1" + d.nodepoolName = defaultNodepoolName + d.volumeType = "storage" + d.csiImageName = "ghcr.io/stackitcloud/cloud-provider-stackit/stackit-csi-plugin" + d.csiImageTag = "v1.0.0" +} + +func setEnvVar(key, value string) { + Expect(os.Setenv(key, value)).To(Succeed()) + DeferCleanup(os.Unsetenv, key) +} + +func providerOptionsFixture() *ske.ProviderOptions { + kubernetesVersion := ske.NewKubernetesVersion() + kubernetesVersion.SetVersion("1.31.0") + + availabilityZone := ske.NewAvailabilityZone() + availabilityZone.SetName(defaultAvailabilityZone) + + machineType := ske.NewMachineType() + machineType.SetName("g1.2") + + imageVersion := ske.NewMachineImageVersion() + imageVersion.SetVersion("v1") + + machineImage := ske.NewMachineImage() + machineImage.SetName("ubuntu") + machineImage.SetVersions([]ske.MachineImageVersion{*imageVersion}) + + volumeType := ske.NewVolumeType() + volumeType.SetName("storage") + + providerOptions := ske.NewProviderOptions() + providerOptions.SetKubernetesVersions([]ske.KubernetesVersion{*kubernetesVersion}) + providerOptions.SetAvailabilityZones([]ske.AvailabilityZone{*availabilityZone}) + providerOptions.SetMachineTypes([]ske.MachineType{*machineType}) + providerOptions.SetMachineImages([]ske.MachineImage{*machineImage}) + providerOptions.SetVolumeTypes([]ske.VolumeType{*volumeType}) + return providerOptions +} + +func healthyClusterFixture() *ske.Cluster { + cluster := ske.NewClusterWithDefaults() + status := ske.NewClusterStatus() + status.SetAggregated(ske.CLUSTERSTATUSSTATE_HEALTHY) + cluster.SetStatus(*status) + return cluster +} + +func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { + status := serviceenablement.NewServiceStatus() + state := serviceenablement.SERVICESTATUSSTATE_DISABLED + status.State = &state + return status +} + +func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { + project := resourcemanager.NewProjectWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func projectResponseFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.GetProjectResponse { + project := resourcemanager.NewGetProjectResponseWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func serviceAccountFixture(email, projectID string) *serviceaccount.ServiceAccount { + serviceAccountObject := serviceaccount.NewServiceAccountWithDefaults() + serviceAccountObject.SetEmail(email) + serviceAccountObject.SetProjectId(projectID) + serviceAccountObject.SetId("service-account-id") + serviceAccountObject.SetInternal(false) + return serviceAccountObject +} + +func createServiceAccountKeyResponseFixture(email string) *serviceaccount.CreateServiceAccountKeyResponse { + credentials := serviceaccount.NewCreateServiceAccountKeyResponseCredentials( + "https://accounts.stackit.cloud", + email, + "00000000-0000-0000-0000-000000000001", + "00000000-0000-0000-0000-000000000002", + ) + credentials.SetPrivateKey("PRIVATE") + credentials.SetTokenEndpoint("https://accounts.stackit.cloud/oauth/v2/token") + + return serviceaccount.NewCreateServiceAccountKeyResponse( + true, + time.Unix(0, 0).UTC(), + *credentials, + "00000000-0000-0000-0000-000000000003", + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYALGORITHM_RSA_2048, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYORIGIN_GENERATED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYTYPE_USER_MANAGED, + "PUBLIC KEY", + ) +} + +var _ types.Options = fakeOptions{} diff --git a/test/kustomize/cloud-config.yaml b/test/kustomize/cloud-config.yaml index a96763e0..dd74e91c 100644 --- a/test/kustomize/cloud-config.yaml +++ b/test/kustomize/cloud-config.yaml @@ -6,7 +6,7 @@ metadata: data: cloud.yaml: |- global: - projectId: REPLACE_WITH_PROJECTID + projectId: stackit-project-id region: eu01 blockStorage: rescanOnResize: true diff --git a/test/kustomize/storageclass.yaml b/test/kustomize/storageclass.yaml index 2bba6f99..0b454e66 100644 --- a/test/kustomize/storageclass.yaml +++ b/test/kustomize/storageclass.yaml @@ -3,5 +3,7 @@ kind: StorageClass metadata: name: premium-perf4-stackit-kubetest2 provisioner: kubetest2.csi.stackit.cloud +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer parameters: type: "storage_premium_perf4"