Add a simulated dev/test environment and simulator-based e2e CI

.github/workflows/e2e-simulator.yml

@@ -0,0 +1,179 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: E2E (CloudStack simulator)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Build the CCM image once and share it with every matrix cell. The
+  # distroless image is small enough that passing it as an artifact is much
+  # cheaper than four redundant builds.
+  build:
+    if: github.repository == 'apache/cloudstack-kubernetes-provider' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Build CCM image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          platforms: linux/amd64
+          tags: apache/cloudstack-kubernetes-provider:e2e
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Export image
+        run: docker save apache/cloudstack-kubernetes-provider:e2e | zstd -T0 -o ccm-image.tar.zst
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ccm-image
+          path: ccm-image.tar.zst
+          retention-days: 1
+
+  e2e:
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s: ['v1.37.0', 'v1.36.4']
+        acs: ['4.22.1.0', '4.20.2.0']
+    env:
+      KIND_NODE_IMAGE: kindest/node:${{ matrix.k8s }}
+      SIM_TAG: ${{ matrix.acs }}
+      CCM_IMAGE: apache/cloudstack-kubernetes-provider:e2e
+      CMK_VERSION: "6.5.0"
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+
+      # Released image tags never change, so cache them per version rather than
+      # pulling on every run. Keyed separately so the ~2 GB simulator tarball is
+      # shared across both Kubernetes versions instead of once per matrix cell.
+      # `docker save` output is already compressed (Docker keeps the layer
+      # blobs), so these are ~2 GB and ~370 MB respectively.
+      - name: Restore simulator image
+        uses: actions/cache@v4
+        with:
+          path: /tmp/images/simulator.tar
+          key: image-simulator-${{ matrix.acs }}
+
+      - name: Restore kind node image
+        uses: actions/cache@v4
+        with:
+          path: /tmp/images/kind-node.tar
+          key: image-kind-node-${{ matrix.k8s }}
+
+      - name: Load images
+        run: |
+          mkdir -p /tmp/images
+          load_or_pull() {
+            if [ -f "$1" ]; then
+              docker load -i "$1"
+            else
+              docker pull "$2"
+              docker save "$2" -o "$1"
+            fi
+          }
+          load_or_pull /tmp/images/simulator.tar "apache/cloudstack-simulator:${SIM_TAG}"
+          load_or_pull /tmp/images/kind-node.tar "${KIND_NODE_IMAGE}"
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: ccm-image
+
+      - name: Load CCM image
+        run: zstd -dc ccm-image.tar.zst | docker load
+
+      - uses: helm/kind-action@v1
+        with:
+          install_only: true
+          version: v0.32.0
+
+      - name: Install cmk (CloudMonkey)
+        run: |
+          curl -fsSL -o /usr/local/bin/cmk \
+            "https://github.com/apache/cloudstack-cloudmonkey/releases/download/${CMK_VERSION}/cmk.linux.x86-64"
+          chmod +x /usr/local/bin/cmk
+          cmk version
+
+      - name: Start simulator and deploy zone
+        run: hack/e2e/10-simulator-up.sh
+
+      - name: Create kind cluster
+        run: hack/e2e/20-kind-up.sh
+
+      - name: Create isolated network topology
+        run: hack/e2e/30-topology-isolated.sh
+
+      - name: Deploy the cloud controller manager
+        run: hack/e2e/40-ccm-deploy.sh
+
+      # These go through the make targets rather than repeating the commands,
+      # so CI and a local run cannot drift apart -- in particular the API
+      # endpoint is derived from SIM_HOST_PORT in one place.
+      - name: Unit and acceptance tests against the live simulator
+        run: |
+          . hack/e2e/_out/keys.env
+          make test
+
+      - name: E2E phase 1 (load balancer, nodes, annotations)
+        run: make test-e2e
+
+      - name: Create VPC topology
+        run: make e2e-vpc
+
+      - name: E2E phase 2 (VPC / network ACL)
+        run: make test-e2e-vpc
+
+      - name: Collect artifacts
+        if: always()
+        run: hack/e2e/90-collect-artifacts.sh
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: e2e-artifacts-${{ matrix.k8s }}-${{ matrix.acs }}
+          path: hack/e2e/_out/artifacts
+          retention-days: 7
+
+      - name: Tear down
+        if: always()
+        run: hack/e2e/99-down.sh

.gitignore

@@ -26,6 +26,12 @@ go.work.sum
 # env file
 .env
 
+# Local dev/test credentials and generated harness state — never commit these
+/cloud-config
+/cmk-config
+/kube-config
+/hack/e2e/_out/
+
 # Editor/IDE
 .idea/
 .vscode/

.golangci.yml

@@ -19,6 +19,9 @@ version: "2"
 run:
   modules-download-mode: readonly
   issues-exit-code: 1
+  # Lint the build-tagged e2e suite too; without this, goheader/gosec silently skip it.
+  build-tags:
+    - e2e
 linters:
   enable:
     - goheader

Makefile

@@ -29,10 +29,21 @@ LDFLAGS="-X k8s.io/kubernetes/pkg/version.gitVersion=${GIT_VERSION} -X k8s.io/ku
 export CGO_ENABLED=0
 export GO111MODULE=on
 
+# Keep these in step with hack/e2e/env.sh: both are overridable from the
+# environment, so `make test-e2e` reaches the same simulator `make e2e-up`
+# published rather than assuming the default port.
+SIM_HOST_PORT ?= 8080
+CS_API_URL ?= http://localhost:$(SIM_HOST_PORT)/client/api
+# Exported so the harness scripts and the acceptance tests in `make test` --
+# which read CS_API_URL from the environment -- agree with the targets here
+# without every caller having to repeat the endpoint.
+export SIM_HOST_PORT
+export CS_API_URL
+
 CMD_SRC=\
 	cmd/cloudstack-ccm/main.go
 
-.PHONY: all clean docker
+.PHONY: all clean docker e2e-up e2e-down e2e-vpc test-e2e test-e2e-vpc
 
 all: cloudstack-ccm
 
@@ -53,6 +64,38 @@ ifneq (${GIT_IS_TAG},NOT_A_TAG)
 	docker tag apache/cloudstack-kubernetes-provider:${GIT_COMMIT_SHORT} apache/cloudstack-kubernetes-provider:${GIT_TAG}
 endif
 
+# Simulator-based e2e environment; see docs/development.md
+e2e-up:
+	hack/e2e/up.sh
+
+e2e-down:
+	hack/e2e/99-down.sh
+
+# go test runs with the package directory as its working directory, so
+# KUBECONFIG must be absolute.
+test-e2e:
+	@test -f hack/e2e/_out/keys.env || (echo "environment not up; run 'make e2e-up' first" && exit 1)
+	. hack/e2e/_out/keys.env && \
+	KUBECONFIG=${CURDIR}/hack/e2e/_out/kubeconfig \
+	CS_API_URL=$(CS_API_URL) \
+	go test -tags e2e -v -timeout 30m ./test/e2e/... -run 'TestLB|TestNode|TestAnnot'
+
+# Phase 2. Run hack/e2e/50-topology-vpc.sh first: it creates the project, VPC
+# and tier, and re-points the CCM at the project. These tests only exercise
+# anything with CS_PROJECT_ID set, and skip otherwise.
+e2e-vpc:
+	hack/e2e/50-topology-vpc.sh
+
+test-e2e-vpc:
+	@test -f hack/e2e/_out/ids.env || (echo "VPC topology not created; run 'make e2e-vpc' first" && exit 1)
+	@grep -q E2E_PROJECT_ID hack/e2e/_out/ids.env || (echo "VPC topology not created; run 'make e2e-vpc' first" && exit 1)
+	. hack/e2e/_out/keys.env && . hack/e2e/_out/ids.env && \
+	KUBECONFIG=${CURDIR}/hack/e2e/_out/kubeconfig \
+	CS_API_URL=$(CS_API_URL) \
+	CS_PROJECT_ID="$$E2E_PROJECT_ID" \
+	E2E_ACL_ID="$$E2E_ACL_ID" E2E_VPC_ID="$$E2E_VPC_ID" \
+	go test -tags e2e -v -timeout 30m ./test/e2e/... -run 'TestVPC'
+
 lint: gofmt
 	@(echo "Running golangci-lint...")
 	golangci-lint run

README.md

@@ -52,6 +52,11 @@ explicitly set `region` in that case.
 
 The access token needs to be able to fetch VM information and deploy load balancers in the project or domain where the nodes reside.
 
+The account must also be allowed to call `listManagementServersMetrics`, which the controller uses
+on startup to determine the management server version. This is a root admin API and is **not**
+included in the default `User` role; without it the controller exits immediately with
+`no management servers found`.
+
 To create the secret, use the following command:
 ```bash
 kubectl -n kube-system create secret generic cloudstack-secret --from-file=cloud-config
@@ -423,9 +428,15 @@ make docker
 
 ### Testing
 
-You need a local instance of the CloudStack Management Server or a 'real' one to connect to.
+Unit tests need nothing but Go:
+
+```bash
+make test
+```
+
+For anything beyond that you need a CloudStack Management Server to talk to.
 The CCM supports the same cloud-config configuration file format used by [the cs tool](https://github.com/exoscale/cs),
-so you can simply point it to that.
+so you can simply point it at one you already have:
 
 ```bash
 ./cloudstack-ccm --cloud-provider external-cloudstack --cloud-config ./cloud-config --kubeconfig ~/.kube/config
@@ -434,45 +445,21 @@ so you can simply point it to that.
 Point `--kubeconfig` at a kubeconfig for your Kubernetes development cluster, and `--cloud-config` at
 a `cloud-config` for the CloudStack installation you want to talk to.
 
-If you don't have a 'real' CloudStack installation, you can also launch a local [simulator instance](https://hub.docker.com/r/cloudstack/simulator) instead. This is very useful for dry-run testing.
-
-### Debugging
-
-You can use the VSCode extension [Go](https://marketplace.visualstudio.com/items?itemName=golang.go) to debug the CCM.
-Add the following configuration to the `.vscode/launch.json` file to launch the CCM and debug it.
-
-```json
-{
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Launch CloudStack CCM",
-            "type": "go",
-            "request": "launch",
-            "mode": "auto",
-            "program": "${workspaceFolder}/cmd/cloudstack-ccm",
-            "env": {},
-            "args": [
-                "--cloud-provider=external-cloudstack",
-                "--cloud-config=${workspaceFolder}/cloud-config",
-                "--kubeconfig=${env:HOME}/.kube/config",
-                "--leader-elect=false",
-                "--v=4"
-            ],
-            "showLog": true,
-            "trace": "verbose"
-        },
-        {
-            "name": "Attach to Process",
-            "type": "go",
-            "request": "attach",
-            "mode": "local",
-            "processId": 0
-        }
-    ]
-}
+If you don't have a 'real' CloudStack installation, you don't need one. The repository ships a
+fully simulated environment — a kind cluster, the
+[CloudStack simulator](https://hub.docker.com/r/apache/cloudstack-simulator) and the CCM, all in
+containers:
+
+```bash
+make e2e-up      # bring the environment up
+make test-e2e    # run the end-to-end suite against it
+make e2e-down    # tear it down
 ```
 
+See [docs/development.md](docs/development.md) for the full walkthrough, how to run the CCM as a
+host process under a debugger, the VPC scenario (`make e2e-vpc` / `make test-e2e-vpc`), and
+troubleshooting.
+
 ## Copyright
 
 Copyright 2019 The Apache Software Foundation

cloudstack.go

@@ -128,7 +128,14 @@ func (cs *CSCloud) getManagementServerVersion() (semver.Version, error) {
 		return semver.Version{}, errors.New("no management servers found")
 	}
 	version := msServersResp.ManagementServersMetrics[0].Version
-	v, err := semver.ParseTolerant(strings.Join(strings.Split(version, ".")[0:3], "."))
+	// Trim to major.minor.patch. Slicing blindly would panic on a version
+	// string with fewer than three components, crashing the controller at
+	// startup instead of reporting a parse failure.
+	parts := strings.Split(version, ".")
+	if len(parts) > 3 {
+		parts = parts[:3]
+	}
+	v, err := semver.ParseTolerant(strings.Join(parts, "."))
 	if err != nil {
 		klog.Errorf("failed to parse management server version: %v", err)
 		return semver.Version{}, err