From 40c0afa404f5e8100d117dccd914f09d0c4bb89d Mon Sep 17 00:00:00 2001 From: Lukas Hoehl Date: Wed, 24 Jun 2026 15:54:25 +0200 Subject: [PATCH 1/6] ccm: sort nics and support primary network in multi network case Signed-off-by: Lukas Hoehl --- pkg/ccm/instances.go | 33 +++++++++++++++++++++++++++++++-- pkg/ccm/instances_test.go | 36 +++++++++++++++++++++++++++++++++++- pkg/ccm/stackit.go | 2 +- pkg/stackit/config/config.go | 8 ++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) diff --git a/pkg/ccm/instances.go b/pkg/ccm/instances.go index 6e1d9a87..dc25d2a1 100644 --- a/pkg/ccm/instances.go +++ b/pkg/ccm/instances.go @@ -21,10 +21,12 @@ import ( "errors" "fmt" "regexp" + "slices" "strings" "github.com/stackitcloud/cloud-provider-stackit/pkg/labels" stackitclient "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/client" + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/config" "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" iaas "github.com/stackitcloud/stackit-sdk-go/services/iaas/v2api" corev1 "k8s.io/api/core/v1" @@ -49,13 +51,15 @@ type Instances struct { regionProviderID bool iaasClient stackitclient.IaaSClient region string + defaultNetwork string } -func NewInstance(client stackitclient.IaaSClient, region string) (*Instances, error) { +func NewInstance(client stackitclient.IaaSClient, region string, opts config.InstanceOpts) (*Instances, error) { return &Instances{ iaasClient: client, region: region, regionProviderID: false, + defaultNetwork: opts.DefaultNetwork, }, nil } @@ -104,7 +108,7 @@ func (i *Instances) InstanceMetadata(ctx context.Context, node *corev1.Node) (*c return nil, fmt.Errorf("server has no network interfaces") } - nics := server.GetNics() + nics := sortNics(server.GetNics(), i.defaultNetwork) for i := range nics { nic := &nics[i] if nic.HasIpv4() { @@ -153,6 +157,31 @@ func (i *Instances) makeInstanceID(server *iaas.Server) string { return fmt.Sprintf("%s://%s", ProviderName, server.GetId()) } +func sortNics(nics []iaas.ServerNetwork, defaultNetwork string) []iaas.ServerNetwork { + // nics are returned by IaaS API in a non-deterministic order + // Sort by network name so that every time we use the same order for node addresses + slices.SortFunc(nics, func(a, b iaas.ServerNetwork) int { + return strings.Compare(a.NetworkName, b.NetworkName) + }) + + if defaultNetwork == "" { + return nics + } + + idx := slices.IndexFunc(nics, func(nic iaas.ServerNetwork) bool { + return nic.NetworkName == defaultNetwork || nic.NetworkId == defaultNetwork + }) + // network not found + if idx == -1 { + return nics + } + defaultNic := nics[idx] + nics = slices.Delete(nics, idx, idx+1) + // prepend default nic + nics = slices.Insert(nics, 0, defaultNic) + return nics +} + // addToNodeAddresses appends the NodeAddresses to the passed-by-pointer slice, // only if they do not already exist func addToNodeAddresses(addresses *[]corev1.NodeAddress, addAddresses ...corev1.NodeAddress) { diff --git a/pkg/ccm/instances_test.go b/pkg/ccm/instances_test.go index 304e1709..77bc2451 100644 --- a/pkg/ccm/instances_test.go +++ b/pkg/ccm/instances_test.go @@ -26,6 +26,7 @@ import ( oapiError "github.com/stackitcloud/stackit-sdk-go/core/oapierror" stackitclientmock "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/client/mock" + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/config" iaas "github.com/stackitcloud/stackit-sdk-go/services/iaas/v2api" "go.uber.org/mock/gomock" corev1 "k8s.io/api/core/v1" @@ -49,7 +50,7 @@ var _ = Describe("Node Controller", func() { nodeMockClient = stackitclientmock.NewMockIaaSClient(ctrl) var err error - instance, err = NewInstance(nodeMockClient, "eu01") + instance, err = NewInstance(nodeMockClient, "eu01", config.InstanceOpts{}) Expect(err).NotTo(HaveOccurred()) }) @@ -281,4 +282,37 @@ var _ = Describe("Node Controller", func() { Expect(metadata).To(BeNil()) }) }) + + Describe("#sortNics", func() { + It("should return the nic of the default network as primary", func() { + nics := []iaas.ServerNetwork{ + { + NetworkName: "abc", + Ipv4: new("10.0.0.69"), + }, + { + NetworkName: "default", + Ipv4: new("192.168.0.123"), + }, + { + NetworkName: "foo", + NetworkId: "123", + Ipv4: new("100.80.0.5"), + }, + } + By("with network name") + newNics := sortNics(nics, "default") + Expect(newNics).To(HaveLen(3)) + Expect(newNics[0].NetworkName).To(Equal("default")) + Expect(newNics[1].NetworkName).To(Equal("abc")) + Expect(newNics[2].NetworkName).To(Equal("foo")) + + By("with network id") + newNics = sortNics(nics, "123") + Expect(newNics).To(HaveLen(3)) + Expect(newNics[0].NetworkName).To(Equal("foo")) + Expect(newNics[1].NetworkName).To(Equal("abc")) + Expect(newNics[2].NetworkName).To(Equal("default")) + }) + }) }) diff --git a/pkg/ccm/stackit.go b/pkg/ccm/stackit.go index 950bd933..10bc5a5f 100644 --- a/pkg/ccm/stackit.go +++ b/pkg/ccm/stackit.go @@ -151,7 +151,7 @@ func NewCloudControllerManager(cfg *stackitconfig.CCMConfig, obs *MetricsRemoteW return nil, fmt.Errorf("failed to create IaaS client: %v", err) } - instances, err := NewInstance(iaasClient, cfg.Global.Region) + instances, err := NewInstance(iaasClient, cfg.Global.Region, cfg.Instance) if err != nil { return nil, err } diff --git a/pkg/stackit/config/config.go b/pkg/stackit/config/config.go index cad4a805..4a88b5d1 100644 --- a/pkg/stackit/config/config.go +++ b/pkg/stackit/config/config.go @@ -19,6 +19,14 @@ type CCMConfig struct { Global GlobalOpts `yaml:"global"` Metadata metadata.Opts `yaml:"metadata"` LoadBalancer LoadBalancerOpts `yaml:"loadBalancer"` + Instance InstanceOpts `yaml:"instance"` +} + +type InstanceOpts struct { + // DefaultNetwork contains the default network to use for a node. + // It can contain either the network name or ID. + // Can be used in mulit-network scenario to indicate which NIC is the primary one. + DefaultNetwork string `yaml:"defaultNetwork"` } type LoadBalancerOpts struct { From f47ca0c5a130f0f01b14fd7b09ebd93032a93f36 Mon Sep 17 00:00:00 2001 From: Lukas Hoehl Date: Wed, 24 Jun 2026 16:03:48 +0200 Subject: [PATCH 2/6] docs Signed-off-by: Lukas Hoehl --- docs/cloud-controller-manager.md | 19 +++++++++++++++++++ docs/migration/configuration.md | 2 ++ 2 files changed, 21 insertions(+) create mode 100644 docs/cloud-controller-manager.md diff --git a/docs/cloud-controller-manager.md b/docs/cloud-controller-manager.md new file mode 100644 index 00000000..28e70c32 --- /dev/null +++ b/docs/cloud-controller-manager.md @@ -0,0 +1,19 @@ +# Cloud controller manager + +## Overview + +The cloud controller manager implements the [Kubernetes cloud-controller-manager contract](https://kubernetes.io/docs/concepts/architecture/cloud-controller/#functions-of-the-ccm). + +### Node controller + +#### Multi Network + +If a server has NICs connected to multiple networks, you can designate the primary network for Node Addresses by setting the default network in the config: + +```yaml +instance: + # either network name or id + defaultNetwork: "foo" +``` + +This ensures the IP address for that network's NIC is listed first in the node status. diff --git a/docs/migration/configuration.md b/docs/migration/configuration.md index a6cbd6fb..7d340bfa 100644 --- a/docs/migration/configuration.md +++ b/docs/migration/configuration.md @@ -79,6 +79,8 @@ loadBalancer: extraLabels: key1: value1 key2: value2 +instance: {} + # defaultNetwork: # used for multi-network nodes ``` ### CSI Configuration From bc5ce0a33fdbd0ecd7f3e62e499b1d462e25f3f3 Mon Sep 17 00:00:00 2001 From: Lukas Hoehl Date: Tue, 7 Jul 2026 13:21:14 +0200 Subject: [PATCH 3/6] address feedback Signed-off-by: Lukas Hoehl --- docs/cloud-controller-manager.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/cloud-controller-manager.md b/docs/cloud-controller-manager.md index 28e70c32..454a37af 100644 --- a/docs/cloud-controller-manager.md +++ b/docs/cloud-controller-manager.md @@ -6,9 +6,13 @@ The cloud controller manager implements the [Kubernetes cloud-controller-manager ### Node controller +The node controller is responsible for updating Node objects when new servers are created in STACKIT infrastructure by obtaining information about the servers. + +For more information check the [Kubernetes documentation](https://kubernetes.io/docs/concepts/architecture/cloud-controller/#node-controller). + #### Multi Network -If a server has NICs connected to multiple networks, you can designate the primary network for Node Addresses by setting the default network in the config: +If a server has NICs connected to multiple networks, you can designate the primary network for [Node Addresses](https://kubernetes.io/docs/reference/node/node-status/#addresses) by setting the default network in the config: ```yaml instance: @@ -16,4 +20,4 @@ instance: defaultNetwork: "foo" ``` -This ensures the IP address for that network's NIC is listed first in the node status. +This ensures the IP address for that network's NIC is listed first in the [Node status](https://kubernetes.io/docs/reference/node/node-status/#addresses). From 1ce0b76b279c1816947de5bbbcc0e86858a90d1c Mon Sep 17 00:00:00 2001 From: Lukas Hoehl Date: Tue, 7 Jul 2026 15:56:17 +0200 Subject: [PATCH 4/6] logline for missing network Signed-off-by: Lukas Hoehl --- pkg/ccm/instances.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/ccm/instances.go b/pkg/ccm/instances.go index dc25d2a1..5ee1b168 100644 --- a/pkg/ccm/instances.go +++ b/pkg/ccm/instances.go @@ -173,6 +173,7 @@ func sortNics(nics []iaas.ServerNetwork, defaultNetwork string) []iaas.ServerNet }) // network not found if idx == -1 { + klog.Infof("no NIC found for default network %s", defaultNetwork) return nics } defaultNic := nics[idx] From 5d9d94266ba565167f6f26649f5e207f4c666bd7 Mon Sep 17 00:00:00 2001 From: Lukas Hoehl Date: Tue, 7 Jul 2026 15:57:52 +0200 Subject: [PATCH 5/6] Apply suggestion from @nschad Co-authored-by: Niclas Schad --- pkg/ccm/instances.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/ccm/instances.go b/pkg/ccm/instances.go index 5ee1b168..08d3613d 100644 --- a/pkg/ccm/instances.go +++ b/pkg/ccm/instances.go @@ -157,6 +157,10 @@ func (i *Instances) makeInstanceID(server *iaas.Server) string { return fmt.Sprintf("%s://%s", ProviderName, server.GetId()) } +// sortNics sorts a slice of server network interfaces alphabetically by their network name +// to ensure a deterministic order. If a non-empty defaultNetwork is provided (matching either +// the NetworkName or NetworkId), that specific network interface is moved to the front (index 0) +// of the returned slice. func sortNics(nics []iaas.ServerNetwork, defaultNetwork string) []iaas.ServerNetwork { // nics are returned by IaaS API in a non-deterministic order // Sort by network name so that every time we use the same order for node addresses From be444436118a72512f92e56b0260410fe9db8ec8 Mon Sep 17 00:00:00 2001 From: Lukas Hoehl Date: Tue, 7 Jul 2026 16:07:02 +0200 Subject: [PATCH 6/6] test for ids Signed-off-by: Lukas Hoehl --- pkg/ccm/instances_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/ccm/instances_test.go b/pkg/ccm/instances_test.go index 77bc2451..0f9e558a 100644 --- a/pkg/ccm/instances_test.go +++ b/pkg/ccm/instances_test.go @@ -288,10 +288,12 @@ var _ = Describe("Node Controller", func() { nics := []iaas.ServerNetwork{ { NetworkName: "abc", + NetworkId: "69", Ipv4: new("10.0.0.69"), }, { NetworkName: "default", + NetworkId: "69", Ipv4: new("192.168.0.123"), }, { @@ -310,9 +312,9 @@ var _ = Describe("Node Controller", func() { By("with network id") newNics = sortNics(nics, "123") Expect(newNics).To(HaveLen(3)) - Expect(newNics[0].NetworkName).To(Equal("foo")) - Expect(newNics[1].NetworkName).To(Equal("abc")) - Expect(newNics[2].NetworkName).To(Equal("default")) + Expect(newNics[0].NetworkId).To(Equal("123")) + Expect(newNics[1].NetworkId).To(Equal("69")) + Expect(newNics[2].NetworkId).To(Equal("69")) }) }) })