From 3537f8233532614d45c08c1710963b936bb79530 Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Sat, 12 Sep 2026 18:54:04 +0200 Subject: [PATCH 1/3] NO-ISSUE: Label primary node and bootstrap OVN SBDB/NBDB RAFT cluster on workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In MicroShift multinode mode both nodes carry node-role.kubernetes.io/master, so ovnkube-master ran on all nodes — each started its own isolated OVN RAFT cluster with no knowledge of the other node. Three fixes: 1. pkg/node/kubelet.go: the primary node (identified by the absence of a bootstrap kubeconfig) receives the label node.microshift.io/role=primary at kubelet startup. 2. assets/components/ovn/multi-node/master/daemonset.yaml — sbdb/nbdb: containers detect whether they run on a worker by reading the [OvnSouth] address from the mounted ovnkube.conf. When the address points to a different host they pass --db-sb-cluster-remote-addr to join the primary's existing RAFT cluster instead of starting a new one. Each node then gets a local unix socket connected to the shared replicated database. 3. assets/components/ovn/multi-node/master/daemonset.yaml — ovnkube-master / ovnkube-cluster-manager: on worker nodes these containers idle (exec sleep infinity) because cluster-wide controller work must run only on the primary. The primary wins the NBDB leader election; workers' RAFT followers handle only database replication. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../ovn/multi-node/master/daemonset.yaml | 49 +++++++++++++++++++ pkg/node/kubelet.go | 3 ++ 2 files changed, 52 insertions(+) diff --git a/assets/components/ovn/multi-node/master/daemonset.yaml b/assets/components/ovn/multi-node/master/daemonset.yaml index fe79c19640..eb1756feee 100644 --- a/assets/components/ovn/multi-node/master/daemonset.yaml +++ b/assets/components/ovn/multi-node/master/daemonset.yaml @@ -141,6 +141,19 @@ spec: --db-nb-cluster-local-proto=tcp \ --no-monitor" + # On worker nodes the configmap contains [OvnNorth] address= pointing to + # the primary. Join the primary's NBDB RAFT cluster so this node gets a + # local unix socket connected to the shared database. + NB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnNorth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${NB_ADDR}" =~ ^tcp: ]] && [[ "${NB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + PRIMARY_IP="${NB_ADDR#tcp:}"; PRIMARY_IP="${PRIMARY_IP%%:*}" + OVN_ARGS="${OVN_ARGS} \ + --db-nb-cluster-remote-addr=$(bracketify ${PRIMARY_IP}) \ + --db-nb-cluster-remote-port=9643 \ + --db-nb-cluster-remote-proto=tcp" + fi + rm -f /run/ovn/ovnnb_db.sock echo "$(date -Iseconds) - starting nbdb" @@ -234,6 +247,8 @@ spec: name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /run/ovnkube-config/ + name: ovnkube-config - mountPath: /env name: env-overrides resources: @@ -288,6 +303,19 @@ spec: --db-sb-cluster-local-proto=tcp \ --no-monitor" + # On worker nodes the configmap contains [OvnSouth] address= pointing to + # the primary. Join the primary's SBDB RAFT cluster so this node gets a + # local unix socket connected to the shared database. + SB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnSouth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${SB_ADDR}" =~ ^tcp: ]] && [[ "${SB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + PRIMARY_IP="${SB_ADDR#tcp:}"; PRIMARY_IP="${PRIMARY_IP%%:*}" + OVN_ARGS="${OVN_ARGS} \ + --db-sb-cluster-remote-addr=$(bracketify ${PRIMARY_IP}) \ + --db-sb-cluster-remote-port=9644 \ + --db-sb-cluster-remote-proto=tcp" + fi + rm -f /run/ovn/ovnsb_db.sock echo "$(date -Iseconds) - starting sbdb " @@ -345,6 +373,8 @@ spec: name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /run/ovnkube-config/ + name: ovnkube-config - mountPath: /env name: env-overrides resources: @@ -371,6 +401,16 @@ spec: set +o allexport fi + # cluster-manager handles cluster-wide IPAM — only the primary should run it. + # Worker nodes run sbdb/nbdb as RAFT followers; the primary wins the NBDB + # leader election and handles all cluster-manager work. + SB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnSouth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${SB_ADDR}" =~ ^tcp: ]] && [[ "${SB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + echo "$(date -Iseconds) - worker node: cluster-manager is primary-only, idling" + exec sleep infinity + fi + echo "$(date -Iseconds) - starting ovnkube-cluster-manager, Node: ${K8S_NODE}" exec /usr/bin/ovnkube \ --init-cluster-manager "${K8S_NODE}" \ @@ -411,6 +451,15 @@ spec: set +o allexport fi + # ovnkube-master handles per-node network setup — only the primary should + # run the controller role; workers' local NBDB/SBDB are RAFT followers. + SB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnSouth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${SB_ADDR}" =~ ^tcp: ]] && [[ "${SB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + echo "$(date -Iseconds) - worker node: ovnkube-master is primary-only, idling" + exec sleep infinity + fi + # K8S_NODE_IP triggers reconcilation of this daemon when node IP changes echo "$(date -Iseconds) - starting ovnkube-master, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}" diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 6a0c8ce6d8..54385e4a74 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -89,6 +89,9 @@ func (s *KubeletServer) configure(cfg *config.Config) { kubeletFlags.NodeLabels["node-role.kubernetes.io/worker"] = "" kubeletFlags.NodeLabels["node.openshift.io/os_id"] = osID kubeletFlags.NodeLabels["node.kubernetes.io/instance-type"] = "rhde" + if !cfg.BootstrapKubeConfigExists() { + kubeletFlags.NodeLabels["node.microshift.io/role"] = "primary" + } kubeletConfig, err := loadConfigFile(filepath.Join(config.DataDir, "/resources/kubelet/config/config.yaml")) From d4af2309632c6e7cb7a95082a333c010aaa6cd09 Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Sat, 12 Sep 2026 18:54:04 +0200 Subject: [PATCH 2/3] NO-ISSUE: Fix OVN multinode SBDB connectivity for primary and worker nodes Commit 8a0f4f23eb removed --nb-address/--sb-address from ovnkube --init-node, which were the only mechanism pointing worker nodes at the primary's NB/SB databases. After that removal ovnkube fell back to local unix sockets causing each node to form an isolated OVN RAFT cluster. Three related fixes: 1. ovnkube.conf (configmap): in multinode mode add [OvnNorth]/[OvnSouth] stanzas with the primary's IP and NB/SB port. The sbdb/nbdb containers also use these to discover the primary RAFT cluster to join. 2. networking.go: only the primary writes the ovnkube-config ConfigMap so workers cannot overwrite the correct primary IP. Passes MultiNodeEnabled render param for the configmap template. 3. node/daemonset.yaml (ovnkube-node): on worker nodes, export OVN_NB_DB and OVN_SB_DB so ovnkube connects to the primary's databases over TCP (the ovnkube binary cannot parse [OvnNorth]/[OvnSouth] address= from the config file). Also set ovn-encap-type=geneve and ovn-encap-ip in OVS so ovn-controller can register the chassis and form geneve tunnels. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- assets/components/ovn/common/configmap.yaml | 8 ++++++ .../ovn/multi-node/node/daemonset.yaml | 15 +++++++++++ pkg/components/networking.go | 27 ++++++++++++------- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/assets/components/ovn/common/configmap.yaml b/assets/components/ovn/common/configmap.yaml index 7279ce1fb8..1f435a2d7f 100644 --- a/assets/components/ovn/common/configmap.yaml +++ b/assets/components/ovn/common/configmap.yaml @@ -34,3 +34,11 @@ data: election-lease-duration=137 election-renew-deadline=107 election-retry-period=26 +{{- if .MultiNodeEnabled}} + + [OvnNorth] + address=tcp:{{.NodeIP}}:{{.OVN_NB_PORT}} + + [OvnSouth] + address=tcp:{{.NodeIP}}:{{.OVN_SB_PORT}} +{{- end}} diff --git a/assets/components/ovn/multi-node/node/daemonset.yaml b/assets/components/ovn/multi-node/node/daemonset.yaml index 7d01ee561d..934837e1f5 100644 --- a/assets/components/ovn/multi-node/node/daemonset.yaml +++ b/assets/components/ovn/multi-node/node/daemonset.yaml @@ -146,6 +146,21 @@ spec: # the functionality depends on ip_forwarding being enabled fi + # On worker nodes, ovnkube cannot parse [OvnNorth]/[OvnSouth] address= from + # the config file (upstream config format mismatch). Export OVN_NB_DB and + # OVN_SB_DB so ovnkube connects to the primary's databases over TCP, and set + # the geneve encap external_ids so ovn-controller can register the chassis. + NB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnNorth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + SB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnSouth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${SB_ADDR}" =~ ^tcp: ]] && [[ "${SB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + export OVN_NB_DB="${NB_ADDR}" OVN_SB_DB="${SB_ADDR}" + ovs-vsctl --timeout=5 set Open_vSwitch . \ + "external_ids:ovn-encap-type=geneve" \ + "external_ids:ovn-encap-ip=${K8S_NODE_IP}" || true + fi + echo "I$(date "+%m%d %H:%M:%S.%N") - ovnkube-node - start ovnkube --init-node ${K8S_NODE}" exec /usr/bin/ovnkube \ --init-node "${K8S_NODE}" \ diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 2e6ccad927..31606affb4 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -110,17 +110,24 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri return err } - // Multinode only params: OVN_NB_PORT, OVN_SB_PORT + // Multinode only params: OVN_NB_PORT, OVN_SB_PORT, MultiNodeEnabled extraParams := assets.RenderParams{ - "OVNConfig": ovnConfig, - "KubeconfigPath": kubeconfigPath, - "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), - "OVN_NB_PORT": ovn.OVN_NB_PORT, - "OVN_SB_PORT": ovn.OVN_SB_PORT, - } - if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { - klog.Warningf("Failed to apply configMap %v %v", cm, err) - return err + "OVNConfig": ovnConfig, + "KubeconfigPath": kubeconfigPath, + "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), + "OVN_NB_PORT": ovn.OVN_NB_PORT, + "OVN_SB_PORT": ovn.OVN_SB_PORT, + "MultiNodeEnabled": cfg.MultiNode.Enabled, + } + // In multinode mode the configmap contains [OvnNorth]/[OvnSouth] stanzas + // with the primary's IP. Only the primary may write it; a worker applying + // the configmap would overwrite the primary IP with its own, breaking SBDB + // connectivity for every node that reads the configmap afterwards. + if !cfg.MultiNode.Enabled || !cfg.BootstrapKubeConfigExists() { + if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { + klog.Warningf("Failed to apply configMap %v %v", cm, err) + return err + } } if err := assets.ApplyDaemonSets(ctx, apps, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply apps %v %v", apps, err) From 0970af298c49e99912454cab4a5030bce86a7dc1 Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Sat, 12 Sep 2026 18:54:05 +0200 Subject: [PATCH 3/3] NO-ISSUE: Temporarily disable CNCF conformance exclusion for CI validation TODO: Revert before merging. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- test/bin/ci_phase_boot_and_test.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 79bd22e8c6..8cfb4dcf51 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -22,9 +22,12 @@ prepare_scenario_sources() { rm -rf "${SCENARIOS_TO_RUN}" mkdir -p "${SCENARIOS_TO_RUN}" cp "${SCENARIO_SOURCES}"/*.sh "${SCENARIOS_TO_RUN}"/ - if ${EXCLUDE_CNCF_CONFORMANCE}; then - find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete - fi + # TODO: Temporarily disabled so that the CNCF conformance scenario runs + # unconditionally while the multinode OVN SBDB fix (PR #7344) is validated + # in CI. Revert once the job is confirmed green. + # if ${EXCLUDE_CNCF_CONFORMANCE}; then + # find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete + # fi } # Log output automatically