perf(net): reduce health-check probe and reconfiguration overhead

api/net/v1alpha1/types.go

@@ -92,12 +92,14 @@ type HealthCheckSettings struct {
 
 	// ReceiveInterval is the minimum interval between received health check packets.
 	// Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds.
+	// Defaults to 15s when omitted from the selected health check scope.
 	// +kubebuilder:validation:XIntOrString
 	// +optional
 	ReceiveInterval *intstr.IntOrString `json:"receiveInterval,omitempty"`
 
 	// TransmitInterval is the minimum interval between transmitted health check packets.
 	// Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds.
+	// Defaults to 15s when omitted from the selected health check scope.
 	// +kubebuilder:validation:XIntOrString
 	// +optional
 	TransmitInterval *intstr.IntOrString `json:"transmitInterval,omitempty"`

cmd/kubectl-unbounded/app/net/create.go

@@ -35,16 +35,16 @@ type healthCheckFlags struct {
 // addToFlags registers health check flags with a command.
 func (b *healthCheckFlags) addToFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolVar(&b.enabled, "health-check-enabled", false, "Enable UDP health probes over tunnels")
-	cmd.Flags().Int32Var(&b.detectMultiplier, "health-check-detect-multiplier", 0, "Number of missed probes before marking a peer down")
-	cmd.Flags().StringVar(&b.receiveInterval, "health-check-receive-interval", "", "Min interval between received probes before declaring down, e.g. 300ms")
-	cmd.Flags().StringVar(&b.transmitInterval, "health-check-transmit-interval", "", "Interval between transmitted health probes, e.g. 300ms")
+	cmd.Flags().Int32Var(&b.detectMultiplier, "health-check-detect-multiplier", 0, "Number of missed probes before marking a peer down (node default: 3)")
+	cmd.Flags().StringVar(&b.receiveInterval, "health-check-receive-interval", "", "Expected interval between received probes; down timeout is detect multiplier * max(receive, transmit), e.g. 300ms (node default: 15s)")
+	cmd.Flags().StringVar(&b.transmitInterval, "health-check-transmit-interval", "", "Interval between transmitted health probes, e.g. 300ms (node default: 15s)")
 	cmd.Flags().Int32Var(&b.tunnelMTU, "tunnel-mtu", 0, "MTU for tunnel interfaces in this scope")
 	cmd.Flags().StringVar(&b.tunnelProtocol, "tunnel-protocol", "", "Tunnel encapsulation protocol (WireGuard, GENEVE, or Auto)")
 	_ = cmd.RegisterFlagCompletionFunc("tunnel-protocol", func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { //nolint:errcheck
 		return []string{"WireGuard", "GENEVE", "Auto"}, cobra.ShellCompDirectiveNoFileComp
 	})
 	_ = cmd.RegisterFlagCompletionFunc("health-check-receive-interval", func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { //nolint:errcheck
-		return cobra.AppendActiveHelp(nil, "Minimum interval between received health probes before declaring down. Duration, e.g. 300ms or 1s"), cobra.ShellCompDirectiveNoFileComp
+		return cobra.AppendActiveHelp(nil, "Expected interval between received health probes. Down timeout is detect multiplier * max(receive, transmit). Duration, e.g. 300ms or 1s"), cobra.ShellCompDirectiveNoFileComp
 	})
 	_ = cmd.RegisterFlagCompletionFunc("health-check-transmit-interval", func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { //nolint:errcheck
 		return cobra.AppendActiveHelp(nil, "Interval between transmitted health probes. Duration, e.g. 300ms or 1s"), cobra.ShellCompDirectiveNoFileComp

cmd/kubectl-unbounded/app/net/create_test.go

@@ -7,8 +7,49 @@ import (
 	"bytes"
 	"strings"
 	"testing"
+
+	"github.com/spf13/cobra"
 )
 
+func TestHealthCheckFlagsPreserveRuntimeDefaults(t *testing.T) {
+	cmd := &cobra.Command{}
+	flags := &healthCheckFlags{}
+	flags.addToFlags(cmd)
+	flags.selectedFrom(cmd)
+
+	if flags.toObject() != nil {
+		t.Fatal("omitted health flags must preserve the node's runtime defaults")
+	}
+
+	for _, name := range []string{"health-check-transmit-interval", "health-check-receive-interval"} {
+		flag := cmd.Flags().Lookup(name)
+		if flag.DefValue != "" || !strings.Contains(flag.Usage, "15s") {
+			t.Fatalf("flag %s must document the inherited 15s default without serializing it", name)
+		}
+	}
+
+	multiplier := cmd.Flags().Lookup("health-check-detect-multiplier")
+	if multiplier.DefValue != "0" || !strings.Contains(multiplier.Usage, "node default: 3") {
+		t.Fatal("detect multiplier must document the inherited default without serializing it")
+	}
+
+	receive := cmd.Flags().Lookup("health-check-receive-interval")
+	if !strings.Contains(receive.Usage, "detect multiplier * max(receive, transmit)") {
+		t.Fatal("receive interval must document how it contributes to the down timeout")
+	}
+
+	if err := cmd.Flags().Set("health-check-transmit-interval", "60s"); err != nil {
+		t.Fatal(err)
+	}
+
+	flags.selectedFrom(cmd)
+
+	got := flags.toObject()
+	if len(got) != 1 || got["transmitInterval"] != "60s" {
+		t.Fatalf("explicit interval or partial settings changed: %v", got)
+	}
+}
+
 func TestCreateSiteUsesSharedSiteAPI(t *testing.T) {
 	t.Parallel()
 

cmd/unbounded-net-node/cni_reconcile_test.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 
 	unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3"
+	"github.com/Azure/unbounded/internal/net/healthcheck"
 	unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink"
 )
 
@@ -57,7 +58,7 @@ func TestCNIReconciliationDisablesAndRecoversWithoutMTUChange(t *testing.T) {
 	})
 
 	ensureCNIBridgeMTUFunc = func(string, int, *unboundednetnetlink.NetlinkCache, bool) error { return nil }
-	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error {
+	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error {
 		return nil
 	}
 

cmd/unbounded-net-node/gateway_pool_peering_protocol_test.go

@@ -15,6 +15,7 @@ import (
 
 	unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3"
 	unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1"
+	"github.com/Azure/unbounded/internal/net/healthcheck"
 	unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink"
 )
 
@@ -180,7 +181,7 @@ func newPoolPeeringProtocolFixture(t *testing.T, gateway bool) *poolPeeringProto
 	}
 
 	original := configureWireGuardFunc
-	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, mesh []meshPeerInfo, gateways []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error {
+	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, mesh []meshPeerInfo, gateways []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error {
 		f.configureCalls++
 
 		f.gateways = append([]gatewayPeerInfo(nil), gateways...)

cmd/unbounded-net-node/main_update_test.go

@@ -17,6 +17,7 @@ import (
 
 	unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3"
 	unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1"
+	"github.com/Azure/unbounded/internal/net/healthcheck"
 	unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink"
 )
 
@@ -140,7 +141,7 @@ func TestUpdateWireGuardFromSlices_SitePodCIDRPoolChanges(t *testing.T) {
 			)
 
 			origConfigure := configureWireGuardFunc
-			configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, state *wireGuardState) error {
+			configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, state *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error {
 				configureCalls++
 
 				gotPools = append([]string(nil), state.sitePodCIDRPools...)
@@ -334,7 +335,7 @@ func TestUpdateWireGuardFromSlices_GatewayMeshPeersUseOnlyDirectConnectedSites(t
 	var gotPeers []meshPeerInfo
 
 	origConfigure := configureWireGuardFunc
-	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error {
+	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error {
 		gotPeers = append([]meshPeerInfo(nil), peers...)
 		return nil
 	}
@@ -439,7 +440,7 @@ func TestUpdateWireGuardFromSlices_ExternalGatewayIncludesAssignedNonDirectSites
 	var gotPeers []meshPeerInfo
 
 	origConfigure := configureWireGuardFunc
-	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error {
+	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error {
 		gotPeers = append([]meshPeerInfo(nil), peers...)
 		return nil
 	}
@@ -524,7 +525,7 @@ func TestUpdateWireGuardFromSlices_NonGatewayMeshPeersUseOnlyPeeredSites(t *test
 	var gotPeers []meshPeerInfo
 
 	origConfigure := configureWireGuardFunc
-	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error {
+	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error {
 		gotPeers = append([]meshPeerInfo(nil), peers...)
 		return nil
 	}
@@ -610,7 +611,7 @@ func TestUpdateWireGuardFromSlices_ManageCniPluginFalseSkipsPodCIDRRoutesForSame
 		var gotGatewayPeers []gatewayPeerInfo
 
 		origConfigure := configureWireGuardFunc
-		configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error {
+		configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error {
 			gotGatewayPeers = append([]gatewayPeerInfo(nil), gatewayPeers...)
 			return nil
 		}
@@ -697,7 +698,7 @@ func TestUpdateWireGuardFromSlices_ManageCniPluginFalseSkipsPodCIDRRoutesForSame
 		var gotPeers []meshPeerInfo
 
 		origConfigure := configureWireGuardFunc
-		configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error {
+		configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error {
 			gotPeers = append([]meshPeerInfo(nil), peers...)
 			return nil
 		}
@@ -791,7 +792,7 @@ func TestUpdateWireGuardFromSlices_ManageCniPluginFalseKeepsRemotePeeredMeshPeer
 	var gotPeers []meshPeerInfo
 
 	origConfigure := configureWireGuardFunc
-	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error {
+	configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error {
 		gotPeers = append([]meshPeerInfo(nil), peers...)
 		return nil
 	}

cmd/unbounded-net-node/peer_healthcheck.go

@@ -4,13 +4,38 @@
 package main
 
 import (
+	"errors"
+	"fmt"
 	"net"
+	"time"
 
 	"k8s.io/klog/v2"
 
 	"github.com/Azure/unbounded/internal/net/healthcheck"
 )
 
+// A disabled association must block fallback to less-specific enabled profiles.
+const disabledHealthCheckProfile = "disabled"
+
+var errRegisterHealthChecks = errors.New("health check registration failed")
+
+func resolvedHealthCheckSettings(name string, profiles map[string]healthcheck.HealthCheckSettings, maxBackoff time.Duration) (healthcheck.HealthCheckSettings, bool, error) {
+	if name == "" || name == disabledHealthCheckProfile {
+		return healthcheck.HealthCheckSettings{}, false, nil
+	}
+
+	settings, ok := profiles[name]
+	if !ok {
+		return healthcheck.HealthCheckSettings{}, false, fmt.Errorf("health check profile %q is missing from the current reconciliation", name)
+	}
+
+	if maxBackoff > 0 {
+		settings.MaxBackoff = maxBackoff
+	}
+
+	return settings, true, nil
+}
+
 // registerPeersWithHealthCheck registers mesh and gateway peers with the
 // healthcheck manager, resolving HC profiles for each peer. It sets
 // state.meshPeerHealthCheckEnabled and state.gatewayPeerHealthCheckEnabled
@@ -24,7 +49,8 @@ import (
 // when no pool/assignment-level profile is found for a gateway peer. This is
 // used by GENEVE which has no WireGuard handshake as a liveness signal.
 //
-// Returns the set of peer names that were registered (desiredHCPeers).
+// Returns desired peer names and registration errors. On error, names are retained
+// so a failed configuration does not remove an existing healthy session.
 func registerPeersWithHealthCheck(
 	meshPeers []meshPeerInfo,
 	gatewayPeers []gatewayPeerInfo,
@@ -35,15 +61,18 @@ func registerPeersWithHealthCheck(
 	assignmentSiteHCProfileNames map[string]string,
 	assignmentPoolHCProfileNames map[string]string,
 	poolHCProfileNames map[string]string,
+	profiles map[string]healthcheck.HealthCheckSettings,
 	state *wireGuardState,
 	peerIfaceNameFn func(gatewayPeerInfo) string,
 	useSiteFallbackForGateway bool,
-) map[string]bool {
+) (map[string]bool, error) {
 	desiredHCPeers := make(map[string]bool)
 	if state.healthCheckManager == nil {
-		return desiredHCPeers
+		return desiredHCPeers, nil
 	}
 
+	var registrationErrors []error
+
 	// Mesh peers.
 	for _, peer := range meshPeers {
 		overlayIP := getHealthIPFromPodCIDRs(peer.PodCIDRs)
@@ -53,7 +82,16 @@ func registerPeersWithHealthCheck(
 
 		hcProfileName := resolveMeshPeerHealthCheckProfileName(isGatewayNode, peer, mySiteName,
 			siteHCProfileNames, peeringHCProfileNames, assignmentSiteHCProfileNames)
-		if hcProfileName == "" {
+
+		settings, enabled, err := resolvedHealthCheckSettings(hcProfileName, profiles, state.healthFlapMaxBackoff)
+		if err != nil {
+			desiredHCPeers[peer.Name] = true
+			registrationErrors = append(registrationErrors, fmt.Errorf("mesh peer %s: %w", peer.Name, err))
+
+			continue
+		}
+
+		if !enabled {
 			continue
 		}
 
@@ -64,13 +102,8 @@ func registerPeersWithHealthCheck(
 			state.mu.Unlock()
 		}
 
-		settings := healthcheck.DefaultSettings()
-		if state.healthFlapMaxBackoff > 0 {
-			settings.MaxBackoff = state.healthFlapMaxBackoff
-		}
-
 		if err := state.healthCheckManager.AddPeer(peer.Name, net.ParseIP(overlayIP), settings); err != nil {
-			klog.V(2).Infof("Healthcheck: failed to register mesh peer %s at %s: %v", peer.Name, overlayIP, err)
+			registrationErrors = append(registrationErrors, fmt.Errorf("register mesh peer %s at %s: %w", peer.Name, overlayIP, err))
 		} else {
 			klog.V(4).Infof("Healthcheck: registered mesh peer %s at %s", peer.Name, overlayIP)
 		}
@@ -94,7 +127,15 @@ func registerPeersWithHealthCheck(
 			hcProfileName = siteHCProfileNames[mySiteName]
 		}
 
-		if hcProfileName == "" {
+		settings, enabled, err := resolvedHealthCheckSettings(hcProfileName, profiles, state.healthFlapMaxBackoff)
+		if err != nil {
+			desiredHCPeers[gwPeer.Name] = true
+			registrationErrors = append(registrationErrors, fmt.Errorf("gateway peer %s: %w", gwPeer.Name, err))
+
+			continue
+		}
+
+		if !enabled {
 			continue
 		}
 
@@ -104,19 +145,18 @@ func registerPeersWithHealthCheck(
 		state.gatewayPeerHealthCheckEnabled[ifName] = true
 		state.mu.Unlock()
 
-		settings := healthcheck.DefaultSettings()
-		if state.healthFlapMaxBackoff > 0 {
-			settings.MaxBackoff = state.healthFlapMaxBackoff
-		}
-
 		if err := state.healthCheckManager.AddPeer(gwPeer.Name, net.ParseIP(overlayIP), settings); err != nil {
-			klog.V(2).Infof("Healthcheck: failed to register gateway peer %s at %s: %v", gwPeer.Name, overlayIP, err)
+			registrationErrors = append(registrationErrors, fmt.Errorf("register gateway peer %s at %s: %w", gwPeer.Name, overlayIP, err))
 		} else {
 			klog.V(4).Infof("Healthcheck: registered gateway peer %s at %s (iface %s)", gwPeer.Name, overlayIP, ifName)
 		}
 	}
 
-	return desiredHCPeers
+	if len(registrationErrors) > 0 {
+		return desiredHCPeers, fmt.Errorf("%w: %w", errRegisterHealthChecks, errors.Join(registrationErrors...))
+	}
+
+	return desiredHCPeers, nil
 }
 
 // peerIfaceNameWireGuard maps a gateway peer to its WireGuard interface name