net: preserve diagnostic and resource-online summary fidelity

cmd/unbounded-net-controller/node_overview.go

@@ -18,7 +18,9 @@ func (c *NodeStatusCache) StoreOverview(nodeName string, overview statusv1alpha1
 	}
 
 	if overview.PeerCount < 0 || overview.HealthyPeers < 0 ||
-		overview.HealthyPeers > overview.PeerCount || overview.RouteCount < 0 {
+		overview.HealthyPeers > overview.PeerCount || overview.RouteCount < 0 ||
+		overview.RouteMismatchCount < 0 || overview.UnhealthyPeerLinks < 0 ||
+		(overview.RouteMismatchCount > 0 && !overview.RouteMismatch) {
 		return 0, fmt.Errorf("summary contains invalid observed counts")
 	}
 

cmd/unbounded-net-controller/node_overview_test.go

@@ -86,6 +86,9 @@ func TestNodeOverviewCacheRejectsInvalidFacts(t *testing.T) {
 		{HealthyPeers: -1},
 		{PeerCount: 1, HealthyPeers: 2},
 		{RouteCount: -1},
+		{RouteMismatchCount: -1},
+		{UnhealthyPeerLinks: -1},
+		{RouteMismatchCount: 1},
 	} {
 		cache := NewNodeStatusCache()
 		if _, err := cache.StoreOverview("node", overview, "ws"); err == nil || cache.Len() != 0 {

cmd/unbounded-net-controller/overview_diagnostics_test.go

@@ -0,0 +1,119 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"encoding/json"
+	"slices"
+	"testing"
+	"time"
+
+	statuspkg "github.com/Azure/unbounded/internal/net/status"
+	statusproto "github.com/Azure/unbounded/internal/net/status/proto"
+	statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1"
+)
+
+func TestOverviewDiagnosticMessagesMatchFullProblems(t *testing.T) {
+	now := time.Now()
+	yes := true
+	node := &NodeStatusResponse{
+		NodeInfo: NodeInfo{Name: "node", K8sReady: "Ready", ProviderID: "azure://vm", WireGuard: &WireGuardStatusInfo{Interface: "wg0"}},
+		Peers: []WireGuardPeerStatus{
+			{Name: "same", Tunnel: PeerTunnelStatus{Protocol: "IPIP"}, HealthCheck: &HealthCheckPeerStatus{Status: " UP "}},
+			{Name: "same", Tunnel: PeerTunnelStatus{LastHandshake: now}, HealthCheck: &HealthCheckPeerStatus{Status: " down "}},
+			{Name: "same", HealthCheck: &HealthCheckPeerStatus{Enabled: true, Status: "DOWN"}},
+		},
+		RoutingTable: RoutingTableInfo{Routes: []RouteEntry{{
+			NextHops: []NextHop{{Expected: &yes}, {Present: &yes}, {Expected: &yes}},
+		}}},
+	}
+
+	fullProblems := collectClusterProblems(&ClusterStatusResponse{Nodes: []*NodeStatusResponse{node}})
+	if len(fullProblems) != 1 || len(fullProblems[0].Errors) != 3 {
+		t.Fatalf("legacy diagnostic fixture changed: %+v", fullProblems)
+	}
+
+	overview := statuspkg.OverviewFromStatus(node, now)
+	overview.NodeInfo.ProviderID = ""
+	messages := statuspkg.OverviewDiagnosticMessages(overview, node.NodeInfo.ProviderID)
+	slices.Sort(messages)
+
+	want := slices.Clone(fullProblems[0].Errors)
+	slices.Sort(want)
+
+	if !slices.Equal(messages, want) || overview.RouteMismatchCount != 3 || overview.UnhealthyPeerLinks != 2 {
+		t.Fatalf("summary diagnostics=%v, legacy=%v, overview=%+v", messages, want, overview)
+	}
+
+	if otherCloud := statuspkg.OverviewDiagnosticMessages(overview, "other://vm"); len(otherCloud) != 2 {
+		t.Fatalf("IPIP warning applied outside Azure: %v", otherCloud)
+	}
+
+	overview.UsesIPIP = false
+	if noIPIP := statuspkg.OverviewDiagnosticMessages(overview, node.NodeInfo.ProviderID); len(noIPIP) != 2 {
+		t.Fatalf("Azure warning applied without IPIP: %v", noIPIP)
+	}
+}
+
+func TestProtoOverviewPreservesDiagnosticFacts(t *testing.T) {
+	got := protoToNodeOverview(&statusproto.NodeStatusOverview{
+		NodeInfo:  &statusproto.NodeInfo{Name: "node", ProviderId: "azure://vm"},
+		PeerCount: 4, HealthyPeers: 1, RouteCount: 2, RouteMismatch: true,
+		RouteMismatchCount: 3, UnhealthyPeerLinks: 2, UsesIpip: true,
+	})
+	if got.RouteMismatchCount != 3 || got.UnhealthyPeerLinks != 2 || !got.UsesIPIP ||
+		got.PeerCount != 4 || got.HealthyPeers != 1 || !got.RouteMismatch || got.NodeInfo.ProviderID != "azure://vm" {
+		t.Fatalf("overview converter lost diagnostics: %+v", got)
+	}
+}
+
+func TestViewerSummaryPreservesInterfaceOnlineIndependentOfCNI(t *testing.T) {
+	for _, native := range []bool{false, true} {
+		for _, tc := range []struct {
+			name   string
+			status *WireGuardStatusInfo
+			online bool
+		}{
+			{"missing", nil, false},
+			{"public key without interface", &WireGuardStatusInfo{PublicKey: "key"}, false},
+			{"interface with CNI failure", &WireGuardStatusInfo{Interface: "wg0"}, true},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				node := &NodeStatusResponse{
+					NodeInfo:   NodeInfo{Name: "node", WireGuard: tc.status},
+					NodeErrors: []NodeError{{Type: "cni", Message: "blocked"}},
+				}
+
+				cluster := &ClusterStatusResponse{Nodes: []*NodeStatusResponse{node}}
+				if native {
+					cluster.NodeOverviews = map[string]*statusv1alpha1.NodeStatusOverview{
+						"node": {NodeInfo: node.NodeInfo, NodeErrors: node.NodeErrors},
+					}
+				}
+
+				summary := buildClusterSummary(cluster).NodeSummaries[0]
+				if summary.WireGuardOnline != tc.online || summary.ErrorCount != 1 ||
+					summary.FirstError != "blocked" || summary.CniStatus != "Errors" {
+					t.Fatalf("interface/CNI facts conflated: %+v", summary)
+				}
+
+				data, err := json.Marshal(summary)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				var decoded struct {
+					Online *bool `json:"wireGuardOnline"`
+				}
+				if err := json.Unmarshal(data, &decoded); err != nil {
+					t.Fatal(err)
+				}
+
+				if decoded.Online == nil || *decoded.Online != tc.online {
+					t.Fatalf("explicit online/offline fact omitted or changed: %s", data)
+				}
+			})
+		}
+	}
+}

cmd/unbounded-net-controller/status_overview_proto.go

@@ -18,6 +18,8 @@ func protoToNodeOverview(msg *statusproto.NodeStatusOverview) statusv1alpha1.Nod
 		NodePodInfo: protoToNodePodInfo(msg.NodePodInfo),
 		PeerCount:   int(msg.PeerCount), HealthyPeers: int(msg.HealthyPeers),
 		RouteCount: int(msg.RouteCount), RouteMismatch: msg.RouteMismatch,
+		RouteMismatchCount: int(msg.RouteMismatchCount),
+		UnhealthyPeerLinks: int(msg.UnhealthyPeerLinks), UsesIPIP: msg.UsesIpip,
 	}
 	if msg.NodeInfo != nil {
 		overview.NodeInfo = protoToNodeInfo(msg.NodeInfo)

cmd/unbounded-net-controller/status_types.go

@@ -163,20 +163,21 @@ type ClusterSummary struct {
 
 // NodeSummary is a compact per-node summary for use in ClusterSummary.
 type NodeSummary struct {
-	Name          string `json:"name"`
-	SiteName      string `json:"siteName,omitempty"`
-	IsGateway     bool   `json:"isGateway,omitempty"`
-	K8sReady      string `json:"k8sReady,omitempty"`
-	StatusSource  string `json:"statusSource,omitempty"`
-	CniStatus     string `json:"cniStatus,omitempty"`
-	CniTone       string `json:"cniTone,omitempty"`
-	ErrorCount    int    `json:"errorCount,omitempty"`
-	FirstError    string `json:"firstError,omitempty"`
-	PeerCount     int    `json:"peerCount,omitempty"`
-	HealthyPeers  int    `json:"healthyPeers,omitempty"`
-	RouteCount    int    `json:"routeCount,omitempty"`
-	RouteMismatch bool   `json:"routeMismatch,omitempty"`
-	FetchError    string `json:"fetchError,omitempty"`
+	Name            string `json:"name"`
+	SiteName        string `json:"siteName,omitempty"`
+	IsGateway       bool   `json:"isGateway,omitempty"`
+	K8sReady        string `json:"k8sReady,omitempty"`
+	StatusSource    string `json:"statusSource,omitempty"`
+	CniStatus       string `json:"cniStatus,omitempty"`
+	CniTone         string `json:"cniTone,omitempty"`
+	ErrorCount      int    `json:"errorCount,omitempty"`
+	FirstError      string `json:"firstError,omitempty"`
+	PeerCount       int    `json:"peerCount,omitempty"`
+	HealthyPeers    int    `json:"healthyPeers,omitempty"`
+	RouteCount      int    `json:"routeCount,omitempty"`
+	RouteMismatch   bool   `json:"routeMismatch,omitempty"`
+	FetchError      string `json:"fetchError,omitempty"`
+	WireGuardOnline bool   `json:"wireGuardOnline"`
 }
 
 // buildClusterSummary extracts a ClusterSummary from a full ClusterStatusResponse.
@@ -195,17 +196,18 @@ func buildClusterSummary(status *ClusterStatusResponse) *ClusterSummary {
 		}
 
 		ns := NodeSummary{
-			Name:          node.NodeInfo.Name,
-			SiteName:      node.NodeInfo.SiteName,
-			IsGateway:     node.NodeInfo.IsGateway,
-			K8sReady:      node.NodeInfo.K8sReady,
-			StatusSource:  node.StatusSource,
-			PeerCount:     overview.PeerCount,
-			HealthyPeers:  overview.HealthyPeers,
-			RouteCount:    overview.RouteCount,
-			RouteMismatch: overview.RouteMismatch,
-			FetchError:    node.FetchError,
-			ErrorCount:    len(node.NodeErrors),
+			Name:            node.NodeInfo.Name,
+			SiteName:        node.NodeInfo.SiteName,
+			IsGateway:       node.NodeInfo.IsGateway,
+			K8sReady:        node.NodeInfo.K8sReady,
+			StatusSource:    node.StatusSource,
+			PeerCount:       overview.PeerCount,
+			HealthyPeers:    overview.HealthyPeers,
+			RouteCount:      overview.RouteCount,
+			RouteMismatch:   overview.RouteMismatch,
+			FetchError:      node.FetchError,
+			ErrorCount:      len(node.NodeErrors),
+			WireGuardOnline: node.NodeInfo.WireGuard != nil && node.NodeInfo.WireGuard.Interface != "",
 		}
 
 		// Include first error message so the frontend can show it inline

cmd/unbounded-net-node/status_proto.go

@@ -24,17 +24,20 @@ func nodeSummaryToProto(summary *NodeStatusOverview) *statusproto.NodeStatusOver
 	}
 
 	result := &statusproto.NodeStatusOverview{
-		TimestampUnixNs: statusUnixNano(summary.Timestamp),
-		NodeInfo:        nodeInfoToProto(&summary.NodeInfo),
-		HealthCheck:     healthCheckStatusToProto(summary.HealthCheck),
-		NodeErrors:      nodeErrorsToProto(summary.NodeErrors),
-		FetchError:      summary.FetchError,
-		StatusSource:    summary.StatusSource,
-		NodePodInfo:     nodePodInfoToProto(summary.NodePodInfo),
-		PeerCount:       int32(summary.PeerCount),
-		HealthyPeers:    int32(summary.HealthyPeers),
-		RouteCount:      int32(summary.RouteCount),
-		RouteMismatch:   summary.RouteMismatch,
+		TimestampUnixNs:    statusUnixNano(summary.Timestamp),
+		NodeInfo:           nodeInfoToProto(&summary.NodeInfo),
+		HealthCheck:        healthCheckStatusToProto(summary.HealthCheck),
+		NodeErrors:         nodeErrorsToProto(summary.NodeErrors),
+		FetchError:         summary.FetchError,
+		StatusSource:       summary.StatusSource,
+		NodePodInfo:        nodePodInfoToProto(summary.NodePodInfo),
+		PeerCount:          int32(summary.PeerCount),
+		HealthyPeers:       int32(summary.HealthyPeers),
+		RouteCount:         int32(summary.RouteCount),
+		RouteMismatch:      summary.RouteMismatch,
+		RouteMismatchCount: int32(summary.RouteMismatchCount),
+		UnhealthyPeerLinks: int32(summary.UnhealthyPeerLinks),
+		UsesIpip:           summary.UsesIPIP,
 	}
 	if summary.LastPushTime != nil {
 		result.LastPushTimeUnixNs = statusUnixNano(*summary.LastPushTime)

cmd/unbounded-net-node/status_summary.go

@@ -32,6 +32,14 @@ func (s *nodeStatusServer) getNodeSummary() *NodeStatusOverview {
 			summary.HealthyPeers++
 		}
 
+		if !netstatus.PeerLinkHealthyForDiagnostics(&peer, now) {
+			summary.UnhealthyPeerLinks++
+		}
+
+		if peer.Tunnel.Protocol == "IPIP" {
+			summary.UsesIPIP = true
+		}
+
 		previous, seen := interfaceHealthy[peer.Tunnel.Interface]
 		interfaceHealthy[peer.Tunnel.Interface] = (!seen || previous) && peerStatusHealthy(peer, now)
 		routePeers = append(routePeers, routeplan.Peer{
@@ -46,7 +54,8 @@ func (s *nodeStatusServer) getNodeSummary() *NodeStatusOverview {
 	summary.NodeInfo = facts.NodeInfo
 	summary.NodeErrors = facts.NodeErrors
 	summary.HealthCheck = facts.HealthCheck
-	summary.RouteCount, summary.RouteMismatch = s.collectRouteSummary(routePeers, facts.NodeInfo.SiteName)
+	summary.RouteCount, summary.RouteMismatchCount = s.collectRouteSummary(routePeers, facts.NodeInfo.SiteName)
+	summary.RouteMismatch = summary.RouteMismatchCount > 0
 
 	if s.state.linkStatsMonitor != nil {
 		for _, warning := range s.state.linkStatsMonitor.GetWarnings() {

cmd/unbounded-net-node/status_summary_routes.go

@@ -44,7 +44,7 @@ func newRouteSummaryFamily(plan []routeplan.ExpectedRoute) *routeSummaryFamily {
 // collectRouteSummary preserves annotation counts, including synthetic missing
 // routes and the per-family unbounded0 suppression of missing tunnel hops.
 // It never creates status route arrays or fills the full-detail route cache.
-func (s *nodeStatusServer) collectRouteSummary(peers []routeplan.Peer, localSite string) (count int, mismatch bool) {
+func (s *nodeStatusServer) collectRouteSummary(peers []routeplan.Peer, localSite string) (count, mismatchCount int) {
 	defer func() {
 		if r := recover(); r != nil {
 			klog.Warningf("route summary recovered from panic: %v", r)
@@ -90,14 +90,14 @@ func (s *nodeStatusServer) collectRouteSummary(peers []routeplan.Peer, localSite
 			} else {
 				// Kernel inspection only emits "kernel" route types, so the
 				// connected/local host-route exception cannot apply here.
-				mismatch = true
+				mismatchCount++
 			}
 		}
 	})
 
 	// The legacy collector does not annotate an entirely empty kernel result.
 	if count == 0 {
-		return count, mismatch
+		return count, mismatchCount
 	}
 
 	for _, family := range families {
@@ -110,7 +110,7 @@ func (s *nodeStatusServer) collectRouteSummary(peers []routeplan.Peer, localSite
 				continue
 			}
 
-			mismatch = true
+			mismatchCount++
 
 			if !family.destinations[expected.destination] {
 				family.destinations[expected.destination] = true
@@ -119,5 +119,5 @@ func (s *nodeStatusServer) collectRouteSummary(peers []routeplan.Peer, localSite
 		}
 	}
 
-	return count, mismatch
+	return count, mismatchCount
 }

cmd/unbounded-net-node/status_summary_routes_test.go

@@ -50,17 +50,18 @@ func TestRouteSummaryParity(t *testing.T) {
 	}
 
 	for _, tc := range []struct {
-		name        string
-		v4          []netlink.Route
-		v6          []netlink.Route
-		table       []netlink.Route
-		withoutPeer bool
+		name               string
+		v4                 []netlink.Route
+		v6                 []netlink.Route
+		table              []netlink.Route
+		withoutPeer        bool
+		exactMismatchCount int
 	}{
 		{name: "empty kernel does not synthesize"},
-		{name: "missing expected", v4: []netlink.Route{summaryRoute("10.9.0.0/24", 1, 0, 0)}},
+		{name: "missing expected", exactMismatchCount: 5, v4: []netlink.Route{summaryRoute("10.9.0.0/24", 1, 0, 0)}},
 		{name: "matched", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 0)}},
 		{name: "unexpected without peers", withoutPeer: true, v4: []netlink.Route{summaryRoute("10.9.0.0/24", 1, 0, 0)}},
-		{name: "unbounded suppresses only its family", v4: []netlink.Route{summaryRoute("10.42.0.0/16", 2, 0, 0)}},
+		{name: "unbounded suppresses only its family", exactMismatchCount: 2, v4: []netlink.Route{summaryRoute("10.42.0.0/16", 2, 0, 0)}},
 		{name: "unbounded both families", v4: []netlink.Route{summaryRoute("10.42.0.0/16", 2, 0, 0)}, v6: []netlink.Route{summaryRoute("fd00::/48", 2, 0, 0)}},
 		{name: "duplicate prefix distinct tables", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 0)}, table: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 100, 0)}},
 		{name: "duplicate next hop", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 0), summaryRoute("10.42.1.0/24", 1, 0, 100)}},
@@ -70,6 +71,10 @@ func TestRouteSummaryParity(t *testing.T) {
 			Dst:       summaryRoute("10.42.1.0/24", 1, 0, 0).Dst,
 			MultiPath: []*netlink.NexthopInfo{{LinkIndex: 1}, {LinkIndex: 3}, {LinkIndex: 4}},
 		}}},
+		{name: "two unexpected tunnel hops", withoutPeer: true, exactMismatchCount: 2, v4: []netlink.Route{{
+			Dst:       summaryRoute("10.99.0.0/24", 1, 0, 0).Dst,
+			MultiPath: []*netlink.NexthopInfo{{LinkIndex: 1}, {LinkIndex: 4}},
+		}}},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			s := summaryRouteFixture()
@@ -95,10 +100,17 @@ func TestRouteSummaryParity(t *testing.T) {
 				}
 			}
 
-			count, mismatch := s.collectRouteSummary(peers, "local")
+			count, mismatchCount := s.collectRouteSummary(peers, "local")
+
+			mismatch := mismatchCount > 0
 			if count != len(full.RoutingTable.Routes) || mismatch != wantMismatch {
 				t.Fatalf("summary=(%d,%v), legacy=(%d,%v): %+v", count, mismatch, len(full.RoutingTable.Routes), wantMismatch, full.RoutingTable.Routes)
 			}
+
+			wantMismatchCount := netstatus.RouteMismatchCount(full.RoutingTable.Routes)
+			if mismatchCount != wantMismatchCount || (tc.exactMismatchCount > 0 && mismatchCount != tc.exactMismatchCount) {
+				t.Fatalf("mismatched hops=%d, legacy=%d, explicit=%d: %+v", mismatchCount, wantMismatchCount, tc.exactMismatchCount, full.RoutingTable.Routes)
+			}
 		})
 	}
 }

cmd/unbounded-net-node/status_summary_test.go

@@ -121,7 +121,9 @@ func TestNodeSummaryParityAndNoBPF(t *testing.T) {
 
 			legacy := netstatus.OverviewFromStatus(full, time.Now())
 			if summary.PeerCount != legacy.PeerCount || summary.HealthyPeers != legacy.HealthyPeers ||
-				summary.RouteCount != legacy.RouteCount || summary.RouteMismatch != legacy.RouteMismatch {
+				summary.RouteCount != legacy.RouteCount || summary.RouteMismatch != legacy.RouteMismatch ||
+				summary.RouteMismatchCount != legacy.RouteMismatchCount || summary.UnhealthyPeerLinks != legacy.UnhealthyPeerLinks ||
+				summary.UsesIPIP != legacy.UsesIPIP {
 				t.Fatalf("counts differ: summary=%+v full peers=%+v routes=%+v", summary, full.Peers, full.RoutingTable)
 			}
 
@@ -261,6 +263,7 @@ func TestNodeSummaryToProto(t *testing.T) {
 	summary := &NodeStatusOverview{
 		Timestamp: now, NodeInfo: NodeInfo{Name: "node", K8sReady: "Unknown"},
 		PeerCount: 10, HealthyPeers: 4, RouteCount: 12, RouteMismatch: true,
+		RouteMismatchCount: 3, UnhealthyPeerLinks: 2, UsesIPIP: true,
 		FetchError: "unavailable", StatusSource: "error", LastPushTime: &now,
 		NodeErrors:  []NodeError{{Type: "failure", Message: "failed"}},
 		HealthCheck: &HealthCheckStatus{Healthy: false, Summary: "unhealthy"},
@@ -278,6 +281,7 @@ func TestNodeSummaryToProto(t *testing.T) {
 	}
 
 	if !proto.Equal(encoded, &decoded) || decoded.PeerCount != 10 || decoded.HealthyPeers != 4 || decoded.RouteCount != 12 ||
+		decoded.RouteMismatchCount != 3 || decoded.UnhealthyPeerLinks != 2 || !decoded.UsesIpip ||
 		!decoded.RouteMismatch || decoded.FetchError != "unavailable" || decoded.NodeInfo.K8SReady != "Unknown" ||
 		decoded.LastPushTimeUnixNs != now.UnixNano() || decoded.StatusSource != "error" || len(decoded.NodeErrors) != 1 {
 		t.Fatalf("summary conversion lost facts: %v", &decoded)