Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 24 additions & 161 deletions cmd/unbounded-net-controller/cluster_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"k8s.io/klog/v2"

"github.com/Azure/unbounded/internal/net/controller"
statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1"
"github.com/Azure/unbounded/internal/version"
)

Expand Down Expand Up @@ -236,6 +237,13 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo
}

cachedStatuses := health.statusCache.GetAll()
status.NodeOverviews = make(map[string]*statusv1alpha1.NodeStatusOverview)

for name, cached := range cachedStatuses {
if cached.Overview != nil {
status.NodeOverviews[name] = cached.Overview
}
}

type pullNode struct{ nodeName, nodeIP string }

Expand Down Expand Up @@ -371,6 +379,7 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo
} else if result.status != nil {
result.status.StatusSource = "pull"
cachedResults[result.nodeName] = *result.status
delete(status.NodeOverviews, result.nodeName)
}
}
}
Expand Down Expand Up @@ -447,6 +456,9 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo
if pubKey := node.Annotations[controller.WireGuardPubKeyAnnotation]; pubKey != "" {
if nodeStatus.NodeInfo.WireGuard == nil {
nodeStatus.NodeInfo.WireGuard = &WireGuardStatusInfo{}
} else {
wireguard := *nodeStatus.NodeInfo.WireGuard
nodeStatus.NodeInfo.WireGuard = &wireguard
}

nodeStatus.NodeInfo.WireGuard.PublicKey = pubKey
Expand Down Expand Up @@ -710,7 +722,6 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo
}

sort.Slice(status.Peerings, func(i, j int) bool { return status.Peerings[i].Name < status.Peerings[j].Name })
status.ConnectivityMatrix = buildConnectivityMatrix(status.Nodes, status.GatewayPools)
status.Problems = collectClusterProblems(status)

return status
Expand Down Expand Up @@ -809,6 +820,18 @@ func collectClusterProblems(status *ClusterStatusResponse) []StatusProblem {
appendProblem("node", nodeName, summary)
}

if overview := status.NodeOverviews[node.NodeInfo.Name]; overview != nil {
if overview.RouteMismatch {
appendProblem("node", nodeName, "Route next-hop mismatches (expected vs present)")
}

if unhealthy := overview.PeerCount - overview.HealthyPeers; unhealthy > 0 {
appendProblem("node", nodeName, fmt.Sprintf("%d peers are not healthy", unhealthy))
}

continue
}

if mismatchCount := routeMismatchCount(node); mismatchCount > 0 {
appendProblem("node", nodeName, fmt.Sprintf("%d route next-hop mismatches (expected vs present)", mismatchCount))
}
Expand Down Expand Up @@ -1038,163 +1061,3 @@ func latestNodeUpdateTime(node *corev1.Node) time.Time {

return latest
}

// buildConnectivityMatrix builds health check connectivity matrices from node peer data.
func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []GatewayPoolStatus) map[string]*SiteMatrix {
siteNodes := make(map[string]map[string]bool)
nodePeers := make(map[string][]WireGuardPeerStatus)
nodeByName := make(map[string]*NodeStatusResponse)

for _, n := range nodes {
name := n.NodeInfo.Name

site := n.NodeInfo.SiteName
if name == "" || site == "" {
continue
}

nodeByName[name] = n

if siteNodes[site] == nil {
siteNodes[site] = make(map[string]bool)
}

siteNodes[site][name] = true

// Keep the immutable snapshot's slice; filter when reading rather
// than copying every peer, including for scopes above the size limit.
nodePeers[name] = n.Peers

for _, p := range n.Peers {
if p.PeerType == "gateway" && p.Name != "" && p.SiteName == site {
siteNodes[site][p.Name] = true
}
}
}

if len(siteNodes) == 0 {
siteNodes = make(map[string]map[string]bool)
}

result := make(map[string]*SiteMatrix)
selfMatrixStatusFromCNI := func(node *NodeStatusResponse) string {
if node.NodeInfo.WireGuard != nil && strings.TrimSpace(node.NodeInfo.WireGuard.Interface) != "" {
return "up"
}

return ""
}

buildScopeMatrix := func(nodeSet map[string]bool) *SiteMatrix {
if len(nodeSet) == 0 || len(nodeSet) > 100 {
return nil
}

nodeNames := make([]string, 0, len(nodeSet))
for name := range nodeSet {
nodeNames = append(nodeNames, name)
}

sort.Strings(nodeNames)

results := make(map[string]map[string]string)
for _, srcNode := range nodeNames {
results[srcNode] = make(map[string]string)
if node, ok := nodeByName[srcNode]; ok {
results[srcNode][srcNode] = selfMatrixStatusFromCNI(node)
}

for _, peer := range nodePeers[srcNode] {
if !isConnectivityMatrixPeer(peer) {
continue
}

tgtNode := peer.Name
if tgtNode == "" || tgtNode == srcNode || !nodeSet[tgtNode] {
continue
}

cellStatus := ""
if peer.HealthCheck != nil {
cellStatus = peer.HealthCheck.Status
} else if peer.PeerType == "gateway" && !peer.Tunnel.LastHandshake.IsZero() {
cellStatus = "up"
}

results[srcNode][tgtNode] = cellStatus
}
}

return &SiteMatrix{Nodes: nodeNames, Results: results}
}

for site, nodeSet := range siteNodes {
scopeMatrix := buildScopeMatrix(nodeSet)
if scopeMatrix != nil {
result[site] = scopeMatrix
}
}

for _, pool := range gatewayPools {
poolName := strings.TrimSpace(pool.Name)
if poolName == "" {
continue
}

poolNodeSet := make(map[string]bool)

for _, gatewayName := range pool.Gateways {
name := strings.TrimSpace(gatewayName)
if name == "" {
continue
}

poolNodeSet[name] = true
for _, peer := range nodePeers[name] {
if !isConnectivityMatrixPeer(peer) {
continue
}

peerName := strings.TrimSpace(peer.Name)
if peerName == "" {
continue
}

if _, ok := nodeByName[peerName]; ok {
poolNodeSet[peerName] = true
}
}

for srcNodeName, peers := range nodePeers {
for _, peer := range peers {
if !isConnectivityMatrixPeer(peer) {
continue
}

if strings.TrimSpace(peer.Name) == name {
if _, ok := nodeByName[srcNodeName]; ok {
poolNodeSet[srcNodeName] = true
}

break
}
}
}
}

scopeMatrix := buildScopeMatrix(poolNodeSet)
if scopeMatrix != nil {
result["pool:"+poolName] = scopeMatrix
}
}

if len(result) == 0 {
return nil
}

return result
}

func isConnectivityMatrixPeer(peer WireGuardPeerStatus) bool {
return peer.PeerType == "site" || peer.PeerType == "gateway"
}
34 changes: 32 additions & 2 deletions cmd/unbounded-net-controller/cluster_status_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package main

import (
"context"
"maps"
"reflect"
"slices"
"sync"
Expand All @@ -15,6 +16,8 @@ import (

unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1"
"github.com/Azure/unbounded/internal/net/controller"
statuspkg "github.com/Azure/unbounded/internal/net/status"
statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1"
)

// ClusterStatusCache maintains a pre-built ClusterStatusResponse in memory,
Expand Down Expand Up @@ -115,6 +118,15 @@ func (c *ClusterStatusCache) Rebuild(ctx context.Context) {
// PatchNode updates a single node's cached status in-place without a full
// rebuild.
func (c *ClusterStatusCache) PatchNode(nodeName string, nodeStatus NodeStatusResponse) {
c.patchNode(nodeName, nodeStatus, nil)
}

// PatchOverview updates metadata and observed facts without collecting details.
func (c *ClusterStatusCache) PatchOverview(nodeName string, overview statusv1alpha1.NodeStatusOverview) {
c.patchNode(nodeName, statuspkg.OverviewMetadata(overview), &overview)
}

func (c *ClusterStatusCache) patchNode(nodeName string, nodeStatus NodeStatusResponse, overview *statusv1alpha1.NodeStatusOverview) {
now := time.Now()
nodeStatus.NodeInfo.ExternalIPs = c.resolveNodeExternalIPs(nodeName, now)

Expand All @@ -125,6 +137,16 @@ func (c *ClusterStatusCache) PatchNode(nodeName string, nodeStatus NodeStatusRes
return
}

if overview == nil {
delete(c.status.NodeOverviews, nodeName)
} else {
if c.status.NodeOverviews == nil {
c.status.NodeOverviews = make(map[string]*statusv1alpha1.NodeStatusOverview)
}

c.status.NodeOverviews[nodeName] = overview
}

if i, ok := c.nodeIndex[nodeName]; ok && i < len(c.status.Nodes) {
// Preserve controller-enriched fields across node-agent status updates.
existing := c.status.Nodes[i]
Expand Down Expand Up @@ -225,13 +247,21 @@ func (c *ClusterStatusCache) MarkFullRebuildNeeded() {
}
}

// Get returns the current pre-built status (read-locked, fast).
// Get snapshots mutable containers; nested node data remains immutable and shared.
// Returns nil if the status has not been built yet.
func (c *ClusterStatusCache) Get() *ClusterStatusResponse {
c.mu.RLock()
defer c.mu.RUnlock()

return c.status
if c.status == nil {
return nil
}

snapshot := *c.status
snapshot.Nodes = slices.Clone(c.status.Nodes)
snapshot.NodeOverviews = maps.Clone(c.status.NodeOverviews)

return &snapshot
}

// GetSeq returns the current sequence number.
Expand Down
Loading