From 6946d6ee42c0c2da43feec24f39a8a2716bd6f03 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:13:46 +0000 Subject: [PATCH 01/34] ui(net): retire dashboard connectivity graph and matrix views Remove the Network card and heatmap navigation without changing Site summaries, node filtering, or per-node diagnostics. Remove unused graph edge-health derivation. Retain the detached topology implementation for an isolated follow-up deletion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- docs/net/operations.md | 7 +- frontend/src/App.tsx | 72 +-- .../src/components/dashboard/NetworkCard.tsx | 118 ----- .../network/ConnectivityHeatmap.tsx | 422 ------------------ frontend/src/hooks/useDashboardData.ts | 102 +---- 5 files changed, 5 insertions(+), 716 deletions(-) delete mode 100644 frontend/src/components/dashboard/NetworkCard.tsx delete mode 100644 frontend/src/components/network/ConnectivityHeatmap.tsx diff --git a/docs/net/operations.md b/docs/net/operations.md index 77d3a073c..735d7cfd1 100644 --- a/docs/net/operations.md +++ b/docs/net/operations.md @@ -212,7 +212,6 @@ kubectl unbounded-system controller proxy The dashboard displays: - **Overview**: Cluster health summary with node counts, site counts, and gateway status - **Sites**: All configured sites with node counts and health indicators -- **Connectivity Matrix**: Visual representation of node-to-node connectivity (pingmesh results) - **Nodes**: Detailed list of all nodes with filtering, sorting, and pagination - Tunnel peer status (WireGuard peers or eBPF tunnel endpoints) - Gateway health for each node @@ -225,12 +224,10 @@ The dashboard uses **WebSocket** for real-time updates with delta compression, f - Filtering nodes by name, site, or role (gateway/worker) - Sorting by any column - Auto-sizing pagination based on screen height -- Expandable connectivity matrix with zoom and labels - Dark/light theme toggle -Connectivity matrices are omitted for site or gateway-pool scopes containing -more than 100 nodes. Smaller scopes remain visible even when other scopes -exceed that limit. +The dashboard does not render a site connectivity graph or connectivity matrix. +Use the Site summaries and filtered node list to inspect individual resources. ### Health Endpoints diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 63bb576f4..02cac42a7 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4,7 +4,6 @@ import * as React from 'react'; import { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import NodeTable from './components/nodes/NodesTable'; -import NetworkCard from './components/dashboard/NetworkCard'; import SitesCard from './components/network/SitesCard'; import StatusJsonModal from './components/status/StatusJsonModal'; import ErrorBoundary from './components/common/ErrorBoundary'; @@ -23,7 +22,6 @@ export default function App() { } = useClusterStatus(); const nodes = status?.nodes || []; const sites = summary?.sites || status?.sites || []; - const peerings = summary?.peerings || status?.peerings || []; const gatewayPools = summary?.gatewayPools || status?.gatewayPools || []; const nodeSummaries = summary?.nodeSummaries || []; const [hiddenSites, setHiddenSites] = useState>(new Set()); @@ -32,8 +30,7 @@ export default function App() { const [selectedNodeDetailTab, setSelectedNodeDetailTab] = useState<'peerings' | 'routes' | 'bpf'>('peerings'); const [pullEnabledOptimistic, setPullEnabledOptimistic] = useState(null); const [selectedNodeTypesFilter, setSelectedNodeTypesFilter] = useState>(new Set(['Gateway', 'Worker'])); - const [networkTab, setNetworkTab] = useState<'siteTopology' | 'matrix'>('siteTopology'); - const [maximizedPanel, setMaximizedPanel] = useState<'nodes' | 'siteTopology' | 'matrix' | null>(null); + const [maximizedPanel, setMaximizedPanel] = useState<'nodes' | null>(null); const [infoOpen, setInfoOpen] = useState(false); const [statusJsonOpen, setStatusJsonOpen] = useState(false); const [errorsDismissed, setErrorsDismissed] = useState(false); @@ -52,12 +49,6 @@ export default function App() { window.localStorage.setItem('theme', theme); }, [theme]); - useEffect(() => { - if (maximizedPanel === 'siteTopology' || maximizedPanel === 'matrix') { - setNetworkTab(maximizedPanel); - } - }, [maximizedPanel]); - useEffect(() => { if (!maximizedPanel) return; const onKeyDown = (event: KeyboardEvent) => { @@ -217,9 +208,7 @@ export default function App() { }, []); const { - activeNetworkTab, activeSelectedNode, - edgeHealthCheckCounts, effectivePullEnabled, gatewayByNode, nodeK8sStatusMap, @@ -227,7 +216,6 @@ export default function App() { nodeTotalCount, peerHealth, poolCounts, - poolToSite, siteCounts, visibleNodeSummaries } = useDashboardData({ @@ -239,8 +227,6 @@ export default function App() { gatewayPoolHiddenNames: hiddenGatewayPools, hiddenSites, selectedNodeTypesFilter, - networkTab, - maximizedPanel, pullEnabledOptimistic, selectedNodeName, nodeDetail @@ -272,18 +258,6 @@ export default function App() { ? 'Polling only' : 'No data'; - const onSelectNetworkTab = (tab: 'siteTopology' | 'matrix') => { - if (maximizedPanel === 'siteTopology' || maximizedPanel === 'matrix') { - setMaximizedPanel(tab); - return; - } - setNetworkTab(tab); - }; - - const onToggleNetworkMaximize = (isMaximized: boolean) => { - setMaximizedPanel(isMaximized ? null : activeNetworkTab); - }; - const renderNodesCard = (isMaximized: boolean) => { const content = (
setMaximizedPanel(null)}>
- {maximizedPanel === 'nodes' ? renderNodesCard(true) : ( - onToggleNetworkMaximize(true)} - /> - )} + {renderNodesCard(true)} - onToggleNetworkMaximize(false)} - />
{loading &&
Loading...
} diff --git a/frontend/src/components/dashboard/NetworkCard.tsx b/frontend/src/components/dashboard/NetworkCard.tsx deleted file mode 100644 index aa599c22e..000000000 --- a/frontend/src/components/dashboard/NetworkCard.tsx +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import React, { Suspense } from 'react'; -import { CloseXIcon, MagnifyPlusIcon } from '../nodes/shared/index'; -import ConnectivityHeatmap from '../network/ConnectivityHeatmap'; -const Topology = React.lazy(() => import('../network/Topology')); -import { GatewayPoolStatus, NodeStatus, PeeringStatus, SiteMatrix, SiteStatus } from '../../types'; - -type NetworkTab = 'siteTopology' | 'matrix'; - -type NetworkCardProps = { - activeNetworkTab: NetworkTab; - edgeHealthCheckCounts: Map; - gatewayByNode: Map; - gatewayPools: GatewayPoolStatus[]; - hiddenGatewayPools: Set; - hiddenSites: Set; - isMaximized: boolean; - nodeStatuses: NodeStatus[]; - peerings: PeeringStatus[]; - poolCounts: Map; - poolToSite: Map; - siteCounts: Map; - sites: SiteStatus[]; - statusMatrix?: Record; - theme: 'dark' | 'light'; - onSelectTab: (tab: NetworkTab) => void; - onToggleMaximize: () => void; -}; - -function NetworkCard({ - activeNetworkTab, - edgeHealthCheckCounts, - gatewayByNode, - gatewayPools, - hiddenGatewayPools, - hiddenSites, - isMaximized, - nodeStatuses, - peerings, - poolCounts, - poolToSite, - siteCounts, - sites, - statusMatrix, - theme, - onSelectTab, - onToggleMaximize -}: NetworkCardProps) { - return ( -
-
-
- - -
-
- -
-
- {activeNetworkTab === 'siteTopology' && ( -
- Loading topology...
}> - - -
- )} - {activeNetworkTab === 'matrix' && ( -
- -
- )} -
- ); -} - -export default NetworkCard; diff --git a/frontend/src/components/network/ConnectivityHeatmap.tsx b/frontend/src/components/network/ConnectivityHeatmap.tsx deleted file mode 100644 index 0e6d24a4c..000000000 --- a/frontend/src/components/network/ConnectivityHeatmap.tsx +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import { useEffect, useMemo, useRef, useState } from 'react'; -import { GatewayPoolStatus, NodeStatus, SiteMatrix, SiteStatus } from '../../types'; -import { getCniStatus } from '../nodes/shared/index'; - -function getGatewayPoolNodeNames(pool: GatewayPoolStatus): string[] { - const nodeNames = (pool.nodes || []) - .map((node) => node.name) - .filter((name): name is string => Boolean(name)); - if (nodeNames.length > 0) { - return nodeNames; - } - return (pool.gateways || []).filter((name): name is string => Boolean(name)); -} - -function ConnectivityHeatmap({ - matrix, - hiddenSites, - hiddenGatewayPools, - sites, - gatewayPools, - siteCounts, - nodeStatuses -}: { - matrix?: Record; - hiddenSites: Set; - hiddenGatewayPools: Set; - sites: SiteStatus[]; - gatewayPools: GatewayPoolStatus[]; - siteCounts: Map; - nodeStatuses: NodeStatus[]; -}) { - const [activeScope, setActiveScope] = useState('all'); - const matrixRef = useRef(null); - const [matrixSize, setMatrixSize] = useState({ width: 420, height: 420 }); - const [tooltip, setTooltip] = useState<{ - x: number; - y: number; - src: string; - dst: string; - value: number; - } | null>(null); - const siteNames = useMemo(() => { - return Object.keys(matrix || {}) - .filter((name) => !name.startsWith('pool:')) - .filter((name) => !hiddenSites.has(name)) - .sort(); - }, [matrix, hiddenSites]); - const gatewayPoolNames = useMemo(() => { - return (gatewayPools || []) - .map((pool) => pool.name || '') - .filter((name) => Boolean(name) && !hiddenGatewayPools.has(name)) - .sort(); - }, [gatewayPools, hiddenGatewayPools]); - const selectorOptions = useMemo( - () => [ - { key: 'all', label: 'All', kind: 'all' as const }, - ...siteNames.map((site) => ({ key: `site:${site}`, label: site, kind: 'site' as const })), - ...gatewayPoolNames.map((pool) => ({ key: `pool:${pool}`, label: pool, kind: 'pool' as const })) - ], - [siteNames, gatewayPoolNames] - ); - const siteLookup = useMemo(() => { - const map = new Map(); - for (const site of sites) { - if (site.name) { - map.set(site.name, site); - } - } - return map; - }, [sites]); - const nodeStatusByName = useMemo(() => { - const map = new Map(); - for (const node of nodeStatuses) { - const name = node.nodeInfo?.name; - if (name) { - map.set(name, node); - } - } - return map; - }, [nodeStatuses]); - const visibleNodeNames = useMemo(() => { - return nodeStatuses - .filter((node) => { - const nodeName = node.nodeInfo?.name || ''; - if (!nodeName) return false; - const siteName = node.nodeInfo?.siteName; - if (siteName && hiddenSites.has(siteName)) { - return false; - } - const poolName = (gatewayPools || []).find((pool) => { - const poolNameValue = pool.name || ''; - if (!poolNameValue) return false; - return getGatewayPoolNodeNames(pool).includes(nodeName); - })?.name; - if (poolName && hiddenGatewayPools.has(poolName)) { - return false; - } - return true; - }) - .map((node) => node.nodeInfo?.name || '') - .filter((name) => Boolean(name)); - }, [nodeStatuses, hiddenSites, hiddenGatewayPools, gatewayPools]); - - const adjacency = useMemo(() => { - const map = new Map>(); - const allVisible = new Set(visibleNodeNames); - for (const node of nodeStatuses) { - const src = node.nodeInfo?.name; - if (!src || !allVisible.has(src)) continue; - if (!map.has(src)) map.set(src, new Set()); - for (const peer of node.peers || []) { - const dst = peer.name; - if (!dst || !allVisible.has(dst) || dst === src) continue; - map.get(src)?.add(dst); - if (!map.has(dst)) map.set(dst, new Set()); - map.get(dst)?.add(src); - } - } - return map; - }, [nodeStatuses, visibleNodeNames]); - - const healthCheckStatusByPair = useMemo(() => { - const map = new Map(); - for (const siteMatrix of Object.values(matrix || {})) { - const results = siteMatrix?.results || {}; - for (const [src, row] of Object.entries(results)) { - for (const [dst, cell] of Object.entries(row || {})) { - const key = src < dst ? `${src}|${dst}` : `${dst}|${src}`; - if (!map.has(key) && cell) { - map.set(key, typeof cell === 'string' ? cell : (cell as { healthCheckStatus?: string })?.healthCheckStatus || ''); - } - } - } - } - // Also extract health check status from per-node peer data (covers cross-site links) - for (const node of nodeStatuses) { - const src = node.nodeInfo?.name; - if (!src) continue; - for (const peer of node.peers || []) { - const dst = peer.name; - if (!dst || dst === src) continue; - const key = src < dst ? `${src}|${dst}` : `${dst}|${src}`; - if (map.has(key)) continue; // matrix data takes priority - const status = peer.healthCheck?.status; - if (status) { - map.set(key, status); - } - } - } - return map; - }, [matrix, nodeStatuses]); - - const selectedNodeNames = useMemo(() => { - const visibleSet = new Set(visibleNodeNames); - if (activeScope === 'all') { - return [...visibleNodeNames].sort(); - } - - if (activeScope.startsWith('site:')) { - const siteName = activeScope.slice(5); - const fromMatrix = (matrix?.[siteName]?.nodes || []) - .map((name) => String(name)) - .filter((name) => visibleSet.has(name)); - if (fromMatrix.length > 0) { - return Array.from(new Set(fromMatrix)).sort(); - } - return nodeStatuses - .map((node) => node.nodeInfo?.name || '') - .filter((name) => { - if (!name || !visibleSet.has(name)) return false; - const site = nodeStatusByName.get(name)?.nodeInfo?.siteName; - return site === siteName; - }) - .sort(); - } - - if (activeScope.startsWith('pool:')) { - const poolName = activeScope.slice(5); - const poolMatrixNodes = (matrix?.[`pool:${poolName}`]?.nodes || []) - .map((name) => String(name)) - .filter((name) => visibleSet.has(name)); - if (poolMatrixNodes.length > 0) { - return Array.from(new Set(poolMatrixNodes)).sort(); - } - const pool = gatewayPools.find((item) => item.name === poolName); - if (!pool) return []; - const members = new Set(); - for (const nodeName of getGatewayPoolNodeNames(pool)) { - if (!visibleSet.has(nodeName)) continue; - members.add(nodeName); - for (const peerName of adjacency.get(nodeName) || []) { - if (visibleSet.has(peerName)) { - members.add(peerName); - } - } - } - return Array.from(members).sort(); - } - - return []; - }, [activeScope, visibleNodeNames, matrix, nodeStatuses, nodeStatusByName, gatewayPools, adjacency]); - - useEffect(() => { - if (!selectorOptions.some((option) => option.key === activeScope)) { - setActiveScope('all'); - } - }, [activeScope, selectorOptions]); - - useEffect(() => { - if (!matrixRef.current) return; - const updateSize = () => { - const rect = matrixRef.current?.getBoundingClientRect(); - if (!rect) return; - const width = Math.max(0, Math.floor(rect.width)); - const height = Math.max(0, Math.floor(rect.height)); - if (width > 0 && height > 0) { - setMatrixSize({ width, height }); - } - }; - updateSize(); - const observer = new ResizeObserver(updateSize); - observer.observe(matrixRef.current); - return () => observer.disconnect(); - }, []); - - if ((!matrix || siteNames.length === 0) && selectedNodeNames.length === 0) { - return
No connectivity data available.
; - } - - const matrixNodes = selectedNodeNames; - if (matrixNodes.length === 0) { - return
No connectivity data available.
; - } - const cells: Array<{ row: number; col: number; value: number; src: string; dst: string }> = []; - const hasExpectedLink = (src: string, dst: string) => { - if (src === dst) return true; - return adjacency.get(src)?.has(dst) || adjacency.get(dst)?.has(src) || false; - }; - - const selfCellValueFromCniStatus = (node?: NodeStatus) => { - if (!node) { - return 0; - } - if (node.statusSource === 'apiserver-push' || node.statusSource === 'apiserver-ws') { - return 2; - } - const cni = getCniStatus(node); - if (cni.tone === 'success') return 1; - if (cni.tone === 'warning') return 2; - return 0; - }; - - for (let i = 0; i < matrixNodes.length; i++) { - const src = matrixNodes[i]; - for (let j = 0; j < matrixNodes.length; j++) { - const dst = matrixNodes[j]; - const pairKey = src < dst ? `${src}|${dst}` : `${dst}|${src}`; - const hcStatus = (healthCheckStatusByPair.get(pairKey) || '').trim().toLowerCase(); - let value = hcStatus === 'up' ? 1 : hcStatus === 'mixed' ? 2 : hcStatus ? 0 : -1; - if (src === dst) { - const node = nodeStatusByName.get(src); - value = selfCellValueFromCniStatus(node); - } else if (!hasExpectedLink(src, dst)) { - value = -2; - } - cells.push({ row: i, col: j, value, src, dst }); - } - } - - const { width, height } = matrixSize; - - const getColor = (value: number) => { - if (value === 1) return '#4ade80'; - if (value === 2) return '#facc15'; - if (value === 0) return '#f87171'; - if (value === -2) return 'transparent'; - return '#6b7280'; - }; - - const renderMatrix = (width: number, height: number) => { - if (width <= 0 || height <= 0) { - return
; - } - const showAxisLabels = false; - const margin = { top: 12, left: 12, right: 12, bottom: 12 }; - const innerWidth = width - margin.left - margin.right; - const innerHeight = height - margin.top - margin.bottom; - const minInner = Math.min(innerWidth, innerHeight); - const gap = Math.max(1, Math.round(minInner * 0.005)); - const cellSize = Math.floor((minInner - gap * (matrixNodes.length - 1)) / matrixNodes.length); - const gridWidth = cellSize * matrixNodes.length + gap * (matrixNodes.length - 1); - const gridHeight = cellSize * matrixNodes.length + gap * (matrixNodes.length - 1); - const offsetX = Math.max(0, Math.floor((innerWidth - gridWidth) / 2)); - const offsetY = Math.max(0, Math.floor((innerHeight - gridHeight) / 2)); - const scale = 1; - - return ( -
-
- - - {cells.map((cell) => { - const x = cell.col * (cellSize + gap); - const y = cell.row * (cellSize + gap); - const w = cellSize; - const h = cellSize; - return ( - { - setTooltip({ - x: event.clientX + 12, - y: event.clientY + 12, - src: cell.src, - dst: cell.dst, - value: cell.value - }); - }} - onMouseLeave={() => setTooltip(null)} - /> - ); - })} - {showAxisLabels && - matrixNodes.map((label, index) => ( - - {label} - - ))} - {showAxisLabels && - matrixNodes.map((label, index) => ( - - {label} - - ))} - - -
- {tooltip && ( -
-
{tooltip.src} {'->'} {tooltip.dst}
-
- {tooltip.src === tooltip.dst - ? (tooltip.value === 1 - ? 'CNI Healthy' - : tooltip.value === 2 - ? `CNI Warning: ${getCniStatus(nodeStatusByName.get(tooltip.src))?.label || 'Warning'}` - : 'CNI No Data') - : tooltip.value === 1 - ? 'HC Up' - : tooltip.value === 2 - ? 'HC Mixed' - : tooltip.value === 0 - ? 'HC Down' - : tooltip.value === -2 - ? 'No Link Expected' - : 'No Data'} -
-
- )} -
- ); - }; - - return ( -
-
- {selectorOptions.map((option) => { - const isActive = option.key === activeScope; - const isSite = option.kind === 'site'; - const isPool = option.kind === 'pool'; - const siteInfo = isSite ? siteLookup.get(option.label) : undefined; - const counts = isSite ? siteCounts.get(option.label) : undefined; - const online = counts?.online ?? siteInfo?.onlineCount ?? 0; - const total = counts?.total ?? siteInfo?.nodeCount ?? 0; - const status = isSite - ? (online === 0 && total > 0 ? 'danger' : online < total ? 'warning' : 'success') - : isPool - ? 'info' - : 'all'; - return ( - - ); - })} -
- {renderMatrix(width, height)} -
- ); -} - - -export default ConnectivityHeatmap; diff --git a/frontend/src/hooks/useDashboardData.ts b/frontend/src/hooks/useDashboardData.ts index f604a6766..1e4fde281 100644 --- a/frontend/src/hooks/useDashboardData.ts +++ b/frontend/src/hooks/useDashboardData.ts @@ -14,8 +14,6 @@ type DashboardDataParams = { gatewayPoolHiddenNames: Set; hiddenSites: Set; selectedNodeTypesFilter: Set; - networkTab: 'siteTopology' | 'matrix'; - maximizedPanel: 'nodes' | 'siteTopology' | 'matrix' | null; pullEnabledOptimistic: boolean | null; selectedNodeName: string | null; nodeDetail: (name: string) => NodeStatus | undefined; @@ -30,8 +28,6 @@ function useDashboardData({ gatewayPoolHiddenNames, hiddenSites, selectedNodeTypesFilter, - networkTab, - maximizedPanel, pullEnabledOptimistic, selectedNodeName, nodeDetail @@ -163,96 +159,7 @@ function useDashboardData({ return counts; }, [gatewayPools, nodes, nodeSummaries]); - const edgeHealthCheckCounts = useMemo(() => { - const nodeToEntity = new Map(); - const nodeNameSet = new Set(); - - // Build node-to-entity mapping from full nodes or summaries - const nodeSource = nodes.length > 0 - ? nodes.map((n) => ({ name: n.nodeInfo?.name, siteName: n.nodeInfo?.siteName })) - : nodeSummaries.map((ns) => ({ name: ns.name, siteName: ns.siteName })); - for (const node of nodeSource) { - const name = node.name; - if (!name) continue; - nodeNameSet.add(name); - const poolName = gatewayByNode.get(name); - if (poolName) { - nodeToEntity.set(name, `pool:${poolName}`); - } else if (node.siteName) { - nodeToEntity.set(name, `site:${node.siteName}`); - } - } - - const counts = new Map(); - - if (nodes.length > 0) { - // Full nodes available: use peer-level health check data - for (const node of nodes) { - const name = node.nodeInfo?.name; - if (!name) continue; - const srcEntity = nodeToEntity.get(name); - if (!srcEntity) continue; - for (const peer of node.peers || []) { - if (!peer.healthCheck?.enabled && !peer.healthCheck) continue; - const peerSite = peer.siteName; - if (!peerSite) continue; - let dstEntity: string | undefined; - if (peer.name) { - if (!nodeNameSet.has(peer.name)) continue; - dstEntity = nodeToEntity.get(peer.name); - } - if (!dstEntity) dstEntity = `site:${peerSite}`; - if (srcEntity === dstEntity) continue; - const edgeKey = srcEntity < dstEntity - ? `${srcEntity}|${dstEntity}` - : `${dstEntity}|${srcEntity}`; - const current = counts.get(edgeKey) || { up: 0, total: 0 }; - current.total++; - const rawStatus = (peer.healthCheck?.status || '').trim().toLowerCase(); - if (rawStatus === 'up') current.up++; - counts.set(edgeKey, current); - } - } - } else { - // Summary mode: derive counts from connectivity matrix - const matrix = summary?.connectivityMatrix; - if (matrix) { - for (const [, siteMatrix] of Object.entries(matrix)) { - const results = siteMatrix?.results || {}; - for (const [src, row] of Object.entries(results)) { - const srcEntity = nodeToEntity.get(src); - if (!srcEntity) continue; - for (const [dst, cellStatus] of Object.entries(row || {})) { - if (src >= dst) continue; // count each pair once - const dstEntity = nodeToEntity.get(dst); - if (!dstEntity || srcEntity === dstEntity) continue; - const edgeKey = srcEntity < dstEntity - ? `${srcEntity}|${dstEntity}` - : `${dstEntity}|${srcEntity}`; - const current = counts.get(edgeKey) || { up: 0, total: 0 }; - current.total++; - const status = (typeof cellStatus === 'string' ? cellStatus : '').trim().toLowerCase(); - if (status === 'up') current.up++; - counts.set(edgeKey, current); - } - } - } - } - } - return counts; - }, [nodes, nodeSummaries, gatewayByNode, summary]); - - const poolToSite = useMemo(() => { - const map = new Map(); - for (const pool of gatewayPools) { - if (pool.name && pool.siteName) { - map.set(pool.name, pool.siteName); - } - } - return map; - }, [gatewayPools]); - - // Visible full nodes (for NetworkCard and other components needing full NodeStatus) + // Visible full nodes provide backward compatibility for summary filtering. const visibleNodes = useMemo(() => { return nodes.filter((node) => { const nodeName = node.nodeInfo?.name || ''; @@ -324,10 +231,6 @@ function useDashboardData({ return { healthy, total }; }, [nodes, nodeSummaries]); - const activeNetworkTab = maximizedPanel === 'siteTopology' || maximizedPanel === 'matrix' - ? maximizedPanel - : networkTab; - const effectivePullEnabled = pullEnabledOptimistic ?? Boolean(summary?.pullEnabled ?? status?.pullEnabled); // Active selected node detail from the cache @@ -340,9 +243,7 @@ function useDashboardData({ }, [selectedNodeName, nodeDetail, nodes]); return { - activeNetworkTab, activeSelectedNode, - edgeHealthCheckCounts, effectivePullEnabled, gatewayByNode, nodeK8sStatusMap, @@ -350,7 +251,6 @@ function useDashboardData({ nodeTotalCount, peerHealth, poolCounts, - poolToSite, siteCounts, visibleNodes, visibleNodeSummaries From 9fce058aeab7996d837c57d5f032d47e57843e03 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:14:15 +0000 Subject: [PATCH 02/34] ui(net): remove retired site topology implementation Isolate the 949-line deletion of the topology component after its render paths were removed. No remaining imports reference it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- frontend/src/components/network/Topology.tsx | 949 ------------------- 1 file changed, 949 deletions(-) delete mode 100644 frontend/src/components/network/Topology.tsx diff --git a/frontend/src/components/network/Topology.tsx b/frontend/src/components/network/Topology.tsx deleted file mode 100644 index 9107d77af..000000000 --- a/frontend/src/components/network/Topology.tsx +++ /dev/null @@ -1,949 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import * as React from 'react'; -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { createPortal } from 'react-dom'; -import { - getTopologyNodeGlyph, - renderTopologyNodeGlyph, - useTopologyNodeIconTextures -} from '../common/topologyIcons'; -import { GatewayPoolStatus, NodeStatus, PeeringStatus, SiteStatus } from '../../types'; -import { getNodeStatus, ReagraphModule } from '../nodes/shared/index'; - -function buildGraph( - allSiteNames: string[], - allPoolNames: string[], - peerings: PeeringStatus[], - hiddenSites: Set, - hiddenGatewayPools: Set, - existingGatewayPools: Set, - palette: { site: string; siteEmpty: string; siteWarn: string; siteDanger: string; pool: string; poolEmpty: string; poolWarn: string; poolDanger: string; edge: string; edgeDim: string; edgeUp: string; edgeWarn: string; edgeDanger: string }, - siteCounts?: Map, - poolCounts?: Map, - edgeHealthCheckCounts?: Map, - poolToSite?: Map -) { - // Dim a hex color by reducing its opacity (blend toward background) - const dimColor = (hex: string) => { - const r = parseInt(hex.slice(1, 3), 16); - const g = parseInt(hex.slice(3, 5), 16); - const b = parseInt(hex.slice(5, 7), 16); - const mix = (c: number) => Math.round(c * 0.3 + 30 * 0.7); - return `#${mix(r).toString(16).padStart(2, '0')}${mix(g).toString(16).padStart(2, '0')}${mix(b).toString(16).padStart(2, '0')}`; - }; - const nodes: { id: string; label: string; fill: string; activeFill: string; data: { level: number; group: string; peerings: string[] } }[] = []; - const edges: { id: string; source: string; target: string; fill?: string; size?: number; data?: { peerings: string[]; hcUp: number; hcTotal: number } }[] = []; - const nodeSet = new Set(); - const edgeSet = new Set(); - const edgePeerings = new Map>(); - const sitePeerings = new Map>(); - const poolPeerings = new Map>(); - - for (const peering of peerings) { - const peeringName = peering.name || 'peering'; - for (const site of peering.sites || []) { - if (!sitePeerings.has(site)) { - sitePeerings.set(site, new Set()); - } - sitePeerings.get(site)?.add(peeringName); - } - for (const pool of peering.gatewayPools || []) { - if (!existingGatewayPools.has(pool)) { - continue; - } - if (!poolPeerings.has(pool)) { - poolPeerings.set(pool, new Set()); - } - poolPeerings.get(pool)?.add(peeringName); - } - } - - const addNode = (id: string, label: string, group: string, hidden: boolean) => { - if (!nodeSet.has(id)) { - nodeSet.add(id); - let fill = group === 'pool' ? palette.pool : palette.site; - if (group === 'site' && siteCounts) { - const counts = siteCounts.get(label); - if (counts) { - if (counts.total === 0) { - fill = palette.siteEmpty; - } else if ((counts.danger || 0) > 0) { - fill = palette.siteDanger; - } else if ((counts.warning || 0) > 0) { - fill = palette.siteWarn; - } - } - } - if (group === 'pool' && poolCounts) { - const counts = poolCounts.get(label); - if (counts) { - // No matching node entries for this pool in current cluster status: - // render gray (empty) instead of green. - if (counts.total === 0) { - fill = palette.poolEmpty; - } else if ((counts.danger || 0) > 0) { - fill = palette.poolDanger; - } else if ((counts.warning || 0) > 0) { - fill = palette.poolWarn; - } else if (counts.online === 0 && counts.total > 0) { - fill = palette.poolDanger; - } else if (counts.online < counts.total) { - fill = palette.poolWarn; - } - } - } - if (hidden) { - fill = dimColor(fill); - } - const peeringsForNode = group === 'pool' - ? Array.from(poolPeerings.get(label) || []) - : Array.from(sitePeerings.get(label) || []); - nodes.push({ id, label, fill, activeFill: fill, data: { level: group === 'pool' ? 0 : 1, group, peerings: peeringsForNode } }); - } - }; - - const addEdge = (a: string, b: string) => { - if (a === b) return; - const key = a < b ? `${a}|${b}` : `${b}|${a}`; - if (edgeSet.has(key)) return; - edgeSet.add(key); - const isHidden = (nodeId: string) => { - if (nodeId.startsWith('site:')) return hiddenSites.has(nodeId.slice(5)); - if (nodeId.startsWith('pool:')) return hiddenGatewayPools.has(nodeId.slice(5)); - return false; - }; - let edgeFill = palette.edge; - if (isHidden(a) || isHidden(b)) { - edgeFill = palette.edgeDim; - } else if (edgeHealthCheckCounts) { - const counts = edgeHealthCheckCounts.get(key); - // No matching health check entries for this edge (missing map entry or total=0): - // keep default gray edge color. Only color healthy/warn/danger with real data. - if (!counts || counts.total === 0) { - edgeFill = palette.edge; - } else { - if (counts.up === counts.total) { - edgeFill = palette.edgeUp; - } else if (counts.total - counts.up > counts.total / 2) { - edgeFill = palette.edgeDanger; - } else { - edgeFill = palette.edgeWarn; - } - } - } - edges.push({ id: key, source: a, target: b, fill: edgeFill, size: 3.5 }); - }; - - // Seed all known sites and gateway pools so isolated entities still render without edges. - for (const site of allSiteNames) { - addNode(`site:${site}`, site, 'site', hiddenSites.has(site)); - } - for (const pool of allPoolNames) { - addNode(`pool:${pool}`, pool, 'pool', hiddenGatewayPools.has(pool)); - } - - for (const peering of peerings) { - const sites = peering.sites || []; - const pools = (peering.gatewayPools || []).filter((pool) => existingGatewayPools.has(pool)); - const pName = peering.name || 'peering'; - const isPoolPeering = pName.startsWith('poolpeering/'); - - for (const site of sites) { - addNode(`site:${site}`, site, 'site', hiddenSites.has(site)); - } - for (const pool of pools) { - addNode(`pool:${pool}`, pool, 'pool', hiddenGatewayPools.has(pool)); - } - - // Track which peerings each edge belongs to - const trackEdgePeering = (a: string, b: string) => { - const key = a < b ? `${a}|${b}` : `${b}|${a}`; - if (!edgePeerings.has(key)) edgePeerings.set(key, new Set()); - edgePeerings.get(key)?.add(pName); - }; - - if (pools.length > 0) { - for (const site of sites) { - for (const pool of pools) { - trackEdgePeering(`site:${site}`, `pool:${pool}`); - addEdge(`site:${site}`, `pool:${pool}`); - } - } - if (isPoolPeering && pools.length > 1) { - for (let i = 0; i < pools.length; i++) { - for (let j = i + 1; j < pools.length; j++) { - trackEdgePeering(`pool:${pools[i]}`, `pool:${pools[j]}`); - addEdge(`pool:${pools[i]}`, `pool:${pools[j]}`); - } - } - } - } else if (sites.length > 1) { - for (let i = 0; i < sites.length; i++) { - for (let j = i + 1; j < sites.length; j++) { - trackEdgePeering(`site:${sites[i]}`, `site:${sites[j]}`); - addEdge(`site:${sites[i]}`, `site:${sites[j]}`); - } - } - } - } - - // Attach peering names and health check counts to edge data - for (const edge of edges) { - const pNames = edgePeerings.get(edge.id); - const hc = edgeHealthCheckCounts?.get(edge.id); - edge.data = { - peerings: pNames ? Array.from(pNames) : [], - hcUp: hc?.up ?? 0, - hcTotal: hc?.total ?? 0 - }; - } - - // Sort nodes lexicographically by label so layout is deterministic - nodes.sort((a, b) => a.label.localeCompare(b.label)); - - return { nodes, edges }; -} - -function buildNodeGraph( - nodes: NodeStatus[], - gatewayByNode: Map, - palette: { - nodeHealthy: string; - nodeWarn: string; - nodeDanger: string; - edge: string; - edgeUp: string; - edgeWarn: string; - edgeDanger: string; - }, - options?: { - edgeSize?: number; - } -) { - const graphNodes: { - id: string; - label: string; - fill: string; - activeFill: string; - data: { level: number; group: string; peerings: string[]; lines: string[] }; - }[] = []; - const graphEdges: { - id: string; - source: string; - target: string; - fill?: string; - size?: number; - data?: { - peerings: string[]; - hcUp: number; - hcTotal: number; - sourceLabel: string; - targetLabel: string; - }; - }[] = []; - - const nodeByName = new Map(); - for (const node of nodes) { - const nodeName = node.nodeInfo?.name; - if (!nodeName) continue; - nodeByName.set(nodeName, node); - } - - for (const [nodeName, node] of nodeByName.entries()) { - const isGateway = node.nodeInfo?.isGateway || gatewayByNode.has(nodeName); - const status = getNodeStatus(node); - let fill = palette.nodeHealthy; - if (status === 'warning') { - fill = palette.nodeWarn; - } else if (status === 'danger') { - fill = palette.nodeDanger; - } - - const siteName = node.nodeInfo?.siteName || '-'; - const poolName = gatewayByNode.get(nodeName) || '-'; - const lines = isGateway - ? [`Gateway Pool: ${poolName}`, `Site: ${siteName}`] - : [`Site: ${siteName}`]; - - graphNodes.push({ - id: `node:${nodeName}`, - label: nodeName, - fill, - activeFill: fill, - data: { - level: isGateway ? 0 : 1, - group: isGateway ? 'gateway-node' : 'worker-node', - peerings: [], - lines - } - }); - } - - const edgeCounts = new Map(); - for (const [nodeName, node] of nodeByName.entries()) { - for (const peer of node.peers || []) { - const peerName = peer.name; - if (!peerName || !nodeByName.has(peerName) || peerName === nodeName) { - continue; - } - const srcId = `node:${nodeName}`; - const dstId = `node:${peerName}`; - const key = srcId < dstId ? `${srcId}|${dstId}` : `${dstId}|${srcId}`; - const current = edgeCounts.get(key) || { up: 0, total: 0 }; - if (peer.healthCheck?.enabled || peer.healthCheck) { - current.total += 1; - const rawStatus = (peer.healthCheck?.status || '').trim().toLowerCase(); - if (rawStatus === 'up') { - current.up += 1; - } - } - edgeCounts.set(key, current); - } - } - - for (const [edgeId, counts] of edgeCounts.entries()) { - const [source, target] = edgeId.split('|'); - let edgeFill = palette.edge; - if (counts.total > 0) { - if (counts.up === counts.total) { - edgeFill = palette.edgeUp; - } else if (counts.total - counts.up > counts.total / 2) { - edgeFill = palette.edgeDanger; - } else { - edgeFill = palette.edgeWarn; - } - } - - graphEdges.push({ - id: edgeId, - source, - target, - fill: edgeFill, - size: options?.edgeSize ?? 2, - data: { - peerings: [], - hcUp: counts.up, - hcTotal: counts.total, - sourceLabel: source.startsWith('node:') ? source.slice(5) : source, - targetLabel: target.startsWith('node:') ? target.slice(5) : target - } - }); - } - - graphNodes.sort((a, b) => a.label.localeCompare(b.label)); - graphEdges.sort((a, b) => a.id.localeCompare(b.id)); - - return { nodes: graphNodes, edges: graphEdges }; -} - -function Topology({ - mode, - isMaximized, - sites, - peerings, - nodes, - gatewayPools, - gatewayByNode, - hiddenSites, - hiddenGatewayPools, - theme, - siteCounts, - poolCounts, - edgeHealthCheckCounts, - poolToSite -}: { - mode: 'sitesAndPools' | 'nodes'; - isMaximized: boolean; - sites: SiteStatus[]; - peerings: PeeringStatus[]; - nodes: NodeStatus[]; - gatewayPools: GatewayPoolStatus[]; - gatewayByNode: Map; - hiddenSites: Set; - hiddenGatewayPools: Set; - theme: 'dark' | 'light'; - siteCounts: Map; - poolCounts: Map; - edgeHealthCheckCounts: Map; - poolToSite: Map; -}) { - const graphRef = useRef(null); - const topologyWrapperRef = useRef(null); - const [reagraphModule, setReagraphModule] = useState(null); - - useEffect(() => { - let canceled = false; - import('reagraph') - .then((module) => { - if (canceled) return; - setReagraphModule({ - GraphCanvas: module.GraphCanvas as React.ComponentType, - darkTheme: module.darkTheme as Record, - lightTheme: module.lightTheme as Record - }); - }) - .catch((error) => { - console.error('Failed to load topology renderer', error); - }); - - return () => { - canceled = true; - }; - }, []); - - const palette = useMemo( - () => - theme === 'light' - ? { - site: '#16a34a', - siteEmpty: '#6b7280', - siteWarn: '#facc15', - siteDanger: '#dc2626', - pool: '#15803d', - poolEmpty: '#6b7280', - poolWarn: '#facc15', - poolDanger: '#dc2626', - edge: '#94a3b8', - edgeDim: '#cbd5e1', - edgeUp: '#16a34a', - edgeWarn: '#facc15', - edgeDanger: '#dc2626', - nodeHealthy: '#16a34a', - nodeWarn: '#facc15', - nodeDanger: '#dc2626', - label: '#1f2937', - background: '#ffffff' - } - : { - site: '#1DE9AC', - siteEmpty: '#6b7280', - siteWarn: '#facc15', - siteDanger: '#ef4444', - pool: '#1DE9AC', - poolEmpty: '#6b7280', - poolWarn: '#facc15', - poolDanger: '#ef4444', - edge: '#4b5563', - edgeDim: '#334155', - edgeUp: '#1DE9AC', - edgeWarn: '#facc15', - edgeDanger: '#ef4444', - nodeHealthy: '#1DE9AC', - nodeWarn: '#facc15', - nodeDanger: '#ef4444', - label: '#e0e0e0', - background: '#1a1a1a' - }, - [theme] - ); - const graphTheme = useMemo(() => { - if (!reagraphModule) return null; - const base = theme === 'light' ? reagraphModule.lightTheme : reagraphModule.darkTheme; - return { - ...base, - canvas: { - ...base.canvas, - background: palette.background, - fog: null - }, - edge: { - ...base.edge, - fill: palette.edge, - activeFill: palette.edge, - opacity: 1, - inactiveOpacity: 0.25 - }, - node: { - ...base.node, - activeFill: 'rgba(0,0,0,0)', - inactiveOpacity: 1, - hoverOpacity: 0, - label: { - ...base.node.label, - color: palette.label, - activeColor: palette.label, - fontSize: 16, - stroke: palette.background, - strokeColor: palette.background - } - } - }; - }, [palette, reagraphModule, theme]); - - const [hovered, setHovered] = useState<{ - x: number; - y: number; - label: string; - group: string; - lines: string[]; - } | null>(null); - const [edgeHovered, setEdgeHovered] = useState<{ - x: number; - y: number; - title: string; - detail: string; - } | null>(null); - const [hoveredNodeId, setHoveredNodeId] = useState(null); - const [showZoomHint, setShowZoomHint] = useState(false); - const [zoomHintRect, setZoomHintRect] = useState<{ left: number; top: number; width: number; height: number } | null>(null); - const zoomHintTimerRef = useRef(null); - const topologyIconTextures = useTopologyNodeIconTextures(); - const allSiteNames = useMemo( - () => sites - .map((site) => (site.name || '').trim()) - .filter((name): name is string => name.length > 0), - [sites] - ); - const allPoolNames = useMemo( - () => gatewayPools - .map((pool) => (pool.name || '').trim()) - .filter((name): name is string => name.length > 0), - [gatewayPools] - ); - const topologySiteCounts = useMemo(() => { - const counts = new Map(); - for (const siteName of allSiteNames) { - counts.set(siteName, { online: 0, total: 0, warning: 0, danger: 0 }); - } - - if (nodes.length > 0) { - for (const node of nodes) { - const siteName = node.nodeInfo?.siteName; - if (!siteName) continue; - const nodeName = node.nodeInfo?.name; - const isGatewayNode = node.nodeInfo?.isGateway || (nodeName ? gatewayByNode.has(nodeName) : false); - if (isGatewayNode) continue; - - const current = counts.get(siteName) || { online: 0, total: 0, warning: 0, danger: 0 }; - current.total += 1; - - const status = getNodeStatus(node); - if (status === 'success') { - current.online += 1; - } else if (status === 'warning') { - current.warning += 1; - } else { - current.danger += 1; - } - - counts.set(siteName, current); - } - } else { - // Summary mode: use siteCounts prop (online/total only). - for (const [siteName, sc] of siteCounts.entries()) { - const offline = Math.max(0, sc.total - sc.online); - counts.set(siteName, { online: sc.online, total: sc.total, warning: 0, danger: offline }); - } - } - - return counts; - }, [allSiteNames, gatewayByNode, nodes, siteCounts]); - const topologyPoolCounts = useMemo(() => { - const counts = new Map(); - - if (nodes.length > 0) { - const nodeByName = new Map(); - for (const node of nodes) { - const nodeName = node.nodeInfo?.name; - if (!nodeName) continue; - nodeByName.set(nodeName, node); - } - - for (const poolName of allPoolNames) { - const baseline = poolCounts.get(poolName) || { online: 0, total: 0 }; - const current = { online: 0, total: baseline.total, warning: 0, danger: 0 }; - - for (const [nodeName, nodePoolName] of gatewayByNode.entries()) { - if (nodePoolName !== poolName) continue; - const node = nodeByName.get(nodeName); - if (!node) continue; - const status = getNodeStatus(node); - if (status === 'success') { - current.online += 1; - } else if (status === 'warning') { - current.warning += 1; - } else { - current.danger += 1; - } - } - - const accounted = current.online + current.warning + current.danger; - if (current.total < accounted) { - current.total = accounted; - } else if (current.total > accounted) { - current.danger += current.total - accounted; - } - - counts.set(poolName, current); - } - } else { - // Summary mode: use poolCounts prop (online/total only). - for (const poolName of allPoolNames) { - const pc = poolCounts.get(poolName) || { online: 0, total: 0 }; - const offline = Math.max(0, pc.total - pc.online); - counts.set(poolName, { online: pc.online, total: pc.total, warning: 0, danger: offline }); - } - } - - return counts; - }, [allPoolNames, gatewayByNode, nodes, poolCounts]); - - const sitePoolGraph = useMemo( - () => buildGraph( - allSiteNames, - allPoolNames, - peerings, - hiddenSites, - hiddenGatewayPools, - new Set(gatewayPools.map((pool) => pool.name).filter((name): name is string => Boolean(name))), - palette, - topologySiteCounts, - topologyPoolCounts, - edgeHealthCheckCounts, - poolToSite - ), - [allPoolNames, allSiteNames, peerings, gatewayPools, hiddenSites, hiddenGatewayPools, palette, topologySiteCounts, topologyPoolCounts, edgeHealthCheckCounts, poolToSite] - ); - const nodeGraph = useMemo( - () => buildNodeGraph(nodes, gatewayByNode, palette, { edgeSize: 1.4 }), - [nodes, gatewayByNode, palette] - ); - const graph = mode === 'nodes' ? nodeGraph : sitePoolGraph; - const dimNodeColor = useCallback((color: string) => { - const hex = (color || '').trim(); - const match = /^#([0-9a-fA-F]{6})$/.exec(hex); - if (!match) return palette.edgeDim; - const value = match[1]; - const r = parseInt(value.slice(0, 2), 16); - const g = parseInt(value.slice(2, 4), 16); - const b = parseInt(value.slice(4, 6), 16); - const mix = (channel: number) => Math.round(channel * 0.35 + 40 * 0.65); - return `#${mix(r).toString(16).padStart(2, '0')}${mix(g).toString(16).padStart(2, '0')}${mix(b).toString(16).padStart(2, '0')}`; - }, [palette.edgeDim]); - - const graphNodesForRender = useMemo(() => { - const baseNodes = mode === 'nodes' - ? graph.nodes.map((node) => ({ ...node, label: '' })) - : graph.nodes; - - if (!hoveredNodeId) return baseNodes; - const connectedNodeIds = new Set([hoveredNodeId]); - for (const edge of graph.edges) { - if (edge.source === hoveredNodeId) { - connectedNodeIds.add(edge.target); - } else if (edge.target === hoveredNodeId) { - connectedNodeIds.add(edge.source); - } - } - - return baseNodes.map((node) => { - if (connectedNodeIds.has(node.id)) { - return node; - } - const dimmed = dimNodeColor(node.fill || palette.site); - return { - ...node, - fill: dimmed, - activeFill: dimmed - }; - }); - }, [graph.edges, graph.nodes, hoveredNodeId, dimNodeColor, mode, palette.site]); - - const graphEdgesForRender = useMemo(() => { - if (!hoveredNodeId) return graph.edges; - return graph.edges.map((edge) => { - const connected = edge.source === hoveredNodeId || edge.target === hoveredNodeId; - if (connected) return edge; - return { - ...edge, - fill: palette.edgeDim - }; - }); - }, [graph.edges, hoveredNodeId, palette.edgeDim]); - - const topologyConfig = useMemo(() => { - const nodeCount = graph.nodes.length; - - const siteTopologyConfig = { - minCameraDistance: 2, - nodeSize: nodeCount <= 6 ? 68 : 50, - layoutOverrides: { - radius: 25, - concentricSpacing: 25 - } - }; - - const nodeTopologyConfig = { - ...siteTopologyConfig, - minCameraDistance: 6, - - layoutOverrides: { - ...siteTopologyConfig.layoutOverrides, - radius: 25, - concentricSpacing: 50 - } - }; - - return mode === 'nodes' ? nodeTopologyConfig : siteTopologyConfig; - }, [graph.nodes.length, mode]); - const topologyLayoutType = mode === 'nodes' ? 'concentric2d' : 'concentric2d'; - - useEffect(() => { - if (mode !== 'sitesAndPools') return; - if (!graphRef.current) return; - - const fit = () => graphRef.current?.fitNodesInView(); - const first = requestAnimationFrame(() => { - const second = requestAnimationFrame(fit); - (fit as unknown as { _second?: number })._second = second; - }); - - return () => { - cancelAnimationFrame(first); - const second = (fit as unknown as { _second?: number })._second; - if (typeof second === 'number') { - cancelAnimationFrame(second); - } - }; - }, [mode, graph.nodes.length, graph.edges.length]); - - useEffect(() => () => { - if (zoomHintTimerRef.current !== null) { - window.clearTimeout(zoomHintTimerRef.current); - zoomHintTimerRef.current = null; - } - }, []); - - const updateZoomHintRect = useCallback(() => { - const rect = topologyWrapperRef.current?.getBoundingClientRect(); - if (!rect) { - setZoomHintRect(null); - return; - } - setZoomHintRect({ left: rect.left, top: rect.top, width: rect.width, height: rect.height }); - }, []); - - const showCtrlZoomHint = useCallback(() => { - updateZoomHintRect(); - setShowZoomHint(true); - if (zoomHintTimerRef.current !== null) { - window.clearTimeout(zoomHintTimerRef.current); - } - zoomHintTimerRef.current = window.setTimeout(() => { - setShowZoomHint(false); - setZoomHintRect(null); - zoomHintTimerRef.current = null; - }, 1400); - }, [updateZoomHintRect]); - - useEffect(() => { - if (!showZoomHint) { - return; - } - const update = () => updateZoomHintRect(); - window.addEventListener('resize', update, { passive: true }); - window.addEventListener('scroll', update, { passive: true, capture: true }); - document.addEventListener('scroll', update, { passive: true, capture: true }); - return () => { - window.removeEventListener('resize', update); - window.removeEventListener('scroll', update, true); - document.removeEventListener('scroll', update, true); - }; - }, [showZoomHint, updateZoomHintRect]); - - const handleWheelCapture = useCallback((event: React.WheelEvent) => { - if (event.ctrlKey) { - return; - } - event.stopPropagation(); - showCtrlZoomHint(); - }, [showCtrlZoomHint]); - - if (graph.nodes.length === 0) { - return
No peering data available.
; - } - - if (!reagraphModule || !graphTheme) { - return
Loading topology renderer...
; - } - - const GraphCanvas = reagraphModule.GraphCanvas; - const zoomHintStyle = (() => { - if (!zoomHintRect) { - return { left: '50vw', top: '50vh' } as React.CSSProperties; - } - return { left: zoomHintRect.left + zoomHintRect.width / 2, top: zoomHintRect.top + zoomHintRect.height / 2 }; - })(); - const zoomHintOverlayStyle = (() => { - if (!zoomHintRect) { - return null; - } - return { left: zoomHintRect.left, top: zoomHintRect.top, width: zoomHintRect.width, height: zoomHintRect.height }; - })(); - - return ( -
-
- - { - const color = n.fill || palette.site; - const group = (n.data as { group?: string } | undefined)?.group; - const glyph = getTopologyNodeGlyph(group); - const material = ( - - ); - const iconNode = renderTopologyNodeGlyph({ - glyph, - size, - color, - textures: topologyIconTextures, - workerOffsetScale: -0.16 - }); - if (iconNode) return iconNode; - - return ( - - - {material} - - ); - }} - onNodePointerOver={(node, event) => { - const tooltipLabel = (node.label || '').trim() - || (node.id.startsWith('node:') - ? node.id.slice(5) - : node.id.startsWith('site:') - ? node.id.slice(5) - : node.id.startsWith('pool:') - ? node.id.slice(5) - : node.id); - setHoveredNodeId(node.id); - setHovered({ - x: event.clientX + 12, - y: event.clientY + 12, - label: tooltipLabel, - group: (node.data as { group?: string })?.group || 'site', - lines: (() => { - const data = node.data as { lines?: string[]; peerings?: string[] } | undefined; - if (data?.lines && data.lines.length > 0) { - return data.lines; - } - const peerings = data?.peerings || []; - return [`Peerings: ${peerings.length > 0 ? peerings.join(', ') : '-'}`]; - })() - }); - }} - onNodePointerOut={() => { - setHovered(null); - setHoveredNodeId(null); - }} - onEdgePointerOver={(edge, event) => { - if (!event) return; - setHovered(null); - setHoveredNodeId(null); - const data = edge.data as { - peerings?: string[]; - hcUp?: number; - hcTotal?: number; - sourceLabel?: string; - targetLabel?: string; - } | undefined; - const hcDetail = data?.hcTotal && data.hcTotal > 0 - ? `${data.hcUp ?? 0}/${data.hcTotal} links up` - : 'No health check data'; - const title = mode === 'nodes' - ? `${data?.sourceLabel || edge.source} <-> ${data?.targetLabel || edge.target}` - : (data?.peerings?.join(', ') || 'Unknown peering'); - setEdgeHovered({ - x: event.clientX + 12, - y: event.clientY + 12, - title, - detail: hcDetail - }); - }} - onEdgePointerOut={() => setEdgeHovered(null)} - onCanvasPointerOut={() => { - setHovered(null); - setHoveredNodeId(null); - setEdgeHovered(null); - }} - onCanvasClick={() => { - setHovered(null); - setHoveredNodeId(null); - setEdgeHovered(null); - }} - /> - {hovered && createPortal( -
-
{hovered.label}
-
Type: { - hovered.group === 'pool' - ? 'Gateway Pool' - : hovered.group === 'site' - ? 'Site' - : hovered.group === 'gateway-node' - ? 'Gateway Node' - : 'Node' - }
- {hovered.lines.map((line, index) => ( -
{line}
- ))} -
, - document.body - )} - {edgeHovered && createPortal( -
-
{edgeHovered.title}
-
{edgeHovered.detail}
-
, - document.body - )} - {showZoomHint && zoomHintOverlayStyle && createPortal( -
, - document.body - )} - {showZoomHint && createPortal( -
- Hold Ctrl and scroll to zoom graph -
, - document.body - )} -
-
- ); -} - - -export default Topology; From bd2c6e16b03ccb4402e84eec428392c632bd78a5 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:19:48 +0000 Subject: [PATCH 03/34] net: remove retired connectivity matrix computation and contracts Remove matrix construction and broadcast fields, obsolete matrix-only coverage, and detached graph helpers. Preserve node overview and resource summaries and unrelated status tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- .../cluster_status.go | 161 ------------- .../cluster_status_test.go | 83 ------- .../matrix_memory_test.go | 113 --------- .../memory_bench_test.go | 34 --- cmd/unbounded-net-controller/status_types.go | 168 ++++++-------- cmd/unbounded-net-controller/websocket.go | 7 +- frontend/src/api.ts | 4 - .../src/components/common/topologyIcons.tsx | 217 ------------------ frontend/src/components/nodes/shared/index.ts | 2 - frontend/src/components/nodes/shared/types.ts | 12 - frontend/src/hooks/useClusterStatus.ts | 2 - frontend/src/types.ts | 9 - frontend/vite.config.ts | 4 +- 13 files changed, 79 insertions(+), 737 deletions(-) delete mode 100644 cmd/unbounded-net-controller/matrix_memory_test.go delete mode 100644 frontend/src/components/common/topologyIcons.tsx delete mode 100644 frontend/src/components/nodes/shared/types.ts diff --git a/cmd/unbounded-net-controller/cluster_status.go b/cmd/unbounded-net-controller/cluster_status.go index 6185da8c8..2f21c62ba 100644 --- a/cmd/unbounded-net-controller/cluster_status.go +++ b/cmd/unbounded-net-controller/cluster_status.go @@ -710,7 +710,6 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo } sort.Slice(status.Peerings, func(i, j int) bool { return status.Peerings[i].Name < status.Peerings[j].Name }) - status.ConnectivityMatrix = buildConnectivityMatrix(status.Nodes, status.GatewayPools) status.Problems = collectClusterProblems(status) return status @@ -1038,163 +1037,3 @@ func latestNodeUpdateTime(node *corev1.Node) time.Time { return latest } - -// buildConnectivityMatrix builds health check connectivity matrices from node peer data. -func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []GatewayPoolStatus) map[string]*SiteMatrix { - siteNodes := make(map[string]map[string]bool) - nodePeers := make(map[string][]WireGuardPeerStatus) - nodeByName := make(map[string]*NodeStatusResponse) - - for _, n := range nodes { - name := n.NodeInfo.Name - - site := n.NodeInfo.SiteName - if name == "" || site == "" { - continue - } - - nodeByName[name] = n - - if siteNodes[site] == nil { - siteNodes[site] = make(map[string]bool) - } - - siteNodes[site][name] = true - - // Keep the immutable snapshot's slice; filter when reading rather - // than copying every peer, including for scopes above the size limit. - nodePeers[name] = n.Peers - - for _, p := range n.Peers { - if p.PeerType == "gateway" && p.Name != "" && p.SiteName == site { - siteNodes[site][p.Name] = true - } - } - } - - if len(siteNodes) == 0 { - siteNodes = make(map[string]map[string]bool) - } - - result := make(map[string]*SiteMatrix) - selfMatrixStatusFromCNI := func(node *NodeStatusResponse) string { - if node.NodeInfo.WireGuard != nil && strings.TrimSpace(node.NodeInfo.WireGuard.Interface) != "" { - return "up" - } - - return "" - } - - buildScopeMatrix := func(nodeSet map[string]bool) *SiteMatrix { - if len(nodeSet) == 0 || len(nodeSet) > 100 { - return nil - } - - nodeNames := make([]string, 0, len(nodeSet)) - for name := range nodeSet { - nodeNames = append(nodeNames, name) - } - - sort.Strings(nodeNames) - - results := make(map[string]map[string]string) - for _, srcNode := range nodeNames { - results[srcNode] = make(map[string]string) - if node, ok := nodeByName[srcNode]; ok { - results[srcNode][srcNode] = selfMatrixStatusFromCNI(node) - } - - for _, peer := range nodePeers[srcNode] { - if !isConnectivityMatrixPeer(peer) { - continue - } - - tgtNode := peer.Name - if tgtNode == "" || tgtNode == srcNode || !nodeSet[tgtNode] { - continue - } - - cellStatus := "" - if peer.HealthCheck != nil { - cellStatus = peer.HealthCheck.Status - } else if peer.PeerType == "gateway" && !peer.Tunnel.LastHandshake.IsZero() { - cellStatus = "up" - } - - results[srcNode][tgtNode] = cellStatus - } - } - - return &SiteMatrix{Nodes: nodeNames, Results: results} - } - - for site, nodeSet := range siteNodes { - scopeMatrix := buildScopeMatrix(nodeSet) - if scopeMatrix != nil { - result[site] = scopeMatrix - } - } - - for _, pool := range gatewayPools { - poolName := strings.TrimSpace(pool.Name) - if poolName == "" { - continue - } - - poolNodeSet := make(map[string]bool) - - for _, gatewayName := range pool.Gateways { - name := strings.TrimSpace(gatewayName) - if name == "" { - continue - } - - poolNodeSet[name] = true - for _, peer := range nodePeers[name] { - if !isConnectivityMatrixPeer(peer) { - continue - } - - peerName := strings.TrimSpace(peer.Name) - if peerName == "" { - continue - } - - if _, ok := nodeByName[peerName]; ok { - poolNodeSet[peerName] = true - } - } - - for srcNodeName, peers := range nodePeers { - for _, peer := range peers { - if !isConnectivityMatrixPeer(peer) { - continue - } - - if strings.TrimSpace(peer.Name) == name { - if _, ok := nodeByName[srcNodeName]; ok { - poolNodeSet[srcNodeName] = true - } - - break - } - } - } - } - - scopeMatrix := buildScopeMatrix(poolNodeSet) - if scopeMatrix != nil { - result["pool:"+poolName] = scopeMatrix - } - } - - if len(result) == 0 { - return nil - } - - return result -} - -func isConnectivityMatrixPeer(peer WireGuardPeerStatus) bool { - return peer.PeerType == "site" || peer.PeerType == "gateway" -} diff --git a/cmd/unbounded-net-controller/cluster_status_test.go b/cmd/unbounded-net-controller/cluster_status_test.go index 1caaeb41f..00d717c1d 100644 --- a/cmd/unbounded-net-controller/cluster_status_test.go +++ b/cmd/unbounded-net-controller/cluster_status_test.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "slices" - "strconv" "strings" "testing" "time" @@ -447,88 +446,6 @@ func TestNodeReadinessAndLatestUpdateTime(t *testing.T) { } } -// TestBuildConnectivityMatrix tests BuildConnectivityMatrix. -func TestBuildConnectivityMatrix(t *testing.T) { - now := time.Now().Add(-75 * time.Second) - - nodes := []*NodeStatusResponse{ - { - NodeInfo: NodeInfo{Name: "node-a", SiteName: "site-a", WireGuard: &WireGuardStatusInfo{Interface: "wg51820"}}, - Peers: []WireGuardPeerStatus{ - {Name: "node-b", PeerType: "site", HealthCheck: &HealthCheckPeerStatus{Status: "up", Uptime: "15s"}}, - {Name: "gw-a", PeerType: "gateway", SiteName: "site-a", Tunnel: PeerTunnelStatus{LastHandshake: now}}, - {Name: "gw-remote", PeerType: "gateway", SiteName: "site-b", Tunnel: PeerTunnelStatus{LastHandshake: now}}, - }, - }, - { - NodeInfo: NodeInfo{Name: "node-b", SiteName: "site-a"}, - Peers: []WireGuardPeerStatus{ - {Name: "node-a", PeerType: "site", HealthCheck: &HealthCheckPeerStatus{Status: "down", Uptime: "3s"}}, - }, - }, - } - for i := 0; i < 101; i++ { - nodes = append(nodes, &NodeStatusResponse{NodeInfo: NodeInfo{Name: "big-" + strconv.Itoa(i), SiteName: "site-big"}}) - } - - gatewayPools := []GatewayPoolStatus{{ - Name: "pool-a", - Gateways: []string{"gw-a"}, - }} - - matrix := buildConnectivityMatrix(nodes, gatewayPools) - if matrix == nil { - t.Fatalf("expected non-nil connectivity matrix") - } - - if _, ok := matrix["site-big"]; ok { - t.Fatalf("expected site-big to be skipped when >100 nodes") - } - - site := matrix["site-a"] - if site == nil { - t.Fatalf("expected site-a matrix") - } - - if !slices.Equal(site.Nodes, []string{"gw-a", "node-a", "node-b"}) { - t.Fatalf("unexpected node list: %#v", site.Nodes) - } - - if got := site.Results["node-a"]["node-b"]; got != "up" { - t.Fatalf("unexpected node-a->node-b status: %q", got) - } - - gatewayCell := site.Results["node-a"]["gw-a"] - if gatewayCell != "up" { - t.Fatalf("unexpected gateway fallback cell: %q", gatewayCell) - } - - if _, ok := site.Results["node-a"]["gw-remote"]; ok { - t.Fatalf("did not expect remote-site gateway in site matrix") - } - - if got := site.Results["node-a"]["node-a"]; got != "up" { - t.Fatalf("expected self cell for node-a to be up from CNI health, got %q", got) - } - - if got := site.Results["node-b"]["node-b"]; got != "" { - t.Fatalf("expected self cell for node-b to be unknown when CNI health is unavailable, got %q", got) - } - - pool := matrix["pool:pool-a"] - if pool == nil { - t.Fatalf("expected pool:pool-a matrix") - } - - if !slices.Equal(pool.Nodes, []string{"gw-a", "node-a"}) { - t.Fatalf("unexpected pool node list: %#v", pool.Nodes) - } - - if got := pool.Results["node-a"]["gw-a"]; got != "up" { - t.Fatalf("unexpected node-a->gw-a pool status: %q", got) - } -} - // TestCollectClusterProblemsIncludesUnhealthySignals tests CollectClusterProblemsIncludesUnhealthySignals. func TestCollectClusterProblemsIncludesUnhealthySignals(t *testing.T) { expectedTrue := true diff --git a/cmd/unbounded-net-controller/matrix_memory_test.go b/cmd/unbounded-net-controller/matrix_memory_test.go deleted file mode 100644 index e7ed3f048..000000000 --- a/cmd/unbounded-net-controller/matrix_memory_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "slices" - "testing" - "time" -) - -func TestConnectivityMatrixMixedScopesPreservesPeers(t *testing.T) { - nodes := make([]*NodeStatusResponse, 0, 102) - for i := range 101 { - nodes = append(nodes, &NodeStatusResponse{NodeInfo: NodeInfo{ - Name: fmt.Sprintf("node-%d", i), SiteName: "large", - }}) - } - - nodes[0].Peers = []WireGuardPeerStatus{{ - Name: "gateway", PeerType: "gateway", SiteName: "small", - Tunnel: PeerTunnelStatus{LastHandshake: time.Unix(1, 0)}, - }} - nodes[1].Peers = []WireGuardPeerStatus{{Name: "gateway", PeerType: "ignored"}} - nodes = append(nodes, &NodeStatusResponse{ - NodeInfo: NodeInfo{Name: "gateway", SiteName: "small"}, - Peers: []WireGuardPeerStatus{ - {Name: "node-0", PeerType: "site", HealthCheck: &HealthCheckPeerStatus{Status: "up"}}, - {Name: "node-0", PeerType: "ignored", HealthCheck: &HealthCheckPeerStatus{Status: "down"}}, - {Name: "node-2", PeerType: "ignored"}, - {Name: "gateway", PeerType: "ignored", HealthCheck: &HealthCheckPeerStatus{Status: "down"}}, - }, - }) - - before, err := json.Marshal(nodes) - if err != nil { - t.Fatal(err) - } - - matrix := buildConnectivityMatrix(nodes, []GatewayPoolStatus{{Name: "pool", Gateways: []string{"gateway"}}}) - if _, ok := matrix["large"]; ok { - t.Fatal("oversized site produced a matrix") - } - - if matrix["small"] == nil || !slices.Equal(matrix["small"].Nodes, []string{"gateway"}) { - t.Fatalf("small site lost its matrix: %+v", matrix["small"]) - } - - pool := matrix["pool:pool"] - if pool == nil || !slices.Equal(pool.Nodes, []string{"gateway", "node-0"}) { - t.Fatalf("small pool crossing a large site has wrong membership: %+v", pool) - } - - if pool.Results["gateway"]["node-0"] != "up" || pool.Results["node-0"]["gateway"] != "up" { - t.Fatalf("pool connectivity changed: %+v", pool.Results) - } - - after, err := json.Marshal(nodes) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(before, after) { - t.Fatal("matrix construction mutated the shared node snapshots") - } -} - -func TestConnectivityMatrixDoesNotCopyPeerSlices(t *testing.T) { - nodes := matrixBenchmarkNodes(200) - - peers := matrixBenchmarkNodes(2000)[0].Peers - for _, node := range nodes { - node.Peers = peers - } - - var matrix map[string]*SiteMatrix - - result := testing.Benchmark(func(b *testing.B) { - for b.Loop() { - matrix = buildConnectivityMatrix(nodes, nil) - } - }) - - if matrix != nil { - t.Fatal("oversized site produced a matrix") - } - // Allow map bookkeeping, but not storage proportional to every peer. - if allocated := result.AllocedBytesPerOp(); allocated > 512*1024 { - t.Fatalf("matrix copied peer data: %d bytes per call", allocated) - } -} - -func TestConnectivityMatrixSizeBoundary(t *testing.T) { - for _, count := range []int{0, 100, 101} { - t.Run(fmt.Sprintf("nodes-%d", count), func(t *testing.T) { - matrix := buildConnectivityMatrix(matrixBenchmarkNodes(count), nil) - if count != 100 { - if matrix != nil { - t.Fatal("empty or oversized site produced a matrix") - } - - return - } - - if matrix["site-a"] == nil || len(matrix["site-a"].Nodes) != count { - t.Fatal("site at the size limit lost its matrix") - } - }) - } -} diff --git a/cmd/unbounded-net-controller/memory_bench_test.go b/cmd/unbounded-net-controller/memory_bench_test.go index ca548d0a6..1ccc15873 100644 --- a/cmd/unbounded-net-controller/memory_bench_test.go +++ b/cmd/unbounded-net-controller/memory_bench_test.go @@ -68,37 +68,3 @@ func BenchmarkProtoWSStatusFrame(b *testing.B) { }) } } - -func matrixBenchmarkNodes(count int) []*NodeStatusResponse { - peers := make([]WireGuardPeerStatus, count) - - nodes := make([]*NodeStatusResponse, count) - for i := range count { - name := fmt.Sprintf("node-%d", i) - peers[i] = WireGuardPeerStatus{Name: name, PeerType: "site", SiteName: "site-a"} - nodes[i] = &NodeStatusResponse{ - NodeInfo: NodeInfo{Name: name, SiteName: "site-a"}, - Peers: peers, - } - } - - return nodes -} - -func BenchmarkBuildConnectivityMatrix(b *testing.B) { - for _, count := range []int{100, 101, 2000} { - b.Run(fmt.Sprintf("nodes-%d", count), func(b *testing.B) { - nodes := matrixBenchmarkNodes(count) - - b.ReportAllocs() - b.ResetTimer() - - for b.Loop() { - matrix := buildConnectivityMatrix(nodes, nil) - if count > 100 && matrix != nil { - b.Fatal("oversized site produced a matrix") - } - } - }) - } -} diff --git a/cmd/unbounded-net-controller/status_types.go b/cmd/unbounded-net-controller/status_types.go index 98b72c1bd..567a13be5 100644 --- a/cmd/unbounded-net-controller/status_types.go +++ b/cmd/unbounded-net-controller/status_types.go @@ -14,42 +14,40 @@ import ( // ClusterStatusResponse is the top-level status response for the cluster. type ClusterStatusResponse struct { - Seq uint64 `json:"seq"` - Timestamp time.Time `json:"timestamp"` - NodeCount int `json:"nodeCount"` - SiteCount int `json:"siteCount"` - AzureTenantID string `json:"azureTenantId,omitempty"` - LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` - BuildInfo *BuildInfo `json:"buildInfo,omitempty"` - Nodes []*NodeStatusResponse `json:"nodes"` - Sites []SiteStatus `json:"sites"` - GatewayPools []GatewayPoolStatus `json:"gatewayPools"` - Peerings []PeeringStatus `json:"peerings"` - Errors []string `json:"errors,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Problems []StatusProblem `json:"problems"` - ConnectivityMatrix map[string]*SiteMatrix `json:"connectivityMatrix,omitempty"` - PullEnabled bool `json:"pullEnabled"` + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + NodeCount int `json:"nodeCount"` + SiteCount int `json:"siteCount"` + AzureTenantID string `json:"azureTenantId,omitempty"` + LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` + BuildInfo *BuildInfo `json:"buildInfo,omitempty"` + Nodes []*NodeStatusResponse `json:"nodes"` + Sites []SiteStatus `json:"sites"` + GatewayPools []GatewayPoolStatus `json:"gatewayPools"` + Peerings []PeeringStatus `json:"peerings"` + Errors []string `json:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Problems []StatusProblem `json:"problems"` + PullEnabled bool `json:"pullEnabled"` } // ClusterStatusDelta is a WebSocket delta update. type ClusterStatusDelta struct { - Seq uint64 `json:"seq"` - Timestamp time.Time `json:"timestamp"` - NodeCount int `json:"nodeCount"` - SiteCount int `json:"siteCount"` - AzureTenantID string `json:"azureTenantId,omitempty"` - LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` - Errors []string `json:"errors,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Problems []StatusProblem `json:"problems"` - UpdatedNodes []json.RawMessage `json:"updatedNodes,omitempty"` - RemovedNodes []string `json:"removedNodes,omitempty"` - Sites []SiteStatus `json:"sites"` - GatewayPools []GatewayPoolStatus `json:"gatewayPools"` - Peerings []PeeringStatus `json:"peerings"` - ConnectivityMatrix map[string]*SiteMatrix `json:"connectivityMatrix,omitempty"` - PullEnabled bool `json:"pullEnabled"` + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + NodeCount int `json:"nodeCount"` + SiteCount int `json:"siteCount"` + AzureTenantID string `json:"azureTenantId,omitempty"` + LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` + Errors []string `json:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Problems []StatusProblem `json:"problems"` + UpdatedNodes []json.RawMessage `json:"updatedNodes,omitempty"` + RemovedNodes []string `json:"removedNodes,omitempty"` + Sites []SiteStatus `json:"sites"` + GatewayPools []GatewayPoolStatus `json:"gatewayPools"` + Peerings []PeeringStatus `json:"peerings"` + PullEnabled bool `json:"pullEnabled"` } // StatusProblem describes one unhealthy condition surfaced in cluster status. @@ -149,22 +147,21 @@ type BpfEntry = statusv1alpha1.BpfEntry // everything from ClusterStatusResponse except detailed per-node status // (routes, peers, health check details), replacing those with NodeSummary rows. type ClusterSummary struct { - Seq uint64 `json:"seq"` - Timestamp time.Time `json:"timestamp"` - NodeCount int `json:"nodeCount"` - SiteCount int `json:"siteCount"` - AzureTenantID string `json:"azureTenantId,omitempty"` - LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` - BuildInfo *BuildInfo `json:"buildInfo,omitempty"` - Sites []SiteStatus `json:"sites"` - GatewayPools []GatewayPoolStatus `json:"gatewayPools"` - Peerings []PeeringStatus `json:"peerings"` - Errors []string `json:"errors,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Problems []StatusProblem `json:"problems"` - PullEnabled bool `json:"pullEnabled"` - NodeSummaries []NodeSummary `json:"nodeSummaries"` - ConnectivityMatrix map[string]*SiteMatrix `json:"connectivityMatrix,omitempty"` + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + NodeCount int `json:"nodeCount"` + SiteCount int `json:"siteCount"` + AzureTenantID string `json:"azureTenantId,omitempty"` + LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` + BuildInfo *BuildInfo `json:"buildInfo,omitempty"` + Sites []SiteStatus `json:"sites"` + GatewayPools []GatewayPoolStatus `json:"gatewayPools"` + Peerings []PeeringStatus `json:"peerings"` + Errors []string `json:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Problems []StatusProblem `json:"problems"` + PullEnabled bool `json:"pullEnabled"` + NodeSummaries []NodeSummary `json:"nodeSummaries"` } // NodeSummary is a compact per-node summary for use in ClusterSummary. @@ -251,22 +248,21 @@ func buildClusterSummary(status *ClusterStatusResponse) *ClusterSummary { sort.Slice(summaries, func(i, j int) bool { return summaries[i].Name < summaries[j].Name }) return &ClusterSummary{ - Seq: status.Seq, - Timestamp: status.Timestamp, - NodeCount: status.NodeCount, - SiteCount: status.SiteCount, - AzureTenantID: status.AzureTenantID, - LeaderInfo: status.LeaderInfo, - BuildInfo: status.BuildInfo, - Sites: status.Sites, - GatewayPools: status.GatewayPools, - Peerings: status.Peerings, - Errors: status.Errors, - Warnings: status.Warnings, - Problems: status.Problems, - PullEnabled: status.PullEnabled, - NodeSummaries: summaries, - ConnectivityMatrix: status.ConnectivityMatrix, + Seq: status.Seq, + Timestamp: status.Timestamp, + NodeCount: status.NodeCount, + SiteCount: status.SiteCount, + AzureTenantID: status.AzureTenantID, + LeaderInfo: status.LeaderInfo, + BuildInfo: status.BuildInfo, + Sites: status.Sites, + GatewayPools: status.GatewayPools, + Peerings: status.Peerings, + Errors: status.Errors, + Warnings: status.Warnings, + Problems: status.Problems, + PullEnabled: status.PullEnabled, + NodeSummaries: summaries, } } @@ -330,33 +326,26 @@ type PeeringStatus struct { HealthCheckEnabled bool `json:"healthCheckEnabled,omitempty"` } -// SiteMatrix contains connectivity results across site nodes. -type SiteMatrix struct { - Nodes []string `json:"nodes"` - Results map[string]map[string]string `json:"results"` // src -> dst -> status -} - // ClusterSummaryDelta contains only the fields of ClusterSummary that changed // since the last broadcast. NodeSummaries contains only added/changed entries; // RemovedNodes lists nodes that disappeared. type ClusterSummaryDelta struct { - Seq uint64 `json:"seq"` - Timestamp time.Time `json:"timestamp"` - NodeCount *int `json:"nodeCount,omitempty"` - SiteCount *int `json:"siteCount,omitempty"` - AzureTenantID *string `json:"azureTenantId,omitempty"` - LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` - BuildInfo *BuildInfo `json:"buildInfo,omitempty"` - Sites []SiteStatus `json:"sites,omitempty"` - GatewayPools []GatewayPoolStatus `json:"gatewayPools,omitempty"` - Peerings []PeeringStatus `json:"peerings,omitempty"` - Errors []string `json:"errors,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Problems []StatusProblem `json:"problems,omitempty"` - PullEnabled *bool `json:"pullEnabled,omitempty"` - NodeSummaries []NodeSummary `json:"nodeSummaries,omitempty"` - RemovedNodes []string `json:"removedNodes,omitempty"` - ConnectivityMatrix map[string]*SiteMatrix `json:"connectivityMatrix,omitempty"` + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + NodeCount *int `json:"nodeCount,omitempty"` + SiteCount *int `json:"siteCount,omitempty"` + AzureTenantID *string `json:"azureTenantId,omitempty"` + LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` + BuildInfo *BuildInfo `json:"buildInfo,omitempty"` + Sites []SiteStatus `json:"sites,omitempty"` + GatewayPools []GatewayPoolStatus `json:"gatewayPools,omitempty"` + Peerings []PeeringStatus `json:"peerings,omitempty"` + Errors []string `json:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Problems []StatusProblem `json:"problems,omitempty"` + PullEnabled *bool `json:"pullEnabled,omitempty"` + NodeSummaries []NodeSummary `json:"nodeSummaries,omitempty"` + RemovedNodes []string `json:"removedNodes,omitempty"` } // computeClusterSummaryDelta computes a delta between two ClusterSummary snapshots. @@ -437,11 +426,6 @@ func computeClusterSummaryDelta(prev, curr *ClusterSummary) *ClusterSummaryDelta changed = true } - if !jsonEqual(prev.ConnectivityMatrix, curr.ConnectivityMatrix) { - delta.ConnectivityMatrix = curr.ConnectivityMatrix - changed = true - } - // NodeSummaries: diff by name prevByName := make(map[string]NodeSummary, len(prev.NodeSummaries)) for _, ns := range prev.NodeSummaries { diff --git a/cmd/unbounded-net-controller/websocket.go b/cmd/unbounded-net-controller/websocket.go index 0a9ceaf82..7a4bda15d 100644 --- a/cmd/unbounded-net-controller/websocket.go +++ b/cmd/unbounded-net-controller/websocket.go @@ -321,8 +321,8 @@ func (b *WSBroadcaster) broadcastUpdate(ctx context.Context) { return } - klog.V(4).Infof("WebSocket: summary delta: %d nodeSummaries, %d removed, sites=%v pools=%v matrix=%v", - len(delta.NodeSummaries), len(delta.RemovedNodes), delta.Sites != nil, delta.GatewayPools != nil, delta.ConnectivityMatrix != nil) + klog.V(4).Infof("WebSocket: summary delta: %d nodeSummaries, %d removed, sites=%v pools=%v", + len(delta.NodeSummaries), len(delta.RemovedNodes), delta.Sites != nil, delta.GatewayPools != nil) msg := WSMessage{Type: "cluster_summary_delta", Data: delta} summaryData, _ = json.Marshal(msg) //nolint:errcheck } else { @@ -422,9 +422,6 @@ func (b *WSBroadcaster) broadcastUpdate(ctx context.Context) { PullEnabled: status.PullEnabled, } - // Always include ConnectivityMatrix so link-state-only changes refresh clients. - delta.ConnectivityMatrix = status.ConnectivityMatrix - msg := WSMessage{Type: "cluster_status_delta", Data: delta} deltaData, err := json.Marshal(msg) diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 4036872df..49bce2631 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -90,10 +90,6 @@ export function mergeDelta(current: ClusterStatus | null, delta: ClusterStatusDe merged.nodes = Object.values(nodeMap); } - if (delta.connectivityMatrix !== undefined && delta.connectivityMatrix !== null) { - merged.connectivityMatrix = delta.connectivityMatrix || undefined; - } - return merged; } diff --git a/frontend/src/components/common/topologyIcons.tsx b/frontend/src/components/common/topologyIcons.tsx deleted file mode 100644 index 0c81c0156..000000000 --- a/frontend/src/components/common/topologyIcons.tsx +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import * as React from 'react'; -import { useEffect, useMemo } from 'react'; -import * as THREE from 'three'; - -const gatewayPoolSvgIcon = ` - - - - - - - - - - - - - - -`; - -const workerSiteSvgIcon = ` - - - - - - - - - - - - - - - - - - - - - - - - -`; - -const maskFromSVG = (svg: string): string => { - return svg - .split('#00bbf1').join('#ffffff') - .split('#c5c5c5').join('#ffffff') - .split('#e6f8fe').join('#ffffff') - .split('#ccf1fc').join('#ffffff') - .split('#80ddf8').join('#ffffff') - .split('#919191').join('#ffffff'); -}; - -type TopologyNodeIconTextures = { - gatewayPoolIconTexture: THREE.Texture; - workerSiteIconTexture: THREE.Texture; - gatewayPoolMaskTexture: THREE.Texture; - workerSiteMaskTexture: THREE.Texture; -}; - -type RenderTopologyNodeGlyphArgs = { - glyph: 'router' | 'vm' | 'default'; - size: number; - color: string; - textures: TopologyNodeIconTextures; - workerOffsetScale?: number; -}; - -export function getTopologyNodeGlyph(group?: string): 'router' | 'vm' | 'default' { - if (group === 'pool' || group === 'gateway-node') { - return 'router'; - } - if (group === 'site' || group === 'worker-node') { - return 'vm'; - } - return 'default'; -} - -export function useTopologyNodeIconTextures(): TopologyNodeIconTextures { - const gatewayPoolIconTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(gatewayPoolSvgIcon)}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - const workerSiteIconTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(workerSiteSvgIcon)}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - const gatewayPoolMaskTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(maskFromSVG(gatewayPoolSvgIcon))}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - const workerSiteMaskTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(maskFromSVG(workerSiteSvgIcon))}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - useEffect(() => { - return () => { - gatewayPoolIconTexture.dispose(); - workerSiteIconTexture.dispose(); - gatewayPoolMaskTexture.dispose(); - workerSiteMaskTexture.dispose(); - }; - }, [gatewayPoolIconTexture, workerSiteIconTexture, gatewayPoolMaskTexture, workerSiteMaskTexture]); - - return { - gatewayPoolIconTexture, - workerSiteIconTexture, - gatewayPoolMaskTexture, - workerSiteMaskTexture - }; -} - -export function renderTopologyNodeGlyph({ - glyph, - size, - color, - textures, - workerOffsetScale = -0.16 -}: RenderTopologyNodeGlyphArgs): React.ReactNode { - if (glyph === 'router') { - return ( - - - - - - - - - - - ); - } - - if (glyph === 'vm') { - return ( - - - - - - - - - - - ); - } - - return null; -} diff --git a/frontend/src/components/nodes/shared/index.ts b/frontend/src/components/nodes/shared/index.ts index 6e8a4b51b..e57457d0f 100644 --- a/frontend/src/components/nodes/shared/index.ts +++ b/frontend/src/components/nodes/shared/index.ts @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 -export type { ReagraphModule } from './types'; - export { uiDiag } from './uiDiag'; export { CloseXIcon, MagnifyPlusIcon, TableFilterButton, useDismissOnOutside } from './tableUi'; export { diff --git a/frontend/src/components/nodes/shared/types.ts b/frontend/src/components/nodes/shared/types.ts deleted file mode 100644 index f4a4994bc..000000000 --- a/frontend/src/components/nodes/shared/types.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import * as React from 'react'; - -type ReagraphModule = { - GraphCanvas: React.ComponentType; - darkTheme: Record; - lightTheme: Record; -}; - -export type { ReagraphModule }; diff --git a/frontend/src/hooks/useClusterStatus.ts b/frontend/src/hooks/useClusterStatus.ts index c987b9fe8..7e9500520 100644 --- a/frontend/src/hooks/useClusterStatus.ts +++ b/frontend/src/hooks/useClusterStatus.ts @@ -160,7 +160,6 @@ function buildSummaryFromFullStatus(cs: ClusterStatus): ClusterSummary { problems: cs.problems, pullEnabled: cs.pullEnabled, nodeSummaries: nodes.map(buildNodeSummaryFromNodeStatus), - connectivityMatrix: cs.connectivityMatrix, }; } @@ -274,7 +273,6 @@ function useClusterStatus() { if (delta.warnings) merged.warnings = delta.warnings; if (delta.problems) merged.problems = delta.problems; if (delta.pullEnabled != null) merged.pullEnabled = delta.pullEnabled; - if (delta.connectivityMatrix !== undefined) merged.connectivityMatrix = delta.connectivityMatrix; if (delta.nodeSummaries || delta.removedNodes) { const byName = new Map(); for (const ns of prev.nodeSummaries || []) { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 79530d5a5..c6ad4a4ff 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -10,7 +10,6 @@ export type ClusterStatus = { sites?: SiteStatus[]; gatewayPools?: GatewayPoolStatus[]; peerings?: PeeringStatus[]; - connectivityMatrix?: Record; buildInfo?: BuildInfo; leaderInfo?: LeaderInfo; errors?: string[]; @@ -195,11 +194,6 @@ export type PeeringStatus = { healthCheckEnabled?: boolean; }; -export type SiteMatrix = { - nodes?: string[]; - results?: Record>; -}; - export type ClusterStatusDelta = { seq?: number; timestamp?: string; @@ -212,7 +206,6 @@ export type ClusterStatusDelta = { sites?: SiteStatus[]; gatewayPools?: GatewayPoolStatus[]; peerings?: PeeringStatus[]; - connectivityMatrix?: Record | null; buildInfo?: BuildInfo; leaderInfo?: LeaderInfo; errors?: string[]; @@ -237,7 +230,6 @@ export type ClusterSummary = { problems?: StatusProblem[]; pullEnabled?: boolean; nodeSummaries?: NodeSummary[]; - connectivityMatrix?: Record; }; export type ClusterSummaryDelta = { @@ -257,7 +249,6 @@ export type ClusterSummaryDelta = { pullEnabled?: boolean; nodeSummaries?: NodeSummary[]; removedNodes?: string[]; - connectivityMatrix?: Record; }; export type NodeSummary = { diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 9a28582bd..9bf92c9b4 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -28,9 +28,7 @@ export default defineConfig(({ mode }) => { output: { codeSplitting: { groups: [ - { name: 'tanstack', test: /node_modules[\\/]@tanstack[\\/](react-table|table-core)[\\/]/ }, - { name: 'reagraph', test: /node_modules[\\/]reagraph[\\/]/ }, - { name: 'three', test: /node_modules[\\/]three[\\/]/ } + { name: 'tanstack', test: /node_modules[\\/]@tanstack[\\/](react-table|table-core)[\\/]/ } ] } } From 4032b99a16e15884093d68eda3a1c09e662fc7eb Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:13:37 +0000 Subject: [PATCH 04/34] feat(net): add lightweight node status overview contracts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- internal/net/status/overview_test.go | 80 ++++++++ internal/net/status/proto/status.pb.go | 250 ++++++++++++++++++++----- internal/net/status/proto/status.proto | 19 +- internal/net/status/v1alpha1/types.go | 17 ++ 4 files changed, 323 insertions(+), 43 deletions(-) create mode 100644 internal/net/status/overview_test.go diff --git a/internal/net/status/overview_test.go b/internal/net/status/overview_test.go new file mode 100644 index 000000000..7b40088c4 --- /dev/null +++ b/internal/net/status/overview_test.go @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "encoding/json" + "testing" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestOverviewJSONFields(t *testing.T) { + data, err := json.Marshal(v1alpha1.NodeStatusOverview{}) + if err != nil { + t.Fatal(err) + } + + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + t.Fatal(err) + } + + for _, name := range []string{"peerCount", "healthyPeers", "routeCount", "routeMismatch"} { + if _, ok := fields[name]; !ok { + t.Errorf("zero-valued overview fact %q omitted", name) + } + } + + for _, name := range []string{"peers", "routingTable", "bpfEntries", "peerMeasurements"} { + if _, ok := fields[name]; ok { + t.Errorf("detail field %q present", name) + } + } +} + +func TestOverviewProtoContract(t *testing.T) { + want := &statusproto.NodeStatusMessage{ + Type: "node_status_summary", NodeName: "node", + Summary: &statusproto.NodeStatusOverview{ + NodeInfo: &statusproto.NodeInfo{Name: "node"}, + PeerCount: 7, HealthyPeers: 5, RouteCount: 11, RouteMismatch: true, + NodeErrors: []*statusproto.NodeError{{Type: "cni", Message: "not ready"}}, + }, + } + + data, err := proto.Marshal(want) + if err != nil { + t.Fatal(err) + } + + got := &statusproto.NodeStatusMessage{} + if err := proto.Unmarshal(data, got); err != nil { + t.Fatal(err) + } + + if !proto.Equal(got, want) || got.Status != nil || got.Delta != nil { + t.Fatalf("summary round trip changed payload: %v", got) + } + + fields := got.ProtoReflect().Descriptor().Fields() + for name, number := range map[protoreflect.Name]protoreflect.FieldNumber{ + "type": 1, "node_name": 2, "base_revision": 3, "status": 4, "delta": 5, "summary": 6, + } { + if field := fields.ByName(name); field == nil || field.Number() != number { + t.Errorf("field %s no longer has number %d", name, number) + } + } + + summaryFields := got.Summary.ProtoReflect().Descriptor().Fields() + for _, name := range []protoreflect.Name{"peers", "routing_table", "bpf_entries", "peer_measurements"} { + if summaryFields.ByName(name) != nil { + t.Errorf("summary exposes detail field %q", name) + } + } +} diff --git a/internal/net/status/proto/status.pb.go b/internal/net/status/proto/status.pb.go index 3330a065b..5c7505f40 100644 --- a/internal/net/status/proto/status.pb.go +++ b/internal/net/status/proto/status.pb.go @@ -27,11 +27,12 @@ const ( // NodeStatusMessage wraps all node-to-controller status messages. type NodeStatusMessage struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // "node_status_full" or "node_status_delta" + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // "node_status_full", "node_status_delta", or "node_status_summary" NodeName string `protobuf:"bytes,2,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` BaseRevision uint64 `protobuf:"varint,3,opt,name=base_revision,json=baseRevision,proto3" json:"base_revision,omitempty"` - Status *NodeStatusFull `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` // set for full updates - Delta *NodeStatusDelta `protobuf:"bytes,5,opt,name=delta,proto3" json:"delta,omitempty"` // set for delta updates + Status *NodeStatusFull `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` // set for full updates + Delta *NodeStatusDelta `protobuf:"bytes,5,opt,name=delta,proto3" json:"delta,omitempty"` // set for delta updates + Summary *NodeStatusOverview `protobuf:"bytes,6,opt,name=summary,proto3" json:"summary,omitempty"` // complete overview, including on resync unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -101,6 +102,13 @@ func (x *NodeStatusMessage) GetDelta() *NodeStatusDelta { return nil } +func (x *NodeStatusMessage) GetSummary() *NodeStatusOverview { + if x != nil { + return x.Summary + } + return nil +} + // NodeStatusAck is the acknowledgment returned by the controller for push updates. type NodeStatusAck struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1822,17 +1830,151 @@ func (x *BpfEntry) GetHealthy() bool { return false } +// NodeStatusOverview carries observed overview facts without detailed arrays. +type NodeStatusOverview struct { + state protoimpl.MessageState `protogen:"open.v1"` + TimestampUnixNs int64 `protobuf:"varint,1,opt,name=timestamp_unix_ns,json=timestampUnixNs,proto3" json:"timestamp_unix_ns,omitempty"` + NodeInfo *NodeInfo `protobuf:"bytes,2,opt,name=node_info,json=nodeInfo,proto3" json:"node_info,omitempty"` + HealthCheck *HealthCheckStatus `protobuf:"bytes,3,opt,name=health_check,json=healthCheck,proto3" json:"health_check,omitempty"` + NodeErrors []*NodeError `protobuf:"bytes,4,rep,name=node_errors,json=nodeErrors,proto3" json:"node_errors,omitempty"` + FetchError string `protobuf:"bytes,5,opt,name=fetch_error,json=fetchError,proto3" json:"fetch_error,omitempty"` + LastPushTimeUnixNs int64 `protobuf:"varint,6,opt,name=last_push_time_unix_ns,json=lastPushTimeUnixNs,proto3" json:"last_push_time_unix_ns,omitempty"` // 0 means unset + StatusSource string `protobuf:"bytes,7,opt,name=status_source,json=statusSource,proto3" json:"status_source,omitempty"` + NodePodInfo *NodePodInfo `protobuf:"bytes,8,opt,name=node_pod_info,json=nodePodInfo,proto3" json:"node_pod_info,omitempty"` + PeerCount int32 `protobuf:"varint,9,opt,name=peer_count,json=peerCount,proto3" json:"peer_count,omitempty"` + HealthyPeers int32 `protobuf:"varint,10,opt,name=healthy_peers,json=healthyPeers,proto3" json:"healthy_peers,omitempty"` + RouteCount int32 `protobuf:"varint,11,opt,name=route_count,json=routeCount,proto3" json:"route_count,omitempty"` + RouteMismatch bool `protobuf:"varint,12,opt,name=route_mismatch,json=routeMismatch,proto3" json:"route_mismatch,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NodeStatusOverview) Reset() { + *x = NodeStatusOverview{} + mi := &file_status_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NodeStatusOverview) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeStatusOverview) ProtoMessage() {} + +func (x *NodeStatusOverview) ProtoReflect() protoreflect.Message { + mi := &file_status_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeStatusOverview.ProtoReflect.Descriptor instead. +func (*NodeStatusOverview) Descriptor() ([]byte, []int) { + return file_status_proto_rawDescGZIP(), []int{21} +} + +func (x *NodeStatusOverview) GetTimestampUnixNs() int64 { + if x != nil { + return x.TimestampUnixNs + } + return 0 +} + +func (x *NodeStatusOverview) GetNodeInfo() *NodeInfo { + if x != nil { + return x.NodeInfo + } + return nil +} + +func (x *NodeStatusOverview) GetHealthCheck() *HealthCheckStatus { + if x != nil { + return x.HealthCheck + } + return nil +} + +func (x *NodeStatusOverview) GetNodeErrors() []*NodeError { + if x != nil { + return x.NodeErrors + } + return nil +} + +func (x *NodeStatusOverview) GetFetchError() string { + if x != nil { + return x.FetchError + } + return "" +} + +func (x *NodeStatusOverview) GetLastPushTimeUnixNs() int64 { + if x != nil { + return x.LastPushTimeUnixNs + } + return 0 +} + +func (x *NodeStatusOverview) GetStatusSource() string { + if x != nil { + return x.StatusSource + } + return "" +} + +func (x *NodeStatusOverview) GetNodePodInfo() *NodePodInfo { + if x != nil { + return x.NodePodInfo + } + return nil +} + +func (x *NodeStatusOverview) GetPeerCount() int32 { + if x != nil { + return x.PeerCount + } + return 0 +} + +func (x *NodeStatusOverview) GetHealthyPeers() int32 { + if x != nil { + return x.HealthyPeers + } + return 0 +} + +func (x *NodeStatusOverview) GetRouteCount() int32 { + if x != nil { + return x.RouteCount + } + return 0 +} + +func (x *NodeStatusOverview) GetRouteMismatch() bool { + if x != nil { + return x.RouteMismatch + } + return false +} + var File_status_proto protoreflect.FileDescriptor const file_status_proto_rawDesc = "" + "\n" + - "\fstatus.proto\x12\x16unboundednet.status.v1\"\xe8\x01\n" + + "\fstatus.proto\x12\x16unboundednet.status.v1\"\xae\x02\n" + "\x11NodeStatusMessage\x12\x12\n" + "\x04type\x18\x01 \x01(\tR\x04type\x12\x1b\n" + "\tnode_name\x18\x02 \x01(\tR\bnodeName\x12#\n" + "\rbase_revision\x18\x03 \x01(\x04R\fbaseRevision\x12>\n" + "\x06status\x18\x04 \x01(\v2&.unboundednet.status.v1.NodeStatusFullR\x06status\x12=\n" + - "\x05delta\x18\x05 \x01(\v2'.unboundednet.status.v1.NodeStatusDeltaR\x05delta\"\x88\x01\n" + + "\x05delta\x18\x05 \x01(\v2'.unboundednet.status.v1.NodeStatusDeltaR\x05delta\x12D\n" + + "\asummary\x18\x06 \x01(\v2*.unboundednet.status.v1.NodeStatusOverviewR\asummary\"\x88\x01\n" + "\rNodeStatusAck\x12\x16\n" + "\x06status\x18\x01 \x01(\tR\x06status\x12\x1a\n" + "\brevision\x18\x02 \x01(\x04R\brevision\x12\x16\n" + @@ -2011,7 +2153,25 @@ const file_status_proto_rawDesc = "" + "\x03vni\x18\x06 \x01(\rR\x03vni\x12\x10\n" + "\x03mtu\x18\a \x01(\x05R\x03mtu\x12\x18\n" + "\aifindex\x18\b \x01(\rR\aifindex\x12\x18\n" + - "\ahealthy\x18\t \x01(\bR\ahealthyB6Z4github.com/Azure/unbounded/internal/net/status/protob\x06proto3" + "\ahealthy\x18\t \x01(\bR\ahealthy\"\xe0\x04\n" + + "\x12NodeStatusOverview\x12*\n" + + "\x11timestamp_unix_ns\x18\x01 \x01(\x03R\x0ftimestampUnixNs\x12=\n" + + "\tnode_info\x18\x02 \x01(\v2 .unboundednet.status.v1.NodeInfoR\bnodeInfo\x12L\n" + + "\fhealth_check\x18\x03 \x01(\v2).unboundednet.status.v1.HealthCheckStatusR\vhealthCheck\x12B\n" + + "\vnode_errors\x18\x04 \x03(\v2!.unboundednet.status.v1.NodeErrorR\n" + + "nodeErrors\x12\x1f\n" + + "\vfetch_error\x18\x05 \x01(\tR\n" + + "fetchError\x122\n" + + "\x16last_push_time_unix_ns\x18\x06 \x01(\x03R\x12lastPushTimeUnixNs\x12#\n" + + "\rstatus_source\x18\a \x01(\tR\fstatusSource\x12G\n" + + "\rnode_pod_info\x18\b \x01(\v2#.unboundednet.status.v1.NodePodInfoR\vnodePodInfo\x12\x1d\n" + + "\n" + + "peer_count\x18\t \x01(\x05R\tpeerCount\x12#\n" + + "\rhealthy_peers\x18\n" + + " \x01(\x05R\fhealthyPeers\x12\x1f\n" + + "\vroute_count\x18\v \x01(\x05R\n" + + "routeCount\x12%\n" + + "\x0eroute_mismatch\x18\f \x01(\bR\rrouteMismatchB6Z4github.com/Azure/unbounded/internal/net/status/protob\x06proto3" var ( file_status_proto_rawDescOnce sync.Once @@ -2025,7 +2185,7 @@ func file_status_proto_rawDescGZIP() []byte { return file_status_proto_rawDescData } -var file_status_proto_msgTypes = make([]protoimpl.MessageInfo, 23) +var file_status_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_status_proto_goTypes = []any{ (*NodeStatusMessage)(nil), // 0: unboundednet.status.v1.NodeStatusMessage (*NodeStatusAck)(nil), // 1: unboundednet.status.v1.NodeStatusAck @@ -2048,44 +2208,50 @@ var file_status_proto_goTypes = []any{ (*NodeError)(nil), // 18: unboundednet.status.v1.NodeError (*NodePodInfo)(nil), // 19: unboundednet.status.v1.NodePodInfo (*BpfEntry)(nil), // 20: unboundednet.status.v1.BpfEntry - nil, // 21: unboundednet.status.v1.NodeInfo.K8sLabelsEntry - nil, // 22: unboundednet.status.v1.PeerStatus.RouteDistancesEntry + (*NodeStatusOverview)(nil), // 21: unboundednet.status.v1.NodeStatusOverview + nil, // 22: unboundednet.status.v1.NodeInfo.K8sLabelsEntry + nil, // 23: unboundednet.status.v1.PeerStatus.RouteDistancesEntry } var file_status_proto_depIdxs = []int32{ 2, // 0: unboundednet.status.v1.NodeStatusMessage.status:type_name -> unboundednet.status.v1.NodeStatusFull 3, // 1: unboundednet.status.v1.NodeStatusMessage.delta:type_name -> unboundednet.status.v1.NodeStatusDelta - 5, // 2: unboundednet.status.v1.NodeStatusFull.node_info:type_name -> unboundednet.status.v1.NodeInfo - 8, // 3: unboundednet.status.v1.NodeStatusFull.peers:type_name -> unboundednet.status.v1.PeerStatus - 10, // 4: unboundednet.status.v1.NodeStatusFull.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo - 16, // 5: unboundednet.status.v1.NodeStatusFull.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus - 18, // 6: unboundednet.status.v1.NodeStatusFull.node_errors:type_name -> unboundednet.status.v1.NodeError - 19, // 7: unboundednet.status.v1.NodeStatusFull.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo - 20, // 8: unboundednet.status.v1.NodeStatusFull.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry - 5, // 9: unboundednet.status.v1.NodeStatusDelta.node_info:type_name -> unboundednet.status.v1.NodeInfo - 8, // 10: unboundednet.status.v1.NodeStatusDelta.peers:type_name -> unboundednet.status.v1.PeerStatus - 10, // 11: unboundednet.status.v1.NodeStatusDelta.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo - 16, // 12: unboundednet.status.v1.NodeStatusDelta.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus - 18, // 13: unboundednet.status.v1.NodeStatusDelta.node_errors:type_name -> unboundednet.status.v1.NodeError - 20, // 14: unboundednet.status.v1.NodeStatusDelta.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry - 4, // 15: unboundednet.status.v1.NodeStatusDelta.peer_measurements:type_name -> unboundednet.status.v1.PeerMeasurements - 19, // 16: unboundednet.status.v1.NodeStatusDelta.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo - 6, // 17: unboundednet.status.v1.NodeInfo.build_info:type_name -> unboundednet.status.v1.BuildInfo - 7, // 18: unboundednet.status.v1.NodeInfo.wire_guard:type_name -> unboundednet.status.v1.WireGuardStatusInfo - 21, // 19: unboundednet.status.v1.NodeInfo.k8s_labels:type_name -> unboundednet.status.v1.NodeInfo.K8sLabelsEntry - 22, // 20: unboundednet.status.v1.PeerStatus.route_distances:type_name -> unboundednet.status.v1.PeerStatus.RouteDistancesEntry - 9, // 21: unboundednet.status.v1.PeerStatus.tunnel:type_name -> unboundednet.status.v1.PeerTunnelStatus - 17, // 22: unboundednet.status.v1.PeerStatus.health_check:type_name -> unboundednet.status.v1.HealthCheckPeerStatus - 11, // 23: unboundednet.status.v1.RoutingTableInfo.routes:type_name -> unboundednet.status.v1.RouteEntry - 12, // 24: unboundednet.status.v1.RouteEntry.next_hops:type_name -> unboundednet.status.v1.NextHop - 15, // 25: unboundednet.status.v1.NextHop.route_types:type_name -> unboundednet.status.v1.RouteType - 13, // 26: unboundednet.status.v1.NextHop.expected:type_name -> unboundednet.status.v1.OptionalBool - 13, // 27: unboundednet.status.v1.NextHop.present:type_name -> unboundednet.status.v1.OptionalBool - 14, // 28: unboundednet.status.v1.NextHop.info:type_name -> unboundednet.status.v1.NextHopInfo - 29, // [29:29] is the sub-list for method output_type - 29, // [29:29] is the sub-list for method input_type - 29, // [29:29] is the sub-list for extension type_name - 29, // [29:29] is the sub-list for extension extendee - 0, // [0:29] is the sub-list for field type_name + 21, // 2: unboundednet.status.v1.NodeStatusMessage.summary:type_name -> unboundednet.status.v1.NodeStatusOverview + 5, // 3: unboundednet.status.v1.NodeStatusFull.node_info:type_name -> unboundednet.status.v1.NodeInfo + 8, // 4: unboundednet.status.v1.NodeStatusFull.peers:type_name -> unboundednet.status.v1.PeerStatus + 10, // 5: unboundednet.status.v1.NodeStatusFull.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo + 16, // 6: unboundednet.status.v1.NodeStatusFull.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 7: unboundednet.status.v1.NodeStatusFull.node_errors:type_name -> unboundednet.status.v1.NodeError + 19, // 8: unboundednet.status.v1.NodeStatusFull.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 20, // 9: unboundednet.status.v1.NodeStatusFull.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry + 5, // 10: unboundednet.status.v1.NodeStatusDelta.node_info:type_name -> unboundednet.status.v1.NodeInfo + 8, // 11: unboundednet.status.v1.NodeStatusDelta.peers:type_name -> unboundednet.status.v1.PeerStatus + 10, // 12: unboundednet.status.v1.NodeStatusDelta.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo + 16, // 13: unboundednet.status.v1.NodeStatusDelta.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 14: unboundednet.status.v1.NodeStatusDelta.node_errors:type_name -> unboundednet.status.v1.NodeError + 20, // 15: unboundednet.status.v1.NodeStatusDelta.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry + 4, // 16: unboundednet.status.v1.NodeStatusDelta.peer_measurements:type_name -> unboundednet.status.v1.PeerMeasurements + 19, // 17: unboundednet.status.v1.NodeStatusDelta.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 6, // 18: unboundednet.status.v1.NodeInfo.build_info:type_name -> unboundednet.status.v1.BuildInfo + 7, // 19: unboundednet.status.v1.NodeInfo.wire_guard:type_name -> unboundednet.status.v1.WireGuardStatusInfo + 22, // 20: unboundednet.status.v1.NodeInfo.k8s_labels:type_name -> unboundednet.status.v1.NodeInfo.K8sLabelsEntry + 23, // 21: unboundednet.status.v1.PeerStatus.route_distances:type_name -> unboundednet.status.v1.PeerStatus.RouteDistancesEntry + 9, // 22: unboundednet.status.v1.PeerStatus.tunnel:type_name -> unboundednet.status.v1.PeerTunnelStatus + 17, // 23: unboundednet.status.v1.PeerStatus.health_check:type_name -> unboundednet.status.v1.HealthCheckPeerStatus + 11, // 24: unboundednet.status.v1.RoutingTableInfo.routes:type_name -> unboundednet.status.v1.RouteEntry + 12, // 25: unboundednet.status.v1.RouteEntry.next_hops:type_name -> unboundednet.status.v1.NextHop + 15, // 26: unboundednet.status.v1.NextHop.route_types:type_name -> unboundednet.status.v1.RouteType + 13, // 27: unboundednet.status.v1.NextHop.expected:type_name -> unboundednet.status.v1.OptionalBool + 13, // 28: unboundednet.status.v1.NextHop.present:type_name -> unboundednet.status.v1.OptionalBool + 14, // 29: unboundednet.status.v1.NextHop.info:type_name -> unboundednet.status.v1.NextHopInfo + 5, // 30: unboundednet.status.v1.NodeStatusOverview.node_info:type_name -> unboundednet.status.v1.NodeInfo + 16, // 31: unboundednet.status.v1.NodeStatusOverview.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 32: unboundednet.status.v1.NodeStatusOverview.node_errors:type_name -> unboundednet.status.v1.NodeError + 19, // 33: unboundednet.status.v1.NodeStatusOverview.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 34, // [34:34] is the sub-list for method output_type + 34, // [34:34] is the sub-list for method input_type + 34, // [34:34] is the sub-list for extension type_name + 34, // [34:34] is the sub-list for extension extendee + 0, // [0:34] is the sub-list for field type_name } func init() { file_status_proto_init() } @@ -2099,7 +2265,7 @@ func file_status_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_status_proto_rawDesc), len(file_status_proto_rawDesc)), NumEnums: 0, - NumMessages: 23, + NumMessages: 24, NumExtensions: 0, NumServices: 0, }, diff --git a/internal/net/status/proto/status.proto b/internal/net/status/proto/status.proto index 98903e2d8..e01ed59cc 100644 --- a/internal/net/status/proto/status.proto +++ b/internal/net/status/proto/status.proto @@ -7,11 +7,12 @@ option go_package = "github.com/Azure/unbounded/internal/net/status/proto"; // NodeStatusMessage wraps all node-to-controller status messages. message NodeStatusMessage { - string type = 1; // "node_status_full" or "node_status_delta" + string type = 1; // "node_status_full", "node_status_delta", or "node_status_summary" string node_name = 2; uint64 base_revision = 3; NodeStatusFull status = 4; // set for full updates NodeStatusDelta delta = 5; // set for delta updates + NodeStatusOverview summary = 6; // complete overview, including on resync } // NodeStatusAck is the acknowledgment returned by the controller for push updates. @@ -220,3 +221,19 @@ message BpfEntry { uint32 ifindex = 8; bool healthy = 9; } + +// NodeStatusOverview carries observed overview facts without detailed arrays. +message NodeStatusOverview { + int64 timestamp_unix_ns = 1; + NodeInfo node_info = 2; + HealthCheckStatus health_check = 3; + repeated NodeError node_errors = 4; + string fetch_error = 5; + int64 last_push_time_unix_ns = 6; // 0 means unset + string status_source = 7; + NodePodInfo node_pod_info = 8; + int32 peer_count = 9; + int32 healthy_peers = 10; + int32 route_count = 11; + bool route_mismatch = 12; +} diff --git a/internal/net/status/v1alpha1/types.go b/internal/net/status/v1alpha1/types.go index e2eb291d1..34a7766f4 100644 --- a/internal/net/status/v1alpha1/types.go +++ b/internal/net/status/v1alpha1/types.go @@ -5,6 +5,23 @@ package v1alpha1 import "time" +// NodeStatusOverview contains routine status without peer, route, or BPF details. +// Counts and mismatch state are observed facts, not inferred from missing details. +type NodeStatusOverview struct { + Timestamp time.Time `json:"timestamp"` + NodeInfo NodeInfo `json:"nodeInfo"` + HealthCheck *HealthCheckStatus `json:"healthCheck,omitempty"` + NodeErrors []NodeError `json:"nodeErrors,omitempty"` + FetchError string `json:"fetchError,omitempty"` + LastPushTime *time.Time `json:"lastPushTime,omitempty"` + StatusSource string `json:"statusSource,omitempty"` + NodePodInfo *NodePodInfo `json:"nodePodInfo,omitempty"` + PeerCount int `json:"peerCount"` + HealthyPeers int `json:"healthyPeers"` + RouteCount int `json:"routeCount"` + RouteMismatch bool `json:"routeMismatch"` +} + // NodeStatusResponse is the top-level status response for a node. type NodeStatusResponse struct { Timestamp time.Time `json:"timestamp"` From feaf0cdd531c2b63fd95c5efd0ef5ef851681203 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:15:05 +0000 Subject: [PATCH 05/34] feat(net): add correlated detail request and capability contracts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- internal/net/status/details.go | 81 +++++++++ internal/net/status/details_test.go | 151 ++++++++++++++++ internal/net/status/proto/status.pb.go | 211 +++++++++++++++++------ internal/net/status/proto/status.proto | 14 +- internal/net/status/v1alpha1/messages.go | 54 ++++++ 5 files changed, 457 insertions(+), 54 deletions(-) create mode 100644 internal/net/status/details.go create mode 100644 internal/net/status/details_test.go create mode 100644 internal/net/status/v1alpha1/messages.go diff --git a/internal/net/status/details.go b/internal/net/status/details.go new file mode 100644 index 000000000..4eeedb3bd --- /dev/null +++ b/internal/net/status/details.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "fmt" + "time" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// ValidateDetailRequest rejects missing identities and expired/unset deadlines. +func ValidateDetailRequest(request *statusv1alpha1.DetailRequest, now time.Time) error { + if request == nil || request.RequestID == "" { + return fmt.Errorf("detail request ID is required") + } + + if request.Deadline.IsZero() || !request.Deadline.After(now) { + return fmt.Errorf("detail request deadline must be in the future") + } + + return nil +} + +// DetailRequestToProto preserves nil requests and uses zero for unset deadlines. +func DetailRequestToProto(request *statusv1alpha1.DetailRequest) *statusproto.DetailRequest { + if request == nil { + return nil + } + + result := &statusproto.DetailRequest{RequestId: request.RequestID} + if !request.Deadline.IsZero() { + result.DeadlineUnixNs = request.Deadline.UnixNano() + } + + return result +} + +// DetailRequestFromProto preserves nil requests and unset deadlines. +func DetailRequestFromProto(request *statusproto.DetailRequest) *statusv1alpha1.DetailRequest { + if request == nil { + return nil + } + + result := &statusv1alpha1.DetailRequest{RequestID: request.RequestId} + if request.DeadlineUnixNs != 0 { + result.Deadline = time.Unix(0, request.DeadlineUnixNs).UTC() + } + + return result +} + +// NodeStatusAckToProto converts shared ACKs without conflating request IDs and revisions. +func NodeStatusAckToProto(ack *statusv1alpha1.NodeStatusAck) *statusproto.NodeStatusAck { + if ack == nil { + return nil + } + + return &statusproto.NodeStatusAck{ + Status: ack.Status, Revision: ack.Revision, Reason: ack.Reason, + PeerMeasurements: ack.PeerMeasurements, + DetailRequest: DetailRequestToProto(ack.DetailRequest), + SummarySupported: ack.SummarySupported, DetailRequestId: ack.DetailRequestID, + } +} + +// NodeStatusAckFromProto converts shared ACKs for either response transport. +func NodeStatusAckFromProto(ack *statusproto.NodeStatusAck) *statusv1alpha1.NodeStatusAck { + if ack == nil { + return nil + } + + return &statusv1alpha1.NodeStatusAck{ + Status: ack.Status, Revision: ack.Revision, Reason: ack.Reason, + PeerMeasurements: ack.PeerMeasurements, + DetailRequest: DetailRequestFromProto(ack.DetailRequest), + SummarySupported: ack.SummarySupported, DetailRequestID: ack.DetailRequestId, + } +} diff --git a/internal/net/status/details_test.go b/internal/net/status/details_test.go new file mode 100644 index 000000000..fc73a1f97 --- /dev/null +++ b/internal/net/status/details_test.go @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestDetailRequestValidation(t *testing.T) { + now := time.Unix(100, 0) + for _, tc := range []struct { + name string + request *statusv1alpha1.DetailRequest + valid bool + }{ + {"nil", nil, false}, + {"missing ID", &statusv1alpha1.DetailRequest{Deadline: now.Add(time.Second)}, false}, + {"missing deadline", &statusv1alpha1.DetailRequest{RequestID: "id"}, false}, + {"expired", &statusv1alpha1.DetailRequest{RequestID: "id", Deadline: now.Add(-time.Nanosecond)}, false}, + {"at deadline", &statusv1alpha1.DetailRequest{RequestID: "id", Deadline: now}, false}, + {"live", &statusv1alpha1.DetailRequest{RequestID: "id", Deadline: now.Add(time.Nanosecond)}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := ValidateDetailRequest(tc.request, now); (err == nil) != tc.valid { + t.Fatalf("ValidateDetailRequest() = %v, valid = %v", err, tc.valid) + } + }) + } +} + +func TestDetailACKRoundTrip(t *testing.T) { + request := &statusv1alpha1.DetailRequest{RequestID: "request", Deadline: time.Unix(100, 123).UTC()} + for _, tc := range []struct { + name string + ack *statusv1alpha1.NodeStatusAck + publication bool + }{ + {"nil", nil, false}, + {"legacy", &statusv1alpha1.NodeStatusAck{Status: "ok", Revision: 5}, true}, + {"resync", &statusv1alpha1.NodeStatusAck{Status: "resync_required", Reason: "base expired"}, true}, + {"command", &statusv1alpha1.NodeStatusAck{Status: statusv1alpha1.DetailRequestStatus, DetailRequest: request}, false}, + {"details", &statusv1alpha1.NodeStatusAck{Status: "ok", DetailRequestID: "request"}, false}, + {"unknown", &statusv1alpha1.NodeStatusAck{Status: "unknown"}, false}, + {"piggyback", &statusv1alpha1.NodeStatusAck{ + Status: "ok", Revision: 9, DetailRequest: request, SummarySupported: true, PeerMeasurements: true, + }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + pb := NodeStatusAckToProto(tc.ack) + if pb != nil { + data, err := proto.Marshal(pb) + if err != nil { + t.Fatal(err) + } + + pb = &statusproto.NodeStatusAck{} + if err := proto.Unmarshal(data, pb); err != nil { + t.Fatal(err) + } + } + + got := NodeStatusAckFromProto(pb) + if !reflect.DeepEqual(got, tc.ack) || got.IsPublicationAck() != tc.publication { + t.Fatalf("ACK changed or misclassified: %+v", got) + } + + data, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + + var jsonAck *statusv1alpha1.NodeStatusAck + if err := json.Unmarshal(data, &jsonAck); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(jsonAck, tc.ack) || jsonAck.IsPublicationAck() != tc.publication { + t.Fatalf("JSON ACK changed or misclassified: %s", data) + } + }) + } + + if got := DetailRequestFromProto(DetailRequestToProto(&statusv1alpha1.DetailRequest{RequestID: "unset"})); !got.Deadline.IsZero() { + t.Fatalf("unset deadline changed: %v", got.Deadline) + } +} + +func TestDetailWireFields(t *testing.T) { + for _, tc := range []struct { + message proto.Message + fields map[protoreflect.Name]protoreflect.FieldNumber + }{ + {&statusproto.NodeStatusMessage{}, map[protoreflect.Name]protoreflect.FieldNumber{ + "status": 4, "summary": 6, "detail_request_id": 7, "supports_details": 8, + }}, + {&statusproto.NodeStatusAck{}, map[protoreflect.Name]protoreflect.FieldNumber{ + "status": 1, "revision": 2, "reason": 3, "peer_measurements": 4, + "detail_request": 5, "summary_supported": 6, "detail_request_id": 7, + }}, + {&statusproto.DetailRequest{}, map[protoreflect.Name]protoreflect.FieldNumber{ + "request_id": 1, "deadline_unix_ns": 2, + }}, + } { + fields := tc.message.ProtoReflect().Descriptor().Fields() + for name, number := range tc.fields { + if field := fields.ByName(name); field == nil || field.Number() != number { + t.Errorf("%T field %q no longer has number %d", tc.message, name, number) + } + } + } + + legacy := &statusv1alpha1.NodeStatusAck{Status: "ok", Revision: 1} + + data, err := json.Marshal(legacy) + if err != nil { + t.Fatal(err) + } + + if string(data) != `{"status":"ok","revision":1}` { + t.Fatalf("legacy JSON shape changed: %s", data) + } + + message := &statusv1alpha1.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusDetailsType, NodeName: "node", DetailRequestID: "request", + SupportsDetails: true, Status: &statusv1alpha1.NodeStatusResponse{}, + } + + data, err = json.Marshal(message) + if err != nil { + t.Fatal(err) + } + + var got statusv1alpha1.NodeStatusMessage + if err := json.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(&got, message) || got.Summary != nil || got.BaseRevision != 0 { + t.Fatalf("detail JSON round trip changed payload: %s", data) + } +} diff --git a/internal/net/status/proto/status.pb.go b/internal/net/status/proto/status.pb.go index 5c7505f40..79eff0d73 100644 --- a/internal/net/status/proto/status.pb.go +++ b/internal/net/status/proto/status.pb.go @@ -26,15 +26,17 @@ const ( // NodeStatusMessage wraps all node-to-controller status messages. type NodeStatusMessage struct { - state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // "node_status_full", "node_status_delta", or "node_status_summary" - NodeName string `protobuf:"bytes,2,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` - BaseRevision uint64 `protobuf:"varint,3,opt,name=base_revision,json=baseRevision,proto3" json:"base_revision,omitempty"` - Status *NodeStatusFull `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` // set for full updates - Delta *NodeStatusDelta `protobuf:"bytes,5,opt,name=delta,proto3" json:"delta,omitempty"` // set for delta updates - Summary *NodeStatusOverview `protobuf:"bytes,6,opt,name=summary,proto3" json:"summary,omitempty"` // complete overview, including on resync - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // "node_status_full", "node_status_delta", "node_status_summary", or "node_status_details" + NodeName string `protobuf:"bytes,2,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + BaseRevision uint64 `protobuf:"varint,3,opt,name=base_revision,json=baseRevision,proto3" json:"base_revision,omitempty"` + Status *NodeStatusFull `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` // set for full updates + Delta *NodeStatusDelta `protobuf:"bytes,5,opt,name=delta,proto3" json:"delta,omitempty"` // set for delta updates + Summary *NodeStatusOverview `protobuf:"bytes,6,opt,name=summary,proto3" json:"summary,omitempty"` // complete overview, including on resync + DetailRequestId string `protobuf:"bytes,7,opt,name=detail_request_id,json=detailRequestId,proto3" json:"detail_request_id,omitempty"` // correlates one-shot details in status, never a revision + SupportsDetails bool `protobuf:"varint,8,opt,name=supports_details,json=supportsDetails,proto3" json:"supports_details,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeStatusMessage) Reset() { @@ -109,6 +111,20 @@ func (x *NodeStatusMessage) GetSummary() *NodeStatusOverview { return nil } +func (x *NodeStatusMessage) GetDetailRequestId() string { + if x != nil { + return x.DetailRequestId + } + return "" +} + +func (x *NodeStatusMessage) GetSupportsDetails() bool { + if x != nil { + return x.SupportsDetails + } + return false +} + // NodeStatusAck is the acknowledgment returned by the controller for push updates. type NodeStatusAck struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -116,6 +132,9 @@ type NodeStatusAck struct { Revision uint64 `protobuf:"varint,2,opt,name=revision,proto3" json:"revision,omitempty"` Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` PeerMeasurements bool `protobuf:"varint,4,opt,name=peer_measurements,json=peerMeasurements,proto3" json:"peer_measurements,omitempty"` // Positive capability, scoped to this WebSocket connection. + DetailRequest *DetailRequest `protobuf:"bytes,5,opt,name=detail_request,json=detailRequest,proto3" json:"detail_request,omitempty"` // unsolicited WS command or piggybacked HTTP response + SummarySupported bool `protobuf:"varint,6,opt,name=summary_supported,json=summarySupported,proto3" json:"summary_supported,omitempty"` + DetailRequestId string `protobuf:"bytes,7,opt,name=detail_request_id,json=detailRequestId,proto3" json:"detail_request_id,omitempty"` // a detail ACK must not acknowledge a routine publication unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -178,6 +197,27 @@ func (x *NodeStatusAck) GetPeerMeasurements() bool { return false } +func (x *NodeStatusAck) GetDetailRequest() *DetailRequest { + if x != nil { + return x.DetailRequest + } + return nil +} + +func (x *NodeStatusAck) GetSummarySupported() bool { + if x != nil { + return x.SummarySupported + } + return false +} + +func (x *NodeStatusAck) GetDetailRequestId() string { + if x != nil { + return x.DetailRequestId + } + return "" +} + // NodeStatusFull mirrors the complete NodeStatusResponse payload. type NodeStatusFull struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1963,23 +2003,82 @@ func (x *NodeStatusOverview) GetRouteMismatch() bool { return false } +// DetailRequest uses the same deadline across all delivery attempts. +// Standalone commands use ACK status "detail_request", not "ok". +type DetailRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + DeadlineUnixNs int64 `protobuf:"varint,2,opt,name=deadline_unix_ns,json=deadlineUnixNs,proto3" json:"deadline_unix_ns,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DetailRequest) Reset() { + *x = DetailRequest{} + mi := &file_status_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DetailRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DetailRequest) ProtoMessage() {} + +func (x *DetailRequest) ProtoReflect() protoreflect.Message { + mi := &file_status_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DetailRequest.ProtoReflect.Descriptor instead. +func (*DetailRequest) Descriptor() ([]byte, []int) { + return file_status_proto_rawDescGZIP(), []int{22} +} + +func (x *DetailRequest) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +func (x *DetailRequest) GetDeadlineUnixNs() int64 { + if x != nil { + return x.DeadlineUnixNs + } + return 0 +} + var File_status_proto protoreflect.FileDescriptor const file_status_proto_rawDesc = "" + "\n" + - "\fstatus.proto\x12\x16unboundednet.status.v1\"\xae\x02\n" + + "\fstatus.proto\x12\x16unboundednet.status.v1\"\x85\x03\n" + "\x11NodeStatusMessage\x12\x12\n" + "\x04type\x18\x01 \x01(\tR\x04type\x12\x1b\n" + "\tnode_name\x18\x02 \x01(\tR\bnodeName\x12#\n" + "\rbase_revision\x18\x03 \x01(\x04R\fbaseRevision\x12>\n" + "\x06status\x18\x04 \x01(\v2&.unboundednet.status.v1.NodeStatusFullR\x06status\x12=\n" + "\x05delta\x18\x05 \x01(\v2'.unboundednet.status.v1.NodeStatusDeltaR\x05delta\x12D\n" + - "\asummary\x18\x06 \x01(\v2*.unboundednet.status.v1.NodeStatusOverviewR\asummary\"\x88\x01\n" + + "\asummary\x18\x06 \x01(\v2*.unboundednet.status.v1.NodeStatusOverviewR\asummary\x12*\n" + + "\x11detail_request_id\x18\a \x01(\tR\x0fdetailRequestId\x12)\n" + + "\x10supports_details\x18\b \x01(\bR\x0fsupportsDetails\"\xaf\x02\n" + "\rNodeStatusAck\x12\x16\n" + "\x06status\x18\x01 \x01(\tR\x06status\x12\x1a\n" + "\brevision\x18\x02 \x01(\x04R\brevision\x12\x16\n" + "\x06reason\x18\x03 \x01(\tR\x06reason\x12+\n" + - "\x11peer_measurements\x18\x04 \x01(\bR\x10peerMeasurements\"\x9c\x05\n" + + "\x11peer_measurements\x18\x04 \x01(\bR\x10peerMeasurements\x12L\n" + + "\x0edetail_request\x18\x05 \x01(\v2%.unboundednet.status.v1.DetailRequestR\rdetailRequest\x12+\n" + + "\x11summary_supported\x18\x06 \x01(\bR\x10summarySupported\x12*\n" + + "\x11detail_request_id\x18\a \x01(\tR\x0fdetailRequestId\"\x9c\x05\n" + "\x0eNodeStatusFull\x12*\n" + "\x11timestamp_unix_ns\x18\x01 \x01(\x03R\x0ftimestampUnixNs\x12=\n" + "\tnode_info\x18\x02 \x01(\v2 .unboundednet.status.v1.NodeInfoR\bnodeInfo\x128\n" + @@ -2171,7 +2270,11 @@ const file_status_proto_rawDesc = "" + " \x01(\x05R\fhealthyPeers\x12\x1f\n" + "\vroute_count\x18\v \x01(\x05R\n" + "routeCount\x12%\n" + - "\x0eroute_mismatch\x18\f \x01(\bR\rrouteMismatchB6Z4github.com/Azure/unbounded/internal/net/status/protob\x06proto3" + "\x0eroute_mismatch\x18\f \x01(\bR\rrouteMismatch\"X\n" + + "\rDetailRequest\x12\x1d\n" + + "\n" + + "request_id\x18\x01 \x01(\tR\trequestId\x12(\n" + + "\x10deadline_unix_ns\x18\x02 \x01(\x03R\x0edeadlineUnixNsB6Z4github.com/Azure/unbounded/internal/net/status/protob\x06proto3" var ( file_status_proto_rawDescOnce sync.Once @@ -2185,7 +2288,7 @@ func file_status_proto_rawDescGZIP() []byte { return file_status_proto_rawDescData } -var file_status_proto_msgTypes = make([]protoimpl.MessageInfo, 24) +var file_status_proto_msgTypes = make([]protoimpl.MessageInfo, 25) var file_status_proto_goTypes = []any{ (*NodeStatusMessage)(nil), // 0: unboundednet.status.v1.NodeStatusMessage (*NodeStatusAck)(nil), // 1: unboundednet.status.v1.NodeStatusAck @@ -2209,49 +2312,51 @@ var file_status_proto_goTypes = []any{ (*NodePodInfo)(nil), // 19: unboundednet.status.v1.NodePodInfo (*BpfEntry)(nil), // 20: unboundednet.status.v1.BpfEntry (*NodeStatusOverview)(nil), // 21: unboundednet.status.v1.NodeStatusOverview - nil, // 22: unboundednet.status.v1.NodeInfo.K8sLabelsEntry - nil, // 23: unboundednet.status.v1.PeerStatus.RouteDistancesEntry + (*DetailRequest)(nil), // 22: unboundednet.status.v1.DetailRequest + nil, // 23: unboundednet.status.v1.NodeInfo.K8sLabelsEntry + nil, // 24: unboundednet.status.v1.PeerStatus.RouteDistancesEntry } var file_status_proto_depIdxs = []int32{ 2, // 0: unboundednet.status.v1.NodeStatusMessage.status:type_name -> unboundednet.status.v1.NodeStatusFull 3, // 1: unboundednet.status.v1.NodeStatusMessage.delta:type_name -> unboundednet.status.v1.NodeStatusDelta 21, // 2: unboundednet.status.v1.NodeStatusMessage.summary:type_name -> unboundednet.status.v1.NodeStatusOverview - 5, // 3: unboundednet.status.v1.NodeStatusFull.node_info:type_name -> unboundednet.status.v1.NodeInfo - 8, // 4: unboundednet.status.v1.NodeStatusFull.peers:type_name -> unboundednet.status.v1.PeerStatus - 10, // 5: unboundednet.status.v1.NodeStatusFull.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo - 16, // 6: unboundednet.status.v1.NodeStatusFull.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus - 18, // 7: unboundednet.status.v1.NodeStatusFull.node_errors:type_name -> unboundednet.status.v1.NodeError - 19, // 8: unboundednet.status.v1.NodeStatusFull.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo - 20, // 9: unboundednet.status.v1.NodeStatusFull.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry - 5, // 10: unboundednet.status.v1.NodeStatusDelta.node_info:type_name -> unboundednet.status.v1.NodeInfo - 8, // 11: unboundednet.status.v1.NodeStatusDelta.peers:type_name -> unboundednet.status.v1.PeerStatus - 10, // 12: unboundednet.status.v1.NodeStatusDelta.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo - 16, // 13: unboundednet.status.v1.NodeStatusDelta.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus - 18, // 14: unboundednet.status.v1.NodeStatusDelta.node_errors:type_name -> unboundednet.status.v1.NodeError - 20, // 15: unboundednet.status.v1.NodeStatusDelta.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry - 4, // 16: unboundednet.status.v1.NodeStatusDelta.peer_measurements:type_name -> unboundednet.status.v1.PeerMeasurements - 19, // 17: unboundednet.status.v1.NodeStatusDelta.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo - 6, // 18: unboundednet.status.v1.NodeInfo.build_info:type_name -> unboundednet.status.v1.BuildInfo - 7, // 19: unboundednet.status.v1.NodeInfo.wire_guard:type_name -> unboundednet.status.v1.WireGuardStatusInfo - 22, // 20: unboundednet.status.v1.NodeInfo.k8s_labels:type_name -> unboundednet.status.v1.NodeInfo.K8sLabelsEntry - 23, // 21: unboundednet.status.v1.PeerStatus.route_distances:type_name -> unboundednet.status.v1.PeerStatus.RouteDistancesEntry - 9, // 22: unboundednet.status.v1.PeerStatus.tunnel:type_name -> unboundednet.status.v1.PeerTunnelStatus - 17, // 23: unboundednet.status.v1.PeerStatus.health_check:type_name -> unboundednet.status.v1.HealthCheckPeerStatus - 11, // 24: unboundednet.status.v1.RoutingTableInfo.routes:type_name -> unboundednet.status.v1.RouteEntry - 12, // 25: unboundednet.status.v1.RouteEntry.next_hops:type_name -> unboundednet.status.v1.NextHop - 15, // 26: unboundednet.status.v1.NextHop.route_types:type_name -> unboundednet.status.v1.RouteType - 13, // 27: unboundednet.status.v1.NextHop.expected:type_name -> unboundednet.status.v1.OptionalBool - 13, // 28: unboundednet.status.v1.NextHop.present:type_name -> unboundednet.status.v1.OptionalBool - 14, // 29: unboundednet.status.v1.NextHop.info:type_name -> unboundednet.status.v1.NextHopInfo - 5, // 30: unboundednet.status.v1.NodeStatusOverview.node_info:type_name -> unboundednet.status.v1.NodeInfo - 16, // 31: unboundednet.status.v1.NodeStatusOverview.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus - 18, // 32: unboundednet.status.v1.NodeStatusOverview.node_errors:type_name -> unboundednet.status.v1.NodeError - 19, // 33: unboundednet.status.v1.NodeStatusOverview.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo - 34, // [34:34] is the sub-list for method output_type - 34, // [34:34] is the sub-list for method input_type - 34, // [34:34] is the sub-list for extension type_name - 34, // [34:34] is the sub-list for extension extendee - 0, // [0:34] is the sub-list for field type_name + 22, // 3: unboundednet.status.v1.NodeStatusAck.detail_request:type_name -> unboundednet.status.v1.DetailRequest + 5, // 4: unboundednet.status.v1.NodeStatusFull.node_info:type_name -> unboundednet.status.v1.NodeInfo + 8, // 5: unboundednet.status.v1.NodeStatusFull.peers:type_name -> unboundednet.status.v1.PeerStatus + 10, // 6: unboundednet.status.v1.NodeStatusFull.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo + 16, // 7: unboundednet.status.v1.NodeStatusFull.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 8: unboundednet.status.v1.NodeStatusFull.node_errors:type_name -> unboundednet.status.v1.NodeError + 19, // 9: unboundednet.status.v1.NodeStatusFull.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 20, // 10: unboundednet.status.v1.NodeStatusFull.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry + 5, // 11: unboundednet.status.v1.NodeStatusDelta.node_info:type_name -> unboundednet.status.v1.NodeInfo + 8, // 12: unboundednet.status.v1.NodeStatusDelta.peers:type_name -> unboundednet.status.v1.PeerStatus + 10, // 13: unboundednet.status.v1.NodeStatusDelta.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo + 16, // 14: unboundednet.status.v1.NodeStatusDelta.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 15: unboundednet.status.v1.NodeStatusDelta.node_errors:type_name -> unboundednet.status.v1.NodeError + 20, // 16: unboundednet.status.v1.NodeStatusDelta.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry + 4, // 17: unboundednet.status.v1.NodeStatusDelta.peer_measurements:type_name -> unboundednet.status.v1.PeerMeasurements + 19, // 18: unboundednet.status.v1.NodeStatusDelta.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 6, // 19: unboundednet.status.v1.NodeInfo.build_info:type_name -> unboundednet.status.v1.BuildInfo + 7, // 20: unboundednet.status.v1.NodeInfo.wire_guard:type_name -> unboundednet.status.v1.WireGuardStatusInfo + 23, // 21: unboundednet.status.v1.NodeInfo.k8s_labels:type_name -> unboundednet.status.v1.NodeInfo.K8sLabelsEntry + 24, // 22: unboundednet.status.v1.PeerStatus.route_distances:type_name -> unboundednet.status.v1.PeerStatus.RouteDistancesEntry + 9, // 23: unboundednet.status.v1.PeerStatus.tunnel:type_name -> unboundednet.status.v1.PeerTunnelStatus + 17, // 24: unboundednet.status.v1.PeerStatus.health_check:type_name -> unboundednet.status.v1.HealthCheckPeerStatus + 11, // 25: unboundednet.status.v1.RoutingTableInfo.routes:type_name -> unboundednet.status.v1.RouteEntry + 12, // 26: unboundednet.status.v1.RouteEntry.next_hops:type_name -> unboundednet.status.v1.NextHop + 15, // 27: unboundednet.status.v1.NextHop.route_types:type_name -> unboundednet.status.v1.RouteType + 13, // 28: unboundednet.status.v1.NextHop.expected:type_name -> unboundednet.status.v1.OptionalBool + 13, // 29: unboundednet.status.v1.NextHop.present:type_name -> unboundednet.status.v1.OptionalBool + 14, // 30: unboundednet.status.v1.NextHop.info:type_name -> unboundednet.status.v1.NextHopInfo + 5, // 31: unboundednet.status.v1.NodeStatusOverview.node_info:type_name -> unboundednet.status.v1.NodeInfo + 16, // 32: unboundednet.status.v1.NodeStatusOverview.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 33: unboundednet.status.v1.NodeStatusOverview.node_errors:type_name -> unboundednet.status.v1.NodeError + 19, // 34: unboundednet.status.v1.NodeStatusOverview.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 35, // [35:35] is the sub-list for method output_type + 35, // [35:35] is the sub-list for method input_type + 35, // [35:35] is the sub-list for extension type_name + 35, // [35:35] is the sub-list for extension extendee + 0, // [0:35] is the sub-list for field type_name } func init() { file_status_proto_init() } @@ -2265,7 +2370,7 @@ func file_status_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_status_proto_rawDesc), len(file_status_proto_rawDesc)), NumEnums: 0, - NumMessages: 24, + NumMessages: 25, NumExtensions: 0, NumServices: 0, }, diff --git a/internal/net/status/proto/status.proto b/internal/net/status/proto/status.proto index e01ed59cc..867fafa46 100644 --- a/internal/net/status/proto/status.proto +++ b/internal/net/status/proto/status.proto @@ -7,12 +7,14 @@ option go_package = "github.com/Azure/unbounded/internal/net/status/proto"; // NodeStatusMessage wraps all node-to-controller status messages. message NodeStatusMessage { - string type = 1; // "node_status_full", "node_status_delta", or "node_status_summary" + string type = 1; // "node_status_full", "node_status_delta", "node_status_summary", or "node_status_details" string node_name = 2; uint64 base_revision = 3; NodeStatusFull status = 4; // set for full updates NodeStatusDelta delta = 5; // set for delta updates NodeStatusOverview summary = 6; // complete overview, including on resync + string detail_request_id = 7; // correlates one-shot details in status, never a revision + bool supports_details = 8; } // NodeStatusAck is the acknowledgment returned by the controller for push updates. @@ -21,6 +23,9 @@ message NodeStatusAck { uint64 revision = 2; string reason = 3; bool peer_measurements = 4; // Positive capability, scoped to this WebSocket connection. + DetailRequest detail_request = 5; // unsolicited WS command or piggybacked HTTP response + bool summary_supported = 6; + string detail_request_id = 7; // a detail ACK must not acknowledge a routine publication } // NodeStatusFull mirrors the complete NodeStatusResponse payload. @@ -237,3 +242,10 @@ message NodeStatusOverview { int32 route_count = 11; bool route_mismatch = 12; } + +// DetailRequest uses the same deadline across all delivery attempts. +// Standalone commands use ACK status "detail_request", not "ok". +message DetailRequest { + string request_id = 1; + int64 deadline_unix_ns = 2; +} diff --git a/internal/net/status/v1alpha1/messages.go b/internal/net/status/v1alpha1/messages.go new file mode 100644 index 000000000..3ec66ced8 --- /dev/null +++ b/internal/net/status/v1alpha1/messages.go @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "encoding/json" + "time" +) + +const ( + NodeStatusSummaryType = "node_status_summary" + NodeStatusDetailsType = "node_status_details" + DetailRequestStatus = "detail_request" +) + +// DetailRequest is a node-bound diagnostic command, independent of revisions. +type DetailRequest struct { + RequestID string `json:"requestId"` + Deadline time.Time `json:"deadline"` +} + +// NodeStatusMessage is the JSON equivalent of the protobuf status envelope. +// Status holds legacy full publications or one-shot node_status_details replies. +type NodeStatusMessage struct { + Type string `json:"type"` + NodeName string `json:"nodeName,omitempty"` + BaseRevision uint64 `json:"baseRevision,omitempty"` + Status *NodeStatusResponse `json:"status,omitempty"` + Delta map[string]json.RawMessage `json:"delta,omitempty"` + Summary *NodeStatusOverview `json:"summary,omitempty"` + DetailRequestID string `json:"detailRequestId,omitempty"` + SupportsDetails bool `json:"supportsDetails,omitempty"` +} + +// NodeStatusAck is shared by HTTP responses and WebSocket ACK/command data. +// An unsolicited command uses DetailRequestStatus and leaves Revision unset. +// An HTTP publication ACK may also carry a pending DetailRequest. +type NodeStatusAck struct { + Status string `json:"status"` + Revision uint64 `json:"revision,omitempty"` + Reason string `json:"reason,omitempty"` + PeerMeasurements bool `json:"peerMeasurements,omitempty"` + DetailRequest *DetailRequest `json:"detailRequest,omitempty"` + SummarySupported bool `json:"summarySupported,omitempty"` + DetailRequestID string `json:"detailRequestId,omitempty"` +} + +// IsPublicationAck excludes standalone commands and one-shot detail ACKs. +// Only publication ACKs may clear a routine publisher's pending ACK state. +func (a *NodeStatusAck) IsPublicationAck() bool { + return a != nil && a.DetailRequestID == "" && + (a.Status == "ok" || a.Status == "resync_required") +} From 71647ce266a4d24f55409b67bd6c2d18384f6afb Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:19:23 +0000 Subject: [PATCH 06/34] feat(net): prepare startup configuration for lightweight status Preserve full publication by default until the later runtime activation layer. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-controller/main.go | 24 +++++ .../main_config_test.go | 2 + .../status_detail_config_test.go | 63 +++++++++++ cmd/unbounded-net-node/main.go | 11 ++ cmd/unbounded-net-node/main_config_test.go | 3 + .../status_detail_config_test.go | 53 +++++++++ deploy/net/01-configmap.yaml.tmpl | 5 + .../reference/networking/configuration.md | 17 +++ docs/net/configuration.md | 17 +++ internal/net/config/config.go | 12 +++ internal/net/config/config_test.go | 6 +- internal/net/config/runtime_config.go | 37 +++++++ internal/net/config/status_detail_test.go | 101 ++++++++++++++++++ 13 files changed, 350 insertions(+), 1 deletion(-) create mode 100644 cmd/unbounded-net-controller/status_detail_config_test.go create mode 100644 cmd/unbounded-net-node/status_detail_config_test.go create mode 100644 internal/net/config/status_detail_test.go diff --git a/cmd/unbounded-net-controller/main.go b/cmd/unbounded-net-controller/main.go index 67b04c973..b604dfbea 100644 --- a/cmd/unbounded-net-controller/main.go +++ b/cmd/unbounded-net-controller/main.go @@ -79,6 +79,8 @@ func main() { RequireDashboardAuth: true, StatusWSKeepaliveInterval: 10 * time.Second, StatusWSKeepaliveFailureCount: 2, + StatusDetailCacheTTL: config.DefaultStatusDetailCacheTTL, + StatusDetailRequestTimeout: config.DefaultStatusDetailRequestTimeout, ManagedKubeProxyEnabled: true, NodeTokenLifetime: 4 * time.Hour, ViewerTokenLifetime: 30 * time.Minute, @@ -127,6 +129,8 @@ on site configuration, and maintain SiteNodeSlice and GatewayPool status.`, flags.IntVar(&cfg.HealthPort, "health-port", 9999, "Port for health check HTTP server (0 to disable)") flags.IntVar(&cfg.NodeAgentHealthPort, "node-agent-health-port", 9998, "Port where node agents serve their health/status endpoints") flags.DurationVar(&cfg.StatusStaleThreshold, "status-stale-threshold", 90*time.Second, "Duration after which a node's pushed status is considered stale") + flags.DurationVar(&cfg.StatusDetailCacheTTL, "status-detail-cache-ttl", config.DefaultStatusDetailCacheTTL, "Lifetime of received node details (positive duration; preparatory)") + flags.DurationVar(&cfg.StatusDetailRequestTimeout, "status-detail-request-timeout", config.DefaultStatusDetailRequestTimeout, "End-to-end node detail request timeout (positive duration; preparatory)") flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "Interval between websocket keepalive pings on controller node status streams (0 to disable)") flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "Sequential websocket keepalive ping failures before closing node status websocket") flags.BoolVar(&cfg.RegisterAggregatedAPIServer, "register-aggregated-apiserver", true, "Serve node status push endpoints via aggregated API server paths") @@ -164,6 +168,24 @@ func applyControllerRuntimeConfig(cmd *cobra.Command, cfg *config.Config, config flags := cmd.Flags() + if !flags.Changed("status-detail-cache-ttl") && runtimeCfg.Controller.StatusDetailCacheTTL != "" { + d, parseErr := config.ParsePositiveDurationField(runtimeCfg.Controller.StatusDetailCacheTTL, "controller.statusDetailCacheTTL") + if parseErr != nil { + return parseErr + } + + cfg.StatusDetailCacheTTL = d + } + + if !flags.Changed("status-detail-request-timeout") && runtimeCfg.Controller.StatusDetailRequestTimeout != "" { + d, parseErr := config.ParsePositiveDurationField(runtimeCfg.Controller.StatusDetailRequestTimeout, "controller.statusDetailRequestTimeout") + if parseErr != nil { + return parseErr + } + + cfg.StatusDetailRequestTimeout = d + } + if !flags.Changed("informer-resync-period") { if d, parseErr := config.ParseDurationField(runtimeCfg.Controller.InformerResyncPeriod, "controller.informerResyncPeriod"); parseErr != nil { return parseErr @@ -321,6 +343,8 @@ General Flags: --managed-kube-proxy Create kube-proxy DaemonSets for unbounded-managed site nodes not covered by provider kube-proxy (default true) --managed-kube-proxy-image string kube-proxy image for managed site DaemonSets --status-stale-threshold duration Duration after which a node's pushed status is considered stale (default 90s) + --status-detail-cache-ttl duration Lifetime of received node details; preparatory (default 5m0s) + --status-detail-request-timeout duration End-to-end node detail request timeout; preparatory (default 2m0s) --status-ws-keepalive-interval duration Interval between websocket keepalive pings on controller node status streams (0 to disable) (default 10s) --status-ws-keepalive-failure-count int Sequential websocket keepalive ping failures before closing node status websocket (default 2) diff --git a/cmd/unbounded-net-controller/main_config_test.go b/cmd/unbounded-net-controller/main_config_test.go index cb184733a..3359cc03e 100644 --- a/cmd/unbounded-net-controller/main_config_test.go +++ b/cmd/unbounded-net-controller/main_config_test.go @@ -21,6 +21,8 @@ func newControllerConfigTestCommand(cfg *config.Config) *cobra.Command { flags.IntVar(&cfg.HealthPort, "health-port", 9999, "") flags.IntVar(&cfg.NodeAgentHealthPort, "node-agent-health-port", 9998, "") flags.DurationVar(&cfg.StatusStaleThreshold, "status-stale-threshold", 40*time.Second, "") + flags.DurationVar(&cfg.StatusDetailCacheTTL, "status-detail-cache-ttl", config.DefaultStatusDetailCacheTTL, "") + flags.DurationVar(&cfg.StatusDetailRequestTimeout, "status-detail-request-timeout", config.DefaultStatusDetailRequestTimeout, "") flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "") flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "") flags.BoolVar(&cfg.RegisterAggregatedAPIServer, "register-aggregated-apiserver", true, "") diff --git a/cmd/unbounded-net-controller/status_detail_config_test.go b/cmd/unbounded-net-controller/status_detail_config_test.go new file mode 100644 index 000000000..ce5c6d447 --- /dev/null +++ b/cmd/unbounded-net-controller/status_detail_config_test.go @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/Azure/unbounded/internal/net/config" +) + +func TestControllerStatusDetailConfig(t *testing.T) { + for _, tc := range []struct { + name, yaml, flagName, flagValue string + ttl, timeout time.Duration + invalid bool + }{ + {name: "defaults", yaml: "controller: {}", ttl: 300 * time.Second, timeout: 120 * time.Second}, + {name: "configured", yaml: "controller:\n statusDetailCacheTTL: 30s\n statusDetailRequestTimeout: 10s", ttl: 30 * time.Second, timeout: 10 * time.Second}, + {name: "TTL zero", yaml: "controller:\n statusDetailCacheTTL: 0s", invalid: true}, + {name: "TTL negative", yaml: "controller:\n statusDetailCacheTTL: -1s", invalid: true}, + {name: "TTL malformed", yaml: "controller:\n statusDetailCacheTTL: invalid", invalid: true}, + {name: "timeout zero", yaml: "controller:\n statusDetailRequestTimeout: 0s", invalid: true}, + {name: "timeout negative", yaml: "controller:\n statusDetailRequestTimeout: -1s", invalid: true}, + {name: "timeout malformed", yaml: "controller:\n statusDetailRequestTimeout: invalid", invalid: true}, + {name: "TTL flag wins", yaml: "controller:\n statusDetailCacheTTL: invalid", flagName: "status-detail-cache-ttl", flagValue: "15s", ttl: 15 * time.Second, timeout: 120 * time.Second}, + {name: "timeout flag wins", yaml: "controller:\n statusDetailRequestTimeout: invalid", flagName: "status-detail-request-timeout", flagValue: "15s", ttl: 300 * time.Second, timeout: 15 * time.Second}, + {name: "zero flag", yaml: "controller: {}", flagName: "status-detail-cache-ttl", flagValue: "0s", invalid: true}, + {name: "negative flag", yaml: "controller: {}", flagName: "status-detail-request-timeout", flagValue: "-1s", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0o600); err != nil { + t.Fatal(err) + } + + cfg := &config.Config{} + + cmd := newControllerConfigTestCommand(cfg) + if tc.flagName != "" { + if err := cmd.Flags().Set(tc.flagName, tc.flagValue); err != nil { + t.Fatal(err) + } + } + + err := applyControllerRuntimeConfig(cmd, cfg, path) + if err == nil { + err = cfg.Validate() + } + + if (err != nil) != tc.invalid { + t.Fatalf("startup config validation = %v", err) + } + + if !tc.invalid && (cfg.StatusDetailCacheTTL != tc.ttl || cfg.StatusDetailRequestTimeout != tc.timeout) { + t.Errorf("lifetimes = %s/%s, want %s/%s", cfg.StatusDetailCacheTTL, cfg.StatusDetailRequestTimeout, tc.ttl, tc.timeout) + } + }) + } +} diff --git a/cmd/unbounded-net-node/main.go b/cmd/unbounded-net-node/main.go index 58d166093..a52c39ce7 100644 --- a/cmd/unbounded-net-node/main.go +++ b/cmd/unbounded-net-node/main.go @@ -93,6 +93,7 @@ type config struct { StatusPushInterval time.Duration // Interval between status pushes to controller StatusPushAPIServerInterval time.Duration // Interval between status pushes via aggregated API server StatusPushDelta bool // Whether periodic HTTP pushes use deltas + StatusDetailMode string // Startup-loaded; publication wiring follows separately. StatusWSEnabled bool // Whether websocket push is enabled StatusWSURL string // Controller websocket URL for status push StatusWSAPIServerMode string // API server fallback mode: never, fallback, preferred (alias for fallback) @@ -230,6 +231,7 @@ func main() { StatusPushInterval: 10 * time.Second, // Default 10s push interval StatusPushAPIServerInterval: 30 * time.Second, StatusPushDelta: true, + StatusDetailMode: configpkg.DefaultStatusDetailMode, StatusWSEnabled: true, StatusWSAPIServerMode: statusWSAPIServerModeFallback, StatusWSAPIServerStartupDelay: 60 * time.Second, @@ -328,6 +330,7 @@ then annotates the node with the public key.`, flags.DurationVar(&cfg.StatusPushInterval, "status-push-interval", 60*time.Second, "Interval between status pushes to controller") flags.DurationVar(&cfg.StatusPushAPIServerInterval, "status-push-apiserver-interval", 60*time.Second, "Interval between status pushes via aggregated API server") flags.BoolVar(&cfg.StatusPushDelta, "status-push-delta", true, "Enable delta mode for periodic HTTP status push") + flags.StringVar(&cfg.StatusDetailMode, "status-detail-mode", configpkg.DefaultStatusDetailMode, "Routine status detail mode: summary or full (preparatory; publication behavior unchanged)") flags.BoolVar(&cfg.StatusWSEnabled, "status-ws-enabled", true, "Enable websocket status push to controller") flags.StringVar(&cfg.StatusWSURL, "status-ws-url", "", "Controller websocket URL for status push (default: ws://service/status/nodews)") flags.StringVar(&cfg.StatusWSAPIServerMode, "status-ws-apiserver-mode", statusWSAPIServerModeFallback, "API server fallback mode: never, fallback, preferred (alias for fallback); direct controller endpoints are tried first") @@ -365,6 +368,14 @@ func applyNodeRuntimeConfig(cmd *cobra.Command, cfg *config) error { flags := cmd.Flags() nodeCfg := runtimeCfg.Node + if !flags.Changed("status-detail-mode") && nodeCfg.StatusDetailMode != "" { + cfg.StatusDetailMode = nodeCfg.StatusDetailMode + } + + if err := configpkg.ValidateStatusDetailMode(cfg.StatusDetailMode); err != nil { + return err + } + if !flags.Changed("informer-resync-period") { if d, parseErr := configpkg.ParseDurationField(nodeCfg.InformerResyncPeriod, "node.informerResyncPeriod"); parseErr != nil { return parseErr diff --git a/cmd/unbounded-net-node/main_config_test.go b/cmd/unbounded-net-node/main_config_test.go index 97b3835fc..1a87b877e 100644 --- a/cmd/unbounded-net-node/main_config_test.go +++ b/cmd/unbounded-net-node/main_config_test.go @@ -11,6 +11,8 @@ import ( "time" "github.com/spf13/cobra" + + configpkg "github.com/Azure/unbounded/internal/net/config" ) func newNodeConfigTestCommand(cfg *config) *cobra.Command { @@ -35,6 +37,7 @@ func newNodeConfigTestCommand(cfg *config) *cobra.Command { flags.DurationVar(&cfg.StatusPushInterval, "status-push-interval", 10*time.Second, "") flags.DurationVar(&cfg.StatusPushAPIServerInterval, "status-push-apiserver-interval", 30*time.Second, "") flags.BoolVar(&cfg.StatusPushDelta, "status-push-delta", true, "") + flags.StringVar(&cfg.StatusDetailMode, "status-detail-mode", configpkg.DefaultStatusDetailMode, "") flags.BoolVar(&cfg.StatusWSEnabled, "status-ws-enabled", true, "") flags.StringVar(&cfg.StatusWSURL, "status-ws-url", "", "") flags.StringVar(&cfg.StatusWSAPIServerMode, "status-ws-apiserver-mode", statusWSAPIServerModeFallback, "") diff --git a/cmd/unbounded-net-node/status_detail_config_test.go b/cmd/unbounded-net-node/status_detail_config_test.go new file mode 100644 index 000000000..31e5ba0b9 --- /dev/null +++ b/cmd/unbounded-net-node/status_detail_config_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestNodeStatusDetailConfig(t *testing.T) { + for _, tc := range []struct { + name, yaml, flag, want string + invalid bool + }{ + {name: "default", yaml: "node: {}", want: "full"}, + {name: "summary", yaml: "node:\n statusDetailMode: summary", want: "summary"}, + {name: "full", yaml: "node:\n statusDetailMode: full", want: "full"}, + {name: "invalid YAML value", yaml: "node:\n statusDetailMode: invalid", invalid: true}, + {name: "flag wins", yaml: "node:\n statusDetailMode: summary", flag: "full", want: "full"}, + {name: "flag overrides invalid YAML", yaml: "node:\n statusDetailMode: invalid", flag: "summary", want: "summary"}, + {name: "invalid flag", yaml: "node: {}", flag: "invalid", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0o600); err != nil { + t.Fatal(err) + } + + cfg := &config{ + ConfigFile: path, GeneveInterfaceName: "geneve0", VXLANInterfaceName: "vxlan0", + IPIPInterfaceName: "ipip0", WireGuardInterfacePrefix: "wg", + } + + cmd := newNodeConfigTestCommand(cfg) + if tc.flag != "" { + if err := cmd.Flags().Set("status-detail-mode", tc.flag); err != nil { + t.Fatal(err) + } + } + + err := applyNodeRuntimeConfig(cmd, cfg) + if (err != nil) != tc.invalid { + t.Fatalf("applyNodeRuntimeConfig() = %v", err) + } + + if !tc.invalid && cfg.StatusDetailMode != tc.want { + t.Errorf("mode = %q, want %q", cfg.StatusDetailMode, tc.want) + } + }) + } +} diff --git a/deploy/net/01-configmap.yaml.tmpl b/deploy/net/01-configmap.yaml.tmpl index d4dc83a36..e742250ad 100644 --- a/deploy/net/01-configmap.yaml.tmpl +++ b/deploy/net/01-configmap.yaml.tmpl @@ -20,6 +20,9 @@ data: nodeAgentHealthPort: {{ default "9998" .ControllerNodeAgentHealthPort }} informerResyncPeriod: "{{ default "300s" .ControllerInformerResyncPeriod }}" statusStaleThreshold: "{{ default "90s" .ControllerStatusStaleThreshold }}" + # Preparatory, startup-only detail lifetimes; cache/request wiring follows. + statusDetailCacheTTL: "{{ default "300s" .ControllerStatusDetailCacheTTL }}" + statusDetailRequestTimeout: "{{ default "120s" .ControllerStatusDetailRequestTimeout }}" statusWebsocketKeepaliveInterval: "{{ default "30s" .ControllerStatusWebsocketKeepaliveInterval }}" statusWsKeepaliveFailureCount: {{ default "3" .ControllerStatusWsKeepaliveFailureCount }} registerAggregatedAPIServer: {{ default "true" .ControllerRegisterAggregatedAPIServer }} @@ -67,6 +70,8 @@ data: statusPushEnabled: {{ default "true" .NodeStatusPushEnabled }} statusPushURL: "{{ default "" .NodeStatusPushURL }}" statusPushDelta: {{ default "true" .NodeStatusPushDelta }} + # Preparatory, startup-only; keep full until summary rollout is activated. + statusDetailMode: "{{ default "full" .NodeStatusDetailMode }}" statusPushInterval: "{{ default "60s" .NodeStatusPushInterval }}" statusPushApiserverInterval: "{{ default "60s" .NodeStatusPushApiserverInterval }}" healthCheckPort: "{{ default "9997" .NodeHealthCheckPort }}" diff --git a/docs/content/reference/networking/configuration.md b/docs/content/reference/networking/configuration.md index 95d70787a..290958c6e 100644 --- a/docs/content/reference/networking/configuration.md +++ b/docs/content/reference/networking/configuration.md @@ -17,6 +17,23 @@ file mounted from the `unbounded-net-config` ConfigMap. - Startup behavior: fail-fast if the config file is missing or invalid. - CLI flags still work as explicit overrides when set. +### Preparatory detail status settings + +The lightweight-status rollout adds startup-only settings. This preparatory +layer parses and validates them without changing publication behavior. `full` +remains the default until the later collector, cache, and consumer activation. +Changes require restarting the affected controller or node pod. + +| Runtime setting | CLI override | Current default | Allowed values | +|-----------------|--------------|-----------------|----------------| +| `node.statusDetailMode` | `--status-detail-mode` | `full` | `summary`, `full` | +| `controller.statusDetailCacheTTL` | `--status-detail-cache-ttl` | `300s` | Strictly positive duration | +| `controller.statusDetailRequestTimeout` | `--status-detail-request-timeout` | `120s` | Strictly positive duration | + +The intended cache lifetime is measured from actual detail receipt; summaries +and reads do not extend it. Request timeout spans all delivery attempts. +Upgrade controllers before enabling summary publication in the completed rollout. + ### Config Structure ```yaml diff --git a/docs/net/configuration.md b/docs/net/configuration.md index 28ad70f67..0574e27da 100644 --- a/docs/net/configuration.md +++ b/docs/net/configuration.md @@ -13,6 +13,23 @@ Both binaries now load runtime settings from a shared YAML file mounted from the - Startup behavior: fail-fast if the config file is missing or invalid - CLI flags still work as explicit overrides when set +### Preparatory detail status settings + +These startup-only settings prepare the lightweight-status rollout. They are +parsed and validated now; collection, caching, and request delivery are wired in +subsequent layers. Publication behavior remains unchanged, with `full` as the +default until final activation. Changing these settings requires a pod restart. + +| Runtime setting | CLI override | Current default | Allowed values | +|-----------------|--------------|-----------------|----------------| +| `node.statusDetailMode` | `--status-detail-mode` | `full` | `summary`, `full` | +| `controller.statusDetailCacheTTL` | `--status-detail-cache-ttl` | `300s` | Strictly positive duration | +| `controller.statusDetailRequestTimeout` | `--status-detail-request-timeout` | `120s` | Strictly positive duration | + +The intended cache lifetime starts when actual details arrive, not on summary +updates or reads. The request timeout covers all delivery attempts together. +Upgrade controllers before enabling summary publication in the completed rollout. + ### Runtime config structure ```yaml diff --git a/internal/net/config/config.go b/internal/net/config/config.go index eab564c2c..32e793e48 100644 --- a/internal/net/config/config.go +++ b/internal/net/config/config.go @@ -35,6 +35,10 @@ type Config struct { // StatusStaleThreshold is the duration after which a node's pushed status is considered stale. // When stale, the controller falls back to pulling status directly from the node. StatusStaleThreshold time.Duration + // StatusDetailCacheTTL is the startup-loaded lifetime of received node details. + StatusDetailCacheTTL time.Duration + // StatusDetailRequestTimeout is the startup-loaded end-to-end detail request deadline. + StatusDetailRequestTimeout time.Duration // RegisterAggregatedAPIServer controls whether the controller serves aggregated API status endpoints. RegisterAggregatedAPIServer bool // StatusWSKeepaliveInterval controls websocket ping cadence for node status streams. @@ -114,5 +118,13 @@ func (c *Config) Validate() error { return fmt.Errorf("status websocket keepalive failure count must be >= 1") } + if c.StatusDetailCacheTTL <= 0 { + return fmt.Errorf("controller.statusDetailCacheTTL must be greater than zero") + } + + if c.StatusDetailRequestTimeout <= 0 { + return fmt.Errorf("controller.statusDetailRequestTimeout must be greater than zero") + } + return nil } diff --git a/internal/net/config/config_test.go b/internal/net/config/config_test.go index 5bc3ff640..078704233 100644 --- a/internal/net/config/config_test.go +++ b/internal/net/config/config_test.go @@ -40,7 +40,11 @@ func TestDefaultLeaderElectionConfig(t *testing.T) { // TestConfigValidate tests ConfigValidate. func TestConfigValidate(t *testing.T) { - cfg := &Config{StatusWSKeepaliveFailureCount: 2} + cfg := &Config{ + StatusWSKeepaliveFailureCount: 2, + StatusDetailCacheTTL: DefaultStatusDetailCacheTTL, + StatusDetailRequestTimeout: DefaultStatusDetailRequestTimeout, + } if err := cfg.Validate(); err != nil { t.Fatalf("expected nil validation error, got %v", err) } diff --git a/internal/net/config/runtime_config.go b/internal/net/config/runtime_config.go index d52776fdf..da6cb7396 100644 --- a/internal/net/config/runtime_config.go +++ b/internal/net/config/runtime_config.go @@ -31,6 +31,8 @@ type ControllerRuntimeConfig struct { HealthPort *int `yaml:"healthPort"` NodeAgentHealthPort *int `yaml:"nodeAgentHealthPort"` StatusStaleThreshold string `yaml:"statusStaleThreshold"` + StatusDetailCacheTTL string `yaml:"statusDetailCacheTTL"` + StatusDetailRequestTimeout string `yaml:"statusDetailRequestTimeout"` StatusWSKeepaliveInterval string `yaml:"statusWebsocketKeepaliveInterval"` StatusWSKeepaliveFailCount *int `yaml:"statusWsKeepaliveFailureCount"` RegisterAggregatedAPIServer *bool `yaml:"registerAggregatedAPIServer"` @@ -82,6 +84,7 @@ type NodeRuntimeConfig struct { StatusPushInterval string `yaml:"statusPushInterval"` StatusPushAPIServerInterval string `yaml:"statusPushApiserverInterval"` StatusPushDelta *bool `yaml:"statusPushDelta"` + StatusDetailMode string `yaml:"statusDetailMode"` StatusWSEnabled *bool `yaml:"statusWebsocketEnabled"` StatusWSURL string `yaml:"statusWebsocketURL"` StatusWSAPIServerMode string `yaml:"statusWebsocketApiserverMode"` @@ -140,3 +143,37 @@ func ParseDurationField(raw, fieldName string) (time.Duration, error) { return value, nil } + +// ParsePositiveDurationField parses a configured lifetime; empty means unset. +func ParsePositiveDurationField(raw, fieldName string) (time.Duration, error) { + value, err := ParseDurationField(raw, fieldName) + if err != nil { + return 0, err + } + + if raw != "" && value <= 0 { + return 0, fmt.Errorf("%s must be greater than zero", fieldName) + } + + return value, nil +} + +const ( + StatusDetailModeSummary = "summary" + StatusDetailModeFull = "full" + // DefaultStatusDetailMode preserves legacy publication during preparatory rollout. + // Summary becomes the default only after collectors and consumers are wired. + DefaultStatusDetailMode = StatusDetailModeFull + DefaultStatusDetailCacheTTL = 300 * time.Second + DefaultStatusDetailRequestTimeout = 120 * time.Second +) + +// ValidateStatusDetailMode checks the startup-loaded publication mode. +func ValidateStatusDetailMode(mode string) error { + switch mode { + case StatusDetailModeSummary, StatusDetailModeFull: + return nil + default: + return fmt.Errorf("invalid node.statusDetailMode %q: must be summary or full", mode) + } +} diff --git a/internal/net/config/status_detail_test.go b/internal/net/config/status_detail_test.go new file mode 100644 index 000000000..a6fc06aee --- /dev/null +++ b/internal/net/config/status_detail_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "strings" + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +func TestStatusDetailMode(t *testing.T) { + if DefaultStatusDetailMode != "full" { + t.Fatal("preparatory default must preserve full publication") + } + + for _, mode := range []string{"summary", "full", "", "SUMMARY", "other", " full "} { + valid := mode == "summary" || mode == "full" + if err := ValidateStatusDetailMode(mode); (err == nil) != valid { + t.Errorf("ValidateStatusDetailMode(%q) = %v", mode, err) + } + } +} + +func TestPositiveStatusDetailDurations(t *testing.T) { + for _, tc := range []struct { + raw string + want time.Duration + valid bool + }{ + {"", 0, true}, + {"300s", 300 * time.Second, true}, + {"1ns", time.Nanosecond, true}, + {"0s", 0, false}, + {"-1s", 0, false}, + {"invalid", 0, false}, + } { + got, err := ParsePositiveDurationField(tc.raw, "controller.statusDetailCacheTTL") + if (err == nil) != tc.valid || got != tc.want { + t.Errorf("ParsePositiveDurationField(%q) = %v, %v", tc.raw, got, err) + } + + if err != nil && !strings.Contains(err.Error(), "controller.statusDetailCacheTTL") { + t.Errorf("missing field name in error: %v", err) + } + } + + for _, field := range []string{"cache", "request"} { + for _, duration := range []time.Duration{0, -time.Second, time.Nanosecond} { + cfg := &Config{ + StatusWSKeepaliveFailureCount: 2, + StatusDetailCacheTTL: DefaultStatusDetailCacheTTL, + StatusDetailRequestTimeout: DefaultStatusDetailRequestTimeout, + } + if field == "cache" { + cfg.StatusDetailCacheTTL = duration + } else { + cfg.StatusDetailRequestTimeout = duration + } + + if err := cfg.Validate(); (err == nil) != (duration > 0) { + t.Errorf("Validate(%s=%s) = %v", field, duration, err) + } + } + } +} + +func TestStatusDetailRuntimeYAMLRoundTrip(t *testing.T) { + for _, mode := range []string{"", "summary", "full"} { + want := RuntimeConfig{ + Node: NodeRuntimeConfig{StatusDetailMode: mode}, + Controller: ControllerRuntimeConfig{ + StatusDetailCacheTTL: "300s", StatusDetailRequestTimeout: "120s", + }, + } + + data, err := yaml.Marshal(want) + if err != nil { + t.Fatal(err) + } + + for _, field := range []string{"statusDetailMode:", "statusDetailCacheTTL: 300s", "statusDetailRequestTimeout: 120s"} { + if !strings.Contains(string(data), field) { + t.Errorf("missing YAML setting %q", field) + } + } + + var got RuntimeConfig + if err := yaml.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + + if got.Node.StatusDetailMode != mode || + got.Controller.StatusDetailCacheTTL != want.Controller.StatusDetailCacheTTL || + got.Controller.StatusDetailRequestTimeout != want.Controller.StatusDetailRequestTimeout { + t.Fatalf("settings changed after YAML round trip: %+v", got) + } + } +} From adfa7cbd01c25c46e9bbd80dc2e16668f1eb5ae5 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:21:00 +0000 Subject: [PATCH 07/34] net-controller: add standalone expiring node detail cache Keep detailed snapshots in one immutable shallow-owned TTL store. Add read-time and proactive deadline expiry, lifecycle cancellation and restart, and deterministic ownership and concurrency tests. Leave existing status ingestion and broadcasts unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-controller/detail_cache.go | 209 ++++++++ .../detail_cache_test.go | 448 ++++++++++++++++++ 2 files changed, 657 insertions(+) create mode 100644 cmd/unbounded-net-controller/detail_cache.go create mode 100644 cmd/unbounded-net-controller/detail_cache_test.go diff --git a/cmd/unbounded-net-controller/detail_cache.go b/cmd/unbounded-net-controller/detail_cache.go new file mode 100644 index 000000000..bd51416e4 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_cache.go @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "sync" + "time" + + "k8s.io/utils/clock" +) + +// nodeDetailSnapshot carries immutable details, separate from routine status. +// Status and all its nested data must remain read-only, including for callers +// retaining a returned snapshot after its cache entry expires. +type nodeDetailSnapshot struct { + NodeName string + RequestID string + CollectedAt time.Time + ReceivedAt time.Time + ExpiresAt time.Time + Status *NodeStatusResponse +} + +// nodeDetailCache is a leader-local, TTL-only store. It owns no second result +// history or per-entry timers. TTL bounds retention time, not peak memory. +// Construct it with newNodeDetailCache and run one Run loop for proactive expiry. +type nodeDetailCache struct { + mu sync.Mutex + ttl time.Duration + clock clock.Clock // May be replaced in tests before concurrent use. + entries map[string]nodeDetailSnapshot + changed chan struct{} + running bool +} + +func newNodeDetailCache(ttl time.Duration) (*nodeDetailCache, error) { + if ttl <= 0 { + return nil, errors.New("node detail cache TTL must be positive") + } + + return &nodeDetailCache{ + ttl: ttl, + clock: clock.RealClock{}, + entries: make(map[string]nodeDetailSnapshot), + changed: make(chan struct{}, 1), + }, nil +} + +// Store accepts actual detailed data only; callers must not pass summaries or +// failed fetches. It shallow-copies status without modifying it. Nested slices, +// maps, and pointers remain shared and must not be mutated by the caller. +// Only Store renews the receipt-based TTL; request validation belongs upstream. +func (c *nodeDetailCache) Store(nodeName, requestID string, collectedAt time.Time, status *NodeStatusResponse) (nodeDetailSnapshot, error) { + if nodeName == "" { + return nodeDetailSnapshot{}, errors.New("node detail cache requires a node name") + } + + if status == nil { + return nodeDetailSnapshot{}, errors.New("node detail cache requires detailed status") + } + + statusCopy := *status + + c.mu.Lock() + defer c.mu.Unlock() + + now := c.clock.Now() + snapshot := nodeDetailSnapshot{ + NodeName: nodeName, + RequestID: requestID, + CollectedAt: collectedAt, + ReceivedAt: now, + ExpiresAt: now.Add(c.ttl), + Status: &statusCopy, + } + c.entries[nodeName] = snapshot + c.notify() + + return snapshot, nil +} + +// Get does not refresh TTL. At the deadline it drops the cache's ownership, +// even when the proactive expiry loop has not yet been scheduled. +func (c *nodeDetailCache) Get(nodeName string) (nodeDetailSnapshot, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + snapshot, ok := c.entries[nodeName] + if !ok { + return nodeDetailSnapshot{}, false + } + + if !c.clock.Now().Before(snapshot.ExpiresAt) { + delete(c.entries, nodeName) + c.notify() + + return nodeDetailSnapshot{}, false + } + + return snapshot, true +} + +func (c *nodeDetailCache) Delete(nodeName string) { + c.mu.Lock() + defer c.mu.Unlock() + + delete(c.entries, nodeName) + c.notify() +} + +// Clear releases all cache-owned details without mutating shared payloads. +// It does not stop Run; subsequent stores can be expired by the same loop. +func (c *nodeDetailCache) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + + clear(c.entries) + c.notify() +} + +// Expire removes entries at or past their deadline and returns their count. +func (c *nodeDetailCache) Expire() int { + c.mu.Lock() + defer c.mu.Unlock() + + removed, _ := c.expireLocked(c.clock.Now()) + c.notify() + + return removed +} + +func (c *nodeDetailCache) expireLocked(now time.Time) (int, time.Time) { + removed := 0 + + var next time.Time + + for name, snapshot := range c.entries { + if !now.Before(snapshot.ExpiresAt) { + delete(c.entries, name) + + removed++ + } else if next.IsZero() || snapshot.ExpiresAt.Before(next) { + next = snapshot.ExpiresAt + } + } + + return removed, next +} + +func (c *nodeDetailCache) notify() { + select { + case c.changed <- struct{}{}: + default: + } +} + +// Run blocks until cancellation, then stops its timer and clears all entries. +// Wait for Run to return before restarting it or storing for a new leadership +// term. Concurrent Run calls are rejected; no goroutine is started internally. +func (c *nodeDetailCache) Run(ctx context.Context) error { + c.mu.Lock() + if c.running { + c.mu.Unlock() + + return errors.New("node detail cache expiry loop is already running") + } + + c.running = true + c.mu.Unlock() + + defer func() { + c.mu.Lock() + defer c.mu.Unlock() + + clear(c.entries) + c.running = false + }() + + for ctx.Err() == nil { + c.mu.Lock() + _, next := c.expireLocked(c.clock.Now()) + c.mu.Unlock() + + var ( + timer clock.Timer + timerC <-chan time.Time + ) + + if !next.IsZero() { + timer = c.clock.NewTimer(next.Sub(c.clock.Now())) + timerC = timer.C() + } + + select { + case <-ctx.Done(): + case <-c.changed: + case <-timerC: + } + + if timer != nil { + timer.Stop() + } + } + + return nil +} diff --git a/cmd/unbounded-net-controller/detail_cache_test.go b/cmd/unbounded-net-controller/detail_cache_test.go new file mode 100644 index 000000000..629204a3b --- /dev/null +++ b/cmd/unbounded-net-controller/detail_cache_test.go @@ -0,0 +1,448 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "sync" + "testing" + "testing/synctest" + "time" + + testingclock "k8s.io/utils/clock/testing" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func newTestNodeDetailCache(t *testing.T) (*nodeDetailCache, *testingclock.FakeClock) { + t.Helper() + + cache, err := newNodeDetailCache(time.Minute) + if err != nil { + t.Fatal(err) + } + + fakeClock := testingclock.NewFakeClock(time.Now()) + cache.clock = fakeClock + + return cache, fakeClock +} + +func storeTestNodeDetails(t *testing.T, cache *nodeDetailCache, requestID string) nodeDetailSnapshot { + t.Helper() + + snapshot, err := cache.Store("node", requestID, cache.clock.Now().Add(-time.Hour), &NodeStatusResponse{}) + if err != nil { + t.Fatal(err) + } + + return snapshot +} + +func assertNodeDetailEntries(t *testing.T, cache *nodeDetailCache, want int) { + t.Helper() + cache.mu.Lock() + defer cache.mu.Unlock() + + if got := len(cache.entries); got != want { + t.Fatalf("cache owns %d entries, want %d", got, want) + } +} + +func startNodeDetailLoop(t *testing.T, cache *nodeDetailCache) (context.CancelFunc, <-chan error) { + t.Helper() + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + + go func() { + done <- cache.Run(ctx) + }() + + t.Cleanup(cancel) + synctest.Wait() + + return cancel, done +} + +func TestNodeDetailCacheValidation(t *testing.T) { + for _, ttl := range []time.Duration{-time.Second, 0} { + if cache, err := newNodeDetailCache(ttl); err == nil || cache != nil { + t.Fatalf("TTL %v: got cache %v, error %v", ttl, cache, err) + } + } + + if _, err := newNodeDetailCache(time.Nanosecond); err != nil { + t.Fatalf("positive TTL rejected: %v", err) + } + + cache, fakeClock := newTestNodeDetailCache(t) + initial := storeTestNodeDetails(t, cache, "original") + + fakeClock.Step(time.Second) + + if _, err := cache.Store("", "invalid", fakeClock.Now(), &NodeStatusResponse{}); err == nil { + t.Fatal("empty node name accepted") + } + + if _, err := cache.Store("node", "invalid", fakeClock.Now(), nil); err == nil { + t.Fatal("nil details accepted") + } + + if got, ok := cache.Get("node"); !ok || got != initial { + t.Fatal("invalid Store replaced or refreshed existing details") + } + + if got, ok := cache.Get("missing"); ok || got != (nodeDetailSnapshot{}) { + t.Fatal("missing entry did not return an empty snapshot") + } + + assertNodeDetailEntries(t, cache, 1) +} + +func TestNodeDetailCacheDeadlineAndReadNonrefresh(t *testing.T) { + for _, offset := range []time.Duration{-time.Nanosecond, 0, time.Nanosecond} { + t.Run(offset.String(), func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + initial := storeTestNodeDetails(t, cache, "request") + + if initial.NodeName != "node" || initial.RequestID != "request" || + !initial.ReceivedAt.Equal(fakeClock.Now()) || + !initial.CollectedAt.Equal(fakeClock.Now().Add(-time.Hour)) || + !initial.ExpiresAt.Equal(fakeClock.Now().Add(cache.ttl)) { + t.Fatalf("incorrect receipt metadata: %+v", initial) + } + + fakeClock.Step(cache.ttl / 2) + + for range 3 { + got, ok := cache.Get("node") + if !ok || got != initial { + t.Fatal("read changed the snapshot") + } + + got.RequestID = "local-copy-only" + } + + fakeClock.Step(cache.ttl/2 + offset) + + got, ok := cache.Get("node") + if offset < 0 { + if !ok || got != initial { + t.Fatal("details expired before the deadline") + } + + assertNodeDetailEntries(t, cache, 1) + } else { + if ok || got != (nodeDetailSnapshot{}) { + t.Fatal("expired details returned") + } + + assertNodeDetailEntries(t, cache, 0) + } + }) + } +} + +func TestNodeDetailCacheReplacement(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + initial := storeTestNodeDetails(t, cache, "first") + fakeClock.Step(cache.ttl / 2) + replacement := storeTestNodeDetails(t, cache, "second") + + if replacement.Status == initial.Status || replacement.ExpiresAt != initial.ExpiresAt.Add(cache.ttl/2) { + t.Fatal("replacement did not renew details and TTL") + } + + fakeClock.Step(cache.ttl / 2) + + if removed := cache.Expire(); removed != 0 { + t.Fatalf("old deadline removed %d replacements", removed) + } + + if got, ok := cache.Get("node"); !ok || got != replacement { + t.Fatal("replacement missing at the old deadline") + } + + fakeClock.Step(cache.ttl / 2) + + if removed := cache.Expire(); removed != 1 { + t.Fatalf("removed %d entries at replacement deadline, want 1", removed) + } + + if removed := cache.Expire(); removed != 0 { + t.Fatalf("repeated expiry removed %d entries", removed) + } + + assertNodeDetailEntries(t, cache, 0) +} + +func TestNodeDetailCacheReleasesHeavyReferences(t *testing.T) { + for _, operation := range []string{"replace", "delete", "clear", "expire", "get-expired"} { + t.Run(operation, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + status := &NodeStatusResponse{ + NodeInfo: NodeInfo{K8sLabels: map[string]string{"label": "original"}}, + Peers: make([]statusv1alpha1.PeerStatus, 1024), + RoutingTable: RoutingTableInfo{Routes: []statusv1alpha1.RouteEntry{{ + NextHops: make([]statusv1alpha1.NextHop, 1024), + }}}, + BpfEntries: make([]BpfEntry, 1024), + } + + snapshot, err := cache.Store("node", "heavy", fakeClock.Now(), status) + if err != nil { + t.Fatal(err) + } + + if snapshot.Status == status || &snapshot.Status.Peers[0] != &status.Peers[0] || + &snapshot.Status.RoutingTable.Routes[0] != &status.RoutingTable.Routes[0] || + &snapshot.Status.BpfEntries[0] != &status.BpfEntries[0] { + t.Fatal("Store must copy the top-level value but share nested details") + } + + switch operation { + case "replace": + replacement := storeTestNodeDetails(t, cache, "light") + cache.mu.Lock() + stored := cache.entries["node"] + cache.mu.Unlock() + + if stored != replacement || stored.Status == snapshot.Status { + t.Fatal("map still owns the heavy snapshot") + } + case "delete": + cache.Delete("missing") + assertNodeDetailEntries(t, cache, 1) + cache.Delete("node") + cache.Delete("node") + case "clear": + if _, err := cache.Store("other", "", fakeClock.Now(), status); err != nil { + t.Fatal(err) + } + + cache.Clear() + cache.Clear() + case "expire": + fakeClock.Step(cache.ttl) + cache.Expire() + case "get-expired": + fakeClock.Step(cache.ttl) + cache.Get("node") + } + + if operation != "replace" { + assertNodeDetailEntries(t, cache, 0) + } + + if status.NodeInfo.K8sLabels["label"] != "original" || len(status.Peers) != 1024 || + len(status.RoutingTable.Routes[0].NextHops) != 1024 || len(status.BpfEntries) != 1024 || + len(snapshot.Status.Peers) != 1024 || snapshot.Status.NodeInfo.K8sLabels["label"] != "original" { + t.Fatal("removing ownership mutated a shared payload") + } + }) + } +} + +func TestNodeDetailCacheRunExpiryAndReplacement(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + cancel, done := startNodeDetailLoop(t, cache) + storeTestNodeDetails(t, cache, "first") + synctest.Wait() + fakeClock.Step(cache.ttl / 2) + storeTestNodeDetails(t, cache, "replacement") + synctest.Wait() + fakeClock.Step(cache.ttl / 2) + synctest.Wait() + + // Inspect the map, not Get: readers must not be required for cleanup. + assertNodeDetailEntries(t, cache, 1) + fakeClock.Step(cache.ttl / 2) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + + if fakeClock.HasWaiters() { + t.Fatal("expiry loop left an active timer") + } + }) +} + +func TestNodeDetailCacheRunClearCancelRestart(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + storeTestNodeDetails(t, cache, "before-run") + cancel, done := startNodeDetailLoop(t, cache) + + if err := cache.Run(t.Context()); err == nil { + t.Fatal("concurrent expiry loop accepted") + } + + cache.Clear() + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + + if fakeClock.HasWaiters() { + t.Fatal("Clear left an active timer") + } + + storeTestNodeDetails(t, cache, "after-clear") + synctest.Wait() + fakeClock.Step(cache.ttl) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + storeTestNodeDetails(t, cache, "before-cancel") + synctest.Wait() + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + + assertNodeDetailEntries(t, cache, 0) + + if fakeClock.HasWaiters() { + t.Fatal("cancellation left an active timer") + } + + storeTestNodeDetails(t, cache, "restart") + cancel, done = startNodeDetailLoop(t, cache) + fakeClock.Step(cache.ttl) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + }) +} + +func TestNodeDetailCacheRunMultipleDeadlines(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + storeTestNodeDetails(t, cache, "already-expired") + fakeClock.Step(cache.ttl) + cancel, done := startNodeDetailLoop(t, cache) + assertNodeDetailEntries(t, cache, 0) + + storeTestNodeDetails(t, cache, "earliest") + synctest.Wait() + fakeClock.Step(cache.ttl / 2) + + if _, err := cache.Store("later", "", fakeClock.Now(), &NodeStatusResponse{}); err != nil { + t.Fatal(err) + } + + synctest.Wait() + fakeClock.Step(cache.ttl / 2) + synctest.Wait() + assertNodeDetailEntries(t, cache, 1) + + cache.mu.Lock() + _, earliestRetained := cache.entries["node"] + _, laterRetained := cache.entries["later"] + cache.mu.Unlock() + + if earliestRetained || !laterRetained { + t.Fatal("loop did not expire only the earliest deadline") + } + + fakeClock.Step(cache.ttl / 2) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + }) +} + +func TestNodeDetailCacheRunAlreadyCanceled(t *testing.T) { + cache, _ := newTestNodeDetailCache(t) + storeTestNodeDetails(t, cache, "request") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + if err := cache.Run(ctx); err != nil { + t.Fatal(err) + } + + assertNodeDetailEntries(t, cache, 0) +} + +func TestNodeDetailCacheConcurrentAccess(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + cancel, done := startNodeDetailLoop(t, cache) + + var workers sync.WaitGroup + + for worker := range 8 { + workers.Go(func() { + for iteration := range 100 { + name := fmt.Sprintf("node-%d", iteration%4) + request := fmt.Sprintf("%d-%d", worker, iteration) + + if _, err := cache.Store(name, request, fakeClock.Now(), &NodeStatusResponse{}); err != nil { + t.Error(err) + + return + } + + cache.Get(name) + fakeClock.Step(time.Second) + cache.Expire() + cache.Delete(name) + + if iteration%10 == 0 { + cache.Clear() + } + } + }) + } + + workers.Wait() + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + + assertNodeDetailEntries(t, cache, 0) + + if fakeClock.HasWaiters() { + t.Fatal("concurrent operations left an active timer") + } + }) +} + +func TestNodeDetailCacheRunRealClock(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, err := newNodeDetailCache(time.Minute) + if err != nil { + t.Fatal(err) + } + + storeTestNodeDetails(t, cache, "request") + cancel, done := startNodeDetailLoop(t, cache) + + // synctest advances virtual standard-library time without a real sleep. + time.Sleep(cache.ttl) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + }) +} From 701d3b6a02c5ba7687ea411f49f9055d1a6198d7 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:28:50 +0000 Subject: [PATCH 08/34] net: share exact overview projection and health semantics Factor legacy peer health and observed-route mismatch calculations into reusable status helpers. Preserve lightweight metadata without carrying peer, route, or BPF arrays. Reuse the calculations for existing controller summaries without changing their output. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-controller/status_types.go | 56 +++------ internal/net/status/overview.go | 68 +++++++++++ .../net/status/overview_projection_test.go | 109 ++++++++++++++++++ 3 files changed, 191 insertions(+), 42 deletions(-) create mode 100644 internal/net/status/overview.go create mode 100644 internal/net/status/overview_projection_test.go diff --git a/cmd/unbounded-net-controller/status_types.go b/cmd/unbounded-net-controller/status_types.go index 567a13be5..e6145a30f 100644 --- a/cmd/unbounded-net-controller/status_types.go +++ b/cmd/unbounded-net-controller/status_types.go @@ -9,6 +9,7 @@ import ( "sort" "time" + statuspkg "github.com/Azure/unbounded/internal/net/status" statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) @@ -183,23 +184,26 @@ type NodeSummary struct { } // buildClusterSummary extracts a ClusterSummary from a full ClusterStatusResponse. -// This is O(N) in nodes with simple field reads -- no route annotation work. +// Legacy payloads require scanning peers and route next hops for overview facts. func buildClusterSummary(status *ClusterStatusResponse) *ClusterSummary { summaries := make([]NodeSummary, 0, len(status.Nodes)) now := time.Now() for i := range status.Nodes { node := status.Nodes[i] + overview := statuspkg.OverviewFromStatus(node, now) ns := NodeSummary{ - Name: node.NodeInfo.Name, - SiteName: node.NodeInfo.SiteName, - IsGateway: node.NodeInfo.IsGateway, - K8sReady: node.NodeInfo.K8sReady, - StatusSource: node.StatusSource, - PeerCount: len(node.Peers), - RouteCount: len(node.RoutingTable.Routes), - FetchError: node.FetchError, - ErrorCount: len(node.NodeErrors), + Name: node.NodeInfo.Name, + SiteName: node.NodeInfo.SiteName, + IsGateway: node.NodeInfo.IsGateway, + K8sReady: node.NodeInfo.K8sReady, + StatusSource: node.StatusSource, + PeerCount: overview.PeerCount, + HealthyPeers: overview.HealthyPeers, + RouteCount: overview.RouteCount, + RouteMismatch: overview.RouteMismatch, + FetchError: node.FetchError, + ErrorCount: len(node.NodeErrors), } // Include first error message so the frontend can show it inline @@ -208,38 +212,6 @@ func buildClusterSummary(status *ClusterStatusResponse) *ClusterSummary { ns.FirstError = node.NodeErrors[0].Message } - // Count healthy peers - for j := range node.Peers { - peer := &node.Peers[j] - if peer.HealthCheck != nil && peer.HealthCheck.Enabled { - if peer.HealthCheck.Status == "up" || peer.HealthCheck.Status == "Up" { - ns.HealthyPeers++ - } - } else { - // Fall back to handshake freshness - if !peer.Tunnel.LastHandshake.IsZero() && now.Sub(peer.Tunnel.LastHandshake) < 3*time.Minute { - ns.HealthyPeers++ - } - } - } - - // Route mismatch check - for _, route := range node.RoutingTable.Routes { - for _, hop := range route.NextHops { - expected := hop.Expected != nil && *hop.Expected - - present := hop.Present != nil && *hop.Present - if expected != present { - ns.RouteMismatch = true - break - } - } - - if ns.RouteMismatch { - break - } - } - // Derive CNI status and tone ns.CniStatus, ns.CniTone = deriveCniStatusAndTone(node, ns.RouteMismatch) summaries = append(summaries, ns) diff --git a/internal/net/status/overview.go b/internal/net/status/overview.go new file mode 100644 index 000000000..1f06c3189 --- /dev/null +++ b/internal/net/status/overview.go @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "time" + + "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// OverviewFromStatus projects a legacy detailed publication into observed facts. +// Node summary collection should compute these facts without collecting details. +func OverviewFromStatus(full *v1alpha1.NodeStatusResponse, now time.Time) v1alpha1.NodeStatusOverview { + overview := v1alpha1.NodeStatusOverview{ + Timestamp: full.Timestamp, NodeInfo: full.NodeInfo, HealthCheck: full.HealthCheck, + NodeErrors: full.NodeErrors, FetchError: full.FetchError, + LastPushTime: full.LastPushTime, StatusSource: full.StatusSource, NodePodInfo: full.NodePodInfo, + PeerCount: len(full.Peers), RouteCount: len(full.RoutingTable.Routes), + } + for i := range full.Peers { + if PeerHealthyForOverview(&full.Peers[i], now) { + overview.HealthyPeers++ + } + } + + for _, route := range full.RoutingTable.Routes { + if RouteMismatchForOverview(route) { + overview.RouteMismatch = true + break + } + } + + return overview +} + +// OverviewMetadata preserves lightweight fields for controller enrichment. +// It contains no details and must not be returned as a diagnostic snapshot. +func OverviewMetadata(overview v1alpha1.NodeStatusOverview) v1alpha1.NodeStatusResponse { + return v1alpha1.NodeStatusResponse{ + Timestamp: overview.Timestamp, NodeInfo: overview.NodeInfo, HealthCheck: overview.HealthCheck, + NodeErrors: overview.NodeErrors, FetchError: overview.FetchError, + LastPushTime: overview.LastPushTime, StatusSource: overview.StatusSource, NodePodInfo: overview.NodePodInfo, + } +} + +// PeerHealthyForOverview preserves the dashboard's probe/handshake fallback. +func PeerHealthyForOverview(peer *v1alpha1.PeerStatus, now time.Time) bool { + if peer.HealthCheck != nil && peer.HealthCheck.Enabled { + return peer.HealthCheck.Status == "up" || peer.HealthCheck.Status == "Up" + } + + return !peer.Tunnel.LastHandshake.IsZero() && now.Sub(peer.Tunnel.LastHandshake) < 3*time.Minute +} + +// RouteMismatchForOverview compares observed and expected next-hop presence. +func RouteMismatchForOverview(route v1alpha1.RouteEntry) bool { + for _, hop := range route.NextHops { + expected := hop.Expected != nil && *hop.Expected + + present := hop.Present != nil && *hop.Present + if expected != present { + return true + } + } + + return false +} diff --git a/internal/net/status/overview_projection_test.go b/internal/net/status/overview_projection_test.go new file mode 100644 index 000000000..bcecb795e --- /dev/null +++ b/internal/net/status/overview_projection_test.go @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "reflect" + "testing" + "time" + + "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestPeerHealthyForOverview(t *testing.T) { + now := time.Unix(1000, 0) + + for _, tc := range []struct { + name string + check *v1alpha1.HealthCheckPeerStatus + age time.Duration + missing bool + want bool + }{ + {name: "unknown", missing: true}, + {name: "recent", age: time.Minute, want: true}, + {name: "boundary", age: 3 * time.Minute}, + {name: "stale", age: 4 * time.Minute}, + {name: "future preserves legacy behavior", age: -time.Minute, want: true}, + {name: "up", check: &v1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "up"}, missing: true, want: true}, + {name: "Up", check: &v1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "Up"}, missing: true, want: true}, + {name: "UP is not up", check: &v1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "UP"}}, + {name: "enabled down overrides handshake", check: &v1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "down"}}, + {name: "disabled uses handshake", check: &v1alpha1.HealthCheckPeerStatus{Status: "down"}, want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + peer := v1alpha1.PeerStatus{HealthCheck: tc.check} + if !tc.missing { + peer.Tunnel.LastHandshake = now.Add(-tc.age) + } + + if got := PeerHealthyForOverview(&peer, now); got != tc.want { + t.Fatalf("healthy = %v, want %v", got, tc.want) + } + }) + } +} + +func TestRouteMismatchForOverview(t *testing.T) { + yes, no := true, false + for _, expected := range []*bool{nil, &no, &yes} { + for _, present := range []*bool{nil, &no, &yes} { + route := v1alpha1.RouteEntry{ + NextHops: []v1alpha1.NextHop{{}, {Expected: expected, Present: present}}, + } + + want := (expected != nil && *expected) != (present != nil && *present) + if got := RouteMismatchForOverview(route); got != want { + t.Fatalf("expected=%v present=%v mismatch=%v, want %v", expected, present, got, want) + } + } + } + + if RouteMismatchForOverview(v1alpha1.RouteEntry{}) { + t.Fatal("an empty route has no mismatched hops") + } +} + +func TestOverviewProjectionPreservesFactsAndMetadata(t *testing.T) { + now := time.Unix(1000, 0) + yes := true + full := v1alpha1.NodeStatusResponse{ + Timestamp: now, + NodeInfo: v1alpha1.NodeInfo{ + Name: "node", SiteName: "site", IsGateway: true, K8sReady: "NotReady", + WireGuard: &v1alpha1.WireGuardStatusInfo{Interface: "wg0"}, + }, + HealthCheck: &v1alpha1.HealthCheckStatus{Healthy: false, Summary: "blocked"}, + NodeErrors: []v1alpha1.NodeError{{Type: "cni", Message: "bootstrap blocked"}}, + FetchError: "stale", LastPushTime: &now, StatusSource: "stale-cache", + NodePodInfo: &v1alpha1.NodePodInfo{PodName: "pod"}, + Peers: []v1alpha1.PeerStatus{ + {HealthCheck: &v1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "up"}}, + {HealthCheck: &v1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "down"}}, + }, + RoutingTable: v1alpha1.RoutingTableInfo{Routes: []v1alpha1.RouteEntry{ + {}, {NextHops: []v1alpha1.NextHop{{Expected: &yes}}}, + }}, + BpfEntries: []v1alpha1.BpfEntry{{CIDR: "10.0.0.0/24"}}, + } + + overview := OverviewFromStatus(&full, now) + if overview.PeerCount != 2 || overview.HealthyPeers != 1 || overview.RouteCount != 2 || !overview.RouteMismatch { + t.Fatalf("observed facts changed: %+v", overview) + } + + metadata := OverviewMetadata(overview) + want := full + want.Peers = nil + want.RoutingTable = v1alpha1.RoutingTableInfo{} + + want.BpfEntries = nil + if !reflect.DeepEqual(metadata, want) { + t.Fatalf("metadata changed: got %+v, want %+v", metadata, want) + } + + if len(full.Peers) != 2 || len(full.RoutingTable.Routes) != 2 || len(full.BpfEntries) != 1 { + t.Fatal("projection mutated the original snapshot") + } +} From 7fe4956283426c7d3cde16e4b6f31020f5a170bf Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:33:49 +0000 Subject: [PATCH 09/34] net-controller: add overview-only routine cache state Keep explicit node-reported counts separate from lightweight metadata and reject legacy deltas against summary bases. Preserve enrichment and summary health indicators, and snapshot mutable cluster containers so concurrent patches cannot change a returned overview. Existing legacy full publication and bulk behavior remain unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- .../cluster_status.go | 24 +++ .../cluster_status_cache.go | 34 ++- cmd/unbounded-net-controller/node_overview.go | 61 ++++++ .../node_overview_test.go | 198 ++++++++++++++++++ cmd/unbounded-net-controller/node_status.go | 27 ++- cmd/unbounded-net-controller/status_types.go | 41 ++-- 6 files changed, 360 insertions(+), 25 deletions(-) create mode 100644 cmd/unbounded-net-controller/node_overview.go create mode 100644 cmd/unbounded-net-controller/node_overview_test.go diff --git a/cmd/unbounded-net-controller/cluster_status.go b/cmd/unbounded-net-controller/cluster_status.go index 2f21c62ba..d33dcce34 100644 --- a/cmd/unbounded-net-controller/cluster_status.go +++ b/cmd/unbounded-net-controller/cluster_status.go @@ -18,6 +18,7 @@ import ( "k8s.io/klog/v2" "github.com/Azure/unbounded/internal/net/controller" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" "github.com/Azure/unbounded/internal/version" ) @@ -236,6 +237,13 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo } cachedStatuses := health.statusCache.GetAll() + status.NodeOverviews = make(map[string]*statusv1alpha1.NodeStatusOverview) + + for name, cached := range cachedStatuses { + if cached.Overview != nil { + status.NodeOverviews[name] = cached.Overview + } + } type pullNode struct{ nodeName, nodeIP string } @@ -371,6 +379,7 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo } else if result.status != nil { result.status.StatusSource = "pull" cachedResults[result.nodeName] = *result.status + delete(status.NodeOverviews, result.nodeName) } } } @@ -447,6 +456,9 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo if pubKey := node.Annotations[controller.WireGuardPubKeyAnnotation]; pubKey != "" { if nodeStatus.NodeInfo.WireGuard == nil { nodeStatus.NodeInfo.WireGuard = &WireGuardStatusInfo{} + } else { + wireguard := *nodeStatus.NodeInfo.WireGuard + nodeStatus.NodeInfo.WireGuard = &wireguard } nodeStatus.NodeInfo.WireGuard.PublicKey = pubKey @@ -808,6 +820,18 @@ func collectClusterProblems(status *ClusterStatusResponse) []StatusProblem { appendProblem("node", nodeName, summary) } + if overview := status.NodeOverviews[node.NodeInfo.Name]; overview != nil { + if overview.RouteMismatch { + appendProblem("node", nodeName, "Route next-hop mismatches (expected vs present)") + } + + if unhealthy := overview.PeerCount - overview.HealthyPeers; unhealthy > 0 { + appendProblem("node", nodeName, fmt.Sprintf("%d peers are not healthy", unhealthy)) + } + + continue + } + if mismatchCount := routeMismatchCount(node); mismatchCount > 0 { appendProblem("node", nodeName, fmt.Sprintf("%d route next-hop mismatches (expected vs present)", mismatchCount)) } diff --git a/cmd/unbounded-net-controller/cluster_status_cache.go b/cmd/unbounded-net-controller/cluster_status_cache.go index 6a9b5d13e..33437a4a8 100644 --- a/cmd/unbounded-net-controller/cluster_status_cache.go +++ b/cmd/unbounded-net-controller/cluster_status_cache.go @@ -5,6 +5,7 @@ package main import ( "context" + "maps" "reflect" "slices" "sync" @@ -15,6 +16,8 @@ import ( unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" "github.com/Azure/unbounded/internal/net/controller" + statuspkg "github.com/Azure/unbounded/internal/net/status" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) // ClusterStatusCache maintains a pre-built ClusterStatusResponse in memory, @@ -115,6 +118,15 @@ func (c *ClusterStatusCache) Rebuild(ctx context.Context) { // PatchNode updates a single node's cached status in-place without a full // rebuild. func (c *ClusterStatusCache) PatchNode(nodeName string, nodeStatus NodeStatusResponse) { + c.patchNode(nodeName, nodeStatus, nil) +} + +// PatchOverview updates metadata and observed facts without collecting details. +func (c *ClusterStatusCache) PatchOverview(nodeName string, overview statusv1alpha1.NodeStatusOverview) { + c.patchNode(nodeName, statuspkg.OverviewMetadata(overview), &overview) +} + +func (c *ClusterStatusCache) patchNode(nodeName string, nodeStatus NodeStatusResponse, overview *statusv1alpha1.NodeStatusOverview) { now := time.Now() nodeStatus.NodeInfo.ExternalIPs = c.resolveNodeExternalIPs(nodeName, now) @@ -125,6 +137,16 @@ func (c *ClusterStatusCache) PatchNode(nodeName string, nodeStatus NodeStatusRes return } + if overview == nil { + delete(c.status.NodeOverviews, nodeName) + } else { + if c.status.NodeOverviews == nil { + c.status.NodeOverviews = make(map[string]*statusv1alpha1.NodeStatusOverview) + } + + c.status.NodeOverviews[nodeName] = overview + } + if i, ok := c.nodeIndex[nodeName]; ok && i < len(c.status.Nodes) { // Preserve controller-enriched fields across node-agent status updates. existing := c.status.Nodes[i] @@ -225,13 +247,21 @@ func (c *ClusterStatusCache) MarkFullRebuildNeeded() { } } -// Get returns the current pre-built status (read-locked, fast). +// Get snapshots mutable containers; nested node data remains immutable and shared. // Returns nil if the status has not been built yet. func (c *ClusterStatusCache) Get() *ClusterStatusResponse { c.mu.RLock() defer c.mu.RUnlock() - return c.status + if c.status == nil { + return nil + } + + snapshot := *c.status + snapshot.Nodes = slices.Clone(c.status.Nodes) + snapshot.NodeOverviews = maps.Clone(c.status.NodeOverviews) + + return &snapshot } // GetSeq returns the current sequence number. diff --git a/cmd/unbounded-net-controller/node_overview.go b/cmd/unbounded-net-controller/node_overview.go new file mode 100644 index 000000000..fe5f79225 --- /dev/null +++ b/cmd/unbounded-net-controller/node_overview.go @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "time" + + statuspkg "github.com/Azure/unbounded/internal/net/status" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// StoreOverview replaces routine wire state without retaining diagnostic arrays. +func (c *NodeStatusCache) StoreOverview(nodeName string, overview statusv1alpha1.NodeStatusOverview, source string) (uint64, error) { + if nodeName == "" || (overview.NodeInfo.Name != "" && overview.NodeInfo.Name != nodeName) { + return 0, fmt.Errorf("summary identity does not match node %q", nodeName) + } + + if overview.PeerCount < 0 || overview.HealthyPeers < 0 || + overview.HealthyPeers > overview.PeerCount || overview.RouteCount < 0 { + return 0, fmt.Errorf("summary contains invalid observed counts") + } + + overview.NodeInfo.Name = nodeName + + if source == "" { + source = "push" + } + + overview.StatusSource = source + metadata := statuspkg.OverviewMetadata(overview) + + c.mu.Lock() + + revision := uint64(1) + if previous := c.entries[nodeName]; previous != nil { + revision = previous.Revision + 1 + } + + c.entries[nodeName] = &CachedNodeStatus{ + Status: &metadata, Overview: &overview, Source: source, + Revision: revision, ReceivedAt: time.Now(), + } + fn := c.onOverviewChange + c.mu.Unlock() + + if fn != nil { + fn(nodeName, overview) + } + + return revision, nil +} + +// SetOnOverviewChange registers the summary-only cache mutation callback. +func (c *NodeStatusCache) SetOnOverviewChange(fn func(string, statusv1alpha1.NodeStatusOverview)) { + c.mu.Lock() + defer c.mu.Unlock() + + c.onOverviewChange = fn +} diff --git a/cmd/unbounded-net-controller/node_overview_test.go b/cmd/unbounded-net-controller/node_overview_test.go new file mode 100644 index 000000000..7b5734891 --- /dev/null +++ b/cmd/unbounded-net-controller/node_overview_test.go @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestNodeOverviewCacheReplacesOnlyRoutineState(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node", NodeStatusResponse{Peers: []WireGuardPeerStatus{{Name: "peer"}}}, "push") + + var notified statusv1alpha1.NodeStatusOverview + + cache.SetOnOverviewChange(func(name string, overview statusv1alpha1.NodeStatusOverview) { + if name != "node" || cache.Len() != 1 { + t.Error("notification has wrong identity or ran under the cache lock") + } + + notified = overview + }) + overview := statusv1alpha1.NodeStatusOverview{ + PeerCount: 20, HealthyPeers: 17, RouteCount: 30, RouteMismatch: true, + NodeErrors: []NodeError{{Type: "cni", Message: "bootstrap blocked"}}, + } + + revision, err := cache.StoreOverview("node", overview, "ws") + if err != nil || revision != 2 { + t.Fatalf("store: revision=%d error=%v", revision, err) + } + + cached, ok := cache.Get("node") + if !ok || cached.Overview == nil || cached.Overview.PeerCount != 20 || cached.peerIdentity != nil { + t.Fatalf("unexpected overview wire state: %+v", cached) + } + + if cached.Status.Peers != nil || cached.Status.RoutingTable.Routes != nil || cached.Status.BpfEntries != nil { + t.Fatal("summary retained diagnostic arrays") + } + + cached.Overview.PeerCount = 99 + + unchanged, _ := cache.Get("node") + if unchanged.Overview.PeerCount != 20 { + t.Fatal("changing the returned overview mutated the cache") + } + + if notified.NodeInfo.Name != "node" || notified.StatusSource != "ws" || len(cached.Status.NodeErrors) != 1 { + t.Fatal("notification or metadata lost identity, source, or errors") + } + + rev, resync, err := cache.ApplyDelta("node", revision, map[string]json.RawMessage{}, "push") + if err != nil || !resync || rev != revision { + t.Fatalf("legacy delta must not apply to a summary: %d %v %v", rev, resync, err) + } + + snapshot := cache.GetAll() + cache.UpdateSource("node", "apiserver-ws") + + if snapshot["node"].Source != "ws" || notified.StatusSource != "apiserver-ws" { + t.Fatal("source change mutated an older snapshot or lost its notification") + } + + if next := cache.StoreFull("node", NodeStatusResponse{Peers: []WireGuardPeerStatus{{Name: "legacy"}}}, "push"); next != 3 { + t.Fatalf("legacy full resync revision=%d", next) + } + + legacy, _ := cache.Get("node") + if legacy.Overview != nil || len(legacy.Status.Peers) != 1 { + t.Fatal("explicit legacy full resync did not replace summary state") + } +} + +func TestNodeOverviewCacheRejectsInvalidFacts(t *testing.T) { + for _, overview := range []statusv1alpha1.NodeStatusOverview{ + {NodeInfo: NodeInfo{Name: "different-node"}}, + {PeerCount: -1}, + {HealthyPeers: -1}, + {PeerCount: 1, HealthyPeers: 2}, + {RouteCount: -1}, + } { + cache := NewNodeStatusCache() + if _, err := cache.StoreOverview("node", overview, "ws"); err == nil || cache.Len() != 0 { + t.Fatalf("invalid summary accepted: %+v", overview) + } + } + + if _, err := NewNodeStatusCache().StoreOverview("", statusv1alpha1.NodeStatusOverview{}, ""); err == nil { + t.Fatal("empty node identity accepted") + } +} + +func TestClusterOverviewPreservesCountsAndEnrichment(t *testing.T) { + c := NewClusterStatusCache(&healthState{}) + c.status = &ClusterStatusResponse{ + Nodes: []*NodeStatusResponse{{NodeInfo: NodeInfo{Name: "node", K8sReady: "Ready", ProviderID: "provider"}}}, + } + c.nodeIndex["node"] = 0 + overview := statusv1alpha1.NodeStatusOverview{ + NodeInfo: NodeInfo{Name: "node", SiteName: "site", WireGuard: &WireGuardStatusInfo{Interface: "wg0"}}, + StatusSource: "ws", PeerCount: 20, HealthyPeers: 17, RouteCount: 30, RouteMismatch: true, + } + c.PatchOverview("node", overview) + snapshot := c.Get() + + row := buildClusterSummary(snapshot).NodeSummaries[0] + if row.PeerCount != 20 || row.HealthyPeers != 17 || row.RouteCount != 30 || !row.RouteMismatch || + row.K8sReady != "Ready" || row.CniStatus != "Route mismatch" || row.SiteName != "site" { + t.Fatalf("summary lost observed facts or enriched fields: %+v", row) + } + + if snapshot.Nodes[0].NodeInfo.ProviderID != "provider" { + t.Fatal("controller enrichment was lost") + } + + problems := collectClusterProblems(snapshot) + if len(problems) != 1 || len(problems[0].Errors) != 2 { + t.Fatalf("summary health/mismatch problems were hidden: %+v", problems) + } + + overview.PeerCount = 25 + overview.NodeErrors = []NodeError{{Type: "cni", Message: "blocked"}} + c.PatchOverview("node", overview) + + if buildClusterSummary(snapshot).NodeSummaries[0] != row { + t.Fatal("patching changed a previously returned snapshot") + } + + nextRow := buildClusterSummary(c.Get()).NodeSummaries[0] + if nextRow.PeerCount != 25 || nextRow.FirstError != "blocked" || nextRow.CniTone != "danger" { + t.Fatalf("summary update lost errors or counts: %+v", nextRow) + } + + c.PatchNode("node", NodeStatusResponse{NodeInfo: overview.NodeInfo, Peers: []WireGuardPeerStatus{{}}}) + + if legacy := buildClusterSummary(c.Get()).NodeSummaries[0]; legacy.PeerCount != 1 { + t.Fatal("legacy update retained stale explicit summary counts") + } +} + +func TestClusterOverviewWireIgnoresDiagnosticArrays(t *testing.T) { + node := &NodeStatusResponse{NodeInfo: NodeInfo{Name: "node"}} + status := &ClusterStatusResponse{ + Nodes: []*NodeStatusResponse{node}, + NodeOverviews: map[string]*statusv1alpha1.NodeStatusOverview{ + "node": {PeerCount: 5, HealthyPeers: 4, RouteCount: 9}, + }, + } + + before, err := json.Marshal(buildClusterSummary(status)) + if err != nil { + t.Fatal(err) + } + + node.Peers = make([]WireGuardPeerStatus, 10000) + node.RoutingTable.Routes = make([]RouteEntry, 10000) + node.BpfEntries = make([]BpfEntry, 10000) + + after, err := json.Marshal(buildClusterSummary(status)) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(before, after) { + t.Fatal("overview wire size or facts depend on diagnostic arrays") + } + + for _, field := range []string{`"peers":`, `"routingTable":`, `"bpfEntries":`, `"NodeOverviews":`} { + if strings.Contains(string(after), field) { + t.Fatalf("overview exposed %s", field) + } + } +} + +func TestClusterOverviewConcurrentSnapshots(t *testing.T) { + c := NewClusterStatusCache(&healthState{}) + c.status = &ClusterStatusResponse{} + + var wg sync.WaitGroup + for range 4 { + wg.Go(func() { + for range 100 { + c.PatchOverview("node", statusv1alpha1.NodeStatusOverview{NodeInfo: NodeInfo{Name: "node"}, Timestamp: time.Now()}) + buildClusterSummary(c.Get()) + } + }) + } + + wg.Wait() +} diff --git a/cmd/unbounded-net-controller/node_status.go b/cmd/unbounded-net-controller/node_status.go index 7fb9b09b1..eca669423 100644 --- a/cmd/unbounded-net-controller/node_status.go +++ b/cmd/unbounded-net-controller/node_status.go @@ -12,6 +12,7 @@ import ( "time" statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) // CachedNodeStatus stores a node's pushed status with timestamp and revision. @@ -20,15 +21,17 @@ type CachedNodeStatus struct { ReceivedAt time.Time Source string Revision uint64 + Overview *statusv1alpha1.NodeStatusOverview peerIdentity *peerIdentityDigest } // NodeStatusCache is a thread-safe cache of node status data pushed from node agents. type NodeStatusCache struct { - mu sync.RWMutex - entries map[string]*CachedNodeStatus - onChange func(nodeName string, status *NodeStatusResponse) + mu sync.RWMutex + entries map[string]*CachedNodeStatus + onChange func(nodeName string, status *NodeStatusResponse) + onOverviewChange func(nodeName string, overview statusv1alpha1.NodeStatusOverview) } // NewNodeStatusCache creates an empty NodeStatusCache. @@ -230,7 +233,7 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, return 0, true, nil } - if (pd.peerMeasurements != nil && baseRevision == 0) || (baseRevision != 0 && entry.Revision != baseRevision) { + if entry.Overview != nil || (pd.peerMeasurements != nil && baseRevision == 0) || (baseRevision != 0 && entry.Revision != baseRevision) { rev := entry.Revision c.mu.RUnlock() @@ -399,6 +402,11 @@ func (c *NodeStatusCache) Get(nodeName string) (*CachedNodeStatus, bool) { copy := *entry copy.Status = &statusCopy + if entry.Overview != nil { + overviewCopy := *entry.Overview + copy.Overview = &overviewCopy + } + return ©, true } @@ -445,12 +453,19 @@ func (c *NodeStatusCache) UpdateSource(nodeName, source string) bool { return true } - entry.Source = source + updated := *entry + updated.Source = source + c.entries[nodeName] = &updated fn := c.onChange + overviewFn := c.onOverviewChange statusCopy := entry.Status c.mu.Unlock() - if fn != nil { + if entry.Overview != nil && overviewFn != nil { + overview := *entry.Overview + overview.StatusSource = source + overviewFn(nodeName, overview) + } else if fn != nil { fn(nodeName, statusCopy) } diff --git a/cmd/unbounded-net-controller/status_types.go b/cmd/unbounded-net-controller/status_types.go index e6145a30f..c5efb89f7 100644 --- a/cmd/unbounded-net-controller/status_types.go +++ b/cmd/unbounded-net-controller/status_types.go @@ -15,21 +15,22 @@ import ( // ClusterStatusResponse is the top-level status response for the cluster. type ClusterStatusResponse struct { - Seq uint64 `json:"seq"` - Timestamp time.Time `json:"timestamp"` - NodeCount int `json:"nodeCount"` - SiteCount int `json:"siteCount"` - AzureTenantID string `json:"azureTenantId,omitempty"` - LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` - BuildInfo *BuildInfo `json:"buildInfo,omitempty"` - Nodes []*NodeStatusResponse `json:"nodes"` - Sites []SiteStatus `json:"sites"` - GatewayPools []GatewayPoolStatus `json:"gatewayPools"` - Peerings []PeeringStatus `json:"peerings"` - Errors []string `json:"errors,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Problems []StatusProblem `json:"problems"` - PullEnabled bool `json:"pullEnabled"` + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + NodeCount int `json:"nodeCount"` + SiteCount int `json:"siteCount"` + AzureTenantID string `json:"azureTenantId,omitempty"` + LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` + BuildInfo *BuildInfo `json:"buildInfo,omitempty"` + Nodes []*NodeStatusResponse `json:"nodes"` + Sites []SiteStatus `json:"sites"` + GatewayPools []GatewayPoolStatus `json:"gatewayPools"` + Peerings []PeeringStatus `json:"peerings"` + Errors []string `json:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Problems []StatusProblem `json:"problems"` + PullEnabled bool `json:"pullEnabled"` + NodeOverviews map[string]*statusv1alpha1.NodeStatusOverview `json:"-"` } // ClusterStatusDelta is a WebSocket delta update. @@ -184,14 +185,20 @@ type NodeSummary struct { } // buildClusterSummary extracts a ClusterSummary from a full ClusterStatusResponse. -// Legacy payloads require scanning peers and route next hops for overview facts. +// Only legacy payloads require scanning peers and route next hops. func buildClusterSummary(status *ClusterStatusResponse) *ClusterSummary { summaries := make([]NodeSummary, 0, len(status.Nodes)) now := time.Now() for i := range status.Nodes { node := status.Nodes[i] - overview := statuspkg.OverviewFromStatus(node, now) + + overview := status.NodeOverviews[node.NodeInfo.Name] + if overview == nil { + projected := statuspkg.OverviewFromStatus(node, now) + overview = &projected + } + ns := NodeSummary{ Name: node.NodeInfo.Name, SiteName: node.NodeInfo.SiteName, From f61e8949ac1dd55c4bf670c95b3dfe2cd6a916e8 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:39:41 +0000 Subject: [PATCH 10/34] net-controller: ingest authenticated summary status on both transports Accept summary-only protobuf and JSON over WebSocket and HTTP, validate nested identity and observed counts, reject mixed detail payloads, and advertise summary capability in acknowledgments. Patch overview cache and notify viewers without retaining diagnostic arrays. Preserve legacy publication formats. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-controller/server.go | 81 ++++++- .../status_overview_ingestion_test.go | 197 ++++++++++++++++++ .../status_overview_proto.go | 36 ++++ cmd/unbounded-net-controller/status_proto.go | 58 +++++- cmd/unbounded-net-controller/status_types.go | 28 +-- 5 files changed, 365 insertions(+), 35 deletions(-) create mode 100644 cmd/unbounded-net-controller/status_overview_ingestion_test.go create mode 100644 cmd/unbounded-net-controller/status_overview_proto.go diff --git a/cmd/unbounded-net-controller/server.go b/cmd/unbounded-net-controller/server.go index c706ca95e..c71e8b0e7 100644 --- a/cmd/unbounded-net-controller/server.go +++ b/cmd/unbounded-net-controller/server.go @@ -27,6 +27,7 @@ import ( "github.com/Azure/unbounded/internal/net/html" "github.com/Azure/unbounded/internal/net/metrics" statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" webhookpkg "github.com/Azure/unbounded/internal/net/webhook" ) @@ -58,6 +59,9 @@ type nodeStatusWSIdentity struct { Status *struct { NodeInfo nodeStatusIdentityInfo `json:"nodeInfo"` } `json:"status"` + Summary *struct { + NodeInfo nodeStatusIdentityInfo `json:"nodeInfo"` + } `json:"summary"` Delta map[string]json.RawMessage `json:"delta"` } @@ -76,6 +80,10 @@ func extractNodeNameFromWSMessage(data []byte) (string, error) { nodeNames = append(nodeNames, identity.Status.NodeInfo.Name) } + if identity.Summary != nil { + nodeNames = append(nodeNames, identity.Summary.NodeInfo.Name) + } + // Match ApplyDelta's case-sensitive map lookup, not struct field matching. if raw, ok := identity.Delta["nodeInfo"]; ok { var info nodeStatusIdentityInfo @@ -96,10 +104,10 @@ func rejectDuplicateStatusIdentityFields(data []byte, object string) error { return nil } - fields := []string{"nodeName", "nodeInfo", "status", "delta"} + fields := []string{"nodeName", "nodeInfo", "status", "delta", "summary"} switch object { - case "status", "delta": + case "status", "delta", "summary": fields = []string{"nodeInfo"} case "nodeInfo": fields = []string{"name"} @@ -147,7 +155,7 @@ func rejectDuplicateStatusIdentityFields(data []byte, object string) error { return fmt.Errorf("invalid status identity value: %w", err) } - if matched == "status" || matched == "delta" || matched == "nodeInfo" { + if matched == "status" || matched == "delta" || matched == "nodeInfo" || matched == "summary" { if err := rejectDuplicateStatusIdentityFields(value, matched); err != nil { return err } @@ -233,6 +241,11 @@ func startServer(ctx context.Context, healthPort int, requireDashboardAuth bool, clusterStatusCache.MarkDirty() broadcaster.Notify() }) + health.statusCache.SetOnOverviewChange(func(nodeName string, overview statusv1alpha1.NodeStatusOverview) { + clusterStatusCache.PatchOverview(nodeName, overview) + clusterStatusCache.MarkDirty() + broadcaster.Notify() + }) // Node WebSocket connection semaphore. wsSemaphore := make(chan struct{}, maxConcurrentNodeWS) @@ -1172,7 +1185,9 @@ func handleStatusPushBody(health *healthState, r *http.Request, bodyBytes []byte return handleStatusPushRequestWithSource(health, bodyBytes, source) } -func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, source string) (NodeStatusPushAck, int, error) { +func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, source string) (ack NodeStatusPushAck, code int, err error) { + defer func() { ack.SummarySupported = true }() + if _, err := extractNodeNameFromWSMessage(bodyBytes); err != nil { return NodeStatusPushAck{}, http.StatusBadRequest, err } @@ -1182,7 +1197,23 @@ func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, so return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("invalid request body: %v", err) } - ack := NodeStatusPushAck{Status: "ok"} + ack = NodeStatusPushAck{Status: "ok"} + + if envelope.Type == statusv1alpha1.NodeStatusSummaryType { + if envelope.Mode != "" && envelope.Mode != "summary" { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("conflicting status mode and type") + } + + envelope.Mode = "summary" + } + + if envelope.Summary != nil && envelope.Mode != "summary" { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("overview requires summary mode") + } + + if envelope.Mode == "summary" && envelope.Type != "" && envelope.Type != statusv1alpha1.NodeStatusSummaryType { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("conflicting status mode and type") + } if envelope.Mode == "" { var nodeStatus NodeStatusResponse @@ -1205,11 +1236,26 @@ func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, so nodeName = envelope.Status.NodeInfo.Name } + if envelope.Summary != nil && envelope.Summary.NodeInfo.Name != "" { + nodeName = envelope.Summary.NodeInfo.Name + } + if nodeName == "" { return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("nodeName is required") } switch envelope.Mode { + case "summary": + if envelope.Summary == nil || envelope.Status != nil || envelope.Delta != nil || envelope.DetailRequestID != "" { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("summary must contain only overview data") + } + + ack.Revision, err = health.statusCache.StoreOverview(nodeName, *envelope.Summary, source) + if err != nil { + return NodeStatusPushAck{}, http.StatusBadRequest, err + } + + return ack, http.StatusOK, nil case "full": if envelope.Status == nil { return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("status is required for full mode") @@ -1242,7 +1288,7 @@ func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, so return ack, http.StatusOK, nil default: - return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("mode must be full or delta") + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("unsupported status mode %q", envelope.Mode) } } @@ -1250,7 +1296,9 @@ func handleNodeStatusWSMessage(health *healthState, data []byte) (string, NodeSt return handleNodeStatusWSMessageWithSource(health, data, "ws") } -func handleNodeStatusWSMessageWithSource(health *healthState, data []byte, source string) (string, NodeStatusPushAck) { +func handleNodeStatusWSMessageWithSource(health *healthState, data []byte, source string) (ackType string, ack NodeStatusPushAck) { + defer func() { ack.SummarySupported = true }() + if _, err := extractNodeNameFromWSMessage(data); err != nil { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} } @@ -1260,16 +1308,35 @@ func handleNodeStatusWSMessageWithSource(health *healthState, data []byte, sourc return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "invalid message"} } + if message.Summary != nil && message.Type != statusv1alpha1.NodeStatusSummaryType { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "overview requires summary message type"} + } + nodeName := message.NodeName if message.Status != nil && message.Status.NodeInfo.Name != "" { nodeName = message.Status.NodeInfo.Name } + if message.Summary != nil && message.Summary.NodeInfo.Name != "" { + nodeName = message.Summary.NodeInfo.Name + } + if nodeName == "" { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "nodeName is required"} } switch message.Type { + case statusv1alpha1.NodeStatusSummaryType: + if message.Summary == nil || message.Status != nil || message.Delta != nil || message.DetailRequestID != "" { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "summary must contain only overview data"} + } + + revision, err := health.statusCache.StoreOverview(nodeName, *message.Summary, source) + if err != nil { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} + } + + return "node_status_ack", NodeStatusPushAck{Status: "ok", Revision: revision} case "node_status_full": if message.Status == nil { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "full message missing status"} diff --git a/cmd/unbounded-net-controller/status_overview_ingestion_test.go b/cmd/unbounded-net-controller/status_overview_ingestion_test.go new file mode 100644 index 000000000..87cddf225 --- /dev/null +++ b/cmd/unbounded-net-controller/status_overview_ingestion_test.go @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net/http" + "testing" + "time" + + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func submitOverview(t *testing.T, channel string, health *healthState, message *statusproto.NodeStatusMessage) NodeStatusPushAck { + t.Helper() + + var ( + data []byte + err error + ) + if channel == "proto-http" || channel == "proto-ws" { + data, err = proto.Marshal(message) + } else { + envelope := NodeStatusWSMessage{ + Type: message.Type, NodeName: message.NodeName, DetailRequestID: message.DetailRequestId, + } + if message.Summary != nil { + overview := protoToNodeOverview(message.Summary) + envelope.Summary = &overview + } + + if message.Status != nil { + status := protoToNodeStatus(message.Status) + envelope.Status = &status + } + + if message.Delta != nil { + envelope.Delta = map[string]json.RawMessage{"timestamp": json.RawMessage(`null`)} + } + + data, err = json.Marshal(envelope) + } + + if err != nil { + t.Fatal(err) + } + + switch channel { + case "proto-http": + ack, code, err := handleProtoPushRequest(health, data, "push") + if err != nil || code != http.StatusOK { + return NodeStatusPushAck{Status: "rejected"} + } + + return ack + case "json-http": + ack, code, err := handleStatusPushRequestWithSource(health, data, "push") + if err != nil || code != http.StatusOK { + return NodeStatusPushAck{Status: "rejected"} + } + + return ack + case "proto-ws": + decoded, err := decodeProtoWSMessage(data) + if err != nil { + return NodeStatusPushAck{Status: "rejected"} + } + + _, ack := handleProtoWSMessage(health, decoded, "ws") + + return ack + case "json-ws": + _, ack := handleNodeStatusWSMessageWithSource(health, data, "ws") + return ack + default: + t.Fatalf("unknown test channel %s", channel) + return NodeStatusPushAck{} + } +} + +func TestOverviewIngestionAllChannels(t *testing.T) { + for _, channel := range []string{"proto-http", "json-http", "proto-ws", "json-ws"} { + t.Run(channel, func(t *testing.T) { + health := &healthState{statusCache: NewNodeStatusCache()} + message := &statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusSummaryType, NodeName: "node", + Summary: &statusproto.NodeStatusOverview{ + TimestampUnixNs: 1000, LastPushTimeUnixNs: 2000, + NodeInfo: &statusproto.NodeInfo{ + Name: "node", SiteName: "site", WireGuard: &statusproto.WireGuardStatusInfo{Interface: "wg0"}, + }, + PeerCount: 10, HealthyPeers: 8, RouteCount: 20, RouteMismatch: true, + NodeErrors: []*statusproto.NodeError{{Type: "cni", Message: "blocked"}}, + HealthCheck: &statusproto.HealthCheckStatus{Summary: "not healthy"}, + NodePodInfo: &statusproto.NodePodInfo{PodName: "agent"}, + }, + } + + ack := submitOverview(t, channel, health, message) + if ack.Status != "ok" || ack.Revision != 1 || !ack.SummarySupported { + t.Fatalf("unexpected ACK: %+v", ack) + } + + entry, ok := health.statusCache.Get("node") + if !ok || entry.Overview == nil { + t.Fatal("overview was not stored") + } + + overview := entry.Overview + if overview.PeerCount != 10 || overview.HealthyPeers != 8 || overview.RouteCount != 20 || !overview.RouteMismatch || + overview.NodeErrors[0].Message != "blocked" || overview.NodeInfo.SiteName != "site" || + overview.NodeInfo.WireGuard.Interface != "wg0" || overview.NodePodInfo.PodName != "agent" || + overview.HealthCheck.Summary != "not healthy" || + !overview.Timestamp.Equal(time.Unix(0, 1000)) || !overview.LastPushTime.Equal(time.Unix(0, 2000)) { + t.Fatalf("overview changed during ingestion: %+v", overview) + } + + if entry.Status.Peers != nil || entry.Status.RoutingTable.Routes != nil || entry.Status.BpfEntries != nil { + t.Fatal("summary retained diagnostic arrays") + } + + if next := submitOverview(t, channel, health, message); next.Revision != 2 { + t.Fatal("summary resync did not advance routine revision") + } + }) + } +} + +func TestOverviewIngestionRejectsInvalidEnvelopes(t *testing.T) { + for _, channel := range []string{"proto-http", "json-http", "proto-ws", "json-ws"} { + for _, tc := range []struct { + name string + mutate func(*statusproto.NodeStatusMessage) + }{ + {"missing", func(m *statusproto.NodeStatusMessage) { m.Summary = nil }}, + {"identity mismatch", func(m *statusproto.NodeStatusMessage) { m.Summary.NodeInfo.Name = "other" }}, + {"negative counts", func(m *statusproto.NodeStatusMessage) { m.Summary.PeerCount = -1 }}, + {"impossible counts", func(m *statusproto.NodeStatusMessage) { m.Summary.HealthyPeers = 1 }}, + {"full mixed with summary", func(m *statusproto.NodeStatusMessage) { m.Status = &statusproto.NodeStatusFull{} }}, + {"delta mixed with summary", func(m *statusproto.NodeStatusMessage) { m.Delta = &statusproto.NodeStatusDelta{} }}, + {"detail correlation on summary", func(m *statusproto.NodeStatusMessage) { m.DetailRequestId = "request" }}, + {"summary in full", func(m *statusproto.NodeStatusMessage) { m.Type = "node_status_full" }}, + } { + t.Run(channel+"/"+tc.name, func(t *testing.T) { + health := &healthState{statusCache: NewNodeStatusCache()} + message := &statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusSummaryType, NodeName: "node", + Summary: &statusproto.NodeStatusOverview{NodeInfo: &statusproto.NodeInfo{Name: "node"}}, + } + tc.mutate(message) + + ack := submitOverview(t, channel, health, message) + if ack.Status == "ok" || health.statusCache.Len() != 0 { + t.Fatalf("invalid summary accepted: %+v", ack) + } + }) + } + } +} + +func TestOverviewIdentityRejectsDuplicateAndConflictingFields(t *testing.T) { + for _, data := range []string{ + `{"nodeName":"node","summary":{"nodeInfo":{"name":"other"}}}`, + `{"nodeName":"node","summary":{"nodeInfo":{"name":"other"}},"Summary":null}`, + `{"summary":{"nodeInfo":{"name":"other","Name":"node"}}}`, + `{"summary":{"nodeInfo":{"name":"other"},"NodeInfo":{"name":"node"}}}`, + } { + if _, err := extractNodeNameFromWSMessage([]byte(data)); err == nil { + t.Fatalf("ambiguous summary identity accepted: %s", data) + } + } + + name, err := extractNodeNameFromWSMessage([]byte(`{"summary":{"nodeInfo":{"name":"node"}}}`)) + if err != nil || name != "node" { + t.Fatalf("summary-only identity lost: %q %v", name, err) + } +} + +func TestOverviewCapabilityProtoAck(t *testing.T) { + data, err := marshalProtoAck("node_status_ack", NodeStatusPushAck{Status: "ok", Revision: 7}) + if err != nil { + t.Fatal(err) + } + + var ack statusproto.NodeStatusAck + if err := proto.Unmarshal(data, &ack); err != nil { + t.Fatal(err) + } + + if !ack.SummarySupported || !ack.PeerMeasurements || ack.Revision != 7 { + t.Fatalf("ACK lost capability or revision: %v", &ack) + } +} diff --git a/cmd/unbounded-net-controller/status_overview_proto.go b/cmd/unbounded-net-controller/status_overview_proto.go new file mode 100644 index 000000000..04613df38 --- /dev/null +++ b/cmd/unbounded-net-controller/status_overview_proto.go @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "time" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func protoToNodeOverview(msg *statusproto.NodeStatusOverview) statusv1alpha1.NodeStatusOverview { + overview := statusv1alpha1.NodeStatusOverview{ + HealthCheck: protoToHealthCheckStatus(msg.HealthCheck), + NodeErrors: protoToNodeErrors(msg.NodeErrors), + FetchError: msg.FetchError, StatusSource: msg.StatusSource, + NodePodInfo: protoToNodePodInfo(msg.NodePodInfo), + PeerCount: int(msg.PeerCount), HealthyPeers: int(msg.HealthyPeers), + RouteCount: int(msg.RouteCount), RouteMismatch: msg.RouteMismatch, + } + if msg.NodeInfo != nil { + overview.NodeInfo = protoToNodeInfo(msg.NodeInfo) + } + + if msg.TimestampUnixNs != 0 { + overview.Timestamp = time.Unix(0, msg.TimestampUnixNs) + } + + if msg.LastPushTimeUnixNs != 0 { + lastPush := time.Unix(0, msg.LastPushTimeUnixNs) + overview.LastPushTime = &lastPush + } + + return overview +} diff --git a/cmd/unbounded-net-controller/status_proto.go b/cmd/unbounded-net-controller/status_proto.go index cb10196b6..40c91f4e9 100644 --- a/cmd/unbounded-net-controller/status_proto.go +++ b/cmd/unbounded-net-controller/status_proto.go @@ -9,6 +9,7 @@ import ( "google.golang.org/protobuf/proto" + statuspkg "github.com/Azure/unbounded/internal/net/status" statusproto "github.com/Azure/unbounded/internal/net/status/proto" statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) @@ -468,20 +469,41 @@ func validatedProtoNodeName(msg *statusproto.NodeStatusMessage) (string, error) nodeNames = append(nodeNames, msg.Delta.NodeInfo.Name) } + if msg.Summary != nil && msg.Summary.NodeInfo != nil { + nodeNames = append(nodeNames, msg.Summary.NodeInfo.Name) + } + return validatedNodeNames(nodeNames) } // handleProtoWSMessage applies the same decoded message used for authorization. -func handleProtoWSMessage(health *healthState, decoded *decodedProtoWSMessage, source string) (string, NodeStatusPushAck) { +func handleProtoWSMessage(health *healthState, decoded *decodedProtoWSMessage, source string) (ackType string, ack NodeStatusPushAck) { + defer func() { ack.SummarySupported = true }() + msg := &decoded.message nodeName := decoded.nodeName + if msg.Summary != nil && msg.Type != statusv1alpha1.NodeStatusSummaryType { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "overview requires summary message type"} + } + if msg.Delta.GetPeerMeasurements() != nil && (msg.Type != "node_status_delta" || msg.Status != nil) { peerMeasurementUpdatesTotal.WithLabelValues("error").Inc() return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "full status conflicts with measurements"} } switch msg.Type { + case statusv1alpha1.NodeStatusSummaryType: + if msg.Summary == nil || msg.Status != nil || msg.Delta != nil || msg.DetailRequestId != "" { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "summary must contain only overview data"} + } + + revision, err := health.statusCache.StoreOverview(nodeName, protoToNodeOverview(msg.Summary), source) + if err != nil { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} + } + + return "node_status_ack", NodeStatusPushAck{Status: "ok", Revision: revision} case "node_status_full": if msg.Status == nil { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "full message missing status"} @@ -518,7 +540,9 @@ func handleProtoWSMessage(health *healthState, decoded *decodedProtoWSMessage, s } // handleProtoPushRequest processes an HTTP push request with protobuf body. -func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string) (NodeStatusPushAck, int, error) { +func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string) (ack NodeStatusPushAck, code int, err error) { + defer func() { ack.SummarySupported = true }() + var msg statusproto.NodeStatusMessage if err := proto.Unmarshal(bodyBytes, &msg); err != nil { return NodeStatusPushAck{}, 400, fmt.Errorf("invalid protobuf body: %v", err) @@ -533,13 +557,29 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string return NodeStatusPushAck{}, 400, fmt.Errorf("nodeName is required") } - ack := NodeStatusPushAck{Status: "ok"} + ack = NodeStatusPushAck{Status: "ok"} + + if msg.Summary != nil && msg.Type != statusv1alpha1.NodeStatusSummaryType { + return NodeStatusPushAck{}, 400, fmt.Errorf("overview requires summary message type") + } + if msg.Delta.GetPeerMeasurements() != nil && (msg.Type != "node_status_delta" || msg.Status != nil) { peerMeasurementUpdatesTotal.WithLabelValues("error").Inc() return NodeStatusPushAck{Status: "resync_required", Reason: "full status conflicts with measurements"}, 429, nil } switch msg.Type { + case statusv1alpha1.NodeStatusSummaryType: + if msg.Summary == nil || msg.Status != nil || msg.Delta != nil || msg.DetailRequestId != "" { + return NodeStatusPushAck{}, 400, fmt.Errorf("summary must contain only overview data") + } + + ack.Revision, err = health.statusCache.StoreOverview(nodeName, protoToNodeOverview(msg.Summary), source) + if err != nil { + return NodeStatusPushAck{}, 400, err + } + + return ack, 200, nil case "node_status_full": if msg.Status == nil { return NodeStatusPushAck{}, 400, fmt.Errorf("status is required for full mode") @@ -573,18 +613,14 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string return ack, 200, nil default: - return NodeStatusPushAck{}, 400, fmt.Errorf("type must be node_status_full or node_status_delta") + return NodeStatusPushAck{}, 400, fmt.Errorf("unsupported status message type %q", msg.Type) } } // marshalProtoAck serializes a NodeStatusPushAck into a protobuf NodeStatusAck. func marshalProtoAck(ackType string, ack NodeStatusPushAck) ([]byte, error) { - pbAck := &statusproto.NodeStatusAck{ - PeerMeasurements: true, - Status: ack.Status, - Revision: ack.Revision, - Reason: ack.Reason, - } + ack.PeerMeasurements = true + ack.SummarySupported = true - return proto.Marshal(pbAck) + return proto.Marshal(statuspkg.NodeStatusAckToProto(&ack)) } diff --git a/cmd/unbounded-net-controller/status_types.go b/cmd/unbounded-net-controller/status_types.go index c5efb89f7..2c3cf41d9 100644 --- a/cmd/unbounded-net-controller/status_types.go +++ b/cmd/unbounded-net-controller/status_types.go @@ -77,28 +77,22 @@ type NodeStatusResponse = statusv1alpha1.NodeStatusResponse // NodeStatusPushEnvelope carries a push status update from a node. type NodeStatusPushEnvelope struct { - Mode string `json:"mode,omitempty"` - NodeName string `json:"nodeName,omitempty"` - BaseRevision uint64 `json:"baseRevision,omitempty"` - Status *NodeStatusResponse `json:"status,omitempty"` - Delta map[string]json.RawMessage `json:"delta,omitempty"` + Type string `json:"type,omitempty"` + Mode string `json:"mode,omitempty"` + NodeName string `json:"nodeName,omitempty"` + BaseRevision uint64 `json:"baseRevision,omitempty"` + Status *NodeStatusResponse `json:"status,omitempty"` + Delta map[string]json.RawMessage `json:"delta,omitempty"` + Summary *statusv1alpha1.NodeStatusOverview `json:"summary,omitempty"` + DetailRequestID string `json:"detailRequestId,omitempty"` + SupportsDetails bool `json:"supportsDetails,omitempty"` } // NodeStatusPushAck is the acknowledgment returned for push updates. -type NodeStatusPushAck struct { - Status string `json:"status"` - Revision uint64 `json:"revision,omitempty"` - Reason string `json:"reason,omitempty"` -} +type NodeStatusPushAck = statusv1alpha1.NodeStatusAck // NodeStatusWSMessage is the status message format used over WebSockets. -type NodeStatusWSMessage struct { - Type string `json:"type"` - NodeName string `json:"nodeName,omitempty"` - BaseRevision uint64 `json:"baseRevision,omitempty"` - Status *NodeStatusResponse `json:"status,omitempty"` - Delta map[string]json.RawMessage `json:"delta,omitempty"` -} +type NodeStatusWSMessage = statusv1alpha1.NodeStatusMessage // NodePodInfo aliases the shared node pod status schema. type NodePodInfo = statusv1alpha1.NodePodInfo From bc80e9380ccd491be64662c4b7b20a0975413303 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:22:28 +0000 Subject: [PATCH 11/34] refactor(net-node): share streamed peer and kernel route inspection Separate detail snapshot construction from peer visitation and kernel route filtering so summary collection can reuse inspection without retaining detail arrays. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-node/status_server.go | 151 ++++++++++++++---------- 1 file changed, 86 insertions(+), 65 deletions(-) diff --git a/cmd/unbounded-net-node/status_server.go b/cmd/unbounded-net-node/status_server.go index 5472aa39c..7eed77a49 100644 --- a/cmd/unbounded-net-node/status_server.go +++ b/cmd/unbounded-net-node/status_server.go @@ -2389,8 +2389,16 @@ func (s *nodeStatusServer) startRouteChangeWatcher(ctx context.Context) { }() } -// getNodeStatus collects all status information about this node -func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { +type nodeStatusFacts struct { + Timestamp time.Time + NodeInfo NodeInfo + NodeErrors []NodeError + HealthCheck *HealthCheckStatus +} + +// inspectNodePeers visits peers without retaining an outbound peer array. +// The visitor must not acquire state.mu: non-WireGuard peers are visited under it. +func (s *nodeStatusServer) inspectNodePeers(visit func(WireGuardPeerStatus)) *nodeStatusFacts { // Snapshot state under the lock - copy all fields we need, then release. // Expensive operations (WireGuard GetDevice, collectRoutingTable) happen outside the lock. lockStart := time.Now() @@ -2398,7 +2406,7 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { s.state.mu.Lock() lockWait := time.Since(lockStart) - status := &NodeStatusResponse{ + status := &nodeStatusFacts{ Timestamp: time.Now(), NodeInfo: NodeInfo{ Name: s.cfg.NodeName, @@ -2608,7 +2616,7 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } } - status.Peers = append(status.Peers, peer) + visit(peer) } } } @@ -2663,7 +2671,8 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } } - status.Peers = append(status.Peers, peer) + visit(peer) + addedPeerNames[gw.gatewayName] = true } } @@ -2727,7 +2736,7 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } } - status.Peers = append(status.Peers, peer) + visit(peer) } for _, gp := range s.state.gatewayPeers { @@ -2782,10 +2791,35 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } } - status.Peers = append(status.Peers, peer) + visit(peer) } s.state.mu.Unlock() + expensiveDuration := time.Since(expensiveStart) + + totalDuration := time.Since(lockStart) + if totalDuration > 2*time.Second { + klog.Warningf("inspectNodePeers() slow: total=%v (lock_wait=%v, snapshot=%v, expensive=%v)", + totalDuration, lockWait, snapshotDuration-lockWait, expensiveDuration) + } else { + klog.V(4).Infof("inspectNodePeers() timing: total=%v (lock_wait=%v, snapshot=%v, expensive=%v)", + totalDuration, lockWait, snapshotDuration-lockWait, expensiveDuration) + } + + return status +} + +// getNodeStatus collects all status information about this node. +func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { + status := &NodeStatusResponse{} + facts := s.inspectNodePeers(func(peer WireGuardPeerStatus) { + status.Peers = append(status.Peers, peer) + }) + status.Timestamp = facts.Timestamp + status.NodeInfo = facts.NodeInfo + status.NodeErrors = facts.NodeErrors + status.HealthCheck = facts.HealthCheck + sortStatusPeers(status.Peers) // Collect routing table from kernel via netlink @@ -2811,17 +2845,6 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { // Collect BPF trie entries. status.BpfEntries = s.collectBpfEntries() - expensiveDuration := time.Since(expensiveStart) - - totalDuration := time.Since(lockStart) - if totalDuration > 2*time.Second { - klog.Warningf("getNodeStatus() slow: total=%v (lock_wait=%v, snapshot=%v, expensive=%v)", - totalDuration, lockWait, snapshotDuration-lockWait, expensiveDuration) - } else { - klog.V(4).Infof("getNodeStatus() timing: total=%v (lock_wait=%v, snapshot=%v, expensive=%v)", - totalDuration, lockWait, snapshotDuration-lockWait, expensiveDuration) - } - return status } @@ -2957,6 +2980,37 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { s.routingTableCacheMu.RUnlock() + s.inspectKernelRoutes(func(family, destination string, table int, hops []observedNextHop) { + entry := RouteEntry{Destination: destination, Family: family, Table: table} + for _, hop := range hops { + entry.NextHops = append(entry.NextHops, NextHop{ + Gateway: hop.gateway, Device: hop.device, Distance: hop.distance, MTU: hop.mtu, + RouteTypes: []RouteType{{Type: "kernel", Attributes: []string{"fib"}}}, + }) + } + + info.Routes = append(info.Routes, entry) + }) + + s.routingTableCacheMu.Lock() + s.routingTableCache = info + s.routingTableCachedAt = time.Now() + s.routingTableCacheMu.Unlock() + s.routingTableDirty.Store(false) + + return info +} + +type observedNextHop struct { + gateway string + device string + distance int + mtu int +} + +// inspectKernelRoutes shares filtering and deduplication without constructing +// status routes, annotations, or a full-detail routing cache. +func (s *nodeStatusServer) inspectKernelRoutes(visit func(family, destination string, table int, hops []observedNextHop)) { // Build a set of managed route prefixes from the route manager so we can // include routes on non-tunnel interfaces (e.g. eth0 with tunnelProtocol: None). managedPrefixes := make(map[string]bool) @@ -2967,7 +3021,7 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { } } - collect := func(family int, familyLabel string) []RouteEntry { + collect := func(family int, familyLabel string) { // Collect routes from the main table and, if configured, our dedicated table. // RouteList(nil, family) only returns routes from the main table, so we // explicitly request routes from our dedicated table via RouteListFiltered. @@ -3023,7 +3077,7 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { type destEntry struct { destination string table int - nexthops map[nhKey]NextHop + nexthops map[nhKey]observedNextHop nhOrder []nhKey } @@ -3095,7 +3149,7 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { de, exists := destMap[mapKey] if !exists { - de = &destEntry{destination: prefix, table: table, nexthops: make(map[nhKey]NextHop)} + de = &destEntry{destination: prefix, table: table, nexthops: make(map[nhKey]observedNextHop)} destMap[mapKey] = de destOrder = append(destOrder, mapKey) } @@ -3103,12 +3157,11 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { for _, wh := range wgHops { nk := nhKey{gateway: wh.gwStr, device: wh.devName} if _, nhExists := de.nexthops[nk]; !nhExists { - nh := NextHop{ - Gateway: wh.gwStr, Device: wh.devName, Distance: r.Priority, - RouteTypes: []RouteType{{Type: "kernel", Attributes: []string{"fib"}}}, + nh := observedNextHop{ + gateway: wh.gwStr, device: wh.devName, distance: r.Priority, } if r.MTU > 0 { - nh.MTU = r.MTU + nh.mtu = r.MTU } de.nexthops[nk] = nh @@ -3152,7 +3205,7 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { de, exists := destMap[mapKey] if !exists { - de = &destEntry{destination: prefix, table: table, nexthops: make(map[nhKey]NextHop)} + de = &destEntry{destination: prefix, table: table, nexthops: make(map[nhKey]observedNextHop)} destMap[mapKey] = de destOrder = append(destOrder, mapKey) } @@ -3164,17 +3217,11 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { nk := nhKey{gateway: gwStr, device: devName} if _, nhExists := de.nexthops[nk]; !nhExists { - nh := NextHop{ - Gateway: gwStr, - Device: devName, - Distance: r.Priority, - RouteTypes: []RouteType{{ - Type: "kernel", - Attributes: []string{"fib"}, - }}, + nh := observedNextHop{ + gateway: gwStr, device: devName, distance: r.Priority, } if r.MTU > 0 { - nh.MTU = r.MTU + nh.mtu = r.MTU } de.nexthops[nk] = nh @@ -3182,46 +3229,20 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { } } - result := make([]RouteEntry, 0, len(destOrder)) for _, mapKey := range destOrder { de := destMap[mapKey] - nhs := make([]NextHop, 0, len(de.nhOrder)) + nhs := make([]observedNextHop, 0, len(de.nhOrder)) for _, nk := range de.nhOrder { nhs = append(nhs, de.nexthops[nk]) } - result = append(result, RouteEntry{ - Destination: de.destination, - Family: familyLabel, - Table: de.table, - NextHops: nhs, - }) + visit(familyLabel, de.destination, de.table, nhs) } - - return result - } - - v4Routes := collect(netlink.FAMILY_V4, "IPv4") - v6Routes := collect(netlink.FAMILY_V6, "IPv6") - - if v4Routes == nil { - v4Routes = []RouteEntry{} - } - - if v6Routes == nil { - v6Routes = []RouteEntry{} } - info.Routes = append(v4Routes, v6Routes...) - - s.routingTableCacheMu.Lock() - s.routingTableCache = info - s.routingTableCachedAt = time.Now() - s.routingTableCacheMu.Unlock() - s.routingTableDirty.Store(false) - - return info + collect(netlink.FAMILY_V4, "IPv4") + collect(netlink.FAMILY_V6, "IPv6") } // isManagedTunnelInterface returns true for the interfaces created by the From ef4bbe19db7776107f5f650f546a06965be7ae70 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:25:03 +0000 Subject: [PATCH 12/34] feat(net-node): count annotated routes without detail snapshots Reuse kernel inspection and expected-route planning while preserving legacy missing-route and unbounded0 annotation semantics. Add table, family, missing, mismatch, and multipath parity fixtures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- .../status_summary_routes.go | 123 ++++++++++++++++++ .../status_summary_routes_test.go | 105 +++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 cmd/unbounded-net-node/status_summary_routes.go create mode 100644 cmd/unbounded-net-node/status_summary_routes_test.go diff --git a/cmd/unbounded-net-node/status_summary_routes.go b/cmd/unbounded-net-node/status_summary_routes.go new file mode 100644 index 000000000..70f91638b --- /dev/null +++ b/cmd/unbounded-net-node/status_summary_routes.go @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + + "k8s.io/klog/v2" + + "github.com/Azure/unbounded/internal/net/routeplan" +) + +type routeSummaryFamily struct { + expected map[routeKey]expectedRoute + lowest map[string]int + destinations map[string]bool + hasUnbounded bool +} + +func newRouteSummaryFamily(plan []routeplan.ExpectedRoute) *routeSummaryFamily { + family := &routeSummaryFamily{ + expected: make(map[routeKey]expectedRoute), + lowest: make(map[string]int), + destinations: make(map[string]bool), + } + + for _, route := range plan { + distance := effectiveRouteDistance(route.Distance) + key := routeKey{destination: route.Destination, gateway: route.Gateway, device: route.Device, distance: distance, weight: route.Weight} + + family.expected[key] = expectedRoute{ + destination: route.Destination, + nextHop: NextHop{Gateway: route.Gateway, Device: route.Device, Distance: distance, Weight: route.Weight}, + } + if previous, ok := family.lowest[route.Destination]; !ok || distance < previous { + family.lowest[route.Destination] = distance + } + } + + return family +} + +// collectRouteSummary preserves annotation counts, including synthetic missing +// routes and the per-family unbounded0 suppression of missing tunnel hops. +// It never creates status route arrays or fills the full-detail route cache. +func (s *nodeStatusServer) collectRouteSummary(peers []routeplan.Peer, localSite string) (count int, mismatch bool) { + defer func() { + if r := recover(); r != nil { + klog.Warningf("route summary recovered from panic: %v", r) + } + }() + + actx := buildAnnotationContext(s.siteInformer, s.sliceInformer, s.gatewayPoolInformer, s.sitePeeringInformer) + for i := range peers { + peers[i].SitePeered = sitesAreDirectlyPeered(strings.TrimSpace(localSite), peers[i].SiteName, actx.directSitePeerings) + } + + ipv4, ipv6 := routeplan.BuildExpectedWireGuardRoutes(peers, actx.routeNodes, routeplan.InterfaceNames{ + WireGuardPrefix: s.cfg.WireGuardInterfacePrefix, + Geneve: s.cfg.GeneveInterfaceName, + VXLAN: s.cfg.VXLANInterfaceName, + IPIP: s.cfg.IPIPInterfaceName, + }) + families := map[string]*routeSummaryFamily{ + "IPv4": newRouteSummaryFamily(ipv4), + "IPv6": newRouteSummaryFamily(ipv6), + } + + s.inspectKernelRoutes(func(familyName, destination string, _ int, hops []observedNextHop) { + family := families[familyName] + count++ + family.destinations[destination] = true + normalized, _ := normalizeRouteDestination(destination) + + for _, observed := range hops { + if observed.device == unbounded0DeviceName { + family.hasUnbounded = true + } + + if !isPeerRoutingInterface(s.cfg, observed.device) { + continue + } + + hop := NextHop{Gateway: observed.gateway, Device: observed.device, Distance: observed.distance} + + key, matched := findExpectedWireGuardMatch(family.expected, normalized, &hop) + if matched { + delete(family.expected, key) + } else { + // Kernel inspection only emits "kernel" route types, so the + // connected/local host-route exception cannot apply here. + mismatch = true + } + } + }) + + // The legacy collector does not annotate an entirely empty kernel result. + if count == 0 { + return count, mismatch + } + + for _, family := range families { + for _, expected := range family.expected { + if effectiveRouteDistance(expected.nextHop.Distance) > family.lowest[expected.destination] { + continue + } + + if family.hasUnbounded { + continue + } + + mismatch = true + + if !family.destinations[expected.destination] { + family.destinations[expected.destination] = true + count++ + } + } + } + + return count, mismatch +} diff --git a/cmd/unbounded-net-node/status_summary_routes_test.go b/cmd/unbounded-net-node/status_summary_routes_test.go new file mode 100644 index 000000000..6bd1a7863 --- /dev/null +++ b/cmd/unbounded-net-node/status_summary_routes_test.go @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net" + "testing" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "github.com/Azure/unbounded/internal/net/routeplan" +) + +func summaryRoute(destination string, index, table, distance int) netlink.Route { + _, prefix, _ := net.ParseCIDR(destination) + + return netlink.Route{Dst: prefix, LinkIndex: index, Table: table, Priority: distance, Protocol: unix.RTPROT_BOOT} +} + +func summaryRouteFixture() *nodeStatusServer { + return &nodeStatusServer{ + cfg: &config{ + NodeName: "local", WireGuardInterfacePrefix: "wg", WireGuardPort: 51820, + GeneveInterfaceName: "gn0", VXLANInterfaceName: "vx0", IPIPInterfaceName: "ip0", + }, + state: &wireGuardState{routeTableID: 100}, + netlinkOps: &fakeNetlinkOps{ + links: map[int]netlink.Link{ + 1: &fakeLink{attrs: netlink.LinkAttrs{Index: 1, Name: "wg51820"}}, + 2: &fakeLink{attrs: netlink.LinkAttrs{Index: 2, Name: unbounded0DeviceName}}, + 3: &fakeLink{attrs: netlink.LinkAttrs{Index: 3, Name: "eth0"}}, + 4: &fakeLink{attrs: netlink.LinkAttrs{Index: 4, Name: "gn0"}}, + }, + }, + } +} + +func TestRouteSummaryParity(t *testing.T) { + peer := WireGuardPeerStatus{ + Name: "peer", PeerType: "site", SiteName: "local", + PodCIDRGateways: []string{"10.42.1.1", "fd00:1::1"}, + Tunnel: PeerTunnelStatus{Interface: "wg51820", AllowedIPs: []string{"10.42.1.0/24", "fd00:1::/64"}}, + } + planPeer := routeplan.Peer{ + Name: peer.Name, PeerType: peer.PeerType, SiteName: peer.SiteName, + Interface: peer.Tunnel.Interface, AllowedIPs: peer.Tunnel.AllowedIPs, PodCIDRGateways: peer.PodCIDRGateways, + } + + for _, tc := range []struct { + name string + v4 []netlink.Route + v6 []netlink.Route + table []netlink.Route + withoutPeer bool + }{ + {name: "empty kernel does not synthesize"}, + {name: "missing expected", v4: []netlink.Route{summaryRoute("10.9.0.0/24", 1, 0, 0)}}, + {name: "matched", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 0)}}, + {name: "unexpected without peers", withoutPeer: true, v4: []netlink.Route{summaryRoute("10.9.0.0/24", 1, 0, 0)}}, + {name: "unbounded suppresses only its family", v4: []netlink.Route{summaryRoute("10.42.0.0/16", 2, 0, 0)}}, + {name: "unbounded both families", v4: []netlink.Route{summaryRoute("10.42.0.0/16", 2, 0, 0)}, v6: []netlink.Route{summaryRoute("fd00::/48", 2, 0, 0)}}, + {name: "duplicate prefix distinct tables", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 0)}, table: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 100, 0)}}, + {name: "duplicate next hop", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 0), summaryRoute("10.42.1.0/24", 1, 0, 100)}}, + {name: "wrong distance", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 500)}}, + {name: "unmanaged ignored", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 3, 0, 0)}}, + {name: "multipath", v4: []netlink.Route{{ + Dst: summaryRoute("10.42.1.0/24", 1, 0, 0).Dst, + MultiPath: []*netlink.NexthopInfo{{LinkIndex: 1}, {LinkIndex: 3}, {LinkIndex: 4}}, + }}}, + } { + t.Run(tc.name, func(t *testing.T) { + s := summaryRouteFixture() + ops := s.netlinkOps.(*fakeNetlinkOps) + ops.mainRoutes = map[int][]netlink.Route{netlink.FAMILY_V4: tc.v4, netlink.FAMILY_V6: tc.v6} + ops.tableRoutes = map[int]map[int][]netlink.Route{netlink.FAMILY_V4: {100: tc.table}} + full := &NodeStatusResponse{NodeInfo: NodeInfo{SiteName: "local"}, RoutingTable: s.collectRoutingTableFromKernel()} + + var peers []routeplan.Peer + + if !tc.withoutPeer { + full.Peers = []WireGuardPeerStatus{peer} + peers = []routeplan.Peer{planPeer} + } + + annotateNodeRoutes(full, s.cfg, nil, nil, nil, nil) + + wantMismatch := false + + for _, route := range full.RoutingTable.Routes { + for _, hop := range route.NextHops { + if (hop.Expected != nil && *hop.Expected) != (hop.Present != nil && *hop.Present) { + wantMismatch = true + } + } + } + + count, mismatch := s.collectRouteSummary(peers, "local") + if count != len(full.RoutingTable.Routes) || mismatch != wantMismatch { + t.Fatalf("summary=(%d,%v), legacy=(%d,%v): %+v", count, mismatch, len(full.RoutingTable.Routes), wantMismatch, full.RoutingTable.Routes) + } + }) + } +} From 7e150d1ecf6bd74a53d7d175dfc98fbe6ab3d40e Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:33:48 +0000 Subject: [PATCH 13/34] feat(net-node): add lightweight overview collection and local endpoint Collect shared metadata, bootstrap errors, aggregate health and exact peer/route summary facts without BPF traversal or outbound detail arrays. Add /status/summary using the existing local status policy and an additive protobuf converter. Leave all publishers and full endpoints unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-node/status_proto.go | 25 ++ cmd/unbounded-net-node/status_server.go | 66 +++- cmd/unbounded-net-node/status_summary.go | 110 ++++++ cmd/unbounded-net-node/status_summary_test.go | 320 ++++++++++++++++++ 4 files changed, 503 insertions(+), 18 deletions(-) create mode 100644 cmd/unbounded-net-node/status_summary.go create mode 100644 cmd/unbounded-net-node/status_summary_test.go diff --git a/cmd/unbounded-net-node/status_proto.go b/cmd/unbounded-net-node/status_proto.go index 256a64711..fa1350f67 100644 --- a/cmd/unbounded-net-node/status_proto.go +++ b/cmd/unbounded-net-node/status_proto.go @@ -18,6 +18,31 @@ func statusUnixNano(t time.Time) int64 { return t.UnixNano() } +func nodeSummaryToProto(summary *NodeStatusOverview) *statusproto.NodeStatusOverview { + if summary == nil { + return nil + } + + result := &statusproto.NodeStatusOverview{ + TimestampUnixNs: statusUnixNano(summary.Timestamp), + NodeInfo: nodeInfoToProto(&summary.NodeInfo), + HealthCheck: healthCheckStatusToProto(summary.HealthCheck), + NodeErrors: nodeErrorsToProto(summary.NodeErrors), + FetchError: summary.FetchError, + StatusSource: summary.StatusSource, + NodePodInfo: nodePodInfoToProto(summary.NodePodInfo), + PeerCount: int32(summary.PeerCount), + HealthyPeers: int32(summary.HealthyPeers), + RouteCount: int32(summary.RouteCount), + RouteMismatch: summary.RouteMismatch, + } + if summary.LastPushTime != nil { + result.LastPushTimeUnixNs = statusUnixNano(*summary.LastPushTime) + } + + return result +} + // nodeStatusToProto converts a Go NodeStatusResponse to the protobuf NodeStatusFull message. func nodeStatusToProto(status *NodeStatusResponse) *statusproto.NodeStatusFull { if status == nil { diff --git a/cmd/unbounded-net-node/status_server.go b/cmd/unbounded-net-node/status_server.go index 7eed77a49..3cf303936 100644 --- a/cmd/unbounded-net-node/status_server.go +++ b/cmd/unbounded-net-node/status_server.go @@ -25,6 +25,7 @@ import ( "github.com/coder/websocket" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "google.golang.org/protobuf/proto" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -465,7 +466,7 @@ func (tm *hmacTokenManager) requestToken() error { return fmt.Errorf("all HMAC token endpoints failed: %s", strings.Join(endpointErrors, "; ")) } -func startHealthServer(port int, healthState *nodeHealthState) { +func newHealthMux(healthState *nodeHealthState) *http.ServeMux { mux := http.NewServeMux() metrics.Register(mux) @@ -528,6 +529,13 @@ func startHealthServer(port int, healthState *nodeHealthState) { } }) + // Same local routing and authentication policy as the full status endpoint. + mux.HandleFunc("/status/summary", healthState.handleStatusSummary) + + return mux +} + +func startHealthServer(port int, healthState *nodeHealthState) { addr := fmt.Sprintf(":%d", port) klog.Infof("Starting health server on %s", addr) @@ -535,7 +543,7 @@ func startHealthServer(port int, healthState *nodeHealthState) { server := &http.Server{ Addr: addr, - Handler: httpMiddleware.Wrap("all", mux), + Handler: httpMiddleware.Wrap("all", newHealthMux(healthState)), ReadHeaderTimeout: 10 * time.Second, } if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { @@ -2291,6 +2299,18 @@ type nodeStatusServer struct { // the host kernel's actual routing table. Production callers leave this // nil and the helpers fall back to the real netlink package. netlinkOps statusServerNetlinkOps + + // Optional collector override for tests; summaries never invoke this. + bpfCollector func() []BpfEntry + wireGuardDevice func(*unboundednetnetlink.WireGuardManager) (*wgtypes.Device, error) +} + +func (s *nodeStatusServer) getWireGuardDevice(manager *unboundednetnetlink.WireGuardManager) (*wgtypes.Device, error) { + if s.wireGuardDevice != nil { + return s.wireGuardDevice(manager) + } + + return manager.GetDevice() } // statusServerNetlinkOps abstracts the netlink reads that @@ -2550,7 +2570,7 @@ func (s *nodeStatusServer) inspectNodePeers(visit func(WireGuardPeerStatus)) *no // Get WireGuard device info if available (netlink syscall) if wgManager != nil { - if device, err := wgManager.GetDevice(); err == nil { + if device, err := s.getWireGuardDevice(wgManager); err == nil { status.NodeInfo.WireGuard.ListenPort = device.ListenPort status.NodeInfo.WireGuard.PeerCount = len(device.Peers) @@ -2627,7 +2647,7 @@ func (s *nodeStatusServer) inspectNodePeers(visit func(WireGuardPeerStatus)) *no for _, gw := range gwSnapshots { // Get WireGuard peer info for this gateway interface (netlink syscall) if gw.wgManager != nil { - if device, err := gw.wgManager.GetDevice(); err == nil && len(device.Peers) > 0 { + if device, err := s.getWireGuardDevice(gw.wgManager); err == nil && len(device.Peers) > 0 { wgPeer := device.Peers[0] // Each gateway interface has one peer peer := WireGuardPeerStatus{ @@ -2843,7 +2863,11 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } // Collect BPF trie entries. - status.BpfEntries = s.collectBpfEntries() + if s.bpfCollector != nil { + status.BpfEntries = s.bpfCollector() + } else { + status.BpfEntries = s.collectBpfEntries() + } return status } @@ -2866,26 +2890,32 @@ func linkStatsWarningsAsNodeErrors(warnings []string, peers []WireGuardPeerStatu } func suppressHealthyWireGuardRxErrors(warning string, peers []WireGuardPeerStatus, now time.Time) bool { - iface, deltas, ok := parseLinkStatsWarning(warning) - if !ok || len(deltas) != 1 || !strings.HasPrefix(deltas[0], "rx_errors +") { - return false - } + return suppressHealthyInterfaceRxErrors(warning, func(iface string) bool { + matched := false - matched := false + for _, peer := range peers { + if peer.Tunnel.Interface != iface { + continue + } - for _, peer := range peers { - if peer.Tunnel.Interface != iface { - continue + matched = true + + if !peerStatusHealthy(peer, now) { + return false + } } - matched = true + return matched + }) +} - if !peerStatusHealthy(peer, now) { - return false - } +func suppressHealthyInterfaceRxErrors(warning string, healthy func(string) bool) bool { + iface, deltas, ok := parseLinkStatsWarning(warning) + if !ok || len(deltas) != 1 || !strings.HasPrefix(deltas[0], "rx_errors +") { + return false } - return matched + return healthy(iface) } func parseLinkStatsWarning(warning string) (string, []string, bool) { diff --git a/cmd/unbounded-net-node/status_summary.go b/cmd/unbounded-net-node/status_summary.go new file mode 100644 index 000000000..5d088de91 --- /dev/null +++ b/cmd/unbounded-net-node/status_summary.go @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net/http" + "time" + + "k8s.io/klog/v2" + + "github.com/Azure/unbounded/internal/net/routeplan" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +type NodeStatusOverview = statusv1alpha1.NodeStatusOverview + +// getNodeSummary collects overview facts directly. Only route-planning inputs +// survive peer visitation; full peer, route, and BPF snapshots are never built. +func (s *nodeStatusServer) getNodeSummary() *NodeStatusOverview { + summary := &NodeStatusOverview{} + + var routePeers []routeplan.Peer + + interfaceHealthy := make(map[string]bool) + now := time.Now() + facts := s.inspectNodePeers(func(peer WireGuardPeerStatus) { + summary.PeerCount++ + if summaryPeerHealthy(peer, now) { + summary.HealthyPeers++ + } + + previous, seen := interfaceHealthy[peer.Tunnel.Interface] + interfaceHealthy[peer.Tunnel.Interface] = (!seen || previous) && peerStatusHealthy(peer, now) + routePeers = append(routePeers, routeplan.Peer{ + Name: peer.Name, PeerType: peer.PeerType, SiteName: peer.SiteName, + SkipPodCIDRRoutes: peer.SkipPodCIDRRoutes, + Interface: peer.Tunnel.Interface, Endpoint: peer.Tunnel.Endpoint, + PodCIDRGateways: peer.PodCIDRGateways, AllowedIPs: peer.Tunnel.AllowedIPs, + RouteDistances: peer.RouteDistances, + }) + }) + summary.Timestamp = facts.Timestamp + summary.NodeInfo = facts.NodeInfo + summary.NodeErrors = facts.NodeErrors + summary.HealthCheck = facts.HealthCheck + summary.RouteCount, summary.RouteMismatch = s.collectRouteSummary(routePeers, facts.NodeInfo.SiteName) + + if s.state.linkStatsMonitor != nil { + for _, warning := range s.state.linkStatsMonitor.GetWarnings() { + if !suppressHealthyInterfaceRxErrors(warning, func(iface string) bool { return interfaceHealthy[iface] }) { + summary.NodeErrors = append(summary.NodeErrors, NodeError{Type: "link-stats", Message: warning}) + } + } + } + + return summary +} + +// This is deliberately stricter than link-warning suppression: the controller +// counts only "up"/"Up" when enabled, and otherwise uses handshake freshness. +func summaryPeerHealthy(peer WireGuardPeerStatus, now time.Time) bool { + if peer.HealthCheck != nil && peer.HealthCheck.Enabled { + return peer.HealthCheck.Status == "up" || peer.HealthCheck.Status == "Up" + } + + return !peer.Tunnel.LastHandshake.IsZero() && now.Sub(peer.Tunnel.LastHandshake) < 3*time.Minute +} + +func (h *nodeHealthState) getSummarySnapshot() *NodeStatusOverview { + h.mu.RLock() + srv := h.statusServer + + summary := &NodeStatusOverview{ + Timestamp: time.Now(), + NodeInfo: NodeInfo{ + Name: h.nodeName, SiteName: h.siteName, IsGateway: h.isGateway, + PodCIDRs: append([]string(nil), h.podCIDRs...), BuildInfo: nodeAgentBuildInfo(), + }, + } + if h.pubKey != "" { + summary.NodeInfo.WireGuard = &WireGuardStatusInfo{PublicKey: h.pubKey} + } + + cniManaged, cniReady, cniReason := h.cniManaged, h.cniReady, h.cniReason + transientErrors := append([]NodeError(nil), h.transientErrors...) + h.mu.RUnlock() + + if srv != nil { + summary = srv.getNodeSummary() + } + + summary.NodeErrors = mergeNodeErrors(summary.NodeErrors, filterExpiredNodeErrors(transientErrors, time.Now(), time.Minute)) + + summary.NodeErrors = removeNodeErrorsByType(summary.NodeErrors, configPodCIDRGuard) + if cniManaged && !cniReady && cniReason != "" { + summary.NodeErrors = append(summary.NodeErrors, NodeError{Type: configPodCIDRGuard, Message: cniReason}) + } + + return summary +} + +func (h *nodeHealthState) handleStatusSummary(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if err := json.NewEncoder(w).Encode(h.getSummarySnapshot()); err != nil { + klog.V(4).Infof("status summary json encode failed: %v", err) + } +} diff --git a/cmd/unbounded-net-node/status_summary_test.go b/cmd/unbounded-net-node/status_summary_test.go new file mode 100644 index 000000000..75cecacfe --- /dev/null +++ b/cmd/unbounded-net-node/status_summary_test.go @@ -0,0 +1,320 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "reflect" + "sync" + "testing" + "time" + + "github.com/vishvananda/netlink" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "google.golang.org/protobuf/proto" + + "github.com/Azure/unbounded/internal/net/healthcheck" + unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func TestSummaryPeerHealthy(t *testing.T) { + now := time.Now() + for _, tc := range []struct { + name string + health *HealthCheckPeerStatus + handshake time.Time + want bool + }{ + {"up", &HealthCheckPeerStatus{Enabled: true, Status: "up"}, time.Time{}, true}, + {"Up", &HealthCheckPeerStatus{Enabled: true, Status: "Up"}, time.Time{}, true}, + {"uppercase is not counted", &HealthCheckPeerStatus{Enabled: true, Status: "UP"}, now, false}, + {"enabled unknown", &HealthCheckPeerStatus{Enabled: true}, now, false}, + {"enabled down", &HealthCheckPeerStatus{Enabled: true, Status: "down"}, now, false}, + {"disabled uses handshake", &HealthCheckPeerStatus{Status: "down"}, now, true}, + {"missing health uses handshake", nil, now.Add(-time.Minute), true}, + {"missing handshake", nil, time.Time{}, false}, + {"boundary", nil, now.Add(-3 * time.Minute), false}, + {"future handshake", nil, now.Add(time.Minute), true}, + } { + t.Run(tc.name, func(t *testing.T) { + peer := WireGuardPeerStatus{HealthCheck: tc.health, Tunnel: PeerTunnelStatus{LastHandshake: tc.handshake}} + if got := summaryPeerHealthy(peer, now); got != tc.want { + t.Fatalf("healthy=%v, want %v", got, tc.want) + } + }) + } +} + +func TestNodeSummaryParityAndNoBPF(t *testing.T) { + for _, deviceFailure := range []bool{false, true} { + t.Run(map[bool]string{false: "device success", true: "device failure"}[deviceFailure], func(t *testing.T) { + s := summaryRouteFixture() + s.state.siteName = "local" + s.state.nodePodCIDRs = []string{"10.42.0.0/24"} + s.state.nodeInternalIPs = []string{"192.0.2.1"} + s.state.nodeExternalIPs = []string{"198.51.100.1"} + s.state.nodeErrors = []NodeError{{Type: "test", Message: "failure"}} + s.state.wireguardManager = &unboundednetnetlink.WireGuardManager{} + + manager, err := healthcheck.NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + + if err := manager.AddPeer("down", net.ParseIP("10.42.1.1"), healthcheck.DefaultSettings()); err != nil { + t.Fatal(err) + } + + s.state.healthCheckManager = manager + s.state.meshPeerHealthCheckEnabled = map[string]bool{"down": true, "missing": true} + s.state.peers = []meshPeerInfo{ + {Name: "down", WireGuardPublicKey: "down", TunnelProtocol: "GENEVE", InternalIPs: []string{"192.0.2.2"}, PodCIDRs: []string{"10.42.1.0/24"}}, + {Name: "missing", WireGuardPublicKey: "missing", TunnelProtocol: "VXLAN", InternalIPs: []string{"192.0.2.3"}, PodCIDRs: []string{"10.42.2.0/24"}}, + {Name: "no-address", TunnelProtocol: "IPIP"}, + } + s.state.gatewayPeers = []gatewayPeerInfo{ + {Name: "gateway", TunnelProtocol: "IPIP", InternalIPs: []string{"192.0.2.4"}, PodCIDRs: []string{"10.42.3.0/24"}}, + } + s.state.gatewayHealthEndpoints = map[string]string{"wg51821": "10.42.3.1"} + s.state.gatewayNames = map[string]string{"wg51821": "gateway"} + s.state.gatewayWireguardManagers = map[string]*unboundednetnetlink.WireGuardManager{"wg51821": {}} + s.state.linkStatsMonitor = &linkStatsMonitor{warnings: []string{ + "interface /wg51820: rx_errors +24", "interface /gn0: rx_errors +24", "interface /eth0: tx_errors +2", + }} + s.wireGuardDevice = func(*unboundednetnetlink.WireGuardManager) (*wgtypes.Device, error) { + if deviceFailure { + return nil, errors.New("device unavailable") + } + + return &wgtypes.Device{ListenPort: 51820, Peers: []wgtypes.Peer{ + {LastHandshakeTime: time.Now().Add(-time.Minute)}, + }}, nil + } + s.netlinkOps.(*fakeNetlinkOps).mainRoutes = map[int][]netlink.Route{ + netlink.FAMILY_V4: {summaryRoute("10.42.0.0/16", 2, 0, 0)}, + } + bpfCalls := 0 + s.bpfCollector = func() []BpfEntry { bpfCalls++; return nil } + + summary := s.getNodeSummary() + if bpfCalls != 0 || !s.routingTableCachedAt.IsZero() || len(s.routingTableCache.Routes) != 0 { + t.Fatal("summary collected BPF or populated the full route cache") + } + + full := s.getNodeStatus() + + if bpfCalls != 1 { + t.Fatal("legacy full collection no longer collects BPF") + } + + if !reflect.DeepEqual(summary.NodeInfo, full.NodeInfo) || !reflect.DeepEqual(summary.NodeErrors, full.NodeErrors) { + t.Fatalf("metadata/errors differ: summary=%+v full=%+v", summary, full) + } + + wantHealthy := 0 + + for _, peer := range full.Peers { + if summaryPeerHealthy(peer, time.Now()) { + wantHealthy++ + } + } + + if summary.PeerCount != len(full.Peers) || summary.HealthyPeers != wantHealthy || summary.RouteCount != len(full.RoutingTable.Routes) { + t.Fatalf("counts differ: summary=%+v full peers=%+v routes=%+v", summary, full.Peers, full.RoutingTable) + } + + wantPeers, wantHealthyPeers := 4, 2 + if deviceFailure { + wantPeers, wantHealthyPeers = 3, 0 + } + + if summary.PeerCount != wantPeers || summary.HealthyPeers != wantHealthyPeers || summary.RouteMismatch { + t.Fatalf("unexpected observed counts/mismatch: %+v", summary) + } + + if summary.HealthCheck == nil || summary.HealthCheck.Healthy != full.HealthCheck.Healthy || + summary.HealthCheck.PeerCount != full.HealthCheck.PeerCount || summary.HealthCheck.Summary != full.HealthCheck.Summary { + t.Fatalf("health aggregates differ: %v vs %v", summary.HealthCheck, full.HealthCheck) + } + + assertSummaryHasNoDetails(t, summary) + }) + } +} + +func assertSummaryHasNoDetails(t *testing.T, summary *NodeStatusOverview) { + t.Helper() + + data, err := json.Marshal(summary) + if err != nil { + t.Fatal(err) + } + + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + t.Fatal(err) + } + + for _, name := range []string{"peers", "routingTable", "bpfEntries", "peerMeasurements"} { + if _, exists := fields[name]; exists { + t.Fatalf("summary contains detail field %q", name) + } + } +} + +func TestSummaryBootstrapRecoveryAndLocalRouting(t *testing.T) { + h := blockedBootstrapHealthState() + h.transientErrors = []NodeError{{Type: "transport", Message: "failed"}, {Type: "expired", Message: "old", Timestamp: time.Now().Add(-2 * time.Minute)}} + mux := newHealthMux(h) + + for _, path := range []string{"/status/summary", "/status", "/status/json"} { + recorder := httptest.NewRecorder() + mux.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path, nil)) + + if recorder.Code != http.StatusOK || recorder.Header().Get("Content-Type") != "application/json" { + t.Fatalf("%s: code=%d headers=%v", path, recorder.Code, recorder.Header()) + } + + var summary NodeStatusOverview + if err := json.Unmarshal(recorder.Body.Bytes(), &summary); err != nil { + t.Fatal(err) + } + + if summary.NodeInfo.Name != "node-a" || len(summary.NodeErrors) != 2 || summary.NodeErrors[1].Type != configPodCIDRGuard { + t.Fatalf("%s: lost bootstrap identity/errors: %+v", path, summary) + } + + var fields map[string]json.RawMessage + if err := json.Unmarshal(recorder.Body.Bytes(), &fields); err != nil { + t.Fatal(err) + } + + _, hasPeers := fields["peers"] + if hasPeers == (path == "/status/summary") { + t.Fatalf("%s: endpoint returned wrong representation", path) + } + } + + h.setCNIReady("cbr0", []string{"10.244.7.0/24"}) + + summary := h.getSummarySnapshot() + if len(summary.NodeErrors) != 1 || summary.NodeErrors[0].Type != "transport" { + t.Fatalf("guard did not recover: %+v", summary.NodeErrors) + } + + summary.NodeInfo.PodCIDRs[0] = "mutated" + if h.getSummarySnapshot().NodeInfo.PodCIDRs[0] == "mutated" { + t.Fatal("bootstrap CIDRs alias shared state") + } + + s := summaryRouteFixture() + s.state.nodeErrors = []NodeError{{Type: configPodCIDRGuard, Message: "obsolete guard"}} + h.setStatusServer(s) + + if got := h.getSummarySnapshot(); got.NodeInfo.Name != "local" || len(got.NodeErrors) != 1 || got.NodeErrors[0].Type != "transport" { + t.Fatalf("initialized summary lost recovery/transport state: %+v", got) + } +} + +func TestSummaryHealthyAggregate(t *testing.T) { + s := summaryRouteFixture() + + manager, err := healthcheck.NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + + s.state.healthCheckManager = manager + + summary := s.getNodeSummary() + if summary.HealthCheck == nil || !summary.HealthCheck.Healthy || summary.HealthCheck.PeerCount != 0 || + summary.HealthCheck.Summary != "all peers healthy" || summary.HealthCheck.CheckedAt.IsZero() { + t.Fatalf("lost healthy aggregate: %+v", summary.HealthCheck) + } +} + +func TestSummaryConcurrentBootstrapState(t *testing.T) { + h := blockedBootstrapHealthState() + + var wg sync.WaitGroup + for range 4 { + wg.Go(func() { + for range 20 { + h.beginManagedCNI("cbr0") + h.getSummarySnapshot() + h.setCNIReady("cbr0", []string{"10.244.7.0/24"}) + } + }) + } + + wg.Wait() +} + +func TestNodeSummaryToProto(t *testing.T) { + if nodeSummaryToProto(nil) != nil { + t.Fatal("nil summary converted") + } + + now := time.Now() + summary := &NodeStatusOverview{ + Timestamp: now, NodeInfo: NodeInfo{Name: "node", K8sReady: "Unknown"}, + PeerCount: 10, HealthyPeers: 4, RouteCount: 12, RouteMismatch: true, + FetchError: "unavailable", StatusSource: "error", LastPushTime: &now, + NodeErrors: []NodeError{{Type: "failure", Message: "failed"}}, + HealthCheck: &HealthCheckStatus{Healthy: false, Summary: "unhealthy"}, + } + encoded := nodeSummaryToProto(summary) + + data, err := proto.Marshal(encoded) + if err != nil { + t.Fatal(err) + } + + var decoded statusproto.NodeStatusOverview + if err := proto.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + + if !proto.Equal(encoded, &decoded) || decoded.PeerCount != 10 || decoded.HealthyPeers != 4 || decoded.RouteCount != 12 || + !decoded.RouteMismatch || decoded.FetchError != "unavailable" || decoded.NodeInfo.K8SReady != "Unknown" || + decoded.LastPushTimeUnixNs != now.UnixNano() || decoded.StatusSource != "error" || len(decoded.NodeErrors) != 1 { + t.Fatalf("summary conversion lost facts: %v", &decoded) + } +} + +func BenchmarkNodeSummaryCollection(b *testing.B) { + for _, full := range []bool{false, true} { + b.Run(map[bool]string{false: "summary", true: "full"}[full], func(b *testing.B) { + s := summaryRouteFixture() + s.bpfCollector = func() []BpfEntry { return nil } + + s.netlinkOps.(*fakeNetlinkOps).mainRoutes = map[int][]netlink.Route{ + netlink.FAMILY_V4: {summaryRoute("10.0.0.0/8", 2, 0, 0)}, + } + for i := range 2000 { + s.state.peers = append(s.state.peers, meshPeerInfo{ + Name: fmt.Sprintf("peer-%d", i), WireGuardPublicKey: fmt.Sprintf("key-%d", i), + TunnelProtocol: "GENEVE", InternalIPs: []string{"192.0.2.2"}, + PodCIDRs: []string{fmt.Sprintf("10.%d.%d.0/24", i/256, i%256)}, + }) + } + + b.ReportAllocs() + + for b.Loop() { + if full { + s.getNodeStatus() + } else { + s.getNodeSummary() + } + } + }) + } +} From 190d868980d7e4f8969a12d427901e4b19bb9100 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:39:42 +0000 Subject: [PATCH 14/34] refactor(net-node): reuse shared overview health semantics Use PeerHealthyForOverview during streamed summary collection and shared legacy projection/mismatch helpers in parity tests. Do not collect full snapshots in the production summary path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-node/status_summary.go | 13 ++----------- .../status_summary_routes_test.go | 7 +++---- cmd/unbounded-net-node/status_summary_test.go | 15 +++++---------- 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/cmd/unbounded-net-node/status_summary.go b/cmd/unbounded-net-node/status_summary.go index 5d088de91..4866afe85 100644 --- a/cmd/unbounded-net-node/status_summary.go +++ b/cmd/unbounded-net-node/status_summary.go @@ -11,6 +11,7 @@ import ( "k8s.io/klog/v2" "github.com/Azure/unbounded/internal/net/routeplan" + netstatus "github.com/Azure/unbounded/internal/net/status" statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) @@ -27,7 +28,7 @@ func (s *nodeStatusServer) getNodeSummary() *NodeStatusOverview { now := time.Now() facts := s.inspectNodePeers(func(peer WireGuardPeerStatus) { summary.PeerCount++ - if summaryPeerHealthy(peer, now) { + if netstatus.PeerHealthyForOverview(&peer, now) { summary.HealthyPeers++ } @@ -58,16 +59,6 @@ func (s *nodeStatusServer) getNodeSummary() *NodeStatusOverview { return summary } -// This is deliberately stricter than link-warning suppression: the controller -// counts only "up"/"Up" when enabled, and otherwise uses handshake freshness. -func summaryPeerHealthy(peer WireGuardPeerStatus, now time.Time) bool { - if peer.HealthCheck != nil && peer.HealthCheck.Enabled { - return peer.HealthCheck.Status == "up" || peer.HealthCheck.Status == "Up" - } - - return !peer.Tunnel.LastHandshake.IsZero() && now.Sub(peer.Tunnel.LastHandshake) < 3*time.Minute -} - func (h *nodeHealthState) getSummarySnapshot() *NodeStatusOverview { h.mu.RLock() srv := h.statusServer diff --git a/cmd/unbounded-net-node/status_summary_routes_test.go b/cmd/unbounded-net-node/status_summary_routes_test.go index 6bd1a7863..7d8e598b3 100644 --- a/cmd/unbounded-net-node/status_summary_routes_test.go +++ b/cmd/unbounded-net-node/status_summary_routes_test.go @@ -11,6 +11,7 @@ import ( "golang.org/x/sys/unix" "github.com/Azure/unbounded/internal/net/routeplan" + netstatus "github.com/Azure/unbounded/internal/net/status" ) func summaryRoute(destination string, index, table, distance int) netlink.Route { @@ -89,10 +90,8 @@ func TestRouteSummaryParity(t *testing.T) { wantMismatch := false for _, route := range full.RoutingTable.Routes { - for _, hop := range route.NextHops { - if (hop.Expected != nil && *hop.Expected) != (hop.Present != nil && *hop.Present) { - wantMismatch = true - } + if netstatus.RouteMismatchForOverview(route) { + wantMismatch = true } } diff --git a/cmd/unbounded-net-node/status_summary_test.go b/cmd/unbounded-net-node/status_summary_test.go index 75cecacfe..215b1c2c0 100644 --- a/cmd/unbounded-net-node/status_summary_test.go +++ b/cmd/unbounded-net-node/status_summary_test.go @@ -21,6 +21,7 @@ import ( "github.com/Azure/unbounded/internal/net/healthcheck" unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" + netstatus "github.com/Azure/unbounded/internal/net/status" statusproto "github.com/Azure/unbounded/internal/net/status/proto" ) @@ -45,7 +46,7 @@ func TestSummaryPeerHealthy(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { peer := WireGuardPeerStatus{HealthCheck: tc.health, Tunnel: PeerTunnelStatus{LastHandshake: tc.handshake}} - if got := summaryPeerHealthy(peer, now); got != tc.want { + if got := netstatus.PeerHealthyForOverview(&peer, now); got != tc.want { t.Fatalf("healthy=%v, want %v", got, tc.want) } }) @@ -118,15 +119,9 @@ func TestNodeSummaryParityAndNoBPF(t *testing.T) { t.Fatalf("metadata/errors differ: summary=%+v full=%+v", summary, full) } - wantHealthy := 0 - - for _, peer := range full.Peers { - if summaryPeerHealthy(peer, time.Now()) { - wantHealthy++ - } - } - - if summary.PeerCount != len(full.Peers) || summary.HealthyPeers != wantHealthy || summary.RouteCount != len(full.RoutingTable.Routes) { + legacy := netstatus.OverviewFromStatus(full, time.Now()) + if summary.PeerCount != legacy.PeerCount || summary.HealthyPeers != legacy.HealthyPeers || + summary.RouteCount != legacy.RouteCount || summary.RouteMismatch != legacy.RouteMismatch { t.Fatalf("counts differ: summary=%+v full peers=%+v routes=%+v", summary, full.Peers, full.RoutingTable) } From b714aa40b764f0278b9e019296a6114e6587bad9 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:29:58 +0000 Subject: [PATCH 15/34] net-controller: add leader-local detail request lifecycle Coalesce per-node requests, reuse UID-bound cached details, dispatch through a WebSocket hook or bounded HTTP pull, and expose POST polling fallback. Keep only request metadata outside the TTL detail cache; reject stale or mismatched replies and proactively retire requests. Add shared detail result JSON types and deterministic lifecycle tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-controller/detail_cache.go | 11 +- .../detail_requests.go | 401 ++++++++++++++++++ .../detail_requests_test.go | 316 ++++++++++++++ internal/net/status/v1alpha1/details.go | 37 ++ 4 files changed, 757 insertions(+), 8 deletions(-) create mode 100644 cmd/unbounded-net-controller/detail_requests.go create mode 100644 cmd/unbounded-net-controller/detail_requests_test.go create mode 100644 internal/net/status/v1alpha1/details.go diff --git a/cmd/unbounded-net-controller/detail_cache.go b/cmd/unbounded-net-controller/detail_cache.go index bd51416e4..ccd64fe2c 100644 --- a/cmd/unbounded-net-controller/detail_cache.go +++ b/cmd/unbounded-net-controller/detail_cache.go @@ -10,19 +10,14 @@ import ( "time" "k8s.io/utils/clock" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) // nodeDetailSnapshot carries immutable details, separate from routine status. // Status and all its nested data must remain read-only, including for callers // retaining a returned snapshot after its cache entry expires. -type nodeDetailSnapshot struct { - NodeName string - RequestID string - CollectedAt time.Time - ReceivedAt time.Time - ExpiresAt time.Time - Status *NodeStatusResponse -} +type nodeDetailSnapshot = statusv1alpha1.NodeDetailSnapshot // nodeDetailCache is a leader-local, TTL-only store. It owns no second result // history or per-entry timers. TTL bounds retention time, not peak memory. diff --git a/cmd/unbounded-net-controller/detail_requests.go b/cmd/unbounded-net-controller/detail_requests.go new file mode 100644 index 000000000..e2cc5bcac --- /dev/null +++ b/cmd/unbounded-net-controller/detail_requests.go @@ -0,0 +1,401 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "sync" + "time" + + "k8s.io/apimachinery/pkg/types" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// Hooks must honor cancellation. Resolve reads the current informer identity; +// Dispatch returns true only when an active WebSocket accepted the command. +type nodeDetailRequestHooks struct { + Resolve func(string) (types.UID, error) + Dispatch func(context.Context, string, statusv1alpha1.DetailRequest) (bool, error) + Pull func(context.Context, string) (*NodeStatusResponse, error) +} + +// A request owns metadata and cancellation only, never a result payload. +type nodeDetailRequest struct { + nodeName string + uid types.UID + command statusv1alpha1.DetailRequest + state statusv1alpha1.NodeDetailState + message string + wakeAt time.Time + poll bool + cancel context.CancelFunc +} + +type nodeDetailRequests struct { + mu sync.Mutex + ctx context.Context + cancel context.CancelFunc + done chan struct{} + changed chan struct{} + workers sync.WaitGroup + closed bool + timeout time.Duration + cache *nodeDetailCache + hooks nodeDetailRequestHooks + requests map[string]*nodeDetailRequest + active map[string]*nodeDetailRequest +} + +// newNodeDetailRequests takes exclusive lifecycle ownership of cache. All +// snapshots must enter through Complete so their node UID binding is known. +func newNodeDetailRequests(ctx context.Context, cache *nodeDetailCache, timeout time.Duration, hooks nodeDetailRequestHooks) (*nodeDetailRequests, error) { + if cache == nil || timeout <= 0 || hooks.Resolve == nil { + return nil, errors.New("detail requests require a cache, positive timeout, and node resolver") + } + + ctx, cancel := context.WithCancel(ctx) + m := &nodeDetailRequests{ + ctx: ctx, cancel: cancel, done: make(chan struct{}), changed: make(chan struct{}, 1), + timeout: timeout, cache: cache, hooks: hooks, + requests: make(map[string]*nodeDetailRequest), active: make(map[string]*nodeDetailRequest), + } + cache.Clear() + + go m.run() + + return m, nil +} + +// Close cancels dispatches, clears leader-local state, and waits for all workers. +func (m *nodeDetailRequests) Close() { + m.cancel() + <-m.done +} + +func (m *nodeDetailRequests) Request(nodeName string, forceRefresh bool) statusv1alpha1.NodeDetailResult { + m.mu.Lock() + defer m.mu.Unlock() + + if m.ctx.Err() != nil || m.closed { + return detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailRetryable, "detail request leader is unavailable") + } + + m.expireLocked(time.Now()) + + uid, err := m.hooks.Resolve(nodeName) + if err != nil || uid == "" { + return detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailUnavailable, "node identity is unavailable") + } + + for _, request := range m.requests { + if request.nodeName == nodeName && request.uid != uid { + m.invalidateLocked(request) + } + } + + if !forceRefresh { + if snapshot, ok := m.cache.Get(nodeName); ok { + if request := m.requests[snapshot.RequestID]; request != nil && request.uid == uid { + return m.resultLocked(request) + } + } + } + + if request := m.active[nodeName]; request != nil { + return m.resultLocked(request) + } + + now := time.Now() + request := &nodeDetailRequest{ + nodeName: nodeName, uid: uid, state: statusv1alpha1.NodeDetailPending, + command: statusv1alpha1.DetailRequest{RequestID: rand.Text(), Deadline: now.Add(m.timeout)}, + wakeAt: now.Add(m.timeout), + } + ctx, cancel := context.WithDeadline(m.ctx, request.command.Deadline) + request.cancel = cancel + m.requests[request.command.RequestID] = request + m.active[nodeName] = request + m.notify() + m.workers.Go(func() { m.dispatch(ctx, nodeName, request.command) }) + + return m.resultLocked(request) +} + +func (m *nodeDetailRequests) Result(nodeName, requestID string) statusv1alpha1.NodeDetailResult { + m.mu.Lock() + defer m.mu.Unlock() + + if m.ctx.Err() != nil || m.closed { + return detailRequestFailure(nodeName, requestID, statusv1alpha1.NodeDetailRetryable, "detail request leader is unavailable") + } + + m.expireLocked(time.Now()) + + request := m.requests[requestID] + if request == nil || request.nodeName != nodeName { + return detailRequestFailure(nodeName, requestID, statusv1alpha1.NodeDetailRetryable, "request is no longer known; retry on the current leader") + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + } + + return m.resultLocked(request) +} + +// Complete is idempotent while a completed request is retained. It rejects +// mismatched, late, and expired replies without replacing data or renewing TTL. +func (m *nodeDetailRequests) Complete(nodeName, requestID string, status *NodeStatusResponse) error { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + request := m.requests[requestID] + if m.ctx.Err() != nil || m.closed || request == nil || request.nodeName != nodeName { + return errors.New("detail request is no longer available") + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + + return errors.New("detail request node was deleted or replaced") + } + + if status == nil || status.NodeInfo.Name != nodeName || status.FetchError != "" { + return errors.New("detail response has missing data, a fetch error, or a mismatched node name") + } + + if request.state == statusv1alpha1.NodeDetailComplete { + return nil + } + + if request.state != statusv1alpha1.NodeDetailPending { + return errors.New("detail request is no longer pending") + } + + snapshot, err := m.cache.Store(nodeName, requestID, status.Timestamp, status) + if err != nil { + return err + } + + request.state = statusv1alpha1.NodeDetailComplete + request.message = "" + request.poll = false + request.wakeAt = snapshot.ExpiresAt + request.cancel() + delete(m.active, nodeName) + m.notify() + + return nil +} + +// Pending exposes only a failed-pull fallback command, without refreshing its +// deadline. Returning it repeatedly is safe until a valid reply completes it. +func (m *nodeDetailRequests) Pending(nodeName string) (statusv1alpha1.DetailRequest, bool) { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + request := m.active[nodeName] + if m.ctx.Err() != nil || request == nil || !request.poll { + return statusv1alpha1.DetailRequest{}, false + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + + return statusv1alpha1.DetailRequest{}, false + } + + return request.command, true +} + +// InvalidateNode handles deletion/replacement of a specific UID. A delayed old +// informer event cannot cancel a request for a newer node with the same name. +func (m *nodeDetailRequests) InvalidateNode(nodeName string, uid types.UID) { + m.mu.Lock() + defer m.mu.Unlock() + + for _, request := range m.requests { + if request.nodeName == nodeName && request.uid == uid { + m.invalidateLocked(request) + } + } +} + +func (m *nodeDetailRequests) invalidateLocked(request *nodeDetailRequest) { + if request.state == statusv1alpha1.NodeDetailUnavailable { + return + } + + request.cancel() + request.state = statusv1alpha1.NodeDetailUnavailable + request.message = "node was deleted or replaced" + request.poll = false + request.wakeAt = time.Now().Add(m.timeout) + + if m.active[request.nodeName] == request { + delete(m.active, request.nodeName) + } + + if snapshot, ok := m.cache.Get(request.nodeName); ok && snapshot.RequestID == request.command.RequestID { + m.cache.Delete(request.nodeName) + } + + m.notify() +} + +func (m *nodeDetailRequests) resultLocked(request *nodeDetailRequest) statusv1alpha1.NodeDetailResult { + result := statusv1alpha1.NodeDetailResult{ + NodeName: request.nodeName, RequestID: request.command.RequestID, Deadline: request.command.Deadline, + State: request.state, Error: request.message, + } + if request.state == statusv1alpha1.NodeDetailComplete { + if snapshot, ok := m.cache.Get(request.nodeName); ok && snapshot.RequestID == request.command.RequestID { + result.Details = &snapshot + } else { + result.State = statusv1alpha1.NodeDetailExpired + result.Error = "details expired or were replaced" + } + } + + return result +} + +func detailRequestFailure(nodeName, requestID string, state statusv1alpha1.NodeDetailState, message string) statusv1alpha1.NodeDetailResult { + return statusv1alpha1.NodeDetailResult{NodeName: nodeName, RequestID: requestID, State: state, Error: message} +} + +func (m *nodeDetailRequests) dispatch(ctx context.Context, nodeName string, command statusv1alpha1.DetailRequest) { + if m.hooks.Dispatch != nil { + if sent, err := m.hooks.Dispatch(ctx, nodeName, command); sent && err == nil { + return + } + } + + err := errors.New("node HTTP detail pull is unavailable") + + if m.hooks.Pull != nil && ctx.Err() == nil { + pullCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + status, pullErr := m.hooks.Pull(pullCtx, nodeName) + + cancel() + + err = pullErr + if err == nil { + err = m.Complete(nodeName, command.RequestID, status) + } + + if err == nil { + return + } + } + + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + if request := m.active[nodeName]; request != nil && request.command.RequestID == command.RequestID && m.ctx.Err() == nil { + request.poll = true + request.message = fmt.Sprintf("HTTP detail pull failed; waiting for status POST: %v", err) + } +} + +func (m *nodeDetailRequests) expireLocked(now time.Time) time.Time { + var next time.Time + + for id, request := range m.requests { + if !now.Before(request.wakeAt) { + if request.state == statusv1alpha1.NodeDetailPending || request.state == statusv1alpha1.NodeDetailComplete { + request.cancel() + request.state = statusv1alpha1.NodeDetailExpired + request.message = "detail request or snapshot expired" + request.poll = false + request.wakeAt = request.wakeAt.Add(m.timeout) + + if m.active[request.nodeName] == request { + delete(m.active, request.nodeName) + } + } + + if !now.Before(request.wakeAt) { + delete(m.requests, id) + + continue + } + } + + if next.IsZero() || request.wakeAt.Before(next) { + next = request.wakeAt + } + } + + return next +} + +func (m *nodeDetailRequests) notify() { + select { + case m.changed <- struct{}{}: + default: + } +} + +func (m *nodeDetailRequests) run() { + cacheDone := make(chan struct{}) + go func() { + defer close(cacheDone) + + if err := m.cache.Run(m.ctx); err != nil { + m.cancel() + } + }() + + for m.ctx.Err() == nil { + m.mu.Lock() + next := m.expireLocked(time.Now()) + m.mu.Unlock() + + var ( + timer *time.Timer + timerC <-chan time.Time + ) + + if !next.IsZero() { + timer = time.NewTimer(time.Until(next)) + timerC = timer.C + } + + select { + case <-m.ctx.Done(): + case <-m.changed: + case <-timerC: + } + + if timer != nil { + timer.Stop() + } + } + + m.mu.Lock() + m.closed = true + + for _, request := range m.requests { + request.cancel() + } + + clear(m.requests) + clear(m.active) + m.cache.Clear() + m.mu.Unlock() + m.workers.Wait() + <-cacheDone + close(m.done) +} diff --git a/cmd/unbounded-net-controller/detail_requests_test.go b/cmd/unbounded-net-controller/detail_requests_test.go new file mode 100644 index 000000000..af5c38567 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_requests_test.go @@ -0,0 +1,316 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "k8s.io/apimachinery/pkg/types" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func testDetailRequests(t *testing.T, hooks nodeDetailRequestHooks) *nodeDetailRequests { + t.Helper() + + if hooks.Resolve == nil { + hooks.Resolve = func(string) (types.UID, error) { return "uid", nil } + } + + cache, err := newNodeDetailCache(10 * time.Second) + if err != nil { + t.Fatal(err) + } + + manager, err := newNodeDetailRequests(t.Context(), cache, 3*time.Second, hooks) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(manager.Close) + + return manager +} + +func testDetailStatus() *NodeStatusResponse { + return &NodeStatusResponse{Timestamp: time.Now(), NodeInfo: NodeInfo{Name: "node"}} +} + +func TestDetailRequestsValidation(t *testing.T) { + cache, _ := newNodeDetailCache(time.Second) + resolve := func(string) (types.UID, error) { return "uid", nil } + + for _, tc := range []struct { + cache *nodeDetailCache + timeout time.Duration + hooks nodeDetailRequestHooks + }{ + {nil, time.Second, nodeDetailRequestHooks{Resolve: resolve}}, + {cache, 0, nodeDetailRequestHooks{Resolve: resolve}}, + {cache, -time.Second, nodeDetailRequestHooks{Resolve: resolve}}, + {cache, time.Second, nodeDetailRequestHooks{}}, + } { + if _, err := newNodeDetailRequests(t.Context(), tc.cache, tc.timeout, tc.hooks); err == nil { + t.Fatal("invalid constructor accepted") + } + } +} + +func TestDetailRequestsCoalesceAndDispatch(t *testing.T) { + for _, activeWS := range []bool{false, true} { + t.Run(map[bool]string{false: "HTTP", true: "WebSocket"}[activeWS], func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var pulls, sends atomic.Int32 + + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: func(context.Context, string, statusv1alpha1.DetailRequest) (bool, error) { + sends.Add(1) + + return activeWS, nil + }, + Pull: func(ctx context.Context, _ string) (*NodeStatusResponse, error) { + pulls.Add(1) + <-ctx.Done() + + return nil, ctx.Err() + }, + }) + first := manager.Request("node", false) + + var workers sync.WaitGroup + + for range 32 { + workers.Go(func() { + result := manager.Request("node", true) + if result.State != statusv1alpha1.NodeDetailPending || result.RequestID != first.RequestID || result.Deadline != first.Deadline { + t.Error("concurrent refresh did not coalesce") + } + }) + } + + workers.Wait() + synctest.Wait() + + if sends.Load() != 1 || pulls.Load() != map[bool]int32{false: 1, true: 0}[activeWS] { + t.Fatal("unexpected dispatch count") + } + + if _, ok := manager.Pending("node"); ok { + t.Fatal("poll command available before pull failure") + } + }) + }) + } +} + +func TestDetailRequestsCacheRefreshAndDuplicate(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + request := manager.Request("node", false) + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + original := manager.Result("node", request.RequestID) + + time.Sleep(time.Second) + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + duplicate := manager.Request("node", false) + if duplicate.State != statusv1alpha1.NodeDetailComplete || duplicate.Details == nil || + *duplicate.Details != *original.Details { + t.Fatal("duplicate changed data or receipt TTL") + } + + refresh := manager.Request("node", true) + if refresh.RequestID == request.RequestID || refresh.State != statusv1alpha1.NodeDetailPending { + t.Fatal("refresh did not create a fresh request") + } + + if cached := manager.Request("node", false); cached.RequestID != request.RequestID || cached.Details == nil { + t.Fatal("pending refresh prevented reuse of valid data") + } + + if err := manager.Complete("node", refresh.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + if old := manager.Result("node", request.RequestID); old.State != statusv1alpha1.NodeDetailExpired || old.Details != nil { + t.Fatal("old request retained a second result") + } + + time.Sleep(10 * time.Second) + synctest.Wait() + + if result := manager.Result("node", refresh.RequestID); result.State != statusv1alpha1.NodeDetailExpired || result.Details != nil { + t.Fatal("result still available at TTL boundary") + } + + assertNodeDetailEntries(t, manager.cache, 0) + }) +} + +func TestDetailRequestsFallbackDeadlineAndCleanup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: func(context.Context, string, statusv1alpha1.DetailRequest) (bool, error) { + return true, errors.New("socket closed") + }, + Pull: func(ctx context.Context, _ string) (*NodeStatusResponse, error) { + deadline, ok := ctx.Deadline() + if !ok || deadline != time.Now().Add(3*time.Second) { + t.Error("pull did not inherit the overall deadline") + } + + time.Sleep(time.Second) + + return nil, errors.New("unreachable") + }, + }) + request := manager.Request("node", false) + + synctest.Wait() + time.Sleep(time.Second) + synctest.Wait() + + for range 2 { + command, ok := manager.Pending("node") + if !ok || command.RequestID != request.RequestID || command.Deadline != request.Deadline { + t.Fatal("failed pull did not expose the unchanged polling command") + } + } + + time.Sleep(2*time.Second - time.Nanosecond) + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailPending { + t.Fatal("request expired too early") + } + + time.Sleep(time.Nanosecond) + synctest.Wait() + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailExpired { + t.Fatal("request remained pending at deadline") + } + + if _, ok := manager.Pending("node"); ok { + t.Fatal("expired polling command retained") + } + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("late result accepted") + } + + assertNodeDetailEntries(t, manager.cache, 0) + time.Sleep(manager.timeout) + synctest.Wait() + manager.mu.Lock() + count := len(manager.requests) + len(manager.active) + manager.mu.Unlock() + + if count != 0 { + t.Fatal("terminal metadata was not proactively removed") + } + }) +} + +func TestDetailRequestsBindingAndDeletion(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + uid := types.UID("old") + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Resolve: func(string) (types.UID, error) { return uid, nil }, + }) + request := manager.Request("node", false) + + for _, status := range []*NodeStatusResponse{nil, {}, {NodeInfo: NodeInfo{Name: "wrong"}}, {NodeInfo: NodeInfo{Name: "node"}, FetchError: "failed"}} { + if err := manager.Complete("node", request.RequestID, status); err == nil { + t.Fatal("invalid details accepted") + } + } + + if err := manager.Complete("other", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("wrong node binding accepted") + } + + if err := manager.Complete("node", "unknown", testDetailStatus()); err == nil { + t.Fatal("unknown request accepted") + } + + synctest.Wait() + + uid = "replacement" + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("replaced node accepted") + } + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailUnavailable { + t.Fatal("replacement did not invalidate request") + } + + fresh := manager.Request("node", false) + manager.InvalidateNode("node", "old") + + if err := manager.Complete("node", fresh.RequestID, testDetailStatus()); err != nil { + t.Fatalf("old informer event invalidated new node: %v", err) + } + + manager.InvalidateNode("node", "replacement") + assertNodeDetailEntries(t, manager.cache, 0) + + if result := manager.Result("node", fresh.RequestID); result.State != statusv1alpha1.NodeDetailUnavailable { + t.Fatal("node deletion did not invalidate cached details") + } + }) +} + +func TestDetailRequestsHTTPCompletionAndShutdown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Pull: func(context.Context, string) (*NodeStatusResponse, error) { return testDetailStatus(), nil }, + }) + request := manager.Request("node", false) + + synctest.Wait() + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailComplete || result.Details == nil { + t.Fatal("HTTP pull did not complete") + } + + manager.Close() + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailRetryable || result.Details != nil { + t.Fatal("shutdown did not make results retryable") + } + + if result := manager.Request("node", true); result.State != statusv1alpha1.NodeDetailRetryable { + t.Fatal("shutdown accepted a new request") + } + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("shutdown accepted late details") + } + + assertNodeDetailEntries(t, manager.cache, 0) + restarted := testDetailRequests(t, nodeDetailRequestHooks{}) + + if result := restarted.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailRetryable { + t.Fatal("new leader pretended to own an old request") + } + + if result := restarted.Request("node", false); result.RequestID == request.RequestID { + t.Fatal("restart reused an old request ID") + } + }) +} diff --git a/internal/net/status/v1alpha1/details.go b/internal/net/status/v1alpha1/details.go new file mode 100644 index 000000000..ae0f3609a --- /dev/null +++ b/internal/net/status/v1alpha1/details.go @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import "time" + +type NodeDetailState string + +const ( + NodeDetailPending NodeDetailState = "pending" + NodeDetailComplete NodeDetailState = "complete" + NodeDetailExpired NodeDetailState = "expired" + NodeDetailUnavailable NodeDetailState = "unavailable" + NodeDetailRetryable NodeDetailState = "retryable" +) + +// NodeDetailSnapshot is a single expiring, immutable diagnostic result. +type NodeDetailSnapshot struct { + NodeName string `json:"nodeName"` + RequestID string `json:"requestId"` + CollectedAt time.Time `json:"collectedAt"` + ReceivedAt time.Time `json:"receivedAt"` + ExpiresAt time.Time `json:"expiresAt"` + Status *NodeStatusResponse `json:"status"` +} + +// NodeDetailResult describes a leader-local request. Details are absent unless +// the corresponding snapshot is still available in the detail cache. +type NodeDetailResult struct { + State NodeDetailState `json:"state"` + NodeName string `json:"nodeName"` + RequestID string `json:"requestId,omitempty"` + Deadline time.Time `json:"deadline,omitempty"` + Error string `json:"error,omitempty"` + Details *NodeDetailSnapshot `json:"details,omitempty"` +} From ba3d61b043ae479ffd0d63b13e1e575ff4da1721 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:41:18 +0000 Subject: [PATCH 16/34] net-controller: serve authorized asynchronous node detail requests Add per-node POST and GET detail APIs using existing viewer and aggregated authorization. Start request and cache lifecycles with the leader context and configured durations, invalidate replaced or deleted node UIDs, and use informer-resolved HTTP pulls regardless of background pull settings. Reject oversized decoded HTTP detail payloads without imposing a cache or WebSocket cap. Preserve legacy status routes and publication behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-controller/detail_api.go | 118 ++++++ .../detail_api_test.go | 374 ++++++++++++++++++ .../detail_lifecycle.go | 164 ++++++++ cmd/unbounded-net-controller/health_state.go | 9 + cmd/unbounded-net-controller/main.go | 14 +- cmd/unbounded-net-controller/server.go | 2 + 6 files changed, 679 insertions(+), 2 deletions(-) create mode 100644 cmd/unbounded-net-controller/detail_api.go create mode 100644 cmd/unbounded-net-controller/detail_api_test.go create mode 100644 cmd/unbounded-net-controller/detail_lifecycle.go diff --git a/cmd/unbounded-net-controller/detail_api.go b/cmd/unbounded-net-controller/detail_api.go new file mode 100644 index 000000000..ba720e8e3 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_api.go @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + "k8s.io/klog/v2" + + "github.com/Azure/unbounded/internal/net/authn" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" + webhookpkg "github.com/Azure/unbounded/internal/net/webhook" +) + +func registerNodeDetailHandlers(mux *http.ServeMux, health *healthState, requireAuth bool, webhookServer *webhookpkg.Server, authorizer *dashboardAuthorizer, issuer *authn.TokenIssuer) { + mux.HandleFunc("/status/node/{name}/details", func(w http.ResponseWriter, r *http.Request) { + if !authorizeDashboardOrAggregated(requireAuth, issuer, authorizer, webhookServer, r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + + return + } + + nodeName := r.PathValue("name") + manager := health.getDetailRequests() + + if !health.isLeader.Load() || manager == nil { + writeNodeDetailResult(w, http.StatusServiceUnavailable, + detailRequestFailure(nodeName, r.URL.Query().Get("requestId"), statusv1alpha1.NodeDetailRetryable, "detail request leader is unavailable")) + + return + } + + var result statusv1alpha1.NodeDetailResult + + switch r.Method { + case http.MethodPost: + var input *struct { + ForceRefresh bool `json:"forceRefresh"` + } + + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + + err := decoder.Decode(&input) + if err == nil { + var extra any + + err = decoder.Decode(&extra) + if errors.Is(err, io.EOF) && input != nil { + err = nil + } else if err == nil || input == nil { + err = errors.New("expected one JSON object") + } + } + + if err != nil { + code := http.StatusBadRequest + + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + code = http.StatusRequestEntityTooLarge + } + + writeNodeDetailResult(w, code, detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailUnavailable, err.Error())) + + return + } + + result = manager.Request(nodeName, input.ForceRefresh) + case http.MethodGet: + requestID := r.URL.Query().Get("requestId") + if requestID == "" { + writeNodeDetailResult(w, http.StatusBadRequest, + detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailUnavailable, "requestId is required")) + + return + } + + result = manager.Result(nodeName, requestID) + default: + w.Header().Set("Allow", "GET, POST") + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + + return + } + + code := http.StatusOK + + switch result.State { + case statusv1alpha1.NodeDetailPending: + code = http.StatusAccepted + case statusv1alpha1.NodeDetailExpired: + code = http.StatusGone + case statusv1alpha1.NodeDetailUnavailable: + code = http.StatusNotFound + case statusv1alpha1.NodeDetailRetryable: + code = http.StatusServiceUnavailable + case statusv1alpha1.NodeDetailComplete: + } + + writeNodeDetailResult(w, code, result) + }) +} + +func writeNodeDetailResult(w http.ResponseWriter, code int, result statusv1alpha1.NodeDetailResult) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(code) + + if err := json.NewEncoder(w).Encode(result); err != nil { + klog.V(4).Infof("node detail response encode failed: %v", err) + } +} diff --git a/cmd/unbounded-net-controller/detail_api_test.go b/cmd/unbounded-net-controller/detail_api_test.go new file mode 100644 index 000000000..c09e13f20 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_api_test.go @@ -0,0 +1,374 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "testing/synctest" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func serveDetailRequest(t *testing.T, mux *http.ServeMux, method, path, body string) (*httptest.ResponseRecorder, statusv1alpha1.NodeDetailResult) { + t.Helper() + + recorder := httptest.NewRecorder() + mux.ServeHTTP(recorder, httptest.NewRequest(method, path, strings.NewReader(body))) + + var result statusv1alpha1.NodeDetailResult + if recorder.Header().Get("Content-Type") == "application/json" { + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + } + + return recorder, result +} + +func TestDetailAPIRequestAndResult(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + health := &healthState{detailRequests: manager} + health.isLeader.Store(true) + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, false, nil, nil, nil) + + path := "/status/node/node/details" + response, pending := serveDetailRequest(t, mux, http.MethodPost, path, `{}`) + + if response.Code != http.StatusAccepted || pending.State != statusv1alpha1.NodeDetailPending || pending.RequestID == "" { + t.Fatalf("POST did not return a pending request: %d %s", response.Code, response.Body.String()) + } + + if err := manager.Complete("node", pending.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + response, complete := serveDetailRequest(t, mux, http.MethodGet, path+"?requestId="+pending.RequestID, "") + if response.Code != http.StatusOK || complete.State != statusv1alpha1.NodeDetailComplete || + complete.Details == nil || complete.Details.Status.NodeInfo.Name != "node" || + response.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("GET did not return cached details: %d %s", response.Code, response.Body.String()) + } + + response, reused := serveDetailRequest(t, mux, http.MethodPost, path, `{"forceRefresh":false}`) + if response.Code != http.StatusOK || reused.RequestID != pending.RequestID { + t.Fatal("POST did not reuse existing details") + } + + response, refresh := serveDetailRequest(t, mux, http.MethodPost, path, `{"forceRefresh":true}`) + if response.Code != http.StatusAccepted || refresh.RequestID == pending.RequestID { + t.Fatal("forced refresh did not create a new request") + } + + time.Sleep(manager.timeout) + synctest.Wait() + + response, expired := serveDetailRequest(t, mux, http.MethodGet, path+"?requestId="+refresh.RequestID, "") + + if response.Code != http.StatusGone || expired.State != statusv1alpha1.NodeDetailExpired || expired.Details != nil { + t.Fatal("expired request was not explicit") + } + + health.setLeader(false) + + response, stopped := serveDetailRequest(t, mux, http.MethodGet, path+"?requestId="+pending.RequestID, "") + + if response.Code != http.StatusServiceUnavailable || stopped.State != statusv1alpha1.NodeDetailRetryable || stopped.Details != nil { + t.Fatal("leadership loss did not produce retryable failure") + } + + assertNodeDetailEntries(t, manager.cache, 0) + }) +} + +func TestDetailAPIMethodsAndErrors(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + health := &healthState{detailRequests: manager} + health.isLeader.Store(true) + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, false, nil, nil, nil) + + for _, tc := range []struct { + method string + query string + body string + code int + }{ + {http.MethodDelete, "", "", http.StatusMethodNotAllowed}, + {http.MethodGet, "", "", http.StatusBadRequest}, + {http.MethodGet, "?requestId=unknown", "", http.StatusServiceUnavailable}, + {http.MethodPost, "", "", http.StatusBadRequest}, + {http.MethodPost, "", "null", http.StatusBadRequest}, + {http.MethodPost, "", "{} {}", http.StatusBadRequest}, + {http.MethodPost, "", `{"forceRefresh":"yes"}`, http.StatusBadRequest}, + {http.MethodPost, "", `{"url":"http://caller-controlled"}`, http.StatusBadRequest}, + {http.MethodPost, "", strings.Repeat(" ", 1<<20) + "{}", http.StatusRequestEntityTooLarge}, + } { + response, _ := serveDetailRequest(t, mux, tc.method, "/status/node/node/details"+tc.query, tc.body) + if response.Code != tc.code { + t.Fatalf("%s %s: got %d, want %d: %s", tc.method, tc.query, response.Code, tc.code, response.Body.String()) + } + + if tc.code == http.StatusMethodNotAllowed && response.Header().Get("Allow") != "GET, POST" { + t.Fatal("missing Allow header") + } + } + + manager.mu.Lock() + count := len(manager.requests) + manager.mu.Unlock() + + if count != 0 { + t.Fatal("invalid API requests created work") + } + }) +} + +func TestDetailAPIAuthorization(t *testing.T) { + for _, allowed := range []bool{false, true} { + t.Run(strconv.FormatBool(allowed), func(t *testing.T) { + client := k8sfake.NewClientset() + client.PrependReactor("create", "subjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + review := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SubjectAccessReview) + if review.Spec.ResourceAttributes.Name != "dashboard" || review.Spec.ResourceAttributes.Verb != "get" { + t.Error("detail API changed the existing authorization resource") + } + + return true, &authorizationv1.SubjectAccessReview{Status: authorizationv1.SubjectAccessReviewStatus{Allowed: allowed}}, nil + }) + + issuer := testTokenIssuer(t) + + viewer, _, err := issuer.IssueViewerToken("viewer", nil, time.Hour) + if err != nil { + t.Fatal(err) + } + + health := &healthState{detailRequests: testDetailRequests(t, nodeDetailRequestHooks{})} + health.isLeader.Store(true) + + proxy, trustedTLS := testNodeTokenFrontProxy(t) + mux := http.NewServeMux() + registerStatusHandlers(mux, health, true, proxy, newDashboardAuthorizer(client), issuer) + + for _, token := range []string{"", "invalid", testNodeToken(t, issuer), viewer} { + request := httptest.NewRequest(http.MethodPost, "/status/node/node/details", strings.NewReader("{}")) + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + + want := http.StatusUnauthorized + if token == viewer && allowed { + want = http.StatusAccepted + } + + if response.Code != want { + t.Fatalf("authorization: got %d, want %d", response.Code, want) + } + } + + request := httptest.NewRequest(http.MethodPost, "/status/node/node/details", strings.NewReader("{}")) + request.TLS = trustedTLS + request.Header.Set("X-Remote-User", "aggregated-viewer") + + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + + if response.Code != http.StatusAccepted { + t.Fatalf("trusted aggregated request rejected: %d %s", response.Code, response.Body.String()) + } + }) + } +} + +func testDetailLifecycle(t *testing.T, port int) (*healthState, cache.SharedIndexInformer, *nodeDetailRequests) { + t.Helper() + + health := &healthState{ + statusDetailCacheTTL: 10 * time.Second, statusDetailRequestTimeout: 3 * time.Second, + nodeAgentHealthPort: port, + } + health.isLeader.Store(true) + + factory := informers.NewSharedInformerFactory(k8sfake.NewClientset(), 0) + informer := factory.Core().V1().Nodes().Informer() + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node", UID: "uid"}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "127.0.0.1"}}}, + } + if err := informer.GetIndexer().Add(node); err != nil { + t.Fatal(err) + } + + manager, err := health.startDetailRequests(t.Context(), informer) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(manager.Close) + + return health, informer, manager +} + +func TestDetailAPIHTTPPull(t *testing.T) { + for _, mode := range []string{"success", "failure", "wrong-node", "oversized"} { + t.Run(mode, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/status/json" || r.Method != http.MethodGet { + t.Error("incorrect node detail pull endpoint") + } + + if mode == "failure" { + http.Error(w, "unreachable", http.StatusServiceUnavailable) + + return + } + + status := testDetailStatus() + if mode == "wrong-node" { + status.NodeInfo.Name = "other" + } + + if mode == "oversized" { + status.NodeInfo.K8sLabels = map[string]string{"large": strings.Repeat("x", 1<<20)} + } + + json.NewEncoder(w).Encode(status) + })) + defer server.Close() + + _, portText, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) + if err != nil { + t.Fatal(err) + } + + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatal(err) + } + + health, _, manager := testDetailLifecycle(t, port) + if health.pullEnabled.Load() { + t.Fatal("test must exercise disabled background pulls") + } + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, false, nil, nil, nil) + _, request := serveDetailRequest(t, mux, http.MethodPost, "/status/node/node/details", "{}") + + deadline := time.NewTimer(time.Second) + defer deadline.Stop() + + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + + for { + response, result := serveDetailRequest(t, mux, http.MethodGet, "/status/node/node/details?requestId="+request.RequestID, "") + if mode == "success" && result.State == statusv1alpha1.NodeDetailComplete { + if response.Code != http.StatusOK || result.Details == nil || result.Details.Status.NodeInfo.Name != "node" { + t.Fatal("HTTP pull result is incomplete") + } + + break + } + + if mode != "success" { + if _, ok := manager.Pending("node"); ok { + result = manager.Result("node", request.RequestID) + + if result.Details != nil || result.Error == "" { + t.Fatal("failed pull returned success-shaped details") + } + + if mode == "oversized" && !strings.Contains(result.Error, "1 MiB") { + t.Fatal("oversized response was not explicit") + } + + break + } + } + + select { + case <-deadline.C: + t.Fatalf("HTTP pull did not settle: %+v", result) + case <-ticker.C: + } + } + }) + } +} + +func TestDetailLifecycleInvalidationAndShutdown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + health, informer, manager := testDetailLifecycle(t, 0) + + request := manager.Request("node", false) + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + oldNode := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", UID: "uid"}} + newNode := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", UID: "new"}} + detailNodeEvents(manager).OnUpdate(oldNode, newNode) + assertNodeDetailEntries(t, manager.cache, 0) + + if _, err := health.startDetailRequests(t.Context(), informer); err == nil { + t.Fatal("duplicate lifecycle initialized") + } + + health.setLeader(false) + synctest.Wait() + + if health.getDetailRequests() != nil { + t.Fatal("leadership loss retained manager") + } + + ctx, cancel := context.WithCancel(t.Context()) + + health.isLeader.Store(true) + + restarted, err := health.startDetailRequests(ctx, informer) + if err != nil { + t.Fatal(err) + } + + restarted.Request("node", true) + detailNodeEvents(restarted).OnDelete(cache.DeletedFinalStateUnknown{Obj: oldNode}) + detailNodeEvents(restarted).OnDelete("invalid") + cancel() + restarted.Close() + synctest.Wait() + + if health.getDetailRequests() != nil { + t.Fatal("context cancellation retained manager") + } + + assertNodeDetailEntries(t, restarted.cache, 0) + }) +} diff --git a/cmd/unbounded-net-controller/detail_lifecycle.go b/cmd/unbounded-net-controller/detail_lifecycle.go new file mode 100644 index 000000000..4f3ccf013 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_lifecycle.go @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +func (h *healthState) getDetailRequests() *nodeDetailRequests { + h.detailMu.Lock() + defer h.detailMu.Unlock() + + return h.detailRequests +} + +func (h *healthState) stopDetailRequests() { + h.detailMu.Lock() + manager := h.detailRequests + h.detailRequests = nil + h.detailMu.Unlock() + + if manager != nil { + manager.Close() + } +} + +// startDetailRequests is called once per leadership term, with that term's +// context and node informer. Neither hooks nor workers use the HTTP caller's +// context, so disconnecting a viewer does not cancel another viewer's request. +func (h *healthState) startDetailRequests(ctx context.Context, nodeInformer cache.SharedIndexInformer) (*nodeDetailRequests, error) { + if nodeInformer == nil { + return nil, errors.New("node detail requests require a node informer") + } + + detailCache, err := newNodeDetailCache(h.statusDetailCacheTTL) + if err != nil { + return nil, err + } + + lister := corev1listers.NewNodeLister(nodeInformer.GetIndexer()) + + manager, err := newNodeDetailRequests(ctx, detailCache, h.statusDetailRequestTimeout, nodeDetailRequestHooks{ + Resolve: func(name string) (types.UID, error) { + node, err := lister.Get(name) + if err != nil { + return "", err + } + + return node.UID, nil + }, + Pull: func(ctx context.Context, name string) (*NodeStatusResponse, error) { + node, err := lister.Get(name) + if err != nil { + return nil, err + } + + for _, address := range node.Status.Addresses { + if address.Type != corev1.NodeInternalIP || net.ParseIP(address.Address) == nil { + continue + } + + host := address.Address + if net.ParseIP(host).To4() == nil { + host = "[" + host + "]" + } + + status, err := fetchNodeStatus(ctx, host, h.nodeAgentHealthPort) + if err != nil { + return nil, err + } + + // The legacy pull decoder is unbounded. Enforce the HTTP status + // payload limit before retaining its result, not in the cache + // shared with transports that have different frame limits. + payload, err := json.Marshal(status) + if err != nil { + return nil, fmt.Errorf("encode pulled node details: %w", err) + } + + if len(payload) > 1<<20 { + return nil, errors.New("HTTP detail response exceeds the 1 MiB status payload limit") + } + + return status, nil + } + + return nil, fmt.Errorf("node %q has no valid InternalIP", name) + }, + }) + if err != nil { + return nil, err + } + + registration, err := nodeInformer.AddEventHandler(detailNodeEvents(manager)) + if err != nil { + manager.Close() + + return nil, fmt.Errorf("register detail node invalidation: %w", err) + } + + h.detailMu.Lock() + if !h.isLeader.Load() || ctx.Err() != nil || h.detailRequests != nil { + h.detailMu.Unlock() + manager.Close() + + if err := nodeInformer.RemoveEventHandler(registration); err != nil { + klog.Warningf("Removing node detail event handler: %v", err) + } + + return nil, errors.New("detail request leadership is unavailable or already initialized") + } + + h.detailRequests = manager + h.detailMu.Unlock() + + go func() { + <-manager.done + + if err := nodeInformer.RemoveEventHandler(registration); err != nil { + klog.Warningf("Removing node detail event handler: %v", err) + } + + h.detailMu.Lock() + if h.detailRequests == manager { + h.detailRequests = nil + } + h.detailMu.Unlock() + }() + + return manager, nil +} + +func detailNodeEvents(manager *nodeDetailRequests) cache.ResourceEventHandlerFuncs { + return cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(oldObj, newObj any) { + oldNode, oldOK := oldObj.(*corev1.Node) + + newNode, newOK := newObj.(*corev1.Node) + if oldOK && newOK && oldNode.UID != newNode.UID { + manager.InvalidateNode(oldNode.Name, oldNode.UID) + } + }, + DeleteFunc: func(obj any) { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + + if node, ok := obj.(*corev1.Node); ok { + manager.InvalidateNode(node.Name, node.UID) + } + }, + } +} diff --git a/cmd/unbounded-net-controller/health_state.go b/cmd/unbounded-net-controller/health_state.go index fcc3eadb6..1f09d89fd 100644 --- a/cmd/unbounded-net-controller/health_state.go +++ b/cmd/unbounded-net-controller/health_state.go @@ -63,6 +63,11 @@ type healthState struct { nodeTokenVerifier serviceAccountTokenVerifier nodeAuthReady func() bool // Required only by the startup-selected local OIDC verifier. + detailMu sync.Mutex + detailRequests *nodeDetailRequests + statusDetailCacheTTL time.Duration + statusDetailRequestTimeout time.Duration + // Pull fallback toggle (controlled via dashboard WS message; default: disabled). pullEnabled atomic.Bool // registerAggregatedAPIServer controls serving aggregated API status push endpoints. @@ -166,6 +171,10 @@ func (h *healthState) setLeader(leader bool) { h.controllerReady.Store(false) } + if !leader { + h.stopDetailRequests() + } + if leader { leaderIsLeader.Set(1) klog.Info("Health: marked as leader") diff --git a/cmd/unbounded-net-controller/main.go b/cmd/unbounded-net-controller/main.go index b604dfbea..f88cc4804 100644 --- a/cmd/unbounded-net-controller/main.go +++ b/cmd/unbounded-net-controller/main.go @@ -129,8 +129,8 @@ on site configuration, and maintain SiteNodeSlice and GatewayPool status.`, flags.IntVar(&cfg.HealthPort, "health-port", 9999, "Port for health check HTTP server (0 to disable)") flags.IntVar(&cfg.NodeAgentHealthPort, "node-agent-health-port", 9998, "Port where node agents serve their health/status endpoints") flags.DurationVar(&cfg.StatusStaleThreshold, "status-stale-threshold", 90*time.Second, "Duration after which a node's pushed status is considered stale") - flags.DurationVar(&cfg.StatusDetailCacheTTL, "status-detail-cache-ttl", config.DefaultStatusDetailCacheTTL, "Lifetime of received node details (positive duration; preparatory)") - flags.DurationVar(&cfg.StatusDetailRequestTimeout, "status-detail-request-timeout", config.DefaultStatusDetailRequestTimeout, "End-to-end node detail request timeout (positive duration; preparatory)") + flags.DurationVar(&cfg.StatusDetailCacheTTL, "status-detail-cache-ttl", config.DefaultStatusDetailCacheTTL, "Lifetime of received node details (positive duration)") + flags.DurationVar(&cfg.StatusDetailRequestTimeout, "status-detail-request-timeout", config.DefaultStatusDetailRequestTimeout, "End-to-end node detail request timeout (positive duration)") flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "Interval between websocket keepalive pings on controller node status streams (0 to disable)") flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "Sequential websocket keepalive ping failures before closing node status websocket") flags.BoolVar(&cfg.RegisterAggregatedAPIServer, "register-aggregated-apiserver", true, "Serve node status push endpoints via aggregated API server paths") @@ -505,6 +505,8 @@ func run(cfg *config.Config, forceNotLeader bool) error { nodeName: os.Getenv("NODE_NAME"), statusCache: NewNodeStatusCache(), staleThreshold: cfg.StatusStaleThreshold, + statusDetailCacheTTL: cfg.StatusDetailCacheTTL, + statusDetailRequestTimeout: cfg.StatusDetailRequestTimeout, tokenAuth: newTokenAuthenticator(nodeTokenVerifier, []string{fmt.Sprintf("%s:unbounded-net-node", controllerNamespace)}), nodeServiceAccount: fmt.Sprintf("%s:unbounded-net-node", controllerNamespace), nodeTokenVerifier: nodeTokenVerifier, @@ -585,6 +587,14 @@ func run(cfg *config.Config, forceNotLeader bool) error { // Set informers in health state for efficient lookups in status endpoints healthState.setInformers(siteCtrl.GetNodeLister(), podLister, siteCtrl.GetSiteInformer(), gatewayPoolInformer, sitePeeringInformer, assignmentInformer, poolPeeringInformer) + detailRequests, err := healthState.startDetailRequests(ctx, informerFactory.Core().V1().Nodes().Informer()) + if err != nil { + klog.Errorf("Failed to start node detail requests: %v", err) + + return + } + defer detailRequests.Close() + healthState.siteController = siteCtrl if healthState.clusterStatusCache != nil { healthState.clusterStatusCache.MarkFullRebuildNeeded() diff --git a/cmd/unbounded-net-controller/server.go b/cmd/unbounded-net-controller/server.go index c71e8b0e7..b625f0c5c 100644 --- a/cmd/unbounded-net-controller/server.go +++ b/cmd/unbounded-net-controller/server.go @@ -396,6 +396,8 @@ func serveStatusJSON(health *healthState, w http.ResponseWriter, r *http.Request } func registerStatusHandlers(mux *http.ServeMux, health *healthState, requireDashboardAuth bool, webhookServer *webhookpkg.Server, dashAuthorizer *dashboardAuthorizer, tokenIssuer *authn.TokenIssuer) { + registerNodeDetailHandlers(mux, health, requireDashboardAuth, webhookServer, dashAuthorizer, tokenIssuer) + mux.HandleFunc("/status/json", func(w http.ResponseWriter, r *http.Request) { if !authorizeDashboardOrAggregated(requireDashboardAuth, tokenIssuer, dashAuthorizer, webhookServer, r) { http.Error(w, "Unauthorized", http.StatusUnauthorized) From 79e5235c3c47b52172d362b425c47bf0d7284d52 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:51:44 +0000 Subject: [PATCH 17/34] net-controller: preserve existing HTTP detail pull response behavior Do not apply the status POST wire-size limit to an uncompressed HTTP GET response. Preserve existing explicit-pull behavior, avoid an extra full JSON allocation, and cover a response larger than one MiB. WebSocket and status POST frame limits are unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- .../detail_api_test.go | 15 ++++++++------ .../detail_lifecycle.go | 20 +------------------ 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/cmd/unbounded-net-controller/detail_api_test.go b/cmd/unbounded-net-controller/detail_api_test.go index c09e13f20..42579d0f7 100644 --- a/cmd/unbounded-net-controller/detail_api_test.go +++ b/cmd/unbounded-net-controller/detail_api_test.go @@ -239,6 +239,8 @@ func testDetailLifecycle(t *testing.T, port int) (*healthState, cache.SharedInde func TestDetailAPIHTTPPull(t *testing.T) { for _, mode := range []string{"success", "failure", "wrong-node", "oversized"} { t.Run(mode, func(t *testing.T) { + pullSucceeds := mode == "success" || mode == "oversized" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/status/json" || r.Method != http.MethodGet { t.Error("incorrect node detail pull endpoint") @@ -290,15 +292,20 @@ func TestDetailAPIHTTPPull(t *testing.T) { for { response, result := serveDetailRequest(t, mux, http.MethodGet, "/status/node/node/details?requestId="+request.RequestID, "") - if mode == "success" && result.State == statusv1alpha1.NodeDetailComplete { + if pullSucceeds && result.State == statusv1alpha1.NodeDetailComplete { if response.Code != http.StatusOK || result.Details == nil || result.Details.Status.NodeInfo.Name != "node" { t.Fatal("HTTP pull result is incomplete") } + // The status POST body limit does not limit legacy HTTP pull responses. + if mode == "oversized" && len(result.Details.Status.NodeInfo.K8sLabels["large"]) != 1<<20 { + t.Fatal("HTTP pull response was truncated to the POST body limit") + } + break } - if mode != "success" { + if !pullSucceeds { if _, ok := manager.Pending("node"); ok { result = manager.Result("node", request.RequestID) @@ -306,10 +313,6 @@ func TestDetailAPIHTTPPull(t *testing.T) { t.Fatal("failed pull returned success-shaped details") } - if mode == "oversized" && !strings.Contains(result.Error, "1 MiB") { - t.Fatal("oversized response was not explicit") - } - break } } diff --git a/cmd/unbounded-net-controller/detail_lifecycle.go b/cmd/unbounded-net-controller/detail_lifecycle.go index 4f3ccf013..3da73cdb8 100644 --- a/cmd/unbounded-net-controller/detail_lifecycle.go +++ b/cmd/unbounded-net-controller/detail_lifecycle.go @@ -5,7 +5,6 @@ package main import ( "context" - "encoding/json" "errors" "fmt" "net" @@ -75,24 +74,7 @@ func (h *healthState) startDetailRequests(ctx context.Context, nodeInformer cach host = "[" + host + "]" } - status, err := fetchNodeStatus(ctx, host, h.nodeAgentHealthPort) - if err != nil { - return nil, err - } - - // The legacy pull decoder is unbounded. Enforce the HTTP status - // payload limit before retaining its result, not in the cache - // shared with transports that have different frame limits. - payload, err := json.Marshal(status) - if err != nil { - return nil, fmt.Errorf("encode pulled node details: %w", err) - } - - if len(payload) > 1<<20 { - return nil, errors.New("HTTP detail response exceeds the 1 MiB status payload limit") - } - - return status, nil + return fetchNodeStatus(ctx, host, h.nodeAgentHealthPort) } return nil, fmt.Errorf("node %q has no valid InternalIP", name) From 415f62187206698d1795e651043c995cbff23217 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 22:00:46 +0000 Subject: [PATCH 18/34] net-controller: dispatch detail commands through active node WebSockets Use authenticated capability-aware connection handles and context-aware serialized writes. Preserve request identity, deadline, and a single dispatcher through reconnects; fall back to HTTP when a command write fails. Old connection teardown cannot remove its replacement. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- .../detail_dispatch.go | 92 ++++++++ .../detail_dispatch_test.go | 206 ++++++++++++++++++ .../detail_lifecycle.go | 1 + .../detail_requests.go | 22 +- cmd/unbounded-net-controller/health_state.go | 41 ++-- cmd/unbounded-net-controller/server.go | 76 +++++-- 6 files changed, 384 insertions(+), 54 deletions(-) create mode 100644 cmd/unbounded-net-controller/detail_dispatch.go create mode 100644 cmd/unbounded-net-controller/detail_dispatch_test.go diff --git a/cmd/unbounded-net-controller/detail_dispatch.go b/cmd/unbounded-net-controller/detail_dispatch.go new file mode 100644 index 000000000..193d4db48 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_dispatch.go @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +type nodeWSConnection struct { + cancel context.CancelFunc + send func(context.Context, statusv1alpha1.DetailRequest) error +} + +func (h *healthState) setNodeWSDetailSender(nodeName string, connection *nodeWSConnection, send func(context.Context, statusv1alpha1.DetailRequest) error) { + h.nodeWSMu.Lock() + + ready := false + if connection != nil && h.nodeWSRegistry[nodeName] == connection { + ready = connection.send == nil + connection.send = send + } + h.nodeWSMu.Unlock() + + if ready { + h.retryNodeDetails(nodeName) + } +} + +func (h *healthState) dispatchNodeDetail(ctx context.Context, nodeName string, command statusv1alpha1.DetailRequest) (bool, error) { + h.nodeWSMu.Lock() + + var send func(context.Context, statusv1alpha1.DetailRequest) error + if connection := h.nodeWSRegistry[nodeName]; connection != nil { + send = connection.send + } + h.nodeWSMu.Unlock() + + if send == nil { + return false, nil + } + + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + err := send(writeCtx, command) + + return err == nil, err +} + +func (h *healthState) retryNodeDetails(nodeName string) { + if manager := h.getDetailRequests(); manager != nil { + manager.Retry(nodeName) + } +} + +// Retry wakes an existing request after a transport change, retaining its ID, +// deadline, and one-dispatch-at-a-time ownership. +func (m *nodeDetailRequests) Retry(nodeName string) { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + if request := m.active[nodeName]; request != nil && m.ctx.Err() == nil && !m.closed { + request.retry = true + if !request.dispatching { + m.startDispatchLocked(request) + } + } +} + +func (m *nodeDetailRequests) startDispatchLocked(request *nodeDetailRequest) { + request.dispatching = true + request.retry = false + request.poll = false + + m.workers.Go(func() { + m.dispatch(request.ctx, request.nodeName, request.command) + + m.mu.Lock() + defer m.mu.Unlock() + + request.dispatching = false + if m.active[request.nodeName] == request && request.retry && m.ctx.Err() == nil && !m.closed { + m.startDispatchLocked(request) + } + }) +} diff --git a/cmd/unbounded-net-controller/detail_dispatch_test.go b/cmd/unbounded-net-controller/detail_dispatch_test.go new file mode 100644 index 000000000..8c75514f6 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_dispatch_test.go @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestDetailDispatchUsesOnlyCurrentCapableConnection(t *testing.T) { + health := &healthState{} + + var canceled, sent atomic.Int32 + + cancel := func() { canceled.Add(1) } + old := health.registerNodeWS("node", cancel) + current := health.registerNodeWS("node", cancel) + health.unregisterNodeWS("node", old) + + if canceled.Load() != 1 { + t.Fatal("replaced connection was not canceled") + } + + command := statusv1alpha1.DetailRequest{RequestID: "request", Deadline: time.Now().Add(time.Minute)} + if ok, err := health.dispatchNodeDetail(t.Context(), "node", command); ok || err != nil { + t.Fatal("legacy connection was treated as command-capable") + } + + health.setNodeWSDetailSender("node", old, func(context.Context, statusv1alpha1.DetailRequest) error { + t.Error("an obsolete connection sent a command") + return nil + }) + health.setNodeWSDetailSender("node", current, func(_ context.Context, got statusv1alpha1.DetailRequest) error { + if got != command { + t.Error("command identity or deadline changed") + } + + sent.Add(1) + + return nil + }) + + if ok, err := health.dispatchNodeDetail(t.Context(), "node", command); !ok || err != nil || sent.Load() != 1 { + t.Fatalf("current connection did not receive command: %v %v", ok, err) + } + + health.unregisterNodeWS("node", current) + + if ok, _ := health.dispatchNodeDetail(t.Context(), "node", command); ok { + t.Fatal("closed connection remained usable") + } +} + +func TestDetailDispatchDisconnectAndReconnectPreserveDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + health := &healthState{} + commands := make(chan statusv1alpha1.DetailRequest, 3) + sender := func(_ context.Context, command statusv1alpha1.DetailRequest) error { + select { + case commands <- command: + return nil + default: + return errors.New("unexpected extra dispatch") + } + } + + var pulls atomic.Int32 + + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: health.dispatchNodeDetail, + Pull: func(context.Context, string) (*NodeStatusResponse, error) { + pulls.Add(1) + return nil, errors.New("unreachable") + }, + }) + health.detailRequests = manager + connection := health.registerNodeWS("node", func() {}) + health.setNodeWSDetailSender("node", connection, sender) + + request := manager.Request("node", true) + + synctest.Wait() + + if len(commands) != 1 { + t.Fatal("expected one WebSocket command") + } + + first := <-commands + if pulls.Load() != 0 || first.RequestID != request.RequestID { + t.Fatal("active WebSocket did not take priority") + } + + time.Sleep(time.Second) + health.unregisterNodeWS("node", connection) + synctest.Wait() + + pending, ok := manager.Pending("node") + if !ok || pulls.Load() != 1 || pending != first { + t.Fatal("disconnect failed to pull and retain the original POST command") + } + + reconnected := health.registerNodeWS("node", func() {}) + health.setNodeWSDetailSender("node", reconnected, sender) + synctest.Wait() + + if len(commands) != 1 { + t.Fatal("expected one command on reconnect") + } + + if next := <-commands; next != first { + t.Fatal("reconnect reset request identity or deadline") + } + + health.setNodeWSDetailSender("node", reconnected, sender) + synctest.Wait() + + if len(commands) != 0 { + t.Fatal("a routine capability update dispatched another collection") + } + + if err := manager.Complete("node", first.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + health.unregisterNodeWS("node", reconnected) + synctest.Wait() + + if pulls.Load() != 1 { + t.Fatal("a completed request restarted after disconnect") + } + }) +} + +func TestDetailDispatchRetryCoalescesAndCancels(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var active, peak atomic.Int32 + + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: func(ctx context.Context, _ string, _ statusv1alpha1.DetailRequest) (bool, error) { + n := active.Add(1) + if n > peak.Load() { + peak.Store(n) + } + + defer active.Add(-1) + + <-ctx.Done() + + return false, ctx.Err() + }, + }) + manager.Request("node", true) + synctest.Wait() + + for range 10 { + manager.Retry("node") + } + + synctest.Wait() + manager.Close() + + if peak.Load() != 1 || active.Load() != 0 { + t.Fatal("retry created concurrent dispatches or shutdown left one running") + } + }) +} + +func TestDetailDispatchWriteTimeoutFallsBackWithinOriginalDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + health := &healthState{} + connection := health.registerNodeWS("node", func() {}) + health.setNodeWSDetailSender("node", connection, func(ctx context.Context, _ statusv1alpha1.DetailRequest) error { + <-ctx.Done() + return ctx.Err() + }) + + var pulls atomic.Int32 + + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: health.dispatchNodeDetail, + Pull: func(context.Context, string) (*NodeStatusResponse, error) { + pulls.Add(1) + return testDetailStatus(), nil + }, + }) + manager.timeout = 20 * time.Second + health.detailRequests = manager + request := manager.Request("node", true) + + synctest.Wait() + time.Sleep(5 * time.Second) + synctest.Wait() + + result := manager.Result("node", request.RequestID) + if pulls.Load() != 1 || result.State != statusv1alpha1.NodeDetailComplete || !result.Deadline.Equal(request.Deadline) { + t.Fatalf("failed WebSocket write did not fall back within the original deadline: %+v", result) + } + }) +} diff --git a/cmd/unbounded-net-controller/detail_lifecycle.go b/cmd/unbounded-net-controller/detail_lifecycle.go index 3da73cdb8..ce2f49b9f 100644 --- a/cmd/unbounded-net-controller/detail_lifecycle.go +++ b/cmd/unbounded-net-controller/detail_lifecycle.go @@ -50,6 +50,7 @@ func (h *healthState) startDetailRequests(ctx context.Context, nodeInformer cach lister := corev1listers.NewNodeLister(nodeInformer.GetIndexer()) manager, err := newNodeDetailRequests(ctx, detailCache, h.statusDetailRequestTimeout, nodeDetailRequestHooks{ + Dispatch: h.dispatchNodeDetail, Resolve: func(name string) (types.UID, error) { node, err := lister.Get(name) if err != nil { diff --git a/cmd/unbounded-net-controller/detail_requests.go b/cmd/unbounded-net-controller/detail_requests.go index e2cc5bcac..0477b9a81 100644 --- a/cmd/unbounded-net-controller/detail_requests.go +++ b/cmd/unbounded-net-controller/detail_requests.go @@ -26,14 +26,17 @@ type nodeDetailRequestHooks struct { // A request owns metadata and cancellation only, never a result payload. type nodeDetailRequest struct { - nodeName string - uid types.UID - command statusv1alpha1.DetailRequest - state statusv1alpha1.NodeDetailState - message string - wakeAt time.Time - poll bool - cancel context.CancelFunc + nodeName string + uid types.UID + command statusv1alpha1.DetailRequest + state statusv1alpha1.NodeDetailState + message string + wakeAt time.Time + poll bool + cancel context.CancelFunc + ctx context.Context + dispatching bool + retry bool } type nodeDetailRequests struct { @@ -118,10 +121,11 @@ func (m *nodeDetailRequests) Request(nodeName string, forceRefresh bool) statusv } ctx, cancel := context.WithDeadline(m.ctx, request.command.Deadline) request.cancel = cancel + request.ctx = ctx m.requests[request.command.RequestID] = request m.active[nodeName] = request m.notify() - m.workers.Go(func() { m.dispatch(ctx, nodeName, request.command) }) + m.startDispatchLocked(request) return m.resultLocked(request) } diff --git a/cmd/unbounded-net-controller/health_state.go b/cmd/unbounded-net-controller/health_state.go index 1f09d89fd..9b20ca3ed 100644 --- a/cmd/unbounded-net-controller/health_state.go +++ b/cmd/unbounded-net-controller/health_state.go @@ -85,11 +85,11 @@ type healthState struct { // kubeProxyMonitor checks the local kube-proxy health endpoint. kubeProxyMonitor *kubeProxyMonitor - // nodeWSRegistry tracks the active WS cancel function per node name. + // nodeWSRegistry tracks the active authenticated connection per node name. // When a node reconnects, the previous connection is canceled to avoid // duplicate connections consuming resources. nodeWSMu sync.Mutex - nodeWSRegistry map[string]context.CancelFunc + nodeWSRegistry map[string]*nodeWSConnection } const defaultMaxPullConcurrency = 20 @@ -97,45 +97,44 @@ const defaultMaxPullConcurrency = 20 // registerNodeWS registers a WS connection for a node. If an existing // connection is registered for the same node, its context is canceled // to force it to close (preventing duplicate connections). -func (h *healthState) registerNodeWS(nodeName string, cancel context.CancelFunc) { +func (h *healthState) registerNodeWS(nodeName string, cancel context.CancelFunc) *nodeWSConnection { if nodeName == "" { - return + return nil } h.nodeWSMu.Lock() defer h.nodeWSMu.Unlock() if h.nodeWSRegistry == nil { - h.nodeWSRegistry = make(map[string]context.CancelFunc) + h.nodeWSRegistry = make(map[string]*nodeWSConnection) } if prev, ok := h.nodeWSRegistry[nodeName]; ok { - prev() // cancel the old connection + prev.cancel() } - h.nodeWSRegistry[nodeName] = cancel + connection := &nodeWSConnection{cancel: cancel} + h.nodeWSRegistry[nodeName] = connection + + return connection } -// unregisterNodeWS removes a node's WS registration. Only removes if the -// cancel function matches (to avoid unregistering a newer connection). -func (h *healthState) unregisterNodeWS(nodeName string, cancel context.CancelFunc) { - if nodeName == "" { +// unregisterNodeWS cannot remove a newer connection with the same node identity. +func (h *healthState) unregisterNodeWS(nodeName string, connection *nodeWSConnection) { + if connection == nil { return } h.nodeWSMu.Lock() - defer h.nodeWSMu.Unlock() - if h.nodeWSRegistry == nil { - return + removed := h.nodeWSRegistry[nodeName] == connection + if removed { + delete(h.nodeWSRegistry, nodeName) } - // Only remove if it's still our registration (not replaced by a newer connection) - if existing, ok := h.nodeWSRegistry[nodeName]; ok { - // Compare by pointer identity -- Go func values aren't comparable, - // but context.CancelFunc from the same WithCancel call is the same pointer. - if fmt.Sprintf("%p", existing) == fmt.Sprintf("%p", cancel) { - delete(h.nodeWSRegistry, nodeName) - } + h.nodeWSMu.Unlock() + + if removed { + h.retryNodeDetails(nodeName) } } diff --git a/cmd/unbounded-net-controller/server.go b/cmd/unbounded-net-controller/server.go index b625f0c5c..203044441 100644 --- a/cmd/unbounded-net-controller/server.go +++ b/cmd/unbounded-net-controller/server.go @@ -661,37 +661,54 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer } }() - send := func(frameType websocket.MessageType, ackMsgType string, ack NodeStatusPushAck) { - if frameType == websocket.MessageBinary { - payload, marshalErr := marshalProtoAck(ackMsgType, ack) - if marshalErr != nil { - klog.V(4).Infof("Node WebSocket proto ack marshal failed (source=%s, node=%s): %v", source, nodeNameForLog(), marshalErr) - return - } + wsCtx, wsCancel := context.WithCancel(r.Context()) + defer wsCancel() - if writeErr := conn.Write(r.Context(), websocket.MessageBinary, payload); writeErr != nil { - klog.V(4).Infof("Node WebSocket ack write failed (source=%s, node=%s): %v", source, nodeNameForLog(), writeErr) - } + var registration *nodeWSConnection + defer func() { health.unregisterNodeWS(lastWSNodeName, registration) }() - return + writeGate := make(chan struct{}, 1) + + sendContext := func(ctx context.Context, frameType websocket.MessageType, ackMsgType string, ack NodeStatusPushAck) error { + select { + case writeGate <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + case <-wsCtx.Done(): + return wsCtx.Err() + } + + defer func() { <-writeGate }() + + var ( + payload []byte + marshalErr error + ) + if frameType == websocket.MessageBinary { + payload, marshalErr = marshalProtoAck(ackMsgType, ack) + } else { + payload, marshalErr = json.Marshal(map[string]interface{}{"type": ackMsgType, "data": ack}) } - payload, marshalErr := json.Marshal(map[string]interface{}{"type": ackMsgType, "data": ack}) if marshalErr != nil { - klog.V(4).Infof("Node WebSocket ack marshal failed (source=%s, node=%s): %v", source, nodeNameForLog(), marshalErr) - return + return marshalErr } - if writeErr := conn.Write(r.Context(), websocket.MessageText, payload); writeErr != nil { - klog.V(4).Infof("Node WebSocket ack write failed (source=%s, node=%s): %v", source, nodeNameForLog(), writeErr) + return conn.Write(ctx, frameType, payload) + } + send := func(frameType websocket.MessageType, ackMsgType string, ack NodeStatusPushAck) { + if err := sendContext(wsCtx, frameType, ackMsgType, ack); err != nil { + klog.V(4).Infof("Node WebSocket ack failed (source=%s): %v", source, err) + wsCancel() } } - - wsCtx, wsCancel := context.WithCancel(r.Context()) - defer wsCancel() - defer func() { - health.unregisterNodeWS(lastWSNodeName, wsCancel) - }() + enableDetails := func(nodeName string, frameType websocket.MessageType) { + health.setNodeWSDetailSender(nodeName, registration, func(ctx context.Context, command statusv1alpha1.DetailRequest) error { + return sendContext(ctx, frameType, "node_status_ack", NodeStatusPushAck{ + Status: statusv1alpha1.DetailRequestStatus, DetailRequest: &command, SummarySupported: true, + }) + }) + } recvCh := make(chan wsFrame) errCh := make(chan error, 1) @@ -832,7 +849,7 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer if nodeName != "" { if lastWSNodeName == "" { // First message identifies the node -- register and evict old connections. - health.registerNodeWS(nodeName, wsCancel) + registration = health.registerNodeWS(nodeName, wsCancel) } lastWSNodeName = nodeName @@ -840,6 +857,10 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer ackType, ack := handleProtoWSMessage(health, decoded, source) send(websocket.MessageBinary, ackType, ack) + + if ack.Status == "ok" && decoded.message.SupportsDetails { + enableDetails(nodeName, websocket.MessageBinary) + } } else { nodeName, identityErr := extractNodeNameFromWSMessage(frame.data) if identityErr != nil || nodeName == "" { @@ -867,7 +888,7 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer if nodeName != "" { if lastWSNodeName == "" { - health.registerNodeWS(nodeName, wsCancel) + registration = health.registerNodeWS(nodeName, wsCancel) } lastWSNodeName = nodeName @@ -875,6 +896,13 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer ackType, ack := handleNodeStatusWSMessageWithSource(health, frame.data, source) send(websocket.MessageText, ackType, ack) + + var capability struct { + SupportsDetails bool `json:"supportsDetails"` + } + if err := json.Unmarshal(frame.data, &capability); err == nil && ack.Status == "ok" && capability.SupportsDetails { + enableDetails(nodeName, websocket.MessageText) + } } case <-keepaliveCh: // Skip ping if we received a message recently From 0035abd7d027e7c30b736d4bccc7945abc21458d Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 22:13:40 +0000 Subject: [PATCH 19/34] net-controller: correlate one-shot details and poll commands through status ACKs Complete requested details without mutating routine wire revisions, send pending commands in authenticated status POST ACKs, and preserve all capability and correlation fields in actual protobuf HTTP responses. Add explicit failure state and real authenticated JSON/protobuf transport coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- .../detail_responses.go | 74 +++++ .../detail_transport_test.go | 303 ++++++++++++++++++ cmd/unbounded-net-controller/server.go | 36 ++- cmd/unbounded-net-controller/status_proto.go | 26 ++ 4 files changed, 432 insertions(+), 7 deletions(-) create mode 100644 cmd/unbounded-net-controller/detail_responses.go create mode 100644 cmd/unbounded-net-controller/detail_transport_test.go diff --git a/cmd/unbounded-net-controller/detail_responses.go b/cmd/unbounded-net-controller/detail_responses.go new file mode 100644 index 000000000..49c3f0e8d --- /dev/null +++ b/cmd/unbounded-net-controller/detail_responses.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// Fail records a correlated collection failure without deleting an older, +// still-valid snapshot that a viewer may display during a failed refresh. +func (m *nodeDetailRequests) Fail(nodeName, requestID, reason string) error { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + request := m.requests[requestID] + if reason == "" || m.ctx.Err() != nil || m.closed || request == nil || request.nodeName != nodeName { + return errors.New("detail failure does not match an available request") + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + return errors.New("detail request node was deleted or replaced") + } + + if request.state == statusv1alpha1.NodeDetailComplete { + return nil + } + + if request.state != statusv1alpha1.NodeDetailPending { + return errors.New("detail request is no longer pending") + } + + request.cancel() + request.state = statusv1alpha1.NodeDetailUnavailable + request.message = reason + request.poll = false + request.wakeAt = time.Now().Add(m.timeout) + delete(m.active, nodeName) + m.notify() + + return nil +} + +func handleNodeDetailResponse(health *healthState, nodeName, requestID string, status *NodeStatusResponse, failure string) NodeStatusPushAck { + ack := NodeStatusPushAck{Status: "error", DetailRequestID: requestID, SummarySupported: true} + + manager := health.getDetailRequests() + if manager == nil || requestID == "" { + ack.Reason = "detail request leader or request identity is unavailable" + return ack + } + + var err error + if failure != "" { + err = manager.Fail(nodeName, requestID, failure) + } else { + err = manager.Complete(nodeName, requestID, status) + } + + if err != nil { + ack.Reason = err.Error() + return ack + } + + ack.Status = "ok" + + return ack +} diff --git a/cmd/unbounded-net-controller/detail_transport_test.go b/cmd/unbounded-net-controller/detail_transport_test.go new file mode 100644 index 000000000..573e9615b --- /dev/null +++ b/cmd/unbounded-net-controller/detail_transport_test.go @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "testing/synctest" + "time" + + "github.com/coder/websocket" + "google.golang.org/protobuf/proto" + + statuspkg "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func encodeDetailTransportMessage(t *testing.T, binary bool, requestID string) []byte { + t.Helper() + + message := &statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusSummaryType, NodeName: "node-a", SupportsDetails: true, + Summary: &statusproto.NodeStatusOverview{NodeInfo: &statusproto.NodeInfo{Name: "node-a"}, PeerCount: 1}, + } + jsonMessage := NodeStatusWSMessage{Type: message.Type, NodeName: message.NodeName, SupportsDetails: true} + overview := protoToNodeOverview(message.Summary) + jsonMessage.Summary = &overview + + if requestID != "" { + message.Type = statusv1alpha1.NodeStatusDetailsType + message.Summary = nil + message.DetailRequestId = requestID + message.Status = &statusproto.NodeStatusFull{ + NodeInfo: &statusproto.NodeInfo{Name: "node-a"}, + Peers: []*statusproto.PeerStatus{{Name: "peer"}}, + } + full := protoToNodeStatus(message.Status) + jsonMessage.Type = message.Type + jsonMessage.Summary = nil + jsonMessage.Status = &full + jsonMessage.DetailRequestID = requestID + } + + var ( + data []byte + err error + ) + if binary { + data, err = proto.Marshal(message) + } else { + data, err = json.Marshal(jsonMessage) + } + + if err != nil { + t.Fatal(err) + } + + return data +} + +func decodeDetailTransportAck(t *testing.T, binary, ws bool, data []byte) *NodeStatusPushAck { + t.Helper() + + if binary { + var ack statusproto.NodeStatusAck + if err := proto.Unmarshal(data, &ack); err != nil { + t.Fatal(err) + } + + return statuspkg.NodeStatusAckFromProto(&ack) + } + + var ack NodeStatusPushAck + if ws { + var envelope struct { + Data NodeStatusPushAck `json:"data"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + t.Fatal(err) + } + + ack = envelope.Data + } else if err := json.Unmarshal(data, &ack); err != nil { + t.Fatal(err) + } + + return &ack +} + +func awaitDetailTransport(t *testing.T, ready func() bool) { + t.Helper() + + timeout := time.NewTimer(3 * time.Second) + defer timeout.Stop() + + tick := time.NewTicker(time.Millisecond) + defer tick.Stop() + + for !ready() { + select { + case <-timeout.C: + t.Fatal("detail transport did not become ready") + case <-tick.C: + } + } +} + +func TestDetailWebSocketCommandAndResponse(t *testing.T) { + for _, binary := range []bool{false, true} { + t.Run(map[bool]string{false: "json", true: "protobuf"}[binary], func(t *testing.T) { + health := newJSONIdentityHealth() + health.registerAggregatedAPIServer = false + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: health.dispatchNodeDetail, + Pull: func(context.Context, string) (*NodeStatusResponse, error) { + t.Error("active WebSocket request unexpectedly used HTTP") + return nil, nil + }, + }) + health.detailRequests = manager + issuer := testTokenIssuer(t) + mux := http.NewServeMux() + registerPushHandlers(mux, health, nil, make(chan struct{}, 1), issuer) + + server := httptest.NewServer(mux) + defer server.Close() + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + conn, _, err := websocket.Dial(ctx, server.URL+"/status/nodews", &websocket.DialOptions{ + HTTPHeader: http.Header{"Authorization": {"Bearer " + testNodeToken(t, issuer)}}, + }) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + + frameType := websocket.MessageText + if binary { + frameType = websocket.MessageBinary + } + + send := func(id string) { + t.Helper() + + if err := conn.Write(ctx, frameType, encodeDetailTransportMessage(t, binary, id)); err != nil { + t.Fatal(err) + } + } + read := func() *NodeStatusPushAck { + t.Helper() + + _, data, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + + return decodeDetailTransportAck(t, binary, true, data) + } + + send("") + + publication := read() + + awaitDetailTransport(t, func() bool { + health.nodeWSMu.Lock() + defer health.nodeWSMu.Unlock() + + return health.nodeWSRegistry["node-a"] != nil && health.nodeWSRegistry["node-a"].send != nil + }) + + request := manager.Request("node-a", true) + + command := read() + if command.IsPublicationAck() || command.DetailRequest == nil || + command.DetailRequest.RequestID != request.RequestID || !command.SummarySupported { + t.Fatalf("wire command was not distinct from an ordinary ACK: %+v", command) + } + + send(request.RequestID) + + ack := read() + if ack.Status != "ok" || ack.DetailRequestID != request.RequestID || ack.Revision != 0 || ack.IsPublicationAck() { + t.Fatalf("invalid correlated ACK: %+v", ack) + } + + result := manager.Result("node-a", request.RequestID) + if result.Details == nil || len(result.Details.Status.Peers) != 1 { + t.Fatal("one-shot details did not complete the request") + } + + cached, _ := health.statusCache.Get("node-a") + if cached.Revision != publication.Revision || cached.Status.Peers != nil { + t.Fatal("one-shot reply became the routine delta base") + } + + expires := result.Details.ExpiresAt + + send(request.RequestID) + + if duplicate := read(); duplicate.Status != "ok" { + t.Fatal("duplicate detail reply was not idempotent") + } + + if !manager.Result("node-a", request.RequestID).Details.ExpiresAt.Equal(expires) { + t.Fatal("duplicate detail reply renewed TTL") + } + }) + } +} + +func TestDetailHTTPPollingCommandAndResponse(t *testing.T) { + for _, binary := range []bool{false, true} { + t.Run(map[bool]string{false: "json", true: "protobuf"}[binary], func(t *testing.T) { + health := newJSONIdentityHealth() + health.registerAggregatedAPIServer = false + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + health.detailRequests = manager + issuer := testTokenIssuer(t) + token := testNodeToken(t, issuer) + mux := http.NewServeMux() + registerPushHandlers(mux, health, nil, make(chan struct{}, 1), issuer) + + request := manager.Request("node-a", true) + + awaitDetailTransport(t, func() bool { _, ok := manager.Pending("node-a"); return ok }) + + post := func(id string) *NodeStatusPushAck { + t.Helper() + r := httptest.NewRequest(http.MethodPost, "/status/push", bytes.NewReader(encodeDetailTransportMessage(t, binary, id))) + r.Header.Set("Authorization", "Bearer "+token) + + if binary { + r.Header.Set("Content-Type", "application/x-protobuf") + } + + w := httptest.NewRecorder() + mux.ServeHTTP(w, r) + + if w.Code != http.StatusOK { + t.Fatalf("POST failed: %d %s", w.Code, w.Body.String()) + } + + return decodeDetailTransportAck(t, binary, false, w.Body.Bytes()) + } + + publication := post("") + if !publication.SummarySupported || !publication.IsPublicationAck() || + publication.DetailRequest == nil || publication.DetailRequest.RequestID != request.RequestID { + t.Fatalf("POST ACK lost capabilities or polling command: %+v", publication) + } + + detailAck := post(request.RequestID) + if detailAck.Status != "ok" || detailAck.DetailRequestID != request.RequestID || detailAck.IsPublicationAck() || detailAck.DetailRequest != nil { + t.Fatalf("POST detail ACK corrupted publication/polling state: %+v", detailAck) + } + + cached, _ := health.statusCache.Get("node-a") + if cached.Revision != publication.Revision || manager.Result("node-a", request.RequestID).Details == nil { + t.Fatal("POST detail response changed routine state or failed to complete") + } + }) + } +} + +func TestDetailFailureKeepsPreviousValidSnapshot(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + + first := manager.Request("node", true) + if err := manager.Complete("node", first.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + before, _ := manager.cache.Get("node") + + refresh := manager.Request("node", true) + if err := manager.Fail("other", refresh.RequestID, "bad"); err == nil { + t.Fatal("failure for a different node was accepted") + } + + if err := manager.Fail("node", refresh.RequestID, "response exceeds the transport frame limit"); err != nil { + t.Fatal(err) + } + + result := manager.Result("node", refresh.RequestID) + if result.State != statusv1alpha1.NodeDetailUnavailable || result.Error == "" || result.Details != nil { + t.Fatal("collection failure was not explicit") + } + + after, ok := manager.cache.Get("node") + if !ok || after.Status != before.Status || !after.ExpiresAt.Equal(before.ExpiresAt) { + t.Fatal("failed refresh destroyed or renewed the old valid snapshot") + } + }) +} diff --git a/cmd/unbounded-net-controller/server.go b/cmd/unbounded-net-controller/server.go index 203044441..79ecc9e27 100644 --- a/cmd/unbounded-net-controller/server.go +++ b/cmd/unbounded-net-controller/server.go @@ -563,17 +563,19 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer return } + if ack.IsPublicationAck() && ack.Status == "ok" { + if manager := health.getDetailRequests(); manager != nil { + if command, ok := manager.Pending(authorizedNodeName(r)); ok { + ack.DetailRequest = &command + } + } + } + isProto := isProtobufContentType(r) if isProto { w.Header().Set("Content-Type", "application/x-protobuf") - pbAck := &statusproto.NodeStatusAck{ - Status: ack.Status, - Revision: ack.Revision, - Reason: ack.Reason, - } - - data, marshalErr := proto.Marshal(pbAck) + data, marshalErr := marshalProtoAck("node_status_ack", ack) if marshalErr != nil { klog.V(4).Infof("status push proto ack marshal failed: %v", marshalErr) http.Error(w, "internal error", http.StatusInternalServerError) @@ -1237,6 +1239,14 @@ func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, so envelope.Mode = "summary" } + if envelope.Type == statusv1alpha1.NodeStatusDetailsType { + if envelope.Mode != "" && envelope.Mode != "details" { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("conflicting status mode and type") + } + + envelope.Mode = "details" + } + if envelope.Summary != nil && envelope.Mode != "summary" { return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("overview requires summary mode") } @@ -1275,6 +1285,12 @@ func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, so } switch envelope.Mode { + case "details": + if envelope.Delta != nil { + return NodeStatusPushAck{Status: "error", DetailRequestID: envelope.DetailRequestID, Reason: "details cannot include a delta"}, http.StatusOK, nil + } + + return handleNodeDetailResponse(health, nodeName, envelope.DetailRequestID, envelope.Status, ""), http.StatusOK, nil case "summary": if envelope.Summary == nil || envelope.Status != nil || envelope.Delta != nil || envelope.DetailRequestID != "" { return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("summary must contain only overview data") @@ -1356,6 +1372,12 @@ func handleNodeStatusWSMessageWithSource(health *healthState, data []byte, sourc } switch message.Type { + case statusv1alpha1.NodeStatusDetailsType: + if message.Delta != nil { + return "node_status_ack", NodeStatusPushAck{Status: "error", DetailRequestID: message.DetailRequestID, Reason: "details cannot include a delta"} + } + + return "node_status_ack", handleNodeDetailResponse(health, nodeName, message.DetailRequestID, message.Status, "") case statusv1alpha1.NodeStatusSummaryType: if message.Summary == nil || message.Status != nil || message.Delta != nil || message.DetailRequestID != "" { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "summary must contain only overview data"} diff --git a/cmd/unbounded-net-controller/status_proto.go b/cmd/unbounded-net-controller/status_proto.go index 40c91f4e9..5969e3789 100644 --- a/cmd/unbounded-net-controller/status_proto.go +++ b/cmd/unbounded-net-controller/status_proto.go @@ -493,6 +493,19 @@ func handleProtoWSMessage(health *healthState, decoded *decodedProtoWSMessage, s } switch msg.Type { + case statusv1alpha1.NodeStatusDetailsType: + if msg.Delta != nil { + return "node_status_ack", NodeStatusPushAck{Status: "error", DetailRequestID: msg.DetailRequestId, Reason: "details cannot include a delta"} + } + + var status *NodeStatusResponse + + if msg.Status != nil { + full := protoToNodeStatus(msg.Status) + status = &full + } + + return "node_status_ack", handleNodeDetailResponse(health, nodeName, msg.DetailRequestId, status, "") case statusv1alpha1.NodeStatusSummaryType: if msg.Summary == nil || msg.Status != nil || msg.Delta != nil || msg.DetailRequestId != "" { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "summary must contain only overview data"} @@ -569,6 +582,19 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string } switch msg.Type { + case statusv1alpha1.NodeStatusDetailsType: + if msg.Delta != nil { + return NodeStatusPushAck{Status: "error", DetailRequestID: msg.DetailRequestId, Reason: "details cannot include a delta"}, 200, nil + } + + var status *NodeStatusResponse + + if msg.Status != nil { + full := protoToNodeStatus(msg.Status) + status = &full + } + + return handleNodeDetailResponse(health, nodeName, msg.DetailRequestId, status, ""), 200, nil case statusv1alpha1.NodeStatusSummaryType: if msg.Summary == nil || msg.Status != nil || msg.Delta != nil || msg.DetailRequestId != "" { return NodeStatusPushAck{}, 400, fmt.Errorf("summary must contain only overview data") From cff44037a55be189daef747274ec861d85c9ba37 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:43:06 +0000 Subject: [PATCH 20/34] frontend: add summary projection and detail response contracts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- frontend/src/state/clusterSummary.ts | 120 ++++++++++++++++++++++++++ frontend/src/types.ts | 19 ++++ frontend/tests/clusterSummary.test.ts | 79 +++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 frontend/src/state/clusterSummary.ts create mode 100644 frontend/tests/clusterSummary.test.ts diff --git a/frontend/src/state/clusterSummary.ts b/frontend/src/state/clusterSummary.ts new file mode 100644 index 000000000..78fcf58d7 --- /dev/null +++ b/frontend/src/state/clusterSummary.ts @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +import type { ClusterStatus, ClusterStatusDelta, ClusterSummary, ClusterSummaryDelta, NodeStatus, NodeSummary } from '../types'; + +function cniState(source?: string, fetchError?: string, errorCount = 0, routeMismatch = false) { + const [cniStatus, cniTone] = source === 'no-data' ? ['No data', 'warning'] + : fetchError ? ['Fetch error', 'danger'] + : errorCount ? ['Errors', 'danger'] + : routeMismatch ? ['Route mismatch', 'warning'] + : source === 'stale' || source === 'error' ? ['Stale', 'warning'] + : !source ? ['Unknown', 'warning'] : ['Healthy', 'success']; + return { cniStatus, cniTone }; +} + +export function summarizeNode(node: NodeStatus, now = Date.now()): NodeSummary { + const peers = node.peers || []; + const healthyPeers = peers.filter((peer) => peer.healthCheck?.enabled + ? peer.healthCheck.status?.toLowerCase() === 'up' + : Boolean(peer.tunnel?.lastHandshake && now - Date.parse(peer.tunnel.lastHandshake) < 180000)).length; + const routeMismatch = (node.routingTable?.routes || []).some((route) => + (route.nextHops || []).some((hop) => (hop.expected === true) !== (hop.present === true))); + const errorCount = node.nodeErrors?.length || 0; + const source = node.statusSource; + return { + name: node.nodeInfo?.name, + siteName: node.nodeInfo?.siteName, + isGateway: node.nodeInfo?.isGateway, + k8sReady: node.nodeInfo?.k8sReady, + statusSource: source, + ...cniState(source, node.fetchError, errorCount, routeMismatch), errorCount, + firstError: node.nodeErrors?.[0]?.message, + peerCount: peers.length, + healthyPeers, + routeCount: node.routingTable?.routes?.length || 0, + routeMismatch, + fetchError: node.fetchError, + wireGuardOnline: Boolean(node.nodeInfo?.wireGuard?.interface), + }; +} + +// Whitelist wire fields, including for already-summary input. Never spread full +// cluster/node objects into persistent state or the global JSON export. +function summaryNode(node: NodeSummary): NodeSummary { + return { + name: node.name, siteName: node.siteName, isGateway: node.isGateway, + k8sReady: node.k8sReady, statusSource: node.statusSource, + cniStatus: node.cniStatus, cniTone: node.cniTone, + errorCount: node.errorCount, firstError: node.firstError, + peerCount: node.peerCount, healthyPeers: node.healthyPeers, + routeCount: node.routeCount, routeMismatch: node.routeMismatch, + fetchError: node.fetchError, wireGuardOnline: node.wireGuardOnline, + }; +} + +export function toClusterSummary(input: ClusterSummary | ClusterStatus, now = Date.now()): ClusterSummary { + const summary = input as ClusterSummary; + return { + seq: summary.seq, timestamp: input.timestamp, + nodeCount: input.nodeCount, siteCount: input.siteCount, + azureTenantId: input.azureTenantId, leaderInfo: input.leaderInfo, + buildInfo: input.buildInfo, sites: input.sites, + gatewayPools: input.gatewayPools, peerings: input.peerings, + errors: input.errors, warnings: input.warnings, problems: input.problems, + pullEnabled: input.pullEnabled, + nodeSummaries: summary.nodeSummaries != null + ? summary.nodeSummaries.map(summaryNode) + : ((input as ClusterStatus).nodes || []).map((node) => summarizeNode(node, now)), + }; +} + +export function mergeSummary( + current: ClusterSummary | null, delta: ClusterSummaryDelta +): ClusterSummary | null { + if (!current) return null; + if (current.seq != null && delta.seq != null && delta.seq <= current.seq) return current; + const metadata = toClusterSummary(delta); + const merged = { ...current }; + for (const key of Object.keys(metadata) as (keyof ClusterSummary)[]) { + if (key !== 'nodeSummaries' && metadata[key] !== undefined) { + Object.assign(merged, { [key]: metadata[key] }); + } + } + if (delta.nodeSummaries || delta.removedNodes) { + const nodes = new Map((current.nodeSummaries || []).map((node) => [node.name, node])); + for (const name of delta.removedNodes || []) nodes.delete(name); + for (const node of delta.nodeSummaries || []) nodes.set(node.name, summaryNode(node)); + merged.nodeSummaries = [...nodes.values()].sort((a, b) => (a.name || '').localeCompare(b.name || '')); + } + return merged; +} + +export function mergeLegacySummary(current: ClusterSummary | null, delta: ClusterStatusDelta): ClusterSummary | null { + if (delta.nodes) return toClusterSummary({ ...current, ...delta, nodeSummaries: undefined } as ClusterStatus); + // Legacy updatedNodes contains changed top-level fields. Preserve summary + // counts when a patch omits the corresponding full array. + const previous = new Map((current?.nodeSummaries || []).map((node) => [node.name, node])); + const nodeSummaries = (delta.updatedNodes || []).map((node) => { + const next = summarizeNode(node); + const old = previous.get(next.name); + if (!old) return next; + const merged = { + ...old, ...next, + siteName: node.nodeInfo ? next.siteName : old.siteName, + isGateway: node.nodeInfo ? next.isGateway : old.isGateway, + k8sReady: node.nodeInfo ? next.k8sReady : old.k8sReady, + wireGuardOnline: node.nodeInfo ? next.wireGuardOnline : old.wireGuardOnline, + peerCount: node.peers ? next.peerCount : old.peerCount, + healthyPeers: node.peers ? next.healthyPeers : old.healthyPeers, + routeCount: node.routingTable ? next.routeCount : old.routeCount, + routeMismatch: node.routingTable ? next.routeMismatch : old.routeMismatch, + errorCount: node.nodeErrors ? next.errorCount : old.errorCount, + firstError: node.nodeErrors ? next.firstError : old.firstError, + statusSource: node.statusSource ?? old.statusSource, + fetchError: node.fetchError ?? old.fetchError, + }; + return { ...merged, ...cniState(merged.statusSource, merged.fetchError, merged.errorCount, merged.routeMismatch) }; + }); + return mergeSummary(current, { ...toClusterSummary(delta), nodeSummaries, removedNodes: delta.removedNodes }); +} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index c6ad4a4ff..551489cf1 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -266,4 +266,23 @@ export type NodeSummary = { routeCount?: number; routeMismatch?: boolean; fetchError?: string; + wireGuardOnline?: boolean; +}; + +export type NodeDetailSnapshot = { + nodeName: string; + requestId: string; + collectedAt: string; + receivedAt: string; + expiresAt: string; + status: NodeStatus; +}; + +export type NodeDetailResult = { + state: 'pending' | 'complete' | 'expired' | 'unavailable' | 'error'; + nodeName: string; + requestId?: string; + deadline?: string; + error?: string; + details?: NodeDetailSnapshot; }; diff --git a/frontend/tests/clusterSummary.test.ts b/frontend/tests/clusterSummary.test.ts new file mode 100644 index 000000000..53d5bfb33 --- /dev/null +++ b/frontend/tests/clusterSummary.test.ts @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mergeLegacySummary, mergeSummary, summarizeNode, toClusterSummary } from '../src/state/clusterSummary.ts'; + +test('full compatibility projection preserves counts and drops detail arrays', () => { + const node = { + nodeInfo: { name: 'worker', siteName: 'site', wireGuard: { interface: 'wg0' } }, + statusSource: 'push', + peers: [ + { healthCheck: { enabled: true, status: 'up' } }, + { healthCheck: { enabled: true, status: 'down' } }, + { tunnel: { lastHandshake: new Date(99000).toISOString() } }, + { tunnel: { lastHandshake: new Date(0).toISOString() } }, + ], + routingTable: { routes: [{ nextHops: [{ expected: true, present: false }] }] }, + bpfEntries: [{ cidr: 'hidden' }], + }; + const projected = toClusterSummary({ + nodes: [node], nodeCount: 1, siteCount: 1, + sites: [{ name: 'site' }], gatewayPools: [{ name: 'pool' }], peerings: [{ name: 'peering' }], + }, 200000); + assert.equal(projected.nodeSummaries[0].healthyPeers, 2); + assert.equal(projected.nodeSummaries[0].peerCount, 4); + assert.equal(projected.nodeSummaries[0].routeCount, 1); + assert.equal(projected.nodeSummaries[0].cniStatus, 'Route mismatch'); + assert.equal(projected.nodeSummaries[0].wireGuardOnline, true); + assert.deepEqual(projected.peerings, [{ name: 'peering' }]); + assert.deepEqual(projected.sites, [{ name: 'site' }]); + assert.deepEqual(projected.gatewayPools, [{ name: 'pool' }]); + for (const key of ['"nodes"', '"peers"', '"routingTable"', '"nextHops"', '"bpfEntries"']) { + assert.equal(JSON.stringify(projected).includes(key), false, key); + } +}); + +test('summary input wins over legacy full fields and is itself whitelisted', () => { + const summary = toClusterSummary({ + nodeSummaries: [{ name: 'node', peerCount: 19, peers: [{ name: 'hidden' }] }], + nodes: [{ nodeInfo: { name: 'hidden' } }], + } as never); + assert.equal(summary.nodeSummaries[0].peerCount, 19); + assert.equal(JSON.stringify(summary).includes('hidden'), false); +}); + +test('CNI priority preserves no-data, errors, unknown and health', () => { + assert.equal(summarizeNode({ statusSource: 'no-data', fetchError: 'error' }).cniStatus, 'No data'); + assert.equal(summarizeNode({ fetchError: 'error' }).cniStatus, 'Fetch error'); + const errors = summarizeNode({ nodeErrors: [{ message: 'bootstrap blocked' }] }); + assert.equal(errors.firstError, 'bootstrap blocked'); + assert.equal(errors.cniTone, 'danger'); + assert.equal(summarizeNode({}).cniStatus, 'Unknown'); + assert.equal(summarizeNode({ statusSource: 'stale' }).cniStatus, 'Stale'); + assert.equal(summarizeNode({ statusSource: 'push' }).cniTone, 'success'); +}); + +test('summary deltas reject stale sequence and remove nodes without losing resources', () => { + const initial = toClusterSummary({ seq: 4, sites: [{ name: 'site' }], nodeSummaries: [{ name: 'old' }] }); + assert.equal(mergeSummary(initial, { seq: 4, removedNodes: ['old'] }), initial); + const next = mergeSummary(initial, { seq: 5, removedNodes: ['old'], nodeSummaries: [{ name: 'new' }] }); + assert.deepEqual(next.nodeSummaries, toClusterSummary({ nodeSummaries: [{ name: 'new' }] }).nodeSummaries); + assert.deepEqual(next.sites, initial.sites); +}); + +test('legacy partial updates keep counts and CNI facts without retaining full base', () => { + const initial = toClusterSummary({ nodes: [{ + nodeInfo: { name: 'node', siteName: 'site' }, statusSource: 'push', + peers: [{ healthCheck: { enabled: true, status: 'up' } }], + routingTable: { routes: [{ nextHops: [{ expected: true }] }] }, + }] }); + const next = mergeLegacySummary(initial, { updatedNodes: [{ nodeInfo: { name: 'node' }, lastPushTime: 'changed' }] }); + assert.equal(next.nodeSummaries[0].peerCount, 1); + assert.equal(next.nodeSummaries[0].routeMismatch, true); + assert.equal(next.nodeSummaries[0].cniStatus, 'Route mismatch'); + assert.equal(JSON.stringify(next).includes('nextHops'), false); + const empty = mergeLegacySummary(next, { nodes: [] }); + assert.deepEqual(empty.nodeSummaries, []); +}); From 7ae03d45ff2a8ecc15e0df3f646315535f698f4b Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 22:16:02 +0000 Subject: [PATCH 21/34] frontend: remove obsolete node table prop during summary migration NodeTable receives Kubernetes readiness in NodeSummary and does not accept the old nodeK8sStatusMap prop. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- frontend/src/App.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 02cac42a7..fcf712c6c 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -273,7 +273,6 @@ export default function App() { onToggleGatewayPool={toggleGatewayPool} onShowAllGatewayPools={showAllGatewayPools} gatewayByNode={gatewayByNode} - nodeK8sStatusMap={nodeK8sStatusMap} selectedNodeTypes={selectedNodeTypesFilter} onSelectedNodeTypesChange={setSelectedNodeTypesFilter} pullEnabled={effectivePullEnabled} From f03e25314c3a4671d02f364addbe3552634e36e2 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:44:16 +0000 Subject: [PATCH 22/34] frontend: add explicit expiring node detail client lifecycle Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- frontend/src/api.ts | 33 ++++- frontend/src/state/nodeDetails.ts | 189 +++++++++++++++++++++++++++++ frontend/tests/nodeDetails.test.ts | 143 ++++++++++++++++++++++ 3 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 frontend/src/state/nodeDetails.ts create mode 100644 frontend/tests/nodeDetails.test.ts diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 49bce2631..e41322d9a 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 -import { ClusterStatus, ClusterStatusDelta, ClusterSummary, ClusterSummaryDelta, NodeStatus } from './types'; +import { ClusterStatus, ClusterStatusDelta, ClusterSummary, ClusterSummaryDelta, NodeStatus, NodeDetailResult } from './types'; export type StatusEvent = { type: 'cluster_status' | 'cluster_status_delta' | 'cluster_summary' | 'cluster_summary_delta' | 'node_detail_response' | 'node_detail_update'; @@ -15,6 +15,37 @@ function buildControllerUrl(path: string): string { return path; } +async function fetchNodeDetails(path: string, options: RequestInit): Promise { + const response = await fetch(buildControllerUrl(path), { credentials: 'same-origin', ...options }); + const text = await response.text(); + let result: NodeDetailResult; + try { + result = JSON.parse(text); + } catch { + throw new Error(`Detail request failed (${response.status} ${response.statusText})${text ? `: ${text}` : ''}`); + } + if (!response.ok) { + throw new Error(result?.error || `Detail request failed (${response.status} ${response.statusText})`); + } + if (!result || typeof result.state !== 'string') { + throw new Error('Invalid detail response from controller'); + } + return result; +} + +export function requestNodeDetails(name: string, forceRefresh: boolean, signal: AbortSignal) { + return fetchNodeDetails(`/status/node/${encodeURIComponent(name)}/details`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ forceRefresh }), signal, + }); +} + +export function pollNodeDetails(name: string, requestId: string, signal: AbortSignal) { + return fetchNodeDetails(`/status/node/${encodeURIComponent(name)}/details?requestId=${encodeURIComponent(requestId)}`, { + signal, cache: 'no-store', + }); +} + export async function fetchClusterStatus(): Promise { const url = buildControllerUrl('/status/json'); try { diff --git a/frontend/src/state/nodeDetails.ts b/frontend/src/state/nodeDetails.ts new file mode 100644 index 000000000..e438e54d6 --- /dev/null +++ b/frontend/src/state/nodeDetails.ts @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +import type { NodeDetailResult, NodeDetailSnapshot } from '../types'; + +export type DetailView = { + state: 'not-loaded' | 'loading' | 'loaded' | 'expired' | 'error'; + snapshot?: NodeDetailSnapshot; + error?: string; + deadline?: string; +}; +type Timer = ReturnType; +type Operation = { abort: AbortController; deadline: number; requestId?: string; timer?: Timer }; +type Transport = { + request: (name: string, forceRefresh: boolean, signal: AbortSignal) => Promise; + poll: (name: string, requestId: string, signal: AbortSignal) => Promise; +}; +type Clock = { + now: () => number; + setTimeout: (callback: () => void, delay: number) => Timer; + clearTimeout: (timer: Timer) => void; +}; +const browserClock: Clock = { now: Date.now, setTimeout, clearTimeout }; +const notLoaded: DetailView = { state: 'not-loaded' }; + +// Only this store owns retained snapshots. React subscribes to a version, not a +// second cache. Timers and async operations capture names/IDs, never old data. +export class NodeDetails { + private views = new Map(); + private operations = new Map(); + private expiryTimers = new Map(); + private transport: Transport; + private changed: () => void; + private clock: Clock; + + constructor(transport: Transport, changed: () => void, clock: Clock = browserClock) { + this.transport = transport; + this.changed = changed; + this.clock = clock; + } + + read(name: string): DetailView { + this.expire(name); + return this.views.get(name) || notLoaded; + } + + private expire(name: string) { + const view = this.views.get(name); + if (view?.snapshot && !(Date.parse(view.snapshot.expiresAt) > this.clock.now())) { + this.clearExpiry(name); + this.views.set(name, { + state: view.state === 'loading' ? 'loading' : view.error ? 'error' : 'expired', + error: view.error, deadline: view.deadline, + }); + } + } + + private clearExpiry(name: string) { + const timer = this.expiryTimers.get(name); + if (timer !== undefined) this.clock.clearTimeout(timer); + this.expiryTimers.delete(name); + } + + private publish(name: string, view: DetailView) { + this.views.set(name, view); + this.changed(); + } + + cancel(name: string) { + const op = this.operations.get(name); + if (!op) return; + this.operations.delete(name); + op.abort.abort(); + if (op.timer !== undefined) this.clock.clearTimeout(op.timer); + const view = this.read(name); + this.publish(name, { state: view.snapshot ? 'loaded' : 'not-loaded', snapshot: view.snapshot }); + } + + dispose() { + for (const op of this.operations.values()) { + op.abort.abort(); + if (op.timer !== undefined) this.clock.clearTimeout(op.timer); + } + for (const timer of this.expiryTimers.values()) this.clock.clearTimeout(timer); + this.operations.clear(); + this.expiryTimers.clear(); + this.views.clear(); + } + + load(name: string, forceRefresh = false) { + if (!name) return; + const view = this.read(name); + if (!forceRefresh && view.snapshot) { + this.publish(name, { state: 'loaded', snapshot: view.snapshot }); + return; + } + this.cancel(name); + const op: Operation = { abort: new AbortController(), deadline: Infinity }; + this.operations.set(name, op); + this.publish(name, { state: 'loading', snapshot: this.read(name).snapshot }); + // Bound an unresponsive initial POST too; once pending arrives, only the + // controller's fixed deadline governs polling. + op.timer = this.clock.setTimeout(() => this.fail(name, op, 'Detail request timed out'), 120000); + void this.transport.request(name, forceRefresh, op.abort.signal) + .then((result) => this.accept(name, op, result)) + .catch((error) => this.fail(name, op, String(error.message || error))); + } + + private current(name: string, op: Operation) { + return this.operations.get(name) === op && !op.abort.signal.aborted; + } + + private finish(name: string, op: Operation) { + if (op.timer !== undefined) this.clock.clearTimeout(op.timer); + this.operations.delete(name); + op.abort.abort(); + } + + private fail(name: string, op: Operation, error: string, state: 'error' | 'expired' = 'error') { + if (!this.current(name, op)) return; + this.finish(name, op); + this.publish(name, { state, error, snapshot: this.read(name).snapshot }); + } + + private accept(name: string, op: Operation, result: NodeDetailResult) { + if (!this.current(name, op)) return; + if (this.clock.now() >= op.deadline) { + this.fail(name, op, 'Detail request deadline expired', 'expired'); + return; + } + if (result.nodeName !== name || (op.requestId && result.requestId !== op.requestId)) { + this.fail(name, op, 'Mismatched node or detail request ID'); + return; + } + if (result.state === 'pending') { + const deadline = Date.parse(result.deadline || ''); + if (!result.requestId || !Number.isFinite(deadline)) { + this.fail(name, op, 'Pending detail response is missing a request ID or deadline'); + return; + } + op.requestId = result.requestId; + op.deadline = Math.min(op.deadline, deadline); + if (op.timer !== undefined) this.clock.clearTimeout(op.timer); + const remaining = op.deadline - this.clock.now(); + if (remaining <= 0) { + this.fail(name, op, 'Detail request deadline expired', 'expired'); + return; + } + this.publish(name, { ...this.read(name), state: 'loading', deadline: new Date(op.deadline).toISOString() }); + op.timer = this.clock.setTimeout(() => { + if (!this.current(name, op)) return; + if (this.clock.now() >= op.deadline) { + this.fail(name, op, 'Detail request deadline expired', 'expired'); + return; + } + op.timer = this.clock.setTimeout( + () => this.fail(name, op, 'Detail request deadline expired', 'expired'), + op.deadline - this.clock.now() + ); + void this.transport.poll(name, op.requestId!, op.abort.signal) + .then((next) => this.accept(name, op, next)) + .catch((error) => this.fail(name, op, String(error.message || error))); + }, Math.min(1000, remaining)); + return; + } + if (result.state !== 'complete') { + this.fail(name, op, result.error || `Detail request ${result.state}`, result.state === 'expired' ? 'expired' : 'error'); + return; + } + const snapshot = result.details; + if (!snapshot || !result.requestId || snapshot.nodeName !== name || + snapshot.status?.nodeInfo?.name !== name || snapshot.requestId !== result.requestId) { + this.fail(name, op, 'Invalid detail snapshot identity'); + return; + } + const expiry = Date.parse(snapshot.expiresAt); + if (!(expiry > this.clock.now())) { + this.fail(name, op, 'Detail snapshot expired', 'expired'); + return; + } + this.finish(name, op); + this.clearExpiry(name); + this.publish(name, { state: 'loaded', snapshot }); + this.expiryTimers.set(name, this.clock.setTimeout(() => { + this.expire(name); + this.changed(); + }, expiry - this.clock.now())); + } +} diff --git a/frontend/tests/nodeDetails.test.ts b/frontend/tests/nodeDetails.test.ts new file mode 100644 index 000000000..0214eaa42 --- /dev/null +++ b/frontend/tests/nodeDetails.test.ts @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { NodeDetails } from '../src/state/nodeDetails.ts'; + +const tick = async () => { for (let i = 0; i < 5; i++) await Promise.resolve(); }; +const time = (value: number) => new Date(value).toISOString(); +function fixture() { + let now = 1000; + let id = 0; + const timers = new Map void }>(); + const calls: { name: string; force?: boolean; signal: AbortSignal; resolve: (value: any) => void; reject: (error: Error) => void }[] = []; + const send = (name: string, signal: AbortSignal, force?: boolean) => + new Promise((resolve, reject) => calls.push({ name, signal, force, resolve, reject })); + const store = new NodeDetails({ + request: (name, force, signal) => send(name, signal, force), + poll: (name, _id, signal) => send(name, signal), + }, () => {}, { + now: () => now, + setTimeout: (run, delay) => { timers.set(++id, { at: now + delay, run }); return id as any; }, + clearTimeout: (id) => { timers.delete(id as any); }, + }); + const advance = (value: number, runTimers = true) => { + now = value; + if (runTimers) for (const [id, timer] of [...timers]) { + if (timer.at <= now) { timers.delete(id); timer.run(); } + } + }; + const complete = (requestId = 'a', expiresAt = 5000, nodeName = 'node') => ({ + state: 'complete', nodeName, requestId, + details: { + nodeName, requestId, collectedAt: time(1000), receivedAt: time(1000), expiresAt: time(expiresAt), + status: { nodeInfo: { name: nodeName }, peers: [{ name: 'peer' }], bpfEntries: [{ cidr: 'heavy' }] }, + }, + }); + return { store, calls, timers, advance, complete }; +} + +test('reads/open/summary/reconnect do not load; explicit load reuses cache and refresh forces', async () => { + const f = fixture(); + for (let i = 0; i < 5; i++) assert.equal(f.store.read('node').state, 'not-loaded'); + assert.equal(f.calls.length, 0); + f.store.load('node'); + assert.equal(f.calls[0].force, false); + f.calls[0].resolve(f.complete()); + await tick(); + f.store.load('node'); + assert.equal(f.calls.length, 1); + f.store.load('node', true); + assert.equal(f.calls[1].force, true); + f.calls[1].reject(new Error('leadership unavailable')); + await tick(); + assert.equal(f.store.read('node').state, 'error'); + assert.equal(f.store.read('node').error, 'leadership unavailable'); + assert.equal(f.store.read('node').snapshot.requestId, 'a'); + f.advance(5000); + assert.equal(f.store.read('node').snapshot, undefined); + assert.equal(f.calls.length, 2); +}); + +test('expiry actively releases data and read-time expiry cannot extend TTL', async () => { + for (const runTimers of [true, false]) { + const f = fixture(); + f.store.load('node'); + f.calls[0].resolve(f.complete()); + await tick(); + f.advance(4999); + assert.equal(f.store.read('node').state, 'loaded'); + f.advance(5000, runTimers); + assert.deepEqual(f.store.read('node'), { state: 'expired', error: undefined, deadline: undefined }); + assert.equal(f.calls.length, 1); + } +}); + +test('selection cancellation and superseded refresh reject stale async payloads', async () => { + const f = fixture(); + f.store.load('node'); + f.store.cancel('node'); + assert.equal(f.calls[0].signal.aborted, true); + f.calls[0].resolve(f.complete()); + await tick(); + assert.equal(f.store.read('node').snapshot, undefined); + f.store.load('node'); + f.store.load('node', true); + f.calls[2].resolve(f.complete('new', 9000)); + f.calls[1].resolve(f.complete('old')); + await tick(); + assert.equal(f.store.read('node').snapshot.requestId, 'new'); + f.advance(5000); + assert.equal(f.store.read('node').snapshot.requestId, 'new'); +}); + +test('pending polling honors original deadline, aborts in-flight GET, rejects late replies', async () => { + const f = fixture(); + f.store.load('node'); + f.calls[0].resolve({ state: 'pending', nodeName: 'node', requestId: 'a', deadline: time(4000) }); + await tick(); + f.advance(2000); + assert.equal(f.calls.length, 2); + f.calls[1].resolve({ state: 'pending', nodeName: 'node', requestId: 'a', deadline: time(9000) }); + await tick(); + assert.equal(f.store.read('node').deadline, time(4000)); + f.advance(3000); + f.advance(4000); + assert.equal(f.calls[2].signal.aborted, true); + assert.equal(f.store.read('node').state, 'expired'); + f.calls[2].resolve(f.complete()); + await tick(); + assert.equal(f.store.read('node').snapshot, undefined); +}); + +test('invalid and expired responses surface errors, not empty success', async () => { + for (const result of [ + { state: 'unavailable', nodeName: 'node', error: 'offline' }, + { state: 'pending', nodeName: 'node', requestId: 'a' }, + { state: 'complete', nodeName: 'other' }, + ]) { + const f = fixture(); + f.store.load('node'); + f.calls[0].resolve(result); + await tick(); + assert.equal(f.store.read('node').state, 'error'); + assert.equal(f.store.read('node').snapshot, undefined); + } + const f = fixture(); + f.store.load('node'); + f.calls[0].resolve(f.complete('a', 1000)); + await tick(); + assert.equal(f.store.read('node').state, 'expired'); +}); + +test('dispose cancels pending work, clears timers/cache, and ignores late results', async () => { + const f = fixture(); + f.store.load('node'); + f.store.dispose(); + assert.equal(f.calls[0].signal.aborted, true); + assert.equal(f.timers.size, 0); + f.calls[0].resolve(f.complete()); + await tick(); + assert.equal(f.store.read('node').snapshot, undefined); +}); From 0b32d4c5aa27068f4e05ce01e0c9952dd9bfdd46 Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 21:45:08 +0000 Subject: [PATCH 23/34] frontend: prepare explicit detail controls and summary-only JSON Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- .../src/components/nodes/NodeDetailDialog.tsx | 79 +++++++++++++++++++ .../src/components/nodes/NodeDetailModal.tsx | 3 + .../src/components/status/StatusJsonModal.tsx | 36 ++++----- frontend/src/hooks/useNodeDetails.ts | 30 +++++++ 4 files changed, 129 insertions(+), 19 deletions(-) create mode 100644 frontend/src/components/nodes/NodeDetailDialog.tsx create mode 100644 frontend/src/hooks/useNodeDetails.ts diff --git a/frontend/src/components/nodes/NodeDetailDialog.tsx b/frontend/src/components/nodes/NodeDetailDialog.tsx new file mode 100644 index 000000000..84220f08b --- /dev/null +++ b/frontend/src/components/nodes/NodeDetailDialog.tsx @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +import { useEffect, useState } from 'react'; +import type { ComponentProps } from 'react'; +import type { DetailView } from '../../state/nodeDetails'; +import NodeDetailModal from './NodeDetailModal'; +import { CloseXIcon, formatDateAndAge } from './shared/index'; + +type Props = Omit, 'node' | 'detailControls'> & { + detail: DetailView; + onLoad: (forceRefresh?: boolean) => void; +}; + +export default function NodeDetailDialog({ detail, onLoad, ...props }: Props) { + const [jsonOpen, setJsonOpen] = useState(false); + const [, setClock] = useState(0); + const expiresAt = detail.snapshot?.expiresAt; + useEffect(() => { + if (!expiresAt) return; + const timer = window.setInterval(() => setClock((clock) => clock + 1), 1000); + return () => window.clearInterval(timer); + }, [expiresAt]); + useEffect(() => setJsonOpen(false), [props.nodeName, expiresAt]); + useEffect(() => { + if (!props.nodeName) return; + const onKeyDown = (event: KeyboardEvent) => { if (event.key === 'Escape') props.onClose(); }; + window.addEventListener('keydown', onKeyDown); + return () => window.removeEventListener('keydown', onKeyDown); + }, [props.nodeName, props.onClose]); + if (!props.nodeName) return null; + const snapshot = detail.snapshot && Date.parse(detail.snapshot.expiresAt) > Date.now() ? detail.snapshot : undefined; + const busy = detail.state === 'loading'; + const collected = snapshot ? formatDateAndAge(snapshot.collectedAt) : undefined; + const state = !snapshot && detail.state === 'loaded' ? 'expired' : detail.state; + const message = state === 'not-loaded' ? 'Detailed data is not loaded. Choose Load data to inspect peers, routes, BPF entries, and full node JSON.' + : state === 'loading' ? 'Loading node details...' + : state === 'expired' ? 'Detailed data expired and was removed. Choose Load data to request it again.' + : state === 'error' ? 'The detail request failed. Choose Load data to retry or Refresh to force a fresh collection.' + : 'Detailed data loaded.'; + const controls = ( +
+
{message}
+ {detail.error &&
Request error: {detail.error}
} + {snapshot && ( +
+ {detail.error || busy ? 'Showing previous still-valid snapshot. ' : ''} + Collected {collected?.age} ({collected?.absolute}). + {' '}Received {snapshot.receivedAt}. Expires {snapshot.expiresAt}. +
+ )} + {busy && detail.deadline &&
Request deadline: {detail.deadline}
} +
+ + + {snapshot && } +
+ {snapshot && jsonOpen && 
{JSON.stringify(snapshot.status, null, 2)}
} +
+ ); + // Unmount the heavy tables when data expires so their memoized rows, maps, + // column callbacks and serialized JSON cannot keep the snapshot alive. + if (snapshot) { + return ; + } + return ( +
+
event.stopPropagation()}> +
+
{props.nodeName}
+ +
+ {controls} +
+
+ ); +} diff --git a/frontend/src/components/nodes/NodeDetailModal.tsx b/frontend/src/components/nodes/NodeDetailModal.tsx index 1fa1eee1d..297520fb9 100644 --- a/frontend/src/components/nodes/NodeDetailModal.tsx +++ b/frontend/src/components/nodes/NodeDetailModal.tsx @@ -42,6 +42,7 @@ import NodeInfoPanel from './detail/NodeInfoPanel'; import NodeDetailTabsHeader from './detail/NodeDetailTabsHeader'; function NodeDetailModal({ + detailControls, nodeName, node, allNodeNames, @@ -55,6 +56,7 @@ function NodeDetailModal({ onSelectNode, onClose }: { + detailControls?: React.ReactNode; nodeName: string | null; node: NodeStatus | null; allNodeNames: string[]; @@ -1705,6 +1707,7 @@ function NodeDetailModal({
+ {detailControls}
void; }) { - const [snapshotStatus, setSnapshotStatus] = useState(null); + const [snapshotStatus, setSnapshotStatus] = useState(null); const [fetchError, setFetchError] = useState(null); const [fetching, setFetching] = useState(false); const [collapseAllVersion, setCollapseAllVersion] = useState(0); - const wasOpenRef = useRef(false); useEffect(() => { if (!open) return; @@ -30,31 +31,28 @@ function StatusJsonModal({ }, [open, onClose]); useEffect(() => { - if (open && !wasOpenRef.current) { - // Fetch fresh data from HTTP on each open - wasOpenRef.current = true; + let cancelled = false; + if (open) { setFetching(true); setFetchError(null); - fetch('/status/json') - .then((res) => { - if (!res.ok) throw new Error(`HTTP ${res.status}`); - return res.json(); - }) + fetchClusterStatus() .then((data) => { - setSnapshotStatus(data as ClusterStatus); + if (cancelled) return; + setSnapshotStatus(toClusterSummary(data)); setCollapseAllVersion((version) => version + 1); }) .catch((err) => { + if (cancelled) return; setFetchError((err as Error).message); }) .finally(() => { + if (cancelled) return; setFetching(false); }); - return; - } - if (!open) { - wasOpenRef.current = false; + } else { + setSnapshotStatus(null); } + return () => { cancelled = true; }; }, [open]); const statusJsonValue = useMemo(() => { @@ -70,10 +68,10 @@ function StatusJsonModal({ return a.localeCompare(b); }; - const sortedStatus: ClusterStatus = { + const sortedStatus: ClusterSummary = { ...snapshotStatus, sites: [...(snapshotStatus.sites || [])].sort((a, b) => compareNames(a.name, b.name)), - nodes: [...(snapshotStatus.nodes || [])].sort((a, b) => compareNames(a.nodeInfo?.name, b.nodeInfo?.name)), + nodeSummaries: [...(snapshotStatus.nodeSummaries || [])].sort((a, b) => compareNames(a.name, b.name)), gatewayPools: [...(snapshotStatus.gatewayPools || [])].sort((a, b) => compareNames(a.name, b.name)) }; @@ -86,7 +84,7 @@ function StatusJsonModal({
event.stopPropagation()}>
-
Cluster Status JSON
+
Cluster Status JSON (summary only)