diff --git a/cmd/kubectl-unbounded/app/net/detail_client.go b/cmd/kubectl-unbounded/app/net/detail_client.go new file mode 100644 index 000000000..ae4f8abf6 --- /dev/null +++ b/cmd/kubectl-unbounded/app/net/detail_client.go @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package net + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +type statusRequest func(context.Context, string, string, []byte) ([]byte, error) + +type nodeDetailClient struct { + request statusRequest + pollInterval time.Duration +} + +func (c nodeDetailClient) fetch(ctx context.Context, nodeName string, forceRefresh bool) (*statusv1alpha1.NodeStatusResponse, error) { + if len(validation.IsDNS1123Subdomain(nodeName)) != 0 { + return nil, fmt.Errorf("invalid node name %q", nodeName) + } + + body, err := json.Marshal(struct { + ForceRefresh bool `json:"forceRefresh"` + }{ForceRefresh: forceRefresh}) + if err != nil { + return nil, err + } + + path := "/status/node/" + nodeName + "/details" + + raw, requestErr := c.request(ctx, http.MethodPost, path, body) + if ctx.Err() != nil { + return nil, ctx.Err() + } + + initial, err := decodeNodeDetailResult(raw, requestErr, nodeName, "") + if err != nil { + return nil, err + } + + if initial.State == statusv1alpha1.NodeDetailComplete { + return validateNodeDetails(initial, time.Now()) + } + + requestCtx, cancel := context.WithDeadline(ctx, initial.Deadline) + defer cancel() + + for { + interval := c.pollInterval + if interval <= 0 { + interval = time.Second + } + + timer := time.NewTimer(interval) + select { + case <-requestCtx.Done(): + timer.Stop() + return nil, fmt.Errorf("node %q detail request canceled or deadline exceeded: %w", nodeName, requestCtx.Err()) + case <-timer.C: + } + + raw, requestErr = c.request(requestCtx, http.MethodGet, path+"?requestId="+url.QueryEscape(initial.RequestID), nil) + if requestCtx.Err() != nil { + return nil, fmt.Errorf("node %q detail request canceled or deadline exceeded: %w", nodeName, requestCtx.Err()) + } + + result, err := decodeNodeDetailResult(raw, requestErr, nodeName, initial.RequestID) + if err != nil { + return nil, err + } + + if result.State == statusv1alpha1.NodeDetailComplete { + return validateNodeDetails(result, time.Now()) + } + + if !result.Deadline.Equal(initial.Deadline) { + return nil, fmt.Errorf("malformed node detail response: request deadline changed") + } + } +} + +func decodeNodeDetailResult(raw []byte, requestErr error, nodeName, requestID string) (statusv1alpha1.NodeDetailResult, error) { + var result statusv1alpha1.NodeDetailResult + + decodeErr := json.Unmarshal(raw, &result) + if requestErr != nil && (decodeErr != nil || result.State == "") { + return result, fmt.Errorf("node %q detail API unavailable or unsupported: %w", nodeName, requestErr) + } + + if decodeErr != nil { + return result, fmt.Errorf("malformed node detail response: %w", decodeErr) + } + + if result.NodeName != nodeName || (requestID != "" && result.RequestID != requestID) { + return result, fmt.Errorf("malformed node detail response: node or request identity mismatch") + } + + switch result.State { + case statusv1alpha1.NodeDetailComplete, statusv1alpha1.NodeDetailPending: + if requestErr != nil { + return result, fmt.Errorf("node detail request failed: %w", requestErr) + } + + if result.State == statusv1alpha1.NodeDetailPending && + (result.RequestID == "" || result.Deadline.IsZero() || result.Details != nil) { + return result, fmt.Errorf("malformed pending node detail response") + } + + return result, nil + case statusv1alpha1.NodeDetailExpired, statusv1alpha1.NodeDetailUnavailable, statusv1alpha1.NodeDetailRetryable: + return result, fmt.Errorf("node %q details %s: %s", nodeName, result.State, result.Error) + default: + return result, fmt.Errorf("malformed or unsupported node detail state %q: %s", result.State, result.Error) + } +} + +func validateNodeDetails(result statusv1alpha1.NodeDetailResult, now time.Time) (*statusv1alpha1.NodeStatusResponse, error) { + details := result.Details + if result.RequestID == "" || details == nil || details.Status == nil || + details.NodeName != result.NodeName || details.RequestID != result.RequestID || + details.Status.NodeInfo.Name != result.NodeName || details.CollectedAt.IsZero() || + details.ReceivedAt.IsZero() || !details.ExpiresAt.After(details.ReceivedAt) || result.Error != "" { + return nil, fmt.Errorf("malformed completed node detail response") + } + + if !details.ExpiresAt.After(now) { + return nil, fmt.Errorf("node %q details expired; request them again", result.NodeName) + } + + if details.Status.FetchError != "" { + return nil, fmt.Errorf("node %q detail collection failed: %s", result.NodeName, details.Status.FetchError) + } + + return details.Status, nil +} + +// newStatusRequest reuses kubectl credentials and authenticated port-forward fallback. +// Each HTTP attempt is bounded independently; the caller owns the overall deadline. +func newStatusRequest(rt *pluginRuntime, opts nodeStatusFetchOptions) (statusRequest, error) { + ns, err := rt.namespace() + if err != nil { + return nil, err + } + + client, err := rt.kubeClient() + if err != nil { + return nil, err + } + + cfg, err := rt.restConfig() + if err != nil { + return nil, err + } + + return func(ctx context.Context, method, path string, body []byte) ([]byte, error) { + attemptCtx, cancel := context.WithTimeout(ctx, opts.timeout) + defer cancel() + + raw, err := requestStatusViaAggregatedAPI(attemptCtx, client, method, path, body) + if err == nil || len(raw) > 0 || ctx.Err() != nil { + return raw, err + } + + fallbackCtx, fallbackCancel := context.WithTimeout(ctx, opts.timeout) + defer fallbackCancel() + + return requestStatusViaPortForward(fallbackCtx, client, cfg, ns, + opts.controllerDeploy, opts.controllerSelector, opts.controllerPort, opts.timeout, + method, path, body) + }, nil +} + +func requestStatusViaAggregatedAPI(ctx context.Context, client *kubernetes.Clientset, method, path string, body []byte) ([]byte, error) { + target, err := url.ParseRequestURI(path) + if err != nil || target.IsAbs() || target.Host != "" { + return nil, fmt.Errorf("invalid controller status path %q", path) + } + + request := client.CoreV1().RESTClient().Verb(method). + AbsPath("/apis/status.net.unbounded-cloud.io/v1alpha1" + target.Path) + for key, values := range target.Query() { + for _, value := range values { + request.Param(key, value) + } + } + + if body != nil { + request.SetHeader("Content-Type", "application/json").Body(body) + } + + return request.DoRaw(ctx) +} diff --git a/cmd/kubectl-unbounded/app/net/detail_client_test.go b/cmd/kubectl-unbounded/app/net/detail_client_test.go new file mode 100644 index 000000000..44d3fd78a --- /dev/null +++ b/cmd/kubectl-unbounded/app/net/detail_client_test.go @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package net + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +func newDetailTestRuntime(t *testing.T, serverURL string) *pluginRuntime { + t.Helper() + + path := filepath.Join(t.TempDir(), "kubeconfig") + + data := fmt.Sprintf(`apiVersion: v1 +kind: Config +current-context: test +clusters: +- name: test + cluster: + server: %s +contexts: +- name: test + context: + cluster: test + user: test +users: +- name: test + user: + token: test-token +`, serverURL) + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + + rt := newPluginRuntime() + *rt.configFlags.KubeConfig = path + *rt.configFlags.Namespace = "unbounded-system" + + return rt +} + +func TestStatusRequestAggregatedTransport(t *testing.T) { + for _, method := range []string{http.MethodPost, http.MethodGet} { + t.Run(method, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != method || r.URL.Path != "/apis/status.net.unbounded-cloud.io/v1alpha1/status/node/node-a/details" { + t.Errorf("unexpected request %s %s", r.Method, r.URL) + } + + if r.Header.Get("Authorization") != "Bearer test-token" { + t.Error("request did not reuse Kubernetes authentication") + } + + if method == http.MethodPost { + body, err := io.ReadAll(r.Body) + if err != nil || string(body) != `{"forceRefresh":true}` || r.Header.Get("Content-Type") != "application/json" { + t.Errorf("unexpected request body %s, %v", body, err) + } + } else if r.URL.Query().Get("requestId") != "id/+ ?" { + t.Errorf("request ID changed: %q", r.URL.Query().Get("requestId")) + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _, _ = io.WriteString(w, `{"state":"pending"}`) + })) + defer server.Close() + + client, err := kubernetes.NewForConfig(&rest.Config{Host: server.URL, BearerToken: "test-token"}) + if err != nil { + t.Fatal(err) + } + + path := "/status/node/node-a/details" + + var body []byte + if method == http.MethodPost { + body = []byte(`{"forceRefresh":true}`) + } else { + path += "?requestId=id%2F%2B+%3F" + } + + raw, err := requestStatusViaAggregatedAPI(context.Background(), client, method, path, body) + if err != nil || string(raw) != `{"state":"pending"}` { + t.Fatalf("request = %s, %v", raw, err) + } + }) + } +} + +func TestStatusRequestPreservesHTTPFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = io.WriteString(w, `{"state":"unavailable","error":"leadership changed"}`) + })) + defer server.Close() + + client, err := kubernetes.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + + raw, err := requestStatusViaAggregatedAPI(context.Background(), client, http.MethodGet, "/status/node/node-a/details", nil) + if err == nil || !strings.Contains(string(raw), "leadership changed") { + t.Fatalf("lost explicit controller failure: %s, %v", raw, err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if _, err := requestStatusViaAggregatedAPI(ctx, client, http.MethodGet, "/status/node/node-a/details", nil); err == nil { + t.Fatal("canceled request succeeded") + } + + for _, path := range []string{"https://other.invalid/status", "://invalid"} { + if _, err := requestStatusViaAggregatedAPI(context.Background(), client, http.MethodGet, path, nil); err == nil { + t.Errorf("accepted invalid path %q", path) + } + } + + request, err := newStatusRequest(newDetailTestRuntime(t, server.URL), defaultNodeStatusFetchOptions()) + if err != nil { + t.Fatal(err) + } + + raw, err = request(context.Background(), http.MethodGet, "/status/node/node-a/details", nil) + if err == nil || !strings.Contains(string(raw), "leadership changed") { + t.Fatalf("fallback hid controller failure: %s, %v", raw, err) + } +} diff --git a/cmd/kubectl-unbounded/app/net/detail_poll_test.go b/cmd/kubectl-unbounded/app/net/detail_poll_test.go new file mode 100644 index 000000000..7ceeab5b3 --- /dev/null +++ b/cmd/kubectl-unbounded/app/net/detail_poll_test.go @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package net + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "strings" + "testing" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func detailResultFixture() statusv1alpha1.NodeDetailResult { + now := time.Now().UTC() + + return statusv1alpha1.NodeDetailResult{ + State: statusv1alpha1.NodeDetailComplete, NodeName: "node-a", RequestID: "request/+ ?", + Deadline: now.Add(-time.Second), + Details: &statusv1alpha1.NodeDetailSnapshot{ + NodeName: "node-a", RequestID: "request/+ ?", CollectedAt: now, ReceivedAt: now, + ExpiresAt: now.Add(time.Minute), + Status: &statusv1alpha1.NodeStatusResponse{ + Timestamp: now, NodeInfo: statusv1alpha1.NodeInfo{Name: "node-a"}, + Peers: []statusv1alpha1.PeerStatus{{Name: "peer"}}, + }, + }, + } +} + +func TestNodeDetailClientCachedAndPending(t *testing.T) { + for _, pending := range []bool{false, true} { + for _, refresh := range []bool{false, true} { + result := detailResultFixture() + calls := 0 + client := nodeDetailClient{ + pollInterval: time.Nanosecond, + request: func(_ context.Context, method, path string, body []byte) ([]byte, error) { + calls++ + if calls == 1 { + if method != http.MethodPost || path != "/status/node/node-a/details" { + t.Fatalf("initial request = %s %s", method, path) + } + + var args struct { + Refresh *bool `json:"forceRefresh"` + } + if err := json.Unmarshal(body, &args); err != nil || args.Refresh == nil || *args.Refresh != refresh { + t.Fatalf("refresh body = %s, %v", body, err) + } + + if pending { + result.Deadline = time.Now().Add(time.Minute) + + return json.Marshal(statusv1alpha1.NodeDetailResult{ + State: statusv1alpha1.NodeDetailPending, NodeName: result.NodeName, + RequestID: result.RequestID, Deadline: result.Deadline, + }) + } + } else { + target, err := url.Parse(path) + if err != nil || method != http.MethodGet || target.Query().Get("requestId") != result.RequestID || body != nil { + t.Fatalf("poll request = %s %s, %v", method, path, err) + } + } + + return json.Marshal(result) + }, + } + + got, err := client.fetch(context.Background(), "node-a", refresh) + if err != nil || got.NodeInfo.Name != "node-a" || len(got.Peers) != 1 { + t.Fatalf("fetch = %+v, %v", got, err) + } + + wantCalls := 1 + if pending { + wantCalls = 2 + } + + if calls != wantCalls { + t.Errorf("made %d calls, want %d", calls, wantCalls) + } + } + } +} + +func TestNodeDetailClientFailures(t *testing.T) { + for _, tc := range []struct { + name string + change func(*statusv1alpha1.NodeDetailResult) + want string + }{ + {"wrong node", func(r *statusv1alpha1.NodeDetailResult) { r.NodeName = "other" }, "identity mismatch"}, + {"wrong snapshot", func(r *statusv1alpha1.NodeDetailResult) { r.Details.NodeName = "other" }, "malformed"}, + {"wrong request", func(r *statusv1alpha1.NodeDetailResult) { r.Details.RequestID = "other" }, "malformed"}, + {"wrong payload", func(r *statusv1alpha1.NodeDetailResult) { r.Details.Status.NodeInfo.Name = "other" }, "malformed"}, + {"missing details", func(r *statusv1alpha1.NodeDetailResult) { r.Details = nil }, "malformed"}, + {"missing payload", func(r *statusv1alpha1.NodeDetailResult) { r.Details.Status = nil }, "malformed"}, + {"missing collection time", func(r *statusv1alpha1.NodeDetailResult) { r.Details.CollectedAt = time.Time{} }, "malformed"}, + {"missing receipt time", func(r *statusv1alpha1.NodeDetailResult) { r.Details.ReceivedAt = time.Time{} }, "malformed"}, + {"missing request", func(r *statusv1alpha1.NodeDetailResult) { r.RequestID = "" }, "malformed"}, + {"fetch failure", func(r *statusv1alpha1.NodeDetailResult) { r.Details.Status.FetchError = "unreachable" }, "collection failed"}, + {"expired snapshot", func(r *statusv1alpha1.NodeDetailResult) { + r.Details.ReceivedAt = time.Now().Add(-time.Hour) + r.Details.ExpiresAt = time.Now().Add(-time.Second) + }, "expired"}, + {"missing pending ID", func(r *statusv1alpha1.NodeDetailResult) { + r.State, r.RequestID, r.Details = statusv1alpha1.NodeDetailPending, "", nil + }, "malformed"}, + {"missing pending deadline", func(r *statusv1alpha1.NodeDetailResult) { + r.State, r.Deadline, r.Details = statusv1alpha1.NodeDetailPending, time.Time{}, nil + }, "malformed"}, + } { + t.Run(tc.name, func(t *testing.T) { + result := detailResultFixture() + tc.change(&result) + + client := nodeDetailClient{request: func(context.Context, string, string, []byte) ([]byte, error) { + return json.Marshal(result) + }} + if _, err := client.fetch(context.Background(), "node-a", false); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("fetch error = %v, want %q", err, tc.want) + } + }) + } + + for _, state := range []statusv1alpha1.NodeDetailState{ + statusv1alpha1.NodeDetailExpired, statusv1alpha1.NodeDetailUnavailable, statusv1alpha1.NodeDetailRetryable, "unsupported", "", + } { + client := nodeDetailClient{request: func(context.Context, string, string, []byte) ([]byte, error) { + return json.Marshal(statusv1alpha1.NodeDetailResult{NodeName: "node-a", State: state, Error: "controller explanation"}) + }} + if _, err := client.fetch(context.Background(), "node-a", false); err == nil || + !strings.Contains(err.Error(), string(state)) || !strings.Contains(err.Error(), "controller explanation") { + t.Errorf("state %q error = %v", state, err) + } + } +} + +func TestNodeDetailClientDeadlineAndCancellation(t *testing.T) { + result := detailResultFixture() + result.State, result.Details, result.Deadline = statusv1alpha1.NodeDetailPending, nil, time.Now().Add(-time.Second) + calls := 0 + + client := nodeDetailClient{request: func(context.Context, string, string, []byte) ([]byte, error) { + calls++ + return json.Marshal(result) + }} + if _, err := client.fetch(context.Background(), "node-a", false); !errors.Is(err, context.DeadlineExceeded) || calls != 1 { + t.Fatalf("expired deadline = %v, calls=%d", err, calls) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if _, err := client.fetch(ctx, "node-a", false); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation = %v", err) + } + + client.pollInterval = time.Nanosecond + result.Deadline = time.Now().Add(time.Minute) + + client.request = func(context.Context, string, string, []byte) ([]byte, error) { + result.Deadline = result.Deadline.Add(time.Second) + return json.Marshal(result) + } + if _, err := client.fetch(context.Background(), "node-a", false); err == nil || !strings.Contains(err.Error(), "deadline changed") { + t.Fatalf("deadline extension accepted: %v", err) + } + + client.request = func(context.Context, string, string, []byte) ([]byte, error) { return []byte(`not JSON`), nil } + if _, err := client.fetch(context.Background(), "node-a", false); err == nil || !strings.Contains(err.Error(), "malformed") { + t.Fatalf("malformed response = %v", err) + } + + client.request = func(context.Context, string, string, []byte) ([]byte, error) { return nil, errors.New("HTTP 404") } + if _, err := client.fetch(context.Background(), "node-a", false); err == nil || !strings.Contains(err.Error(), "unsupported") { + t.Fatalf("unsupported API = %v", err) + } + + if _, err := client.fetch(context.Background(), "../other", false); err == nil || !strings.Contains(err.Error(), "invalid node name") { + t.Fatalf("invalid node = %v", err) + } +} diff --git a/cmd/kubectl-unbounded/app/net/node.go b/cmd/kubectl-unbounded/app/net/node.go index 42ac8fa9d..089e3e2cf 100644 --- a/cmd/kubectl-unbounded/app/net/node.go +++ b/cmd/kubectl-unbounded/app/net/node.go @@ -4,6 +4,7 @@ package net import ( + "bytes" "context" "crypto/tls" "crypto/x509" @@ -27,6 +28,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" + netstatus "github.com/Azure/unbounded/internal/net/status" statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) @@ -178,16 +180,16 @@ func runNodeList(rt *pluginRuntime, cmd *cobra.Command, baseFetch nodeStatusFetc fetchOpts.timeout = override.timeout } - status, err := fetchClusterStatus(rt, cmd, fetchOpts) + status, err := fetchClusterSummary(rt, cmd, fetchOpts) if err != nil { return err } - rows := buildNodeRows(status) + rows := buildNodeRowsFromSummary(status) useColor := shouldUseColor(cmd.OutOrStdout(), color) if !suppressWarnings { - printWarnings(cmd.OutOrStdout(), collectWarnings(status), useColor) + printWarnings(cmd.OutOrStdout(), collectWarningsFromSummary(status), useColor) } switch output { @@ -213,19 +215,32 @@ func runNodeList(rt *pluginRuntime, cmd *cobra.Command, baseFetch nodeStatusFetc func fetchClusterStatus(rt *pluginRuntime, cmd *cobra.Command, opts nodeStatusFetchOptions) (clusterStatusResponse, error) { var status clusterStatusResponse - ns, err := rt.namespace() + raw, err := fetchClusterStatusRaw(rt, cmd, opts) if err != nil { return status, err } + if err := json.Unmarshal(raw, &status); err != nil { + return status, fmt.Errorf("decode /status/json: %w", err) + } + + return status, nil +} + +func fetchClusterStatusRaw(rt *pluginRuntime, cmd *cobra.Command, opts nodeStatusFetchOptions) ([]byte, error) { + ns, err := rt.namespace() + if err != nil { + return nil, err + } + client, err := rt.kubeClient() if err != nil { - return status, err + return nil, err } cfg, err := rt.restConfig() if err != nil { - return status, err + return nil, err } ctx, cancel := context.WithTimeout(cmd.Context(), opts.timeout) @@ -235,15 +250,71 @@ func fetchClusterStatus(rt *pluginRuntime, cmd *cobra.Command, opts nodeStatusFe if err != nil { raw, err = fetchStatusViaPortForward(ctx, client, cfg, ns, opts.controllerDeploy, opts.controllerSelector, opts.controllerPort, opts.timeout) if err != nil { - return status, fmt.Errorf("fetch /status/json failed via service proxy (%s) and pod port-forward (%s)", opts.controllerService, err) + return nil, fmt.Errorf("fetch /status/json failed via service proxy (%s) and pod port-forward (%s)", opts.controllerService, err) } } - if err := json.Unmarshal(raw, &status); err != nil { - return status, fmt.Errorf("decode /status/json: %w", err) + return raw, nil +} + +func fetchClusterSummary(rt *pluginRuntime, cmd *cobra.Command, opts nodeStatusFetchOptions) (clusterSummary, error) { + raw, err := fetchClusterStatusRaw(rt, cmd, opts) + if err != nil { + return clusterSummary{}, err } - return status, nil + return decodeClusterSummary(raw) +} + +// decodeClusterSummary projects legacy full responses once, never retaining details. +func decodeClusterSummary(raw []byte) (clusterSummary, error) { + var shape map[string]json.RawMessage + if err := json.Unmarshal(raw, &shape); err != nil { + return clusterSummary{}, fmt.Errorf("decode cluster overview: %w", err) + } + + var summary clusterSummary + if err := json.Unmarshal(raw, &summary); err != nil { + return summary, fmt.Errorf("decode cluster overview: %w", err) + } + + if _, ok := shape["nodeSummaries"]; ok { + return summary, nil + } + + if _, ok := shape["nodes"]; !ok { + return summary, fmt.Errorf("malformed cluster overview: missing nodeSummaries or nodes") + } + + var legacy clusterStatusResponse + if err := json.Unmarshal(raw, &legacy); err != nil { + return summary, fmt.Errorf("decode legacy cluster overview: %w", err) + } + + summary.NodeSummaries = make([]nodeSummary, 0, len(legacy.Nodes)) + + now := time.Now() + for _, node := range legacy.Nodes { + overview := netstatus.OverviewFromStatus(&node, now) + + entry := nodeSummary{ + Name: node.NodeInfo.Name, SiteName: node.NodeInfo.SiteName, + IsGateway: node.NodeInfo.IsGateway, K8sReady: node.NodeInfo.K8sReady, + StatusSource: node.StatusSource, FetchError: node.FetchError, + PeerCount: overview.PeerCount, HealthyPeers: overview.HealthyPeers, RouteCount: overview.RouteCount, + RouteMismatch: overview.RouteMismatch, ErrorCount: len(node.NodeErrors), + CniStatus: cniStatusLabel(node, legacy.PullEnabled), CniTone: statusTone(node, legacy.PullEnabled), + } + if len(node.NodeErrors) > 0 { + entry.FirstError = node.NodeErrors[0].Message + } + + summary.NodeSummaries = append(summary.NodeSummaries, entry) + } + + summary.NodeCount = len(summary.NodeSummaries) + + return summary, nil } // newNodeLogsCommand shows CNI node-agent logs for a specific Kubernetes node. @@ -798,6 +869,18 @@ func fetchStatusViaPortForward( selector string, remotePort string, timeout time.Duration, +) ([]byte, error) { + return requestStatusViaPortForward(ctx, client, cfg, ns, deployName, selector, remotePort, timeout, http.MethodGet, "/status/json", nil) +} + +func requestStatusViaPortForward( + ctx context.Context, + client *kubernetes.Clientset, + cfg *rest.Config, + ns, deployName, selector, remotePort string, + timeout time.Duration, + method, path string, + body []byte, ) ([]byte, error) { pods, err := podsForController(ctx, client, ns, deployName, selector) if err != nil { @@ -825,6 +908,8 @@ func fetchStatusViaPortForward( } stopCh := make(chan struct{}, 1) + defer close(stopCh) + readyCh := make(chan struct{}) errCh := make(chan error, 1) @@ -845,8 +930,6 @@ func fetchStatusViaPortForward( return nil, ctx.Err() } - defer close(stopCh) - fwdPorts, err := fw.GetPorts() if err != nil { return nil, err @@ -861,11 +944,15 @@ func fetchStatusViaPortForward( reqCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, fmt.Sprintf("https://127.0.0.1:%d/status/json", localPort), nil) + req, err := http.NewRequestWithContext(reqCtx, method, fmt.Sprintf("https://127.0.0.1:%d%s", localPort, path), bytes.NewReader(body)) if err != nil { return nil, err } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + // Request an HMAC viewer token for authentication. When port-forwarding // directly to the controller pod, the API server front-proxy is bypassed // so the controller requires an HMAC token. @@ -892,6 +979,7 @@ func fetchStatusViaPortForward( }, }, } + defer tlsClient.CloseIdleConnections() resp, err := tlsClient.Do(req) if err != nil { @@ -900,16 +988,16 @@ func fetchStatusViaPortForward( defer func() { _ = resp.Body.Close() }() //nolint:errcheck - body, err := io.ReadAll(resp.Body) + responseBody, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("controller /status/json returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + return responseBody, fmt.Errorf("controller %s returned %d: %s", path, resp.StatusCode, strings.TrimSpace(string(responseBody))) } - return body, nil + return responseBody, nil } // podsForController returns controller pods from deployment selector, falling back to label selector. diff --git a/cmd/kubectl-unbounded/app/net/summary_poll_test.go b/cmd/kubectl-unbounded/app/net/summary_poll_test.go new file mode 100644 index 000000000..0a99e1809 --- /dev/null +++ b/cmd/kubectl-unbounded/app/net/summary_poll_test.go @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package net + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +const legacyOverviewFixture = `{ + "seq":7,"pullEnabled":false,"leaderInfo":{"podName":"leader"}, + "sites":[{"name":"site","manageCniPlugin":true}], + "gatewayPools":[{"name":"pool","gateways":["node-a"]}], + "warnings":["controller warning"], + "nodes":[{ + "nodeInfo":{"name":"node-a","siteName":"site","isGateway":true,"k8sReady":"Ready"}, + "statusSource":"ws","nodeErrors":[{"type":"cni","message":"not ready"}], + "peers":[{"name":"peer","healthCheck":{"enabled":true,"status":"up"}},{"name":"down"}], + "routingTable":{"routes":[{"nextHops":[{"expected":true,"present":false}]}]}, + "bpfEntries":[{"cidr":"10.0.0.0/24","node":"secret-detail"}] + }] +}` + +func TestDecodeClusterSummaryCompatibility(t *testing.T) { + summary, err := decodeClusterSummary([]byte(legacyOverviewFixture)) + if err != nil { + t.Fatal(err) + } + + if summary.Seq != 7 || summary.LeaderInfo.PodName != "leader" || len(summary.NodeSummaries) != 1 { + t.Fatalf("missing cluster metadata: %+v", summary) + } + + node := summary.NodeSummaries[0] + if node.PeerCount != 2 || node.HealthyPeers != 1 || node.RouteCount != 1 || !node.RouteMismatch || + node.ErrorCount != 1 || node.FirstError != "not ready" || node.Name != "node-a" { + t.Fatalf("overview facts changed: %+v", node) + } + + raw, err := json.Marshal(summary) + if err != nil { + t.Fatal(err) + } + + for _, forbidden := range []string{`"nodes"`, `"peers"`, `"routingTable"`, `"bpfEntries"`, "secret-detail", "10.0.0.0/24"} { + if strings.Contains(string(raw), forbidden) { + t.Errorf("retained detail %q in summary: %s", forbidden, raw) + } + } + + got, err := decodeClusterSummary(raw) + if err != nil || got.NodeSummaries[0] != node { + t.Fatalf("summary round trip changed: %+v, %v", got, err) + } + + for _, raw := range []string{`{`, `{}`, `null`, `{"nodeSummaries":"bad"}`, `{"nodes":"bad"}`} { + if _, err := decodeClusterSummary([]byte(raw)); err == nil { + t.Errorf("accepted malformed response %s", raw) + } + } + + if got, err := decodeClusterSummary([]byte(`{"nodeSummaries":[],"nodes":"ignored legacy field"}`)); err != nil || len(got.NodeSummaries) != 0 { + t.Fatalf("summary must take precedence: %+v, %v", got, err) + } +} + +func TestMergeClusterSummaryDelta(t *testing.T) { + summary := clusterSummary{ + Seq: 1, PullEnabled: true, NodeCount: 2, + NodeSummaries: []nodeSummary{ + {Name: "keep", PeerCount: 3, HealthyPeers: 2}, + {Name: "remove", PeerCount: 4}, + }, + Warnings: []string{"old warning"}, + } + + raw := []byte(`{ + "seq":2,"nodeCount":2,"pullEnabled":false,"warnings":[], + "removedNodes":["remove"], + "nodeSummaries":[{"name":"new","peerCount":7,"healthyPeers":5}] + }`) + if err := mergeClusterSummaryDelta(&summary, raw); err != nil { + t.Fatal(err) + } + + if summary.Seq != 2 || summary.PullEnabled || len(summary.Warnings) != 0 || len(summary.NodeSummaries) != 2 { + t.Fatalf("delta metadata not applied: %+v", summary) + } + + nodes := make(map[string]nodeSummary) + for _, node := range summary.NodeSummaries { + nodes[node.Name] = node + } + + if nodes["keep"].PeerCount != 3 || nodes["new"].HealthyPeers != 5 { + t.Fatalf("delta lost existing/updated facts: %+v", nodes) + } + + if _, ok := nodes["remove"]; ok { + t.Fatal("removed node still retained") + } + + if err := mergeClusterSummaryDelta(&summary, []byte(`{"nodeSummaries":[{"name":"keep","peerCount":0}]}`)); err != nil { + t.Fatal(err) + } + + for _, node := range summary.NodeSummaries { + if node.Name == "keep" && (node.PeerCount != 0 || node.HealthyPeers != 0) { + t.Fatalf("zero-valued update not applied: %+v", node) + } + } + + for _, raw := range []string{`null`, `{`, `{"nodeSummaries":"bad"}`, `{"pullEnabled":"bad"}`} { + if err := mergeClusterSummaryDelta(&summary, []byte(raw)); err == nil { + t.Errorf("accepted malformed summary delta %s", raw) + } + } +} + +func TestNodeListUsesOnlyOverview(t *testing.T) { + for _, fixture := range []string{ + legacyOverviewFixture, + `{"nodeSummaries":[{"name":"node-a","peerCount":2,"healthyPeers":1}],"sites":[],"gatewayPools":[]}`, + } { + for _, output := range []string{"json", "table", "wide"} { + t.Run(output, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/apis/status.net.unbounded-cloud.io/v1alpha1/status/json" { + t.Errorf("list requested non-overview endpoint: %s %s", r.Method, r.URL) + http.Error(w, "unexpected request", http.StatusBadRequest) + + return + } + + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, fixture) + })) + defer server.Close() + + cmd := newNodeRootCommand(newDetailTestRuntime(t, server.URL)) + + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"list", "-o", output, "--color=never", "--suppress-warnings"}) + + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } + + if !strings.Contains(out.String(), "node-a") || !strings.Contains(out.String(), "1/2") { + t.Fatalf("missing summary row: %s", out.String()) + } + + if strings.Contains(out.String(), "secret-detail") { + t.Fatal("list leaked details") + } + }) + } + } +} diff --git a/cmd/kubectl-unbounded/app/net/watch.go b/cmd/kubectl-unbounded/app/net/watch.go index 3d41ad388..dae9f2bf7 100644 --- a/cmd/kubectl-unbounded/app/net/watch.go +++ b/cmd/kubectl-unbounded/app/net/watch.go @@ -47,11 +47,74 @@ type watchOpts struct { // renderSummary, when non-nil, enables the WS summary protocol. On // connect the client sends cluster_summary_subscribe and subsequent // updates arrive as cluster_summary messages rendered via this callback. - // The full-status render callback is still used during HTTP polling - // fallback because the polling endpoint returns full status. + // HTTP polling also projects older controller payloads into this summary. renderSummary func(io.Writer, *clusterSummary, bool) error } +func mergeClusterSummaryDelta(current *clusterSummary, raw []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return err + } + + if fields == nil { + return fmt.Errorf("malformed cluster summary delta") + } + + var nodes struct { + Updated []nodeSummary `json:"nodeSummaries"` + Removed []string `json:"removedNodes"` + } + if err := json.Unmarshal(raw, &nodes); err != nil { + return err + } + + delete(fields, "nodeSummaries") + delete(fields, "removedNodes") + + base, err := json.Marshal(current) + if err != nil { + return err + } + + metadata, err := json.Marshal(fields) + if err != nil { + return err + } + + merged, err := shallowMergeJSON(base, metadata) + if err != nil { + return err + } + + var result clusterSummary + if err := json.Unmarshal(merged, &result); err != nil { + return err + } + + byName := make(map[string]nodeSummary, len(result.NodeSummaries)) + for _, node := range result.NodeSummaries { + byName[node.Name] = node + } + + for _, name := range nodes.Removed { + delete(byName, name) + } + + for _, node := range nodes.Updated { + byName[node.Name] = node + } + + result.NodeSummaries = make([]nodeSummary, 0, len(byName)) + for _, node := range byName { + result.NodeSummaries = append(result.NodeSummaries, node) + } + + *current = result + + return nil +} + // mergeStatusDelta applies an incremental delta to the current cluster status. func mergeStatusDelta(current *clusterStatusResponse, deltaRaw json.RawMessage) error { var delta clusterStatusDelta @@ -411,7 +474,7 @@ func renderWatchScreenSummary( // runWatch connects via WebSocket and renders live-updating data to the terminal. // On WebSocket failure it reconnects with exponential backoff and falls back to -// HTTP polling via fetchClusterStatus while disconnected. +// HTTP overview polling while disconnected. // // When wopts is non-nil and wopts.renderSummary is set, the client subscribes // to the lightweight cluster_summary protocol. If the controller supports it, @@ -439,6 +502,8 @@ func runWatch( ctx, stop := signal.NotifyContext(ctx, os.Interrupt) defer stop() + cmd.SetContext(ctx) + // Enter raw mode for 'q' keypress detection. Cleanup is in the main // function scope so the terminal is always restored, even on error exits. var terminalRestored bool @@ -506,6 +571,7 @@ func runWatch( var currentSummary *clusterSummary summaryInitialized := false + summaryProtocol := false const ( backoffMin = 1 * time.Second @@ -515,6 +581,48 @@ func runWatch( backoff := backoffMin + poll := func() error { + var pollErr error + + if summaryMode { + var summary clusterSummary + + summary, pollErr = fetchClusterSummary(rt, cmd, fetchOpts) + if pollErr == nil { + currentSummary = &summary + summaryInitialized = true + lastSeq = summary.Seq + } + } else { + var status clusterStatusResponse + + status, pollErr = fetchClusterStatus(rt, cmd, fetchOpts) + if pollErr == nil { + current = status + initialized = true + } + } + + if pollErr == nil { + lastUpdate = time.Now() + connState = "Polling" + } else if !initialized && !summaryInitialized { + connState = "Disconnected" + } + + if summaryMode && summaryInitialized { + return renderWatchScreenSummary(os.Stdout, currentSummary, connState, + lastSeq, lastUpdate, useColor, wopts.renderSummary) + } + + if initialized { + return renderWatchScreen(os.Stdout, os.Stdout, current, connState, + lastSeq, lastUpdate, useColor, render) + } + + return nil + } + for { if ctx.Err() != nil { _, _ = fmt.Fprintln(os.Stdout) //nolint:errcheck @@ -537,6 +645,8 @@ func runWatch( conn.SetReadLimit(32 * 1024 * 1024) + summaryProtocol = false + // Subscribe to the summary protocol when configured. If the // controller does not support it the message is silently ignored // and the client keeps receiving cluster_status / cluster_status_delta. @@ -568,6 +678,19 @@ func runWatch( switch msg.Type { case "cluster_status": + if summaryMode { + summary, err := decodeClusterSummary(msg.Data) + if err != nil { + continue + } + + currentSummary = &summary + summaryInitialized = true + lastUpdate = time.Now() + + break + } + if err := json.Unmarshal(msg.Data, ¤t); err != nil { continue } @@ -575,6 +698,20 @@ func runWatch( initialized = true lastUpdate = time.Now() case "cluster_status_delta": + if summaryMode { + if summaryProtocol { + continue + } + + // An old controller's partial detail delta needs a heavy base. + // Refresh the overview instead of retaining that base. + if err := poll(); err != nil { + return err + } + + continue + } + if !initialized { continue } @@ -602,8 +739,21 @@ func runWatch( currentSummary = &summary summaryInitialized = true + summaryProtocol = true lastSeq = summary.Seq lastUpdate = time.Now() + case "cluster_summary_delta": + if !summaryMode || !summaryInitialized { + continue + } + + if err := mergeClusterSummaryDelta(currentSummary, msg.Data); err != nil { + continue + } + + summaryProtocol = true + lastSeq = currentSummary.Seq + lastUpdate = time.Now() default: continue } @@ -627,24 +777,8 @@ func runWatch( // Polling fallback while waiting to reconnect WebSocket. // Poll once before applying the backoff wait. - pollStatus, pollErr := fetchClusterStatus(rt, cmd, fetchOpts) - if pollErr == nil { - current = pollStatus - initialized = true - lastUpdate = time.Now() - connState = "Polling" - } else { - if !initialized { - connState = "Disconnected" - } - // Keep connState as "Polling" if we had data before. - } - - if initialized { - if err := renderWatchScreen(os.Stdout, os.Stdout, current, connState, - lastSeq, lastUpdate, useColor, render); err != nil { - return err - } + if err := poll(); err != nil { + return err } // Backoff wait before attempting WebSocket reconnection. @@ -666,21 +800,8 @@ func runWatch( waited += sleepDur // Poll during the backoff window. - pollStatus, pollErr := fetchClusterStatus(rt, cmd, fetchOpts) - if pollErr == nil { - current = pollStatus - initialized = true - lastUpdate = time.Now() - connState = "Polling" - } else if !initialized { - connState = "Disconnected" - } - - if initialized { - if err := renderWatchScreen(os.Stdout, os.Stdout, current, connState, - lastSeq, lastUpdate, useColor, render); err != nil { - return err - } + if err := poll(); err != nil { + return err } } diff --git a/cmd/unbounded-net-controller/cluster_status.go b/cmd/unbounded-net-controller/cluster_status.go index 6185da8c8..d33dcce34 100644 --- a/cmd/unbounded-net-controller/cluster_status.go +++ b/cmd/unbounded-net-controller/cluster_status.go @@ -18,6 +18,7 @@ import ( "k8s.io/klog/v2" "github.com/Azure/unbounded/internal/net/controller" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" "github.com/Azure/unbounded/internal/version" ) @@ -236,6 +237,13 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo } cachedStatuses := health.statusCache.GetAll() + status.NodeOverviews = make(map[string]*statusv1alpha1.NodeStatusOverview) + + for name, cached := range cachedStatuses { + if cached.Overview != nil { + status.NodeOverviews[name] = cached.Overview + } + } type pullNode struct{ nodeName, nodeIP string } @@ -371,6 +379,7 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo } else if result.status != nil { result.status.StatusSource = "pull" cachedResults[result.nodeName] = *result.status + delete(status.NodeOverviews, result.nodeName) } } } @@ -447,6 +456,9 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo if pubKey := node.Annotations[controller.WireGuardPubKeyAnnotation]; pubKey != "" { if nodeStatus.NodeInfo.WireGuard == nil { nodeStatus.NodeInfo.WireGuard = &WireGuardStatusInfo{} + } else { + wireguard := *nodeStatus.NodeInfo.WireGuard + nodeStatus.NodeInfo.WireGuard = &wireguard } nodeStatus.NodeInfo.WireGuard.PublicKey = pubKey @@ -710,7 +722,6 @@ func fetchClusterStatus(ctx context.Context, health *healthState, pullEnabled bo } sort.Slice(status.Peerings, func(i, j int) bool { return status.Peerings[i].Name < status.Peerings[j].Name }) - status.ConnectivityMatrix = buildConnectivityMatrix(status.Nodes, status.GatewayPools) status.Problems = collectClusterProblems(status) return status @@ -809,6 +820,18 @@ func collectClusterProblems(status *ClusterStatusResponse) []StatusProblem { appendProblem("node", nodeName, summary) } + if overview := status.NodeOverviews[node.NodeInfo.Name]; overview != nil { + if overview.RouteMismatch { + appendProblem("node", nodeName, "Route next-hop mismatches (expected vs present)") + } + + if unhealthy := overview.PeerCount - overview.HealthyPeers; unhealthy > 0 { + appendProblem("node", nodeName, fmt.Sprintf("%d peers are not healthy", unhealthy)) + } + + continue + } + if mismatchCount := routeMismatchCount(node); mismatchCount > 0 { appendProblem("node", nodeName, fmt.Sprintf("%d route next-hop mismatches (expected vs present)", mismatchCount)) } @@ -1038,163 +1061,3 @@ func latestNodeUpdateTime(node *corev1.Node) time.Time { return latest } - -// buildConnectivityMatrix builds health check connectivity matrices from node peer data. -func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []GatewayPoolStatus) map[string]*SiteMatrix { - siteNodes := make(map[string]map[string]bool) - nodePeers := make(map[string][]WireGuardPeerStatus) - nodeByName := make(map[string]*NodeStatusResponse) - - for _, n := range nodes { - name := n.NodeInfo.Name - - site := n.NodeInfo.SiteName - if name == "" || site == "" { - continue - } - - nodeByName[name] = n - - if siteNodes[site] == nil { - siteNodes[site] = make(map[string]bool) - } - - siteNodes[site][name] = true - - // Keep the immutable snapshot's slice; filter when reading rather - // than copying every peer, including for scopes above the size limit. - nodePeers[name] = n.Peers - - for _, p := range n.Peers { - if p.PeerType == "gateway" && p.Name != "" && p.SiteName == site { - siteNodes[site][p.Name] = true - } - } - } - - if len(siteNodes) == 0 { - siteNodes = make(map[string]map[string]bool) - } - - result := make(map[string]*SiteMatrix) - selfMatrixStatusFromCNI := func(node *NodeStatusResponse) string { - if node.NodeInfo.WireGuard != nil && strings.TrimSpace(node.NodeInfo.WireGuard.Interface) != "" { - return "up" - } - - return "" - } - - buildScopeMatrix := func(nodeSet map[string]bool) *SiteMatrix { - if len(nodeSet) == 0 || len(nodeSet) > 100 { - return nil - } - - nodeNames := make([]string, 0, len(nodeSet)) - for name := range nodeSet { - nodeNames = append(nodeNames, name) - } - - sort.Strings(nodeNames) - - results := make(map[string]map[string]string) - for _, srcNode := range nodeNames { - results[srcNode] = make(map[string]string) - if node, ok := nodeByName[srcNode]; ok { - results[srcNode][srcNode] = selfMatrixStatusFromCNI(node) - } - - for _, peer := range nodePeers[srcNode] { - if !isConnectivityMatrixPeer(peer) { - continue - } - - tgtNode := peer.Name - if tgtNode == "" || tgtNode == srcNode || !nodeSet[tgtNode] { - continue - } - - cellStatus := "" - if peer.HealthCheck != nil { - cellStatus = peer.HealthCheck.Status - } else if peer.PeerType == "gateway" && !peer.Tunnel.LastHandshake.IsZero() { - cellStatus = "up" - } - - results[srcNode][tgtNode] = cellStatus - } - } - - return &SiteMatrix{Nodes: nodeNames, Results: results} - } - - for site, nodeSet := range siteNodes { - scopeMatrix := buildScopeMatrix(nodeSet) - if scopeMatrix != nil { - result[site] = scopeMatrix - } - } - - for _, pool := range gatewayPools { - poolName := strings.TrimSpace(pool.Name) - if poolName == "" { - continue - } - - poolNodeSet := make(map[string]bool) - - for _, gatewayName := range pool.Gateways { - name := strings.TrimSpace(gatewayName) - if name == "" { - continue - } - - poolNodeSet[name] = true - for _, peer := range nodePeers[name] { - if !isConnectivityMatrixPeer(peer) { - continue - } - - peerName := strings.TrimSpace(peer.Name) - if peerName == "" { - continue - } - - if _, ok := nodeByName[peerName]; ok { - poolNodeSet[peerName] = true - } - } - - for srcNodeName, peers := range nodePeers { - for _, peer := range peers { - if !isConnectivityMatrixPeer(peer) { - continue - } - - if strings.TrimSpace(peer.Name) == name { - if _, ok := nodeByName[srcNodeName]; ok { - poolNodeSet[srcNodeName] = true - } - - break - } - } - } - } - - scopeMatrix := buildScopeMatrix(poolNodeSet) - if scopeMatrix != nil { - result["pool:"+poolName] = scopeMatrix - } - } - - if len(result) == 0 { - return nil - } - - return result -} - -func isConnectivityMatrixPeer(peer WireGuardPeerStatus) bool { - return peer.PeerType == "site" || peer.PeerType == "gateway" -} diff --git a/cmd/unbounded-net-controller/cluster_status_cache.go b/cmd/unbounded-net-controller/cluster_status_cache.go index 6a9b5d13e..33437a4a8 100644 --- a/cmd/unbounded-net-controller/cluster_status_cache.go +++ b/cmd/unbounded-net-controller/cluster_status_cache.go @@ -5,6 +5,7 @@ package main import ( "context" + "maps" "reflect" "slices" "sync" @@ -15,6 +16,8 @@ import ( unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" "github.com/Azure/unbounded/internal/net/controller" + statuspkg "github.com/Azure/unbounded/internal/net/status" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) // ClusterStatusCache maintains a pre-built ClusterStatusResponse in memory, @@ -115,6 +118,15 @@ func (c *ClusterStatusCache) Rebuild(ctx context.Context) { // PatchNode updates a single node's cached status in-place without a full // rebuild. func (c *ClusterStatusCache) PatchNode(nodeName string, nodeStatus NodeStatusResponse) { + c.patchNode(nodeName, nodeStatus, nil) +} + +// PatchOverview updates metadata and observed facts without collecting details. +func (c *ClusterStatusCache) PatchOverview(nodeName string, overview statusv1alpha1.NodeStatusOverview) { + c.patchNode(nodeName, statuspkg.OverviewMetadata(overview), &overview) +} + +func (c *ClusterStatusCache) patchNode(nodeName string, nodeStatus NodeStatusResponse, overview *statusv1alpha1.NodeStatusOverview) { now := time.Now() nodeStatus.NodeInfo.ExternalIPs = c.resolveNodeExternalIPs(nodeName, now) @@ -125,6 +137,16 @@ func (c *ClusterStatusCache) PatchNode(nodeName string, nodeStatus NodeStatusRes return } + if overview == nil { + delete(c.status.NodeOverviews, nodeName) + } else { + if c.status.NodeOverviews == nil { + c.status.NodeOverviews = make(map[string]*statusv1alpha1.NodeStatusOverview) + } + + c.status.NodeOverviews[nodeName] = overview + } + if i, ok := c.nodeIndex[nodeName]; ok && i < len(c.status.Nodes) { // Preserve controller-enriched fields across node-agent status updates. existing := c.status.Nodes[i] @@ -225,13 +247,21 @@ func (c *ClusterStatusCache) MarkFullRebuildNeeded() { } } -// Get returns the current pre-built status (read-locked, fast). +// Get snapshots mutable containers; nested node data remains immutable and shared. // Returns nil if the status has not been built yet. func (c *ClusterStatusCache) Get() *ClusterStatusResponse { c.mu.RLock() defer c.mu.RUnlock() - return c.status + if c.status == nil { + return nil + } + + snapshot := *c.status + snapshot.Nodes = slices.Clone(c.status.Nodes) + snapshot.NodeOverviews = maps.Clone(c.status.NodeOverviews) + + return &snapshot } // GetSeq returns the current sequence number. diff --git a/cmd/unbounded-net-controller/cluster_status_test.go b/cmd/unbounded-net-controller/cluster_status_test.go index 1caaeb41f..00d717c1d 100644 --- a/cmd/unbounded-net-controller/cluster_status_test.go +++ b/cmd/unbounded-net-controller/cluster_status_test.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "slices" - "strconv" "strings" "testing" "time" @@ -447,88 +446,6 @@ func TestNodeReadinessAndLatestUpdateTime(t *testing.T) { } } -// TestBuildConnectivityMatrix tests BuildConnectivityMatrix. -func TestBuildConnectivityMatrix(t *testing.T) { - now := time.Now().Add(-75 * time.Second) - - nodes := []*NodeStatusResponse{ - { - NodeInfo: NodeInfo{Name: "node-a", SiteName: "site-a", WireGuard: &WireGuardStatusInfo{Interface: "wg51820"}}, - Peers: []WireGuardPeerStatus{ - {Name: "node-b", PeerType: "site", HealthCheck: &HealthCheckPeerStatus{Status: "up", Uptime: "15s"}}, - {Name: "gw-a", PeerType: "gateway", SiteName: "site-a", Tunnel: PeerTunnelStatus{LastHandshake: now}}, - {Name: "gw-remote", PeerType: "gateway", SiteName: "site-b", Tunnel: PeerTunnelStatus{LastHandshake: now}}, - }, - }, - { - NodeInfo: NodeInfo{Name: "node-b", SiteName: "site-a"}, - Peers: []WireGuardPeerStatus{ - {Name: "node-a", PeerType: "site", HealthCheck: &HealthCheckPeerStatus{Status: "down", Uptime: "3s"}}, - }, - }, - } - for i := 0; i < 101; i++ { - nodes = append(nodes, &NodeStatusResponse{NodeInfo: NodeInfo{Name: "big-" + strconv.Itoa(i), SiteName: "site-big"}}) - } - - gatewayPools := []GatewayPoolStatus{{ - Name: "pool-a", - Gateways: []string{"gw-a"}, - }} - - matrix := buildConnectivityMatrix(nodes, gatewayPools) - if matrix == nil { - t.Fatalf("expected non-nil connectivity matrix") - } - - if _, ok := matrix["site-big"]; ok { - t.Fatalf("expected site-big to be skipped when >100 nodes") - } - - site := matrix["site-a"] - if site == nil { - t.Fatalf("expected site-a matrix") - } - - if !slices.Equal(site.Nodes, []string{"gw-a", "node-a", "node-b"}) { - t.Fatalf("unexpected node list: %#v", site.Nodes) - } - - if got := site.Results["node-a"]["node-b"]; got != "up" { - t.Fatalf("unexpected node-a->node-b status: %q", got) - } - - gatewayCell := site.Results["node-a"]["gw-a"] - if gatewayCell != "up" { - t.Fatalf("unexpected gateway fallback cell: %q", gatewayCell) - } - - if _, ok := site.Results["node-a"]["gw-remote"]; ok { - t.Fatalf("did not expect remote-site gateway in site matrix") - } - - if got := site.Results["node-a"]["node-a"]; got != "up" { - t.Fatalf("expected self cell for node-a to be up from CNI health, got %q", got) - } - - if got := site.Results["node-b"]["node-b"]; got != "" { - t.Fatalf("expected self cell for node-b to be unknown when CNI health is unavailable, got %q", got) - } - - pool := matrix["pool:pool-a"] - if pool == nil { - t.Fatalf("expected pool:pool-a matrix") - } - - if !slices.Equal(pool.Nodes, []string{"gw-a", "node-a"}) { - t.Fatalf("unexpected pool node list: %#v", pool.Nodes) - } - - if got := pool.Results["node-a"]["gw-a"]; got != "up" { - t.Fatalf("unexpected node-a->gw-a pool status: %q", got) - } -} - // TestCollectClusterProblemsIncludesUnhealthySignals tests CollectClusterProblemsIncludesUnhealthySignals. func TestCollectClusterProblemsIncludesUnhealthySignals(t *testing.T) { expectedTrue := true diff --git a/cmd/unbounded-net-controller/detail_api.go b/cmd/unbounded-net-controller/detail_api.go new file mode 100644 index 000000000..ba720e8e3 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_api.go @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + "k8s.io/klog/v2" + + "github.com/Azure/unbounded/internal/net/authn" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" + webhookpkg "github.com/Azure/unbounded/internal/net/webhook" +) + +func registerNodeDetailHandlers(mux *http.ServeMux, health *healthState, requireAuth bool, webhookServer *webhookpkg.Server, authorizer *dashboardAuthorizer, issuer *authn.TokenIssuer) { + mux.HandleFunc("/status/node/{name}/details", func(w http.ResponseWriter, r *http.Request) { + if !authorizeDashboardOrAggregated(requireAuth, issuer, authorizer, webhookServer, r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + + return + } + + nodeName := r.PathValue("name") + manager := health.getDetailRequests() + + if !health.isLeader.Load() || manager == nil { + writeNodeDetailResult(w, http.StatusServiceUnavailable, + detailRequestFailure(nodeName, r.URL.Query().Get("requestId"), statusv1alpha1.NodeDetailRetryable, "detail request leader is unavailable")) + + return + } + + var result statusv1alpha1.NodeDetailResult + + switch r.Method { + case http.MethodPost: + var input *struct { + ForceRefresh bool `json:"forceRefresh"` + } + + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + + err := decoder.Decode(&input) + if err == nil { + var extra any + + err = decoder.Decode(&extra) + if errors.Is(err, io.EOF) && input != nil { + err = nil + } else if err == nil || input == nil { + err = errors.New("expected one JSON object") + } + } + + if err != nil { + code := http.StatusBadRequest + + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + code = http.StatusRequestEntityTooLarge + } + + writeNodeDetailResult(w, code, detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailUnavailable, err.Error())) + + return + } + + result = manager.Request(nodeName, input.ForceRefresh) + case http.MethodGet: + requestID := r.URL.Query().Get("requestId") + if requestID == "" { + writeNodeDetailResult(w, http.StatusBadRequest, + detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailUnavailable, "requestId is required")) + + return + } + + result = manager.Result(nodeName, requestID) + default: + w.Header().Set("Allow", "GET, POST") + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + + return + } + + code := http.StatusOK + + switch result.State { + case statusv1alpha1.NodeDetailPending: + code = http.StatusAccepted + case statusv1alpha1.NodeDetailExpired: + code = http.StatusGone + case statusv1alpha1.NodeDetailUnavailable: + code = http.StatusNotFound + case statusv1alpha1.NodeDetailRetryable: + code = http.StatusServiceUnavailable + case statusv1alpha1.NodeDetailComplete: + } + + writeNodeDetailResult(w, code, result) + }) +} + +func writeNodeDetailResult(w http.ResponseWriter, code int, result statusv1alpha1.NodeDetailResult) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(code) + + if err := json.NewEncoder(w).Encode(result); err != nil { + klog.V(4).Infof("node detail response encode failed: %v", err) + } +} diff --git a/cmd/unbounded-net-controller/detail_api_test.go b/cmd/unbounded-net-controller/detail_api_test.go new file mode 100644 index 000000000..42579d0f7 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_api_test.go @@ -0,0 +1,377 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "testing/synctest" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func serveDetailRequest(t *testing.T, mux *http.ServeMux, method, path, body string) (*httptest.ResponseRecorder, statusv1alpha1.NodeDetailResult) { + t.Helper() + + recorder := httptest.NewRecorder() + mux.ServeHTTP(recorder, httptest.NewRequest(method, path, strings.NewReader(body))) + + var result statusv1alpha1.NodeDetailResult + if recorder.Header().Get("Content-Type") == "application/json" { + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + } + + return recorder, result +} + +func TestDetailAPIRequestAndResult(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + health := &healthState{detailRequests: manager} + health.isLeader.Store(true) + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, false, nil, nil, nil) + + path := "/status/node/node/details" + response, pending := serveDetailRequest(t, mux, http.MethodPost, path, `{}`) + + if response.Code != http.StatusAccepted || pending.State != statusv1alpha1.NodeDetailPending || pending.RequestID == "" { + t.Fatalf("POST did not return a pending request: %d %s", response.Code, response.Body.String()) + } + + if err := manager.Complete("node", pending.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + response, complete := serveDetailRequest(t, mux, http.MethodGet, path+"?requestId="+pending.RequestID, "") + if response.Code != http.StatusOK || complete.State != statusv1alpha1.NodeDetailComplete || + complete.Details == nil || complete.Details.Status.NodeInfo.Name != "node" || + response.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("GET did not return cached details: %d %s", response.Code, response.Body.String()) + } + + response, reused := serveDetailRequest(t, mux, http.MethodPost, path, `{"forceRefresh":false}`) + if response.Code != http.StatusOK || reused.RequestID != pending.RequestID { + t.Fatal("POST did not reuse existing details") + } + + response, refresh := serveDetailRequest(t, mux, http.MethodPost, path, `{"forceRefresh":true}`) + if response.Code != http.StatusAccepted || refresh.RequestID == pending.RequestID { + t.Fatal("forced refresh did not create a new request") + } + + time.Sleep(manager.timeout) + synctest.Wait() + + response, expired := serveDetailRequest(t, mux, http.MethodGet, path+"?requestId="+refresh.RequestID, "") + + if response.Code != http.StatusGone || expired.State != statusv1alpha1.NodeDetailExpired || expired.Details != nil { + t.Fatal("expired request was not explicit") + } + + health.setLeader(false) + + response, stopped := serveDetailRequest(t, mux, http.MethodGet, path+"?requestId="+pending.RequestID, "") + + if response.Code != http.StatusServiceUnavailable || stopped.State != statusv1alpha1.NodeDetailRetryable || stopped.Details != nil { + t.Fatal("leadership loss did not produce retryable failure") + } + + assertNodeDetailEntries(t, manager.cache, 0) + }) +} + +func TestDetailAPIMethodsAndErrors(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + health := &healthState{detailRequests: manager} + health.isLeader.Store(true) + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, false, nil, nil, nil) + + for _, tc := range []struct { + method string + query string + body string + code int + }{ + {http.MethodDelete, "", "", http.StatusMethodNotAllowed}, + {http.MethodGet, "", "", http.StatusBadRequest}, + {http.MethodGet, "?requestId=unknown", "", http.StatusServiceUnavailable}, + {http.MethodPost, "", "", http.StatusBadRequest}, + {http.MethodPost, "", "null", http.StatusBadRequest}, + {http.MethodPost, "", "{} {}", http.StatusBadRequest}, + {http.MethodPost, "", `{"forceRefresh":"yes"}`, http.StatusBadRequest}, + {http.MethodPost, "", `{"url":"http://caller-controlled"}`, http.StatusBadRequest}, + {http.MethodPost, "", strings.Repeat(" ", 1<<20) + "{}", http.StatusRequestEntityTooLarge}, + } { + response, _ := serveDetailRequest(t, mux, tc.method, "/status/node/node/details"+tc.query, tc.body) + if response.Code != tc.code { + t.Fatalf("%s %s: got %d, want %d: %s", tc.method, tc.query, response.Code, tc.code, response.Body.String()) + } + + if tc.code == http.StatusMethodNotAllowed && response.Header().Get("Allow") != "GET, POST" { + t.Fatal("missing Allow header") + } + } + + manager.mu.Lock() + count := len(manager.requests) + manager.mu.Unlock() + + if count != 0 { + t.Fatal("invalid API requests created work") + } + }) +} + +func TestDetailAPIAuthorization(t *testing.T) { + for _, allowed := range []bool{false, true} { + t.Run(strconv.FormatBool(allowed), func(t *testing.T) { + client := k8sfake.NewClientset() + client.PrependReactor("create", "subjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + review := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SubjectAccessReview) + if review.Spec.ResourceAttributes.Name != "dashboard" || review.Spec.ResourceAttributes.Verb != "get" { + t.Error("detail API changed the existing authorization resource") + } + + return true, &authorizationv1.SubjectAccessReview{Status: authorizationv1.SubjectAccessReviewStatus{Allowed: allowed}}, nil + }) + + issuer := testTokenIssuer(t) + + viewer, _, err := issuer.IssueViewerToken("viewer", nil, time.Hour) + if err != nil { + t.Fatal(err) + } + + health := &healthState{detailRequests: testDetailRequests(t, nodeDetailRequestHooks{})} + health.isLeader.Store(true) + + proxy, trustedTLS := testNodeTokenFrontProxy(t) + mux := http.NewServeMux() + registerStatusHandlers(mux, health, true, proxy, newDashboardAuthorizer(client), issuer) + + for _, token := range []string{"", "invalid", testNodeToken(t, issuer), viewer} { + request := httptest.NewRequest(http.MethodPost, "/status/node/node/details", strings.NewReader("{}")) + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + + want := http.StatusUnauthorized + if token == viewer && allowed { + want = http.StatusAccepted + } + + if response.Code != want { + t.Fatalf("authorization: got %d, want %d", response.Code, want) + } + } + + request := httptest.NewRequest(http.MethodPost, "/status/node/node/details", strings.NewReader("{}")) + request.TLS = trustedTLS + request.Header.Set("X-Remote-User", "aggregated-viewer") + + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + + if response.Code != http.StatusAccepted { + t.Fatalf("trusted aggregated request rejected: %d %s", response.Code, response.Body.String()) + } + }) + } +} + +func testDetailLifecycle(t *testing.T, port int) (*healthState, cache.SharedIndexInformer, *nodeDetailRequests) { + t.Helper() + + health := &healthState{ + statusDetailCacheTTL: 10 * time.Second, statusDetailRequestTimeout: 3 * time.Second, + nodeAgentHealthPort: port, + } + health.isLeader.Store(true) + + factory := informers.NewSharedInformerFactory(k8sfake.NewClientset(), 0) + informer := factory.Core().V1().Nodes().Informer() + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node", UID: "uid"}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "127.0.0.1"}}}, + } + if err := informer.GetIndexer().Add(node); err != nil { + t.Fatal(err) + } + + manager, err := health.startDetailRequests(t.Context(), informer) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(manager.Close) + + return health, informer, manager +} + +func TestDetailAPIHTTPPull(t *testing.T) { + for _, mode := range []string{"success", "failure", "wrong-node", "oversized"} { + t.Run(mode, func(t *testing.T) { + pullSucceeds := mode == "success" || mode == "oversized" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/status/json" || r.Method != http.MethodGet { + t.Error("incorrect node detail pull endpoint") + } + + if mode == "failure" { + http.Error(w, "unreachable", http.StatusServiceUnavailable) + + return + } + + status := testDetailStatus() + if mode == "wrong-node" { + status.NodeInfo.Name = "other" + } + + if mode == "oversized" { + status.NodeInfo.K8sLabels = map[string]string{"large": strings.Repeat("x", 1<<20)} + } + + json.NewEncoder(w).Encode(status) + })) + defer server.Close() + + _, portText, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) + if err != nil { + t.Fatal(err) + } + + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatal(err) + } + + health, _, manager := testDetailLifecycle(t, port) + if health.pullEnabled.Load() { + t.Fatal("test must exercise disabled background pulls") + } + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, false, nil, nil, nil) + _, request := serveDetailRequest(t, mux, http.MethodPost, "/status/node/node/details", "{}") + + deadline := time.NewTimer(time.Second) + defer deadline.Stop() + + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + + for { + response, result := serveDetailRequest(t, mux, http.MethodGet, "/status/node/node/details?requestId="+request.RequestID, "") + if pullSucceeds && result.State == statusv1alpha1.NodeDetailComplete { + if response.Code != http.StatusOK || result.Details == nil || result.Details.Status.NodeInfo.Name != "node" { + t.Fatal("HTTP pull result is incomplete") + } + + // The status POST body limit does not limit legacy HTTP pull responses. + if mode == "oversized" && len(result.Details.Status.NodeInfo.K8sLabels["large"]) != 1<<20 { + t.Fatal("HTTP pull response was truncated to the POST body limit") + } + + break + } + + if !pullSucceeds { + if _, ok := manager.Pending("node"); ok { + result = manager.Result("node", request.RequestID) + + if result.Details != nil || result.Error == "" { + t.Fatal("failed pull returned success-shaped details") + } + + break + } + } + + select { + case <-deadline.C: + t.Fatalf("HTTP pull did not settle: %+v", result) + case <-ticker.C: + } + } + }) + } +} + +func TestDetailLifecycleInvalidationAndShutdown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + health, informer, manager := testDetailLifecycle(t, 0) + + request := manager.Request("node", false) + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + oldNode := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", UID: "uid"}} + newNode := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", UID: "new"}} + detailNodeEvents(manager).OnUpdate(oldNode, newNode) + assertNodeDetailEntries(t, manager.cache, 0) + + if _, err := health.startDetailRequests(t.Context(), informer); err == nil { + t.Fatal("duplicate lifecycle initialized") + } + + health.setLeader(false) + synctest.Wait() + + if health.getDetailRequests() != nil { + t.Fatal("leadership loss retained manager") + } + + ctx, cancel := context.WithCancel(t.Context()) + + health.isLeader.Store(true) + + restarted, err := health.startDetailRequests(ctx, informer) + if err != nil { + t.Fatal(err) + } + + restarted.Request("node", true) + detailNodeEvents(restarted).OnDelete(cache.DeletedFinalStateUnknown{Obj: oldNode}) + detailNodeEvents(restarted).OnDelete("invalid") + cancel() + restarted.Close() + synctest.Wait() + + if health.getDetailRequests() != nil { + t.Fatal("context cancellation retained manager") + } + + assertNodeDetailEntries(t, restarted.cache, 0) + }) +} diff --git a/cmd/unbounded-net-controller/detail_cache.go b/cmd/unbounded-net-controller/detail_cache.go new file mode 100644 index 000000000..ccd64fe2c --- /dev/null +++ b/cmd/unbounded-net-controller/detail_cache.go @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "sync" + "time" + + "k8s.io/utils/clock" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// nodeDetailSnapshot carries immutable details, separate from routine status. +// Status and all its nested data must remain read-only, including for callers +// retaining a returned snapshot after its cache entry expires. +type nodeDetailSnapshot = statusv1alpha1.NodeDetailSnapshot + +// nodeDetailCache is a leader-local, TTL-only store. It owns no second result +// history or per-entry timers. TTL bounds retention time, not peak memory. +// Construct it with newNodeDetailCache and run one Run loop for proactive expiry. +type nodeDetailCache struct { + mu sync.Mutex + ttl time.Duration + clock clock.Clock // May be replaced in tests before concurrent use. + entries map[string]nodeDetailSnapshot + changed chan struct{} + running bool +} + +func newNodeDetailCache(ttl time.Duration) (*nodeDetailCache, error) { + if ttl <= 0 { + return nil, errors.New("node detail cache TTL must be positive") + } + + return &nodeDetailCache{ + ttl: ttl, + clock: clock.RealClock{}, + entries: make(map[string]nodeDetailSnapshot), + changed: make(chan struct{}, 1), + }, nil +} + +// Store accepts actual detailed data only; callers must not pass summaries or +// failed fetches. It shallow-copies status without modifying it. Nested slices, +// maps, and pointers remain shared and must not be mutated by the caller. +// Only Store renews the receipt-based TTL; request validation belongs upstream. +func (c *nodeDetailCache) Store(nodeName, requestID string, collectedAt time.Time, status *NodeStatusResponse) (nodeDetailSnapshot, error) { + if nodeName == "" { + return nodeDetailSnapshot{}, errors.New("node detail cache requires a node name") + } + + if status == nil { + return nodeDetailSnapshot{}, errors.New("node detail cache requires detailed status") + } + + statusCopy := *status + + c.mu.Lock() + defer c.mu.Unlock() + + now := c.clock.Now() + snapshot := nodeDetailSnapshot{ + NodeName: nodeName, + RequestID: requestID, + CollectedAt: collectedAt, + ReceivedAt: now, + ExpiresAt: now.Add(c.ttl), + Status: &statusCopy, + } + c.entries[nodeName] = snapshot + c.notify() + + return snapshot, nil +} + +// Get does not refresh TTL. At the deadline it drops the cache's ownership, +// even when the proactive expiry loop has not yet been scheduled. +func (c *nodeDetailCache) Get(nodeName string) (nodeDetailSnapshot, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + snapshot, ok := c.entries[nodeName] + if !ok { + return nodeDetailSnapshot{}, false + } + + if !c.clock.Now().Before(snapshot.ExpiresAt) { + delete(c.entries, nodeName) + c.notify() + + return nodeDetailSnapshot{}, false + } + + return snapshot, true +} + +func (c *nodeDetailCache) Delete(nodeName string) { + c.mu.Lock() + defer c.mu.Unlock() + + delete(c.entries, nodeName) + c.notify() +} + +// Clear releases all cache-owned details without mutating shared payloads. +// It does not stop Run; subsequent stores can be expired by the same loop. +func (c *nodeDetailCache) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + + clear(c.entries) + c.notify() +} + +// Expire removes entries at or past their deadline and returns their count. +func (c *nodeDetailCache) Expire() int { + c.mu.Lock() + defer c.mu.Unlock() + + removed, _ := c.expireLocked(c.clock.Now()) + c.notify() + + return removed +} + +func (c *nodeDetailCache) expireLocked(now time.Time) (int, time.Time) { + removed := 0 + + var next time.Time + + for name, snapshot := range c.entries { + if !now.Before(snapshot.ExpiresAt) { + delete(c.entries, name) + + removed++ + } else if next.IsZero() || snapshot.ExpiresAt.Before(next) { + next = snapshot.ExpiresAt + } + } + + return removed, next +} + +func (c *nodeDetailCache) notify() { + select { + case c.changed <- struct{}{}: + default: + } +} + +// Run blocks until cancellation, then stops its timer and clears all entries. +// Wait for Run to return before restarting it or storing for a new leadership +// term. Concurrent Run calls are rejected; no goroutine is started internally. +func (c *nodeDetailCache) Run(ctx context.Context) error { + c.mu.Lock() + if c.running { + c.mu.Unlock() + + return errors.New("node detail cache expiry loop is already running") + } + + c.running = true + c.mu.Unlock() + + defer func() { + c.mu.Lock() + defer c.mu.Unlock() + + clear(c.entries) + c.running = false + }() + + for ctx.Err() == nil { + c.mu.Lock() + _, next := c.expireLocked(c.clock.Now()) + c.mu.Unlock() + + var ( + timer clock.Timer + timerC <-chan time.Time + ) + + if !next.IsZero() { + timer = c.clock.NewTimer(next.Sub(c.clock.Now())) + timerC = timer.C() + } + + select { + case <-ctx.Done(): + case <-c.changed: + case <-timerC: + } + + if timer != nil { + timer.Stop() + } + } + + return nil +} diff --git a/cmd/unbounded-net-controller/detail_cache_test.go b/cmd/unbounded-net-controller/detail_cache_test.go new file mode 100644 index 000000000..629204a3b --- /dev/null +++ b/cmd/unbounded-net-controller/detail_cache_test.go @@ -0,0 +1,448 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "sync" + "testing" + "testing/synctest" + "time" + + testingclock "k8s.io/utils/clock/testing" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func newTestNodeDetailCache(t *testing.T) (*nodeDetailCache, *testingclock.FakeClock) { + t.Helper() + + cache, err := newNodeDetailCache(time.Minute) + if err != nil { + t.Fatal(err) + } + + fakeClock := testingclock.NewFakeClock(time.Now()) + cache.clock = fakeClock + + return cache, fakeClock +} + +func storeTestNodeDetails(t *testing.T, cache *nodeDetailCache, requestID string) nodeDetailSnapshot { + t.Helper() + + snapshot, err := cache.Store("node", requestID, cache.clock.Now().Add(-time.Hour), &NodeStatusResponse{}) + if err != nil { + t.Fatal(err) + } + + return snapshot +} + +func assertNodeDetailEntries(t *testing.T, cache *nodeDetailCache, want int) { + t.Helper() + cache.mu.Lock() + defer cache.mu.Unlock() + + if got := len(cache.entries); got != want { + t.Fatalf("cache owns %d entries, want %d", got, want) + } +} + +func startNodeDetailLoop(t *testing.T, cache *nodeDetailCache) (context.CancelFunc, <-chan error) { + t.Helper() + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + + go func() { + done <- cache.Run(ctx) + }() + + t.Cleanup(cancel) + synctest.Wait() + + return cancel, done +} + +func TestNodeDetailCacheValidation(t *testing.T) { + for _, ttl := range []time.Duration{-time.Second, 0} { + if cache, err := newNodeDetailCache(ttl); err == nil || cache != nil { + t.Fatalf("TTL %v: got cache %v, error %v", ttl, cache, err) + } + } + + if _, err := newNodeDetailCache(time.Nanosecond); err != nil { + t.Fatalf("positive TTL rejected: %v", err) + } + + cache, fakeClock := newTestNodeDetailCache(t) + initial := storeTestNodeDetails(t, cache, "original") + + fakeClock.Step(time.Second) + + if _, err := cache.Store("", "invalid", fakeClock.Now(), &NodeStatusResponse{}); err == nil { + t.Fatal("empty node name accepted") + } + + if _, err := cache.Store("node", "invalid", fakeClock.Now(), nil); err == nil { + t.Fatal("nil details accepted") + } + + if got, ok := cache.Get("node"); !ok || got != initial { + t.Fatal("invalid Store replaced or refreshed existing details") + } + + if got, ok := cache.Get("missing"); ok || got != (nodeDetailSnapshot{}) { + t.Fatal("missing entry did not return an empty snapshot") + } + + assertNodeDetailEntries(t, cache, 1) +} + +func TestNodeDetailCacheDeadlineAndReadNonrefresh(t *testing.T) { + for _, offset := range []time.Duration{-time.Nanosecond, 0, time.Nanosecond} { + t.Run(offset.String(), func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + initial := storeTestNodeDetails(t, cache, "request") + + if initial.NodeName != "node" || initial.RequestID != "request" || + !initial.ReceivedAt.Equal(fakeClock.Now()) || + !initial.CollectedAt.Equal(fakeClock.Now().Add(-time.Hour)) || + !initial.ExpiresAt.Equal(fakeClock.Now().Add(cache.ttl)) { + t.Fatalf("incorrect receipt metadata: %+v", initial) + } + + fakeClock.Step(cache.ttl / 2) + + for range 3 { + got, ok := cache.Get("node") + if !ok || got != initial { + t.Fatal("read changed the snapshot") + } + + got.RequestID = "local-copy-only" + } + + fakeClock.Step(cache.ttl/2 + offset) + + got, ok := cache.Get("node") + if offset < 0 { + if !ok || got != initial { + t.Fatal("details expired before the deadline") + } + + assertNodeDetailEntries(t, cache, 1) + } else { + if ok || got != (nodeDetailSnapshot{}) { + t.Fatal("expired details returned") + } + + assertNodeDetailEntries(t, cache, 0) + } + }) + } +} + +func TestNodeDetailCacheReplacement(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + initial := storeTestNodeDetails(t, cache, "first") + fakeClock.Step(cache.ttl / 2) + replacement := storeTestNodeDetails(t, cache, "second") + + if replacement.Status == initial.Status || replacement.ExpiresAt != initial.ExpiresAt.Add(cache.ttl/2) { + t.Fatal("replacement did not renew details and TTL") + } + + fakeClock.Step(cache.ttl / 2) + + if removed := cache.Expire(); removed != 0 { + t.Fatalf("old deadline removed %d replacements", removed) + } + + if got, ok := cache.Get("node"); !ok || got != replacement { + t.Fatal("replacement missing at the old deadline") + } + + fakeClock.Step(cache.ttl / 2) + + if removed := cache.Expire(); removed != 1 { + t.Fatalf("removed %d entries at replacement deadline, want 1", removed) + } + + if removed := cache.Expire(); removed != 0 { + t.Fatalf("repeated expiry removed %d entries", removed) + } + + assertNodeDetailEntries(t, cache, 0) +} + +func TestNodeDetailCacheReleasesHeavyReferences(t *testing.T) { + for _, operation := range []string{"replace", "delete", "clear", "expire", "get-expired"} { + t.Run(operation, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + status := &NodeStatusResponse{ + NodeInfo: NodeInfo{K8sLabels: map[string]string{"label": "original"}}, + Peers: make([]statusv1alpha1.PeerStatus, 1024), + RoutingTable: RoutingTableInfo{Routes: []statusv1alpha1.RouteEntry{{ + NextHops: make([]statusv1alpha1.NextHop, 1024), + }}}, + BpfEntries: make([]BpfEntry, 1024), + } + + snapshot, err := cache.Store("node", "heavy", fakeClock.Now(), status) + if err != nil { + t.Fatal(err) + } + + if snapshot.Status == status || &snapshot.Status.Peers[0] != &status.Peers[0] || + &snapshot.Status.RoutingTable.Routes[0] != &status.RoutingTable.Routes[0] || + &snapshot.Status.BpfEntries[0] != &status.BpfEntries[0] { + t.Fatal("Store must copy the top-level value but share nested details") + } + + switch operation { + case "replace": + replacement := storeTestNodeDetails(t, cache, "light") + cache.mu.Lock() + stored := cache.entries["node"] + cache.mu.Unlock() + + if stored != replacement || stored.Status == snapshot.Status { + t.Fatal("map still owns the heavy snapshot") + } + case "delete": + cache.Delete("missing") + assertNodeDetailEntries(t, cache, 1) + cache.Delete("node") + cache.Delete("node") + case "clear": + if _, err := cache.Store("other", "", fakeClock.Now(), status); err != nil { + t.Fatal(err) + } + + cache.Clear() + cache.Clear() + case "expire": + fakeClock.Step(cache.ttl) + cache.Expire() + case "get-expired": + fakeClock.Step(cache.ttl) + cache.Get("node") + } + + if operation != "replace" { + assertNodeDetailEntries(t, cache, 0) + } + + if status.NodeInfo.K8sLabels["label"] != "original" || len(status.Peers) != 1024 || + len(status.RoutingTable.Routes[0].NextHops) != 1024 || len(status.BpfEntries) != 1024 || + len(snapshot.Status.Peers) != 1024 || snapshot.Status.NodeInfo.K8sLabels["label"] != "original" { + t.Fatal("removing ownership mutated a shared payload") + } + }) + } +} + +func TestNodeDetailCacheRunExpiryAndReplacement(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + cancel, done := startNodeDetailLoop(t, cache) + storeTestNodeDetails(t, cache, "first") + synctest.Wait() + fakeClock.Step(cache.ttl / 2) + storeTestNodeDetails(t, cache, "replacement") + synctest.Wait() + fakeClock.Step(cache.ttl / 2) + synctest.Wait() + + // Inspect the map, not Get: readers must not be required for cleanup. + assertNodeDetailEntries(t, cache, 1) + fakeClock.Step(cache.ttl / 2) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + + if fakeClock.HasWaiters() { + t.Fatal("expiry loop left an active timer") + } + }) +} + +func TestNodeDetailCacheRunClearCancelRestart(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + storeTestNodeDetails(t, cache, "before-run") + cancel, done := startNodeDetailLoop(t, cache) + + if err := cache.Run(t.Context()); err == nil { + t.Fatal("concurrent expiry loop accepted") + } + + cache.Clear() + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + + if fakeClock.HasWaiters() { + t.Fatal("Clear left an active timer") + } + + storeTestNodeDetails(t, cache, "after-clear") + synctest.Wait() + fakeClock.Step(cache.ttl) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + storeTestNodeDetails(t, cache, "before-cancel") + synctest.Wait() + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + + assertNodeDetailEntries(t, cache, 0) + + if fakeClock.HasWaiters() { + t.Fatal("cancellation left an active timer") + } + + storeTestNodeDetails(t, cache, "restart") + cancel, done = startNodeDetailLoop(t, cache) + fakeClock.Step(cache.ttl) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + }) +} + +func TestNodeDetailCacheRunMultipleDeadlines(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + storeTestNodeDetails(t, cache, "already-expired") + fakeClock.Step(cache.ttl) + cancel, done := startNodeDetailLoop(t, cache) + assertNodeDetailEntries(t, cache, 0) + + storeTestNodeDetails(t, cache, "earliest") + synctest.Wait() + fakeClock.Step(cache.ttl / 2) + + if _, err := cache.Store("later", "", fakeClock.Now(), &NodeStatusResponse{}); err != nil { + t.Fatal(err) + } + + synctest.Wait() + fakeClock.Step(cache.ttl / 2) + synctest.Wait() + assertNodeDetailEntries(t, cache, 1) + + cache.mu.Lock() + _, earliestRetained := cache.entries["node"] + _, laterRetained := cache.entries["later"] + cache.mu.Unlock() + + if earliestRetained || !laterRetained { + t.Fatal("loop did not expire only the earliest deadline") + } + + fakeClock.Step(cache.ttl / 2) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + }) +} + +func TestNodeDetailCacheRunAlreadyCanceled(t *testing.T) { + cache, _ := newTestNodeDetailCache(t) + storeTestNodeDetails(t, cache, "request") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + if err := cache.Run(ctx); err != nil { + t.Fatal(err) + } + + assertNodeDetailEntries(t, cache, 0) +} + +func TestNodeDetailCacheConcurrentAccess(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, fakeClock := newTestNodeDetailCache(t) + cancel, done := startNodeDetailLoop(t, cache) + + var workers sync.WaitGroup + + for worker := range 8 { + workers.Go(func() { + for iteration := range 100 { + name := fmt.Sprintf("node-%d", iteration%4) + request := fmt.Sprintf("%d-%d", worker, iteration) + + if _, err := cache.Store(name, request, fakeClock.Now(), &NodeStatusResponse{}); err != nil { + t.Error(err) + + return + } + + cache.Get(name) + fakeClock.Step(time.Second) + cache.Expire() + cache.Delete(name) + + if iteration%10 == 0 { + cache.Clear() + } + } + }) + } + + workers.Wait() + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + + assertNodeDetailEntries(t, cache, 0) + + if fakeClock.HasWaiters() { + t.Fatal("concurrent operations left an active timer") + } + }) +} + +func TestNodeDetailCacheRunRealClock(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cache, err := newNodeDetailCache(time.Minute) + if err != nil { + t.Fatal(err) + } + + storeTestNodeDetails(t, cache, "request") + cancel, done := startNodeDetailLoop(t, cache) + + // synctest advances virtual standard-library time without a real sleep. + time.Sleep(cache.ttl) + synctest.Wait() + assertNodeDetailEntries(t, cache, 0) + cancel() + + if err := <-done; err != nil { + t.Fatal(err) + } + }) +} diff --git a/cmd/unbounded-net-controller/detail_dispatch.go b/cmd/unbounded-net-controller/detail_dispatch.go new file mode 100644 index 000000000..193d4db48 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_dispatch.go @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +type nodeWSConnection struct { + cancel context.CancelFunc + send func(context.Context, statusv1alpha1.DetailRequest) error +} + +func (h *healthState) setNodeWSDetailSender(nodeName string, connection *nodeWSConnection, send func(context.Context, statusv1alpha1.DetailRequest) error) { + h.nodeWSMu.Lock() + + ready := false + if connection != nil && h.nodeWSRegistry[nodeName] == connection { + ready = connection.send == nil + connection.send = send + } + h.nodeWSMu.Unlock() + + if ready { + h.retryNodeDetails(nodeName) + } +} + +func (h *healthState) dispatchNodeDetail(ctx context.Context, nodeName string, command statusv1alpha1.DetailRequest) (bool, error) { + h.nodeWSMu.Lock() + + var send func(context.Context, statusv1alpha1.DetailRequest) error + if connection := h.nodeWSRegistry[nodeName]; connection != nil { + send = connection.send + } + h.nodeWSMu.Unlock() + + if send == nil { + return false, nil + } + + writeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + err := send(writeCtx, command) + + return err == nil, err +} + +func (h *healthState) retryNodeDetails(nodeName string) { + if manager := h.getDetailRequests(); manager != nil { + manager.Retry(nodeName) + } +} + +// Retry wakes an existing request after a transport change, retaining its ID, +// deadline, and one-dispatch-at-a-time ownership. +func (m *nodeDetailRequests) Retry(nodeName string) { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + if request := m.active[nodeName]; request != nil && m.ctx.Err() == nil && !m.closed { + request.retry = true + if !request.dispatching { + m.startDispatchLocked(request) + } + } +} + +func (m *nodeDetailRequests) startDispatchLocked(request *nodeDetailRequest) { + request.dispatching = true + request.retry = false + request.poll = false + + m.workers.Go(func() { + m.dispatch(request.ctx, request.nodeName, request.command) + + m.mu.Lock() + defer m.mu.Unlock() + + request.dispatching = false + if m.active[request.nodeName] == request && request.retry && m.ctx.Err() == nil && !m.closed { + m.startDispatchLocked(request) + } + }) +} diff --git a/cmd/unbounded-net-controller/detail_dispatch_test.go b/cmd/unbounded-net-controller/detail_dispatch_test.go new file mode 100644 index 000000000..8c75514f6 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_dispatch_test.go @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestDetailDispatchUsesOnlyCurrentCapableConnection(t *testing.T) { + health := &healthState{} + + var canceled, sent atomic.Int32 + + cancel := func() { canceled.Add(1) } + old := health.registerNodeWS("node", cancel) + current := health.registerNodeWS("node", cancel) + health.unregisterNodeWS("node", old) + + if canceled.Load() != 1 { + t.Fatal("replaced connection was not canceled") + } + + command := statusv1alpha1.DetailRequest{RequestID: "request", Deadline: time.Now().Add(time.Minute)} + if ok, err := health.dispatchNodeDetail(t.Context(), "node", command); ok || err != nil { + t.Fatal("legacy connection was treated as command-capable") + } + + health.setNodeWSDetailSender("node", old, func(context.Context, statusv1alpha1.DetailRequest) error { + t.Error("an obsolete connection sent a command") + return nil + }) + health.setNodeWSDetailSender("node", current, func(_ context.Context, got statusv1alpha1.DetailRequest) error { + if got != command { + t.Error("command identity or deadline changed") + } + + sent.Add(1) + + return nil + }) + + if ok, err := health.dispatchNodeDetail(t.Context(), "node", command); !ok || err != nil || sent.Load() != 1 { + t.Fatalf("current connection did not receive command: %v %v", ok, err) + } + + health.unregisterNodeWS("node", current) + + if ok, _ := health.dispatchNodeDetail(t.Context(), "node", command); ok { + t.Fatal("closed connection remained usable") + } +} + +func TestDetailDispatchDisconnectAndReconnectPreserveDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + health := &healthState{} + commands := make(chan statusv1alpha1.DetailRequest, 3) + sender := func(_ context.Context, command statusv1alpha1.DetailRequest) error { + select { + case commands <- command: + return nil + default: + return errors.New("unexpected extra dispatch") + } + } + + var pulls atomic.Int32 + + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: health.dispatchNodeDetail, + Pull: func(context.Context, string) (*NodeStatusResponse, error) { + pulls.Add(1) + return nil, errors.New("unreachable") + }, + }) + health.detailRequests = manager + connection := health.registerNodeWS("node", func() {}) + health.setNodeWSDetailSender("node", connection, sender) + + request := manager.Request("node", true) + + synctest.Wait() + + if len(commands) != 1 { + t.Fatal("expected one WebSocket command") + } + + first := <-commands + if pulls.Load() != 0 || first.RequestID != request.RequestID { + t.Fatal("active WebSocket did not take priority") + } + + time.Sleep(time.Second) + health.unregisterNodeWS("node", connection) + synctest.Wait() + + pending, ok := manager.Pending("node") + if !ok || pulls.Load() != 1 || pending != first { + t.Fatal("disconnect failed to pull and retain the original POST command") + } + + reconnected := health.registerNodeWS("node", func() {}) + health.setNodeWSDetailSender("node", reconnected, sender) + synctest.Wait() + + if len(commands) != 1 { + t.Fatal("expected one command on reconnect") + } + + if next := <-commands; next != first { + t.Fatal("reconnect reset request identity or deadline") + } + + health.setNodeWSDetailSender("node", reconnected, sender) + synctest.Wait() + + if len(commands) != 0 { + t.Fatal("a routine capability update dispatched another collection") + } + + if err := manager.Complete("node", first.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + health.unregisterNodeWS("node", reconnected) + synctest.Wait() + + if pulls.Load() != 1 { + t.Fatal("a completed request restarted after disconnect") + } + }) +} + +func TestDetailDispatchRetryCoalescesAndCancels(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var active, peak atomic.Int32 + + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: func(ctx context.Context, _ string, _ statusv1alpha1.DetailRequest) (bool, error) { + n := active.Add(1) + if n > peak.Load() { + peak.Store(n) + } + + defer active.Add(-1) + + <-ctx.Done() + + return false, ctx.Err() + }, + }) + manager.Request("node", true) + synctest.Wait() + + for range 10 { + manager.Retry("node") + } + + synctest.Wait() + manager.Close() + + if peak.Load() != 1 || active.Load() != 0 { + t.Fatal("retry created concurrent dispatches or shutdown left one running") + } + }) +} + +func TestDetailDispatchWriteTimeoutFallsBackWithinOriginalDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + health := &healthState{} + connection := health.registerNodeWS("node", func() {}) + health.setNodeWSDetailSender("node", connection, func(ctx context.Context, _ statusv1alpha1.DetailRequest) error { + <-ctx.Done() + return ctx.Err() + }) + + var pulls atomic.Int32 + + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: health.dispatchNodeDetail, + Pull: func(context.Context, string) (*NodeStatusResponse, error) { + pulls.Add(1) + return testDetailStatus(), nil + }, + }) + manager.timeout = 20 * time.Second + health.detailRequests = manager + request := manager.Request("node", true) + + synctest.Wait() + time.Sleep(5 * time.Second) + synctest.Wait() + + result := manager.Result("node", request.RequestID) + if pulls.Load() != 1 || result.State != statusv1alpha1.NodeDetailComplete || !result.Deadline.Equal(request.Deadline) { + t.Fatalf("failed WebSocket write did not fall back within the original deadline: %+v", result) + } + }) +} diff --git a/cmd/unbounded-net-controller/detail_lifecycle.go b/cmd/unbounded-net-controller/detail_lifecycle.go new file mode 100644 index 000000000..ce2f49b9f --- /dev/null +++ b/cmd/unbounded-net-controller/detail_lifecycle.go @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "fmt" + "net" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +func (h *healthState) getDetailRequests() *nodeDetailRequests { + h.detailMu.Lock() + defer h.detailMu.Unlock() + + return h.detailRequests +} + +func (h *healthState) stopDetailRequests() { + h.detailMu.Lock() + manager := h.detailRequests + h.detailRequests = nil + h.detailMu.Unlock() + + if manager != nil { + manager.Close() + } +} + +// startDetailRequests is called once per leadership term, with that term's +// context and node informer. Neither hooks nor workers use the HTTP caller's +// context, so disconnecting a viewer does not cancel another viewer's request. +func (h *healthState) startDetailRequests(ctx context.Context, nodeInformer cache.SharedIndexInformer) (*nodeDetailRequests, error) { + if nodeInformer == nil { + return nil, errors.New("node detail requests require a node informer") + } + + detailCache, err := newNodeDetailCache(h.statusDetailCacheTTL) + if err != nil { + return nil, err + } + + lister := corev1listers.NewNodeLister(nodeInformer.GetIndexer()) + + manager, err := newNodeDetailRequests(ctx, detailCache, h.statusDetailRequestTimeout, nodeDetailRequestHooks{ + Dispatch: h.dispatchNodeDetail, + Resolve: func(name string) (types.UID, error) { + node, err := lister.Get(name) + if err != nil { + return "", err + } + + return node.UID, nil + }, + Pull: func(ctx context.Context, name string) (*NodeStatusResponse, error) { + node, err := lister.Get(name) + if err != nil { + return nil, err + } + + for _, address := range node.Status.Addresses { + if address.Type != corev1.NodeInternalIP || net.ParseIP(address.Address) == nil { + continue + } + + host := address.Address + if net.ParseIP(host).To4() == nil { + host = "[" + host + "]" + } + + return fetchNodeStatus(ctx, host, h.nodeAgentHealthPort) + } + + return nil, fmt.Errorf("node %q has no valid InternalIP", name) + }, + }) + if err != nil { + return nil, err + } + + registration, err := nodeInformer.AddEventHandler(detailNodeEvents(manager)) + if err != nil { + manager.Close() + + return nil, fmt.Errorf("register detail node invalidation: %w", err) + } + + h.detailMu.Lock() + if !h.isLeader.Load() || ctx.Err() != nil || h.detailRequests != nil { + h.detailMu.Unlock() + manager.Close() + + if err := nodeInformer.RemoveEventHandler(registration); err != nil { + klog.Warningf("Removing node detail event handler: %v", err) + } + + return nil, errors.New("detail request leadership is unavailable or already initialized") + } + + h.detailRequests = manager + h.detailMu.Unlock() + + go func() { + <-manager.done + + if err := nodeInformer.RemoveEventHandler(registration); err != nil { + klog.Warningf("Removing node detail event handler: %v", err) + } + + h.detailMu.Lock() + if h.detailRequests == manager { + h.detailRequests = nil + } + h.detailMu.Unlock() + }() + + return manager, nil +} + +func detailNodeEvents(manager *nodeDetailRequests) cache.ResourceEventHandlerFuncs { + return cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(oldObj, newObj any) { + oldNode, oldOK := oldObj.(*corev1.Node) + + newNode, newOK := newObj.(*corev1.Node) + if oldOK && newOK && oldNode.UID != newNode.UID { + manager.InvalidateNode(oldNode.Name, oldNode.UID) + } + }, + DeleteFunc: func(obj any) { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + + if node, ok := obj.(*corev1.Node); ok { + manager.InvalidateNode(node.Name, node.UID) + } + }, + } +} diff --git a/cmd/unbounded-net-controller/detail_requests.go b/cmd/unbounded-net-controller/detail_requests.go new file mode 100644 index 000000000..0477b9a81 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_requests.go @@ -0,0 +1,405 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "sync" + "time" + + "k8s.io/apimachinery/pkg/types" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// Hooks must honor cancellation. Resolve reads the current informer identity; +// Dispatch returns true only when an active WebSocket accepted the command. +type nodeDetailRequestHooks struct { + Resolve func(string) (types.UID, error) + Dispatch func(context.Context, string, statusv1alpha1.DetailRequest) (bool, error) + Pull func(context.Context, string) (*NodeStatusResponse, error) +} + +// A request owns metadata and cancellation only, never a result payload. +type nodeDetailRequest struct { + nodeName string + uid types.UID + command statusv1alpha1.DetailRequest + state statusv1alpha1.NodeDetailState + message string + wakeAt time.Time + poll bool + cancel context.CancelFunc + ctx context.Context + dispatching bool + retry bool +} + +type nodeDetailRequests struct { + mu sync.Mutex + ctx context.Context + cancel context.CancelFunc + done chan struct{} + changed chan struct{} + workers sync.WaitGroup + closed bool + timeout time.Duration + cache *nodeDetailCache + hooks nodeDetailRequestHooks + requests map[string]*nodeDetailRequest + active map[string]*nodeDetailRequest +} + +// newNodeDetailRequests takes exclusive lifecycle ownership of cache. All +// snapshots must enter through Complete so their node UID binding is known. +func newNodeDetailRequests(ctx context.Context, cache *nodeDetailCache, timeout time.Duration, hooks nodeDetailRequestHooks) (*nodeDetailRequests, error) { + if cache == nil || timeout <= 0 || hooks.Resolve == nil { + return nil, errors.New("detail requests require a cache, positive timeout, and node resolver") + } + + ctx, cancel := context.WithCancel(ctx) + m := &nodeDetailRequests{ + ctx: ctx, cancel: cancel, done: make(chan struct{}), changed: make(chan struct{}, 1), + timeout: timeout, cache: cache, hooks: hooks, + requests: make(map[string]*nodeDetailRequest), active: make(map[string]*nodeDetailRequest), + } + cache.Clear() + + go m.run() + + return m, nil +} + +// Close cancels dispatches, clears leader-local state, and waits for all workers. +func (m *nodeDetailRequests) Close() { + m.cancel() + <-m.done +} + +func (m *nodeDetailRequests) Request(nodeName string, forceRefresh bool) statusv1alpha1.NodeDetailResult { + m.mu.Lock() + defer m.mu.Unlock() + + if m.ctx.Err() != nil || m.closed { + return detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailRetryable, "detail request leader is unavailable") + } + + m.expireLocked(time.Now()) + + uid, err := m.hooks.Resolve(nodeName) + if err != nil || uid == "" { + return detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailUnavailable, "node identity is unavailable") + } + + for _, request := range m.requests { + if request.nodeName == nodeName && request.uid != uid { + m.invalidateLocked(request) + } + } + + if !forceRefresh { + if snapshot, ok := m.cache.Get(nodeName); ok { + if request := m.requests[snapshot.RequestID]; request != nil && request.uid == uid { + return m.resultLocked(request) + } + } + } + + if request := m.active[nodeName]; request != nil { + return m.resultLocked(request) + } + + now := time.Now() + request := &nodeDetailRequest{ + nodeName: nodeName, uid: uid, state: statusv1alpha1.NodeDetailPending, + command: statusv1alpha1.DetailRequest{RequestID: rand.Text(), Deadline: now.Add(m.timeout)}, + wakeAt: now.Add(m.timeout), + } + ctx, cancel := context.WithDeadline(m.ctx, request.command.Deadline) + request.cancel = cancel + request.ctx = ctx + m.requests[request.command.RequestID] = request + m.active[nodeName] = request + m.notify() + m.startDispatchLocked(request) + + return m.resultLocked(request) +} + +func (m *nodeDetailRequests) Result(nodeName, requestID string) statusv1alpha1.NodeDetailResult { + m.mu.Lock() + defer m.mu.Unlock() + + if m.ctx.Err() != nil || m.closed { + return detailRequestFailure(nodeName, requestID, statusv1alpha1.NodeDetailRetryable, "detail request leader is unavailable") + } + + m.expireLocked(time.Now()) + + request := m.requests[requestID] + if request == nil || request.nodeName != nodeName { + return detailRequestFailure(nodeName, requestID, statusv1alpha1.NodeDetailRetryable, "request is no longer known; retry on the current leader") + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + } + + return m.resultLocked(request) +} + +// Complete is idempotent while a completed request is retained. It rejects +// mismatched, late, and expired replies without replacing data or renewing TTL. +func (m *nodeDetailRequests) Complete(nodeName, requestID string, status *NodeStatusResponse) error { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + request := m.requests[requestID] + if m.ctx.Err() != nil || m.closed || request == nil || request.nodeName != nodeName { + return errors.New("detail request is no longer available") + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + + return errors.New("detail request node was deleted or replaced") + } + + if status == nil || status.NodeInfo.Name != nodeName || status.FetchError != "" { + return errors.New("detail response has missing data, a fetch error, or a mismatched node name") + } + + if request.state == statusv1alpha1.NodeDetailComplete { + return nil + } + + if request.state != statusv1alpha1.NodeDetailPending { + return errors.New("detail request is no longer pending") + } + + snapshot, err := m.cache.Store(nodeName, requestID, status.Timestamp, status) + if err != nil { + return err + } + + request.state = statusv1alpha1.NodeDetailComplete + request.message = "" + request.poll = false + request.wakeAt = snapshot.ExpiresAt + request.cancel() + delete(m.active, nodeName) + m.notify() + + return nil +} + +// Pending exposes only a failed-pull fallback command, without refreshing its +// deadline. Returning it repeatedly is safe until a valid reply completes it. +func (m *nodeDetailRequests) Pending(nodeName string) (statusv1alpha1.DetailRequest, bool) { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + request := m.active[nodeName] + if m.ctx.Err() != nil || request == nil || !request.poll { + return statusv1alpha1.DetailRequest{}, false + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + + return statusv1alpha1.DetailRequest{}, false + } + + return request.command, true +} + +// InvalidateNode handles deletion/replacement of a specific UID. A delayed old +// informer event cannot cancel a request for a newer node with the same name. +func (m *nodeDetailRequests) InvalidateNode(nodeName string, uid types.UID) { + m.mu.Lock() + defer m.mu.Unlock() + + for _, request := range m.requests { + if request.nodeName == nodeName && request.uid == uid { + m.invalidateLocked(request) + } + } +} + +func (m *nodeDetailRequests) invalidateLocked(request *nodeDetailRequest) { + if request.state == statusv1alpha1.NodeDetailUnavailable { + return + } + + request.cancel() + request.state = statusv1alpha1.NodeDetailUnavailable + request.message = "node was deleted or replaced" + request.poll = false + request.wakeAt = time.Now().Add(m.timeout) + + if m.active[request.nodeName] == request { + delete(m.active, request.nodeName) + } + + if snapshot, ok := m.cache.Get(request.nodeName); ok && snapshot.RequestID == request.command.RequestID { + m.cache.Delete(request.nodeName) + } + + m.notify() +} + +func (m *nodeDetailRequests) resultLocked(request *nodeDetailRequest) statusv1alpha1.NodeDetailResult { + result := statusv1alpha1.NodeDetailResult{ + NodeName: request.nodeName, RequestID: request.command.RequestID, Deadline: request.command.Deadline, + State: request.state, Error: request.message, + } + if request.state == statusv1alpha1.NodeDetailComplete { + if snapshot, ok := m.cache.Get(request.nodeName); ok && snapshot.RequestID == request.command.RequestID { + result.Details = &snapshot + } else { + result.State = statusv1alpha1.NodeDetailExpired + result.Error = "details expired or were replaced" + } + } + + return result +} + +func detailRequestFailure(nodeName, requestID string, state statusv1alpha1.NodeDetailState, message string) statusv1alpha1.NodeDetailResult { + return statusv1alpha1.NodeDetailResult{NodeName: nodeName, RequestID: requestID, State: state, Error: message} +} + +func (m *nodeDetailRequests) dispatch(ctx context.Context, nodeName string, command statusv1alpha1.DetailRequest) { + if m.hooks.Dispatch != nil { + if sent, err := m.hooks.Dispatch(ctx, nodeName, command); sent && err == nil { + return + } + } + + err := errors.New("node HTTP detail pull is unavailable") + + if m.hooks.Pull != nil && ctx.Err() == nil { + pullCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + status, pullErr := m.hooks.Pull(pullCtx, nodeName) + + cancel() + + err = pullErr + if err == nil { + err = m.Complete(nodeName, command.RequestID, status) + } + + if err == nil { + return + } + } + + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + if request := m.active[nodeName]; request != nil && request.command.RequestID == command.RequestID && m.ctx.Err() == nil { + request.poll = true + request.message = fmt.Sprintf("HTTP detail pull failed; waiting for status POST: %v", err) + } +} + +func (m *nodeDetailRequests) expireLocked(now time.Time) time.Time { + var next time.Time + + for id, request := range m.requests { + if !now.Before(request.wakeAt) { + if request.state == statusv1alpha1.NodeDetailPending || request.state == statusv1alpha1.NodeDetailComplete { + request.cancel() + request.state = statusv1alpha1.NodeDetailExpired + request.message = "detail request or snapshot expired" + request.poll = false + request.wakeAt = request.wakeAt.Add(m.timeout) + + if m.active[request.nodeName] == request { + delete(m.active, request.nodeName) + } + } + + if !now.Before(request.wakeAt) { + delete(m.requests, id) + + continue + } + } + + if next.IsZero() || request.wakeAt.Before(next) { + next = request.wakeAt + } + } + + return next +} + +func (m *nodeDetailRequests) notify() { + select { + case m.changed <- struct{}{}: + default: + } +} + +func (m *nodeDetailRequests) run() { + cacheDone := make(chan struct{}) + go func() { + defer close(cacheDone) + + if err := m.cache.Run(m.ctx); err != nil { + m.cancel() + } + }() + + for m.ctx.Err() == nil { + m.mu.Lock() + next := m.expireLocked(time.Now()) + m.mu.Unlock() + + var ( + timer *time.Timer + timerC <-chan time.Time + ) + + if !next.IsZero() { + timer = time.NewTimer(time.Until(next)) + timerC = timer.C + } + + select { + case <-m.ctx.Done(): + case <-m.changed: + case <-timerC: + } + + if timer != nil { + timer.Stop() + } + } + + m.mu.Lock() + m.closed = true + + for _, request := range m.requests { + request.cancel() + } + + clear(m.requests) + clear(m.active) + m.cache.Clear() + m.mu.Unlock() + m.workers.Wait() + <-cacheDone + close(m.done) +} diff --git a/cmd/unbounded-net-controller/detail_requests_test.go b/cmd/unbounded-net-controller/detail_requests_test.go new file mode 100644 index 000000000..af5c38567 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_requests_test.go @@ -0,0 +1,316 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "k8s.io/apimachinery/pkg/types" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func testDetailRequests(t *testing.T, hooks nodeDetailRequestHooks) *nodeDetailRequests { + t.Helper() + + if hooks.Resolve == nil { + hooks.Resolve = func(string) (types.UID, error) { return "uid", nil } + } + + cache, err := newNodeDetailCache(10 * time.Second) + if err != nil { + t.Fatal(err) + } + + manager, err := newNodeDetailRequests(t.Context(), cache, 3*time.Second, hooks) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(manager.Close) + + return manager +} + +func testDetailStatus() *NodeStatusResponse { + return &NodeStatusResponse{Timestamp: time.Now(), NodeInfo: NodeInfo{Name: "node"}} +} + +func TestDetailRequestsValidation(t *testing.T) { + cache, _ := newNodeDetailCache(time.Second) + resolve := func(string) (types.UID, error) { return "uid", nil } + + for _, tc := range []struct { + cache *nodeDetailCache + timeout time.Duration + hooks nodeDetailRequestHooks + }{ + {nil, time.Second, nodeDetailRequestHooks{Resolve: resolve}}, + {cache, 0, nodeDetailRequestHooks{Resolve: resolve}}, + {cache, -time.Second, nodeDetailRequestHooks{Resolve: resolve}}, + {cache, time.Second, nodeDetailRequestHooks{}}, + } { + if _, err := newNodeDetailRequests(t.Context(), tc.cache, tc.timeout, tc.hooks); err == nil { + t.Fatal("invalid constructor accepted") + } + } +} + +func TestDetailRequestsCoalesceAndDispatch(t *testing.T) { + for _, activeWS := range []bool{false, true} { + t.Run(map[bool]string{false: "HTTP", true: "WebSocket"}[activeWS], func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var pulls, sends atomic.Int32 + + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: func(context.Context, string, statusv1alpha1.DetailRequest) (bool, error) { + sends.Add(1) + + return activeWS, nil + }, + Pull: func(ctx context.Context, _ string) (*NodeStatusResponse, error) { + pulls.Add(1) + <-ctx.Done() + + return nil, ctx.Err() + }, + }) + first := manager.Request("node", false) + + var workers sync.WaitGroup + + for range 32 { + workers.Go(func() { + result := manager.Request("node", true) + if result.State != statusv1alpha1.NodeDetailPending || result.RequestID != first.RequestID || result.Deadline != first.Deadline { + t.Error("concurrent refresh did not coalesce") + } + }) + } + + workers.Wait() + synctest.Wait() + + if sends.Load() != 1 || pulls.Load() != map[bool]int32{false: 1, true: 0}[activeWS] { + t.Fatal("unexpected dispatch count") + } + + if _, ok := manager.Pending("node"); ok { + t.Fatal("poll command available before pull failure") + } + }) + }) + } +} + +func TestDetailRequestsCacheRefreshAndDuplicate(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + request := manager.Request("node", false) + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + original := manager.Result("node", request.RequestID) + + time.Sleep(time.Second) + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + duplicate := manager.Request("node", false) + if duplicate.State != statusv1alpha1.NodeDetailComplete || duplicate.Details == nil || + *duplicate.Details != *original.Details { + t.Fatal("duplicate changed data or receipt TTL") + } + + refresh := manager.Request("node", true) + if refresh.RequestID == request.RequestID || refresh.State != statusv1alpha1.NodeDetailPending { + t.Fatal("refresh did not create a fresh request") + } + + if cached := manager.Request("node", false); cached.RequestID != request.RequestID || cached.Details == nil { + t.Fatal("pending refresh prevented reuse of valid data") + } + + if err := manager.Complete("node", refresh.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + if old := manager.Result("node", request.RequestID); old.State != statusv1alpha1.NodeDetailExpired || old.Details != nil { + t.Fatal("old request retained a second result") + } + + time.Sleep(10 * time.Second) + synctest.Wait() + + if result := manager.Result("node", refresh.RequestID); result.State != statusv1alpha1.NodeDetailExpired || result.Details != nil { + t.Fatal("result still available at TTL boundary") + } + + assertNodeDetailEntries(t, manager.cache, 0) + }) +} + +func TestDetailRequestsFallbackDeadlineAndCleanup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: func(context.Context, string, statusv1alpha1.DetailRequest) (bool, error) { + return true, errors.New("socket closed") + }, + Pull: func(ctx context.Context, _ string) (*NodeStatusResponse, error) { + deadline, ok := ctx.Deadline() + if !ok || deadline != time.Now().Add(3*time.Second) { + t.Error("pull did not inherit the overall deadline") + } + + time.Sleep(time.Second) + + return nil, errors.New("unreachable") + }, + }) + request := manager.Request("node", false) + + synctest.Wait() + time.Sleep(time.Second) + synctest.Wait() + + for range 2 { + command, ok := manager.Pending("node") + if !ok || command.RequestID != request.RequestID || command.Deadline != request.Deadline { + t.Fatal("failed pull did not expose the unchanged polling command") + } + } + + time.Sleep(2*time.Second - time.Nanosecond) + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailPending { + t.Fatal("request expired too early") + } + + time.Sleep(time.Nanosecond) + synctest.Wait() + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailExpired { + t.Fatal("request remained pending at deadline") + } + + if _, ok := manager.Pending("node"); ok { + t.Fatal("expired polling command retained") + } + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("late result accepted") + } + + assertNodeDetailEntries(t, manager.cache, 0) + time.Sleep(manager.timeout) + synctest.Wait() + manager.mu.Lock() + count := len(manager.requests) + len(manager.active) + manager.mu.Unlock() + + if count != 0 { + t.Fatal("terminal metadata was not proactively removed") + } + }) +} + +func TestDetailRequestsBindingAndDeletion(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + uid := types.UID("old") + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Resolve: func(string) (types.UID, error) { return uid, nil }, + }) + request := manager.Request("node", false) + + for _, status := range []*NodeStatusResponse{nil, {}, {NodeInfo: NodeInfo{Name: "wrong"}}, {NodeInfo: NodeInfo{Name: "node"}, FetchError: "failed"}} { + if err := manager.Complete("node", request.RequestID, status); err == nil { + t.Fatal("invalid details accepted") + } + } + + if err := manager.Complete("other", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("wrong node binding accepted") + } + + if err := manager.Complete("node", "unknown", testDetailStatus()); err == nil { + t.Fatal("unknown request accepted") + } + + synctest.Wait() + + uid = "replacement" + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("replaced node accepted") + } + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailUnavailable { + t.Fatal("replacement did not invalidate request") + } + + fresh := manager.Request("node", false) + manager.InvalidateNode("node", "old") + + if err := manager.Complete("node", fresh.RequestID, testDetailStatus()); err != nil { + t.Fatalf("old informer event invalidated new node: %v", err) + } + + manager.InvalidateNode("node", "replacement") + assertNodeDetailEntries(t, manager.cache, 0) + + if result := manager.Result("node", fresh.RequestID); result.State != statusv1alpha1.NodeDetailUnavailable { + t.Fatal("node deletion did not invalidate cached details") + } + }) +} + +func TestDetailRequestsHTTPCompletionAndShutdown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Pull: func(context.Context, string) (*NodeStatusResponse, error) { return testDetailStatus(), nil }, + }) + request := manager.Request("node", false) + + synctest.Wait() + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailComplete || result.Details == nil { + t.Fatal("HTTP pull did not complete") + } + + manager.Close() + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailRetryable || result.Details != nil { + t.Fatal("shutdown did not make results retryable") + } + + if result := manager.Request("node", true); result.State != statusv1alpha1.NodeDetailRetryable { + t.Fatal("shutdown accepted a new request") + } + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("shutdown accepted late details") + } + + assertNodeDetailEntries(t, manager.cache, 0) + restarted := testDetailRequests(t, nodeDetailRequestHooks{}) + + if result := restarted.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailRetryable { + t.Fatal("new leader pretended to own an old request") + } + + if result := restarted.Request("node", false); result.RequestID == request.RequestID { + t.Fatal("restart reused an old request ID") + } + }) +} diff --git a/cmd/unbounded-net-controller/detail_responses.go b/cmd/unbounded-net-controller/detail_responses.go new file mode 100644 index 000000000..9b4e3a8a7 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_responses.go @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// Fail records a correlated collection failure without deleting an older, +// still-valid snapshot that a viewer may display during a failed refresh. +func (m *nodeDetailRequests) Fail(nodeName, requestID, reason string) error { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + request := m.requests[requestID] + if reason == "" || m.ctx.Err() != nil || m.closed || request == nil || request.nodeName != nodeName { + return errors.New("detail failure does not match an available request") + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + return errors.New("detail request node was deleted or replaced") + } + + if request.state == statusv1alpha1.NodeDetailComplete || + (request.state == statusv1alpha1.NodeDetailUnavailable && request.message == reason) { + return nil + } + + if request.state != statusv1alpha1.NodeDetailPending { + return errors.New("detail request is no longer pending") + } + + request.cancel() + request.state = statusv1alpha1.NodeDetailUnavailable + request.message = reason + request.poll = false + request.wakeAt = time.Now().Add(m.timeout) + delete(m.active, nodeName) + m.notify() + + return nil +} + +func handleNodeDetailResponse(health *healthState, nodeName, requestID string, status *NodeStatusResponse, failure string) NodeStatusPushAck { + ack := NodeStatusPushAck{Status: "error", DetailRequestID: requestID, SummarySupported: true} + + manager := health.getDetailRequests() + if manager == nil || requestID == "" { + ack.Reason = "detail request leader or request identity is unavailable" + return ack + } + + if failure != "" && status != nil { + ack.Reason = "detail response cannot contain both data and a collection error" + return ack + } + + var err error + if failure != "" { + err = manager.Fail(nodeName, requestID, failure) + } else { + err = manager.Complete(nodeName, requestID, status) + } + + if err != nil { + ack.Reason = err.Error() + return ack + } + + ack.Status = "ok" + + return ack +} diff --git a/cmd/unbounded-net-controller/detail_transport_test.go b/cmd/unbounded-net-controller/detail_transport_test.go new file mode 100644 index 000000000..205dddbf6 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_transport_test.go @@ -0,0 +1,342 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "testing/synctest" + "time" + + "github.com/coder/websocket" + "google.golang.org/protobuf/proto" + + statuspkg "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func encodeDetailTransportMessage(t *testing.T, binary bool, requestID string, failure ...string) []byte { + t.Helper() + + message := &statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusSummaryType, NodeName: "node-a", SupportsDetails: true, + Summary: &statusproto.NodeStatusOverview{NodeInfo: &statusproto.NodeInfo{Name: "node-a"}, PeerCount: 1}, + } + jsonMessage := NodeStatusWSMessage{Type: message.Type, NodeName: message.NodeName, SupportsDetails: true} + overview := protoToNodeOverview(message.Summary) + jsonMessage.Summary = &overview + + if requestID != "" { + message.Type = statusv1alpha1.NodeStatusDetailsType + message.Summary = nil + message.DetailRequestId = requestID + message.Status = &statusproto.NodeStatusFull{ + NodeInfo: &statusproto.NodeInfo{Name: "node-a"}, + Peers: []*statusproto.PeerStatus{{Name: "peer"}}, + } + full := protoToNodeStatus(message.Status) + jsonMessage.Type = message.Type + jsonMessage.Summary = nil + jsonMessage.Status = &full + jsonMessage.DetailRequestID = requestID + } + + if len(failure) > 0 { + message.DetailError = failure[0] + message.Status = nil + jsonMessage.DetailError = failure[0] + jsonMessage.Status = nil + } + + var ( + data []byte + err error + ) + if binary { + data, err = proto.Marshal(message) + } else { + data, err = json.Marshal(jsonMessage) + } + + if err != nil { + t.Fatal(err) + } + + return data +} + +func decodeDetailTransportAck(t *testing.T, binary, ws bool, data []byte) *NodeStatusPushAck { + t.Helper() + + if binary { + var ack statusproto.NodeStatusAck + if err := proto.Unmarshal(data, &ack); err != nil { + t.Fatal(err) + } + + return statuspkg.NodeStatusAckFromProto(&ack) + } + + var ack NodeStatusPushAck + if ws { + var envelope struct { + Data NodeStatusPushAck `json:"data"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + t.Fatal(err) + } + + ack = envelope.Data + } else if err := json.Unmarshal(data, &ack); err != nil { + t.Fatal(err) + } + + return &ack +} + +func awaitDetailTransport(t *testing.T, ready func() bool) { + t.Helper() + + timeout := time.NewTimer(3 * time.Second) + defer timeout.Stop() + + tick := time.NewTicker(time.Millisecond) + defer tick.Stop() + + for !ready() { + select { + case <-timeout.C: + t.Fatal("detail transport did not become ready") + case <-tick.C: + } + } +} + +func TestDetailWebSocketCommandAndResponse(t *testing.T) { + for _, binary := range []bool{false, true} { + t.Run(map[bool]string{false: "json", true: "protobuf"}[binary], func(t *testing.T) { + health := newJSONIdentityHealth() + health.registerAggregatedAPIServer = false + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: health.dispatchNodeDetail, + Pull: func(context.Context, string) (*NodeStatusResponse, error) { + t.Error("active WebSocket request unexpectedly used HTTP") + return nil, nil + }, + }) + health.detailRequests = manager + issuer := testTokenIssuer(t) + mux := http.NewServeMux() + registerPushHandlers(mux, health, nil, make(chan struct{}, 1), issuer) + + server := httptest.NewServer(mux) + defer server.Close() + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + conn, _, err := websocket.Dial(ctx, server.URL+"/status/nodews", &websocket.DialOptions{ + HTTPHeader: http.Header{"Authorization": {"Bearer " + testNodeToken(t, issuer)}}, + }) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + + frameType := websocket.MessageText + if binary { + frameType = websocket.MessageBinary + } + + send := func(id string) { + t.Helper() + + if err := conn.Write(ctx, frameType, encodeDetailTransportMessage(t, binary, id)); err != nil { + t.Fatal(err) + } + } + read := func() *NodeStatusPushAck { + t.Helper() + + _, data, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + + return decodeDetailTransportAck(t, binary, true, data) + } + + send("") + + publication := read() + + awaitDetailTransport(t, func() bool { + health.nodeWSMu.Lock() + defer health.nodeWSMu.Unlock() + + return health.nodeWSRegistry["node-a"] != nil && health.nodeWSRegistry["node-a"].send != nil + }) + + request := manager.Request("node-a", true) + + command := read() + if command.IsPublicationAck() || command.DetailRequest == nil || + command.DetailRequest.RequestID != request.RequestID || !command.SummarySupported { + t.Fatalf("wire command was not distinct from an ordinary ACK: %+v", command) + } + + send(request.RequestID) + + ack := read() + if ack.Status != "ok" || ack.DetailRequestID != request.RequestID || ack.Revision != 0 || ack.IsPublicationAck() { + t.Fatalf("invalid correlated ACK: %+v", ack) + } + + result := manager.Result("node-a", request.RequestID) + if result.Details == nil || len(result.Details.Status.Peers) != 1 { + t.Fatal("one-shot details did not complete the request") + } + + cached, _ := health.statusCache.Get("node-a") + if cached.Revision != publication.Revision || cached.Status.Peers != nil { + t.Fatal("one-shot reply became the routine delta base") + } + + expires := result.Details.ExpiresAt + + send(request.RequestID) + + if duplicate := read(); duplicate.Status != "ok" { + t.Fatal("duplicate detail reply was not idempotent") + } + + if !manager.Result("node-a", request.RequestID).Details.ExpiresAt.Equal(expires) { + t.Fatal("duplicate detail reply renewed TTL") + } + + refresh := manager.Request("node-a", true) + if command := read(); command.DetailRequest == nil || command.DetailRequest.RequestID != refresh.RequestID { + t.Fatal("refresh command was not delivered") + } + + for range 2 { + if err := conn.Write(ctx, frameType, encodeDetailTransportMessage(t, binary, refresh.RequestID, "collection failed")); err != nil { + t.Fatal(err) + } + + if receipt := read(); receipt.Status != "ok" || receipt.DetailRequestID != refresh.RequestID { + t.Fatal("collection error receipt was not idempotently acknowledged") + } + } + + failed := manager.Result("node-a", refresh.RequestID) + if failed.State != statusv1alpha1.NodeDetailUnavailable || failed.Error != "collection failed" || failed.Details != nil { + t.Fatal("collection error did not become an explicit failed request") + } + }) + } +} + +func TestDetailHTTPPollingCommandAndResponse(t *testing.T) { + for _, binary := range []bool{false, true} { + t.Run(map[bool]string{false: "json", true: "protobuf"}[binary], func(t *testing.T) { + health := newJSONIdentityHealth() + health.registerAggregatedAPIServer = false + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + health.detailRequests = manager + issuer := testTokenIssuer(t) + token := testNodeToken(t, issuer) + mux := http.NewServeMux() + registerPushHandlers(mux, health, nil, make(chan struct{}, 1), issuer) + + request := manager.Request("node-a", true) + + awaitDetailTransport(t, func() bool { _, ok := manager.Pending("node-a"); return ok }) + + post := func(id string, failure ...string) *NodeStatusPushAck { + t.Helper() + r := httptest.NewRequest(http.MethodPost, "/status/push", bytes.NewReader(encodeDetailTransportMessage(t, binary, id, failure...))) + r.Header.Set("Authorization", "Bearer "+token) + + if binary { + r.Header.Set("Content-Type", "application/x-protobuf") + } + + w := httptest.NewRecorder() + mux.ServeHTTP(w, r) + + if w.Code != http.StatusOK { + t.Fatalf("POST failed: %d %s", w.Code, w.Body.String()) + } + + return decodeDetailTransportAck(t, binary, false, w.Body.Bytes()) + } + + publication := post("") + if !publication.SummarySupported || !publication.IsPublicationAck() || + publication.DetailRequest == nil || publication.DetailRequest.RequestID != request.RequestID { + t.Fatalf("POST ACK lost capabilities or polling command: %+v", publication) + } + + detailAck := post(request.RequestID) + if detailAck.Status != "ok" || detailAck.DetailRequestID != request.RequestID || detailAck.IsPublicationAck() || detailAck.DetailRequest != nil { + t.Fatalf("POST detail ACK corrupted publication/polling state: %+v", detailAck) + } + + cached, _ := health.statusCache.Get("node-a") + if cached.Revision != publication.Revision || manager.Result("node-a", request.RequestID).Details == nil { + t.Fatal("POST detail response changed routine state or failed to complete") + } + + refresh := manager.Request("node-a", true) + for range 2 { + receipt := post(refresh.RequestID, "response exceeds the transport frame limit") + if receipt.Status != "ok" || receipt.DetailRequestID != refresh.RequestID { + t.Fatal("POST collection failure receipt was not acknowledged") + } + } + + if failed := manager.Result("node-a", refresh.RequestID); failed.State != statusv1alpha1.NodeDetailUnavailable || failed.Error == "" { + t.Fatal("POST collection failure did not terminate the request") + } + }) + } +} + +func TestDetailFailureKeepsPreviousValidSnapshot(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + + first := manager.Request("node", true) + if err := manager.Complete("node", first.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + before, _ := manager.cache.Get("node") + + refresh := manager.Request("node", true) + if err := manager.Fail("other", refresh.RequestID, "bad"); err == nil { + t.Fatal("failure for a different node was accepted") + } + + if err := manager.Fail("node", refresh.RequestID, "response exceeds the transport frame limit"); err != nil { + t.Fatal(err) + } + + result := manager.Result("node", refresh.RequestID) + if result.State != statusv1alpha1.NodeDetailUnavailable || result.Error == "" || result.Details != nil { + t.Fatal("collection failure was not explicit") + } + + after, ok := manager.cache.Get("node") + if !ok || after.Status != before.Status || !after.ExpiresAt.Equal(before.ExpiresAt) { + t.Fatal("failed refresh destroyed or renewed the old valid snapshot") + } + }) +} diff --git a/cmd/unbounded-net-controller/health_state.go b/cmd/unbounded-net-controller/health_state.go index fcc3eadb6..9b20ca3ed 100644 --- a/cmd/unbounded-net-controller/health_state.go +++ b/cmd/unbounded-net-controller/health_state.go @@ -63,6 +63,11 @@ type healthState struct { nodeTokenVerifier serviceAccountTokenVerifier nodeAuthReady func() bool // Required only by the startup-selected local OIDC verifier. + detailMu sync.Mutex + detailRequests *nodeDetailRequests + statusDetailCacheTTL time.Duration + statusDetailRequestTimeout time.Duration + // Pull fallback toggle (controlled via dashboard WS message; default: disabled). pullEnabled atomic.Bool // registerAggregatedAPIServer controls serving aggregated API status push endpoints. @@ -80,11 +85,11 @@ type healthState struct { // kubeProxyMonitor checks the local kube-proxy health endpoint. kubeProxyMonitor *kubeProxyMonitor - // nodeWSRegistry tracks the active WS cancel function per node name. + // nodeWSRegistry tracks the active authenticated connection per node name. // When a node reconnects, the previous connection is canceled to avoid // duplicate connections consuming resources. nodeWSMu sync.Mutex - nodeWSRegistry map[string]context.CancelFunc + nodeWSRegistry map[string]*nodeWSConnection } const defaultMaxPullConcurrency = 20 @@ -92,45 +97,44 @@ const defaultMaxPullConcurrency = 20 // registerNodeWS registers a WS connection for a node. If an existing // connection is registered for the same node, its context is canceled // to force it to close (preventing duplicate connections). -func (h *healthState) registerNodeWS(nodeName string, cancel context.CancelFunc) { +func (h *healthState) registerNodeWS(nodeName string, cancel context.CancelFunc) *nodeWSConnection { if nodeName == "" { - return + return nil } h.nodeWSMu.Lock() defer h.nodeWSMu.Unlock() if h.nodeWSRegistry == nil { - h.nodeWSRegistry = make(map[string]context.CancelFunc) + h.nodeWSRegistry = make(map[string]*nodeWSConnection) } if prev, ok := h.nodeWSRegistry[nodeName]; ok { - prev() // cancel the old connection + prev.cancel() } - h.nodeWSRegistry[nodeName] = cancel + connection := &nodeWSConnection{cancel: cancel} + h.nodeWSRegistry[nodeName] = connection + + return connection } -// unregisterNodeWS removes a node's WS registration. Only removes if the -// cancel function matches (to avoid unregistering a newer connection). -func (h *healthState) unregisterNodeWS(nodeName string, cancel context.CancelFunc) { - if nodeName == "" { +// unregisterNodeWS cannot remove a newer connection with the same node identity. +func (h *healthState) unregisterNodeWS(nodeName string, connection *nodeWSConnection) { + if connection == nil { return } h.nodeWSMu.Lock() - defer h.nodeWSMu.Unlock() - if h.nodeWSRegistry == nil { - return + removed := h.nodeWSRegistry[nodeName] == connection + if removed { + delete(h.nodeWSRegistry, nodeName) } - // Only remove if it's still our registration (not replaced by a newer connection) - if existing, ok := h.nodeWSRegistry[nodeName]; ok { - // Compare by pointer identity -- Go func values aren't comparable, - // but context.CancelFunc from the same WithCancel call is the same pointer. - if fmt.Sprintf("%p", existing) == fmt.Sprintf("%p", cancel) { - delete(h.nodeWSRegistry, nodeName) - } + h.nodeWSMu.Unlock() + + if removed { + h.retryNodeDetails(nodeName) } } @@ -166,6 +170,10 @@ func (h *healthState) setLeader(leader bool) { h.controllerReady.Store(false) } + if !leader { + h.stopDetailRequests() + } + if leader { leaderIsLeader.Set(1) klog.Info("Health: marked as leader") diff --git a/cmd/unbounded-net-controller/main.go b/cmd/unbounded-net-controller/main.go index 67b04c973..f88cc4804 100644 --- a/cmd/unbounded-net-controller/main.go +++ b/cmd/unbounded-net-controller/main.go @@ -79,6 +79,8 @@ func main() { RequireDashboardAuth: true, StatusWSKeepaliveInterval: 10 * time.Second, StatusWSKeepaliveFailureCount: 2, + StatusDetailCacheTTL: config.DefaultStatusDetailCacheTTL, + StatusDetailRequestTimeout: config.DefaultStatusDetailRequestTimeout, ManagedKubeProxyEnabled: true, NodeTokenLifetime: 4 * time.Hour, ViewerTokenLifetime: 30 * time.Minute, @@ -127,6 +129,8 @@ on site configuration, and maintain SiteNodeSlice and GatewayPool status.`, flags.IntVar(&cfg.HealthPort, "health-port", 9999, "Port for health check HTTP server (0 to disable)") flags.IntVar(&cfg.NodeAgentHealthPort, "node-agent-health-port", 9998, "Port where node agents serve their health/status endpoints") flags.DurationVar(&cfg.StatusStaleThreshold, "status-stale-threshold", 90*time.Second, "Duration after which a node's pushed status is considered stale") + flags.DurationVar(&cfg.StatusDetailCacheTTL, "status-detail-cache-ttl", config.DefaultStatusDetailCacheTTL, "Lifetime of received node details (positive duration)") + flags.DurationVar(&cfg.StatusDetailRequestTimeout, "status-detail-request-timeout", config.DefaultStatusDetailRequestTimeout, "End-to-end node detail request timeout (positive duration)") flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "Interval between websocket keepalive pings on controller node status streams (0 to disable)") flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "Sequential websocket keepalive ping failures before closing node status websocket") flags.BoolVar(&cfg.RegisterAggregatedAPIServer, "register-aggregated-apiserver", true, "Serve node status push endpoints via aggregated API server paths") @@ -164,6 +168,24 @@ func applyControllerRuntimeConfig(cmd *cobra.Command, cfg *config.Config, config flags := cmd.Flags() + if !flags.Changed("status-detail-cache-ttl") && runtimeCfg.Controller.StatusDetailCacheTTL != "" { + d, parseErr := config.ParsePositiveDurationField(runtimeCfg.Controller.StatusDetailCacheTTL, "controller.statusDetailCacheTTL") + if parseErr != nil { + return parseErr + } + + cfg.StatusDetailCacheTTL = d + } + + if !flags.Changed("status-detail-request-timeout") && runtimeCfg.Controller.StatusDetailRequestTimeout != "" { + d, parseErr := config.ParsePositiveDurationField(runtimeCfg.Controller.StatusDetailRequestTimeout, "controller.statusDetailRequestTimeout") + if parseErr != nil { + return parseErr + } + + cfg.StatusDetailRequestTimeout = d + } + if !flags.Changed("informer-resync-period") { if d, parseErr := config.ParseDurationField(runtimeCfg.Controller.InformerResyncPeriod, "controller.informerResyncPeriod"); parseErr != nil { return parseErr @@ -321,6 +343,8 @@ General Flags: --managed-kube-proxy Create kube-proxy DaemonSets for unbounded-managed site nodes not covered by provider kube-proxy (default true) --managed-kube-proxy-image string kube-proxy image for managed site DaemonSets --status-stale-threshold duration Duration after which a node's pushed status is considered stale (default 90s) + --status-detail-cache-ttl duration Lifetime of received node details; preparatory (default 5m0s) + --status-detail-request-timeout duration End-to-end node detail request timeout; preparatory (default 2m0s) --status-ws-keepalive-interval duration Interval between websocket keepalive pings on controller node status streams (0 to disable) (default 10s) --status-ws-keepalive-failure-count int Sequential websocket keepalive ping failures before closing node status websocket (default 2) @@ -481,6 +505,8 @@ func run(cfg *config.Config, forceNotLeader bool) error { nodeName: os.Getenv("NODE_NAME"), statusCache: NewNodeStatusCache(), staleThreshold: cfg.StatusStaleThreshold, + statusDetailCacheTTL: cfg.StatusDetailCacheTTL, + statusDetailRequestTimeout: cfg.StatusDetailRequestTimeout, tokenAuth: newTokenAuthenticator(nodeTokenVerifier, []string{fmt.Sprintf("%s:unbounded-net-node", controllerNamespace)}), nodeServiceAccount: fmt.Sprintf("%s:unbounded-net-node", controllerNamespace), nodeTokenVerifier: nodeTokenVerifier, @@ -561,6 +587,14 @@ func run(cfg *config.Config, forceNotLeader bool) error { // Set informers in health state for efficient lookups in status endpoints healthState.setInformers(siteCtrl.GetNodeLister(), podLister, siteCtrl.GetSiteInformer(), gatewayPoolInformer, sitePeeringInformer, assignmentInformer, poolPeeringInformer) + detailRequests, err := healthState.startDetailRequests(ctx, informerFactory.Core().V1().Nodes().Informer()) + if err != nil { + klog.Errorf("Failed to start node detail requests: %v", err) + + return + } + defer detailRequests.Close() + healthState.siteController = siteCtrl if healthState.clusterStatusCache != nil { healthState.clusterStatusCache.MarkFullRebuildNeeded() diff --git a/cmd/unbounded-net-controller/main_config_test.go b/cmd/unbounded-net-controller/main_config_test.go index cb184733a..3359cc03e 100644 --- a/cmd/unbounded-net-controller/main_config_test.go +++ b/cmd/unbounded-net-controller/main_config_test.go @@ -21,6 +21,8 @@ func newControllerConfigTestCommand(cfg *config.Config) *cobra.Command { flags.IntVar(&cfg.HealthPort, "health-port", 9999, "") flags.IntVar(&cfg.NodeAgentHealthPort, "node-agent-health-port", 9998, "") flags.DurationVar(&cfg.StatusStaleThreshold, "status-stale-threshold", 40*time.Second, "") + flags.DurationVar(&cfg.StatusDetailCacheTTL, "status-detail-cache-ttl", config.DefaultStatusDetailCacheTTL, "") + flags.DurationVar(&cfg.StatusDetailRequestTimeout, "status-detail-request-timeout", config.DefaultStatusDetailRequestTimeout, "") flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "") flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "") flags.BoolVar(&cfg.RegisterAggregatedAPIServer, "register-aggregated-apiserver", true, "") diff --git a/cmd/unbounded-net-controller/matrix_memory_test.go b/cmd/unbounded-net-controller/matrix_memory_test.go deleted file mode 100644 index e7ed3f048..000000000 --- a/cmd/unbounded-net-controller/matrix_memory_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "slices" - "testing" - "time" -) - -func TestConnectivityMatrixMixedScopesPreservesPeers(t *testing.T) { - nodes := make([]*NodeStatusResponse, 0, 102) - for i := range 101 { - nodes = append(nodes, &NodeStatusResponse{NodeInfo: NodeInfo{ - Name: fmt.Sprintf("node-%d", i), SiteName: "large", - }}) - } - - nodes[0].Peers = []WireGuardPeerStatus{{ - Name: "gateway", PeerType: "gateway", SiteName: "small", - Tunnel: PeerTunnelStatus{LastHandshake: time.Unix(1, 0)}, - }} - nodes[1].Peers = []WireGuardPeerStatus{{Name: "gateway", PeerType: "ignored"}} - nodes = append(nodes, &NodeStatusResponse{ - NodeInfo: NodeInfo{Name: "gateway", SiteName: "small"}, - Peers: []WireGuardPeerStatus{ - {Name: "node-0", PeerType: "site", HealthCheck: &HealthCheckPeerStatus{Status: "up"}}, - {Name: "node-0", PeerType: "ignored", HealthCheck: &HealthCheckPeerStatus{Status: "down"}}, - {Name: "node-2", PeerType: "ignored"}, - {Name: "gateway", PeerType: "ignored", HealthCheck: &HealthCheckPeerStatus{Status: "down"}}, - }, - }) - - before, err := json.Marshal(nodes) - if err != nil { - t.Fatal(err) - } - - matrix := buildConnectivityMatrix(nodes, []GatewayPoolStatus{{Name: "pool", Gateways: []string{"gateway"}}}) - if _, ok := matrix["large"]; ok { - t.Fatal("oversized site produced a matrix") - } - - if matrix["small"] == nil || !slices.Equal(matrix["small"].Nodes, []string{"gateway"}) { - t.Fatalf("small site lost its matrix: %+v", matrix["small"]) - } - - pool := matrix["pool:pool"] - if pool == nil || !slices.Equal(pool.Nodes, []string{"gateway", "node-0"}) { - t.Fatalf("small pool crossing a large site has wrong membership: %+v", pool) - } - - if pool.Results["gateway"]["node-0"] != "up" || pool.Results["node-0"]["gateway"] != "up" { - t.Fatalf("pool connectivity changed: %+v", pool.Results) - } - - after, err := json.Marshal(nodes) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(before, after) { - t.Fatal("matrix construction mutated the shared node snapshots") - } -} - -func TestConnectivityMatrixDoesNotCopyPeerSlices(t *testing.T) { - nodes := matrixBenchmarkNodes(200) - - peers := matrixBenchmarkNodes(2000)[0].Peers - for _, node := range nodes { - node.Peers = peers - } - - var matrix map[string]*SiteMatrix - - result := testing.Benchmark(func(b *testing.B) { - for b.Loop() { - matrix = buildConnectivityMatrix(nodes, nil) - } - }) - - if matrix != nil { - t.Fatal("oversized site produced a matrix") - } - // Allow map bookkeeping, but not storage proportional to every peer. - if allocated := result.AllocedBytesPerOp(); allocated > 512*1024 { - t.Fatalf("matrix copied peer data: %d bytes per call", allocated) - } -} - -func TestConnectivityMatrixSizeBoundary(t *testing.T) { - for _, count := range []int{0, 100, 101} { - t.Run(fmt.Sprintf("nodes-%d", count), func(t *testing.T) { - matrix := buildConnectivityMatrix(matrixBenchmarkNodes(count), nil) - if count != 100 { - if matrix != nil { - t.Fatal("empty or oversized site produced a matrix") - } - - return - } - - if matrix["site-a"] == nil || len(matrix["site-a"].Nodes) != count { - t.Fatal("site at the size limit lost its matrix") - } - }) - } -} diff --git a/cmd/unbounded-net-controller/memory_bench_test.go b/cmd/unbounded-net-controller/memory_bench_test.go index ca548d0a6..1ccc15873 100644 --- a/cmd/unbounded-net-controller/memory_bench_test.go +++ b/cmd/unbounded-net-controller/memory_bench_test.go @@ -68,37 +68,3 @@ func BenchmarkProtoWSStatusFrame(b *testing.B) { }) } } - -func matrixBenchmarkNodes(count int) []*NodeStatusResponse { - peers := make([]WireGuardPeerStatus, count) - - nodes := make([]*NodeStatusResponse, count) - for i := range count { - name := fmt.Sprintf("node-%d", i) - peers[i] = WireGuardPeerStatus{Name: name, PeerType: "site", SiteName: "site-a"} - nodes[i] = &NodeStatusResponse{ - NodeInfo: NodeInfo{Name: name, SiteName: "site-a"}, - Peers: peers, - } - } - - return nodes -} - -func BenchmarkBuildConnectivityMatrix(b *testing.B) { - for _, count := range []int{100, 101, 2000} { - b.Run(fmt.Sprintf("nodes-%d", count), func(b *testing.B) { - nodes := matrixBenchmarkNodes(count) - - b.ReportAllocs() - b.ResetTimer() - - for b.Loop() { - matrix := buildConnectivityMatrix(nodes, nil) - if count > 100 && matrix != nil { - b.Fatal("oversized site produced a matrix") - } - } - }) - } -} diff --git a/cmd/unbounded-net-controller/node_overview.go b/cmd/unbounded-net-controller/node_overview.go new file mode 100644 index 000000000..8d9864d4e --- /dev/null +++ b/cmd/unbounded-net-controller/node_overview.go @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "time" + + statuspkg "github.com/Azure/unbounded/internal/net/status" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// StoreOverview replaces routine wire state without retaining diagnostic arrays. +func (c *NodeStatusCache) StoreOverview(nodeName string, overview statusv1alpha1.NodeStatusOverview, source string) (uint64, error) { + if nodeName == "" || (overview.NodeInfo.Name != "" && overview.NodeInfo.Name != nodeName) { + return 0, fmt.Errorf("summary identity does not match node %q", nodeName) + } + + if overview.PeerCount < 0 || overview.HealthyPeers < 0 || + overview.HealthyPeers > overview.PeerCount || overview.RouteCount < 0 || + overview.RouteMismatchCount < 0 || overview.UnhealthyPeerLinks < 0 || + (overview.RouteMismatchCount > 0 && !overview.RouteMismatch) { + return 0, fmt.Errorf("summary contains invalid observed counts") + } + + overview.NodeInfo.Name = nodeName + + if source == "" { + source = "push" + } + + overview.StatusSource = source + metadata := statuspkg.OverviewMetadata(overview) + + c.mu.Lock() + + revision := uint64(1) + if previous := c.entries[nodeName]; previous != nil { + revision = previous.Revision + 1 + } + + c.entries[nodeName] = &CachedNodeStatus{ + Status: &metadata, Overview: &overview, Source: source, + Revision: revision, ReceivedAt: time.Now(), + } + fn := c.onOverviewChange + c.mu.Unlock() + + if fn != nil { + fn(nodeName, overview) + } + + return revision, nil +} + +// SetOnOverviewChange registers the summary-only cache mutation callback. +func (c *NodeStatusCache) SetOnOverviewChange(fn func(string, statusv1alpha1.NodeStatusOverview)) { + c.mu.Lock() + defer c.mu.Unlock() + + c.onOverviewChange = fn +} diff --git a/cmd/unbounded-net-controller/node_overview_test.go b/cmd/unbounded-net-controller/node_overview_test.go new file mode 100644 index 000000000..6c7176d91 --- /dev/null +++ b/cmd/unbounded-net-controller/node_overview_test.go @@ -0,0 +1,201 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestNodeOverviewCacheReplacesOnlyRoutineState(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node", NodeStatusResponse{Peers: []WireGuardPeerStatus{{Name: "peer"}}}, "push") + + var notified statusv1alpha1.NodeStatusOverview + + cache.SetOnOverviewChange(func(name string, overview statusv1alpha1.NodeStatusOverview) { + if name != "node" || cache.Len() != 1 { + t.Error("notification has wrong identity or ran under the cache lock") + } + + notified = overview + }) + overview := statusv1alpha1.NodeStatusOverview{ + PeerCount: 20, HealthyPeers: 17, RouteCount: 30, RouteMismatch: true, + NodeErrors: []NodeError{{Type: "cni", Message: "bootstrap blocked"}}, + } + + revision, err := cache.StoreOverview("node", overview, "ws") + if err != nil || revision != 2 { + t.Fatalf("store: revision=%d error=%v", revision, err) + } + + cached, ok := cache.Get("node") + if !ok || cached.Overview == nil || cached.Overview.PeerCount != 20 || cached.peerIdentity != nil { + t.Fatalf("unexpected overview wire state: %+v", cached) + } + + if cached.Status.Peers != nil || cached.Status.RoutingTable.Routes != nil || cached.Status.BpfEntries != nil { + t.Fatal("summary retained diagnostic arrays") + } + + cached.Overview.PeerCount = 99 + + unchanged, _ := cache.Get("node") + if unchanged.Overview.PeerCount != 20 { + t.Fatal("changing the returned overview mutated the cache") + } + + if notified.NodeInfo.Name != "node" || notified.StatusSource != "ws" || len(cached.Status.NodeErrors) != 1 { + t.Fatal("notification or metadata lost identity, source, or errors") + } + + rev, resync, err := cache.ApplyDelta("node", revision, map[string]json.RawMessage{}, "push") + if err != nil || !resync || rev != revision { + t.Fatalf("legacy delta must not apply to a summary: %d %v %v", rev, resync, err) + } + + snapshot := cache.GetAll() + cache.UpdateSource("node", "apiserver-ws") + + if snapshot["node"].Source != "ws" || notified.StatusSource != "apiserver-ws" { + t.Fatal("source change mutated an older snapshot or lost its notification") + } + + if next := cache.StoreFull("node", NodeStatusResponse{Peers: []WireGuardPeerStatus{{Name: "legacy"}}}, "push"); next != 3 { + t.Fatalf("legacy full resync revision=%d", next) + } + + legacy, _ := cache.Get("node") + if legacy.Overview != nil || len(legacy.Status.Peers) != 1 { + t.Fatal("explicit legacy full resync did not replace summary state") + } +} + +func TestNodeOverviewCacheRejectsInvalidFacts(t *testing.T) { + for _, overview := range []statusv1alpha1.NodeStatusOverview{ + {NodeInfo: NodeInfo{Name: "different-node"}}, + {PeerCount: -1}, + {HealthyPeers: -1}, + {PeerCount: 1, HealthyPeers: 2}, + {RouteCount: -1}, + {RouteMismatchCount: -1}, + {UnhealthyPeerLinks: -1}, + {RouteMismatchCount: 1}, + } { + cache := NewNodeStatusCache() + if _, err := cache.StoreOverview("node", overview, "ws"); err == nil || cache.Len() != 0 { + t.Fatalf("invalid summary accepted: %+v", overview) + } + } + + if _, err := NewNodeStatusCache().StoreOverview("", statusv1alpha1.NodeStatusOverview{}, ""); err == nil { + t.Fatal("empty node identity accepted") + } +} + +func TestClusterOverviewPreservesCountsAndEnrichment(t *testing.T) { + c := NewClusterStatusCache(&healthState{}) + c.status = &ClusterStatusResponse{ + Nodes: []*NodeStatusResponse{{NodeInfo: NodeInfo{Name: "node", K8sReady: "Ready", ProviderID: "provider"}}}, + } + c.nodeIndex["node"] = 0 + overview := statusv1alpha1.NodeStatusOverview{ + NodeInfo: NodeInfo{Name: "node", SiteName: "site", WireGuard: &WireGuardStatusInfo{Interface: "wg0"}}, + StatusSource: "ws", PeerCount: 20, HealthyPeers: 17, RouteCount: 30, RouteMismatch: true, + } + c.PatchOverview("node", overview) + snapshot := c.Get() + + row := buildClusterSummary(snapshot).NodeSummaries[0] + if row.PeerCount != 20 || row.HealthyPeers != 17 || row.RouteCount != 30 || !row.RouteMismatch || + row.K8sReady != "Ready" || row.CniStatus != "Route mismatch" || row.SiteName != "site" { + t.Fatalf("summary lost observed facts or enriched fields: %+v", row) + } + + if snapshot.Nodes[0].NodeInfo.ProviderID != "provider" { + t.Fatal("controller enrichment was lost") + } + + problems := collectClusterProblems(snapshot) + if len(problems) != 1 || len(problems[0].Errors) != 2 { + t.Fatalf("summary health/mismatch problems were hidden: %+v", problems) + } + + overview.PeerCount = 25 + overview.NodeErrors = []NodeError{{Type: "cni", Message: "blocked"}} + c.PatchOverview("node", overview) + + if buildClusterSummary(snapshot).NodeSummaries[0] != row { + t.Fatal("patching changed a previously returned snapshot") + } + + nextRow := buildClusterSummary(c.Get()).NodeSummaries[0] + if nextRow.PeerCount != 25 || nextRow.FirstError != "blocked" || nextRow.CniTone != "danger" { + t.Fatalf("summary update lost errors or counts: %+v", nextRow) + } + + c.PatchNode("node", NodeStatusResponse{NodeInfo: overview.NodeInfo, Peers: []WireGuardPeerStatus{{}}}) + + if legacy := buildClusterSummary(c.Get()).NodeSummaries[0]; legacy.PeerCount != 1 { + t.Fatal("legacy update retained stale explicit summary counts") + } +} + +func TestClusterOverviewWireIgnoresDiagnosticArrays(t *testing.T) { + node := &NodeStatusResponse{NodeInfo: NodeInfo{Name: "node"}} + status := &ClusterStatusResponse{ + Nodes: []*NodeStatusResponse{node}, + NodeOverviews: map[string]*statusv1alpha1.NodeStatusOverview{ + "node": {PeerCount: 5, HealthyPeers: 4, RouteCount: 9}, + }, + } + + before, err := json.Marshal(buildClusterSummary(status)) + if err != nil { + t.Fatal(err) + } + + node.Peers = make([]WireGuardPeerStatus, 10000) + node.RoutingTable.Routes = make([]RouteEntry, 10000) + node.BpfEntries = make([]BpfEntry, 10000) + + after, err := json.Marshal(buildClusterSummary(status)) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(before, after) { + t.Fatal("overview wire size or facts depend on diagnostic arrays") + } + + for _, field := range []string{`"peers":`, `"routingTable":`, `"bpfEntries":`, `"NodeOverviews":`} { + if strings.Contains(string(after), field) { + t.Fatalf("overview exposed %s", field) + } + } +} + +func TestClusterOverviewConcurrentSnapshots(t *testing.T) { + c := NewClusterStatusCache(&healthState{}) + c.status = &ClusterStatusResponse{} + + var wg sync.WaitGroup + for range 4 { + wg.Go(func() { + for range 100 { + c.PatchOverview("node", statusv1alpha1.NodeStatusOverview{NodeInfo: NodeInfo{Name: "node"}, Timestamp: time.Now()}) + buildClusterSummary(c.Get()) + } + }) + } + + wg.Wait() +} diff --git a/cmd/unbounded-net-controller/node_status.go b/cmd/unbounded-net-controller/node_status.go index 7fb9b09b1..eca669423 100644 --- a/cmd/unbounded-net-controller/node_status.go +++ b/cmd/unbounded-net-controller/node_status.go @@ -12,6 +12,7 @@ import ( "time" statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) // CachedNodeStatus stores a node's pushed status with timestamp and revision. @@ -20,15 +21,17 @@ type CachedNodeStatus struct { ReceivedAt time.Time Source string Revision uint64 + Overview *statusv1alpha1.NodeStatusOverview peerIdentity *peerIdentityDigest } // NodeStatusCache is a thread-safe cache of node status data pushed from node agents. type NodeStatusCache struct { - mu sync.RWMutex - entries map[string]*CachedNodeStatus - onChange func(nodeName string, status *NodeStatusResponse) + mu sync.RWMutex + entries map[string]*CachedNodeStatus + onChange func(nodeName string, status *NodeStatusResponse) + onOverviewChange func(nodeName string, overview statusv1alpha1.NodeStatusOverview) } // NewNodeStatusCache creates an empty NodeStatusCache. @@ -230,7 +233,7 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, return 0, true, nil } - if (pd.peerMeasurements != nil && baseRevision == 0) || (baseRevision != 0 && entry.Revision != baseRevision) { + if entry.Overview != nil || (pd.peerMeasurements != nil && baseRevision == 0) || (baseRevision != 0 && entry.Revision != baseRevision) { rev := entry.Revision c.mu.RUnlock() @@ -399,6 +402,11 @@ func (c *NodeStatusCache) Get(nodeName string) (*CachedNodeStatus, bool) { copy := *entry copy.Status = &statusCopy + if entry.Overview != nil { + overviewCopy := *entry.Overview + copy.Overview = &overviewCopy + } + return ©, true } @@ -445,12 +453,19 @@ func (c *NodeStatusCache) UpdateSource(nodeName, source string) bool { return true } - entry.Source = source + updated := *entry + updated.Source = source + c.entries[nodeName] = &updated fn := c.onChange + overviewFn := c.onOverviewChange statusCopy := entry.Status c.mu.Unlock() - if fn != nil { + if entry.Overview != nil && overviewFn != nil { + overview := *entry.Overview + overview.StatusSource = source + overviewFn(nodeName, overview) + } else if fn != nil { fn(nodeName, statusCopy) } diff --git a/cmd/unbounded-net-controller/overview_diagnostics_test.go b/cmd/unbounded-net-controller/overview_diagnostics_test.go new file mode 100644 index 000000000..7cebaa0fe --- /dev/null +++ b/cmd/unbounded-net-controller/overview_diagnostics_test.go @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "slices" + "testing" + "time" + + statuspkg "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestOverviewDiagnosticMessagesMatchFullProblems(t *testing.T) { + now := time.Now() + yes := true + node := &NodeStatusResponse{ + NodeInfo: NodeInfo{Name: "node", K8sReady: "Ready", ProviderID: "azure://vm", WireGuard: &WireGuardStatusInfo{Interface: "wg0"}}, + Peers: []WireGuardPeerStatus{ + {Name: "same", Tunnel: PeerTunnelStatus{Protocol: "IPIP"}, HealthCheck: &HealthCheckPeerStatus{Status: " UP "}}, + {Name: "same", Tunnel: PeerTunnelStatus{LastHandshake: now}, HealthCheck: &HealthCheckPeerStatus{Status: " down "}}, + {Name: "same", HealthCheck: &HealthCheckPeerStatus{Enabled: true, Status: "DOWN"}}, + }, + RoutingTable: RoutingTableInfo{Routes: []RouteEntry{{ + NextHops: []NextHop{{Expected: &yes}, {Present: &yes}, {Expected: &yes}}, + }}}, + } + + fullProblems := collectClusterProblems(&ClusterStatusResponse{Nodes: []*NodeStatusResponse{node}}) + if len(fullProblems) != 1 || len(fullProblems[0].Errors) != 3 { + t.Fatalf("legacy diagnostic fixture changed: %+v", fullProblems) + } + + overview := statuspkg.OverviewFromStatus(node, now) + overview.NodeInfo.ProviderID = "" + messages := statuspkg.OverviewDiagnosticMessages(overview, node.NodeInfo.ProviderID) + slices.Sort(messages) + + want := slices.Clone(fullProblems[0].Errors) + slices.Sort(want) + + if !slices.Equal(messages, want) || overview.RouteMismatchCount != 3 || overview.UnhealthyPeerLinks != 2 { + t.Fatalf("summary diagnostics=%v, legacy=%v, overview=%+v", messages, want, overview) + } + + if otherCloud := statuspkg.OverviewDiagnosticMessages(overview, "other://vm"); len(otherCloud) != 2 { + t.Fatalf("IPIP warning applied outside Azure: %v", otherCloud) + } + + overview.UsesIPIP = false + if noIPIP := statuspkg.OverviewDiagnosticMessages(overview, node.NodeInfo.ProviderID); len(noIPIP) != 2 { + t.Fatalf("Azure warning applied without IPIP: %v", noIPIP) + } +} + +func TestProtoOverviewPreservesDiagnosticFacts(t *testing.T) { + got := protoToNodeOverview(&statusproto.NodeStatusOverview{ + NodeInfo: &statusproto.NodeInfo{Name: "node", ProviderId: "azure://vm"}, + PeerCount: 4, HealthyPeers: 1, RouteCount: 2, RouteMismatch: true, + RouteMismatchCount: 3, UnhealthyPeerLinks: 2, UsesIpip: true, + }) + if got.RouteMismatchCount != 3 || got.UnhealthyPeerLinks != 2 || !got.UsesIPIP || + got.PeerCount != 4 || got.HealthyPeers != 1 || !got.RouteMismatch || got.NodeInfo.ProviderID != "azure://vm" { + t.Fatalf("overview converter lost diagnostics: %+v", got) + } +} + +func TestViewerSummaryPreservesInterfaceOnlineIndependentOfCNI(t *testing.T) { + for _, native := range []bool{false, true} { + for _, tc := range []struct { + name string + status *WireGuardStatusInfo + online bool + }{ + {"missing", nil, false}, + {"public key without interface", &WireGuardStatusInfo{PublicKey: "key"}, false}, + {"interface with CNI failure", &WireGuardStatusInfo{Interface: "wg0"}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + node := &NodeStatusResponse{ + NodeInfo: NodeInfo{Name: "node", WireGuard: tc.status}, + NodeErrors: []NodeError{{Type: "cni", Message: "blocked"}}, + } + + cluster := &ClusterStatusResponse{Nodes: []*NodeStatusResponse{node}} + if native { + cluster.NodeOverviews = map[string]*statusv1alpha1.NodeStatusOverview{ + "node": {NodeInfo: node.NodeInfo, NodeErrors: node.NodeErrors}, + } + } + + summary := buildClusterSummary(cluster).NodeSummaries[0] + if summary.WireGuardOnline != tc.online || summary.ErrorCount != 1 || + summary.FirstError != "blocked" || summary.CniStatus != "Errors" { + t.Fatalf("interface/CNI facts conflated: %+v", summary) + } + + data, err := json.Marshal(summary) + if err != nil { + t.Fatal(err) + } + + var decoded struct { + Online *bool `json:"wireGuardOnline"` + } + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + + if decoded.Online == nil || *decoded.Online != tc.online { + t.Fatalf("explicit online/offline fact omitted or changed: %s", data) + } + }) + } + } +} diff --git a/cmd/unbounded-net-controller/server.go b/cmd/unbounded-net-controller/server.go index c706ca95e..965d6e39f 100644 --- a/cmd/unbounded-net-controller/server.go +++ b/cmd/unbounded-net-controller/server.go @@ -27,6 +27,7 @@ import ( "github.com/Azure/unbounded/internal/net/html" "github.com/Azure/unbounded/internal/net/metrics" statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" webhookpkg "github.com/Azure/unbounded/internal/net/webhook" ) @@ -58,6 +59,9 @@ type nodeStatusWSIdentity struct { Status *struct { NodeInfo nodeStatusIdentityInfo `json:"nodeInfo"` } `json:"status"` + Summary *struct { + NodeInfo nodeStatusIdentityInfo `json:"nodeInfo"` + } `json:"summary"` Delta map[string]json.RawMessage `json:"delta"` } @@ -76,6 +80,10 @@ func extractNodeNameFromWSMessage(data []byte) (string, error) { nodeNames = append(nodeNames, identity.Status.NodeInfo.Name) } + if identity.Summary != nil { + nodeNames = append(nodeNames, identity.Summary.NodeInfo.Name) + } + // Match ApplyDelta's case-sensitive map lookup, not struct field matching. if raw, ok := identity.Delta["nodeInfo"]; ok { var info nodeStatusIdentityInfo @@ -96,10 +104,10 @@ func rejectDuplicateStatusIdentityFields(data []byte, object string) error { return nil } - fields := []string{"nodeName", "nodeInfo", "status", "delta"} + fields := []string{"nodeName", "nodeInfo", "status", "delta", "summary"} switch object { - case "status", "delta": + case "status", "delta", "summary": fields = []string{"nodeInfo"} case "nodeInfo": fields = []string{"name"} @@ -147,7 +155,7 @@ func rejectDuplicateStatusIdentityFields(data []byte, object string) error { return fmt.Errorf("invalid status identity value: %w", err) } - if matched == "status" || matched == "delta" || matched == "nodeInfo" { + if matched == "status" || matched == "delta" || matched == "nodeInfo" || matched == "summary" { if err := rejectDuplicateStatusIdentityFields(value, matched); err != nil { return err } @@ -233,6 +241,11 @@ func startServer(ctx context.Context, healthPort int, requireDashboardAuth bool, clusterStatusCache.MarkDirty() broadcaster.Notify() }) + health.statusCache.SetOnOverviewChange(func(nodeName string, overview statusv1alpha1.NodeStatusOverview) { + clusterStatusCache.PatchOverview(nodeName, overview) + clusterStatusCache.MarkDirty() + broadcaster.Notify() + }) // Node WebSocket connection semaphore. wsSemaphore := make(chan struct{}, maxConcurrentNodeWS) @@ -383,6 +396,8 @@ func serveStatusJSON(health *healthState, w http.ResponseWriter, r *http.Request } func registerStatusHandlers(mux *http.ServeMux, health *healthState, requireDashboardAuth bool, webhookServer *webhookpkg.Server, dashAuthorizer *dashboardAuthorizer, tokenIssuer *authn.TokenIssuer) { + registerNodeDetailHandlers(mux, health, requireDashboardAuth, webhookServer, dashAuthorizer, tokenIssuer) + mux.HandleFunc("/status/json", func(w http.ResponseWriter, r *http.Request) { if !authorizeDashboardOrAggregated(requireDashboardAuth, tokenIssuer, dashAuthorizer, webhookServer, r) { http.Error(w, "Unauthorized", http.StatusUnauthorized) @@ -548,17 +563,19 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer return } + if ack.IsPublicationAck() && ack.Status == "ok" { + if manager := health.getDetailRequests(); manager != nil { + if command, ok := manager.Pending(authorizedNodeName(r)); ok { + ack.DetailRequest = &command + } + } + } + isProto := isProtobufContentType(r) if isProto { w.Header().Set("Content-Type", "application/x-protobuf") - pbAck := &statusproto.NodeStatusAck{ - Status: ack.Status, - Revision: ack.Revision, - Reason: ack.Reason, - } - - data, marshalErr := proto.Marshal(pbAck) + data, marshalErr := marshalProtoAck("node_status_ack", ack) if marshalErr != nil { klog.V(4).Infof("status push proto ack marshal failed: %v", marshalErr) http.Error(w, "internal error", http.StatusInternalServerError) @@ -646,37 +663,54 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer } }() - send := func(frameType websocket.MessageType, ackMsgType string, ack NodeStatusPushAck) { - if frameType == websocket.MessageBinary { - payload, marshalErr := marshalProtoAck(ackMsgType, ack) - if marshalErr != nil { - klog.V(4).Infof("Node WebSocket proto ack marshal failed (source=%s, node=%s): %v", source, nodeNameForLog(), marshalErr) - return - } + wsCtx, wsCancel := context.WithCancel(r.Context()) + defer wsCancel() - if writeErr := conn.Write(r.Context(), websocket.MessageBinary, payload); writeErr != nil { - klog.V(4).Infof("Node WebSocket ack write failed (source=%s, node=%s): %v", source, nodeNameForLog(), writeErr) - } + var registration *nodeWSConnection + defer func() { health.unregisterNodeWS(lastWSNodeName, registration) }() - return + writeGate := make(chan struct{}, 1) + + sendContext := func(ctx context.Context, frameType websocket.MessageType, ackMsgType string, ack NodeStatusPushAck) error { + select { + case writeGate <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + case <-wsCtx.Done(): + return wsCtx.Err() + } + + defer func() { <-writeGate }() + + var ( + payload []byte + marshalErr error + ) + if frameType == websocket.MessageBinary { + payload, marshalErr = marshalProtoAck(ackMsgType, ack) + } else { + payload, marshalErr = json.Marshal(map[string]interface{}{"type": ackMsgType, "data": ack}) } - payload, marshalErr := json.Marshal(map[string]interface{}{"type": ackMsgType, "data": ack}) if marshalErr != nil { - klog.V(4).Infof("Node WebSocket ack marshal failed (source=%s, node=%s): %v", source, nodeNameForLog(), marshalErr) - return + return marshalErr } - if writeErr := conn.Write(r.Context(), websocket.MessageText, payload); writeErr != nil { - klog.V(4).Infof("Node WebSocket ack write failed (source=%s, node=%s): %v", source, nodeNameForLog(), writeErr) + return conn.Write(ctx, frameType, payload) + } + send := func(frameType websocket.MessageType, ackMsgType string, ack NodeStatusPushAck) { + if err := sendContext(wsCtx, frameType, ackMsgType, ack); err != nil { + klog.V(4).Infof("Node WebSocket ack failed (source=%s): %v", source, err) + wsCancel() } } - - wsCtx, wsCancel := context.WithCancel(r.Context()) - defer wsCancel() - defer func() { - health.unregisterNodeWS(lastWSNodeName, wsCancel) - }() + enableDetails := func(nodeName string, frameType websocket.MessageType) { + health.setNodeWSDetailSender(nodeName, registration, func(ctx context.Context, command statusv1alpha1.DetailRequest) error { + return sendContext(ctx, frameType, "node_status_ack", NodeStatusPushAck{ + Status: statusv1alpha1.DetailRequestStatus, DetailRequest: &command, SummarySupported: true, + }) + }) + } recvCh := make(chan wsFrame) errCh := make(chan error, 1) @@ -817,7 +851,7 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer if nodeName != "" { if lastWSNodeName == "" { // First message identifies the node -- register and evict old connections. - health.registerNodeWS(nodeName, wsCancel) + registration = health.registerNodeWS(nodeName, wsCancel) } lastWSNodeName = nodeName @@ -825,6 +859,10 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer ackType, ack := handleProtoWSMessage(health, decoded, source) send(websocket.MessageBinary, ackType, ack) + + if ack.Status == "ok" && decoded.message.SupportsDetails { + enableDetails(nodeName, websocket.MessageBinary) + } } else { nodeName, identityErr := extractNodeNameFromWSMessage(frame.data) if identityErr != nil || nodeName == "" { @@ -852,7 +890,7 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer if nodeName != "" { if lastWSNodeName == "" { - health.registerNodeWS(nodeName, wsCancel) + registration = health.registerNodeWS(nodeName, wsCancel) } lastWSNodeName = nodeName @@ -860,6 +898,13 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer ackType, ack := handleNodeStatusWSMessageWithSource(health, frame.data, source) send(websocket.MessageText, ackType, ack) + + var capability struct { + SupportsDetails bool `json:"supportsDetails"` + } + if err := json.Unmarshal(frame.data, &capability); err == nil && ack.Status == "ok" && capability.SupportsDetails { + enableDetails(nodeName, websocket.MessageText) + } } case <-keepaliveCh: // Skip ping if we received a message recently @@ -1172,7 +1217,9 @@ func handleStatusPushBody(health *healthState, r *http.Request, bodyBytes []byte return handleStatusPushRequestWithSource(health, bodyBytes, source) } -func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, source string) (NodeStatusPushAck, int, error) { +func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, source string) (ack NodeStatusPushAck, code int, err error) { + defer func() { ack.SummarySupported = true }() + if _, err := extractNodeNameFromWSMessage(bodyBytes); err != nil { return NodeStatusPushAck{}, http.StatusBadRequest, err } @@ -1182,7 +1229,35 @@ func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, so return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("invalid request body: %v", err) } - ack := NodeStatusPushAck{Status: "ok"} + ack = NodeStatusPushAck{Status: "ok"} + + if envelope.Type == statusv1alpha1.NodeStatusSummaryType { + if envelope.Mode != "" && envelope.Mode != "summary" { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("conflicting status mode and type") + } + + envelope.Mode = "summary" + } + + if envelope.Type == statusv1alpha1.NodeStatusDetailsType { + if envelope.Mode != "" && envelope.Mode != "details" { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("conflicting status mode and type") + } + + envelope.Mode = "details" + } + + if envelope.DetailError != "" && envelope.Mode != "details" { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("collection error requires a detail response") + } + + if envelope.Summary != nil && envelope.Mode != "summary" { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("overview requires summary mode") + } + + if envelope.Mode == "summary" && envelope.Type != "" && envelope.Type != statusv1alpha1.NodeStatusSummaryType { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("conflicting status mode and type") + } if envelope.Mode == "" { var nodeStatus NodeStatusResponse @@ -1205,11 +1280,32 @@ func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, so nodeName = envelope.Status.NodeInfo.Name } + if envelope.Summary != nil && envelope.Summary.NodeInfo.Name != "" { + nodeName = envelope.Summary.NodeInfo.Name + } + if nodeName == "" { return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("nodeName is required") } switch envelope.Mode { + case "details": + if envelope.Delta != nil { + return NodeStatusPushAck{Status: "error", DetailRequestID: envelope.DetailRequestID, Reason: "details cannot include a delta"}, http.StatusOK, nil + } + + return handleNodeDetailResponse(health, nodeName, envelope.DetailRequestID, envelope.Status, envelope.DetailError), http.StatusOK, nil + case "summary": + if envelope.Summary == nil || envelope.Status != nil || envelope.Delta != nil || envelope.DetailRequestID != "" { + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("summary must contain only overview data") + } + + ack.Revision, err = health.statusCache.StoreOverview(nodeName, *envelope.Summary, source) + if err != nil { + return NodeStatusPushAck{}, http.StatusBadRequest, err + } + + return ack, http.StatusOK, nil case "full": if envelope.Status == nil { return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("status is required for full mode") @@ -1242,7 +1338,7 @@ func handleStatusPushRequestWithSource(health *healthState, bodyBytes []byte, so return ack, http.StatusOK, nil default: - return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("mode must be full or delta") + return NodeStatusPushAck{}, http.StatusBadRequest, fmt.Errorf("unsupported status mode %q", envelope.Mode) } } @@ -1250,7 +1346,9 @@ func handleNodeStatusWSMessage(health *healthState, data []byte) (string, NodeSt return handleNodeStatusWSMessageWithSource(health, data, "ws") } -func handleNodeStatusWSMessageWithSource(health *healthState, data []byte, source string) (string, NodeStatusPushAck) { +func handleNodeStatusWSMessageWithSource(health *healthState, data []byte, source string) (ackType string, ack NodeStatusPushAck) { + defer func() { ack.SummarySupported = true }() + if _, err := extractNodeNameFromWSMessage(data); err != nil { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} } @@ -1260,16 +1358,45 @@ func handleNodeStatusWSMessageWithSource(health *healthState, data []byte, sourc return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "invalid message"} } + if message.Summary != nil && message.Type != statusv1alpha1.NodeStatusSummaryType { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "overview requires summary message type"} + } + + if message.DetailError != "" && message.Type != statusv1alpha1.NodeStatusDetailsType { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "collection error requires a detail response"} + } + nodeName := message.NodeName if message.Status != nil && message.Status.NodeInfo.Name != "" { nodeName = message.Status.NodeInfo.Name } + if message.Summary != nil && message.Summary.NodeInfo.Name != "" { + nodeName = message.Summary.NodeInfo.Name + } + if nodeName == "" { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "nodeName is required"} } switch message.Type { + case statusv1alpha1.NodeStatusDetailsType: + if message.Delta != nil { + return "node_status_ack", NodeStatusPushAck{Status: "error", DetailRequestID: message.DetailRequestID, Reason: "details cannot include a delta"} + } + + return "node_status_ack", handleNodeDetailResponse(health, nodeName, message.DetailRequestID, message.Status, message.DetailError) + case statusv1alpha1.NodeStatusSummaryType: + if message.Summary == nil || message.Status != nil || message.Delta != nil || message.DetailRequestID != "" { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "summary must contain only overview data"} + } + + revision, err := health.statusCache.StoreOverview(nodeName, *message.Summary, source) + if err != nil { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} + } + + return "node_status_ack", NodeStatusPushAck{Status: "ok", Revision: revision} case "node_status_full": if message.Status == nil { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "full message missing status"} diff --git a/cmd/unbounded-net-controller/status_detail_config_test.go b/cmd/unbounded-net-controller/status_detail_config_test.go new file mode 100644 index 000000000..ce5c6d447 --- /dev/null +++ b/cmd/unbounded-net-controller/status_detail_config_test.go @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/Azure/unbounded/internal/net/config" +) + +func TestControllerStatusDetailConfig(t *testing.T) { + for _, tc := range []struct { + name, yaml, flagName, flagValue string + ttl, timeout time.Duration + invalid bool + }{ + {name: "defaults", yaml: "controller: {}", ttl: 300 * time.Second, timeout: 120 * time.Second}, + {name: "configured", yaml: "controller:\n statusDetailCacheTTL: 30s\n statusDetailRequestTimeout: 10s", ttl: 30 * time.Second, timeout: 10 * time.Second}, + {name: "TTL zero", yaml: "controller:\n statusDetailCacheTTL: 0s", invalid: true}, + {name: "TTL negative", yaml: "controller:\n statusDetailCacheTTL: -1s", invalid: true}, + {name: "TTL malformed", yaml: "controller:\n statusDetailCacheTTL: invalid", invalid: true}, + {name: "timeout zero", yaml: "controller:\n statusDetailRequestTimeout: 0s", invalid: true}, + {name: "timeout negative", yaml: "controller:\n statusDetailRequestTimeout: -1s", invalid: true}, + {name: "timeout malformed", yaml: "controller:\n statusDetailRequestTimeout: invalid", invalid: true}, + {name: "TTL flag wins", yaml: "controller:\n statusDetailCacheTTL: invalid", flagName: "status-detail-cache-ttl", flagValue: "15s", ttl: 15 * time.Second, timeout: 120 * time.Second}, + {name: "timeout flag wins", yaml: "controller:\n statusDetailRequestTimeout: invalid", flagName: "status-detail-request-timeout", flagValue: "15s", ttl: 300 * time.Second, timeout: 15 * time.Second}, + {name: "zero flag", yaml: "controller: {}", flagName: "status-detail-cache-ttl", flagValue: "0s", invalid: true}, + {name: "negative flag", yaml: "controller: {}", flagName: "status-detail-request-timeout", flagValue: "-1s", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0o600); err != nil { + t.Fatal(err) + } + + cfg := &config.Config{} + + cmd := newControllerConfigTestCommand(cfg) + if tc.flagName != "" { + if err := cmd.Flags().Set(tc.flagName, tc.flagValue); err != nil { + t.Fatal(err) + } + } + + err := applyControllerRuntimeConfig(cmd, cfg, path) + if err == nil { + err = cfg.Validate() + } + + if (err != nil) != tc.invalid { + t.Fatalf("startup config validation = %v", err) + } + + if !tc.invalid && (cfg.StatusDetailCacheTTL != tc.ttl || cfg.StatusDetailRequestTimeout != tc.timeout) { + t.Errorf("lifetimes = %s/%s, want %s/%s", cfg.StatusDetailCacheTTL, cfg.StatusDetailRequestTimeout, tc.ttl, tc.timeout) + } + }) + } +} diff --git a/cmd/unbounded-net-controller/status_overview_ingestion_test.go b/cmd/unbounded-net-controller/status_overview_ingestion_test.go new file mode 100644 index 000000000..05e49ee46 --- /dev/null +++ b/cmd/unbounded-net-controller/status_overview_ingestion_test.go @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net/http" + "testing" + "time" + + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func submitOverview(t *testing.T, channel string, health *healthState, message *statusproto.NodeStatusMessage) NodeStatusPushAck { + t.Helper() + + var ( + data []byte + err error + ) + if channel == "proto-http" || channel == "proto-ws" { + data, err = proto.Marshal(message) + } else { + envelope := NodeStatusWSMessage{ + Type: message.Type, NodeName: message.NodeName, DetailRequestID: message.DetailRequestId, + DetailError: message.DetailError, + } + if message.Summary != nil { + overview := protoToNodeOverview(message.Summary) + envelope.Summary = &overview + } + + if message.Status != nil { + status := protoToNodeStatus(message.Status) + envelope.Status = &status + } + + if message.Delta != nil { + envelope.Delta = map[string]json.RawMessage{"timestamp": json.RawMessage(`null`)} + } + + data, err = json.Marshal(envelope) + } + + if err != nil { + t.Fatal(err) + } + + switch channel { + case "proto-http": + ack, code, err := handleProtoPushRequest(health, data, "push") + if err != nil || code != http.StatusOK { + return NodeStatusPushAck{Status: "rejected"} + } + + return ack + case "json-http": + ack, code, err := handleStatusPushRequestWithSource(health, data, "push") + if err != nil || code != http.StatusOK { + return NodeStatusPushAck{Status: "rejected"} + } + + return ack + case "proto-ws": + decoded, err := decodeProtoWSMessage(data) + if err != nil { + return NodeStatusPushAck{Status: "rejected"} + } + + _, ack := handleProtoWSMessage(health, decoded, "ws") + + return ack + case "json-ws": + _, ack := handleNodeStatusWSMessageWithSource(health, data, "ws") + return ack + default: + t.Fatalf("unknown test channel %s", channel) + return NodeStatusPushAck{} + } +} + +func TestOverviewIngestionAllChannels(t *testing.T) { + for _, channel := range []string{"proto-http", "json-http", "proto-ws", "json-ws"} { + t.Run(channel, func(t *testing.T) { + health := &healthState{statusCache: NewNodeStatusCache()} + message := &statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusSummaryType, NodeName: "node", + Summary: &statusproto.NodeStatusOverview{ + TimestampUnixNs: 1000, LastPushTimeUnixNs: 2000, + NodeInfo: &statusproto.NodeInfo{ + Name: "node", SiteName: "site", WireGuard: &statusproto.WireGuardStatusInfo{Interface: "wg0"}, + }, + PeerCount: 10, HealthyPeers: 8, RouteCount: 20, RouteMismatch: true, + NodeErrors: []*statusproto.NodeError{{Type: "cni", Message: "blocked"}}, + HealthCheck: &statusproto.HealthCheckStatus{Summary: "not healthy"}, + NodePodInfo: &statusproto.NodePodInfo{PodName: "agent"}, + }, + } + + ack := submitOverview(t, channel, health, message) + if ack.Status != "ok" || ack.Revision != 1 || !ack.SummarySupported { + t.Fatalf("unexpected ACK: %+v", ack) + } + + entry, ok := health.statusCache.Get("node") + if !ok || entry.Overview == nil { + t.Fatal("overview was not stored") + } + + overview := entry.Overview + if overview.PeerCount != 10 || overview.HealthyPeers != 8 || overview.RouteCount != 20 || !overview.RouteMismatch || + overview.NodeErrors[0].Message != "blocked" || overview.NodeInfo.SiteName != "site" || + overview.NodeInfo.WireGuard.Interface != "wg0" || overview.NodePodInfo.PodName != "agent" || + overview.HealthCheck.Summary != "not healthy" || + !overview.Timestamp.Equal(time.Unix(0, 1000)) || !overview.LastPushTime.Equal(time.Unix(0, 2000)) { + t.Fatalf("overview changed during ingestion: %+v", overview) + } + + if entry.Status.Peers != nil || entry.Status.RoutingTable.Routes != nil || entry.Status.BpfEntries != nil { + t.Fatal("summary retained diagnostic arrays") + } + + if next := submitOverview(t, channel, health, message); next.Revision != 2 { + t.Fatal("summary resync did not advance routine revision") + } + }) + } +} + +func TestOverviewIngestionRejectsInvalidEnvelopes(t *testing.T) { + for _, channel := range []string{"proto-http", "json-http", "proto-ws", "json-ws"} { + for _, tc := range []struct { + name string + mutate func(*statusproto.NodeStatusMessage) + }{ + {"missing", func(m *statusproto.NodeStatusMessage) { m.Summary = nil }}, + {"identity mismatch", func(m *statusproto.NodeStatusMessage) { m.Summary.NodeInfo.Name = "other" }}, + {"negative counts", func(m *statusproto.NodeStatusMessage) { m.Summary.PeerCount = -1 }}, + {"impossible counts", func(m *statusproto.NodeStatusMessage) { m.Summary.HealthyPeers = 1 }}, + {"full mixed with summary", func(m *statusproto.NodeStatusMessage) { m.Status = &statusproto.NodeStatusFull{} }}, + {"delta mixed with summary", func(m *statusproto.NodeStatusMessage) { m.Delta = &statusproto.NodeStatusDelta{} }}, + {"detail correlation on summary", func(m *statusproto.NodeStatusMessage) { m.DetailRequestId = "request" }}, + {"collection error on summary", func(m *statusproto.NodeStatusMessage) { m.DetailError = "failed" }}, + {"summary in full", func(m *statusproto.NodeStatusMessage) { m.Type = "node_status_full" }}, + } { + t.Run(channel+"/"+tc.name, func(t *testing.T) { + health := &healthState{statusCache: NewNodeStatusCache()} + message := &statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusSummaryType, NodeName: "node", + Summary: &statusproto.NodeStatusOverview{NodeInfo: &statusproto.NodeInfo{Name: "node"}}, + } + tc.mutate(message) + + ack := submitOverview(t, channel, health, message) + if ack.Status == "ok" || health.statusCache.Len() != 0 { + t.Fatalf("invalid summary accepted: %+v", ack) + } + }) + } + } +} + +func TestOverviewIdentityRejectsDuplicateAndConflictingFields(t *testing.T) { + for _, data := range []string{ + `{"nodeName":"node","summary":{"nodeInfo":{"name":"other"}}}`, + `{"nodeName":"node","summary":{"nodeInfo":{"name":"other"}},"Summary":null}`, + `{"summary":{"nodeInfo":{"name":"other","Name":"node"}}}`, + `{"summary":{"nodeInfo":{"name":"other"},"NodeInfo":{"name":"node"}}}`, + } { + if _, err := extractNodeNameFromWSMessage([]byte(data)); err == nil { + t.Fatalf("ambiguous summary identity accepted: %s", data) + } + } + + name, err := extractNodeNameFromWSMessage([]byte(`{"summary":{"nodeInfo":{"name":"node"}}}`)) + if err != nil || name != "node" { + t.Fatalf("summary-only identity lost: %q %v", name, err) + } +} + +func TestOverviewCapabilityProtoAck(t *testing.T) { + data, err := marshalProtoAck("node_status_ack", NodeStatusPushAck{Status: "ok", Revision: 7}) + if err != nil { + t.Fatal(err) + } + + var ack statusproto.NodeStatusAck + if err := proto.Unmarshal(data, &ack); err != nil { + t.Fatal(err) + } + + if !ack.SummarySupported || !ack.PeerMeasurements || ack.Revision != 7 { + t.Fatalf("ACK lost capability or revision: %v", &ack) + } +} diff --git a/cmd/unbounded-net-controller/status_overview_proto.go b/cmd/unbounded-net-controller/status_overview_proto.go new file mode 100644 index 000000000..5255dfe91 --- /dev/null +++ b/cmd/unbounded-net-controller/status_overview_proto.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "time" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func protoToNodeOverview(msg *statusproto.NodeStatusOverview) statusv1alpha1.NodeStatusOverview { + overview := statusv1alpha1.NodeStatusOverview{ + HealthCheck: protoToHealthCheckStatus(msg.HealthCheck), + NodeErrors: protoToNodeErrors(msg.NodeErrors), + FetchError: msg.FetchError, StatusSource: msg.StatusSource, + NodePodInfo: protoToNodePodInfo(msg.NodePodInfo), + PeerCount: int(msg.PeerCount), HealthyPeers: int(msg.HealthyPeers), + RouteCount: int(msg.RouteCount), RouteMismatch: msg.RouteMismatch, + RouteMismatchCount: int(msg.RouteMismatchCount), + UnhealthyPeerLinks: int(msg.UnhealthyPeerLinks), UsesIPIP: msg.UsesIpip, + } + if msg.NodeInfo != nil { + overview.NodeInfo = protoToNodeInfo(msg.NodeInfo) + } + + if msg.TimestampUnixNs != 0 { + overview.Timestamp = time.Unix(0, msg.TimestampUnixNs) + } + + if msg.LastPushTimeUnixNs != 0 { + lastPush := time.Unix(0, msg.LastPushTimeUnixNs) + overview.LastPushTime = &lastPush + } + + return overview +} diff --git a/cmd/unbounded-net-controller/status_proto.go b/cmd/unbounded-net-controller/status_proto.go index cb10196b6..635ebc611 100644 --- a/cmd/unbounded-net-controller/status_proto.go +++ b/cmd/unbounded-net-controller/status_proto.go @@ -9,6 +9,7 @@ import ( "google.golang.org/protobuf/proto" + statuspkg "github.com/Azure/unbounded/internal/net/status" statusproto "github.com/Azure/unbounded/internal/net/status/proto" statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) @@ -468,20 +469,58 @@ func validatedProtoNodeName(msg *statusproto.NodeStatusMessage) (string, error) nodeNames = append(nodeNames, msg.Delta.NodeInfo.Name) } + if msg.Summary != nil && msg.Summary.NodeInfo != nil { + nodeNames = append(nodeNames, msg.Summary.NodeInfo.Name) + } + return validatedNodeNames(nodeNames) } // handleProtoWSMessage applies the same decoded message used for authorization. -func handleProtoWSMessage(health *healthState, decoded *decodedProtoWSMessage, source string) (string, NodeStatusPushAck) { +func handleProtoWSMessage(health *healthState, decoded *decodedProtoWSMessage, source string) (ackType string, ack NodeStatusPushAck) { + defer func() { ack.SummarySupported = true }() + msg := &decoded.message nodeName := decoded.nodeName + if msg.DetailError != "" && msg.Type != statusv1alpha1.NodeStatusDetailsType { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "collection error requires a detail response"} + } + + if msg.Summary != nil && msg.Type != statusv1alpha1.NodeStatusSummaryType { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "overview requires summary message type"} + } + if msg.Delta.GetPeerMeasurements() != nil && (msg.Type != "node_status_delta" || msg.Status != nil) { peerMeasurementUpdatesTotal.WithLabelValues("error").Inc() return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "full status conflicts with measurements"} } switch msg.Type { + case statusv1alpha1.NodeStatusDetailsType: + if msg.Delta != nil { + return "node_status_ack", NodeStatusPushAck{Status: "error", DetailRequestID: msg.DetailRequestId, Reason: "details cannot include a delta"} + } + + var status *NodeStatusResponse + + if msg.Status != nil { + full := protoToNodeStatus(msg.Status) + status = &full + } + + return "node_status_ack", handleNodeDetailResponse(health, nodeName, msg.DetailRequestId, status, msg.DetailError) + case statusv1alpha1.NodeStatusSummaryType: + if msg.Summary == nil || msg.Status != nil || msg.Delta != nil || msg.DetailRequestId != "" { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "summary must contain only overview data"} + } + + revision, err := health.statusCache.StoreOverview(nodeName, protoToNodeOverview(msg.Summary), source) + if err != nil { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} + } + + return "node_status_ack", NodeStatusPushAck{Status: "ok", Revision: revision} case "node_status_full": if msg.Status == nil { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "full message missing status"} @@ -518,7 +557,9 @@ func handleProtoWSMessage(health *healthState, decoded *decodedProtoWSMessage, s } // handleProtoPushRequest processes an HTTP push request with protobuf body. -func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string) (NodeStatusPushAck, int, error) { +func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string) (ack NodeStatusPushAck, code int, err error) { + defer func() { ack.SummarySupported = true }() + var msg statusproto.NodeStatusMessage if err := proto.Unmarshal(bodyBytes, &msg); err != nil { return NodeStatusPushAck{}, 400, fmt.Errorf("invalid protobuf body: %v", err) @@ -533,13 +574,46 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string return NodeStatusPushAck{}, 400, fmt.Errorf("nodeName is required") } - ack := NodeStatusPushAck{Status: "ok"} + ack = NodeStatusPushAck{Status: "ok"} + + if msg.DetailError != "" && msg.Type != statusv1alpha1.NodeStatusDetailsType { + return NodeStatusPushAck{}, 400, fmt.Errorf("collection error requires a detail response") + } + + if msg.Summary != nil && msg.Type != statusv1alpha1.NodeStatusSummaryType { + return NodeStatusPushAck{}, 400, fmt.Errorf("overview requires summary message type") + } + if msg.Delta.GetPeerMeasurements() != nil && (msg.Type != "node_status_delta" || msg.Status != nil) { peerMeasurementUpdatesTotal.WithLabelValues("error").Inc() return NodeStatusPushAck{Status: "resync_required", Reason: "full status conflicts with measurements"}, 429, nil } switch msg.Type { + case statusv1alpha1.NodeStatusDetailsType: + if msg.Delta != nil { + return NodeStatusPushAck{Status: "error", DetailRequestID: msg.DetailRequestId, Reason: "details cannot include a delta"}, 200, nil + } + + var status *NodeStatusResponse + + if msg.Status != nil { + full := protoToNodeStatus(msg.Status) + status = &full + } + + return handleNodeDetailResponse(health, nodeName, msg.DetailRequestId, status, msg.DetailError), 200, nil + case statusv1alpha1.NodeStatusSummaryType: + if msg.Summary == nil || msg.Status != nil || msg.Delta != nil || msg.DetailRequestId != "" { + return NodeStatusPushAck{}, 400, fmt.Errorf("summary must contain only overview data") + } + + ack.Revision, err = health.statusCache.StoreOverview(nodeName, protoToNodeOverview(msg.Summary), source) + if err != nil { + return NodeStatusPushAck{}, 400, err + } + + return ack, 200, nil case "node_status_full": if msg.Status == nil { return NodeStatusPushAck{}, 400, fmt.Errorf("status is required for full mode") @@ -573,18 +647,14 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string return ack, 200, nil default: - return NodeStatusPushAck{}, 400, fmt.Errorf("type must be node_status_full or node_status_delta") + return NodeStatusPushAck{}, 400, fmt.Errorf("unsupported status message type %q", msg.Type) } } // marshalProtoAck serializes a NodeStatusPushAck into a protobuf NodeStatusAck. func marshalProtoAck(ackType string, ack NodeStatusPushAck) ([]byte, error) { - pbAck := &statusproto.NodeStatusAck{ - PeerMeasurements: true, - Status: ack.Status, - Revision: ack.Revision, - Reason: ack.Reason, - } + ack.PeerMeasurements = true + ack.SummarySupported = true - return proto.Marshal(pbAck) + return proto.Marshal(statuspkg.NodeStatusAckToProto(&ack)) } diff --git a/cmd/unbounded-net-controller/status_types.go b/cmd/unbounded-net-controller/status_types.go index 98b72c1bd..c54113178 100644 --- a/cmd/unbounded-net-controller/status_types.go +++ b/cmd/unbounded-net-controller/status_types.go @@ -9,47 +9,47 @@ import ( "sort" "time" + statuspkg "github.com/Azure/unbounded/internal/net/status" statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) // ClusterStatusResponse is the top-level status response for the cluster. type ClusterStatusResponse struct { - Seq uint64 `json:"seq"` - Timestamp time.Time `json:"timestamp"` - NodeCount int `json:"nodeCount"` - SiteCount int `json:"siteCount"` - AzureTenantID string `json:"azureTenantId,omitempty"` - LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` - BuildInfo *BuildInfo `json:"buildInfo,omitempty"` - Nodes []*NodeStatusResponse `json:"nodes"` - Sites []SiteStatus `json:"sites"` - GatewayPools []GatewayPoolStatus `json:"gatewayPools"` - Peerings []PeeringStatus `json:"peerings"` - Errors []string `json:"errors,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Problems []StatusProblem `json:"problems"` - ConnectivityMatrix map[string]*SiteMatrix `json:"connectivityMatrix,omitempty"` - PullEnabled bool `json:"pullEnabled"` + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + NodeCount int `json:"nodeCount"` + SiteCount int `json:"siteCount"` + AzureTenantID string `json:"azureTenantId,omitempty"` + LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` + BuildInfo *BuildInfo `json:"buildInfo,omitempty"` + Nodes []*NodeStatusResponse `json:"nodes"` + Sites []SiteStatus `json:"sites"` + GatewayPools []GatewayPoolStatus `json:"gatewayPools"` + Peerings []PeeringStatus `json:"peerings"` + Errors []string `json:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Problems []StatusProblem `json:"problems"` + PullEnabled bool `json:"pullEnabled"` + NodeOverviews map[string]*statusv1alpha1.NodeStatusOverview `json:"-"` } // ClusterStatusDelta is a WebSocket delta update. type ClusterStatusDelta struct { - Seq uint64 `json:"seq"` - Timestamp time.Time `json:"timestamp"` - NodeCount int `json:"nodeCount"` - SiteCount int `json:"siteCount"` - AzureTenantID string `json:"azureTenantId,omitempty"` - LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` - Errors []string `json:"errors,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Problems []StatusProblem `json:"problems"` - UpdatedNodes []json.RawMessage `json:"updatedNodes,omitempty"` - RemovedNodes []string `json:"removedNodes,omitempty"` - Sites []SiteStatus `json:"sites"` - GatewayPools []GatewayPoolStatus `json:"gatewayPools"` - Peerings []PeeringStatus `json:"peerings"` - ConnectivityMatrix map[string]*SiteMatrix `json:"connectivityMatrix,omitempty"` - PullEnabled bool `json:"pullEnabled"` + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + NodeCount int `json:"nodeCount"` + SiteCount int `json:"siteCount"` + AzureTenantID string `json:"azureTenantId,omitempty"` + LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` + Errors []string `json:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Problems []StatusProblem `json:"problems"` + UpdatedNodes []json.RawMessage `json:"updatedNodes,omitempty"` + RemovedNodes []string `json:"removedNodes,omitempty"` + Sites []SiteStatus `json:"sites"` + GatewayPools []GatewayPoolStatus `json:"gatewayPools"` + Peerings []PeeringStatus `json:"peerings"` + PullEnabled bool `json:"pullEnabled"` } // StatusProblem describes one unhealthy condition surfaced in cluster status. @@ -77,28 +77,23 @@ type NodeStatusResponse = statusv1alpha1.NodeStatusResponse // NodeStatusPushEnvelope carries a push status update from a node. type NodeStatusPushEnvelope struct { - Mode string `json:"mode,omitempty"` - NodeName string `json:"nodeName,omitempty"` - BaseRevision uint64 `json:"baseRevision,omitempty"` - Status *NodeStatusResponse `json:"status,omitempty"` - Delta map[string]json.RawMessage `json:"delta,omitempty"` + Type string `json:"type,omitempty"` + Mode string `json:"mode,omitempty"` + NodeName string `json:"nodeName,omitempty"` + BaseRevision uint64 `json:"baseRevision,omitempty"` + Status *NodeStatusResponse `json:"status,omitempty"` + Delta map[string]json.RawMessage `json:"delta,omitempty"` + Summary *statusv1alpha1.NodeStatusOverview `json:"summary,omitempty"` + DetailRequestID string `json:"detailRequestId,omitempty"` + DetailError string `json:"detailError,omitempty"` + SupportsDetails bool `json:"supportsDetails,omitempty"` } // NodeStatusPushAck is the acknowledgment returned for push updates. -type NodeStatusPushAck struct { - Status string `json:"status"` - Revision uint64 `json:"revision,omitempty"` - Reason string `json:"reason,omitempty"` -} +type NodeStatusPushAck = statusv1alpha1.NodeStatusAck // NodeStatusWSMessage is the status message format used over WebSockets. -type NodeStatusWSMessage struct { - Type string `json:"type"` - NodeName string `json:"nodeName,omitempty"` - BaseRevision uint64 `json:"baseRevision,omitempty"` - Status *NodeStatusResponse `json:"status,omitempty"` - Delta map[string]json.RawMessage `json:"delta,omitempty"` -} +type NodeStatusWSMessage = statusv1alpha1.NodeStatusMessage // NodePodInfo aliases the shared node pod status schema. type NodePodInfo = statusv1alpha1.NodePodInfo @@ -149,60 +144,70 @@ type BpfEntry = statusv1alpha1.BpfEntry // everything from ClusterStatusResponse except detailed per-node status // (routes, peers, health check details), replacing those with NodeSummary rows. type ClusterSummary struct { - Seq uint64 `json:"seq"` - Timestamp time.Time `json:"timestamp"` - NodeCount int `json:"nodeCount"` - SiteCount int `json:"siteCount"` - AzureTenantID string `json:"azureTenantId,omitempty"` - LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` - BuildInfo *BuildInfo `json:"buildInfo,omitempty"` - Sites []SiteStatus `json:"sites"` - GatewayPools []GatewayPoolStatus `json:"gatewayPools"` - Peerings []PeeringStatus `json:"peerings"` - Errors []string `json:"errors,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Problems []StatusProblem `json:"problems"` - PullEnabled bool `json:"pullEnabled"` - NodeSummaries []NodeSummary `json:"nodeSummaries"` - ConnectivityMatrix map[string]*SiteMatrix `json:"connectivityMatrix,omitempty"` + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + NodeCount int `json:"nodeCount"` + SiteCount int `json:"siteCount"` + AzureTenantID string `json:"azureTenantId,omitempty"` + LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` + BuildInfo *BuildInfo `json:"buildInfo,omitempty"` + Sites []SiteStatus `json:"sites"` + GatewayPools []GatewayPoolStatus `json:"gatewayPools"` + Peerings []PeeringStatus `json:"peerings"` + Errors []string `json:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Problems []StatusProblem `json:"problems"` + PullEnabled bool `json:"pullEnabled"` + NodeSummaries []NodeSummary `json:"nodeSummaries"` } // NodeSummary is a compact per-node summary for use in ClusterSummary. type NodeSummary struct { - Name string `json:"name"` - SiteName string `json:"siteName,omitempty"` - IsGateway bool `json:"isGateway,omitempty"` - K8sReady string `json:"k8sReady,omitempty"` - StatusSource string `json:"statusSource,omitempty"` - CniStatus string `json:"cniStatus,omitempty"` - CniTone string `json:"cniTone,omitempty"` - ErrorCount int `json:"errorCount,omitempty"` - FirstError string `json:"firstError,omitempty"` - PeerCount int `json:"peerCount,omitempty"` - HealthyPeers int `json:"healthyPeers,omitempty"` - RouteCount int `json:"routeCount,omitempty"` - RouteMismatch bool `json:"routeMismatch,omitempty"` - FetchError string `json:"fetchError,omitempty"` + Name string `json:"name"` + SiteName string `json:"siteName,omitempty"` + IsGateway bool `json:"isGateway,omitempty"` + K8sReady string `json:"k8sReady,omitempty"` + StatusSource string `json:"statusSource,omitempty"` + CniStatus string `json:"cniStatus,omitempty"` + CniTone string `json:"cniTone,omitempty"` + ErrorCount int `json:"errorCount,omitempty"` + FirstError string `json:"firstError,omitempty"` + PeerCount int `json:"peerCount,omitempty"` + HealthyPeers int `json:"healthyPeers,omitempty"` + RouteCount int `json:"routeCount,omitempty"` + RouteMismatch bool `json:"routeMismatch,omitempty"` + FetchError string `json:"fetchError,omitempty"` + WireGuardOnline bool `json:"wireGuardOnline"` } // buildClusterSummary extracts a ClusterSummary from a full ClusterStatusResponse. -// This is O(N) in nodes with simple field reads -- no route annotation work. +// Only legacy payloads require scanning peers and route next hops. func buildClusterSummary(status *ClusterStatusResponse) *ClusterSummary { summaries := make([]NodeSummary, 0, len(status.Nodes)) now := time.Now() for i := range status.Nodes { node := status.Nodes[i] + + overview := status.NodeOverviews[node.NodeInfo.Name] + if overview == nil { + projected := statuspkg.OverviewFromStatus(node, now) + overview = &projected + } + ns := NodeSummary{ - Name: node.NodeInfo.Name, - SiteName: node.NodeInfo.SiteName, - IsGateway: node.NodeInfo.IsGateway, - K8sReady: node.NodeInfo.K8sReady, - StatusSource: node.StatusSource, - PeerCount: len(node.Peers), - RouteCount: len(node.RoutingTable.Routes), - FetchError: node.FetchError, - ErrorCount: len(node.NodeErrors), + Name: node.NodeInfo.Name, + SiteName: node.NodeInfo.SiteName, + IsGateway: node.NodeInfo.IsGateway, + K8sReady: node.NodeInfo.K8sReady, + StatusSource: node.StatusSource, + PeerCount: overview.PeerCount, + HealthyPeers: overview.HealthyPeers, + RouteCount: overview.RouteCount, + RouteMismatch: overview.RouteMismatch, + FetchError: node.FetchError, + ErrorCount: len(node.NodeErrors), + WireGuardOnline: node.NodeInfo.WireGuard != nil && node.NodeInfo.WireGuard.Interface != "", } // Include first error message so the frontend can show it inline @@ -211,38 +216,6 @@ func buildClusterSummary(status *ClusterStatusResponse) *ClusterSummary { ns.FirstError = node.NodeErrors[0].Message } - // Count healthy peers - for j := range node.Peers { - peer := &node.Peers[j] - if peer.HealthCheck != nil && peer.HealthCheck.Enabled { - if peer.HealthCheck.Status == "up" || peer.HealthCheck.Status == "Up" { - ns.HealthyPeers++ - } - } else { - // Fall back to handshake freshness - if !peer.Tunnel.LastHandshake.IsZero() && now.Sub(peer.Tunnel.LastHandshake) < 3*time.Minute { - ns.HealthyPeers++ - } - } - } - - // Route mismatch check - for _, route := range node.RoutingTable.Routes { - for _, hop := range route.NextHops { - expected := hop.Expected != nil && *hop.Expected - - present := hop.Present != nil && *hop.Present - if expected != present { - ns.RouteMismatch = true - break - } - } - - if ns.RouteMismatch { - break - } - } - // Derive CNI status and tone ns.CniStatus, ns.CniTone = deriveCniStatusAndTone(node, ns.RouteMismatch) summaries = append(summaries, ns) @@ -251,22 +224,21 @@ func buildClusterSummary(status *ClusterStatusResponse) *ClusterSummary { sort.Slice(summaries, func(i, j int) bool { return summaries[i].Name < summaries[j].Name }) return &ClusterSummary{ - Seq: status.Seq, - Timestamp: status.Timestamp, - NodeCount: status.NodeCount, - SiteCount: status.SiteCount, - AzureTenantID: status.AzureTenantID, - LeaderInfo: status.LeaderInfo, - BuildInfo: status.BuildInfo, - Sites: status.Sites, - GatewayPools: status.GatewayPools, - Peerings: status.Peerings, - Errors: status.Errors, - Warnings: status.Warnings, - Problems: status.Problems, - PullEnabled: status.PullEnabled, - NodeSummaries: summaries, - ConnectivityMatrix: status.ConnectivityMatrix, + Seq: status.Seq, + Timestamp: status.Timestamp, + NodeCount: status.NodeCount, + SiteCount: status.SiteCount, + AzureTenantID: status.AzureTenantID, + LeaderInfo: status.LeaderInfo, + BuildInfo: status.BuildInfo, + Sites: status.Sites, + GatewayPools: status.GatewayPools, + Peerings: status.Peerings, + Errors: status.Errors, + Warnings: status.Warnings, + Problems: status.Problems, + PullEnabled: status.PullEnabled, + NodeSummaries: summaries, } } @@ -330,33 +302,26 @@ type PeeringStatus struct { HealthCheckEnabled bool `json:"healthCheckEnabled,omitempty"` } -// SiteMatrix contains connectivity results across site nodes. -type SiteMatrix struct { - Nodes []string `json:"nodes"` - Results map[string]map[string]string `json:"results"` // src -> dst -> status -} - // ClusterSummaryDelta contains only the fields of ClusterSummary that changed // since the last broadcast. NodeSummaries contains only added/changed entries; // RemovedNodes lists nodes that disappeared. type ClusterSummaryDelta struct { - Seq uint64 `json:"seq"` - Timestamp time.Time `json:"timestamp"` - NodeCount *int `json:"nodeCount,omitempty"` - SiteCount *int `json:"siteCount,omitempty"` - AzureTenantID *string `json:"azureTenantId,omitempty"` - LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` - BuildInfo *BuildInfo `json:"buildInfo,omitempty"` - Sites []SiteStatus `json:"sites,omitempty"` - GatewayPools []GatewayPoolStatus `json:"gatewayPools,omitempty"` - Peerings []PeeringStatus `json:"peerings,omitempty"` - Errors []string `json:"errors,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Problems []StatusProblem `json:"problems,omitempty"` - PullEnabled *bool `json:"pullEnabled,omitempty"` - NodeSummaries []NodeSummary `json:"nodeSummaries,omitempty"` - RemovedNodes []string `json:"removedNodes,omitempty"` - ConnectivityMatrix map[string]*SiteMatrix `json:"connectivityMatrix,omitempty"` + Seq uint64 `json:"seq"` + Timestamp time.Time `json:"timestamp"` + NodeCount *int `json:"nodeCount,omitempty"` + SiteCount *int `json:"siteCount,omitempty"` + AzureTenantID *string `json:"azureTenantId,omitempty"` + LeaderInfo *LeaderInfo `json:"leaderInfo,omitempty"` + BuildInfo *BuildInfo `json:"buildInfo,omitempty"` + Sites []SiteStatus `json:"sites,omitempty"` + GatewayPools []GatewayPoolStatus `json:"gatewayPools,omitempty"` + Peerings []PeeringStatus `json:"peerings,omitempty"` + Errors []string `json:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Problems []StatusProblem `json:"problems,omitempty"` + PullEnabled *bool `json:"pullEnabled,omitempty"` + NodeSummaries []NodeSummary `json:"nodeSummaries,omitempty"` + RemovedNodes []string `json:"removedNodes,omitempty"` } // computeClusterSummaryDelta computes a delta between two ClusterSummary snapshots. @@ -437,11 +402,6 @@ func computeClusterSummaryDelta(prev, curr *ClusterSummary) *ClusterSummaryDelta changed = true } - if !jsonEqual(prev.ConnectivityMatrix, curr.ConnectivityMatrix) { - delta.ConnectivityMatrix = curr.ConnectivityMatrix - changed = true - } - // NodeSummaries: diff by name prevByName := make(map[string]NodeSummary, len(prev.NodeSummaries)) for _, ns := range prev.NodeSummaries { diff --git a/cmd/unbounded-net-controller/websocket.go b/cmd/unbounded-net-controller/websocket.go index 0a9ceaf82..7a4bda15d 100644 --- a/cmd/unbounded-net-controller/websocket.go +++ b/cmd/unbounded-net-controller/websocket.go @@ -321,8 +321,8 @@ func (b *WSBroadcaster) broadcastUpdate(ctx context.Context) { return } - klog.V(4).Infof("WebSocket: summary delta: %d nodeSummaries, %d removed, sites=%v pools=%v matrix=%v", - len(delta.NodeSummaries), len(delta.RemovedNodes), delta.Sites != nil, delta.GatewayPools != nil, delta.ConnectivityMatrix != nil) + klog.V(4).Infof("WebSocket: summary delta: %d nodeSummaries, %d removed, sites=%v pools=%v", + len(delta.NodeSummaries), len(delta.RemovedNodes), delta.Sites != nil, delta.GatewayPools != nil) msg := WSMessage{Type: "cluster_summary_delta", Data: delta} summaryData, _ = json.Marshal(msg) //nolint:errcheck } else { @@ -422,9 +422,6 @@ func (b *WSBroadcaster) broadcastUpdate(ctx context.Context) { PullEnabled: status.PullEnabled, } - // Always include ConnectivityMatrix so link-state-only changes refresh clients. - delta.ConnectivityMatrix = status.ConnectivityMatrix - msg := WSMessage{Type: "cluster_status_delta", Data: delta} deltaData, err := json.Marshal(msg) diff --git a/cmd/unbounded-net-node/main.go b/cmd/unbounded-net-node/main.go index 58d166093..0e82868fc 100644 --- a/cmd/unbounded-net-node/main.go +++ b/cmd/unbounded-net-node/main.go @@ -93,6 +93,7 @@ type config struct { StatusPushInterval time.Duration // Interval between status pushes to controller StatusPushAPIServerInterval time.Duration // Interval between status pushes via aggregated API server StatusPushDelta bool // Whether periodic HTTP pushes use deltas + StatusDetailMode string // Startup-loaded routine publication mode. StatusWSEnabled bool // Whether websocket push is enabled StatusWSURL string // Controller websocket URL for status push StatusWSAPIServerMode string // API server fallback mode: never, fallback, preferred (alias for fallback) @@ -230,6 +231,7 @@ func main() { StatusPushInterval: 10 * time.Second, // Default 10s push interval StatusPushAPIServerInterval: 30 * time.Second, StatusPushDelta: true, + StatusDetailMode: configpkg.DefaultStatusDetailMode, StatusWSEnabled: true, StatusWSAPIServerMode: statusWSAPIServerModeFallback, StatusWSAPIServerStartupDelay: 60 * time.Second, @@ -328,6 +330,7 @@ then annotates the node with the public key.`, flags.DurationVar(&cfg.StatusPushInterval, "status-push-interval", 60*time.Second, "Interval between status pushes to controller") flags.DurationVar(&cfg.StatusPushAPIServerInterval, "status-push-apiserver-interval", 60*time.Second, "Interval between status pushes via aggregated API server") flags.BoolVar(&cfg.StatusPushDelta, "status-push-delta", true, "Enable delta mode for periodic HTTP status push") + flags.StringVar(&cfg.StatusDetailMode, "status-detail-mode", configpkg.DefaultStatusDetailMode, "Routine status detail mode: summary or full") flags.BoolVar(&cfg.StatusWSEnabled, "status-ws-enabled", true, "Enable websocket status push to controller") flags.StringVar(&cfg.StatusWSURL, "status-ws-url", "", "Controller websocket URL for status push (default: ws://service/status/nodews)") flags.StringVar(&cfg.StatusWSAPIServerMode, "status-ws-apiserver-mode", statusWSAPIServerModeFallback, "API server fallback mode: never, fallback, preferred (alias for fallback); direct controller endpoints are tried first") @@ -365,6 +368,14 @@ func applyNodeRuntimeConfig(cmd *cobra.Command, cfg *config) error { flags := cmd.Flags() nodeCfg := runtimeCfg.Node + if !flags.Changed("status-detail-mode") && nodeCfg.StatusDetailMode != "" { + cfg.StatusDetailMode = nodeCfg.StatusDetailMode + } + + if err := configpkg.ValidateStatusDetailMode(cfg.StatusDetailMode); err != nil { + return err + } + if !flags.Changed("informer-resync-period") { if d, parseErr := configpkg.ParseDurationField(nodeCfg.InformerResyncPeriod, "node.informerResyncPeriod"); parseErr != nil { return parseErr diff --git a/cmd/unbounded-net-node/main_config_test.go b/cmd/unbounded-net-node/main_config_test.go index 97b3835fc..1a87b877e 100644 --- a/cmd/unbounded-net-node/main_config_test.go +++ b/cmd/unbounded-net-node/main_config_test.go @@ -11,6 +11,8 @@ import ( "time" "github.com/spf13/cobra" + + configpkg "github.com/Azure/unbounded/internal/net/config" ) func newNodeConfigTestCommand(cfg *config) *cobra.Command { @@ -35,6 +37,7 @@ func newNodeConfigTestCommand(cfg *config) *cobra.Command { flags.DurationVar(&cfg.StatusPushInterval, "status-push-interval", 10*time.Second, "") flags.DurationVar(&cfg.StatusPushAPIServerInterval, "status-push-apiserver-interval", 30*time.Second, "") flags.BoolVar(&cfg.StatusPushDelta, "status-push-delta", true, "") + flags.StringVar(&cfg.StatusDetailMode, "status-detail-mode", configpkg.DefaultStatusDetailMode, "") flags.BoolVar(&cfg.StatusWSEnabled, "status-ws-enabled", true, "") flags.StringVar(&cfg.StatusWSURL, "status-ws-url", "", "") flags.StringVar(&cfg.StatusWSAPIServerMode, "status-ws-apiserver-mode", statusWSAPIServerModeFallback, "") diff --git a/cmd/unbounded-net-node/status_ack.go b/cmd/unbounded-net-node/status_ack.go index b81a85770..d457a4ce4 100644 --- a/cmd/unbounded-net-node/status_ack.go +++ b/cmd/unbounded-net-node/status_ack.go @@ -5,11 +5,14 @@ package main import ( "encoding/json" + "fmt" "sync/atomic" "google.golang.org/protobuf/proto" + netstatus "github.com/Azure/unbounded/internal/net/status" statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) // statusAckState is created fresh for every connection. One outstanding message @@ -19,31 +22,54 @@ type statusAckState struct { resync atomic.Bool pending atomic.Bool compact atomic.Bool + summary atomic.Bool } -func (s *statusAckState) accept(data []byte) bool { +func decodeNodeStatusAck(data []byte) (*statusv1alpha1.NodeStatusAck, error) { var ack statusproto.NodeStatusAck - if err := proto.Unmarshal(data, &ack); err != nil { - var envelope struct { - Type string `json:"type"` - Data nodeStatusPushAck `json:"data"` - } - if err := json.Unmarshal(data, &envelope); err != nil { - return false - } + if err := proto.Unmarshal(data, &ack); err == nil && ack.Status != "" { + return netstatus.NodeStatusAckFromProto(&ack), nil + } + + var jsonAck statusv1alpha1.NodeStatusAck + if err := json.Unmarshal(data, &jsonAck); err == nil && jsonAck.Status != "" { + return &jsonAck, nil + } + + var envelope struct { + Type string `json:"type"` + Data statusv1alpha1.NodeStatusAck `json:"data"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return nil, err + } - switch envelope.Type { - case "node_status_ack": - ack.Status = "ok" - case "node_status_resync": - ack.Status = "resync_required" - default: - return false + switch envelope.Type { + case "node_status_ack": + if envelope.Data.Status == "" { + envelope.Data.Status = "ok" } + case "node_status_resync": + envelope.Data.Status = "resync_required" + default: + return nil, fmt.Errorf("unrecognized status acknowledgment") + } - ack.Revision = envelope.Data.Revision + return &envelope.Data, nil +} + +func (s *statusAckState) accept(data []byte) bool { + ack, err := decodeNodeStatusAck(data) + return err == nil && s.acceptAck(ack) +} + +func (s *statusAckState) acceptAck(ack *statusv1alpha1.NodeStatusAck) bool { + if !ack.IsPublicationAck() { + return false } + s.summary.Store(ack.SummarySupported) + switch ack.Status { case "ok": s.compact.Store(ack.PeerMeasurements && ack.Revision > 0) diff --git a/cmd/unbounded-net-node/status_detail_config_test.go b/cmd/unbounded-net-node/status_detail_config_test.go new file mode 100644 index 000000000..31e5ba0b9 --- /dev/null +++ b/cmd/unbounded-net-node/status_detail_config_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestNodeStatusDetailConfig(t *testing.T) { + for _, tc := range []struct { + name, yaml, flag, want string + invalid bool + }{ + {name: "default", yaml: "node: {}", want: "full"}, + {name: "summary", yaml: "node:\n statusDetailMode: summary", want: "summary"}, + {name: "full", yaml: "node:\n statusDetailMode: full", want: "full"}, + {name: "invalid YAML value", yaml: "node:\n statusDetailMode: invalid", invalid: true}, + {name: "flag wins", yaml: "node:\n statusDetailMode: summary", flag: "full", want: "full"}, + {name: "flag overrides invalid YAML", yaml: "node:\n statusDetailMode: invalid", flag: "summary", want: "summary"}, + {name: "invalid flag", yaml: "node: {}", flag: "invalid", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0o600); err != nil { + t.Fatal(err) + } + + cfg := &config{ + ConfigFile: path, GeneveInterfaceName: "geneve0", VXLANInterfaceName: "vxlan0", + IPIPInterfaceName: "ipip0", WireGuardInterfacePrefix: "wg", + } + + cmd := newNodeConfigTestCommand(cfg) + if tc.flag != "" { + if err := cmd.Flags().Set("status-detail-mode", tc.flag); err != nil { + t.Fatal(err) + } + } + + err := applyNodeRuntimeConfig(cmd, cfg) + if (err != nil) != tc.invalid { + t.Fatalf("applyNodeRuntimeConfig() = %v", err) + } + + if !tc.invalid && cfg.StatusDetailMode != tc.want { + t.Errorf("mode = %q, want %q", cfg.StatusDetailMode, tc.want) + } + }) + } +} diff --git a/cmd/unbounded-net-node/status_details.go b/cmd/unbounded-net-node/status_details.go new file mode 100644 index 000000000..5dc5e9a02 --- /dev/null +++ b/cmd/unbounded-net-node/status_details.go @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "sync" + "time" + + "google.golang.org/protobuf/proto" + "k8s.io/klog/v2" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +const ( + nodeDetailFrameLimit = 2 * 1024 * 1024 + nodeDetailRetryInterval = time.Second +) + +type nodeDetailReply struct { + request statusv1alpha1.DetailRequest + payload []byte + sending bool + done bool + retryAt time.Time +} + +// One state spans both publishers and reconnects. Successful ACKs retain only +// request identity/deadline markers; no routine publisher owns detail snapshots. +type nodeDetailState struct { + mu sync.Mutex + replies map[string]*nodeDetailReply + wsWake chan struct{} + httpWake chan struct{} +} + +func (h *nodeHealthState) detailState() *nodeDetailState { + h.mu.Lock() + defer h.mu.Unlock() + + if h.details == nil { + h.details = &nodeDetailState{ + replies: make(map[string]*nodeDetailReply), + wsWake: make(chan struct{}, 1), httpWake: make(chan struct{}, 1), + } + } + + return h.details +} + +func (s *nodeDetailState) wake() { + for _, ch := range []chan struct{}{s.wsWake, s.httpWake} { + select { + case ch <- struct{}{}: + default: + } + } +} + +func (s *nodeDetailState) expireLocked(now time.Time) { + for id, reply := range s.replies { + if !reply.request.Deadline.After(now) { + delete(s.replies, id) + } + } +} + +func (s *nodeDetailState) enqueue(request *statusv1alpha1.DetailRequest, now time.Time) error { + if err := netstatus.ValidateDetailRequest(request, now); err != nil { + return err + } + + s.mu.Lock() + s.expireLocked(now) + + if _, exists := s.replies[request.RequestID]; !exists { + s.replies[request.RequestID] = &nodeDetailReply{request: *request} + } + s.mu.Unlock() + s.wake() + + return nil +} + +func (s *nodeDetailState) acknowledge(ack *statusv1alpha1.NodeStatusAck) { + if ack == nil || ack.DetailRequestID == "" || ack.Status != "ok" { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + if reply := s.replies[ack.DetailRequestID]; reply != nil { + reply.payload = nil + reply.done = true + } +} + +type nodeDetailDelivery struct { + id string + deadline time.Time + payload []byte +} + +func (s *nodeDetailState) take(nodeName string, collect func() *NodeStatusResponse, now time.Time) *nodeDetailDelivery { + s.mu.Lock() + s.expireLocked(now) + + var ( + selected *nodeDetailReply + payload []byte + ) + + for _, reply := range s.replies { + if !reply.done && !reply.sending && !now.Before(reply.retryAt) { + selected = reply + selected.sending = true + payload = selected.payload + + break + } + } + s.mu.Unlock() + + if selected == nil { + return nil + } + + if payload == nil { + payload = collectDetailPayload(nodeName, selected.request.RequestID, collect) + } + + s.mu.Lock() + defer s.mu.Unlock() + + if !selected.request.Deadline.After(time.Now()) { + delete(s.replies, selected.request.RequestID) + return nil + } + + if selected.done { + return nil + } + + selected.payload = payload + + return &nodeDetailDelivery{id: selected.request.RequestID, deadline: selected.request.Deadline, payload: payload} +} + +func (s *nodeDetailState) finish(id string) { + s.mu.Lock() + defer s.mu.Unlock() + + if reply := s.replies[id]; reply != nil { + reply.sending = false + reply.retryAt = time.Now().Add(nodeDetailRetryInterval) + } +} + +func detailErrorPayload(nodeName, requestID, message string) []byte { + payload, err := proto.Marshal(&statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusDetailsType, NodeName: nodeName, DetailRequestId: requestID, + DetailError: message, SupportsDetails: true, + }) + if err != nil { + klog.Errorf("Failed to encode correlated detail failure: %v", err) + return nil + } + + return payload +} + +func collectDetailPayload(nodeName, requestID string, collect func() *NodeStatusResponse) (payload []byte) { + defer func() { + if failure := recover(); failure != nil { + payload = detailErrorPayload(nodeName, requestID, fmt.Sprintf("detail collection failed: %v", failure)) + } + }() + + full := collect() + if full == nil { + return detailErrorPayload(nodeName, requestID, "detail collection returned no snapshot") + } + + if full.FetchError != "" { + return detailErrorPayload(nodeName, requestID, full.FetchError) + } + + message := &statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusDetailsType, NodeName: nodeName, DetailRequestId: requestID, + Status: nodeStatusToProto(full), SupportsDetails: true, + } + if proto.Size(message) > nodeDetailFrameLimit { + return detailErrorPayload(nodeName, requestID, "detail response exceeds 2 MiB transport frame limit") + } + + payload, err := proto.Marshal(message) + if err != nil { + return detailErrorPayload(nodeName, requestID, fmt.Sprintf("detail encoding failed: %v", err)) + } + + return payload +} diff --git a/cmd/unbounded-net-node/status_details_test.go b/cmd/unbounded-net-node/status_details_test.go new file mode 100644 index 000000000..3355f220f --- /dev/null +++ b/cmd/unbounded-net-node/status_details_test.go @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestDetailStateCoalescesRetriesAndReleases(t *testing.T) { + h := blockedBootstrapHealthState() + state := h.detailState() + now := time.Now() + + req := &statusv1alpha1.DetailRequest{RequestID: "one", Deadline: now.Add(time.Minute)} + if err := state.enqueue(req, now); err != nil { + t.Fatal(err) + } + + originalDeadline := req.Deadline + + req.Deadline = req.Deadline.Add(time.Hour) + if err := state.enqueue(req, now); err != nil { + t.Fatal(err) + } + + count := 0 + collect := func() *NodeStatusResponse { count++; return h.getStatusSnapshot() } + + first := state.take("node-a", collect, now) + if first == nil || !first.deadline.Equal(originalDeadline) || count != 1 { + t.Fatalf("lost command deadline/collection: %+v count=%d", first, count) + } + + if state.take("node-a", collect, now) != nil { + t.Fatal("same request sent concurrently") + } + + state.finish(first.id) + + retry := state.take("node-a", collect, now.Add(2*time.Second)) + if retry == nil || !bytes.Equal(first.payload, retry.payload) || count != 1 { + t.Fatal("retry recollected or changed the snapshot") + } + + state.acknowledge(&statusv1alpha1.NodeStatusAck{Status: "ok", DetailRequestID: first.id}) + state.finish(first.id) + + if state.replies[first.id].payload != nil || !state.replies[first.id].done { + t.Fatal("ACK retained heavy payload") + } + + if err := state.enqueue(req, now); err != nil { + t.Fatal(err) + } + + if state.take("node-a", collect, now.Add(3*time.Second)) != nil || count != 1 { + t.Fatal("delayed duplicate recollected an acknowledged request") + } + + state.take("node-a", collect, originalDeadline) + + if len(state.replies) != 0 { + t.Fatal("deadline did not remove idempotency marker") + } +} + +func TestDetailStateExpiryAndConcurrentClaims(t *testing.T) { + state := (&nodeHealthState{}).detailState() + + now := time.Now() + if state.enqueue(&statusv1alpha1.DetailRequest{RequestID: "old", Deadline: now}, now) == nil { + t.Fatal("expired command accepted") + } + + req := &statusv1alpha1.DetailRequest{RequestID: "one", Deadline: now.Add(time.Minute)} + + var ( + count atomic.Int32 + wg sync.WaitGroup + ) + for range 8 { + wg.Go(func() { + if err := state.enqueue(req, now); err != nil { + t.Error(err) + } + + state.take("node", func() *NodeStatusResponse { + count.Add(1) + return &NodeStatusResponse{} + }, now) + }) + } + + wg.Wait() + + if count.Load() != 1 { + t.Fatalf("collected %d duplicate snapshots", count.Load()) + } + + state.take("node", nil, req.Deadline) + + if len(state.replies) != 0 { + t.Fatal("expired unacknowledged detail retained") + } +} + +func TestDetailPayloadErrorsAreCorrelated(t *testing.T) { + for _, tc := range []struct { + name string + collect func() *NodeStatusResponse + }{ + {"nil", func() *NodeStatusResponse { return nil }}, + {"panic", func() *NodeStatusResponse { panic("failed syscall") }}, + {"fetch error", func() *NodeStatusResponse { return &NodeStatusResponse{FetchError: "unavailable"} }}, + {"oversized", func() *NodeStatusResponse { + return &NodeStatusResponse{NodeErrors: []NodeError{{Message: strings.Repeat("x", nodeDetailFrameLimit)}}} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + payload := collectDetailPayload("node", "request", tc.collect) + + var message statusproto.NodeStatusMessage + if err := proto.Unmarshal(payload, &message); err != nil { + t.Fatal(err) + } + + if message.Type != statusv1alpha1.NodeStatusDetailsType || message.NodeName != "node" || + message.DetailRequestId != "request" || message.DetailError == "" || message.Status != nil || len(payload) >= nodeDetailFrameLimit { + t.Fatalf("invalid correlated failure: %v", &message) + } + }) + } +} diff --git a/cmd/unbounded-net-node/status_proto.go b/cmd/unbounded-net-node/status_proto.go index 256a64711..7d2992bc3 100644 --- a/cmd/unbounded-net-node/status_proto.go +++ b/cmd/unbounded-net-node/status_proto.go @@ -18,6 +18,34 @@ func statusUnixNano(t time.Time) int64 { return t.UnixNano() } +func nodeSummaryToProto(summary *NodeStatusOverview) *statusproto.NodeStatusOverview { + if summary == nil { + return nil + } + + result := &statusproto.NodeStatusOverview{ + TimestampUnixNs: statusUnixNano(summary.Timestamp), + NodeInfo: nodeInfoToProto(&summary.NodeInfo), + HealthCheck: healthCheckStatusToProto(summary.HealthCheck), + NodeErrors: nodeErrorsToProto(summary.NodeErrors), + FetchError: summary.FetchError, + StatusSource: summary.StatusSource, + NodePodInfo: nodePodInfoToProto(summary.NodePodInfo), + PeerCount: int32(summary.PeerCount), + HealthyPeers: int32(summary.HealthyPeers), + RouteCount: int32(summary.RouteCount), + RouteMismatch: summary.RouteMismatch, + RouteMismatchCount: int32(summary.RouteMismatchCount), + UnhealthyPeerLinks: int32(summary.UnhealthyPeerLinks), + UsesIpip: summary.UsesIPIP, + } + if summary.LastPushTime != nil { + result.LastPushTimeUnixNs = statusUnixNano(*summary.LastPushTime) + } + + return result +} + // nodeStatusToProto converts a Go NodeStatusResponse to the protobuf NodeStatusFull message. func nodeStatusToProto(status *NodeStatusResponse) *statusproto.NodeStatusFull { if status == nil { diff --git a/cmd/unbounded-net-node/status_publication.go b/cmd/unbounded-net-node/status_publication.go new file mode 100644 index 000000000..7d85dc736 --- /dev/null +++ b/cmd/unbounded-net-node/status_publication.go @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "reflect" + "time" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +const nodeErrorSummaryUnsupported = "status-summary-unsupported" + +func collectPublication(health *nodeHealthState, cfg *config, previous *NodeStatusResponse, force bool, revision uint64) (*statusproto.NodeStatusMessage, *NodeStatusResponse) { + if cfg.StatusDetailMode == "summary" { + summary := health.getSummarySnapshot() + + return &statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusSummaryType, NodeName: summary.NodeInfo.Name, + BaseRevision: revision, Summary: nodeSummaryToProto(summary), + }, nil + } + + full := health.getStatusSnapshot() + + msg := &statusproto.NodeStatusMessage{Type: "node_status_full", NodeName: full.NodeInfo.Name} + if cfg.StatusPushDelta && !force { + msg.Delta = typedStatusDelta(previous, full, false, true) + if msg.Delta != nil { + msg.Type, msg.BaseRevision = "node_status_delta", revision + } + } + + if msg.Delta == nil { + msg.Status = nodeStatusToProto(full) + } + + return msg, full +} + +func publicationNodeErrors(errors []NodeError) []NodeError { + result := make([]NodeError, 0, len(errors)) + for _, err := range errors { + switch err.Type { + case nodeErrorTypeDirectPush, nodeErrorTypeDirectWebSocket, nodeErrorTypeFallbackPush, nodeErrorTypeFallbackWS: + continue + default: + result = append(result, err) + } + } + + return result +} + +func equalPublicationSummaries(a, b *NodeStatusOverview) bool { + if a == nil || b == nil { + return a == b + } + + normalize := func(summary *NodeStatusOverview) NodeStatusOverview { + result := *summary + result.Timestamp = time.Time{} + + if summary.HealthCheck != nil { + health := *summary.HealthCheck + health.CheckedAt = time.Time{} + result.HealthCheck = &health + } + + return result + } + + return reflect.DeepEqual(normalize(a), normalize(b)) +} diff --git a/cmd/unbounded-net-node/status_publication_test.go b/cmd/unbounded-net-node/status_publication_test.go new file mode 100644 index 000000000..d362a46a3 --- /dev/null +++ b/cmd/unbounded-net-node/status_publication_test.go @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "compress/gzip" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + "google.golang.org/protobuf/proto" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestSummaryPublicationNeverCollectsDetails(t *testing.T) { + s := summaryRouteFixture() + s.bpfCollector = func() []BpfEntry { t.Fatal("routine summary collected BPF"); return nil } + h := blockedBootstrapHealthState() + h.setStatusServer(s) + + cfg := &config{StatusDetailMode: "summary", StatusPushDelta: true} + for _, force := range []bool{true, false} { + msg, base := collectPublication(h, cfg, &NodeStatusResponse{Peers: make([]WireGuardPeerStatus, 100)}, force, 42) + if base != nil || msg.Status != nil || msg.Delta != nil || msg.Type != statusv1alpha1.NodeStatusSummaryType || msg.Summary == nil { + t.Fatalf("summary retained details: %v base=%v", msg, base) + } + + if len(msg.Summary.NodeErrors) == 0 || msg.BaseRevision != 42 { + t.Fatal("summary lost guard/revision") + } + } +} + +func TestDetailACKDoesNotReleasePublication(t *testing.T) { + state := &statusAckState{} + state.revision.Store(7) + state.pending.Store(true) + + for _, ack := range []*statusv1alpha1.NodeStatusAck{ + {Status: statusv1alpha1.DetailRequestStatus, DetailRequest: &statusv1alpha1.DetailRequest{RequestID: "r", Deadline: time.Now().Add(time.Minute)}}, + {Status: "ok", DetailRequestID: "r", Revision: 99, SummarySupported: true}, + } { + data, err := proto.Marshal(netstatus.NodeStatusAckToProto(ack)) + if err != nil { + t.Fatal(err) + } + + if state.accept(data) || !state.pending.Load() || state.revision.Load() != 7 || state.summary.Load() { + t.Fatal("detail traffic changed publication ACK state") + } + } +} + +func TestRoutineSummaryPublishers(t *testing.T) { + for _, transport := range []string{"HTTP", "WS"} { + for _, supported := range []bool{true, false} { + t.Run(transport+map[bool]string{true: "/supported", false: "/unsupported"}[supported], func(t *testing.T) { + messages := make(chan *statusproto.NodeStatusMessage, 32) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handle := func(data []byte) []byte { + var msg statusproto.NodeStatusMessage + if err := proto.Unmarshal(data, &msg); err != nil { + t.Error(err) + } + + select { + case messages <- &msg: + case <-r.Context().Done(): + } + + ack, _ := proto.Marshal(&statusproto.NodeStatusAck{Status: "resync_required", Revision: 3, SummarySupported: supported}) + + return ack + } + + if transport == "WS" { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer func() { _ = conn.Close(websocket.StatusNormalClosure, "done") }() + + for { + _, data, err := conn.Read(r.Context()) + if err != nil { + return + } + + if err := conn.Write(r.Context(), websocket.MessageBinary, handle(data)); err != nil { + return + } + } + } + + reader, err := gzip.NewReader(r.Body) + if err != nil { + t.Error(err) + return + } + + data, err := io.ReadAll(reader) + _ = reader.Close() + + if err != nil { + t.Error(err) + return + } + + _, _ = w.Write(handle(data)) + })) + defer server.Close() + + cfg := &config{ + NodeName: "node-a", StatusDetailMode: "summary", StatusPushEnabled: transport == "HTTP", + StatusWSEnabled: transport == "WS", StatusPushURL: server.URL, StatusWSURL: "ws" + strings.TrimPrefix(server.URL, "http"), + StatusPushInterval: 5 * time.Millisecond, StatusPushDelta: true, StatusWSAPIServerMode: statusWSAPIServerModeNever, + CriticalDeltaEvery: 5 * time.Millisecond, StatsDeltaEvery: 7 * time.Millisecond, FullSyncEvery: 9 * time.Millisecond, + } + h := blockedBootstrapHealthState() + ctx, cancel := context.WithCancel(t.Context()) + startStatusPublishers(ctx, cfg, h) + + defer func() { cancel(); h.stopStatusPublishers() }() + + want := 3 + if !supported { + want = 1 + } + + for range want { + select { + case msg := <-messages: + if msg.Type != statusv1alpha1.NodeStatusSummaryType || msg.Summary == nil || msg.Status != nil || msg.Delta != nil { + t.Fatalf("unexpected routine message: %v", msg) + } + + if len(msg.Summary.NodeErrors) == 0 || msg.Summary.NodeInfo.Name != "node-a" { + t.Fatal("lost bootstrap guard/identity") + } + case <-time.After(3 * time.Second): + t.Fatal("no summary received") + } + } + + if !supported { + waitForStatusCondition(t, func() bool { + for _, err := range h.getSummarySnapshot().NodeErrors { + if err.Type == nodeErrorSummaryUnsupported { + return true + } + } + + return false + }) + } + }) + } + } +} diff --git a/cmd/unbounded-net-node/status_server.go b/cmd/unbounded-net-node/status_server.go index 5472aa39c..f63513647 100644 --- a/cmd/unbounded-net-node/status_server.go +++ b/cmd/unbounded-net-node/status_server.go @@ -25,6 +25,7 @@ import ( "github.com/coder/websocket" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "google.golang.org/protobuf/proto" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -34,6 +35,7 @@ import ( "github.com/Azure/unbounded/internal/net/metrics" unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) const routingTableRefreshBackstop = 30 * time.Second @@ -59,6 +61,7 @@ type nodeHealthState struct { statusTransportWg *sync.WaitGroup statusTransportCancel context.CancelFunc statusTransportStop sync.Once + details *nodeDetailState mu sync.RWMutex } @@ -465,7 +468,7 @@ func (tm *hmacTokenManager) requestToken() error { return fmt.Errorf("all HMAC token endpoints failed: %s", strings.Join(endpointErrors, "; ")) } -func startHealthServer(port int, healthState *nodeHealthState) { +func newHealthMux(healthState *nodeHealthState) *http.ServeMux { mux := http.NewServeMux() metrics.Register(mux) @@ -528,6 +531,13 @@ func startHealthServer(port int, healthState *nodeHealthState) { } }) + // Same local routing and authentication policy as the full status endpoint. + mux.HandleFunc("/status/summary", healthState.handleStatusSummary) + + return mux +} + +func startHealthServer(port int, healthState *nodeHealthState) { addr := fmt.Sprintf(":%d", port) klog.Infof("Starting health server on %s", addr) @@ -535,7 +545,7 @@ func startHealthServer(port int, healthState *nodeHealthState) { server := &http.Server{ Addr: addr, - Handler: httpMiddleware.Wrap("all", mux), + Handler: httpMiddleware.Wrap("all", newHealthMux(healthState)), ReadHeaderTimeout: 10 * time.Second, } if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { @@ -543,14 +553,6 @@ func startHealthServer(port int, healthState *nodeHealthState) { } } -// nodeStatusPushAck is the JSON acknowledgment returned by the controller for push updates. -// Kept for backward-compatible JSON fallback parsing during protobuf rollout. -type nodeStatusPushAck struct { - Status string `json:"status"` - Revision uint64 `json:"revision,omitempty"` - Reason string `json:"reason,omitempty"` -} - const ( statusWSAPIServerModeNever = "never" statusWSAPIServerModeFallback = "fallback" @@ -1343,6 +1345,7 @@ func runStatusWebSocketPusher( var ( lastSentStatus *NodeStatusResponse + lastSentSummary *NodeStatusOverview lastCriticalSnapshot *NodeStatusResponse acks statusAckState lastAckTimeNs atomic.Int64 @@ -1365,11 +1368,56 @@ func runStatusWebSocketPusher( if acks.accept(data) { lastAckTimeNs.Store(time.Now().UnixNano()) + + if cfg.StatusDetailMode == "summary" && !acks.summary.Load() { + appendNodeError(healthState, nodeErrorSummaryUnsupported, "controller does not advertise summary support; full publication is disabled in summary mode") + return + } + + clearNodeErrorsByTypes(healthState, nodeErrorSummaryUnsupported) } } }() + sendSummary := func(onlyChanged bool) error { + summary := healthState.getSummarySnapshot() + + summary.NodeErrors = publicationNodeErrors(summary.NodeErrors) + if onlyChanged && equalPublicationSummaries(lastSentSummary, summary) { + return nil + } + + msg := &statusproto.NodeStatusMessage{ + Type: statusv1alpha1.NodeStatusSummaryType, NodeName: summary.NodeInfo.Name, + BaseRevision: acks.revision.Load(), Summary: nodeSummaryToProto(summary), + } + + payload, err := proto.Marshal(msg) + if err != nil { + return err + } + + acks.resync.Store(false) + acks.pending.Store(true) + + lastWriteTime = time.Now() + + if err := conn.Write(connCtx, websocket.MessageBinary, payload); err != nil { + return err + } + + lastSentSummary = summary + + clearNodeErrorsByTypes(healthState, nodeErrorTypeDirectPush, nodeErrorTypeDirectWebSocket, nodeErrorTypeFallbackPush, nodeErrorTypeFallbackWS) + + return nil + } + sendFull := func() error { + if cfg.StatusDetailMode == "summary" { + return sendSummary(false) + } + status := healthState.getStatusSnapshot() if len(status.NodeErrors) > 0 { // When the websocket is established, publish a clean snapshot so @@ -1519,6 +1567,14 @@ func runStatusWebSocketPusher( continue } + if cfg.StatusDetailMode == "summary" { + if err := sendSummary(!acks.resync.Load()); err != nil { + break loop + } + + continue + } + if acks.resync.Load() || lastSentStatus == nil { if err := sendFull(); err != nil { klog.V(2).Infof("Status websocket: resync full send failed: %v", err) @@ -1570,6 +1626,14 @@ func runStatusWebSocketPusher( continue } + if cfg.StatusDetailMode == "summary" { + if err := sendSummary(false); err != nil { + break loop + } + + continue + } + if acks.resync.Load() || lastSentStatus == nil { if err := sendFull(); err != nil { klog.V(2).Infof("Status websocket: stats resync failed: %v", err) @@ -1719,6 +1783,15 @@ func runStatusWebSocketPusher( _ = conn.Close(closeCode, closeReason) //nolint:errcheck connCancel() // tear down the detached connection context after graceful close + + if cfg.StatusDetailMode == "summary" && !acks.summary.Load() { + if wsURL == directWSURL { + nextDirectAttemptAt = time.Now().Add(5 * time.Second) + } else { + nextFallbackAttemptAt = time.Now().Add(5 * time.Second) + } + } + klog.V(4).Info("Status websocket disconnected") } } @@ -1954,8 +2027,6 @@ func startStatusPusher( // Collect status and prepare the request body synchronously. collectStart := time.Now() - nodeStatus := healthState.getStatusSnapshot() - collectDuration := time.Since(collectStart) pushStateMu.Lock() currentForceFull := forceFullPush @@ -1963,26 +2034,9 @@ func startStatusPusher( previousStatus := lastSentStatus pushStateMu.Unlock() - mode := "full" - - protoMsg := &statusproto.NodeStatusMessage{ - Type: "node_status_full", - NodeName: nodeStatus.NodeInfo.Name, - } - if cfg.StatusPushDelta && !currentForceFull { - delta := typedStatusDelta(previousStatus, nodeStatus, false, true) - if delta != nil { - mode = "delta" - protoMsg.Type = "node_status_delta" - protoMsg.BaseRevision = currentRevision - protoMsg.Status = nil - protoMsg.Delta = delta - } - } - - if protoMsg.Delta == nil { - protoMsg.Status = nodeStatusToProto(nodeStatus) - } + protoMsg, nodeStatus := collectPublication(healthState, cfg, previousStatus, currentForceFull, currentRevision) + collectDuration := time.Since(collectStart) + mode := protoMsg.Type marshalStart := time.Now() @@ -2141,19 +2195,13 @@ func startStatusPusher( defer func() { _ = resp.Body.Close() }() //nolint:errcheck - var ack statusproto.NodeStatusAck + ack := &statusv1alpha1.NodeStatusAck{} respBody, readErr := io.ReadAll(resp.Body) if readErr != nil { klog.V(4).Infof("Status push: failed to read %s response body: %v", targetLabel, readErr) - } else if protoErr := proto.Unmarshal(respBody, &ack); protoErr != nil { - // Fallback: try JSON for backward compatibility during rollout. - var jsonAck nodeStatusPushAck - if json.Unmarshal(respBody, &jsonAck) == nil { - ack.Revision = jsonAck.Revision - ack.Status = jsonAck.Status - ack.Reason = jsonAck.Reason - } + } else if decoded, err := decodeNodeStatusAck(respBody); err == nil { + ack = decoded } if resp.StatusCode == http.StatusTooManyRequests { @@ -2205,8 +2253,14 @@ func startStatusPusher( return false, false } + if cfg.StatusDetailMode == "summary" && !ack.SummarySupported { + appendNodeError(healthState, nodeErrorSummaryUnsupported, "controller does not advertise summary support; full publication is disabled in summary mode") + return false, true + } + + clearNodeErrorsByTypes(healthState, nodeErrorSummaryUnsupported) pushStateMu.Lock() - if ack.Revision > 0 { + if ack.IsPublicationAck() && ack.Revision > 0 { lastAckRevision = ack.Revision } @@ -2291,6 +2345,18 @@ type nodeStatusServer struct { // the host kernel's actual routing table. Production callers leave this // nil and the helpers fall back to the real netlink package. netlinkOps statusServerNetlinkOps + + // Optional collector override for tests; summaries never invoke this. + bpfCollector func() []BpfEntry + wireGuardDevice func(*unboundednetnetlink.WireGuardManager) (*wgtypes.Device, error) +} + +func (s *nodeStatusServer) getWireGuardDevice(manager *unboundednetnetlink.WireGuardManager) (*wgtypes.Device, error) { + if s.wireGuardDevice != nil { + return s.wireGuardDevice(manager) + } + + return manager.GetDevice() } // statusServerNetlinkOps abstracts the netlink reads that @@ -2389,8 +2455,16 @@ func (s *nodeStatusServer) startRouteChangeWatcher(ctx context.Context) { }() } -// getNodeStatus collects all status information about this node -func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { +type nodeStatusFacts struct { + Timestamp time.Time + NodeInfo NodeInfo + NodeErrors []NodeError + HealthCheck *HealthCheckStatus +} + +// inspectNodePeers visits peers without retaining an outbound peer array. +// The visitor must not acquire state.mu: non-WireGuard peers are visited under it. +func (s *nodeStatusServer) inspectNodePeers(visit func(WireGuardPeerStatus)) *nodeStatusFacts { // Snapshot state under the lock - copy all fields we need, then release. // Expensive operations (WireGuard GetDevice, collectRoutingTable) happen outside the lock. lockStart := time.Now() @@ -2398,7 +2472,7 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { s.state.mu.Lock() lockWait := time.Since(lockStart) - status := &NodeStatusResponse{ + status := &nodeStatusFacts{ Timestamp: time.Now(), NodeInfo: NodeInfo{ Name: s.cfg.NodeName, @@ -2542,7 +2616,7 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { // Get WireGuard device info if available (netlink syscall) if wgManager != nil { - if device, err := wgManager.GetDevice(); err == nil { + if device, err := s.getWireGuardDevice(wgManager); err == nil { status.NodeInfo.WireGuard.ListenPort = device.ListenPort status.NodeInfo.WireGuard.PeerCount = len(device.Peers) @@ -2608,7 +2682,7 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } } - status.Peers = append(status.Peers, peer) + visit(peer) } } } @@ -2619,7 +2693,7 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { for _, gw := range gwSnapshots { // Get WireGuard peer info for this gateway interface (netlink syscall) if gw.wgManager != nil { - if device, err := gw.wgManager.GetDevice(); err == nil && len(device.Peers) > 0 { + if device, err := s.getWireGuardDevice(gw.wgManager); err == nil && len(device.Peers) > 0 { wgPeer := device.Peers[0] // Each gateway interface has one peer peer := WireGuardPeerStatus{ @@ -2663,7 +2737,8 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } } - status.Peers = append(status.Peers, peer) + visit(peer) + addedPeerNames[gw.gatewayName] = true } } @@ -2727,7 +2802,7 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } } - status.Peers = append(status.Peers, peer) + visit(peer) } for _, gp := range s.state.gatewayPeers { @@ -2782,10 +2857,35 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } } - status.Peers = append(status.Peers, peer) + visit(peer) } s.state.mu.Unlock() + expensiveDuration := time.Since(expensiveStart) + + totalDuration := time.Since(lockStart) + if totalDuration > 2*time.Second { + klog.Warningf("inspectNodePeers() slow: total=%v (lock_wait=%v, snapshot=%v, expensive=%v)", + totalDuration, lockWait, snapshotDuration-lockWait, expensiveDuration) + } else { + klog.V(4).Infof("inspectNodePeers() timing: total=%v (lock_wait=%v, snapshot=%v, expensive=%v)", + totalDuration, lockWait, snapshotDuration-lockWait, expensiveDuration) + } + + return status +} + +// getNodeStatus collects all status information about this node. +func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { + status := &NodeStatusResponse{} + facts := s.inspectNodePeers(func(peer WireGuardPeerStatus) { + status.Peers = append(status.Peers, peer) + }) + status.Timestamp = facts.Timestamp + status.NodeInfo = facts.NodeInfo + status.NodeErrors = facts.NodeErrors + status.HealthCheck = facts.HealthCheck + sortStatusPeers(status.Peers) // Collect routing table from kernel via netlink @@ -2809,17 +2909,10 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } // Collect BPF trie entries. - status.BpfEntries = s.collectBpfEntries() - - expensiveDuration := time.Since(expensiveStart) - - totalDuration := time.Since(lockStart) - if totalDuration > 2*time.Second { - klog.Warningf("getNodeStatus() slow: total=%v (lock_wait=%v, snapshot=%v, expensive=%v)", - totalDuration, lockWait, snapshotDuration-lockWait, expensiveDuration) + if s.bpfCollector != nil { + status.BpfEntries = s.bpfCollector() } else { - klog.V(4).Infof("getNodeStatus() timing: total=%v (lock_wait=%v, snapshot=%v, expensive=%v)", - totalDuration, lockWait, snapshotDuration-lockWait, expensiveDuration) + status.BpfEntries = s.collectBpfEntries() } return status @@ -2843,26 +2936,32 @@ func linkStatsWarningsAsNodeErrors(warnings []string, peers []WireGuardPeerStatu } func suppressHealthyWireGuardRxErrors(warning string, peers []WireGuardPeerStatus, now time.Time) bool { - iface, deltas, ok := parseLinkStatsWarning(warning) - if !ok || len(deltas) != 1 || !strings.HasPrefix(deltas[0], "rx_errors +") { - return false - } + return suppressHealthyInterfaceRxErrors(warning, func(iface string) bool { + matched := false - matched := false + for _, peer := range peers { + if peer.Tunnel.Interface != iface { + continue + } - for _, peer := range peers { - if peer.Tunnel.Interface != iface { - continue + matched = true + + if !peerStatusHealthy(peer, now) { + return false + } } - matched = true + return matched + }) +} - if !peerStatusHealthy(peer, now) { - return false - } +func suppressHealthyInterfaceRxErrors(warning string, healthy func(string) bool) bool { + iface, deltas, ok := parseLinkStatsWarning(warning) + if !ok || len(deltas) != 1 || !strings.HasPrefix(deltas[0], "rx_errors +") { + return false } - return matched + return healthy(iface) } func parseLinkStatsWarning(warning string) (string, []string, bool) { @@ -2957,6 +3056,37 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { s.routingTableCacheMu.RUnlock() + s.inspectKernelRoutes(func(family, destination string, table int, hops []observedNextHop) { + entry := RouteEntry{Destination: destination, Family: family, Table: table} + for _, hop := range hops { + entry.NextHops = append(entry.NextHops, NextHop{ + Gateway: hop.gateway, Device: hop.device, Distance: hop.distance, MTU: hop.mtu, + RouteTypes: []RouteType{{Type: "kernel", Attributes: []string{"fib"}}}, + }) + } + + info.Routes = append(info.Routes, entry) + }) + + s.routingTableCacheMu.Lock() + s.routingTableCache = info + s.routingTableCachedAt = time.Now() + s.routingTableCacheMu.Unlock() + s.routingTableDirty.Store(false) + + return info +} + +type observedNextHop struct { + gateway string + device string + distance int + mtu int +} + +// inspectKernelRoutes shares filtering and deduplication without constructing +// status routes, annotations, or a full-detail routing cache. +func (s *nodeStatusServer) inspectKernelRoutes(visit func(family, destination string, table int, hops []observedNextHop)) { // Build a set of managed route prefixes from the route manager so we can // include routes on non-tunnel interfaces (e.g. eth0 with tunnelProtocol: None). managedPrefixes := make(map[string]bool) @@ -2967,7 +3097,7 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { } } - collect := func(family int, familyLabel string) []RouteEntry { + collect := func(family int, familyLabel string) { // Collect routes from the main table and, if configured, our dedicated table. // RouteList(nil, family) only returns routes from the main table, so we // explicitly request routes from our dedicated table via RouteListFiltered. @@ -3023,7 +3153,7 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { type destEntry struct { destination string table int - nexthops map[nhKey]NextHop + nexthops map[nhKey]observedNextHop nhOrder []nhKey } @@ -3095,7 +3225,7 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { de, exists := destMap[mapKey] if !exists { - de = &destEntry{destination: prefix, table: table, nexthops: make(map[nhKey]NextHop)} + de = &destEntry{destination: prefix, table: table, nexthops: make(map[nhKey]observedNextHop)} destMap[mapKey] = de destOrder = append(destOrder, mapKey) } @@ -3103,12 +3233,11 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { for _, wh := range wgHops { nk := nhKey{gateway: wh.gwStr, device: wh.devName} if _, nhExists := de.nexthops[nk]; !nhExists { - nh := NextHop{ - Gateway: wh.gwStr, Device: wh.devName, Distance: r.Priority, - RouteTypes: []RouteType{{Type: "kernel", Attributes: []string{"fib"}}}, + nh := observedNextHop{ + gateway: wh.gwStr, device: wh.devName, distance: r.Priority, } if r.MTU > 0 { - nh.MTU = r.MTU + nh.mtu = r.MTU } de.nexthops[nk] = nh @@ -3152,7 +3281,7 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { de, exists := destMap[mapKey] if !exists { - de = &destEntry{destination: prefix, table: table, nexthops: make(map[nhKey]NextHop)} + de = &destEntry{destination: prefix, table: table, nexthops: make(map[nhKey]observedNextHop)} destMap[mapKey] = de destOrder = append(destOrder, mapKey) } @@ -3164,17 +3293,11 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { nk := nhKey{gateway: gwStr, device: devName} if _, nhExists := de.nexthops[nk]; !nhExists { - nh := NextHop{ - Gateway: gwStr, - Device: devName, - Distance: r.Priority, - RouteTypes: []RouteType{{ - Type: "kernel", - Attributes: []string{"fib"}, - }}, + nh := observedNextHop{ + gateway: gwStr, device: devName, distance: r.Priority, } if r.MTU > 0 { - nh.MTU = r.MTU + nh.mtu = r.MTU } de.nexthops[nk] = nh @@ -3182,46 +3305,20 @@ func (s *nodeStatusServer) collectRoutingTableFromKernel() RoutingTableInfo { } } - result := make([]RouteEntry, 0, len(destOrder)) for _, mapKey := range destOrder { de := destMap[mapKey] - nhs := make([]NextHop, 0, len(de.nhOrder)) + nhs := make([]observedNextHop, 0, len(de.nhOrder)) for _, nk := range de.nhOrder { nhs = append(nhs, de.nexthops[nk]) } - result = append(result, RouteEntry{ - Destination: de.destination, - Family: familyLabel, - Table: de.table, - NextHops: nhs, - }) + visit(familyLabel, de.destination, de.table, nhs) } - - return result - } - - v4Routes := collect(netlink.FAMILY_V4, "IPv4") - v6Routes := collect(netlink.FAMILY_V6, "IPv6") - - if v4Routes == nil { - v4Routes = []RouteEntry{} - } - - if v6Routes == nil { - v6Routes = []RouteEntry{} } - info.Routes = append(v4Routes, v6Routes...) - - s.routingTableCacheMu.Lock() - s.routingTableCache = info - s.routingTableCachedAt = time.Now() - s.routingTableCacheMu.Unlock() - s.routingTableDirty.Store(false) - - return info + collect(netlink.FAMILY_V4, "IPv4") + collect(netlink.FAMILY_V6, "IPv6") } // isManagedTunnelInterface returns true for the interfaces created by the diff --git a/cmd/unbounded-net-node/status_summary.go b/cmd/unbounded-net-node/status_summary.go new file mode 100644 index 000000000..7f999b7e9 --- /dev/null +++ b/cmd/unbounded-net-node/status_summary.go @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net/http" + "time" + + "k8s.io/klog/v2" + + "github.com/Azure/unbounded/internal/net/routeplan" + netstatus "github.com/Azure/unbounded/internal/net/status" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +type NodeStatusOverview = statusv1alpha1.NodeStatusOverview + +// getNodeSummary collects overview facts directly. Only route-planning inputs +// survive peer visitation; full peer, route, and BPF snapshots are never built. +func (s *nodeStatusServer) getNodeSummary() *NodeStatusOverview { + summary := &NodeStatusOverview{} + + var routePeers []routeplan.Peer + + interfaceHealthy := make(map[string]bool) + now := time.Now() + facts := s.inspectNodePeers(func(peer WireGuardPeerStatus) { + summary.PeerCount++ + if netstatus.PeerHealthyForOverview(&peer, now) { + summary.HealthyPeers++ + } + + if !netstatus.PeerLinkHealthyForDiagnostics(&peer, now) { + summary.UnhealthyPeerLinks++ + } + + if peer.Tunnel.Protocol == "IPIP" { + summary.UsesIPIP = true + } + + previous, seen := interfaceHealthy[peer.Tunnel.Interface] + interfaceHealthy[peer.Tunnel.Interface] = (!seen || previous) && peerStatusHealthy(peer, now) + routePeers = append(routePeers, routeplan.Peer{ + Name: peer.Name, PeerType: peer.PeerType, SiteName: peer.SiteName, + SkipPodCIDRRoutes: peer.SkipPodCIDRRoutes, + Interface: peer.Tunnel.Interface, Endpoint: peer.Tunnel.Endpoint, + PodCIDRGateways: peer.PodCIDRGateways, AllowedIPs: peer.Tunnel.AllowedIPs, + RouteDistances: peer.RouteDistances, + }) + }) + summary.Timestamp = facts.Timestamp + summary.NodeInfo = facts.NodeInfo + summary.NodeErrors = facts.NodeErrors + summary.HealthCheck = facts.HealthCheck + summary.RouteCount, summary.RouteMismatchCount = s.collectRouteSummary(routePeers, facts.NodeInfo.SiteName) + summary.RouteMismatch = summary.RouteMismatchCount > 0 + + if s.state.linkStatsMonitor != nil { + for _, warning := range s.state.linkStatsMonitor.GetWarnings() { + if !suppressHealthyInterfaceRxErrors(warning, func(iface string) bool { return interfaceHealthy[iface] }) { + summary.NodeErrors = append(summary.NodeErrors, NodeError{Type: "link-stats", Message: warning}) + } + } + } + + return summary +} + +func (h *nodeHealthState) getSummarySnapshot() *NodeStatusOverview { + h.mu.RLock() + srv := h.statusServer + + summary := &NodeStatusOverview{ + Timestamp: time.Now(), + NodeInfo: NodeInfo{ + Name: h.nodeName, SiteName: h.siteName, IsGateway: h.isGateway, + PodCIDRs: append([]string(nil), h.podCIDRs...), BuildInfo: nodeAgentBuildInfo(), + }, + } + if h.pubKey != "" { + summary.NodeInfo.WireGuard = &WireGuardStatusInfo{PublicKey: h.pubKey} + } + + cniManaged, cniReady, cniReason := h.cniManaged, h.cniReady, h.cniReason + transientErrors := append([]NodeError(nil), h.transientErrors...) + h.mu.RUnlock() + + if srv != nil { + summary = srv.getNodeSummary() + } + + summary.NodeErrors = mergeNodeErrors(summary.NodeErrors, filterExpiredNodeErrors(transientErrors, time.Now(), time.Minute)) + + summary.NodeErrors = removeNodeErrorsByType(summary.NodeErrors, configPodCIDRGuard) + if cniManaged && !cniReady && cniReason != "" { + summary.NodeErrors = append(summary.NodeErrors, NodeError{Type: configPodCIDRGuard, Message: cniReason}) + } + + return summary +} + +func (h *nodeHealthState) handleStatusSummary(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if err := json.NewEncoder(w).Encode(h.getSummarySnapshot()); err != nil { + klog.V(4).Infof("status summary json encode failed: %v", err) + } +} diff --git a/cmd/unbounded-net-node/status_summary_routes.go b/cmd/unbounded-net-node/status_summary_routes.go new file mode 100644 index 000000000..dcf22d69b --- /dev/null +++ b/cmd/unbounded-net-node/status_summary_routes.go @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + + "k8s.io/klog/v2" + + "github.com/Azure/unbounded/internal/net/routeplan" +) + +type routeSummaryFamily struct { + expected map[routeKey]expectedRoute + lowest map[string]int + destinations map[string]bool + hasUnbounded bool +} + +func newRouteSummaryFamily(plan []routeplan.ExpectedRoute) *routeSummaryFamily { + family := &routeSummaryFamily{ + expected: make(map[routeKey]expectedRoute), + lowest: make(map[string]int), + destinations: make(map[string]bool), + } + + for _, route := range plan { + distance := effectiveRouteDistance(route.Distance) + key := routeKey{destination: route.Destination, gateway: route.Gateway, device: route.Device, distance: distance, weight: route.Weight} + + family.expected[key] = expectedRoute{ + destination: route.Destination, + nextHop: NextHop{Gateway: route.Gateway, Device: route.Device, Distance: distance, Weight: route.Weight}, + } + if previous, ok := family.lowest[route.Destination]; !ok || distance < previous { + family.lowest[route.Destination] = distance + } + } + + return family +} + +// collectRouteSummary preserves annotation counts, including synthetic missing +// routes and the per-family unbounded0 suppression of missing tunnel hops. +// It never creates status route arrays or fills the full-detail route cache. +func (s *nodeStatusServer) collectRouteSummary(peers []routeplan.Peer, localSite string) (count, mismatchCount int) { + defer func() { + if r := recover(); r != nil { + klog.Warningf("route summary recovered from panic: %v", r) + } + }() + + actx := buildAnnotationContext(s.siteInformer, s.sliceInformer, s.gatewayPoolInformer, s.sitePeeringInformer) + for i := range peers { + peers[i].SitePeered = sitesAreDirectlyPeered(strings.TrimSpace(localSite), peers[i].SiteName, actx.directSitePeerings) + } + + ipv4, ipv6 := routeplan.BuildExpectedWireGuardRoutes(peers, actx.routeNodes, routeplan.InterfaceNames{ + WireGuardPrefix: s.cfg.WireGuardInterfacePrefix, + Geneve: s.cfg.GeneveInterfaceName, + VXLAN: s.cfg.VXLANInterfaceName, + IPIP: s.cfg.IPIPInterfaceName, + }) + families := map[string]*routeSummaryFamily{ + "IPv4": newRouteSummaryFamily(ipv4), + "IPv6": newRouteSummaryFamily(ipv6), + } + + s.inspectKernelRoutes(func(familyName, destination string, _ int, hops []observedNextHop) { + family := families[familyName] + count++ + family.destinations[destination] = true + normalized, _ := normalizeRouteDestination(destination) + + for _, observed := range hops { + if observed.device == unbounded0DeviceName { + family.hasUnbounded = true + } + + if !isPeerRoutingInterface(s.cfg, observed.device) { + continue + } + + hop := NextHop{Gateway: observed.gateway, Device: observed.device, Distance: observed.distance} + + key, matched := findExpectedWireGuardMatch(family.expected, normalized, &hop) + if matched { + delete(family.expected, key) + } else { + // Kernel inspection only emits "kernel" route types, so the + // connected/local host-route exception cannot apply here. + mismatchCount++ + } + } + }) + + // The legacy collector does not annotate an entirely empty kernel result. + if count == 0 { + return count, mismatchCount + } + + for _, family := range families { + for _, expected := range family.expected { + if effectiveRouteDistance(expected.nextHop.Distance) > family.lowest[expected.destination] { + continue + } + + if family.hasUnbounded { + continue + } + + mismatchCount++ + + if !family.destinations[expected.destination] { + family.destinations[expected.destination] = true + count++ + } + } + } + + return count, mismatchCount +} diff --git a/cmd/unbounded-net-node/status_summary_routes_test.go b/cmd/unbounded-net-node/status_summary_routes_test.go new file mode 100644 index 000000000..b0ec7b7bf --- /dev/null +++ b/cmd/unbounded-net-node/status_summary_routes_test.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net" + "testing" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "github.com/Azure/unbounded/internal/net/routeplan" + netstatus "github.com/Azure/unbounded/internal/net/status" +) + +func summaryRoute(destination string, index, table, distance int) netlink.Route { + _, prefix, _ := net.ParseCIDR(destination) + + return netlink.Route{Dst: prefix, LinkIndex: index, Table: table, Priority: distance, Protocol: unix.RTPROT_BOOT} +} + +func summaryRouteFixture() *nodeStatusServer { + return &nodeStatusServer{ + cfg: &config{ + NodeName: "local", WireGuardInterfacePrefix: "wg", WireGuardPort: 51820, + GeneveInterfaceName: "gn0", VXLANInterfaceName: "vx0", IPIPInterfaceName: "ip0", + }, + state: &wireGuardState{routeTableID: 100}, + netlinkOps: &fakeNetlinkOps{ + links: map[int]netlink.Link{ + 1: &fakeLink{attrs: netlink.LinkAttrs{Index: 1, Name: "wg51820"}}, + 2: &fakeLink{attrs: netlink.LinkAttrs{Index: 2, Name: unbounded0DeviceName}}, + 3: &fakeLink{attrs: netlink.LinkAttrs{Index: 3, Name: "eth0"}}, + 4: &fakeLink{attrs: netlink.LinkAttrs{Index: 4, Name: "gn0"}}, + }, + }, + } +} + +func TestRouteSummaryParity(t *testing.T) { + peer := WireGuardPeerStatus{ + Name: "peer", PeerType: "site", SiteName: "local", + PodCIDRGateways: []string{"10.42.1.1", "fd00:1::1"}, + Tunnel: PeerTunnelStatus{Interface: "wg51820", AllowedIPs: []string{"10.42.1.0/24", "fd00:1::/64"}}, + } + planPeer := routeplan.Peer{ + Name: peer.Name, PeerType: peer.PeerType, SiteName: peer.SiteName, + Interface: peer.Tunnel.Interface, AllowedIPs: peer.Tunnel.AllowedIPs, PodCIDRGateways: peer.PodCIDRGateways, + } + + for _, tc := range []struct { + name string + v4 []netlink.Route + v6 []netlink.Route + table []netlink.Route + withoutPeer bool + exactMismatchCount int + }{ + {name: "empty kernel does not synthesize"}, + {name: "missing expected", exactMismatchCount: 5, v4: []netlink.Route{summaryRoute("10.9.0.0/24", 1, 0, 0)}}, + {name: "matched", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 0)}}, + {name: "unexpected without peers", withoutPeer: true, v4: []netlink.Route{summaryRoute("10.9.0.0/24", 1, 0, 0)}}, + {name: "unbounded suppresses only its family", exactMismatchCount: 2, v4: []netlink.Route{summaryRoute("10.42.0.0/16", 2, 0, 0)}}, + {name: "unbounded both families", v4: []netlink.Route{summaryRoute("10.42.0.0/16", 2, 0, 0)}, v6: []netlink.Route{summaryRoute("fd00::/48", 2, 0, 0)}}, + {name: "duplicate prefix distinct tables", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 0)}, table: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 100, 0)}}, + {name: "duplicate next hop", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 0), summaryRoute("10.42.1.0/24", 1, 0, 100)}}, + {name: "wrong distance", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 1, 0, 500)}}, + {name: "unmanaged ignored", v4: []netlink.Route{summaryRoute("10.42.1.0/24", 3, 0, 0)}}, + {name: "multipath", v4: []netlink.Route{{ + Dst: summaryRoute("10.42.1.0/24", 1, 0, 0).Dst, + MultiPath: []*netlink.NexthopInfo{{LinkIndex: 1}, {LinkIndex: 3}, {LinkIndex: 4}}, + }}}, + {name: "two unexpected tunnel hops", withoutPeer: true, exactMismatchCount: 2, v4: []netlink.Route{{ + Dst: summaryRoute("10.99.0.0/24", 1, 0, 0).Dst, + MultiPath: []*netlink.NexthopInfo{{LinkIndex: 1}, {LinkIndex: 4}}, + }}}, + } { + t.Run(tc.name, func(t *testing.T) { + s := summaryRouteFixture() + ops := s.netlinkOps.(*fakeNetlinkOps) + ops.mainRoutes = map[int][]netlink.Route{netlink.FAMILY_V4: tc.v4, netlink.FAMILY_V6: tc.v6} + ops.tableRoutes = map[int]map[int][]netlink.Route{netlink.FAMILY_V4: {100: tc.table}} + full := &NodeStatusResponse{NodeInfo: NodeInfo{SiteName: "local"}, RoutingTable: s.collectRoutingTableFromKernel()} + + var peers []routeplan.Peer + + if !tc.withoutPeer { + full.Peers = []WireGuardPeerStatus{peer} + peers = []routeplan.Peer{planPeer} + } + + annotateNodeRoutes(full, s.cfg, nil, nil, nil, nil) + + wantMismatch := false + + for _, route := range full.RoutingTable.Routes { + if netstatus.RouteMismatchForOverview(route) { + wantMismatch = true + } + } + + count, mismatchCount := s.collectRouteSummary(peers, "local") + + mismatch := mismatchCount > 0 + if count != len(full.RoutingTable.Routes) || mismatch != wantMismatch { + t.Fatalf("summary=(%d,%v), legacy=(%d,%v): %+v", count, mismatch, len(full.RoutingTable.Routes), wantMismatch, full.RoutingTable.Routes) + } + + wantMismatchCount := netstatus.RouteMismatchCount(full.RoutingTable.Routes) + if mismatchCount != wantMismatchCount || (tc.exactMismatchCount > 0 && mismatchCount != tc.exactMismatchCount) { + t.Fatalf("mismatched hops=%d, legacy=%d, explicit=%d: %+v", mismatchCount, wantMismatchCount, tc.exactMismatchCount, full.RoutingTable.Routes) + } + }) + } +} diff --git a/cmd/unbounded-net-node/status_summary_test.go b/cmd/unbounded-net-node/status_summary_test.go new file mode 100644 index 000000000..1c0562aca --- /dev/null +++ b/cmd/unbounded-net-node/status_summary_test.go @@ -0,0 +1,319 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "reflect" + "sync" + "testing" + "time" + + "github.com/vishvananda/netlink" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "google.golang.org/protobuf/proto" + + "github.com/Azure/unbounded/internal/net/healthcheck" + unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func TestSummaryPeerHealthy(t *testing.T) { + now := time.Now() + for _, tc := range []struct { + name string + health *HealthCheckPeerStatus + handshake time.Time + want bool + }{ + {"up", &HealthCheckPeerStatus{Enabled: true, Status: "up"}, time.Time{}, true}, + {"Up", &HealthCheckPeerStatus{Enabled: true, Status: "Up"}, time.Time{}, true}, + {"uppercase is not counted", &HealthCheckPeerStatus{Enabled: true, Status: "UP"}, now, false}, + {"enabled unknown", &HealthCheckPeerStatus{Enabled: true}, now, false}, + {"enabled down", &HealthCheckPeerStatus{Enabled: true, Status: "down"}, now, false}, + {"disabled uses handshake", &HealthCheckPeerStatus{Status: "down"}, now, true}, + {"missing health uses handshake", nil, now.Add(-time.Minute), true}, + {"missing handshake", nil, time.Time{}, false}, + {"boundary", nil, now.Add(-3 * time.Minute), false}, + {"future handshake", nil, now.Add(time.Minute), true}, + } { + t.Run(tc.name, func(t *testing.T) { + peer := WireGuardPeerStatus{HealthCheck: tc.health, Tunnel: PeerTunnelStatus{LastHandshake: tc.handshake}} + if got := netstatus.PeerHealthyForOverview(&peer, now); got != tc.want { + t.Fatalf("healthy=%v, want %v", got, tc.want) + } + }) + } +} + +func TestNodeSummaryParityAndNoBPF(t *testing.T) { + for _, deviceFailure := range []bool{false, true} { + t.Run(map[bool]string{false: "device success", true: "device failure"}[deviceFailure], func(t *testing.T) { + s := summaryRouteFixture() + s.state.siteName = "local" + s.state.nodePodCIDRs = []string{"10.42.0.0/24"} + s.state.nodeInternalIPs = []string{"192.0.2.1"} + s.state.nodeExternalIPs = []string{"198.51.100.1"} + s.state.nodeErrors = []NodeError{{Type: "test", Message: "failure"}} + s.state.wireguardManager = &unboundednetnetlink.WireGuardManager{} + + manager, err := healthcheck.NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + + if err := manager.AddPeer("down", net.ParseIP("10.42.1.1"), healthcheck.DefaultSettings()); err != nil { + t.Fatal(err) + } + + s.state.healthCheckManager = manager + s.state.meshPeerHealthCheckEnabled = map[string]bool{"down": true, "missing": true} + s.state.peers = []meshPeerInfo{ + {Name: "down", WireGuardPublicKey: "down", TunnelProtocol: "GENEVE", InternalIPs: []string{"192.0.2.2"}, PodCIDRs: []string{"10.42.1.0/24"}}, + {Name: "missing", WireGuardPublicKey: "missing", TunnelProtocol: "VXLAN", InternalIPs: []string{"192.0.2.3"}, PodCIDRs: []string{"10.42.2.0/24"}}, + {Name: "no-address", TunnelProtocol: "IPIP"}, + } + s.state.gatewayPeers = []gatewayPeerInfo{ + {Name: "gateway", TunnelProtocol: "IPIP", InternalIPs: []string{"192.0.2.4"}, PodCIDRs: []string{"10.42.3.0/24"}}, + } + s.state.gatewayHealthEndpoints = map[string]string{"wg51821": "10.42.3.1"} + s.state.gatewayNames = map[string]string{"wg51821": "gateway"} + s.state.gatewayWireguardManagers = map[string]*unboundednetnetlink.WireGuardManager{"wg51821": {}} + s.state.linkStatsMonitor = &linkStatsMonitor{warnings: []string{ + "interface /wg51820: rx_errors +24", "interface /gn0: rx_errors +24", "interface /eth0: tx_errors +2", + }} + s.wireGuardDevice = func(*unboundednetnetlink.WireGuardManager) (*wgtypes.Device, error) { + if deviceFailure { + return nil, errors.New("device unavailable") + } + + return &wgtypes.Device{ListenPort: 51820, Peers: []wgtypes.Peer{ + {LastHandshakeTime: time.Now().Add(-time.Minute)}, + }}, nil + } + s.netlinkOps.(*fakeNetlinkOps).mainRoutes = map[int][]netlink.Route{ + netlink.FAMILY_V4: {summaryRoute("10.42.0.0/16", 2, 0, 0)}, + } + bpfCalls := 0 + s.bpfCollector = func() []BpfEntry { bpfCalls++; return nil } + + summary := s.getNodeSummary() + if bpfCalls != 0 || !s.routingTableCachedAt.IsZero() || len(s.routingTableCache.Routes) != 0 { + t.Fatal("summary collected BPF or populated the full route cache") + } + + full := s.getNodeStatus() + + if bpfCalls != 1 { + t.Fatal("legacy full collection no longer collects BPF") + } + + if !reflect.DeepEqual(summary.NodeInfo, full.NodeInfo) || !reflect.DeepEqual(summary.NodeErrors, full.NodeErrors) { + t.Fatalf("metadata/errors differ: summary=%+v full=%+v", summary, full) + } + + legacy := netstatus.OverviewFromStatus(full, time.Now()) + if summary.PeerCount != legacy.PeerCount || summary.HealthyPeers != legacy.HealthyPeers || + summary.RouteCount != legacy.RouteCount || summary.RouteMismatch != legacy.RouteMismatch || + summary.RouteMismatchCount != legacy.RouteMismatchCount || summary.UnhealthyPeerLinks != legacy.UnhealthyPeerLinks || + summary.UsesIPIP != legacy.UsesIPIP { + t.Fatalf("counts differ: summary=%+v full peers=%+v routes=%+v", summary, full.Peers, full.RoutingTable) + } + + wantPeers, wantHealthyPeers := 4, 2 + if deviceFailure { + wantPeers, wantHealthyPeers = 3, 0 + } + + if summary.PeerCount != wantPeers || summary.HealthyPeers != wantHealthyPeers || summary.RouteMismatch { + t.Fatalf("unexpected observed counts/mismatch: %+v", summary) + } + + if summary.HealthCheck == nil || summary.HealthCheck.Healthy != full.HealthCheck.Healthy || + summary.HealthCheck.PeerCount != full.HealthCheck.PeerCount || summary.HealthCheck.Summary != full.HealthCheck.Summary { + t.Fatalf("health aggregates differ: %v vs %v", summary.HealthCheck, full.HealthCheck) + } + + assertSummaryHasNoDetails(t, summary) + }) + } +} + +func assertSummaryHasNoDetails(t *testing.T, summary *NodeStatusOverview) { + t.Helper() + + data, err := json.Marshal(summary) + if err != nil { + t.Fatal(err) + } + + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + t.Fatal(err) + } + + for _, name := range []string{"peers", "routingTable", "bpfEntries", "peerMeasurements"} { + if _, exists := fields[name]; exists { + t.Fatalf("summary contains detail field %q", name) + } + } +} + +func TestSummaryBootstrapRecoveryAndLocalRouting(t *testing.T) { + h := blockedBootstrapHealthState() + h.transientErrors = []NodeError{{Type: "transport", Message: "failed"}, {Type: "expired", Message: "old", Timestamp: time.Now().Add(-2 * time.Minute)}} + mux := newHealthMux(h) + + for _, path := range []string{"/status/summary", "/status", "/status/json"} { + recorder := httptest.NewRecorder() + mux.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path, nil)) + + if recorder.Code != http.StatusOK || recorder.Header().Get("Content-Type") != "application/json" { + t.Fatalf("%s: code=%d headers=%v", path, recorder.Code, recorder.Header()) + } + + var summary NodeStatusOverview + if err := json.Unmarshal(recorder.Body.Bytes(), &summary); err != nil { + t.Fatal(err) + } + + if summary.NodeInfo.Name != "node-a" || len(summary.NodeErrors) != 2 || summary.NodeErrors[1].Type != configPodCIDRGuard { + t.Fatalf("%s: lost bootstrap identity/errors: %+v", path, summary) + } + + var fields map[string]json.RawMessage + if err := json.Unmarshal(recorder.Body.Bytes(), &fields); err != nil { + t.Fatal(err) + } + + _, hasPeers := fields["peers"] + if hasPeers == (path == "/status/summary") { + t.Fatalf("%s: endpoint returned wrong representation", path) + } + } + + h.setCNIReady("cbr0", []string{"10.244.7.0/24"}) + + summary := h.getSummarySnapshot() + if len(summary.NodeErrors) != 1 || summary.NodeErrors[0].Type != "transport" { + t.Fatalf("guard did not recover: %+v", summary.NodeErrors) + } + + summary.NodeInfo.PodCIDRs[0] = "mutated" + if h.getSummarySnapshot().NodeInfo.PodCIDRs[0] == "mutated" { + t.Fatal("bootstrap CIDRs alias shared state") + } + + s := summaryRouteFixture() + s.state.nodeErrors = []NodeError{{Type: configPodCIDRGuard, Message: "obsolete guard"}} + h.setStatusServer(s) + + if got := h.getSummarySnapshot(); got.NodeInfo.Name != "local" || len(got.NodeErrors) != 1 || got.NodeErrors[0].Type != "transport" { + t.Fatalf("initialized summary lost recovery/transport state: %+v", got) + } +} + +func TestSummaryHealthyAggregate(t *testing.T) { + s := summaryRouteFixture() + + manager, err := healthcheck.NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + + s.state.healthCheckManager = manager + + summary := s.getNodeSummary() + if summary.HealthCheck == nil || !summary.HealthCheck.Healthy || summary.HealthCheck.PeerCount != 0 || + summary.HealthCheck.Summary != "all peers healthy" || summary.HealthCheck.CheckedAt.IsZero() { + t.Fatalf("lost healthy aggregate: %+v", summary.HealthCheck) + } +} + +func TestSummaryConcurrentBootstrapState(t *testing.T) { + h := blockedBootstrapHealthState() + + var wg sync.WaitGroup + for range 4 { + wg.Go(func() { + for range 20 { + h.beginManagedCNI("cbr0") + h.getSummarySnapshot() + h.setCNIReady("cbr0", []string{"10.244.7.0/24"}) + } + }) + } + + wg.Wait() +} + +func TestNodeSummaryToProto(t *testing.T) { + if nodeSummaryToProto(nil) != nil { + t.Fatal("nil summary converted") + } + + now := time.Now() + summary := &NodeStatusOverview{ + Timestamp: now, NodeInfo: NodeInfo{Name: "node", K8sReady: "Unknown"}, + PeerCount: 10, HealthyPeers: 4, RouteCount: 12, RouteMismatch: true, + RouteMismatchCount: 3, UnhealthyPeerLinks: 2, UsesIPIP: true, + FetchError: "unavailable", StatusSource: "error", LastPushTime: &now, + NodeErrors: []NodeError{{Type: "failure", Message: "failed"}}, + HealthCheck: &HealthCheckStatus{Healthy: false, Summary: "unhealthy"}, + } + encoded := nodeSummaryToProto(summary) + + data, err := proto.Marshal(encoded) + if err != nil { + t.Fatal(err) + } + + var decoded statusproto.NodeStatusOverview + if err := proto.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + + if !proto.Equal(encoded, &decoded) || decoded.PeerCount != 10 || decoded.HealthyPeers != 4 || decoded.RouteCount != 12 || + decoded.RouteMismatchCount != 3 || decoded.UnhealthyPeerLinks != 2 || !decoded.UsesIpip || + !decoded.RouteMismatch || decoded.FetchError != "unavailable" || decoded.NodeInfo.K8SReady != "Unknown" || + decoded.LastPushTimeUnixNs != now.UnixNano() || decoded.StatusSource != "error" || len(decoded.NodeErrors) != 1 { + t.Fatalf("summary conversion lost facts: %v", &decoded) + } +} + +func BenchmarkNodeSummaryCollection(b *testing.B) { + for _, full := range []bool{false, true} { + b.Run(map[bool]string{false: "summary", true: "full"}[full], func(b *testing.B) { + s := summaryRouteFixture() + s.bpfCollector = func() []BpfEntry { return nil } + + s.netlinkOps.(*fakeNetlinkOps).mainRoutes = map[int][]netlink.Route{ + netlink.FAMILY_V4: {summaryRoute("10.0.0.0/8", 2, 0, 0)}, + } + for i := range 2000 { + s.state.peers = append(s.state.peers, meshPeerInfo{ + Name: fmt.Sprintf("peer-%d", i), WireGuardPublicKey: fmt.Sprintf("key-%d", i), + TunnelProtocol: "GENEVE", InternalIPs: []string{"192.0.2.2"}, + PodCIDRs: []string{fmt.Sprintf("10.%d.%d.0/24", i/256, i%256)}, + }) + } + + b.ReportAllocs() + + for b.Loop() { + if full { + s.getNodeStatus() + } else { + s.getNodeSummary() + } + } + }) + } +} diff --git a/deploy/net/01-configmap.yaml.tmpl b/deploy/net/01-configmap.yaml.tmpl index d4dc83a36..e742250ad 100644 --- a/deploy/net/01-configmap.yaml.tmpl +++ b/deploy/net/01-configmap.yaml.tmpl @@ -20,6 +20,9 @@ data: nodeAgentHealthPort: {{ default "9998" .ControllerNodeAgentHealthPort }} informerResyncPeriod: "{{ default "300s" .ControllerInformerResyncPeriod }}" statusStaleThreshold: "{{ default "90s" .ControllerStatusStaleThreshold }}" + # Preparatory, startup-only detail lifetimes; cache/request wiring follows. + statusDetailCacheTTL: "{{ default "300s" .ControllerStatusDetailCacheTTL }}" + statusDetailRequestTimeout: "{{ default "120s" .ControllerStatusDetailRequestTimeout }}" statusWebsocketKeepaliveInterval: "{{ default "30s" .ControllerStatusWebsocketKeepaliveInterval }}" statusWsKeepaliveFailureCount: {{ default "3" .ControllerStatusWsKeepaliveFailureCount }} registerAggregatedAPIServer: {{ default "true" .ControllerRegisterAggregatedAPIServer }} @@ -67,6 +70,8 @@ data: statusPushEnabled: {{ default "true" .NodeStatusPushEnabled }} statusPushURL: "{{ default "" .NodeStatusPushURL }}" statusPushDelta: {{ default "true" .NodeStatusPushDelta }} + # Preparatory, startup-only; keep full until summary rollout is activated. + statusDetailMode: "{{ default "full" .NodeStatusDetailMode }}" statusPushInterval: "{{ default "60s" .NodeStatusPushInterval }}" statusPushApiserverInterval: "{{ default "60s" .NodeStatusPushApiserverInterval }}" healthCheckPort: "{{ default "9997" .NodeHealthCheckPort }}" diff --git a/docs/content/reference/networking/configuration.md b/docs/content/reference/networking/configuration.md index 95d70787a..290958c6e 100644 --- a/docs/content/reference/networking/configuration.md +++ b/docs/content/reference/networking/configuration.md @@ -17,6 +17,23 @@ file mounted from the `unbounded-net-config` ConfigMap. - Startup behavior: fail-fast if the config file is missing or invalid. - CLI flags still work as explicit overrides when set. +### Preparatory detail status settings + +The lightweight-status rollout adds startup-only settings. This preparatory +layer parses and validates them without changing publication behavior. `full` +remains the default until the later collector, cache, and consumer activation. +Changes require restarting the affected controller or node pod. + +| Runtime setting | CLI override | Current default | Allowed values | +|-----------------|--------------|-----------------|----------------| +| `node.statusDetailMode` | `--status-detail-mode` | `full` | `summary`, `full` | +| `controller.statusDetailCacheTTL` | `--status-detail-cache-ttl` | `300s` | Strictly positive duration | +| `controller.statusDetailRequestTimeout` | `--status-detail-request-timeout` | `120s` | Strictly positive duration | + +The intended cache lifetime is measured from actual detail receipt; summaries +and reads do not extend it. Request timeout spans all delivery attempts. +Upgrade controllers before enabling summary publication in the completed rollout. + ### Config Structure ```yaml diff --git a/docs/net/configuration.md b/docs/net/configuration.md index 28ad70f67..0574e27da 100644 --- a/docs/net/configuration.md +++ b/docs/net/configuration.md @@ -13,6 +13,23 @@ Both binaries now load runtime settings from a shared YAML file mounted from the - Startup behavior: fail-fast if the config file is missing or invalid - CLI flags still work as explicit overrides when set +### Preparatory detail status settings + +These startup-only settings prepare the lightweight-status rollout. They are +parsed and validated now; collection, caching, and request delivery are wired in +subsequent layers. Publication behavior remains unchanged, with `full` as the +default until final activation. Changing these settings requires a pod restart. + +| Runtime setting | CLI override | Current default | Allowed values | +|-----------------|--------------|-----------------|----------------| +| `node.statusDetailMode` | `--status-detail-mode` | `full` | `summary`, `full` | +| `controller.statusDetailCacheTTL` | `--status-detail-cache-ttl` | `300s` | Strictly positive duration | +| `controller.statusDetailRequestTimeout` | `--status-detail-request-timeout` | `120s` | Strictly positive duration | + +The intended cache lifetime starts when actual details arrive, not on summary +updates or reads. The request timeout covers all delivery attempts together. +Upgrade controllers before enabling summary publication in the completed rollout. + ### Runtime config structure ```yaml diff --git a/docs/net/operations.md b/docs/net/operations.md index 77d3a073c..735d7cfd1 100644 --- a/docs/net/operations.md +++ b/docs/net/operations.md @@ -212,7 +212,6 @@ kubectl unbounded-system controller proxy The dashboard displays: - **Overview**: Cluster health summary with node counts, site counts, and gateway status - **Sites**: All configured sites with node counts and health indicators -- **Connectivity Matrix**: Visual representation of node-to-node connectivity (pingmesh results) - **Nodes**: Detailed list of all nodes with filtering, sorting, and pagination - Tunnel peer status (WireGuard peers or eBPF tunnel endpoints) - Gateway health for each node @@ -225,12 +224,10 @@ The dashboard uses **WebSocket** for real-time updates with delta compression, f - Filtering nodes by name, site, or role (gateway/worker) - Sorting by any column - Auto-sizing pagination based on screen height -- Expandable connectivity matrix with zoom and labels - Dark/light theme toggle -Connectivity matrices are omitted for site or gateway-pool scopes containing -more than 100 nodes. Smaller scopes remain visible even when other scopes -exceed that limit. +The dashboard does not render a site connectivity graph or connectivity matrix. +Use the Site summaries and filtered node list to inspect individual resources. ### Health Endpoints diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 63bb576f4..fcf712c6c 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4,7 +4,6 @@ import * as React from 'react'; import { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import NodeTable from './components/nodes/NodesTable'; -import NetworkCard from './components/dashboard/NetworkCard'; import SitesCard from './components/network/SitesCard'; import StatusJsonModal from './components/status/StatusJsonModal'; import ErrorBoundary from './components/common/ErrorBoundary'; @@ -23,7 +22,6 @@ export default function App() { } = useClusterStatus(); const nodes = status?.nodes || []; const sites = summary?.sites || status?.sites || []; - const peerings = summary?.peerings || status?.peerings || []; const gatewayPools = summary?.gatewayPools || status?.gatewayPools || []; const nodeSummaries = summary?.nodeSummaries || []; const [hiddenSites, setHiddenSites] = useState>(new Set()); @@ -32,8 +30,7 @@ export default function App() { const [selectedNodeDetailTab, setSelectedNodeDetailTab] = useState<'peerings' | 'routes' | 'bpf'>('peerings'); const [pullEnabledOptimistic, setPullEnabledOptimistic] = useState(null); const [selectedNodeTypesFilter, setSelectedNodeTypesFilter] = useState>(new Set(['Gateway', 'Worker'])); - const [networkTab, setNetworkTab] = useState<'siteTopology' | 'matrix'>('siteTopology'); - const [maximizedPanel, setMaximizedPanel] = useState<'nodes' | 'siteTopology' | 'matrix' | null>(null); + const [maximizedPanel, setMaximizedPanel] = useState<'nodes' | null>(null); const [infoOpen, setInfoOpen] = useState(false); const [statusJsonOpen, setStatusJsonOpen] = useState(false); const [errorsDismissed, setErrorsDismissed] = useState(false); @@ -52,12 +49,6 @@ export default function App() { window.localStorage.setItem('theme', theme); }, [theme]); - useEffect(() => { - if (maximizedPanel === 'siteTopology' || maximizedPanel === 'matrix') { - setNetworkTab(maximizedPanel); - } - }, [maximizedPanel]); - useEffect(() => { if (!maximizedPanel) return; const onKeyDown = (event: KeyboardEvent) => { @@ -217,9 +208,7 @@ export default function App() { }, []); const { - activeNetworkTab, activeSelectedNode, - edgeHealthCheckCounts, effectivePullEnabled, gatewayByNode, nodeK8sStatusMap, @@ -227,7 +216,6 @@ export default function App() { nodeTotalCount, peerHealth, poolCounts, - poolToSite, siteCounts, visibleNodeSummaries } = useDashboardData({ @@ -239,8 +227,6 @@ export default function App() { gatewayPoolHiddenNames: hiddenGatewayPools, hiddenSites, selectedNodeTypesFilter, - networkTab, - maximizedPanel, pullEnabledOptimistic, selectedNodeName, nodeDetail @@ -272,18 +258,6 @@ export default function App() { ? 'Polling only' : 'No data'; - const onSelectNetworkTab = (tab: 'siteTopology' | 'matrix') => { - if (maximizedPanel === 'siteTopology' || maximizedPanel === 'matrix') { - setMaximizedPanel(tab); - return; - } - setNetworkTab(tab); - }; - - const onToggleNetworkMaximize = (isMaximized: boolean) => { - setMaximizedPanel(isMaximized ? null : activeNetworkTab); - }; - const renderNodesCard = (isMaximized: boolean) => { const content = (
setMaximizedPanel(null)}>
- {maximizedPanel === 'nodes' ? renderNodesCard(true) : ( - onToggleNetworkMaximize(true)} - /> - )} + {renderNodesCard(true)} - onToggleNetworkMaximize(false)} - />
{loading &&
Loading...
} diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 4036872df..e41322d9a 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 -import { ClusterStatus, ClusterStatusDelta, ClusterSummary, ClusterSummaryDelta, NodeStatus } from './types'; +import { ClusterStatus, ClusterStatusDelta, ClusterSummary, ClusterSummaryDelta, NodeStatus, NodeDetailResult } from './types'; export type StatusEvent = { type: 'cluster_status' | 'cluster_status_delta' | 'cluster_summary' | 'cluster_summary_delta' | 'node_detail_response' | 'node_detail_update'; @@ -15,6 +15,37 @@ function buildControllerUrl(path: string): string { return path; } +async function fetchNodeDetails(path: string, options: RequestInit): Promise { + const response = await fetch(buildControllerUrl(path), { credentials: 'same-origin', ...options }); + const text = await response.text(); + let result: NodeDetailResult; + try { + result = JSON.parse(text); + } catch { + throw new Error(`Detail request failed (${response.status} ${response.statusText})${text ? `: ${text}` : ''}`); + } + if (!response.ok) { + throw new Error(result?.error || `Detail request failed (${response.status} ${response.statusText})`); + } + if (!result || typeof result.state !== 'string') { + throw new Error('Invalid detail response from controller'); + } + return result; +} + +export function requestNodeDetails(name: string, forceRefresh: boolean, signal: AbortSignal) { + return fetchNodeDetails(`/status/node/${encodeURIComponent(name)}/details`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ forceRefresh }), signal, + }); +} + +export function pollNodeDetails(name: string, requestId: string, signal: AbortSignal) { + return fetchNodeDetails(`/status/node/${encodeURIComponent(name)}/details?requestId=${encodeURIComponent(requestId)}`, { + signal, cache: 'no-store', + }); +} + export async function fetchClusterStatus(): Promise { const url = buildControllerUrl('/status/json'); try { @@ -90,10 +121,6 @@ export function mergeDelta(current: ClusterStatus | null, delta: ClusterStatusDe merged.nodes = Object.values(nodeMap); } - if (delta.connectivityMatrix !== undefined && delta.connectivityMatrix !== null) { - merged.connectivityMatrix = delta.connectivityMatrix || undefined; - } - return merged; } diff --git a/frontend/src/components/common/topologyIcons.tsx b/frontend/src/components/common/topologyIcons.tsx deleted file mode 100644 index 0c81c0156..000000000 --- a/frontend/src/components/common/topologyIcons.tsx +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import * as React from 'react'; -import { useEffect, useMemo } from 'react'; -import * as THREE from 'three'; - -const gatewayPoolSvgIcon = ` - - - - - - - - - - - - - - -`; - -const workerSiteSvgIcon = ` - - - - - - - - - - - - - - - - - - - - - - - - -`; - -const maskFromSVG = (svg: string): string => { - return svg - .split('#00bbf1').join('#ffffff') - .split('#c5c5c5').join('#ffffff') - .split('#e6f8fe').join('#ffffff') - .split('#ccf1fc').join('#ffffff') - .split('#80ddf8').join('#ffffff') - .split('#919191').join('#ffffff'); -}; - -type TopologyNodeIconTextures = { - gatewayPoolIconTexture: THREE.Texture; - workerSiteIconTexture: THREE.Texture; - gatewayPoolMaskTexture: THREE.Texture; - workerSiteMaskTexture: THREE.Texture; -}; - -type RenderTopologyNodeGlyphArgs = { - glyph: 'router' | 'vm' | 'default'; - size: number; - color: string; - textures: TopologyNodeIconTextures; - workerOffsetScale?: number; -}; - -export function getTopologyNodeGlyph(group?: string): 'router' | 'vm' | 'default' { - if (group === 'pool' || group === 'gateway-node') { - return 'router'; - } - if (group === 'site' || group === 'worker-node') { - return 'vm'; - } - return 'default'; -} - -export function useTopologyNodeIconTextures(): TopologyNodeIconTextures { - const gatewayPoolIconTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(gatewayPoolSvgIcon)}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - const workerSiteIconTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(workerSiteSvgIcon)}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - const gatewayPoolMaskTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(maskFromSVG(gatewayPoolSvgIcon))}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - const workerSiteMaskTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(maskFromSVG(workerSiteSvgIcon))}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - useEffect(() => { - return () => { - gatewayPoolIconTexture.dispose(); - workerSiteIconTexture.dispose(); - gatewayPoolMaskTexture.dispose(); - workerSiteMaskTexture.dispose(); - }; - }, [gatewayPoolIconTexture, workerSiteIconTexture, gatewayPoolMaskTexture, workerSiteMaskTexture]); - - return { - gatewayPoolIconTexture, - workerSiteIconTexture, - gatewayPoolMaskTexture, - workerSiteMaskTexture - }; -} - -export function renderTopologyNodeGlyph({ - glyph, - size, - color, - textures, - workerOffsetScale = -0.16 -}: RenderTopologyNodeGlyphArgs): React.ReactNode { - if (glyph === 'router') { - return ( - - - - - - - - - - - ); - } - - if (glyph === 'vm') { - return ( - - - - - - - - - - - ); - } - - return null; -} diff --git a/frontend/src/components/dashboard/NetworkCard.tsx b/frontend/src/components/dashboard/NetworkCard.tsx deleted file mode 100644 index aa599c22e..000000000 --- a/frontend/src/components/dashboard/NetworkCard.tsx +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import React, { Suspense } from 'react'; -import { CloseXIcon, MagnifyPlusIcon } from '../nodes/shared/index'; -import ConnectivityHeatmap from '../network/ConnectivityHeatmap'; -const Topology = React.lazy(() => import('../network/Topology')); -import { GatewayPoolStatus, NodeStatus, PeeringStatus, SiteMatrix, SiteStatus } from '../../types'; - -type NetworkTab = 'siteTopology' | 'matrix'; - -type NetworkCardProps = { - activeNetworkTab: NetworkTab; - edgeHealthCheckCounts: Map; - gatewayByNode: Map; - gatewayPools: GatewayPoolStatus[]; - hiddenGatewayPools: Set; - hiddenSites: Set; - isMaximized: boolean; - nodeStatuses: NodeStatus[]; - peerings: PeeringStatus[]; - poolCounts: Map; - poolToSite: Map; - siteCounts: Map; - sites: SiteStatus[]; - statusMatrix?: Record; - theme: 'dark' | 'light'; - onSelectTab: (tab: NetworkTab) => void; - onToggleMaximize: () => void; -}; - -function NetworkCard({ - activeNetworkTab, - edgeHealthCheckCounts, - gatewayByNode, - gatewayPools, - hiddenGatewayPools, - hiddenSites, - isMaximized, - nodeStatuses, - peerings, - poolCounts, - poolToSite, - siteCounts, - sites, - statusMatrix, - theme, - onSelectTab, - onToggleMaximize -}: NetworkCardProps) { - return ( -
-
-
- - -
-
- -
-
- {activeNetworkTab === 'siteTopology' && ( -
- Loading topology...
}> - - -
- )} - {activeNetworkTab === 'matrix' && ( -
- -
- )} -
- ); -} - -export default NetworkCard; diff --git a/frontend/src/components/network/ConnectivityHeatmap.tsx b/frontend/src/components/network/ConnectivityHeatmap.tsx deleted file mode 100644 index 0e6d24a4c..000000000 --- a/frontend/src/components/network/ConnectivityHeatmap.tsx +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import { useEffect, useMemo, useRef, useState } from 'react'; -import { GatewayPoolStatus, NodeStatus, SiteMatrix, SiteStatus } from '../../types'; -import { getCniStatus } from '../nodes/shared/index'; - -function getGatewayPoolNodeNames(pool: GatewayPoolStatus): string[] { - const nodeNames = (pool.nodes || []) - .map((node) => node.name) - .filter((name): name is string => Boolean(name)); - if (nodeNames.length > 0) { - return nodeNames; - } - return (pool.gateways || []).filter((name): name is string => Boolean(name)); -} - -function ConnectivityHeatmap({ - matrix, - hiddenSites, - hiddenGatewayPools, - sites, - gatewayPools, - siteCounts, - nodeStatuses -}: { - matrix?: Record; - hiddenSites: Set; - hiddenGatewayPools: Set; - sites: SiteStatus[]; - gatewayPools: GatewayPoolStatus[]; - siteCounts: Map; - nodeStatuses: NodeStatus[]; -}) { - const [activeScope, setActiveScope] = useState('all'); - const matrixRef = useRef(null); - const [matrixSize, setMatrixSize] = useState({ width: 420, height: 420 }); - const [tooltip, setTooltip] = useState<{ - x: number; - y: number; - src: string; - dst: string; - value: number; - } | null>(null); - const siteNames = useMemo(() => { - return Object.keys(matrix || {}) - .filter((name) => !name.startsWith('pool:')) - .filter((name) => !hiddenSites.has(name)) - .sort(); - }, [matrix, hiddenSites]); - const gatewayPoolNames = useMemo(() => { - return (gatewayPools || []) - .map((pool) => pool.name || '') - .filter((name) => Boolean(name) && !hiddenGatewayPools.has(name)) - .sort(); - }, [gatewayPools, hiddenGatewayPools]); - const selectorOptions = useMemo( - () => [ - { key: 'all', label: 'All', kind: 'all' as const }, - ...siteNames.map((site) => ({ key: `site:${site}`, label: site, kind: 'site' as const })), - ...gatewayPoolNames.map((pool) => ({ key: `pool:${pool}`, label: pool, kind: 'pool' as const })) - ], - [siteNames, gatewayPoolNames] - ); - const siteLookup = useMemo(() => { - const map = new Map(); - for (const site of sites) { - if (site.name) { - map.set(site.name, site); - } - } - return map; - }, [sites]); - const nodeStatusByName = useMemo(() => { - const map = new Map(); - for (const node of nodeStatuses) { - const name = node.nodeInfo?.name; - if (name) { - map.set(name, node); - } - } - return map; - }, [nodeStatuses]); - const visibleNodeNames = useMemo(() => { - return nodeStatuses - .filter((node) => { - const nodeName = node.nodeInfo?.name || ''; - if (!nodeName) return false; - const siteName = node.nodeInfo?.siteName; - if (siteName && hiddenSites.has(siteName)) { - return false; - } - const poolName = (gatewayPools || []).find((pool) => { - const poolNameValue = pool.name || ''; - if (!poolNameValue) return false; - return getGatewayPoolNodeNames(pool).includes(nodeName); - })?.name; - if (poolName && hiddenGatewayPools.has(poolName)) { - return false; - } - return true; - }) - .map((node) => node.nodeInfo?.name || '') - .filter((name) => Boolean(name)); - }, [nodeStatuses, hiddenSites, hiddenGatewayPools, gatewayPools]); - - const adjacency = useMemo(() => { - const map = new Map>(); - const allVisible = new Set(visibleNodeNames); - for (const node of nodeStatuses) { - const src = node.nodeInfo?.name; - if (!src || !allVisible.has(src)) continue; - if (!map.has(src)) map.set(src, new Set()); - for (const peer of node.peers || []) { - const dst = peer.name; - if (!dst || !allVisible.has(dst) || dst === src) continue; - map.get(src)?.add(dst); - if (!map.has(dst)) map.set(dst, new Set()); - map.get(dst)?.add(src); - } - } - return map; - }, [nodeStatuses, visibleNodeNames]); - - const healthCheckStatusByPair = useMemo(() => { - const map = new Map(); - for (const siteMatrix of Object.values(matrix || {})) { - const results = siteMatrix?.results || {}; - for (const [src, row] of Object.entries(results)) { - for (const [dst, cell] of Object.entries(row || {})) { - const key = src < dst ? `${src}|${dst}` : `${dst}|${src}`; - if (!map.has(key) && cell) { - map.set(key, typeof cell === 'string' ? cell : (cell as { healthCheckStatus?: string })?.healthCheckStatus || ''); - } - } - } - } - // Also extract health check status from per-node peer data (covers cross-site links) - for (const node of nodeStatuses) { - const src = node.nodeInfo?.name; - if (!src) continue; - for (const peer of node.peers || []) { - const dst = peer.name; - if (!dst || dst === src) continue; - const key = src < dst ? `${src}|${dst}` : `${dst}|${src}`; - if (map.has(key)) continue; // matrix data takes priority - const status = peer.healthCheck?.status; - if (status) { - map.set(key, status); - } - } - } - return map; - }, [matrix, nodeStatuses]); - - const selectedNodeNames = useMemo(() => { - const visibleSet = new Set(visibleNodeNames); - if (activeScope === 'all') { - return [...visibleNodeNames].sort(); - } - - if (activeScope.startsWith('site:')) { - const siteName = activeScope.slice(5); - const fromMatrix = (matrix?.[siteName]?.nodes || []) - .map((name) => String(name)) - .filter((name) => visibleSet.has(name)); - if (fromMatrix.length > 0) { - return Array.from(new Set(fromMatrix)).sort(); - } - return nodeStatuses - .map((node) => node.nodeInfo?.name || '') - .filter((name) => { - if (!name || !visibleSet.has(name)) return false; - const site = nodeStatusByName.get(name)?.nodeInfo?.siteName; - return site === siteName; - }) - .sort(); - } - - if (activeScope.startsWith('pool:')) { - const poolName = activeScope.slice(5); - const poolMatrixNodes = (matrix?.[`pool:${poolName}`]?.nodes || []) - .map((name) => String(name)) - .filter((name) => visibleSet.has(name)); - if (poolMatrixNodes.length > 0) { - return Array.from(new Set(poolMatrixNodes)).sort(); - } - const pool = gatewayPools.find((item) => item.name === poolName); - if (!pool) return []; - const members = new Set(); - for (const nodeName of getGatewayPoolNodeNames(pool)) { - if (!visibleSet.has(nodeName)) continue; - members.add(nodeName); - for (const peerName of adjacency.get(nodeName) || []) { - if (visibleSet.has(peerName)) { - members.add(peerName); - } - } - } - return Array.from(members).sort(); - } - - return []; - }, [activeScope, visibleNodeNames, matrix, nodeStatuses, nodeStatusByName, gatewayPools, adjacency]); - - useEffect(() => { - if (!selectorOptions.some((option) => option.key === activeScope)) { - setActiveScope('all'); - } - }, [activeScope, selectorOptions]); - - useEffect(() => { - if (!matrixRef.current) return; - const updateSize = () => { - const rect = matrixRef.current?.getBoundingClientRect(); - if (!rect) return; - const width = Math.max(0, Math.floor(rect.width)); - const height = Math.max(0, Math.floor(rect.height)); - if (width > 0 && height > 0) { - setMatrixSize({ width, height }); - } - }; - updateSize(); - const observer = new ResizeObserver(updateSize); - observer.observe(matrixRef.current); - return () => observer.disconnect(); - }, []); - - if ((!matrix || siteNames.length === 0) && selectedNodeNames.length === 0) { - return
No connectivity data available.
; - } - - const matrixNodes = selectedNodeNames; - if (matrixNodes.length === 0) { - return
No connectivity data available.
; - } - const cells: Array<{ row: number; col: number; value: number; src: string; dst: string }> = []; - const hasExpectedLink = (src: string, dst: string) => { - if (src === dst) return true; - return adjacency.get(src)?.has(dst) || adjacency.get(dst)?.has(src) || false; - }; - - const selfCellValueFromCniStatus = (node?: NodeStatus) => { - if (!node) { - return 0; - } - if (node.statusSource === 'apiserver-push' || node.statusSource === 'apiserver-ws') { - return 2; - } - const cni = getCniStatus(node); - if (cni.tone === 'success') return 1; - if (cni.tone === 'warning') return 2; - return 0; - }; - - for (let i = 0; i < matrixNodes.length; i++) { - const src = matrixNodes[i]; - for (let j = 0; j < matrixNodes.length; j++) { - const dst = matrixNodes[j]; - const pairKey = src < dst ? `${src}|${dst}` : `${dst}|${src}`; - const hcStatus = (healthCheckStatusByPair.get(pairKey) || '').trim().toLowerCase(); - let value = hcStatus === 'up' ? 1 : hcStatus === 'mixed' ? 2 : hcStatus ? 0 : -1; - if (src === dst) { - const node = nodeStatusByName.get(src); - value = selfCellValueFromCniStatus(node); - } else if (!hasExpectedLink(src, dst)) { - value = -2; - } - cells.push({ row: i, col: j, value, src, dst }); - } - } - - const { width, height } = matrixSize; - - const getColor = (value: number) => { - if (value === 1) return '#4ade80'; - if (value === 2) return '#facc15'; - if (value === 0) return '#f87171'; - if (value === -2) return 'transparent'; - return '#6b7280'; - }; - - const renderMatrix = (width: number, height: number) => { - if (width <= 0 || height <= 0) { - return
; - } - const showAxisLabels = false; - const margin = { top: 12, left: 12, right: 12, bottom: 12 }; - const innerWidth = width - margin.left - margin.right; - const innerHeight = height - margin.top - margin.bottom; - const minInner = Math.min(innerWidth, innerHeight); - const gap = Math.max(1, Math.round(minInner * 0.005)); - const cellSize = Math.floor((minInner - gap * (matrixNodes.length - 1)) / matrixNodes.length); - const gridWidth = cellSize * matrixNodes.length + gap * (matrixNodes.length - 1); - const gridHeight = cellSize * matrixNodes.length + gap * (matrixNodes.length - 1); - const offsetX = Math.max(0, Math.floor((innerWidth - gridWidth) / 2)); - const offsetY = Math.max(0, Math.floor((innerHeight - gridHeight) / 2)); - const scale = 1; - - return ( -
-
- - - {cells.map((cell) => { - const x = cell.col * (cellSize + gap); - const y = cell.row * (cellSize + gap); - const w = cellSize; - const h = cellSize; - return ( - { - setTooltip({ - x: event.clientX + 12, - y: event.clientY + 12, - src: cell.src, - dst: cell.dst, - value: cell.value - }); - }} - onMouseLeave={() => setTooltip(null)} - /> - ); - })} - {showAxisLabels && - matrixNodes.map((label, index) => ( - - {label} - - ))} - {showAxisLabels && - matrixNodes.map((label, index) => ( - - {label} - - ))} - - -
- {tooltip && ( -
-
{tooltip.src} {'->'} {tooltip.dst}
-
- {tooltip.src === tooltip.dst - ? (tooltip.value === 1 - ? 'CNI Healthy' - : tooltip.value === 2 - ? `CNI Warning: ${getCniStatus(nodeStatusByName.get(tooltip.src))?.label || 'Warning'}` - : 'CNI No Data') - : tooltip.value === 1 - ? 'HC Up' - : tooltip.value === 2 - ? 'HC Mixed' - : tooltip.value === 0 - ? 'HC Down' - : tooltip.value === -2 - ? 'No Link Expected' - : 'No Data'} -
-
- )} -
- ); - }; - - return ( -
-
- {selectorOptions.map((option) => { - const isActive = option.key === activeScope; - const isSite = option.kind === 'site'; - const isPool = option.kind === 'pool'; - const siteInfo = isSite ? siteLookup.get(option.label) : undefined; - const counts = isSite ? siteCounts.get(option.label) : undefined; - const online = counts?.online ?? siteInfo?.onlineCount ?? 0; - const total = counts?.total ?? siteInfo?.nodeCount ?? 0; - const status = isSite - ? (online === 0 && total > 0 ? 'danger' : online < total ? 'warning' : 'success') - : isPool - ? 'info' - : 'all'; - return ( - - ); - })} -
- {renderMatrix(width, height)} -
- ); -} - - -export default ConnectivityHeatmap; diff --git a/frontend/src/components/network/Topology.tsx b/frontend/src/components/network/Topology.tsx deleted file mode 100644 index 9107d77af..000000000 --- a/frontend/src/components/network/Topology.tsx +++ /dev/null @@ -1,949 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import * as React from 'react'; -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { createPortal } from 'react-dom'; -import { - getTopologyNodeGlyph, - renderTopologyNodeGlyph, - useTopologyNodeIconTextures -} from '../common/topologyIcons'; -import { GatewayPoolStatus, NodeStatus, PeeringStatus, SiteStatus } from '../../types'; -import { getNodeStatus, ReagraphModule } from '../nodes/shared/index'; - -function buildGraph( - allSiteNames: string[], - allPoolNames: string[], - peerings: PeeringStatus[], - hiddenSites: Set, - hiddenGatewayPools: Set, - existingGatewayPools: Set, - palette: { site: string; siteEmpty: string; siteWarn: string; siteDanger: string; pool: string; poolEmpty: string; poolWarn: string; poolDanger: string; edge: string; edgeDim: string; edgeUp: string; edgeWarn: string; edgeDanger: string }, - siteCounts?: Map, - poolCounts?: Map, - edgeHealthCheckCounts?: Map, - poolToSite?: Map -) { - // Dim a hex color by reducing its opacity (blend toward background) - const dimColor = (hex: string) => { - const r = parseInt(hex.slice(1, 3), 16); - const g = parseInt(hex.slice(3, 5), 16); - const b = parseInt(hex.slice(5, 7), 16); - const mix = (c: number) => Math.round(c * 0.3 + 30 * 0.7); - return `#${mix(r).toString(16).padStart(2, '0')}${mix(g).toString(16).padStart(2, '0')}${mix(b).toString(16).padStart(2, '0')}`; - }; - const nodes: { id: string; label: string; fill: string; activeFill: string; data: { level: number; group: string; peerings: string[] } }[] = []; - const edges: { id: string; source: string; target: string; fill?: string; size?: number; data?: { peerings: string[]; hcUp: number; hcTotal: number } }[] = []; - const nodeSet = new Set(); - const edgeSet = new Set(); - const edgePeerings = new Map>(); - const sitePeerings = new Map>(); - const poolPeerings = new Map>(); - - for (const peering of peerings) { - const peeringName = peering.name || 'peering'; - for (const site of peering.sites || []) { - if (!sitePeerings.has(site)) { - sitePeerings.set(site, new Set()); - } - sitePeerings.get(site)?.add(peeringName); - } - for (const pool of peering.gatewayPools || []) { - if (!existingGatewayPools.has(pool)) { - continue; - } - if (!poolPeerings.has(pool)) { - poolPeerings.set(pool, new Set()); - } - poolPeerings.get(pool)?.add(peeringName); - } - } - - const addNode = (id: string, label: string, group: string, hidden: boolean) => { - if (!nodeSet.has(id)) { - nodeSet.add(id); - let fill = group === 'pool' ? palette.pool : palette.site; - if (group === 'site' && siteCounts) { - const counts = siteCounts.get(label); - if (counts) { - if (counts.total === 0) { - fill = palette.siteEmpty; - } else if ((counts.danger || 0) > 0) { - fill = palette.siteDanger; - } else if ((counts.warning || 0) > 0) { - fill = palette.siteWarn; - } - } - } - if (group === 'pool' && poolCounts) { - const counts = poolCounts.get(label); - if (counts) { - // No matching node entries for this pool in current cluster status: - // render gray (empty) instead of green. - if (counts.total === 0) { - fill = palette.poolEmpty; - } else if ((counts.danger || 0) > 0) { - fill = palette.poolDanger; - } else if ((counts.warning || 0) > 0) { - fill = palette.poolWarn; - } else if (counts.online === 0 && counts.total > 0) { - fill = palette.poolDanger; - } else if (counts.online < counts.total) { - fill = palette.poolWarn; - } - } - } - if (hidden) { - fill = dimColor(fill); - } - const peeringsForNode = group === 'pool' - ? Array.from(poolPeerings.get(label) || []) - : Array.from(sitePeerings.get(label) || []); - nodes.push({ id, label, fill, activeFill: fill, data: { level: group === 'pool' ? 0 : 1, group, peerings: peeringsForNode } }); - } - }; - - const addEdge = (a: string, b: string) => { - if (a === b) return; - const key = a < b ? `${a}|${b}` : `${b}|${a}`; - if (edgeSet.has(key)) return; - edgeSet.add(key); - const isHidden = (nodeId: string) => { - if (nodeId.startsWith('site:')) return hiddenSites.has(nodeId.slice(5)); - if (nodeId.startsWith('pool:')) return hiddenGatewayPools.has(nodeId.slice(5)); - return false; - }; - let edgeFill = palette.edge; - if (isHidden(a) || isHidden(b)) { - edgeFill = palette.edgeDim; - } else if (edgeHealthCheckCounts) { - const counts = edgeHealthCheckCounts.get(key); - // No matching health check entries for this edge (missing map entry or total=0): - // keep default gray edge color. Only color healthy/warn/danger with real data. - if (!counts || counts.total === 0) { - edgeFill = palette.edge; - } else { - if (counts.up === counts.total) { - edgeFill = palette.edgeUp; - } else if (counts.total - counts.up > counts.total / 2) { - edgeFill = palette.edgeDanger; - } else { - edgeFill = palette.edgeWarn; - } - } - } - edges.push({ id: key, source: a, target: b, fill: edgeFill, size: 3.5 }); - }; - - // Seed all known sites and gateway pools so isolated entities still render without edges. - for (const site of allSiteNames) { - addNode(`site:${site}`, site, 'site', hiddenSites.has(site)); - } - for (const pool of allPoolNames) { - addNode(`pool:${pool}`, pool, 'pool', hiddenGatewayPools.has(pool)); - } - - for (const peering of peerings) { - const sites = peering.sites || []; - const pools = (peering.gatewayPools || []).filter((pool) => existingGatewayPools.has(pool)); - const pName = peering.name || 'peering'; - const isPoolPeering = pName.startsWith('poolpeering/'); - - for (const site of sites) { - addNode(`site:${site}`, site, 'site', hiddenSites.has(site)); - } - for (const pool of pools) { - addNode(`pool:${pool}`, pool, 'pool', hiddenGatewayPools.has(pool)); - } - - // Track which peerings each edge belongs to - const trackEdgePeering = (a: string, b: string) => { - const key = a < b ? `${a}|${b}` : `${b}|${a}`; - if (!edgePeerings.has(key)) edgePeerings.set(key, new Set()); - edgePeerings.get(key)?.add(pName); - }; - - if (pools.length > 0) { - for (const site of sites) { - for (const pool of pools) { - trackEdgePeering(`site:${site}`, `pool:${pool}`); - addEdge(`site:${site}`, `pool:${pool}`); - } - } - if (isPoolPeering && pools.length > 1) { - for (let i = 0; i < pools.length; i++) { - for (let j = i + 1; j < pools.length; j++) { - trackEdgePeering(`pool:${pools[i]}`, `pool:${pools[j]}`); - addEdge(`pool:${pools[i]}`, `pool:${pools[j]}`); - } - } - } - } else if (sites.length > 1) { - for (let i = 0; i < sites.length; i++) { - for (let j = i + 1; j < sites.length; j++) { - trackEdgePeering(`site:${sites[i]}`, `site:${sites[j]}`); - addEdge(`site:${sites[i]}`, `site:${sites[j]}`); - } - } - } - } - - // Attach peering names and health check counts to edge data - for (const edge of edges) { - const pNames = edgePeerings.get(edge.id); - const hc = edgeHealthCheckCounts?.get(edge.id); - edge.data = { - peerings: pNames ? Array.from(pNames) : [], - hcUp: hc?.up ?? 0, - hcTotal: hc?.total ?? 0 - }; - } - - // Sort nodes lexicographically by label so layout is deterministic - nodes.sort((a, b) => a.label.localeCompare(b.label)); - - return { nodes, edges }; -} - -function buildNodeGraph( - nodes: NodeStatus[], - gatewayByNode: Map, - palette: { - nodeHealthy: string; - nodeWarn: string; - nodeDanger: string; - edge: string; - edgeUp: string; - edgeWarn: string; - edgeDanger: string; - }, - options?: { - edgeSize?: number; - } -) { - const graphNodes: { - id: string; - label: string; - fill: string; - activeFill: string; - data: { level: number; group: string; peerings: string[]; lines: string[] }; - }[] = []; - const graphEdges: { - id: string; - source: string; - target: string; - fill?: string; - size?: number; - data?: { - peerings: string[]; - hcUp: number; - hcTotal: number; - sourceLabel: string; - targetLabel: string; - }; - }[] = []; - - const nodeByName = new Map(); - for (const node of nodes) { - const nodeName = node.nodeInfo?.name; - if (!nodeName) continue; - nodeByName.set(nodeName, node); - } - - for (const [nodeName, node] of nodeByName.entries()) { - const isGateway = node.nodeInfo?.isGateway || gatewayByNode.has(nodeName); - const status = getNodeStatus(node); - let fill = palette.nodeHealthy; - if (status === 'warning') { - fill = palette.nodeWarn; - } else if (status === 'danger') { - fill = palette.nodeDanger; - } - - const siteName = node.nodeInfo?.siteName || '-'; - const poolName = gatewayByNode.get(nodeName) || '-'; - const lines = isGateway - ? [`Gateway Pool: ${poolName}`, `Site: ${siteName}`] - : [`Site: ${siteName}`]; - - graphNodes.push({ - id: `node:${nodeName}`, - label: nodeName, - fill, - activeFill: fill, - data: { - level: isGateway ? 0 : 1, - group: isGateway ? 'gateway-node' : 'worker-node', - peerings: [], - lines - } - }); - } - - const edgeCounts = new Map(); - for (const [nodeName, node] of nodeByName.entries()) { - for (const peer of node.peers || []) { - const peerName = peer.name; - if (!peerName || !nodeByName.has(peerName) || peerName === nodeName) { - continue; - } - const srcId = `node:${nodeName}`; - const dstId = `node:${peerName}`; - const key = srcId < dstId ? `${srcId}|${dstId}` : `${dstId}|${srcId}`; - const current = edgeCounts.get(key) || { up: 0, total: 0 }; - if (peer.healthCheck?.enabled || peer.healthCheck) { - current.total += 1; - const rawStatus = (peer.healthCheck?.status || '').trim().toLowerCase(); - if (rawStatus === 'up') { - current.up += 1; - } - } - edgeCounts.set(key, current); - } - } - - for (const [edgeId, counts] of edgeCounts.entries()) { - const [source, target] = edgeId.split('|'); - let edgeFill = palette.edge; - if (counts.total > 0) { - if (counts.up === counts.total) { - edgeFill = palette.edgeUp; - } else if (counts.total - counts.up > counts.total / 2) { - edgeFill = palette.edgeDanger; - } else { - edgeFill = palette.edgeWarn; - } - } - - graphEdges.push({ - id: edgeId, - source, - target, - fill: edgeFill, - size: options?.edgeSize ?? 2, - data: { - peerings: [], - hcUp: counts.up, - hcTotal: counts.total, - sourceLabel: source.startsWith('node:') ? source.slice(5) : source, - targetLabel: target.startsWith('node:') ? target.slice(5) : target - } - }); - } - - graphNodes.sort((a, b) => a.label.localeCompare(b.label)); - graphEdges.sort((a, b) => a.id.localeCompare(b.id)); - - return { nodes: graphNodes, edges: graphEdges }; -} - -function Topology({ - mode, - isMaximized, - sites, - peerings, - nodes, - gatewayPools, - gatewayByNode, - hiddenSites, - hiddenGatewayPools, - theme, - siteCounts, - poolCounts, - edgeHealthCheckCounts, - poolToSite -}: { - mode: 'sitesAndPools' | 'nodes'; - isMaximized: boolean; - sites: SiteStatus[]; - peerings: PeeringStatus[]; - nodes: NodeStatus[]; - gatewayPools: GatewayPoolStatus[]; - gatewayByNode: Map; - hiddenSites: Set; - hiddenGatewayPools: Set; - theme: 'dark' | 'light'; - siteCounts: Map; - poolCounts: Map; - edgeHealthCheckCounts: Map; - poolToSite: Map; -}) { - const graphRef = useRef(null); - const topologyWrapperRef = useRef(null); - const [reagraphModule, setReagraphModule] = useState(null); - - useEffect(() => { - let canceled = false; - import('reagraph') - .then((module) => { - if (canceled) return; - setReagraphModule({ - GraphCanvas: module.GraphCanvas as React.ComponentType, - darkTheme: module.darkTheme as Record, - lightTheme: module.lightTheme as Record - }); - }) - .catch((error) => { - console.error('Failed to load topology renderer', error); - }); - - return () => { - canceled = true; - }; - }, []); - - const palette = useMemo( - () => - theme === 'light' - ? { - site: '#16a34a', - siteEmpty: '#6b7280', - siteWarn: '#facc15', - siteDanger: '#dc2626', - pool: '#15803d', - poolEmpty: '#6b7280', - poolWarn: '#facc15', - poolDanger: '#dc2626', - edge: '#94a3b8', - edgeDim: '#cbd5e1', - edgeUp: '#16a34a', - edgeWarn: '#facc15', - edgeDanger: '#dc2626', - nodeHealthy: '#16a34a', - nodeWarn: '#facc15', - nodeDanger: '#dc2626', - label: '#1f2937', - background: '#ffffff' - } - : { - site: '#1DE9AC', - siteEmpty: '#6b7280', - siteWarn: '#facc15', - siteDanger: '#ef4444', - pool: '#1DE9AC', - poolEmpty: '#6b7280', - poolWarn: '#facc15', - poolDanger: '#ef4444', - edge: '#4b5563', - edgeDim: '#334155', - edgeUp: '#1DE9AC', - edgeWarn: '#facc15', - edgeDanger: '#ef4444', - nodeHealthy: '#1DE9AC', - nodeWarn: '#facc15', - nodeDanger: '#ef4444', - label: '#e0e0e0', - background: '#1a1a1a' - }, - [theme] - ); - const graphTheme = useMemo(() => { - if (!reagraphModule) return null; - const base = theme === 'light' ? reagraphModule.lightTheme : reagraphModule.darkTheme; - return { - ...base, - canvas: { - ...base.canvas, - background: palette.background, - fog: null - }, - edge: { - ...base.edge, - fill: palette.edge, - activeFill: palette.edge, - opacity: 1, - inactiveOpacity: 0.25 - }, - node: { - ...base.node, - activeFill: 'rgba(0,0,0,0)', - inactiveOpacity: 1, - hoverOpacity: 0, - label: { - ...base.node.label, - color: palette.label, - activeColor: palette.label, - fontSize: 16, - stroke: palette.background, - strokeColor: palette.background - } - } - }; - }, [palette, reagraphModule, theme]); - - const [hovered, setHovered] = useState<{ - x: number; - y: number; - label: string; - group: string; - lines: string[]; - } | null>(null); - const [edgeHovered, setEdgeHovered] = useState<{ - x: number; - y: number; - title: string; - detail: string; - } | null>(null); - const [hoveredNodeId, setHoveredNodeId] = useState(null); - const [showZoomHint, setShowZoomHint] = useState(false); - const [zoomHintRect, setZoomHintRect] = useState<{ left: number; top: number; width: number; height: number } | null>(null); - const zoomHintTimerRef = useRef(null); - const topologyIconTextures = useTopologyNodeIconTextures(); - const allSiteNames = useMemo( - () => sites - .map((site) => (site.name || '').trim()) - .filter((name): name is string => name.length > 0), - [sites] - ); - const allPoolNames = useMemo( - () => gatewayPools - .map((pool) => (pool.name || '').trim()) - .filter((name): name is string => name.length > 0), - [gatewayPools] - ); - const topologySiteCounts = useMemo(() => { - const counts = new Map(); - for (const siteName of allSiteNames) { - counts.set(siteName, { online: 0, total: 0, warning: 0, danger: 0 }); - } - - if (nodes.length > 0) { - for (const node of nodes) { - const siteName = node.nodeInfo?.siteName; - if (!siteName) continue; - const nodeName = node.nodeInfo?.name; - const isGatewayNode = node.nodeInfo?.isGateway || (nodeName ? gatewayByNode.has(nodeName) : false); - if (isGatewayNode) continue; - - const current = counts.get(siteName) || { online: 0, total: 0, warning: 0, danger: 0 }; - current.total += 1; - - const status = getNodeStatus(node); - if (status === 'success') { - current.online += 1; - } else if (status === 'warning') { - current.warning += 1; - } else { - current.danger += 1; - } - - counts.set(siteName, current); - } - } else { - // Summary mode: use siteCounts prop (online/total only). - for (const [siteName, sc] of siteCounts.entries()) { - const offline = Math.max(0, sc.total - sc.online); - counts.set(siteName, { online: sc.online, total: sc.total, warning: 0, danger: offline }); - } - } - - return counts; - }, [allSiteNames, gatewayByNode, nodes, siteCounts]); - const topologyPoolCounts = useMemo(() => { - const counts = new Map(); - - if (nodes.length > 0) { - const nodeByName = new Map(); - for (const node of nodes) { - const nodeName = node.nodeInfo?.name; - if (!nodeName) continue; - nodeByName.set(nodeName, node); - } - - for (const poolName of allPoolNames) { - const baseline = poolCounts.get(poolName) || { online: 0, total: 0 }; - const current = { online: 0, total: baseline.total, warning: 0, danger: 0 }; - - for (const [nodeName, nodePoolName] of gatewayByNode.entries()) { - if (nodePoolName !== poolName) continue; - const node = nodeByName.get(nodeName); - if (!node) continue; - const status = getNodeStatus(node); - if (status === 'success') { - current.online += 1; - } else if (status === 'warning') { - current.warning += 1; - } else { - current.danger += 1; - } - } - - const accounted = current.online + current.warning + current.danger; - if (current.total < accounted) { - current.total = accounted; - } else if (current.total > accounted) { - current.danger += current.total - accounted; - } - - counts.set(poolName, current); - } - } else { - // Summary mode: use poolCounts prop (online/total only). - for (const poolName of allPoolNames) { - const pc = poolCounts.get(poolName) || { online: 0, total: 0 }; - const offline = Math.max(0, pc.total - pc.online); - counts.set(poolName, { online: pc.online, total: pc.total, warning: 0, danger: offline }); - } - } - - return counts; - }, [allPoolNames, gatewayByNode, nodes, poolCounts]); - - const sitePoolGraph = useMemo( - () => buildGraph( - allSiteNames, - allPoolNames, - peerings, - hiddenSites, - hiddenGatewayPools, - new Set(gatewayPools.map((pool) => pool.name).filter((name): name is string => Boolean(name))), - palette, - topologySiteCounts, - topologyPoolCounts, - edgeHealthCheckCounts, - poolToSite - ), - [allPoolNames, allSiteNames, peerings, gatewayPools, hiddenSites, hiddenGatewayPools, palette, topologySiteCounts, topologyPoolCounts, edgeHealthCheckCounts, poolToSite] - ); - const nodeGraph = useMemo( - () => buildNodeGraph(nodes, gatewayByNode, palette, { edgeSize: 1.4 }), - [nodes, gatewayByNode, palette] - ); - const graph = mode === 'nodes' ? nodeGraph : sitePoolGraph; - const dimNodeColor = useCallback((color: string) => { - const hex = (color || '').trim(); - const match = /^#([0-9a-fA-F]{6})$/.exec(hex); - if (!match) return palette.edgeDim; - const value = match[1]; - const r = parseInt(value.slice(0, 2), 16); - const g = parseInt(value.slice(2, 4), 16); - const b = parseInt(value.slice(4, 6), 16); - const mix = (channel: number) => Math.round(channel * 0.35 + 40 * 0.65); - return `#${mix(r).toString(16).padStart(2, '0')}${mix(g).toString(16).padStart(2, '0')}${mix(b).toString(16).padStart(2, '0')}`; - }, [palette.edgeDim]); - - const graphNodesForRender = useMemo(() => { - const baseNodes = mode === 'nodes' - ? graph.nodes.map((node) => ({ ...node, label: '' })) - : graph.nodes; - - if (!hoveredNodeId) return baseNodes; - const connectedNodeIds = new Set([hoveredNodeId]); - for (const edge of graph.edges) { - if (edge.source === hoveredNodeId) { - connectedNodeIds.add(edge.target); - } else if (edge.target === hoveredNodeId) { - connectedNodeIds.add(edge.source); - } - } - - return baseNodes.map((node) => { - if (connectedNodeIds.has(node.id)) { - return node; - } - const dimmed = dimNodeColor(node.fill || palette.site); - return { - ...node, - fill: dimmed, - activeFill: dimmed - }; - }); - }, [graph.edges, graph.nodes, hoveredNodeId, dimNodeColor, mode, palette.site]); - - const graphEdgesForRender = useMemo(() => { - if (!hoveredNodeId) return graph.edges; - return graph.edges.map((edge) => { - const connected = edge.source === hoveredNodeId || edge.target === hoveredNodeId; - if (connected) return edge; - return { - ...edge, - fill: palette.edgeDim - }; - }); - }, [graph.edges, hoveredNodeId, palette.edgeDim]); - - const topologyConfig = useMemo(() => { - const nodeCount = graph.nodes.length; - - const siteTopologyConfig = { - minCameraDistance: 2, - nodeSize: nodeCount <= 6 ? 68 : 50, - layoutOverrides: { - radius: 25, - concentricSpacing: 25 - } - }; - - const nodeTopologyConfig = { - ...siteTopologyConfig, - minCameraDistance: 6, - - layoutOverrides: { - ...siteTopologyConfig.layoutOverrides, - radius: 25, - concentricSpacing: 50 - } - }; - - return mode === 'nodes' ? nodeTopologyConfig : siteTopologyConfig; - }, [graph.nodes.length, mode]); - const topologyLayoutType = mode === 'nodes' ? 'concentric2d' : 'concentric2d'; - - useEffect(() => { - if (mode !== 'sitesAndPools') return; - if (!graphRef.current) return; - - const fit = () => graphRef.current?.fitNodesInView(); - const first = requestAnimationFrame(() => { - const second = requestAnimationFrame(fit); - (fit as unknown as { _second?: number })._second = second; - }); - - return () => { - cancelAnimationFrame(first); - const second = (fit as unknown as { _second?: number })._second; - if (typeof second === 'number') { - cancelAnimationFrame(second); - } - }; - }, [mode, graph.nodes.length, graph.edges.length]); - - useEffect(() => () => { - if (zoomHintTimerRef.current !== null) { - window.clearTimeout(zoomHintTimerRef.current); - zoomHintTimerRef.current = null; - } - }, []); - - const updateZoomHintRect = useCallback(() => { - const rect = topologyWrapperRef.current?.getBoundingClientRect(); - if (!rect) { - setZoomHintRect(null); - return; - } - setZoomHintRect({ left: rect.left, top: rect.top, width: rect.width, height: rect.height }); - }, []); - - const showCtrlZoomHint = useCallback(() => { - updateZoomHintRect(); - setShowZoomHint(true); - if (zoomHintTimerRef.current !== null) { - window.clearTimeout(zoomHintTimerRef.current); - } - zoomHintTimerRef.current = window.setTimeout(() => { - setShowZoomHint(false); - setZoomHintRect(null); - zoomHintTimerRef.current = null; - }, 1400); - }, [updateZoomHintRect]); - - useEffect(() => { - if (!showZoomHint) { - return; - } - const update = () => updateZoomHintRect(); - window.addEventListener('resize', update, { passive: true }); - window.addEventListener('scroll', update, { passive: true, capture: true }); - document.addEventListener('scroll', update, { passive: true, capture: true }); - return () => { - window.removeEventListener('resize', update); - window.removeEventListener('scroll', update, true); - document.removeEventListener('scroll', update, true); - }; - }, [showZoomHint, updateZoomHintRect]); - - const handleWheelCapture = useCallback((event: React.WheelEvent) => { - if (event.ctrlKey) { - return; - } - event.stopPropagation(); - showCtrlZoomHint(); - }, [showCtrlZoomHint]); - - if (graph.nodes.length === 0) { - return
No peering data available.
; - } - - if (!reagraphModule || !graphTheme) { - return
Loading topology renderer...
; - } - - const GraphCanvas = reagraphModule.GraphCanvas; - const zoomHintStyle = (() => { - if (!zoomHintRect) { - return { left: '50vw', top: '50vh' } as React.CSSProperties; - } - return { left: zoomHintRect.left + zoomHintRect.width / 2, top: zoomHintRect.top + zoomHintRect.height / 2 }; - })(); - const zoomHintOverlayStyle = (() => { - if (!zoomHintRect) { - return null; - } - return { left: zoomHintRect.left, top: zoomHintRect.top, width: zoomHintRect.width, height: zoomHintRect.height }; - })(); - - return ( -
-
- - { - const color = n.fill || palette.site; - const group = (n.data as { group?: string } | undefined)?.group; - const glyph = getTopologyNodeGlyph(group); - const material = ( - - ); - const iconNode = renderTopologyNodeGlyph({ - glyph, - size, - color, - textures: topologyIconTextures, - workerOffsetScale: -0.16 - }); - if (iconNode) return iconNode; - - return ( - - - {material} - - ); - }} - onNodePointerOver={(node, event) => { - const tooltipLabel = (node.label || '').trim() - || (node.id.startsWith('node:') - ? node.id.slice(5) - : node.id.startsWith('site:') - ? node.id.slice(5) - : node.id.startsWith('pool:') - ? node.id.slice(5) - : node.id); - setHoveredNodeId(node.id); - setHovered({ - x: event.clientX + 12, - y: event.clientY + 12, - label: tooltipLabel, - group: (node.data as { group?: string })?.group || 'site', - lines: (() => { - const data = node.data as { lines?: string[]; peerings?: string[] } | undefined; - if (data?.lines && data.lines.length > 0) { - return data.lines; - } - const peerings = data?.peerings || []; - return [`Peerings: ${peerings.length > 0 ? peerings.join(', ') : '-'}`]; - })() - }); - }} - onNodePointerOut={() => { - setHovered(null); - setHoveredNodeId(null); - }} - onEdgePointerOver={(edge, event) => { - if (!event) return; - setHovered(null); - setHoveredNodeId(null); - const data = edge.data as { - peerings?: string[]; - hcUp?: number; - hcTotal?: number; - sourceLabel?: string; - targetLabel?: string; - } | undefined; - const hcDetail = data?.hcTotal && data.hcTotal > 0 - ? `${data.hcUp ?? 0}/${data.hcTotal} links up` - : 'No health check data'; - const title = mode === 'nodes' - ? `${data?.sourceLabel || edge.source} <-> ${data?.targetLabel || edge.target}` - : (data?.peerings?.join(', ') || 'Unknown peering'); - setEdgeHovered({ - x: event.clientX + 12, - y: event.clientY + 12, - title, - detail: hcDetail - }); - }} - onEdgePointerOut={() => setEdgeHovered(null)} - onCanvasPointerOut={() => { - setHovered(null); - setHoveredNodeId(null); - setEdgeHovered(null); - }} - onCanvasClick={() => { - setHovered(null); - setHoveredNodeId(null); - setEdgeHovered(null); - }} - /> - {hovered && createPortal( -
-
{hovered.label}
-
Type: { - hovered.group === 'pool' - ? 'Gateway Pool' - : hovered.group === 'site' - ? 'Site' - : hovered.group === 'gateway-node' - ? 'Gateway Node' - : 'Node' - }
- {hovered.lines.map((line, index) => ( -
{line}
- ))} -
, - document.body - )} - {edgeHovered && createPortal( -
-
{edgeHovered.title}
-
{edgeHovered.detail}
-
, - document.body - )} - {showZoomHint && zoomHintOverlayStyle && createPortal( -
, - document.body - )} - {showZoomHint && createPortal( -
- Hold Ctrl and scroll to zoom graph -
, - document.body - )} -
-
- ); -} - - -export default Topology; diff --git a/frontend/src/components/nodes/NodeDetailDialog.tsx b/frontend/src/components/nodes/NodeDetailDialog.tsx new file mode 100644 index 000000000..84220f08b --- /dev/null +++ b/frontend/src/components/nodes/NodeDetailDialog.tsx @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +import { useEffect, useState } from 'react'; +import type { ComponentProps } from 'react'; +import type { DetailView } from '../../state/nodeDetails'; +import NodeDetailModal from './NodeDetailModal'; +import { CloseXIcon, formatDateAndAge } from './shared/index'; + +type Props = Omit, 'node' | 'detailControls'> & { + detail: DetailView; + onLoad: (forceRefresh?: boolean) => void; +}; + +export default function NodeDetailDialog({ detail, onLoad, ...props }: Props) { + const [jsonOpen, setJsonOpen] = useState(false); + const [, setClock] = useState(0); + const expiresAt = detail.snapshot?.expiresAt; + useEffect(() => { + if (!expiresAt) return; + const timer = window.setInterval(() => setClock((clock) => clock + 1), 1000); + return () => window.clearInterval(timer); + }, [expiresAt]); + useEffect(() => setJsonOpen(false), [props.nodeName, expiresAt]); + useEffect(() => { + if (!props.nodeName) return; + const onKeyDown = (event: KeyboardEvent) => { if (event.key === 'Escape') props.onClose(); }; + window.addEventListener('keydown', onKeyDown); + return () => window.removeEventListener('keydown', onKeyDown); + }, [props.nodeName, props.onClose]); + if (!props.nodeName) return null; + const snapshot = detail.snapshot && Date.parse(detail.snapshot.expiresAt) > Date.now() ? detail.snapshot : undefined; + const busy = detail.state === 'loading'; + const collected = snapshot ? formatDateAndAge(snapshot.collectedAt) : undefined; + const state = !snapshot && detail.state === 'loaded' ? 'expired' : detail.state; + const message = state === 'not-loaded' ? 'Detailed data is not loaded. Choose Load data to inspect peers, routes, BPF entries, and full node JSON.' + : state === 'loading' ? 'Loading node details...' + : state === 'expired' ? 'Detailed data expired and was removed. Choose Load data to request it again.' + : state === 'error' ? 'The detail request failed. Choose Load data to retry or Refresh to force a fresh collection.' + : 'Detailed data loaded.'; + const controls = ( +
+
{message}
+ {detail.error &&
Request error: {detail.error}
} + {snapshot && ( +
+ {detail.error || busy ? 'Showing previous still-valid snapshot. ' : ''} + Collected {collected?.age} ({collected?.absolute}). + {' '}Received {snapshot.receivedAt}. Expires {snapshot.expiresAt}. +
+ )} + {busy && detail.deadline &&
Request deadline: {detail.deadline}
} +
+ + + {snapshot && } +
+ {snapshot && jsonOpen && 
{JSON.stringify(snapshot.status, null, 2)}
} +
+ ); + // Unmount the heavy tables when data expires so their memoized rows, maps, + // column callbacks and serialized JSON cannot keep the snapshot alive. + if (snapshot) { + return ; + } + return ( +
+
event.stopPropagation()}> +
+
{props.nodeName}
+ +
+ {controls} +
+
+ ); +} diff --git a/frontend/src/components/nodes/NodeDetailModal.tsx b/frontend/src/components/nodes/NodeDetailModal.tsx index 1fa1eee1d..297520fb9 100644 --- a/frontend/src/components/nodes/NodeDetailModal.tsx +++ b/frontend/src/components/nodes/NodeDetailModal.tsx @@ -42,6 +42,7 @@ import NodeInfoPanel from './detail/NodeInfoPanel'; import NodeDetailTabsHeader from './detail/NodeDetailTabsHeader'; function NodeDetailModal({ + detailControls, nodeName, node, allNodeNames, @@ -55,6 +56,7 @@ function NodeDetailModal({ onSelectNode, onClose }: { + detailControls?: React.ReactNode; nodeName: string | null; node: NodeStatus | null; allNodeNames: string[]; @@ -1705,6 +1707,7 @@ function NodeDetailModal({
+ {detailControls}
; - darkTheme: Record; - lightTheme: Record; -}; - -export type { ReagraphModule }; diff --git a/frontend/src/components/status/StatusJsonModal.tsx b/frontend/src/components/status/StatusJsonModal.tsx index bbeba96ce..408960846 100644 --- a/frontend/src/components/status/StatusJsonModal.tsx +++ b/frontend/src/components/status/StatusJsonModal.tsx @@ -2,7 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 import { useEffect, useMemo, useRef, useState } from 'react'; -import { ClusterStatus } from '../../types'; +import { ClusterSummary } from '../../types'; +import { fetchClusterStatus } from '../../api'; +import { toClusterSummary } from '../../state/clusterSummary'; import { CloseXIcon } from '../nodes/shared/index'; function StatusJsonModal({ @@ -12,11 +14,10 @@ function StatusJsonModal({ open: boolean; onClose: () => void; }) { - const [snapshotStatus, setSnapshotStatus] = useState(null); + const [snapshotStatus, setSnapshotStatus] = useState(null); const [fetchError, setFetchError] = useState(null); const [fetching, setFetching] = useState(false); const [collapseAllVersion, setCollapseAllVersion] = useState(0); - const wasOpenRef = useRef(false); useEffect(() => { if (!open) return; @@ -30,31 +31,28 @@ function StatusJsonModal({ }, [open, onClose]); useEffect(() => { - if (open && !wasOpenRef.current) { - // Fetch fresh data from HTTP on each open - wasOpenRef.current = true; + let cancelled = false; + if (open) { setFetching(true); setFetchError(null); - fetch('/status/json') - .then((res) => { - if (!res.ok) throw new Error(`HTTP ${res.status}`); - return res.json(); - }) + fetchClusterStatus() .then((data) => { - setSnapshotStatus(data as ClusterStatus); + if (cancelled) return; + setSnapshotStatus(toClusterSummary(data)); setCollapseAllVersion((version) => version + 1); }) .catch((err) => { + if (cancelled) return; setFetchError((err as Error).message); }) .finally(() => { + if (cancelled) return; setFetching(false); }); - return; - } - if (!open) { - wasOpenRef.current = false; + } else { + setSnapshotStatus(null); } + return () => { cancelled = true; }; }, [open]); const statusJsonValue = useMemo(() => { @@ -70,10 +68,10 @@ function StatusJsonModal({ return a.localeCompare(b); }; - const sortedStatus: ClusterStatus = { + const sortedStatus: ClusterSummary = { ...snapshotStatus, sites: [...(snapshotStatus.sites || [])].sort((a, b) => compareNames(a.name, b.name)), - nodes: [...(snapshotStatus.nodes || [])].sort((a, b) => compareNames(a.nodeInfo?.name, b.nodeInfo?.name)), + nodeSummaries: [...(snapshotStatus.nodeSummaries || [])].sort((a, b) => compareNames(a.name, b.name)), gatewayPools: [...(snapshotStatus.gatewayPools || [])].sort((a, b) => compareNames(a.name, b.name)) }; @@ -86,7 +84,7 @@ function StatusJsonModal({
event.stopPropagation()}>
-
Cluster Status JSON
+
Cluster Status JSON (summary only)