diff --git a/api/net/v1alpha1/types.go b/api/net/v1alpha1/types.go index 04d0d731f..8882224cb 100644 --- a/api/net/v1alpha1/types.go +++ b/api/net/v1alpha1/types.go @@ -92,12 +92,14 @@ type HealthCheckSettings struct { // ReceiveInterval is the minimum interval between received health check packets. // Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + // Defaults to 15s when omitted from the selected health check scope. // +kubebuilder:validation:XIntOrString // +optional ReceiveInterval *intstr.IntOrString `json:"receiveInterval,omitempty"` // TransmitInterval is the minimum interval between transmitted health check packets. // Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + // Defaults to 15s when omitted from the selected health check scope. // +kubebuilder:validation:XIntOrString // +optional TransmitInterval *intstr.IntOrString `json:"transmitInterval,omitempty"` diff --git a/cmd/kubectl-unbounded/app/net/create.go b/cmd/kubectl-unbounded/app/net/create.go index 8a188080d..c2dd74fb9 100644 --- a/cmd/kubectl-unbounded/app/net/create.go +++ b/cmd/kubectl-unbounded/app/net/create.go @@ -36,8 +36,8 @@ type healthCheckFlags struct { func (b *healthCheckFlags) addToFlags(cmd *cobra.Command) { cmd.Flags().BoolVar(&b.enabled, "health-check-enabled", false, "Enable UDP health probes over tunnels") cmd.Flags().Int32Var(&b.detectMultiplier, "health-check-detect-multiplier", 0, "Number of missed probes before marking a peer down") - cmd.Flags().StringVar(&b.receiveInterval, "health-check-receive-interval", "", "Min interval between received probes before declaring down, e.g. 300ms") - cmd.Flags().StringVar(&b.transmitInterval, "health-check-transmit-interval", "", "Interval between transmitted health probes, e.g. 300ms") + cmd.Flags().StringVar(&b.receiveInterval, "health-check-receive-interval", "", "Min interval between received probes before declaring down, e.g. 300ms (node default: 15s)") + cmd.Flags().StringVar(&b.transmitInterval, "health-check-transmit-interval", "", "Interval between transmitted health probes, e.g. 300ms (node default: 15s)") cmd.Flags().Int32Var(&b.tunnelMTU, "tunnel-mtu", 0, "MTU for tunnel interfaces in this scope") cmd.Flags().StringVar(&b.tunnelProtocol, "tunnel-protocol", "", "Tunnel encapsulation protocol (WireGuard, GENEVE, or Auto)") _ = cmd.RegisterFlagCompletionFunc("tunnel-protocol", func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { //nolint:errcheck diff --git a/cmd/kubectl-unbounded/app/net/create_test.go b/cmd/kubectl-unbounded/app/net/create_test.go index d70c48be0..563252c1c 100644 --- a/cmd/kubectl-unbounded/app/net/create_test.go +++ b/cmd/kubectl-unbounded/app/net/create_test.go @@ -7,8 +7,39 @@ import ( "bytes" "strings" "testing" + + "github.com/spf13/cobra" ) +func TestHealthCheckFlagsPreserveRuntimeDefaults(t *testing.T) { + cmd := &cobra.Command{} + flags := &healthCheckFlags{} + flags.addToFlags(cmd) + flags.selectedFrom(cmd) + + if flags.toObject() != nil { + t.Fatal("omitted health flags must preserve the node's runtime defaults") + } + + for _, name := range []string{"health-check-transmit-interval", "health-check-receive-interval"} { + flag := cmd.Flags().Lookup(name) + if flag.DefValue != "" || !strings.Contains(flag.Usage, "15s") { + t.Fatalf("flag %s must document the inherited 15s default without serializing it", name) + } + } + + if err := cmd.Flags().Set("health-check-transmit-interval", "60s"); err != nil { + t.Fatal(err) + } + + flags.selectedFrom(cmd) + + got := flags.toObject() + if len(got) != 1 || got["transmitInterval"] != "60s" { + t.Fatalf("explicit interval or partial settings changed: %v", got) + } +} + func TestCreateSiteUsesSharedSiteAPI(t *testing.T) { t.Parallel() diff --git a/cmd/unbounded-net-controller/cluster_status.go b/cmd/unbounded-net-controller/cluster_status.go index 1681435af..6185da8c8 100644 --- a/cmd/unbounded-net-controller/cluster_status.go +++ b/cmd/unbounded-net-controller/cluster_status.go @@ -1061,15 +1061,9 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway siteNodes[site][name] = true - var allPeers []WireGuardPeerStatus - - for _, p := range n.Peers { - if p.PeerType == "site" || p.PeerType == "gateway" { - allPeers = append(allPeers, p) - } - } - - nodePeers[name] = allPeers + // Keep the immutable snapshot's slice; filter when reading rather + // than copying every peer, including for scopes above the size limit. + nodePeers[name] = n.Peers for _, p := range n.Peers { if p.PeerType == "gateway" && p.Name != "" && p.SiteName == site { @@ -1111,6 +1105,10 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway } for _, peer := range nodePeers[srcNode] { + if !isConnectivityMatrixPeer(peer) { + continue + } + tgtNode := peer.Name if tgtNode == "" || tgtNode == srcNode || !nodeSet[tgtNode] { continue @@ -1153,6 +1151,10 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway poolNodeSet[name] = true for _, peer := range nodePeers[name] { + if !isConnectivityMatrixPeer(peer) { + continue + } + peerName := strings.TrimSpace(peer.Name) if peerName == "" { continue @@ -1165,6 +1167,10 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway for srcNodeName, peers := range nodePeers { for _, peer := range peers { + if !isConnectivityMatrixPeer(peer) { + continue + } + if strings.TrimSpace(peer.Name) == name { if _, ok := nodeByName[srcNodeName]; ok { poolNodeSet[srcNodeName] = true @@ -1188,3 +1194,7 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway return result } + +func isConnectivityMatrixPeer(peer WireGuardPeerStatus) bool { + return peer.PeerType == "site" || peer.PeerType == "gateway" +} diff --git a/cmd/unbounded-net-controller/cni_status_test.go b/cmd/unbounded-net-controller/cni_status_test.go index 044163a09..569a41c07 100644 --- a/cmd/unbounded-net-controller/cni_status_test.go +++ b/cmd/unbounded-net-controller/cni_status_test.go @@ -121,7 +121,7 @@ func TestCNIGuardProtoStatusRecovery(t *testing.T) { if source == "ws" || source == "apiserver-ws" { var messageType string - messageType, ack = handleProtoWSMessage(health, data, source) + messageType, ack = handleProtoWSBytes(health, data, source) if messageType != "node_status_ack" { t.Fatalf("unexpected message type %q: %+v", messageType, ack) } diff --git a/cmd/unbounded-net-controller/gzip_writer_pool.go b/cmd/unbounded-net-controller/gzip_writer_pool.go new file mode 100644 index 000000000..c0b90ba09 --- /dev/null +++ b/cmd/unbounded-net-controller/gzip_writer_pool.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "compress/gzip" + "io" +) + +const gzipIdleWriterLimit = 4 + +type gzipWriterPool struct { + idle chan *gzip.Writer +} + +func newGzipWriterPool() *gzipWriterPool { + return &gzipWriterPool{idle: make(chan *gzip.Writer, gzipIdleWriterLimit)} +} + +func (p *gzipWriterPool) get(output io.Writer) (*gzip.Writer, error) { + var writer *gzip.Writer + + select { + case writer = <-p.idle: + default: + var err error + + writer, err = gzip.NewWriterLevel(io.Discard, gzip.BestSpeed) + if err != nil { + return nil, err + } + } + + writer.Reset(output) + + return writer, nil +} + +func (p *gzipWriterPool) put(writer *gzip.Writer, completed bool) { + closed := false + + defer func() { + // Detach even if Close panics, and reset before another request can + // acquire the writer. Failed or interrupted responses are not pooled. + writer.Reset(io.Discard) + + if !completed || !closed { + return + } + + select { + case p.idle <- writer: + default: + } + }() + + closed = writer.Close() == nil +} diff --git a/cmd/unbounded-net-controller/gzip_writer_pool_test.go b/cmd/unbounded-net-controller/gzip_writer_pool_test.go new file mode 100644 index 000000000..0149c3be4 --- /dev/null +++ b/cmd/unbounded-net-controller/gzip_writer_pool_test.go @@ -0,0 +1,476 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// freshGzipHandler preserves the original handler for behavior and allocation +// comparisons against the reusable-writer path. +func freshGzipHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") || + strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + next.ServeHTTP(w, r) + return + } + + gz, err := gzip.NewWriterLevel(w, gzip.BestSpeed) + if err != nil { + next.ServeHTTP(w, r) + return + } + + defer func() { _ = gz.Close() }() + + w.Header().Set("Content-Encoding", "gzip") + w.Header().Del("Content-Length") + next.ServeHTTP(&gzipResponseWriter{ResponseWriter: w, Writer: gz}, r) + }) +} + +func TestGzipHandlerPreservesBehavior(t *testing.T) { + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Response", r.URL.Path) + + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + + switch r.URL.Path { + case "/empty": + w.WriteHeader(http.StatusAccepted) + case "/no-content": + w.WriteHeader(http.StatusNoContent) + case "/error": + http.Error(w, "not found", http.StatusNotFound) + default: + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusCreated) + + for _, chunk := range []string{"response:", r.URL.Path} { + if _, err := io.WriteString(w, chunk); err != nil { + t.Error(err) + } + } + } + }) + fresh, pooled := freshGzipHandler(next), gzipHandler(next) + + for _, encoding := range []string{"", "br", "gzip", "br, gzip", "gzip;q=0", "GZIP", "xgzip"} { + for _, upgrade := range []string{"", "WebSocket"} { + for _, path := range []string{"/first", "/empty", "/no-content", "/error", "/second"} { + t.Run(encoding+"/"+upgrade+path, func(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.Header.Set("Accept-Encoding", encoding) + request.Header.Set("Upgrade", upgrade) + + want, got := httptest.NewRecorder(), httptest.NewRecorder() + + for _, recorder := range []*httptest.ResponseRecorder{want, got} { + recorder.Header().Set("Content-Length", "123") + recorder.Header().Set("Content-Encoding", "original") + } + + fresh.ServeHTTP(want, request) + pooled.ServeHTTP(got, request) + + if got.Code != want.Code || got.Flushed != want.Flushed || + !reflect.DeepEqual(got.Header(), want.Header()) || !bytes.Equal(got.Body.Bytes(), want.Body.Bytes()) { + t.Fatalf("response behavior changed: status=%d/%d flush=%t/%t headers=%v/%v", got.Code, want.Code, got.Flushed, want.Flushed, got.Header(), want.Header()) + } + }) + } + } + } +} + +func TestGzipHandlerHTTPRoundTrips(t *testing.T) { + handler := gzipHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Response", r.URL.Path) + + if r.URL.Path == "/error" { + http.Error(w, "not found", http.StatusNotFound) + return + } + + if r.URL.Path == "/no-content" { + w.WriteHeader(http.StatusNoContent) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusCreated) + + if r.URL.Path != "/empty" { + if _, err := io.WriteString(w, strings.Repeat(r.URL.Path, 100)); err != nil { + t.Error(err) + } + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := server.Client() + client.Timeout = 10 * time.Second + + check := func(method, path string) { + t.Helper() + + request, err := http.NewRequestWithContext(t.Context(), method, server.URL+path, nil) + if err != nil { + t.Error(err) + return + } + + request.Header.Set("Accept-Encoding", "gzip") + + response, err := client.Do(request) + if err != nil { + t.Error(err) + return + } + defer response.Body.Close() + + var reader io.Reader = response.Body + + if method != http.MethodHead && path != "/no-content" { + compressed, err := gzip.NewReader(response.Body) + if err != nil { + t.Error(err) + return + } + defer compressed.Close() + + reader = compressed + } + + body, err := io.ReadAll(reader) + if err != nil { + t.Error(err) + return + } + + wantStatus, wantBody := http.StatusCreated, strings.Repeat(path, 100) + switch path { + case "/empty": + wantBody = "" + case "/no-content": + wantStatus, wantBody = http.StatusNoContent, "" + case "/error": + wantStatus, wantBody = http.StatusNotFound, "not found\n" + } + + if method == http.MethodHead { + wantBody = "" + } + + if response.StatusCode != wantStatus || string(body) != wantBody || + response.Header.Get("Content-Encoding") != "gzip" || response.Header.Get("X-Response") != path { + t.Errorf("round trip %s: status=%d body=%q headers=%v", path, response.StatusCode, body, response.Header) + } + } + + for _, path := range []string{"/first", "/different", "/empty", "/no-content", "/error", "/last"} { + check(http.MethodGet, path) + } + + check(http.MethodHead, "/head") + check(http.MethodGet, "/after-head") + + var workers sync.WaitGroup + + for worker := range 12 { + workers.Go(func() { + for request := range 4 { + check(http.MethodGet, fmt.Sprintf("/worker-%d-request-%d", worker, request)) + } + }) + } + + workers.Wait() +} + +func TestGzipWriterPoolBoundedAndDetached(t *testing.T) { + pool := newGzipWriterPool() + writers := make([]*gzip.Writer, gzipIdleWriterLimit+3) + outputs := make([]bytes.Buffer, len(writers)) + + for i := range writers { + writer, err := pool.get(&outputs[i]) + if err != nil { + t.Fatal(err) + } + + writers[i] = writer + + if _, err := io.WriteString(writer, strconv.Itoa(i)); err != nil { + t.Fatal(err) + } + } + + for _, writer := range writers { + pool.put(writer, true) + } + + if len(pool.idle) != gzipIdleWriterLimit { + t.Fatalf("idle pool length=%d, want %d", len(pool.idle), gzipIdleWriterLimit) + } + + // Take ownership of all idle writers before inspecting their detached + // outputs. Overflow writers are already outside the pool. + for len(pool.idle) > 0 { + <-pool.idle + } + + for i, writer := range writers { + before := outputs[i].Len() + + if _, err := io.WriteString(writer, "discard this"); err != nil { + t.Fatal(err) + } + + if err := writer.Close(); err != nil { + t.Fatal(err) + } + + if outputs[i].Len() != before { + t.Fatal("released writer retained its response destination") + } + } +} + +func TestGzipWriterPoolResetsHeadersAndSurvivesGC(t *testing.T) { + pool := newGzipWriterPool() + + var first, second bytes.Buffer + + writer, err := pool.get(&first) + if err != nil { + t.Fatal(err) + } + + writer.Name, writer.Comment, writer.Extra, writer.OS = "private-name", "private-comment", []byte("private-extra"), 7 + + if _, err := io.WriteString(writer, "first body"); err != nil { + t.Fatal(err) + } + + pool.put(writer, true) + runtime.GC() + runtime.GC() + + reused, err := pool.get(&second) + if err != nil { + t.Fatal(err) + } + + if reused != writer { + t.Fatal("idle writer was not reused") + } + + if _, err := io.WriteString(reused, "second body"); err != nil { + t.Fatal(err) + } + + pool.put(reused, true) + + reader, err := gzip.NewReader(&second) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + + body, err := io.ReadAll(reader) + if err != nil || string(body) != "second body" { + t.Fatalf("reused body: %q %v", body, err) + } + + if reader.Name != "" || reader.Comment != "" || len(reader.Extra) != 0 || reader.OS != 255 { + t.Fatalf("reused writer retained gzip headers: %+v", reader.Header) + } +} + +type gzipFailureResponseWriter struct { + *httptest.ResponseRecorder + failAt int + panicAt int + writes int +} + +func (w *gzipFailureResponseWriter) Write(data []byte) (int, error) { + w.writes++ + if w.writes == w.panicAt { + panic("gzip test output panic") + } + + if w.writes == w.failAt { + return 0, io.ErrClosedPipe + } + + return w.ResponseRecorder.Write(data) +} + +func invokeGzipHandler(handler http.Handler, writer http.ResponseWriter, request *http.Request) (panicValue any) { + defer func() { panicValue = recover() }() + + handler.ServeHTTP(writer, request) + + return nil +} + +func TestGzipHandlerWriteFailuresAndPanics(t *testing.T) { + for _, test := range []struct { + name string + failAt int + panicAt int + handlerPanic bool + beforeWrite bool + }{ + {name: "write failure", failAt: 1}, + {name: "close failure", failAt: 2}, + {name: "close panic", panicAt: 2}, + {name: "handler panic", handlerPanic: true}, + {name: "handler panic before write", handlerPanic: true, beforeWrite: true}, + } { + t.Run(test.name, func(t *testing.T) { + var used *gzip.Writer + + var writeErr error + + handler := gzipHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + used = w.(*gzipResponseWriter).Writer.(*gzip.Writer) + + if test.beforeWrite && r.URL.Path == "/failure" { + panic("gzip test handler panic") + } + + _, writeErr = io.WriteString(w, r.URL.Path) + + if test.handlerPanic && r.URL.Path == "/failure" { + panic("gzip test handler panic") + } + })) + failed := &gzipFailureResponseWriter{ + ResponseRecorder: httptest.NewRecorder(), failAt: test.failAt, panicAt: test.panicAt, + } + request := httptest.NewRequest(http.MethodGet, "/failure", nil) + request.Header.Set("Accept-Encoding", "gzip") + panicValue := invokeGzipHandler(handler, failed, request) + + wantPanic := "" + if test.panicAt != 0 { + wantPanic = "gzip test output panic" + } else if test.handlerPanic { + wantPanic = "gzip test handler panic" + } + + if wantPanic != "" && panicValue != wantPanic || wantPanic == "" && panicValue != nil { + t.Fatalf("panic behavior changed: got %v, want %q", panicValue, wantPanic) + } + + if test.failAt == 1 && !errors.Is(writeErr, io.ErrClosedPipe) { + t.Fatalf("write error was suppressed: %v", writeErr) + } + + oldWriter, oldWrites := used, failed.writes + if _, err := io.WriteString(oldWriter, "discard this"); err != nil { + t.Fatalf("failed writer was not reset: %v", err) + } + + if err := oldWriter.Close(); err != nil { + t.Fatal(err) + } + + if failed.writes != oldWrites { + t.Fatal("failed or panicking writer retained its response destination") + } + + recorder := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodGet, "/healthy", nil) + request.Header.Set("Accept-Encoding", "gzip") + handler.ServeHTTP(recorder, request) + + if used == oldWriter || writeErr != nil { + t.Fatal("failed or panicking writer was returned to the idle pool") + } + + reader, err := gzip.NewReader(recorder.Body) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + + body, err := io.ReadAll(reader) + if err != nil || string(body) != "/healthy" { + t.Fatalf("failure contaminated the next response: %q %v", body, err) + } + }) + } +} + +func BenchmarkGzipResponseHandler(b *testing.B) { + for _, response := range []struct { + name string + status int + body string + }{ + { + name: "discovery", status: http.StatusOK, + body: `{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"status.net.unbounded-cloud.io/v1alpha1","resources":[{"name":"status/push","singularName":"","namespaced":false,"kind":"NodeStatusPush","verbs":["create"]},{"name":"status/nodews","singularName":"","namespaced":false,"kind":"NodeStatusStream","verbs":["get"]},{"name":"status/json","singularName":"","namespaced":false,"kind":"ClusterStatus","verbs":["get"]},{"name":"token/node","singularName":"","namespaced":false,"kind":"TokenRequest","verbs":["create"]},{"name":"token/viewer","singularName":"","namespaced":false,"kind":"TokenRequest","verbs":["create"]}]}`, + }, + {name: "not-found", status: http.StatusNotFound, body: "404 page not found\n"}, + {name: "empty", status: http.StatusOK}, + } { + for _, mode := range []struct { + name string + wrap func(http.Handler) http.Handler + }{ + {name: "fresh", wrap: freshGzipHandler}, + {name: "pooled", wrap: gzipHandler}, + } { + b.Run(response.name+"/"+mode.name, func(b *testing.B) { + body := []byte(response.body) + handler := mode.wrap(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(response.status) + + if _, err := w.Write(body); err != nil { + b.Fatal(err) + } + })) + request := httptest.NewRequest(http.MethodGet, "/", nil) + request.Header.Set("Accept-Encoding", "gzip") + handler.ServeHTTP(httptest.NewRecorder(), request) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + + if recorder.Code != response.status { + b.Fatal(recorder.Code) + } + } + }) + } + } +} diff --git a/cmd/unbounded-net-controller/health_state.go b/cmd/unbounded-net-controller/health_state.go index 2d067daa7..fcc3eadb6 100644 --- a/cmd/unbounded-net-controller/health_state.go +++ b/cmd/unbounded-net-controller/health_state.go @@ -61,6 +61,7 @@ type healthState struct { tokenAuth *tokenAuthenticator nodeServiceAccount string // expected service account in namespace:name format nodeTokenVerifier serviceAccountTokenVerifier + nodeAuthReady func() bool // Required only by the startup-selected local OIDC verifier. // Pull fallback toggle (controlled via dashboard WS message; default: disabled). pullEnabled atomic.Bool @@ -233,6 +234,10 @@ func (h *healthState) readinessStatus(_ context.Context) (bool, string) { return false, fmt.Sprintf("token verifier not ready: %s", reason) } + if h.nodeAuthReady != nil && !h.nodeAuthReady() { + return false, "node authentication informer caches not ready" + } + _, err := h.clientset.Discovery().ServerVersion() if err != nil { return false, "cannot connect to kubernetes api" diff --git a/cmd/unbounded-net-controller/main.go b/cmd/unbounded-net-controller/main.go index 2873501bc..67b04c973 100644 --- a/cmd/unbounded-net-controller/main.go +++ b/cmd/unbounded-net-controller/main.go @@ -453,10 +453,16 @@ func run(cfg *config.Config, forceNotLeader bool) error { klog.Fatalf("Failed to create token issuer: %v", err) } + // Do not block serving health endpoints on RBAC or initial cache sync. + // Local OIDC authentication fails closed until both caches are ready. + nodeAuthCaches := newNodeAuthInformers(ctx, clientset, controllerNamespace, cfg.InformerResyncPeriod) + nodeAuthCaches.start() + podLister := nodeAuthCaches.pods.Lister() + nodeTokenVerifier, err := initializeNodeTokenVerifier(ctx, clientset, cfg.OIDCIssuerURL, cfg.OIDCAudience, controllerServiceAccountTokenPath, - func(ctx context.Context, issuer, audience string) (serviceAccountTokenVerifier, error) { + nodeAuthCaches.wrapOIDCFactory(func(ctx context.Context, issuer, audience string) (serviceAccountTokenVerifier, error) { return authn.NewKubernetesOIDCVerifier(ctx, issuer, audience) - }) + })) if err != nil { klog.Fatalf("Failed to initialize node token verifier: %v", err) } @@ -478,6 +484,7 @@ func run(cfg *config.Config, forceNotLeader bool) error { tokenAuth: newTokenAuthenticator(nodeTokenVerifier, []string{fmt.Sprintf("%s:unbounded-net-node", controllerNamespace)}), nodeServiceAccount: fmt.Sprintf("%s:unbounded-net-node", controllerNamespace), nodeTokenVerifier: nodeTokenVerifier, + nodeAuthReady: nodeAuthCaches.readinessCheck(nodeTokenVerifier), registerAggregatedAPIServer: cfg.RegisterAggregatedAPIServer, statusWSKeepaliveInterval: cfg.StatusWSKeepaliveInterval, statusWSKeepaliveFailureCount: cfg.StatusWSKeepaliveFailureCount, @@ -514,15 +521,6 @@ func run(cfg *config.Config, forceNotLeader bool) error { // Create informer factory informerFactory := informers.NewSharedInformerFactory(clientset, cfg.InformerResyncPeriod) - // Create pod informer for unbounded-net-node pods (filtered by label selector) - podInformerFactory := informers.NewSharedInformerFactoryWithOptions(clientset, cfg.InformerResyncPeriod, - informers.WithNamespace(controllerNamespace), - informers.WithTweakListOptions(func(opts *metav1.ListOptions) { - opts.LabelSelector = "app.kubernetes.io/name=unbounded-net-node" - }), - ) - podLister := podInformerFactory.Core().V1().Pods().Lister() - var ( gatewayPoolInformer cache.SharedIndexInformer sitePeeringInformer cache.SharedIndexInformer @@ -699,8 +697,11 @@ func run(cfg *config.Config, forceNotLeader bool) error { // Start informers after all informers are created. informerFactory.Start(ctx.Done()) dynamicInformerFactory.Start(ctx.Done()) - podInformerFactory.Start(ctx.Done()) - podInformerFactory.WaitForCacheSync(ctx.Done()) + + if !cache.WaitForCacheSync(ctx.Done(), nodeAuthCaches.pods.Informer().HasSynced) { + klog.Info("Leadership ended before the shared node Pod cache synced") + return + } <-ctx.Done() } diff --git a/cmd/unbounded-net-controller/matrix_memory_test.go b/cmd/unbounded-net-controller/matrix_memory_test.go new file mode 100644 index 000000000..e7ed3f048 --- /dev/null +++ b/cmd/unbounded-net-controller/matrix_memory_test.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "slices" + "testing" + "time" +) + +func TestConnectivityMatrixMixedScopesPreservesPeers(t *testing.T) { + nodes := make([]*NodeStatusResponse, 0, 102) + for i := range 101 { + nodes = append(nodes, &NodeStatusResponse{NodeInfo: NodeInfo{ + Name: fmt.Sprintf("node-%d", i), SiteName: "large", + }}) + } + + nodes[0].Peers = []WireGuardPeerStatus{{ + Name: "gateway", PeerType: "gateway", SiteName: "small", + Tunnel: PeerTunnelStatus{LastHandshake: time.Unix(1, 0)}, + }} + nodes[1].Peers = []WireGuardPeerStatus{{Name: "gateway", PeerType: "ignored"}} + nodes = append(nodes, &NodeStatusResponse{ + NodeInfo: NodeInfo{Name: "gateway", SiteName: "small"}, + Peers: []WireGuardPeerStatus{ + {Name: "node-0", PeerType: "site", HealthCheck: &HealthCheckPeerStatus{Status: "up"}}, + {Name: "node-0", PeerType: "ignored", HealthCheck: &HealthCheckPeerStatus{Status: "down"}}, + {Name: "node-2", PeerType: "ignored"}, + {Name: "gateway", PeerType: "ignored", HealthCheck: &HealthCheckPeerStatus{Status: "down"}}, + }, + }) + + before, err := json.Marshal(nodes) + if err != nil { + t.Fatal(err) + } + + matrix := buildConnectivityMatrix(nodes, []GatewayPoolStatus{{Name: "pool", Gateways: []string{"gateway"}}}) + if _, ok := matrix["large"]; ok { + t.Fatal("oversized site produced a matrix") + } + + if matrix["small"] == nil || !slices.Equal(matrix["small"].Nodes, []string{"gateway"}) { + t.Fatalf("small site lost its matrix: %+v", matrix["small"]) + } + + pool := matrix["pool:pool"] + if pool == nil || !slices.Equal(pool.Nodes, []string{"gateway", "node-0"}) { + t.Fatalf("small pool crossing a large site has wrong membership: %+v", pool) + } + + if pool.Results["gateway"]["node-0"] != "up" || pool.Results["node-0"]["gateway"] != "up" { + t.Fatalf("pool connectivity changed: %+v", pool.Results) + } + + after, err := json.Marshal(nodes) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(before, after) { + t.Fatal("matrix construction mutated the shared node snapshots") + } +} + +func TestConnectivityMatrixDoesNotCopyPeerSlices(t *testing.T) { + nodes := matrixBenchmarkNodes(200) + + peers := matrixBenchmarkNodes(2000)[0].Peers + for _, node := range nodes { + node.Peers = peers + } + + var matrix map[string]*SiteMatrix + + result := testing.Benchmark(func(b *testing.B) { + for b.Loop() { + matrix = buildConnectivityMatrix(nodes, nil) + } + }) + + if matrix != nil { + t.Fatal("oversized site produced a matrix") + } + // Allow map bookkeeping, but not storage proportional to every peer. + if allocated := result.AllocedBytesPerOp(); allocated > 512*1024 { + t.Fatalf("matrix copied peer data: %d bytes per call", allocated) + } +} + +func TestConnectivityMatrixSizeBoundary(t *testing.T) { + for _, count := range []int{0, 100, 101} { + t.Run(fmt.Sprintf("nodes-%d", count), func(t *testing.T) { + matrix := buildConnectivityMatrix(matrixBenchmarkNodes(count), nil) + if count != 100 { + if matrix != nil { + t.Fatal("empty or oversized site produced a matrix") + } + + return + } + + if matrix["site-a"] == nil || len(matrix["site-a"].Nodes) != count { + t.Fatal("site at the size limit lost its matrix") + } + }) + } +} diff --git a/cmd/unbounded-net-controller/memory_bench_test.go b/cmd/unbounded-net-controller/memory_bench_test.go new file mode 100644 index 000000000..ca548d0a6 --- /dev/null +++ b/cmd/unbounded-net-controller/memory_bench_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "testing" + + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func benchmarkProtoStatus(b *testing.B, peerCount int) []byte { + b.Helper() + + status := &statusproto.NodeStatusFull{ + NodeInfo: &statusproto.NodeInfo{Name: "node-a", SiteName: "site-a"}, + } + + for i := range peerCount { + name := fmt.Sprintf("peer-%d", i) + status.Peers = append(status.Peers, &statusproto.PeerStatus{ + Name: name, PeerType: "site", SiteName: "site-a", + Tunnel: &statusproto.PeerTunnelStatus{ + Interface: "wg0", PublicKey: name, + Endpoint: "10.224.0.1:51820", AllowedIps: []string{"10.244.0.0/24"}, + }, + HealthCheck: &statusproto.HealthCheckPeerStatus{Enabled: true, Status: "up"}, + }) + status.BpfEntries = append(status.BpfEntries, &statusproto.BpfEntry{ + Cidr: "10.244.0.0/24", Remote: "10.224.0.1", Node: name, + InterfaceName: "wg0", Protocol: "WireGuard", + }) + } + + data, err := proto.Marshal(&statusproto.NodeStatusMessage{ + Type: "node_status_full", NodeName: "node-a", Status: status, + }) + if err != nil { + b.Fatal(err) + } + + return data +} + +func BenchmarkProtoWSStatusFrame(b *testing.B) { + for _, peerCount := range []int{100, 2000} { + b.Run(fmt.Sprintf("peers-%d", peerCount), func(b *testing.B) { + data := benchmarkProtoStatus(b, peerCount) + health := &healthState{statusCache: NewNodeStatusCache()} + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + decoded, err := decodeProtoWSMessage(data) + if err != nil || decoded.nodeName != "node-a" { + b.Fatalf("unexpected decoded message: %+v, err=%v", decoded, err) + } + + _, ack := handleProtoWSMessage(health, decoded, "ws") + if ack.Status != "ok" { + b.Fatalf("unexpected ack: %+v", ack) + } + } + }) + } +} + +func matrixBenchmarkNodes(count int) []*NodeStatusResponse { + peers := make([]WireGuardPeerStatus, count) + + nodes := make([]*NodeStatusResponse, count) + for i := range count { + name := fmt.Sprintf("node-%d", i) + peers[i] = WireGuardPeerStatus{Name: name, PeerType: "site", SiteName: "site-a"} + nodes[i] = &NodeStatusResponse{ + NodeInfo: NodeInfo{Name: name, SiteName: "site-a"}, + Peers: peers, + } + } + + return nodes +} + +func BenchmarkBuildConnectivityMatrix(b *testing.B) { + for _, count := range []int{100, 101, 2000} { + b.Run(fmt.Sprintf("nodes-%d", count), func(b *testing.B) { + nodes := matrixBenchmarkNodes(count) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + matrix := buildConnectivityMatrix(nodes, nil) + if count > 100 && matrix != nil { + b.Fatal("oversized site produced a matrix") + } + } + }) + } +} diff --git a/cmd/unbounded-net-controller/metrics.go b/cmd/unbounded-net-controller/metrics.go index cc5f7999d..c05469630 100644 --- a/cmd/unbounded-net-controller/metrics.go +++ b/cmd/unbounded-net-controller/metrics.go @@ -14,6 +14,11 @@ const controllerMetricsNamespace = "unbounded_cni_controller" // Status/push metrics. var ( + peerMeasurementUpdatesTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: controllerMetricsNamespace, + Name: "peer_measurement_updates_total", + Help: "Compact peer measurement batches by outcome (applied, resync, error).", + }, []string{"outcome"}) nodeStatusPushesTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: controllerMetricsNamespace, Name: "node_status_pushes_total", diff --git a/cmd/unbounded-net-controller/node_auth_informers.go b/cmd/unbounded-net-controller/node_auth_informers.go new file mode 100644 index 000000000..2dcc23622 --- /dev/null +++ b/cmd/unbounded-net-controller/node_auth_informers.go @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + + "github.com/Azure/unbounded/internal/net/authn" +) + +// These watches run on every serving replica, independently of leadership. +// HasSynced is an initial sync barrier, not proof of ongoing watch freshness. +type nodeAuthInformers struct { + ctx context.Context + namespace string + podFactory informers.SharedInformerFactory + saFactory informers.SharedInformerFactory + pods coreinformers.PodInformer + serviceAccounts coreinformers.ServiceAccountInformer +} + +func newNodeAuthInformers(ctx context.Context, client kubernetes.Interface, namespace string, resync time.Duration) *nodeAuthInformers { + pods := informers.NewSharedInformerFactoryWithOptions(client, resync, + informers.WithNamespace(namespace), + informers.WithTweakListOptions(func(opts *metav1.ListOptions) { + opts.LabelSelector = "app.kubernetes.io/name=unbounded-net-node" + }), + ) + serviceAccounts := informers.NewSharedInformerFactoryWithOptions(client, resync, informers.WithNamespace(namespace)) + result := &nodeAuthInformers{ + ctx: ctx, namespace: namespace, + podFactory: pods, saFactory: serviceAccounts, + pods: pods.Core().V1().Pods(), serviceAccounts: serviceAccounts.Core().V1().ServiceAccounts(), + } + // Instantiate both informers before starting their factories. + result.pods.Informer() + result.serviceAccounts.Informer() + + return result +} + +func (i *nodeAuthInformers) start() { + i.podFactory.Start(i.ctx.Done()) + i.saFactory.Start(i.ctx.Done()) +} + +func (i *nodeAuthInformers) ready() bool { + return i.ctx.Err() == nil && + !i.pods.Informer().IsStopped() && !i.serviceAccounts.Informer().IsStopped() && + i.pods.Informer().HasSynced() && i.serviceAccounts.Informer().HasSynced() +} + +func (i *nodeAuthInformers) readinessCheck(verifier serviceAccountTokenVerifier) func() bool { + if _, ok := verifier.(*authn.PodBoundTokenVerifier); ok { + return i.ready + } + + return nil +} + +func (i *nodeAuthInformers) wrapOIDCFactory(factory oidcVerifierFactory) oidcVerifierFactory { + return func(ctx context.Context, issuer, audience string) (serviceAccountTokenVerifier, error) { + verifier, err := factory(ctx, issuer, audience) + if err != nil { + return nil, err + } + + return authn.NewPodBoundTokenVerifier(verifier, authn.PodBoundTokenVerifierOptions{ + Namespace: i.namespace, ServiceAccount: "unbounded-net-node", + Pods: i.pods.Lister(), ServiceAccounts: i.serviceAccounts.Lister(), Ready: i.ready, + }), nil + } +} diff --git a/cmd/unbounded-net-controller/node_auth_informers_test.go b/cmd/unbounded-net-controller/node_auth_informers_test.go new file mode 100644 index 000000000..190f592c1 --- /dev/null +++ b/cmd/unbounded-net-controller/node_auth_informers_test.go @@ -0,0 +1,374 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + + "github.com/Azure/unbounded/internal/net/authn" +) + +func nodeAuthObjects() (*corev1.Pod, *corev1.ServiceAccount, *authn.KubernetesServiceAccountIdentity) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "unbounded-system", Name: "agent", UID: "pod-uid", + Labels: map[string]string{"app.kubernetes.io/name": "unbounded-net-node"}, + }, + Spec: corev1.PodSpec{ServiceAccountName: "unbounded-net-node", NodeName: "node-a"}, + } + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: pod.Namespace, Name: pod.Spec.ServiceAccountName, UID: "sa-uid"}} + identity := &authn.KubernetesServiceAccountIdentity{ + Subject: "system:serviceaccount:unbounded-system:unbounded-net-node", + Namespace: pod.Namespace, ServiceAccountName: sa.Name, ServiceAccountUID: string(sa.UID), + PodName: pod.Name, PodUID: string(pod.UID), NodeName: pod.Spec.NodeName, + } + + return pod, sa, identity +} + +func waitNodeAuthCondition(t *testing.T, condition func() bool) { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + + for !condition() { + select { + case <-ctx.Done(): + t.Fatal("timed out waiting for authentication cache state") + case <-ticker.C: + } + } +} + +func startNodeAuthTestCaches(t *testing.T, client *k8sfake.Clientset) (*nodeAuthInformers, context.CancelFunc) { + t.Helper() + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + + caches := newNodeAuthInformers(ctx, client, "unbounded-system", 0) + caches.start() + t.Cleanup(func() { + cancel() + caches.podFactory.Shutdown() + caches.saFactory.Shutdown() + }) + waitNodeAuthCondition(t, caches.ready) + + return caches, cancel +} + +func assertNodeAuthProbes(t *testing.T, caches *nodeAuthInformers, client *k8sfake.Clientset, verifier serviceAccountTokenVerifier, ready bool) { + t.Helper() + + health := &healthState{ + clientset: client, + tokenAuth: readyTokenAuthenticator(), + nodeAuthReady: caches.readinessCheck(verifier), + } + mux := http.NewServeMux() + registerProbeHandlers(mux, health) + + for _, path := range []string{"/healthz", "/readyz"} { + response := httptest.NewRecorder() + mux.ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil)) + + want := http.StatusOK + if path == "/readyz" && !ready { + want = http.StatusServiceUnavailable + + if !strings.Contains(response.Body.String(), "node authentication informer caches not ready") { + t.Fatalf("missing cache readiness reason: %s", response.Body.String()) + } + } + + if response.Code != want { + t.Fatalf("%s returned %d, want %d: %s", path, response.Code, want, response.Body.String()) + } + } +} + +func TestNodeAuthInformerReadinessCancellation(t *testing.T) { + client := k8sfake.NewClientset() + caches, cancel := startNodeAuthTestCaches(t, client) + verifier := &authn.PodBoundTokenVerifier{} + assertNodeAuthProbes(t, caches, client, verifier, true) + cancel() + assertNodeAuthProbes(t, caches, client, verifier, false) + assertNodeAuthProbes(t, caches, client, authn.NewKubernetesTokenReviewVerifier(client.AuthenticationV1()), true) +} + +func TestNodeAuthInformersScopeAndLifecycle(t *testing.T) { + pod, sa, _ := nodeAuthObjects() + unlabeled := pod.DeepCopy() + unlabeled.Name = "unlabeled" + unlabeled.Labels = nil + otherPod := pod.DeepCopy() + otherPod.Namespace = "other" + otherSA := sa.DeepCopy() + otherSA.Namespace = "other" + client := k8sfake.NewClientset(pod, sa, unlabeled, otherPod, otherSA) + caches, stop := startNodeAuthTestCaches(t, client) + + if _, err := caches.pods.Lister().Pods(pod.Namespace).Get(pod.Name); err != nil { + t.Fatal(err) + } + + if _, err := caches.serviceAccounts.Lister().ServiceAccounts(sa.Namespace).Get(sa.Name); err != nil { + t.Fatalf("unlabeled service account must be watched: %v", err) + } + + for _, excluded := range []*corev1.Pod{unlabeled, otherPod} { + if _, err := caches.pods.Lister().Pods(excluded.Namespace).Get(excluded.Name); err == nil { + t.Fatalf("cached excluded Pod %s/%s", excluded.Namespace, excluded.Name) + } + } + + if _, err := caches.serviceAccounts.Lister().ServiceAccounts(otherSA.Namespace).Get(otherSA.Name); err == nil { + t.Fatal("cached service account outside controller namespace") + } + + for _, action := range client.Actions() { + if action.GetNamespace() != "unbounded-system" { + t.Fatalf("unexpected cluster-wide cache request: %v", action) + } + + if action.GetVerb() != "list" && action.GetVerb() != "watch" { + t.Fatalf("unexpected informer API request: %v", action) + } + + selector := "" + + switch action := action.(type) { + case k8stesting.ListAction: + selector = action.GetListRestrictions().Labels.String() + case k8stesting.WatchAction: + selector = action.GetWatchRestrictions().Labels.String() + } + + wantSelector := "" + if action.GetResource().Resource == "pods" { + wantSelector = "app.kubernetes.io/name=unbounded-net-node" + } + + if selector != wantSelector { + t.Fatalf("unexpected %s selector %q, want %q", action.GetResource().Resource, selector, wantSelector) + } + } + + leaderCtx, stopLeader := context.WithCancel(t.Context()) + stopLeader() + + if !cache.WaitForCacheSync(leaderCtx.Done(), caches.pods.Informer().HasSynced) && !caches.pods.Informer().HasSynced() { + t.Fatal("shared Pod informer lost its initial sync") + } + + if !caches.ready() { + t.Fatal("leader cancellation stopped process authentication caches") + } + + if err := client.CoreV1().Pods(pod.Namespace).Delete(t.Context(), pod.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + + waitNodeAuthCondition(t, func() bool { + _, err := caches.pods.Lister().Pods(pod.Namespace).Get(pod.Name) + return err != nil + }) + + stop() + + if caches.ready() { + t.Fatal("process cancellation left authentication available") + } + + caches.podFactory.Shutdown() + caches.saFactory.Shutdown() + + if !caches.pods.Informer().IsStopped() || !caches.serviceAccounts.Informer().IsStopped() || caches.ready() { + t.Fatal("stopped informers left authentication available") + } +} + +func TestNodeAuthInformersInitialSyncFailure(t *testing.T) { + for _, resource := range []string{"pods", "serviceaccounts"} { + t.Run(resource, func(t *testing.T) { + pod, sa, identity := nodeAuthObjects() + client := k8sfake.NewClientset(pod, sa) + failed := make(chan struct{}, 1) + + client.PrependReactor("list", resource, func(k8stesting.Action) (bool, runtime.Object, error) { + select { + case failed <- struct{}{}: + default: + } + + return true, nil, errors.New("forbidden") + }) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + caches := newNodeAuthInformers(ctx, client, pod.Namespace, 0) + caches.start() + + defer func() { + cancel() + caches.podFactory.Shutdown() + caches.saFactory.Shutdown() + }() + + select { + case <-failed: + case <-time.After(5 * time.Second): + t.Fatal("informer did not attempt initial list") + } + + verifier, err := caches.wrapOIDCFactory(func(context.Context, string, string) (serviceAccountTokenVerifier, error) { + return fakeServiceAccountTokenVerifier{identity: identity}, nil + })(ctx, "", "") + if err != nil { + t.Fatal(err) + } + + if got, err := verifier.Verify(t.Context(), "token"); got != nil || err == nil { + t.Fatalf("initial list failure bypassed: %+v, %v", got, err) + } + + assertNodeAuthProbes(t, caches, client, verifier, false) + }) + } +} + +func TestNodeAuthInformersStoppedCache(t *testing.T) { + for _, resource := range []string{"pods", "serviceaccounts"} { + t.Run(resource, func(t *testing.T) { + pod, sa, identity := nodeAuthObjects() + client := k8sfake.NewClientset(pod, sa) + caches := newNodeAuthInformers(t.Context(), client, pod.Namespace, 0) + podCtx, stopPods := context.WithCancel(t.Context()) + saCtx, stopSAs := context.WithCancel(t.Context()) + + caches.podFactory.Start(podCtx.Done()) + caches.saFactory.Start(saCtx.Done()) + + defer func() { + stopPods() + stopSAs() + caches.podFactory.Shutdown() + caches.saFactory.Shutdown() + }() + + waitNodeAuthCondition(t, caches.ready) + + verifier, err := caches.wrapOIDCFactory(func(context.Context, string, string) (serviceAccountTokenVerifier, error) { + return fakeServiceAccountTokenVerifier{identity: identity}, nil + })(t.Context(), "", "") + if err != nil { + t.Fatal(err) + } + + if _, err := verifier.Verify(t.Context(), "token"); err != nil { + t.Fatal(err) + } + + assertNodeAuthProbes(t, caches, client, verifier, true) + + if resource == "pods" { + stopPods() + waitNodeAuthCondition(t, caches.pods.Informer().IsStopped) + } else { + stopSAs() + waitNodeAuthCondition(t, caches.serviceAccounts.Informer().IsStopped) + } + + if !caches.pods.Informer().HasSynced() || !caches.serviceAccounts.Informer().HasSynced() || caches.ctx.Err() != nil { + t.Fatal("test requires initially synced caches and a live process context") + } + + if identity, err := verifier.Verify(t.Context(), "token"); err == nil || identity != nil { + t.Fatalf("stopped cache accepted identity: %+v, %v", identity, err) + } + + assertNodeAuthProbes(t, caches, client, verifier, false) + }) + } +} + +func TestInitializedOIDCVerifierChecksBoundObjects(t *testing.T) { + for _, explicit := range []bool{false, true} { + name := "discovered" + if explicit { + name = "explicit" + } + + t.Run(name, func(t *testing.T) { + pod, sa, identity := nodeAuthObjects() + client := k8sfake.NewClientset(pod, sa) + caches, _ := startNodeAuthTestCaches(t, client) + tokenPath := filepath.Join(t.TempDir(), "token") + + token := "header." + base64.RawURLEncoding.EncodeToString([]byte(`{"iss":"https://issuer.example","aud":["api"]}`)) + ".signature" + if err := os.WriteFile(tokenPath, []byte(token), 0o600); err != nil { + t.Fatal(err) + } + + issuer := "" + if explicit { + issuer = "https://issuer.example" + } + + verifier, err := initializeNodeTokenVerifier(t.Context(), client, issuer, "", tokenPath, + caches.wrapOIDCFactory(func(context.Context, string, string) (serviceAccountTokenVerifier, error) { + return fakeServiceAccountTokenVerifier{identity: identity}, nil + })) + if err != nil { + t.Fatal(err) + } + + if got, err := verifier.Verify(t.Context(), "token"); err != nil || got == nil { + t.Fatalf("live Pod authentication failed: %+v, %v", got, err) + } + + if err := client.CoreV1().Pods(pod.Namespace).Delete(t.Context(), pod.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + + waitNodeAuthCondition(t, func() bool { + _, err := caches.pods.Lister().Pods(pod.Namespace).Get(pod.Name) + return err != nil + }) + + if got, err := verifier.Verify(t.Context(), "token"); err == nil || got != nil { + t.Fatalf("revoked Pod authenticated: %+v, %v", got, err) + } + + for _, action := range client.Actions() { + if action.GetResource().Resource == "tokenreviews" { + t.Fatal("cache miss dynamically fell back to TokenReview") + } + } + }) + } +} diff --git a/cmd/unbounded-net-controller/node_status.go b/cmd/unbounded-net-controller/node_status.go index 3f7317184..7fb9b09b1 100644 --- a/cmd/unbounded-net-controller/node_status.go +++ b/cmd/unbounded-net-controller/node_status.go @@ -10,6 +10,8 @@ import ( "net/http" "sync" "time" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" ) // CachedNodeStatus stores a node's pushed status with timestamp and revision. @@ -18,6 +20,8 @@ type CachedNodeStatus struct { ReceivedAt time.Time Source string Revision uint64 + + peerIdentity *peerIdentityDigest } // NodeStatusCache is a thread-safe cache of node status data pushed from node agents. @@ -73,17 +77,19 @@ func (c *NodeStatusCache) StoreFull(nodeName string, status NodeStatusResponse, // parsedDelta holds pre-deserialized delta fields parsed outside the lock. type parsedDelta struct { - timestamp *time.Time - nodeInfo *NodeInfo - peers []WireGuardPeerStatus - routingTable *RoutingTableInfo - healthCheck *HealthCheckStatus - nodeErrors []NodeError - fetchError *string - lastPushTime *time.Time - statusSource *string - nodePodInfo *NodePodInfo - bpfEntries []BpfEntry + peerMeasurements *statusproto.PeerMeasurements + parseError error + timestamp *time.Time + nodeInfo *NodeInfo + peers []WireGuardPeerStatus + routingTable *RoutingTableInfo + healthCheck *HealthCheckStatus + nodeErrors []NodeError + fetchError *string + lastPushTime *time.Time + statusSource *string + nodePodInfo *NodePodInfo + bpfEntries []BpfEntry // nullFields tracks fields explicitly set to null for clearing. nullFields map[string]bool @@ -193,13 +199,28 @@ func (c *NodeStatusCache) ApplyParsedDelta(nodeName string, baseRevision uint64, source = "push" } - return c.applyParsedDelta(nodeName, baseRevision, pd, source) + rev, conflict, err := c.applyParsedDelta(nodeName, baseRevision, pd, source) + if pd.peerMeasurements != nil || pd.parseError != nil { + outcome := "applied" + if err != nil { + outcome = "error" + } else if conflict { + outcome = "resync" + } + + peerMeasurementUpdatesTotal.WithLabelValues(outcome).Inc() + } + + return rev, conflict, err } // applyParsedDelta merges pre-parsed delta fields into a cached node status // under the lock. Both ApplyDelta (JSON) and ApplyParsedDelta (protobuf) // converge here. func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, pd parsedDelta, source string) (uint64, bool, error) { + if pd.parseError != nil { + return 0, false, pd.parseError + } // Phase 1: Read entry under lock, copy it, release lock. c.mu.RLock() @@ -209,7 +230,7 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, return 0, true, nil } - if baseRevision != 0 && entry.Revision != baseRevision { + if (pd.peerMeasurements != nil && baseRevision == 0) || (baseRevision != 0 && entry.Revision != baseRevision) { rev := entry.Revision c.mu.RUnlock() @@ -219,6 +240,7 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, // Snapshot values we need under lock prevStatus := entry.Status prevRevision := entry.Revision + peerIdentity := entry.peerIdentity c.mu.RUnlock() @@ -226,6 +248,20 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, // part (~1MB copy per node) and must not block other goroutines. merged := *prevStatus + if pd.peerMeasurements != nil { + if pd.peers != nil || pd.nullFields["peers"] { + return prevRevision, false, fmt.Errorf("peer replacement conflicts with measurements") + } + + peers, identity, err := applyPeerMeasurementsWithIdentity(prevStatus.Peers, pd.peerMeasurements, peerIdentity) + if err != nil { + return prevRevision, false, err + } + + merged.Peers = peers + peerIdentity = identity + } + if pd.timestamp != nil { merged.Timestamp = *pd.timestamp } else if pd.nullFields["timestamp"] { @@ -240,10 +276,16 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, if pd.peers != nil { merged.Peers = pd.peers + peerIdentity = nil + } else if pd.nullFields["peers"] { + merged.Peers = nil + peerIdentity = nil } if pd.routingTable != nil { merged.RoutingTable = *pd.routingTable + } else if pd.nullFields["routingTable"] { + merged.RoutingTable = RoutingTableInfo{} } if pd.healthCheck != nil { @@ -292,16 +334,21 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, merged.NodeInfo.Name = nodeName } + return c.commitParsedDelta(nodeName, entry, &merged, peerIdentity, source) +} + +func (c *NodeStatusCache) commitParsedDelta(nodeName string, previous *CachedNodeStatus, merged *NodeStatusResponse, peerIdentity *peerIdentityDigest, source string) (uint64, bool, error) { // Phase 3: Write lock for the brief pointer swap. c.mu.Lock() - // Re-check entry still exists and revision hasn't changed - entry, ok = c.entries[nodeName] + // Deletion and recreation can reuse a revision. The identity memo and + // merged status must still belong to the same entry, not just its number. + entry, ok := c.entries[nodeName] if !ok { c.mu.Unlock() return 0, true, nil } - if entry.Revision != prevRevision { + if entry != previous { // Another goroutine updated this node while we were merging. // Our merge is stale; signal resync. rev := entry.Revision @@ -312,10 +359,11 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, revision := entry.Revision + 1 c.entries[nodeName] = &CachedNodeStatus{ - Status: &merged, - ReceivedAt: time.Now(), - Source: source, - Revision: revision, + Status: merged, + ReceivedAt: time.Now(), + Source: source, + Revision: revision, + peerIdentity: peerIdentity, } fn := c.onChange mergedPtr := c.entries[nodeName].Status @@ -336,7 +384,8 @@ func (c *NodeStatusCache) SetOnChange(fn func(nodeName string, status *NodeStatu c.onChange = fn } -// Get returns a copy of the cached status for a node when present. +// Get returns shallow copies of the cached entry and status when present. +// Nested slices and maps remain shared. func (c *NodeStatusCache) Get(nodeName string) (*CachedNodeStatus, bool) { c.mu.RLock() defer c.mu.RUnlock() diff --git a/cmd/unbounded-net-controller/node_token_verifier_test.go b/cmd/unbounded-net-controller/node_token_verifier_test.go index e8cb511d7..8c9a05203 100644 --- a/cmd/unbounded-net-controller/node_token_verifier_test.go +++ b/cmd/unbounded-net-controller/node_token_verifier_test.go @@ -61,8 +61,9 @@ func TestInitializeNodeTokenVerifier(t *testing.T) { return expectedVerifier, tc.factoryErr } client := k8sfake.NewClientset() + caches := newNodeAuthInformers(t.Context(), client, "unbounded-system", 0) - verifier, err := initializeNodeTokenVerifier(t.Context(), client, tc.issuer, tc.audience, tokenPath, factory) + verifier, err := initializeNodeTokenVerifier(t.Context(), client, tc.issuer, tc.audience, tokenPath, caches.wrapOIDCFactory(factory)) if (err != nil) != tc.wantErr { t.Fatalf("error = %v, want error = %v", err, tc.wantErr) } @@ -71,8 +72,22 @@ func TestInitializeNodeTokenVerifier(t *testing.T) { if _, ok := verifier.(*authn.KubernetesTokenReviewVerifier); !ok { t.Fatalf("expected TokenReview fallback, got %T", verifier) } - } else if !tc.wantErr && verifier != expectedVerifier { - t.Fatalf("expected initialized OIDC verifier, got %T", verifier) + + if caches.readinessCheck(verifier) != nil { + t.Fatal("TokenReview readiness must not depend on informer caches") + } + } else if !tc.wantErr { + if _, ok := verifier.(*authn.PodBoundTokenVerifier); !ok { + t.Fatalf("expected cache-validated OIDC verifier, got %T", verifier) + } + + if ready := caches.readinessCheck(verifier); ready == nil || ready() { + t.Fatal("initialized OIDC verifier must wait for informer readiness") + } + + if identity, err := verifier.Verify(t.Context(), "token"); err == nil || identity != nil { + t.Fatalf("unsynced caches bypassed for initialized OIDC verifier: %+v, %v", identity, err) + } } wantCalls := 0 diff --git a/cmd/unbounded-net-controller/peer_identity_cache_test.go b/cmd/unbounded-net-controller/peer_identity_cache_test.go new file mode 100644 index 000000000..ad2c3634d --- /dev/null +++ b/cmd/unbounded-net-controller/peer_identity_cache_test.go @@ -0,0 +1,457 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" + "sync" + "testing" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func TestPeerIdentityHashMatchesProtocol(t *testing.T) { + for _, count := range []int{0, 1, 2000} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + peers := protoToNodeStatus(measurementTestStatus(count)).Peers + if count > 0 { + peers[0].Name = "peer\x00雪" + peers[0].Tunnel.Protocol = "" + peers[0].Tunnel.Interface = "ab" + peers[0].Tunnel.PublicKey = "c" + } + + digest, err := netstatus.PeerIdentityDigest(peers) + if err != nil { + t.Fatal(err) + } + + got := hashPeerIdentities(peers) + if !bytes.Equal(got[:], digest) { + t.Fatal("cached identity encoding differs from the wire protocol") + } + + if count > 0 { + peers[0].Tunnel.Interface = "a" + peers[0].Tunnel.PublicKey = "bc" + + if hashPeerIdentities(peers) == got { + t.Fatal("identity boundaries are ambiguous") + } + } + }) + } +} + +func warmPeerIdentityCache(t *testing.T, cache *NodeStatusCache) *CachedNodeStatus { + t.Helper() + + entry, ok := cache.Get("node-a") + if !ok { + t.Fatal("missing base") + } + + message := measurementMessage(t, *entry.Status, entry.Revision) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "ok" { + t.Fatalf("compact update: %+v", ack) + } + + result, ok := cache.Get("node-a") + if !ok || result.peerIdentity == nil { + t.Fatal("successful compact update did not memoize identity validation") + } + + return result +} + +func TestPeerIdentityCacheReuse(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(2)) + cache := NewNodeStatusCache() + cache.StoreFull("node-a", status, "ws") + first := warmPeerIdentityCache(t, cache) + + before, err := json.Marshal(first.Status) + if err != nil { + t.Fatal(err) + } + + // The compact copy owns its peer value fields, independently of StoreFull's + // original caller. It still shares immutable static maps and slices. + status.Peers[0].Name = "changed original input" + + for range 3 { + next := warmPeerIdentityCache(t, cache) + if next.peerIdentity != first.peerIdentity { + t.Fatal("unchanged identities rebuilt their validation memo") + } + } + + _, conflict, err := cache.ApplyDelta("node-a", 0, map[string]json.RawMessage{"fetchError": []byte(`"unavailable"`)}, "push") + if err != nil || conflict { + t.Fatalf("non-peer delta: %t %v", conflict, err) + } + + cache.UpdateSource("node-a", "ws") + + next := warmPeerIdentityCache(t, cache) + if next.peerIdentity != first.peerIdentity { + t.Fatal("non-peer update invalidated identity validation") + } + + after, err := json.Marshal(first.Status) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("later updates mutated a retained snapshot") + } + + allocations := testing.AllocsPerRun(10, func() { + identity, err := validatePeerIdentity(next.Status.Peers, next.peerIdentity) + if err != nil || identity != next.peerIdentity { + t.Fatalf("reuse: %v", err) + } + }) + if allocations != 0 { + t.Fatalf("identity reuse allocated %g objects", allocations) + } +} + +func TestPeerIdentityCacheInvalidatesReplacementPaths(t *testing.T) { + changes := map[string]func(*statusproto.NodeStatusFull){ + "identical": func(_ *statusproto.NodeStatusFull) {}, + "metadata": func(s *statusproto.NodeStatusFull) { s.Peers[0].SiteName = "another-site" }, + "renamed": func(s *statusproto.NodeStatusFull) { s.Peers[0].Name = "another-peer" }, + "added": func(s *statusproto.NodeStatusFull) { s.Peers = measurementTestStatus(3).Peers }, + "removed": func(s *statusproto.NodeStatusFull) { s.Peers = s.Peers[:1] }, + "reordered": func(s *statusproto.NodeStatusFull) { + s.Peers[0], s.Peers[1] = s.Peers[1], s.Peers[0] + }, + "duplicate": func(s *statusproto.NodeStatusFull) { s.Peers[1] = s.Peers[0] }, + "unnamed": func(s *statusproto.NodeStatusFull) { s.Peers[0].Name = "" }, + "nil": func(s *statusproto.NodeStatusFull) { s.Peers = nil }, + "empty": func(s *statusproto.NodeStatusFull) { s.Peers = []*statusproto.PeerStatus{} }, + } + for _, path := range []string{"StoreFull", "protobuf full", "protobuf delta", "JSON delta"} { + for name, change := range changes { + t.Run(path+"/"+name, func(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "ws") + first := warmPeerIdentityCache(t, cache) + full := measurementTestStatus(2) + change(full) + + status := protoToNodeStatus(full) + if full.Peers != nil && len(full.Peers) == 0 { + status.Peers = []WireGuardPeerStatus{} + } + + var conflict bool + + var err error + + switch path { + case "StoreFull": + cache.StoreFull("node-a", status, "push") + case "protobuf full": + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, &statusproto.NodeStatusMessage{ + Type: "node_status_full", NodeName: "node-a", Status: full, + }) + if ack.Status != "ok" { + t.Fatal(ack) + } + case "protobuf delta": + delta := protoToParsedDelta(&statusproto.NodeStatusDelta{UpdatedFields: []string{"peers"}, Peers: full.Peers}) + _, conflict, err = cache.ApplyParsedDelta("node-a", first.Revision, delta, "ws") + case "JSON delta": + var raw []byte + + raw, err = json.Marshal(status.Peers) + if err == nil { + _, conflict, err = cache.ApplyDelta("node-a", first.Revision, map[string]json.RawMessage{"peers": raw}, "push") + } + } + + if err != nil || conflict { + t.Fatalf("replacement changed legacy behavior: %t %v", conflict, err) + } + + replaced := cache.entries["node-a"] + if replaced.peerIdentity != nil { + t.Fatal("peer replacement retained the old identity memo") + } + + if name == "duplicate" || name == "unnamed" { + message := measurementMessage(t, *first.Status, replaced.Revision) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "resync_required" || cache.entries["node-a"] != replaced { + t.Fatal("invalid replacement accepted compact measurements or changed the cache") + } + + return + } + + if name != "identical" && name != "metadata" { + message := measurementMessage(t, *first.Status, replaced.Revision) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "resync_required" || cache.entries["node-a"] != replaced { + t.Fatal("replacement accepted old-topology measurements") + } + } + + next := warmPeerIdentityCache(t, cache) + if next.peerIdentity == first.peerIdentity { + t.Fatal("replacement did not independently validate its identities") + } + }) + } + } +} + +func TestPeerIdentityCacheDetectsBorrowedIdentityChanges(t *testing.T) { + changes := map[string]func([]WireGuardPeerStatus){ + "name": func(p []WireGuardPeerStatus) { p[0].Name = "another-peer" }, + "protocol": func(p []WireGuardPeerStatus) { p[0].Tunnel.Protocol = "another-protocol" }, + "interface": func(p []WireGuardPeerStatus) { p[0].Tunnel.Interface = "another-interface" }, + "key": func(p []WireGuardPeerStatus) { p[0].Tunnel.PublicKey = "another-key" }, + "reorder": func(p []WireGuardPeerStatus) { p[0], p[1] = p[1], p[0] }, + "duplicate": func(p []WireGuardPeerStatus) { p[1] = p[0] }, + "unnamed": func(p []WireGuardPeerStatus) { p[0].Name = "" }, + } + for name, change := range changes { + t.Run(name, func(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "ws") + first := warmPeerIdentityCache(t, cache) + message := measurementMessage(t, *first.Status, first.Revision) + digest := *first.peerIdentity + change(first.Status.Peers) + + before, err := json.Marshal(first.Status) + if err != nil { + t.Fatal(err) + } + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + + after, marshalErr := json.Marshal(cache.entries["node-a"].Status) + if ack.Status != "resync_required" || marshalErr != nil || !bytes.Equal(before, after) { + t.Fatal("borrowed identity mutation used a stale memo or partially updated the cache") + } + + if *first.peerIdentity != digest || cache.entries["node-a"].Revision != first.Revision { + t.Fatal("rejected update mutated the old memo or revision") + } + + if name != "duplicate" && name != "unnamed" { + next := warmPeerIdentityCache(t, cache) + if next.peerIdentity == first.peerIdentity { + t.Fatal("changed borrowed identity did not force revalidation") + } + } + }) + } +} + +func TestPeerMeasurementsCachedValidationMatchesDirect(t *testing.T) { + changes := map[string]func([]WireGuardPeerStatus, *statusproto.PeerMeasurements){ + "valid": func(_ []WireGuardPeerStatus, _ *statusproto.PeerMeasurements) {}, + "count": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.PeerCount++ }, + "rx": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.RxBytes = nil }, + "tx": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.TxBytes = nil }, + "handshake": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.LastHandshakeUnixNs = nil }, + "uptime": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.Uptime = nil }, + "rtt": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.Rtt = nil }, + "digest": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.IdentityDigest[0] ^= 1 }, + "duplicate": func(p []WireGuardPeerStatus, _ *statusproto.PeerMeasurements) { p[1] = p[0] }, + "unnamed": func(p []WireGuardPeerStatus, _ *statusproto.PeerMeasurements) { p[0].Name = "" }, + "health": func(p []WireGuardPeerStatus, _ *statusproto.PeerMeasurements) { p[0].HealthCheck = nil }, + } + for name, change := range changes { + t.Run(name, func(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(2)) + message := measurementMessage(t, status, 1) + m := message.Delta.PeerMeasurements + + identity, err := validatePeerIdentity(status.Peers, nil) + if err != nil { + t.Fatal(err) + } + + change(status.Peers, m) + direct, directErr := applyPeerMeasurements(status.Peers, m) + + cached, _, cachedErr := applyPeerMeasurementsWithIdentity(status.Peers, m, identity) + if fmt.Sprint(directErr) != fmt.Sprint(cachedErr) || !reflect.DeepEqual(direct, cached) { + t.Fatalf("direct and memoized validation disagree: %v / %v", directErr, cachedErr) + } + + if name != "valid" && directErr == nil { + t.Fatal("invalid measurements were accepted") + } + }) + } +} + +func TestPeerIdentityCacheRevisionAndRemoval(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "ws") + first := warmPeerIdentityCache(t, cache) + fetchError := "must not apply" + delta := parsedDelta{peerMeasurements: &statusproto.PeerMeasurements{}, fetchError: &fetchError} + + for _, revision := range []uint64{0, first.Revision - 1, first.Revision + 1} { + _, conflict, err := cache.ApplyParsedDelta("node-a", revision, delta, "ws") + if !conflict || err != nil { + t.Fatalf("revision %d did not precede malformed identity/column validation: %t %v", revision, conflict, err) + } + } + + if current := cache.entries["node-a"]; current.Revision != first.Revision || current.peerIdentity != first.peerIdentity || current.Status.FetchError != "" { + t.Fatal("revision conflict changed the cache") + } + + for _, remove := range []func(){func() { cache.Delete("node-a") }, func() { cache.CleanupStaleEntries(nil) }} { + remove() + + _, conflict, err := cache.ApplyParsedDelta("node-a", first.Revision, delta, "ws") + if !conflict || err != nil || cache.Len() != 0 { + t.Fatal("missing entry did not request resync before validation") + } + + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "push") + + if cache.entries["node-a"].peerIdentity != nil { + t.Fatal("recreated cache entry inherited an identity memo") + } + + warmPeerIdentityCache(t, cache) + } +} + +func TestPeerIdentityCacheConcurrentUpdates(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2000)), "ws") + first := warmPeerIdentityCache(t, cache) + delta := protoToParsedDelta(measurementMessage(t, *first.Status, first.Revision).Delta) + start := make(chan struct{}) + results := make(chan bool, 8) + + var workers sync.WaitGroup + + for range cap(results) { + workers.Go(func() { + <-start + + _, conflict, err := cache.ApplyParsedDelta("node-a", first.Revision, delta, "ws") + if err != nil { + t.Errorf("concurrent update: %v", err) + } + + results <- !conflict && err == nil + }) + } + + close(start) + workers.Wait() + close(results) + + applied := 0 + + for success := range results { + if success { + applied++ + } + } + + if current := cache.entries["node-a"]; applied != 1 || current.Revision != first.Revision+1 || current.peerIdentity != first.peerIdentity { + t.Fatalf("concurrent update lost revision or identity invariants: successes=%d", applied) + } +} + +func TestPeerIdentityCacheRejectsCommitAfterRecreation(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "ws") + warmPeerIdentityCache(t, cache) + previous := cache.entries["node-a"] + merged := *previous.Status + merged.FetchError = "must not apply" + + cache.Delete("node-a") + + replacement := protoToNodeStatus(measurementTestStatus(1)) + for range previous.Revision { + cache.StoreFull("node-a", replacement, "push") + } + + current := cache.entries["node-a"] + if current.Revision != previous.Revision { + t.Fatal("test requires a recreated entry with the same revision") + } + + called := false + + cache.SetOnChange(func(_ string, _ *NodeStatusResponse) { called = true }) + + revision, conflict, err := cache.commitParsedDelta("node-a", previous, &merged, previous.peerIdentity, "ws") + if err != nil || !conflict || revision != current.Revision || called { + t.Fatalf("stale commit after recreation: revision=%d conflict=%t err=%v callback=%t", revision, conflict, err, called) + } + + if cache.entries["node-a"] != current || current.peerIdentity != nil { + t.Fatal("stale commit overwrote the recreated entry or restored its old memo") + } +} + +func TestPeerIdentityValidationRejectsDuplicatesAfterMemo(t *testing.T) { + peers := protoToNodeStatus(measurementTestStatus(2)).Peers + + identity, err := validatePeerIdentity(peers, nil) + if err != nil { + t.Fatal(err) + } + + peers[1] = peers[0] + + _, err = validatePeerIdentity(peers, identity) + if err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatalf("memo skipped duplicate validation: %v", err) + } +} + +func BenchmarkCompactIdentityCache2000(b *testing.B) { + status := protoToNodeStatus(measurementTestStatus(2000)) + + measurements, err := netstatus.PeerMeasurementsToProto(status.Peers) + if err != nil { + b.Fatal(err) + } + + cache := NewNodeStatusCache() + revision := cache.StoreFull("node-a", status, "ws") + delta := parsedDelta{peerMeasurements: measurements} + + revision, conflict, err := cache.ApplyParsedDelta("node-a", revision, delta, "ws") + if err != nil || conflict { + b.Fatalf("warm cache: conflict=%t err=%v", conflict, err) + } + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + next, conflict, err := cache.ApplyParsedDelta("node-a", revision, delta, "ws") + if err != nil || conflict { + b.Fatalf("apply: conflict=%t err=%v", conflict, err) + } + + revision = next + } +} diff --git a/cmd/unbounded-net-controller/peer_measurements.go b/cmd/unbounded-net-controller/peer_measurements.go new file mode 100644 index 000000000..6b4fba93d --- /dev/null +++ b/cmd/unbounded-net-controller/peer_measurements.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "fmt" + "time" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// peerIdentityDigest records successful name and uniqueness validation. It is +// immutable and bounded to one digest per cached peer topology. +type peerIdentityDigest [sha256.Size]byte + +// hashPeerIdentities uses the same ordered, length-prefixed identity encoding as +// status.PeerIdentityDigest, without constructing its duplicate-detection map. +func hashPeerIdentities(peers []WireGuardPeerStatus) peerIdentityDigest { + digest := sha256.New() + + var size [8]byte + + for _, peer := range peers { + for _, value := range [4]string{peer.Name, peer.Tunnel.Protocol, peer.Tunnel.Interface, peer.Tunnel.PublicKey} { + binary.BigEndian.PutUint64(size[:], uint64(len(value))) + _, _ = digest.Write(size[:]) + _, _ = digest.Write([]byte(value)) + } + } + + var result peerIdentityDigest + + digest.Sum(result[:0]) + + return result +} + +func validatePeerIdentity(peers []WireGuardPeerStatus, previous *peerIdentityDigest) (*peerIdentityDigest, error) { + // StoreFull and Get share nested slices with callers. Re-hashing preserves + // that ownership contract: even an out-of-band identity change must not use + // a stale uniqueness result. A matching SHA-256 avoids rebuilding the map. + if previous != nil && hashPeerIdentities(peers) == *previous { + return previous, nil + } + + digest, err := netstatus.PeerIdentityDigest(peers) + if err != nil { + return nil, err + } + + result := peerIdentityDigest(digest) + + return &result, nil +} + +// applyPeerMeasurements validates every column and identity before creating a +// replacement slice. Static maps/slices remain shared and immutable. +func applyPeerMeasurements(peers []WireGuardPeerStatus, pb *statusproto.PeerMeasurements) ([]WireGuardPeerStatus, error) { + result, _, err := applyPeerMeasurementsWithIdentity(peers, pb, nil) + + return result, err +} + +func applyPeerMeasurementsWithIdentity(peers []WireGuardPeerStatus, pb *statusproto.PeerMeasurements, previous *peerIdentityDigest) ([]WireGuardPeerStatus, *peerIdentityDigest, error) { + count := len(peers) + if uint64(pb.PeerCount) != uint64(count) || + len(pb.RxBytes) != count || len(pb.TxBytes) != count || + len(pb.LastHandshakeUnixNs) != count || len(pb.Uptime) != count || len(pb.Rtt) != count { + return nil, nil, fmt.Errorf("peer measurement column length mismatch") + } + + digest, err := validatePeerIdentity(peers, previous) + if err != nil { + return nil, nil, err + } + + if !bytes.Equal(digest[:], pb.IdentityDigest) { + return nil, nil, fmt.Errorf("peer measurement identity mismatch") + } + + for i, peer := range peers { + if peer.HealthCheck == nil && (pb.Uptime[i] != "" || pb.Rtt[i] != "") { + return nil, nil, fmt.Errorf("peer measurements require existing health metadata") + } + } + + result := make([]WireGuardPeerStatus, count) + copy(result, peers) + + health := make([]statusv1alpha1.HealthCheckPeerStatus, count) + + for i := range result { + peer := &result[i] + peer.Tunnel.RxBytes = pb.RxBytes[i] + peer.Tunnel.TxBytes = pb.TxBytes[i] + + peer.Tunnel.LastHandshake = time.Time{} + if pb.LastHandshakeUnixNs[i] != 0 { + peer.Tunnel.LastHandshake = time.Unix(0, pb.LastHandshakeUnixNs[i]) + } + + if peer.HealthCheck != nil { + health[i] = *peer.HealthCheck + health[i].Uptime = pb.Uptime[i] + health[i].RTT = pb.Rtt[i] + peer.HealthCheck = &health[i] + } + } + + return result, digest, nil +} diff --git a/cmd/unbounded-net-controller/peer_measurements_test.go b/cmd/unbounded-net-controller/peer_measurements_test.go new file mode 100644 index 000000000..5953ae1ad --- /dev/null +++ b/cmd/unbounded-net-controller/peer_measurements_test.go @@ -0,0 +1,444 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "google.golang.org/protobuf/proto" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func measurementTestStatus(count int) *statusproto.NodeStatusFull { + status := &statusproto.NodeStatusFull{NodeInfo: &statusproto.NodeInfo{Name: "node-a"}} + for i := range count { + status.Peers = append(status.Peers, &statusproto.PeerStatus{ + Name: fmt.Sprintf("peer-%d", i), PeerType: "site", SiteName: "site-a", + PodCidrGateways: []string{"10.244.0.1"}, SkipPodCidrRoutes: true, + RouteDistances: map[string]int32{"10.244.0.0/24": 1}, RouteDestinations: []string{"10.244.0.0/24"}, + Tunnel: &statusproto.PeerTunnelStatus{ + Interface: "wg0", Protocol: "wireguard", PublicKey: fmt.Sprintf("key-%d", i), + Endpoint: "10.224.0.1:51820", AllowedIps: []string{"10.244.0.0/24"}, + RxBytes: 100, TxBytes: 200, LastHandshakeUnixNs: 123456789, + }, + HealthCheck: &statusproto.HealthCheckPeerStatus{Enabled: true, Status: "up", Uptime: "1m", Rtt: "1ms"}, + }) + } + + return status +} + +func measurementMessage(t *testing.T, status NodeStatusResponse, rev uint64) *statusproto.NodeStatusMessage { + t.Helper() + + pb, err := netstatus.PeerMeasurementsToProto(status.Peers) + if err != nil { + t.Fatal(err) + } + + return &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: rev, + Delta: &statusproto.NodeStatusDelta{UpdatedFields: []string{"peerMeasurements"}, PeerMeasurements: pb}, + } +} + +func applyMeasurementMessage(t *testing.T, health *healthState, message *statusproto.NodeStatusMessage) NodeStatusPushAck { + t.Helper() + + data, err := proto.Marshal(message) + if err != nil { + t.Fatal(err) + } + + decoded, err := decodeProtoWSMessage(data) + if err != nil { + t.Fatal(err) + } + + _, ack := handleProtoWSMessage(health, decoded, "ws") + + return ack +} + +func TestPeerMeasurementsCacheOwnsDecodedFrameData(t *testing.T) { + cache := NewNodeStatusCache() + health := &healthState{statusCache: cache} + + var buffer []byte + + applyReusedFrame := func(message *statusproto.NodeStatusMessage) NodeStatusPushAck { + t.Helper() + + var err error + + buffer, err = proto.MarshalOptions{}.MarshalAppend(buffer[:0], message) + if err != nil { + t.Fatal(err) + } + + decoded, err := decodeProtoWSMessage(buffer) + if err != nil { + t.Fatal(err) + } + + _, ack := handleProtoWSMessage(health, decoded, "ws") + if ack.Status != "ok" { + t.Fatalf("apply reused frame: %+v", ack) + } + + for i := range buffer { + buffer[i] = 0xa5 + } + + return ack + } + + full := measurementTestStatus(2) + wantFull := protoToNodeStatus(full) + ack := applyReusedFrame(&statusproto.NodeStatusMessage{ + Type: "node_status_full", NodeName: "node-a", Status: full, + }) + oldSnapshot := cache.entries["node-a"].Status + + if !reflect.DeepEqual(*oldSnapshot, wantFull) { + t.Fatal("full status retained overwritten frame data") + } + + message := measurementMessage(t, wantFull, ack.Revision) + measurements := message.Delta.PeerMeasurements + wantNext := protoToNodeStatus(proto.Clone(full).(*statusproto.NodeStatusFull)) + + for i := range wantNext.Peers { + measurements.RxBytes[i], measurements.TxBytes[i] = 321, 654 + measurements.LastHandshakeUnixNs[i] = 987654321 + measurements.Uptime[i], measurements.Rtt[i] = "2h", "7ms" + wantNext.Peers[i].Tunnel.RxBytes, wantNext.Peers[i].Tunnel.TxBytes = 321, 654 + wantNext.Peers[i].Tunnel.LastHandshake = time.Unix(0, 987654321) + wantNext.Peers[i].HealthCheck.Uptime, wantNext.Peers[i].HealthCheck.RTT = "2h", "7ms" + } + + applyReusedFrame(message) + + if !reflect.DeepEqual(*cache.entries["node-a"].Status, wantNext) { + t.Fatal("compact update retained overwritten frame data or lost static metadata") + } + + if !reflect.DeepEqual(*oldSnapshot, wantFull) { + t.Fatal("reusing frame storage or applying measurements mutated the old snapshot") + } + + identity := cache.entries["node-a"].peerIdentity + compactSnapshot := cache.entries["node-a"].Status + message.BaseRevision = cache.entries["node-a"].Revision + applyReusedFrame(message) + + if cache.entries["node-a"].peerIdentity != identity || + !reflect.DeepEqual(*cache.entries["node-a"].Status, wantNext) || + !reflect.DeepEqual(*compactSnapshot, wantNext) { + t.Fatal("warm identity memo retained frame data or mutated a compact snapshot") + } +} + +func TestPeerMeasurementsApplyAndClearImmutable(t *testing.T) { + for _, count := range []int{0, 3} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(count)) + cache := NewNodeStatusCache() + rev := cache.StoreFull("node-a", status, "ws") + old := cache.entries["node-a"] + + before, err := json.Marshal(old.Status) + if err != nil { + t.Fatal(err) + } + + message := measurementMessage(t, status, rev) + + m := message.Delta.PeerMeasurements + for i := range count { + m.RxBytes[i], m.TxBytes[i], m.LastHandshakeUnixNs[i] = 0, 0, 0 + m.Uptime[i], m.Rtt[i] = "", "" + } + + counter := peerMeasurementUpdatesTotal.WithLabelValues("applied") + start := testutil.ToFloat64(counter) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "ok" || ack.Revision != rev+1 { + t.Fatalf("ack: %+v", ack) + } + + if testutil.ToFloat64(counter) != start+1 { + t.Fatal("missing applied metric") + } + + next := cache.entries["node-a"].Status + for i, peer := range next.Peers { + if !netstatus.PeerMetadataEqual(status.Peers[i], peer) { + t.Fatal("static metadata lost") + } + + if peer.Tunnel.RxBytes != 0 || peer.Tunnel.TxBytes != 0 || !peer.Tunnel.LastHandshake.IsZero() || + peer.HealthCheck.Uptime != "" || peer.HealthCheck.RTT != "" { + t.Fatal("measurements not cleared") + } + + if peer.HealthCheck == old.Status.Peers[i].HealthCheck { + t.Fatal("health pointer was reused") + } + } + + after, err := json.Marshal(old.Status) + if err != nil { + t.Fatal(err) + } + + if string(before) != string(after) || cache.entries["node-a"] == old { + t.Fatal("cached snapshot mutated") + } + }) + } +} + +func TestPeerMeasurementsRejectWithoutMutation(t *testing.T) { + tests := map[string]func(*statusproto.NodeStatusMessage, *NodeStatusResponse){ + "zero revision": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.BaseRevision = 0 }, + "stale revision": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.BaseRevision++ }, + "unknown identity": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.PeerMeasurements.IdentityDigest[0] ^= 1 + }, + "missing identity": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.PeerMeasurements.IdentityDigest = nil + }, + "count": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements.PeerCount++ }, + "rx": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements.RxBytes = nil }, + "tx": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements.TxBytes = nil }, + "handshake": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.PeerMeasurements.LastHandshakeUnixNs = nil + }, + "uptime": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements.Uptime = nil }, + "rtt": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.PeerMeasurements.Rtt = append(m.Delta.PeerMeasurements.Rtt, "") + }, + "reordered": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { + s.Peers[0], s.Peers[1] = s.Peers[1], s.Peers[0] + }, + "deleted": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers = s.Peers[:1] }, + "added": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { + s.Peers = append(s.Peers, WireGuardPeerStatus{Name: "new"}) + }, + "duplicate": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1] = s.Peers[0] }, + "renamed": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].Name = "other" }, + "key": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].Tunnel.PublicKey = "other" }, + "interface": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].Tunnel.Interface = "other" }, + "protocol": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].Tunnel.Protocol = "other" }, + "absent health": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].HealthCheck = nil }, + "missing mask": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.UpdatedFields = nil }, + "missing payload": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements = nil }, + "duplicate mask": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.UpdatedFields = append(m.Delta.UpdatedFields, "peerMeasurements") + }, + "peers mask conflict": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.UpdatedFields = append(m.Delta.UpdatedFields, "peers") + }, + "peers payload conflict": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.Peers = []*statusproto.PeerStatus{{Name: "other"}} + }, + "full conflict": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Status = measurementTestStatus(2) }, + "full type conflict": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Type = "node_status_full" + m.Status = measurementTestStatus(2) + }, + } + for name, change := range tests { + t.Run(name, func(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(2)) + message := measurementMessage(t, status, 1) + change(message, &status) + message.Delta.UpdatedFields = append(message.Delta.UpdatedFields, "fetchError") + message.Delta.FetchError = "must not apply" + cache := NewNodeStatusCache() + cache.StoreFull("node-a", status, "ws") + old := cache.entries["node-a"] + + before, err := json.Marshal(old.Status) + if err != nil { + t.Fatal(err) + } + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "resync_required" { + t.Fatalf("accepted malformed batch: %+v", ack) + } + + after, err := json.Marshal(old.Status) + if err != nil { + t.Fatal(err) + } + + if old != cache.entries["node-a"] || string(before) != string(after) { + t.Fatal("partially mutated cache") + } + }) + } + + t.Run("missing cache", func(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(1)) + cache := NewNodeStatusCache() + counter := peerMeasurementUpdatesTotal.WithLabelValues("resync") + before := testutil.ToFloat64(counter) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, measurementMessage(t, status, 1)) + if ack.Status != "resync_required" || cache.Len() != 0 || testutil.ToFloat64(counter) != before+1 { + t.Fatalf("missing-base behavior: %+v", ack) + } + }) +} + +func TestPeerMeasurementsLegacyAndFullResync(t *testing.T) { + cache := NewNodeStatusCache() + health := &healthState{statusCache: cache} + full := measurementTestStatus(2) + + ack := applyMeasurementMessage(t, health, &statusproto.NodeStatusMessage{Type: "node_status_full", NodeName: "node-a", Status: full}) + if ack.Status != "ok" { + t.Fatal(ack) + } + + full.Peers[0].Tunnel.RxBytes++ + + ack = applyMeasurementMessage(t, health, &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: ack.Revision, + Delta: &statusproto.NodeStatusDelta{UpdatedFields: []string{"peers"}, Peers: full.Peers}, + }) + if ack.Status != "ok" || cache.entries["node-a"].Status.Peers[0].Tunnel.RxBytes != 101 { + t.Fatal("old protobuf node failed") + } + + rev, conflict, err := cache.ApplyDelta("node-a", ack.Revision, map[string]json.RawMessage{"peers": []byte("[]")}, "ws") + if err != nil || conflict || len(cache.entries["node-a"].Status.Peers) != 0 { + t.Fatal("legacy JSON clear failed") + } + + stale := measurementMessage(t, protoToNodeStatus(full), rev) + if got := applyMeasurementMessage(t, health, stale); got.Status != "resync_required" { + t.Fatal("missing peers accepted") + } + + ack = applyMeasurementMessage(t, health, &statusproto.NodeStatusMessage{Type: "node_status_full", NodeName: "node-a", Status: full}) + if got := applyMeasurementMessage(t, health, measurementMessage(t, *cache.entries["node-a"].Status, ack.Revision)); got.Status != "ok" { + t.Fatal("full resync did not restore compact path") + } + + data, err := marshalProtoAck("node_status_ack", ack) + if err != nil { + t.Fatal(err) + } + + var pbAck statusproto.NodeStatusAck + if err := proto.Unmarshal(data, &pbAck); err != nil { + t.Fatal(err) + } + + if !pbAck.PeerMeasurements || pbAck.Revision != ack.Revision { + t.Fatal("capability ACK missing") + } +} + +func TestTypedProtoDeltaAllFieldsAndClearings(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", NodeStatusResponse{NodeInfo: NodeInfo{Name: "node-a"}}, "ws") + delta := &statusproto.NodeStatusDelta{ + UpdatedFields: []string{"timestamp", "nodeInfo", "peers", "routingTable", "healthCheck", "nodeErrors", "bpfEntries", "fetchError", "lastPushTime", "statusSource", "nodePodInfo"}, + TimestampUnixNs: 123, NodeInfo: &statusproto.NodeInfo{Name: "node-a", SiteName: "new"}, + Peers: measurementTestStatus(1).Peers, RoutingTable: &statusproto.RoutingTableInfo{ManagedRouteCount: 2}, + HealthCheck: &statusproto.HealthCheckStatus{Healthy: true, CheckedAtUnixNs: 456}, + NodeErrors: []*statusproto.NodeError{{Type: "test", Message: "error"}}, + BpfEntries: []*statusproto.BpfEntry{{Cidr: "10.0.0.0/24"}}, + FetchError: "failed", LastPushTimeUnixNs: 789, StatusSource: "push", NodePodInfo: &statusproto.NodePodInfo{PodName: "pod"}, + } + + rev, conflict, err := cache.ApplyParsedDelta("node-a", 1, protoToParsedDelta(delta), "ws") + if err != nil || conflict { + t.Fatalf("apply: %v %v", err, conflict) + } + + got := cache.entries["node-a"].Status + if !got.Timestamp.Equal(time.Unix(0, 123)) || got.NodeInfo.SiteName != "new" || len(got.Peers) != 1 || + got.RoutingTable.ManagedRouteCount != 2 || got.HealthCheck == nil || !got.HealthCheck.CheckedAt.Equal(time.Unix(0, 456)) || + len(got.NodeErrors) != 1 || len(got.BpfEntries) != 1 || got.FetchError != "failed" || + got.LastPushTime == nil || !got.LastPushTime.Equal(time.Unix(0, 789)) || got.StatusSource != "push" || got.NodePodInfo.PodName != "pod" { + t.Fatalf("typed fields not applied: %+v", got) + } + + clearDelta := &statusproto.NodeStatusDelta{UpdatedFields: delta.UpdatedFields} + + _, conflict, err = cache.ApplyParsedDelta("node-a", rev, protoToParsedDelta(clearDelta), "ws") + if err != nil || conflict { + t.Fatal("clear failed") + } + + got = cache.entries["node-a"].Status + if !got.Timestamp.IsZero() || got.NodeInfo.Name != "node-a" || got.NodeInfo.SiteName != "" || len(got.Peers) != 0 || + !reflect.DeepEqual(got.RoutingTable, RoutingTableInfo{}) || got.HealthCheck != nil || len(got.NodeErrors) != 0 || + len(got.BpfEntries) != 0 || got.FetchError != "" || got.LastPushTime != nil || got.StatusSource != "" || got.NodePodInfo != nil { + t.Fatalf("typed fields not cleared: %+v", got) + } +} + +func BenchmarkApplyPeerDelta2000(b *testing.B) { + full := measurementTestStatus(2000) + status := protoToNodeStatus(full) + + measurements, err := netstatus.PeerMeasurementsToProto(status.Peers) + if err != nil { + b.Fatal(err) + } + + for _, compact := range []bool{false, true} { + name := "full-peers" + delta := &statusproto.NodeStatusDelta{UpdatedFields: []string{"peers"}, Peers: full.Peers} + + if compact { + name = "compact" + delta = &statusproto.NodeStatusDelta{UpdatedFields: []string{"peerMeasurements"}, PeerMeasurements: measurements} + } + + b.Run(name, func(b *testing.B) { + data, err := proto.Marshal(delta) + if err != nil { + b.Fatal(err) + } + + cache := NewNodeStatusCache() + rev := cache.StoreFull("node-a", status, "ws") + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + var decoded statusproto.NodeStatusDelta + if err := proto.Unmarshal(data, &decoded); err != nil { + b.Fatal(err) + } + + next, conflict, err := cache.ApplyParsedDelta("node-a", rev, protoToParsedDelta(&decoded), "ws") + if err != nil || conflict { + b.Fatalf("apply: %v %v", err, conflict) + } + + rev = next + } + + b.ReportMetric(float64(len(data)), "wire-B/op") + }) + } +} diff --git a/cmd/unbounded-net-controller/pod_binding_auth_test.go b/cmd/unbounded-net-controller/pod_binding_auth_test.go new file mode 100644 index 000000000..67d07ab12 --- /dev/null +++ b/cmd/unbounded-net-controller/pod_binding_auth_test.go @@ -0,0 +1,237 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sfake "k8s.io/client-go/kubernetes/fake" + + "github.com/Azure/unbounded/internal/net/authn" +) + +func revokeNodeAuthObject(t *testing.T, caches *nodeAuthInformers, client *k8sfake.Clientset, object, mutation string) { + t.Helper() + + pod, sa, _ := nodeAuthObjects() + deletedAt := metav1.NewTime(time.Now().Add(-61 * time.Second)) + + if object == "Pod" { + switch mutation { + case "missing": + if err := client.CoreV1().Pods(pod.Namespace).Delete(t.Context(), pod.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + case "replaced": + pod.UID = "replacement-pod" + if _, err := client.CoreV1().Pods(pod.Namespace).Update(t.Context(), pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + case "deleting": + pod.DeletionTimestamp = &deletedAt + if _, err := client.CoreV1().Pods(pod.Namespace).Update(t.Context(), pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } + + waitNodeAuthCondition(t, func() bool { + got, err := caches.pods.Lister().Pods(pod.Namespace).Get(pod.Name) + if mutation == "missing" { + return err != nil + } + + return err == nil && got.UID == pod.UID && got.DeletionTimestamp.Equal(pod.DeletionTimestamp) + }) + } else { + switch mutation { + case "missing": + if err := client.CoreV1().ServiceAccounts(sa.Namespace).Delete(t.Context(), sa.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + case "replaced": + sa.UID = "replacement-sa" + if _, err := client.CoreV1().ServiceAccounts(sa.Namespace).Update(t.Context(), sa, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + case "deleting": + sa.DeletionTimestamp = &deletedAt + if _, err := client.CoreV1().ServiceAccounts(sa.Namespace).Update(t.Context(), sa, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } + + waitNodeAuthCondition(t, func() bool { + got, err := caches.serviceAccounts.Lister().ServiceAccounts(sa.Namespace).Get(sa.Name) + if mutation == "missing" { + return err != nil + } + + return err == nil && got.UID == sa.UID && got.DeletionTimestamp.Equal(sa.DeletionTimestamp) + }) + } +} + +func TestPodBoundAuthenticationRevocation(t *testing.T) { + proxy, clientTLS := testNodeTokenFrontProxy(t) + + const ( + saToken = "verified-service-account-token" + subject = "system:serviceaccount:unbounded-system:unbounded-net-node" + payload = `{"mode":"full","type":"node_status_full","nodeName":"node-a","status":{"nodeInfo":{"name":"node-a","siteName":"unchanged"}}}` + ) + + for _, object := range []string{"Pod", "service account"} { + for _, mutation := range []string{"missing", "replaced", "deleting"} { + t.Run(object+"/"+mutation, func(t *testing.T) { + pod, sa, identity := nodeAuthObjects() + client := k8sfake.NewClientset(pod, sa) + caches, _ := startNodeAuthTestCaches(t, client) + + verifier, err := caches.wrapOIDCFactory(func(context.Context, string, string) (serviceAccountTokenVerifier, error) { + return fakeServiceAccountTokenVerifier{identity: identity}, nil + })(t.Context(), "", "") + if err != nil { + t.Fatal(err) + } + + h := newJSONIdentityHealth() + h.nodeTokenVerifier = verifier + issuer := testTokenIssuer(t) + mux := http.NewServeMux() + registerTokenEndpoints(mux, h, proxy, issuer, tokenEndpointConfig{ + nodeServiceAccount: h.nodeServiceAccount, verifier: verifier, + }) + registerPushHandlers(mux, h, proxy, make(chan struct{}, maxConcurrentNodeWS), issuer) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.TLS = clientTLS + mux.ServeHTTP(w, r) + })) + defer server.Close() + + request := func(path, body, bearer string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)) + req.TLS = clientTLS + req.Header.Set("Authorization", "Bearer "+bearer) + req.Header.Set("X-Remote-User", subject) + req.Header.Set(nodeIdentityTokenHeader, saToken) + req.Header.Set("Content-Type", "application/json") + + resp := httptest.NewRecorder() + mux.ServeHTTP(resp, req) + + return resp + } + tokenBody := `{"serviceAccountToken":"` + saToken + `"}` + + var issued tokenNodeResponse + + for _, path := range []string{directTokenNodePath, aggregatedTokenNodePath} { + resp := request(path, tokenBody, saToken) + if resp.Code != http.StatusOK { + t.Fatalf("initial %s exchange: %d %s", path, resp.Code, resp.Body.String()) + } + + if err := json.Unmarshal(resp.Body.Bytes(), &issued); err != nil { + t.Fatal(err) + } + } + + if resp := request(aggregatedNodeStatusPushPath, payload, saToken); resp.Code != http.StatusOK { + t.Fatalf("initial HTTP push: %d %s", resp.Code, resp.Body.String()) + } + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + headers := http.Header{ + "X-Remote-User": []string{subject}, nodeIdentityTokenHeader: []string{saToken}, + } + + existing, _, err := websocket.Dial(ctx, server.URL+aggregatedNodeStatusWebSocketPath, &websocket.DialOptions{HTTPHeader: headers}) + if err != nil { + t.Fatal(err) + } + + defer func() { _ = existing.CloseNow() }() + + assertAck := func() { + if err := existing.Write(ctx, websocket.MessageText, []byte(payload)); err != nil { + t.Fatal(err) + } + + _, data, err := existing.Read(ctx) + if err != nil { + t.Fatal(err) + } + + var reply struct { + Type string `json:"type"` + } + if err := json.Unmarshal(data, &reply); err != nil || reply.Type != "node_status_ack" { + t.Fatalf("existing WebSocket did not acknowledge: %s, %v", data, err) + } + } + assertAck() + revokeNodeAuthObject(t, caches, client, object, mutation) + + before := h.statusCache.GetAll() + + client.ClearActions() + + for _, path := range []string{directTokenNodePath, aggregatedTokenNodePath} { + resp := request(path, tokenBody, saToken) + if resp.Code != http.StatusUnauthorized { + t.Fatalf("revoked %s exchange: %d %s", path, resp.Code, resp.Body.String()) + } + + var result tokenNodeResponse + if json.Unmarshal(resp.Body.Bytes(), &result) == nil && result.Token != "" { + t.Fatal("issued new credentials after revocation") + } + } + + if resp := request(aggregatedNodeStatusPushPath, payload, saToken); resp.Code != http.StatusForbidden { + t.Fatalf("revoked HTTP push: %d %s", resp.Code, resp.Body.String()) + } + + conn, resp, err := websocket.Dial(ctx, server.URL+aggregatedNodeStatusWebSocketPath, &websocket.DialOptions{HTTPHeader: headers}) + if conn != nil { + _ = conn.CloseNow() + } + + if err == nil || resp == nil || resp.StatusCode != http.StatusForbidden { + t.Fatalf("revoked WebSocket handshake: response=%+v error=%v", resp, err) + } + + assertJSONIdentityCache(t, h, before, true) + + if actions := client.Actions(); len(actions) != 0 { + t.Fatalf("authentication made API requests or fell back: %v", actions) + } + + // This PR intentionally does not revoke credentials or connections + // that were issued/authenticated before the cache observed deletion. + claims, err := issuer.Validate(issued.Token) + if err != nil || claims.Role != authn.RoleNode || claims.NodeName != "node-a" { + t.Fatalf("existing HMAC credentials were revoked: %+v, %v", claims, err) + } + + if resp := request("/status/push", payload, issued.Token); resp.Code != http.StatusOK { + t.Fatalf("existing HMAC upload failed: %d %s", resp.Code, resp.Body.String()) + } + + assertAck() + }) + } + } +} diff --git a/cmd/unbounded-net-controller/proto_ws_identity_test.go b/cmd/unbounded-net-controller/proto_ws_identity_test.go new file mode 100644 index 000000000..d90112867 --- /dev/null +++ b/cmd/unbounded-net-controller/proto_ws_identity_test.go @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func TestProtoWSIdentityBeforeMutation(t *testing.T) { + full := func(envelope, nested string) *statusproto.NodeStatusMessage { + return &statusproto.NodeStatusMessage{ + Type: "node_status_full", NodeName: envelope, + Status: &statusproto.NodeStatusFull{ + NodeInfo: &statusproto.NodeInfo{Name: nested, SiteName: "updated"}, + }, + } + } + cases := []struct { + name string + message *statusproto.NodeStatusMessage + corrupt bool + reject bool + wantClosed bool + }{ + {name: "matching full", message: full("node-a", "node-a")}, + {name: "nested identity", message: full("", "node-a")}, + {name: "matching delta", message: &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: 1, + Delta: &statusproto.NodeStatusDelta{ + UpdatedFields: []string{"nodeInfo"}, + NodeInfo: &statusproto.NodeInfo{Name: "node-a", SiteName: "updated"}, + }, + }}, + {name: "wrong authenticated node", message: full("node-b", "node-b"), reject: true, wantClosed: true}, + {name: "conflicting full", message: full("node-a", "node-b"), reject: true}, + {name: "missing identity", message: full("", ""), reject: true}, + {name: "malformed after valid identity", message: full("node-a", "node-a"), corrupt: true, reject: true}, + {name: "conflicting delta", reject: true, message: &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: 1, + Delta: &statusproto.NodeStatusDelta{ + UpdatedFields: []string{"nodeInfo"}, NodeInfo: &statusproto.NodeInfo{Name: "node-b"}, + }, + }}, + {name: "conflicting unused full field", reject: true, message: &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: 1, + Status: &statusproto.NodeStatusFull{NodeInfo: &statusproto.NodeInfo{Name: "node-b"}}, + Delta: &statusproto.NodeStatusDelta{ + UpdatedFields: []string{"nodeInfo"}, NodeInfo: &statusproto.NodeInfo{Name: "node-a"}, + }, + }}, + } + proxy, clientTLS := testNodeTokenFrontProxy(t) + issuer := testTokenIssuer(t) + token := testNodeToken(t, issuer) + + for _, path := range []string{"/status/nodews", aggregatedNodeStatusWebSocketPath} { + for _, tc := range cases { + t.Run(path+"/"+tc.name, func(t *testing.T) { + health := newJSONIdentityHealth() + before := health.statusCache.GetAll() + + var evicted atomic.Bool + + health.registerNodeWS("node-a", func() { evicted.Store(true) }) + health.registerNodeWS("node-b", func() { evicted.Store(true) }) + + mux := http.NewServeMux() + registerPushHandlers(mux, health, proxy, make(chan struct{}, maxConcurrentNodeWS), issuer) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.TLS = clientTLS + mux.ServeHTTP(w, r) + })) + defer server.Close() + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + conn, _, err := websocket.Dial(ctx, server.URL+path, &websocket.DialOptions{HTTPHeader: http.Header{ + "Authorization": []string{"Bearer " + token}, + "X-Remote-User": []string{"system:serviceaccount:unbounded-system:unbounded-net-node"}, + nodeIdentityTokenHeader: []string{"service-account-token"}, + }}) + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := conn.CloseNow(); err != nil { + t.Logf("close websocket: %v", err) + } + }() + + data, err := proto.Marshal(tc.message) + if err != nil { + t.Fatal(err) + } + + if tc.corrupt { + data = append(data, 0xff) + } + + if err := conn.Write(ctx, websocket.MessageBinary, data); err != nil { + t.Fatal(err) + } + + frameType, reply, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + + if frameType != websocket.MessageBinary { + t.Fatal("expected a binary acknowledgment") + } + + var ack statusproto.NodeStatusAck + if err := proto.Unmarshal(reply, &ack); err != nil { + t.Fatal(err) + } + + assertJSONIdentityCache(t, health, before, tc.reject) + + if !tc.reject { + if ack.Status != "ok" || !evicted.Load() { + t.Fatalf("valid frame was not applied and registered: %v", &ack) + } + + return + } + + if ack.Status != "resync_required" || ack.Reason == "" || evicted.Load() { + t.Fatalf("rejected identity mutated connection state or lost its error: %v", &ack) + } + + if tc.wantClosed { + if _, _, err := conn.Read(ctx); err == nil || ctx.Err() != nil { + t.Fatal("authorization rejection did not close the connection") + } + + return + } + + // Invalid binary payloads retain the existing resync-and-retry behavior. + valid, err := proto.Marshal(full("node-a", "node-a")) + if err != nil { + t.Fatal(err) + } + + if err := conn.Write(ctx, websocket.MessageBinary, valid); err != nil { + t.Fatal(err) + } + + _, reply, err = conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + + if err := proto.Unmarshal(reply, &ack); err != nil { + t.Fatal(err) + } + + if ack.Status != "ok" || !evicted.Load() { + t.Fatalf("valid retry failed: %v", &ack) + } + + assertJSONIdentityCache(t, health, before, false) + }) + } + } +} diff --git a/cmd/unbounded-net-controller/server.go b/cmd/unbounded-net-controller/server.go index 31af3830b..c706ca95e 100644 --- a/cmd/unbounded-net-controller/server.go +++ b/cmd/unbounded-net-controller/server.go @@ -635,7 +635,7 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer return } - conn.SetReadLimit(2 * 1024 * 1024) // 2 MiB -- status payloads grow with cluster size + conn.SetReadLimit(maxNodeWSFrameBytes) websocketConnections.Inc() defer func() { @@ -685,16 +685,19 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer defer close(recvCh) for { - msgType, data, readErr := conn.Read(wsCtx) + frame, readErr := nodeWSBuffers.readFrame(wsCtx, conn) if readErr != nil { errCh <- readErr return } select { - case recvCh <- wsFrame{msgType: msgType, data: data}: + case recvCh <- frame: case <-wsCtx.Done(): + nodeWSBuffers.put(frame.data) + errCh <- wsCtx.Err() + return } } @@ -746,7 +749,13 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer pingInFlight := false lastActivity := time.Now() + var frameData []byte + defer func() { nodeWSBuffers.put(frameData) }() + for { + nodeWSBuffers.put(frameData) + frameData = nil + select { case <-wsCtx.Done(): return @@ -781,10 +790,21 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer return } + frameData = frame.data lastActivity = time.Now() if frame.msgType == websocket.MessageBinary { - nodeName := extractNodeNameFromProtoMessage(frame.data) + decoded, decodeErr := decodeProtoWSMessage(frame.data) + if decodeErr != nil { + send(websocket.MessageBinary, "node_status_resync", NodeStatusPushAck{ + Status: "resync_required", + Reason: decodeErr.Error(), + }) + + continue + } + + nodeName := decoded.nodeName if expectedNode := authorizedNodeName(r); expectedNode != "" && nodeName != "" && nodeName != expectedNode { send(websocket.MessageBinary, "node_status_resync", NodeStatusPushAck{ Status: "resync_required", @@ -803,7 +823,7 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer lastWSNodeName = nodeName } - ackType, ack := handleProtoWSMessage(health, frame.data, source) + ackType, ack := handleProtoWSMessage(health, decoded, source) send(websocket.MessageBinary, ackType, ack) } else { nodeName, identityErr := extractNodeNameFromWSMessage(frame.data) @@ -1387,6 +1407,8 @@ func serveUnifiedServer(ctx context.Context, port int, mux *http.ServeMux, certM // advertises gzip support via Accept-Encoding. WebSocket upgrades and // requests without gzip support are passed through unmodified. func gzipHandler(next http.Handler) http.Handler { + pool := newGzipWriterPool() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") || strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { @@ -1394,17 +1416,21 @@ func gzipHandler(next http.Handler) http.Handler { return } - gz, err := gzip.NewWriterLevel(w, gzip.BestSpeed) + gz, err := pool.get(w) if err != nil { next.ServeHTTP(w, r) return } - defer func() { _ = gz.Close() }() //nolint:errcheck + completed := false + + defer func() { pool.put(gz, completed) }() w.Header().Set("Content-Encoding", "gzip") w.Header().Del("Content-Length") next.ServeHTTP(&gzipResponseWriter{ResponseWriter: w, Writer: gz}, r) + + completed = true }) } diff --git a/cmd/unbounded-net-controller/status_proto.go b/cmd/unbounded-net-controller/status_proto.go index a1ce3bfcc..cb10196b6 100644 --- a/cmd/unbounded-net-controller/status_proto.go +++ b/cmd/unbounded-net-controller/status_proto.go @@ -330,13 +330,63 @@ func protoToParsedDelta(msg *statusproto.NodeStatusDelta) parsedDelta { updatedSet := make(map[string]bool, len(msg.UpdatedFields)) for _, f := range msg.UpdatedFields { + if f == "peerMeasurements" && updatedSet[f] { + pd.parseError = fmt.Errorf("duplicate peerMeasurements field mask") + } + updatedSet[f] = true } + if updatedSet["peerMeasurements"] || msg.PeerMeasurements != nil { + pd.peerMeasurements = msg.PeerMeasurements + if !updatedSet["peerMeasurements"] || msg.PeerMeasurements == nil { + pd.parseError = fmt.Errorf("peerMeasurements payload and field mask must agree") + } else if updatedSet["peers"] || len(msg.Peers) != 0 { + pd.parseError = fmt.Errorf("peer replacement conflicts with measurements") + } + } + + if pd.parseError != nil { + return pd + } + + if updatedSet["timestamp"] { + t := time.Time{} + if msg.TimestampUnixNs != 0 { + t = time.Unix(0, msg.TimestampUnixNs) + } + + pd.timestamp = &t + } + + if updatedSet["fetchError"] { + pd.fetchError = &msg.FetchError + } + + if updatedSet["statusSource"] { + pd.statusSource = &msg.StatusSource + } + + if updatedSet["lastPushTime"] { + if msg.LastPushTimeUnixNs == 0 { + pd.nullFields["lastPushTime"] = true + } else { + t := time.Unix(0, msg.LastPushTimeUnixNs) + pd.lastPushTime = &t + } + } + + if updatedSet["nodePodInfo"] { + pd.nodePodInfo = protoToNodePodInfo(msg.NodePodInfo) + pd.nullFields["nodePodInfo"] = msg.NodePodInfo == nil + } + if updatedSet["nodeInfo"] { if msg.NodeInfo != nil { ni := protoToNodeInfo(msg.NodeInfo) pd.nodeInfo = &ni + } else { + pd.nullFields["nodeInfo"] = true } } @@ -352,6 +402,8 @@ func protoToParsedDelta(msg *statusproto.NodeStatusDelta) parsedDelta { if msg.RoutingTable != nil { rt := protoToRoutingTable(msg.RoutingTable) pd.routingTable = &rt + } else { + pd.nullFields["routingTable"] = true } } @@ -380,20 +432,30 @@ func protoToParsedDelta(msg *statusproto.NodeStatusDelta) parsedDelta { return pd } -// extractNodeNameFromProtoMessage extracts the node name from a protobuf -// NodeStatusMessage for early identification on WebSocket connections. -func extractNodeNameFromProtoMessage(data []byte) string { - var msg statusproto.NodeStatusMessage - if err := proto.Unmarshal(data, &msg); err != nil { - return "" +type decodedProtoWSMessage struct { + message statusproto.NodeStatusMessage + nodeName string +} + +// Decode and validate identity before connection registration or cache mutation. +func decodeProtoWSMessage(data []byte) (*decodedProtoWSMessage, error) { + decoded := &decodedProtoWSMessage{} + if err := proto.Unmarshal(data, &decoded.message); err != nil { + return nil, fmt.Errorf("invalid protobuf message") } - nodeName, err := validatedProtoNodeName(&msg) + nodeName, err := validatedProtoNodeName(&decoded.message) if err != nil { - return "" + return nil, err + } + + if nodeName == "" { + return nil, fmt.Errorf("nodeName is required") } - return nodeName + decoded.nodeName = nodeName + + return decoded, nil } func validatedProtoNodeName(msg *statusproto.NodeStatusMessage) (string, error) { @@ -409,21 +471,14 @@ func validatedProtoNodeName(msg *statusproto.NodeStatusMessage) (string, error) return validatedNodeNames(nodeNames) } -// handleProtoWSMessage processes a binary (protobuf) WebSocket message and -// returns the ack type string and ack struct, identical to the JSON path. -func handleProtoWSMessage(health *healthState, data []byte, source string) (string, NodeStatusPushAck) { - var msg statusproto.NodeStatusMessage - if err := proto.Unmarshal(data, &msg); err != nil { - return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "invalid protobuf message"} - } +// handleProtoWSMessage applies the same decoded message used for authorization. +func handleProtoWSMessage(health *healthState, decoded *decodedProtoWSMessage, source string) (string, NodeStatusPushAck) { + msg := &decoded.message + nodeName := decoded.nodeName - nodeName, err := validatedProtoNodeName(&msg) - if err != nil { - return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} - } - - if nodeName == "" { - return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "nodeName is required"} + if msg.Delta.GetPeerMeasurements() != nil && (msg.Type != "node_status_delta" || msg.Status != nil) { + peerMeasurementUpdatesTotal.WithLabelValues("error").Inc() + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "full status conflicts with measurements"} } switch msg.Type { @@ -441,7 +496,7 @@ func handleProtoWSMessage(health *healthState, data []byte, source string) (stri return "node_status_ack", NodeStatusPushAck{Status: "ok", Revision: rev} case "node_status_delta": - if msg.Delta == nil || len(msg.Delta.UpdatedFields) == 0 { + if msg.Delta == nil || (len(msg.Delta.UpdatedFields) == 0 && msg.Delta.PeerMeasurements == nil) { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "delta message missing delta"} } @@ -479,6 +534,10 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string } ack := NodeStatusPushAck{Status: "ok"} + if msg.Delta.GetPeerMeasurements() != nil && (msg.Type != "node_status_delta" || msg.Status != nil) { + peerMeasurementUpdatesTotal.WithLabelValues("error").Inc() + return NodeStatusPushAck{Status: "resync_required", Reason: "full status conflicts with measurements"}, 429, nil + } switch msg.Type { case "node_status_full": @@ -495,7 +554,7 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string return ack, 200, nil case "node_status_delta": - if msg.Delta == nil || len(msg.Delta.UpdatedFields) == 0 { + if msg.Delta == nil || (len(msg.Delta.UpdatedFields) == 0 && msg.Delta.PeerMeasurements == nil) { return NodeStatusPushAck{}, 400, fmt.Errorf("delta is required for delta mode") } @@ -521,9 +580,10 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string // marshalProtoAck serializes a NodeStatusPushAck into a protobuf NodeStatusAck. func marshalProtoAck(ackType string, ack NodeStatusPushAck) ([]byte, error) { pbAck := &statusproto.NodeStatusAck{ - Status: ack.Status, - Revision: ack.Revision, - Reason: ack.Reason, + PeerMeasurements: true, + Status: ack.Status, + Revision: ack.Revision, + Reason: ack.Reason, } return proto.Marshal(pbAck) diff --git a/cmd/unbounded-net-controller/status_proto_test.go b/cmd/unbounded-net-controller/status_proto_test.go index b4160a6f1..d90ebc873 100644 --- a/cmd/unbounded-net-controller/status_proto_test.go +++ b/cmd/unbounded-net-controller/status_proto_test.go @@ -216,11 +216,20 @@ func TestProtoToParsedDeltaEmptyPeers(t *testing.T) { } } +func handleProtoWSBytes(health *healthState, data []byte, source string) (string, NodeStatusPushAck) { + decoded, err := decodeProtoWSMessage(data) + if err != nil { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} + } + + return handleProtoWSMessage(health, decoded, source) +} + func TestHandleProtoWSMessage(t *testing.T) { health := &healthState{statusCache: NewNodeStatusCache()} t.Run("invalid proto", func(t *testing.T) { - msgType, ack := handleProtoWSMessage(health, []byte("not-proto"), "ws") + msgType, ack := handleProtoWSBytes(health, []byte("not-proto"), "ws") if msgType != "node_status_resync" || ack.Status != "resync_required" { t.Fatalf("expected resync on invalid proto, got type=%q ack=%+v", msgType, ack) } @@ -230,7 +239,7 @@ func TestHandleProtoWSMessage(t *testing.T) { msg := &statusproto.NodeStatusMessage{Type: "node_status_full"} data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "nodeName is required" { t.Fatalf("expected nodeName required, got type=%q ack=%+v", msgType, ack) } @@ -246,7 +255,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_ack" || ack.Status != "ok" || ack.Revision == 0 { t.Fatalf("expected full ack success, got type=%q ack=%+v", msgType, ack) } @@ -267,7 +276,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Status != "resync_required" { t.Fatalf("expected conflicting node names to require resync, got type=%q ack=%+v", msgType, ack) } @@ -280,7 +289,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "full message missing status" { t.Fatalf("expected missing status resync, got type=%q ack=%+v", msgType, ack) } @@ -298,7 +307,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_ack" || ack.Status != "ok" || ack.Revision < 2 { t.Fatalf("expected delta ack success, got type=%q ack=%+v", msgType, ack) } @@ -321,7 +330,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "base revision mismatch" { t.Fatalf("expected conflict resync, got type=%q ack=%+v", msgType, ack) } @@ -334,7 +343,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "delta message missing delta" { t.Fatalf("expected missing delta resync, got type=%q ack=%+v", msgType, ack) } @@ -347,7 +356,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "unsupported message type" { t.Fatalf("expected unsupported type resync, got type=%q ack=%+v", msgType, ack) } @@ -471,8 +480,8 @@ func TestExtractNodeNameFromProtoMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - if got := extractNodeNameFromProtoMessage(data); got != "node-x" { - t.Fatalf("expected node-x, got %q", got) + if got, err := decodeProtoWSMessage(data); err != nil || got.nodeName != "node-x" { + t.Fatalf("expected node-x, got %+v, err=%v", got, err) } }) @@ -483,14 +492,14 @@ func TestExtractNodeNameFromProtoMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - if got := extractNodeNameFromProtoMessage(data); got != "node-y" { - t.Fatalf("expected node-y, got %q", got) + if got, err := decodeProtoWSMessage(data); err != nil || got.nodeName != "node-y" { + t.Fatalf("expected node-y, got %+v, err=%v", got, err) } }) t.Run("invalid data", func(t *testing.T) { - if got := extractNodeNameFromProtoMessage([]byte("bad")); got != "" { - t.Fatalf("expected empty, got %q", got) + if got, err := decodeProtoWSMessage([]byte("bad")); err == nil || got != nil { + t.Fatalf("expected invalid protobuf error, got %+v, err=%v", got, err) } }) @@ -501,8 +510,8 @@ func TestExtractNodeNameFromProtoMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - if got := extractNodeNameFromProtoMessage(data); got != "" { - t.Fatalf("expected empty for conflicting node names, got %q", got) + if got, err := decodeProtoWSMessage(data); err == nil || got != nil { + t.Fatalf("expected conflicting node names error, got %+v, err=%v", got, err) } }) } diff --git a/cmd/unbounded-net-controller/ws_frame_buffer.go b/cmd/unbounded-net-controller/ws_frame_buffer.go new file mode 100644 index 000000000..ba7c0fec3 --- /dev/null +++ b/cmd/unbounded-net-controller/ws_frame_buffer.go @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "io" + + "github.com/coder/websocket" + "k8s.io/klog/v2" +) + +const ( + maxNodeWSFrameBytes = 2 * 1024 * 1024 + nodeWSBufferClasses = 9 + nodeWSMinBufferSize = 8 * 1024 + nodeWSBuffersPerClass = 16 +) + +// Idle buffers are capped below 64 MiB across all connections. Active frames +// retain exclusive ownership until processing finishes, not for a connection's lifetime. +type nodeWSBufferPool struct { + buffers [nodeWSBufferClasses]chan []byte +} + +var nodeWSBuffers = newNodeWSBufferPool(nodeWSBuffersPerClass) + +func newNodeWSBufferPool(perClass int) *nodeWSBufferPool { + pool := &nodeWSBufferPool{} + for i := range pool.buffers { + pool.buffers[i] = make(chan []byte, perClass) + } + + return pool +} + +func (p *nodeWSBufferPool) get(size int) []byte { + for i, buffers := range p.buffers { + classSize := nodeWSMinBufferSize << i + if classSize < size { + continue + } + + select { + case data := <-buffers: + return data[:0] + default: + return make([]byte, 0, classSize) + } + } + + panic("node WebSocket buffer request exceeds the frame limit") +} + +func (p *nodeWSBufferPool) put(data []byte) { + for i, buffers := range p.buffers { + if cap(data) != nodeWSMinBufferSize< 0 { + emptyReads = 0 + continue + } + } + + emptyReads++ + if emptyReads >= 100 { + p.put(data) + return nil, io.ErrNoProgress + } + } +} + +func (p *nodeWSBufferPool) readFrame(ctx context.Context, conn *websocket.Conn) (wsFrame, error) { + msgType, reader, err := conn.Reader(ctx) + if err != nil { + return wsFrame{}, err + } + + data, err := p.read(reader) + if errors.Is(err, websocket.ErrMessageTooBig) { + if closeErr := conn.Close(websocket.StatusMessageTooBig, "node status exceeds 2 MiB"); closeErr != nil { + klog.V(4).Infof("Node WebSocket oversized message close failed: %v", closeErr) + } + } + + return wsFrame{msgType: msgType, data: data}, err +} diff --git a/cmd/unbounded-net-controller/ws_frame_buffer_test.go b/cmd/unbounded-net-controller/ws_frame_buffer_test.go new file mode 100644 index 000000000..1723fdef1 --- /dev/null +++ b/cmd/unbounded-net-controller/ws_frame_buffer_test.go @@ -0,0 +1,318 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/coder/websocket" +) + +func TestNodeWSBufferSizes(t *testing.T) { + for _, size := range []int{0, 1, nodeWSMinBufferSize - 1, nodeWSMinBufferSize, nodeWSMinBufferSize + 1, 512 * 1024, maxNodeWSFrameBytes, maxNodeWSFrameBytes + 1} { + t.Run(fmt.Sprint(size), func(t *testing.T) { + pool := newNodeWSBufferPool(2) + payload := bytes.Repeat([]byte{0x5a}, size) + + data, err := pool.read(bytes.NewReader(payload)) + defer pool.put(data) + + if size > maxNodeWSFrameBytes { + if !errors.Is(err, websocket.ErrMessageTooBig) || data != nil { + t.Fatalf("oversized frame returned %d bytes, %v", len(data), err) + } + + return + } + + if err != nil || !bytes.Equal(data, payload) { + t.Fatalf("frame size %d: got %d bytes, %v", size, len(data), err) + } + }) + } +} + +func TestNodeWSBufferOwnershipAndBound(t *testing.T) { + pool := newNodeWSBufferPool(2) + + first, err := pool.read(bytes.NewReader([]byte("first"))) + if err != nil { + t.Fatal(err) + } + + second, err := pool.read(bytes.NewReader([]byte("second"))) + if err != nil { + t.Fatal(err) + } + + pool.put(second) + + reused := pool.get(nodeWSMinBufferSize) + copy(reused[:cap(reused)], []byte("overwrite")) + + if string(first) != "first" { + t.Fatal("reusing another frame mutated an in-flight frame") + } + + pool.put(first) + pool.put(reused) + + retained := 0 + + for i, buffers := range pool.buffers { + size := nodeWSMinBufferSize << i + for range 4 { + pool.put(make([]byte, size)) + } + + if len(buffers) != 2 { + t.Fatalf("class %d retained %d buffers, want 2", i, len(buffers)) + } + + retained += len(buffers) * size + } + + if retained != 2*(2*maxNodeWSFrameBytes-nodeWSMinBufferSize) { + t.Fatalf("unexpected retention: %d bytes", retained) + } +} + +type failingWSReader struct { + err error +} + +func (r failingWSReader) Read([]byte) (int, error) { + return 0, r.err +} + +func TestNodeWSBufferReadErrors(t *testing.T) { + readFailure := errors.New("read failure") + for _, tc := range []struct { + name string + reader io.Reader + want error + }{ + {"failure", failingWSReader{readFailure}, readFailure}, + {"cancellation", failingWSReader{context.Canceled}, context.Canceled}, + {"wrapped-eof", failingWSReader{fmt.Errorf("closed mid-frame: %w", io.EOF)}, io.EOF}, + {"no-progress", failingWSReader{}, io.ErrNoProgress}, + {"failure-after-growth", io.MultiReader(bytes.NewReader(make([]byte, 2*nodeWSMinBufferSize)), failingWSReader{readFailure}), readFailure}, + } { + t.Run(tc.name, func(t *testing.T) { + pool := newNodeWSBufferPool(2) + + data, err := pool.read(tc.reader) + if data != nil || !errors.Is(err, tc.want) { + t.Fatalf("read returned %d bytes, %v", len(data), err) + } + + if len(pool.buffers[0]) == 0 { + t.Fatal("failed read did not release its buffer") + } + }) + } +} + +func TestNodeWSBufferAllocations(t *testing.T) { + pool := newNodeWSBufferPool(2) + reader := bytes.NewReader(make([]byte, 512*1024)) + + allocs := testing.AllocsPerRun(100, func() { + if _, err := reader.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + + data, err := pool.read(reader) + if err != nil { + t.Fatal(err) + } + + pool.put(data) + }) + if allocs > 1 { + t.Fatalf("warmed frame reader allocated %.1f objects; want at most 1", allocs) + } +} + +func TestNodeWSFrameTransport(t *testing.T) { + for _, compression := range []websocket.CompressionMode{websocket.CompressionDisabled, websocket.CompressionContextTakeover} { + for _, size := range []int{0, 32769, maxNodeWSFrameBytes, maxNodeWSFrameBytes + 1} { + t.Run(fmt.Sprintf("compression-%d/bytes-%d", compression, size), func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + pool := newNodeWSBufferPool(2) + result := make(chan error, 1) + payload := bytes.Repeat([]byte("x"), size) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{CompressionMode: compression}) + if err != nil { + result <- err + return + } + defer func() { + if err := conn.CloseNow(); err != nil && !errors.Is(err, net.ErrClosed) { + t.Errorf("server close: %v", err) + } + }() + + conn.SetReadLimit(maxNodeWSFrameBytes) + + frame, err := pool.readFrame(ctx, conn) + defer pool.put(frame.data) + + if err == nil && (frame.msgType != websocket.MessageBinary || !bytes.Equal(frame.data, payload)) { + err = errors.New("frame type or payload changed") + } + + result <- err + })) + defer server.Close() + + conn, _, err := websocket.Dial(ctx, server.URL, &websocket.DialOptions{CompressionMode: compression}) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := conn.CloseNow(); err != nil && !errors.Is(err, net.ErrClosed) { + t.Errorf("client close: %v", err) + } + }() + + if err := conn.Write(ctx, websocket.MessageBinary, payload); err != nil && size <= maxNodeWSFrameBytes { + t.Fatal(err) + } + + if size > maxNodeWSFrameBytes { + _, _, err := conn.Read(ctx) + if websocket.CloseStatus(err) != websocket.StatusMessageTooBig { + t.Fatalf("oversize close: %v", err) + } + } + + select { + case err := <-result: + if size > maxNodeWSFrameBytes { + if !errors.Is(err, websocket.ErrMessageTooBig) { + t.Fatalf("expected size rejection, got %v", err) + } + } else if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + }) + } + } +} + +func TestNodeWSFrameCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + pool := newNodeWSBufferPool(2) + started := make(chan struct{}) + result := make(chan error, 1) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + result <- err + return + } + defer func() { + if err := conn.CloseNow(); err != nil && !errors.Is(err, net.ErrClosed) { + t.Errorf("server close: %v", err) + } + }() + + close(started) + + frame, err := pool.readFrame(ctx, conn) + pool.put(frame.data) + + result <- err + })) + defer server.Close() + + conn, _, err := websocket.Dial(t.Context(), server.URL, nil) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := conn.CloseNow(); err != nil && !errors.Is(err, net.ErrClosed) { + t.Errorf("client close: %v", err) + } + }() + + <-started + cancel() + + select { + case err := <-result: + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation, got %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("canceled reader did not return") + } +} + +func BenchmarkNodeWSFrameBuffer(b *testing.B) { + for _, size := range []int{8 * 1024, 512 * 1024, maxNodeWSFrameBytes} { + for _, pooled := range []bool{false, true} { + b.Run(fmt.Sprintf("bytes-%d/pooled-%t", size, pooled), func(b *testing.B) { + pool := newNodeWSBufferPool(2) + + reader := bytes.NewReader(make([]byte, size)) + if pooled { + data, err := pool.read(reader) + if err != nil { + b.Fatal(err) + } + + pool.put(data) + } + + b.ReportAllocs() + b.SetBytes(int64(size)) + + for b.Loop() { + if _, err := reader.Seek(0, io.SeekStart); err != nil { + b.Fatal(err) + } + + var ( + data []byte + err error + ) + if pooled { + data, err = pool.read(reader) + } else { + data, err = io.ReadAll(reader) + } + + if err != nil || len(data) != size { + b.Fatalf("read returned %d bytes, %v", len(data), err) + } + + if pooled { + pool.put(data) + } + } + }) + } + } +} diff --git a/cmd/unbounded-net-node/bpf_status.go b/cmd/unbounded-net-node/bpf_status.go index f0712a0fe..a13fdf30e 100644 --- a/cmd/unbounded-net-node/bpf_status.go +++ b/cmd/unbounded-net-node/bpf_status.go @@ -139,9 +139,11 @@ func bpfCollectEntries(m *ebpf.Map) ([]statusv1alpha1.BpfEntry, error) { var entries []statusv1alpha1.BpfEntry + resolveInterface := newBpfInterfaceResolver(net.InterfaceByIndex) + iter := m.Iterate() for iter.Next(&key, &val) { - entries = append(entries, bpfMakeEntries(key, val)...) + entries = bpfAppendEntries(entries, key, val, resolveInterface) } if err := iter.Err(); err != nil { @@ -151,17 +153,16 @@ func bpfCollectEntries(m *ebpf.Map) ([]statusv1alpha1.BpfEntry, error) { return entries, nil } -// bpfMakeEntries expands a single LPM trie entry into one BpfEntry per +// bpfAppendEntries expands a single LPM trie entry into one BpfEntry per // nexthop. v4 entries (those whose key is IPv4-mapped) are rendered with // dotted-quad CIDR notation; v6 entries use canonical v6 form. Underlay // addresses follow the same rule. -func bpfMakeEntries(key ebpfpkg.LpmKey, val ebpfpkg.RawTunnelEndpoint) []statusv1alpha1.BpfEntry { +func bpfAppendEntries(entries []statusv1alpha1.BpfEntry, key ebpfpkg.LpmKey, val ebpfpkg.RawTunnelEndpoint, resolveInterface func(uint32) (string, int)) []statusv1alpha1.BpfEntry { cidr := bpfFormatKey(key) - entries := make([]statusv1alpha1.BpfEntry, 0, val.Count) for i := uint32(0); i < val.Count && i < uint32(ebpfpkg.MaxNexthops); i++ { nh := val.Nexthops[i] - ifName, mtu := bpfResolveInterface(nh.Ifindex) + ifName, mtu := resolveInterface(nh.Ifindex) entries = append(entries, statusv1alpha1.BpfEntry{ CIDR: cidr, @@ -192,7 +193,7 @@ func bpfFormatKey(key ebpfpkg.LpmKey) string { return fmt.Sprintf("%s/%d", ip.String(), prefix) } - ip := net.IP(append([]byte(nil), key.Addr[:]...)) + ip := net.IP(key.Addr[:]) return fmt.Sprintf("%s/%d", ip.String(), key.Prefixlen) } @@ -204,7 +205,7 @@ func bpfFormatEndpoint(addr [16]byte) string { return net.IPv4(addr[12], addr[13], addr[14], addr[15]).String() } - return net.IP(append([]byte(nil), addr[:]...)).String() + return net.IP(addr[:]).String() } // bpfProtocolName returns the tunnel protocol name for the given constant. @@ -225,13 +226,28 @@ func bpfProtocolName(proto uint32) string { } } -// bpfResolveInterface returns the interface name and MTU for the given -// ifindex; returns a placeholder if the interface no longer exists. -func bpfResolveInterface(ifindex uint32) (string, int) { - iface, err := net.InterfaceByIndex(int(ifindex)) - if err != nil { - return fmt.Sprintf("if%d", ifindex), 0 +// newBpfInterfaceResolver caches successful lookups for one collection only. +// Failed lookups retain the existing placeholder and are retried on the next entry. +func newBpfInterfaceResolver(lookup func(int) (*net.Interface, error)) func(uint32) (string, int) { + type interfaceInfo struct { + name string + mtu int } - return iface.Name, iface.MTU + interfaces := make(map[uint32]interfaceInfo) + + return func(ifindex uint32) (string, int) { + if info, ok := interfaces[ifindex]; ok { + return info.name, info.mtu + } + + iface, err := lookup(int(ifindex)) + if err != nil { + return fmt.Sprintf("if%d", ifindex), 0 + } + + interfaces[ifindex] = interfaceInfo{name: iface.Name, mtu: iface.MTU} + + return iface.Name, iface.MTU + } } diff --git a/cmd/unbounded-net-node/bpf_status_test.go b/cmd/unbounded-net-node/bpf_status_test.go new file mode 100644 index 000000000..ac5eb0a7c --- /dev/null +++ b/cmd/unbounded-net-node/bpf_status_test.go @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "net" + "net/netip" + "reflect" + "testing" + + ebpfpkg "github.com/Azure/unbounded/internal/net/ebpf" +) + +func TestBpfInterfaceResolverSnapshot(t *testing.T) { + calls := make(map[int]int) + current := net.Interface{Name: "wg51820", MTU: 1400} + lookup := func(index int) (*net.Interface, error) { + calls[index]++ + if index == 9 && calls[index] == 1 { + return nil, errors.New("interface disappeared") + } + + return ¤t, nil + } + + resolve := newBpfInterfaceResolver(lookup) + for range 2000 { + name, mtu := resolve(7) + if name != "wg51820" || mtu != 1400 { + t.Fatalf("unexpected interface: %s, %d", name, mtu) + } + } + + if calls[7] != 1 { + t.Fatalf("repeated interface lookup: %d calls", calls[7]) + } + + if name, mtu := resolve(9); name != "if9" || mtu != 0 { + t.Fatalf("missing-interface fallback changed: %s, %d", name, mtu) + } + + if name, mtu := resolve(9); name != "wg51820" || mtu != 1400 || calls[9] != 2 { + t.Fatalf("failed lookup was not retried: %s, %d, %v", name, mtu, calls) + } + + current.Name, current.MTU = "wg51821", 1500 + + if name, mtu := resolve(7); name != "wg51820" || mtu != 1400 { + t.Fatal("cached interface metadata changed within the snapshot") + } + + next := newBpfInterfaceResolver(lookup) + if name, mtu := next(7); name != "wg51821" || mtu != 1500 || calls[7] != 2 { + t.Fatalf("next snapshot retained stale metadata: %s, %d, %v", name, mtu, calls) + } +} + +func TestBpfAppendEntries(t *testing.T) { + for _, tc := range []struct { + name, address, remote, cidr string + prefix uint32 + }{ + {"v4", "10.1.2.0", "192.0.2.1", "10.1.2.0/24", 120}, + {"v6", "fd00:1::", "2001:db8::1", "fd00:1::/64", 64}, + } { + t.Run(tc.name, func(t *testing.T) { + key := ebpfpkg.LpmKey{Addr: netip.MustParseAddr(tc.address).As16(), Prefixlen: tc.prefix} + + var value ebpfpkg.RawTunnelEndpoint + + value.Count = 2 + value.Nexthops[0].Ifindex = 7 + value.Nexthops[0].RemoteEndpoint = netip.MustParseAddr(tc.remote).As16() + value.Nexthops[0].Protocol = ebpfpkg.TunnelProtoGENEVE + value.Nexthops[0].Healthy = 1 + value.Nexthops[0].Vni = 42 + value.Nexthops[1] = value.Nexthops[0] + value.Nexthops[1].Healthy = 0 + value.Nexthops[1].Protocol = 123 + + resolver := newBpfInterfaceResolver(func(int) (*net.Interface, error) { + return &net.Interface{Name: "ugn0", MTU: 1400}, nil + }) + prefix := BpfEntry{CIDR: "existing"} + got := bpfAppendEntries([]BpfEntry{prefix}, key, value, resolver) + + want := []BpfEntry{ + prefix, + {CIDR: tc.cidr, Remote: tc.remote, Interface: "ugn0", MTU: 1400, IfIndex: 7, VNI: 42, Protocol: "GENEVE", Healthy: true}, + {CIDR: tc.cidr, Remote: tc.remote, Interface: "ugn0", MTU: 1400, IfIndex: 7, VNI: 42, Protocol: "unknown(123)", Healthy: false}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("entry fields or append order changed: got %+v, want %+v", got, want) + } + + value.Count = 0 + if got := bpfAppendEntries(nil, key, value, resolver); len(got) != 0 { + t.Fatalf("zero nexthops produced entries: %+v", got) + } + + value.Count = ^uint32(0) + if got := bpfAppendEntries(nil, key, value, resolver); len(got) != ebpfpkg.MaxNexthops { + t.Fatalf("nexthop bound changed: %d", len(got)) + } + }) + } +} + +func BenchmarkBpfEntryCollection(b *testing.B) { + for _, count := range []int{10, 2000} { + for _, cached := range []bool{false, true} { + b.Run(fmt.Sprintf("entries-%d/cached-%t", count, cached), func(b *testing.B) { + key := ebpfpkg.LpmKey{Addr: netip.MustParseAddr("10.1.0.0").As16(), Prefixlen: 120} + + var value ebpfpkg.RawTunnelEndpoint + + value.Count = 1 + value.Nexthops[0].Ifindex = 7 + lookups := 0 + lookup := func(int) (*net.Interface, error) { + lookups++ + return &net.Interface{Name: "wg51820", MTU: 1400}, nil + } + + b.ReportAllocs() + + for b.Loop() { + var entries []BpfEntry + + resolver := func(index uint32) (string, int) { + iface, err := lookup(int(index)) + if err != nil { + b.Fatal(err) + } + + return iface.Name, iface.MTU + } + if cached { + resolver = newBpfInterfaceResolver(lookup) + } + + for range count { + if cached { + entries = bpfAppendEntries(entries, key, value, resolver) + } else { + entries = append(entries, bpfAppendEntries(make([]BpfEntry, 0, value.Count), key, value, resolver)...) + } + } + + if len(entries) != count { + b.Fatal("incorrect entry count") + } + } + + b.ReportMetric(float64(lookups)/float64(b.N), "lookups/op") + }) + } + } +} diff --git a/cmd/unbounded-net-node/cni_reconcile_test.go b/cmd/unbounded-net-node/cni_reconcile_test.go index eb2ee2b21..54fe326f1 100644 --- a/cmd/unbounded-net-node/cni_reconcile_test.go +++ b/cmd/unbounded-net-node/cni_reconcile_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/client-go/kubernetes/fake" unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/net/healthcheck" unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" ) @@ -57,7 +58,7 @@ func TestCNIReconciliationDisablesAndRecoversWithoutMTUChange(t *testing.T) { }) ensureCNIBridgeMTUFunc = func(string, int, *unboundednetnetlink.NetlinkCache, bool) error { return nil } - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { return nil } diff --git a/cmd/unbounded-net-node/gateway_pool_peering_protocol_test.go b/cmd/unbounded-net-node/gateway_pool_peering_protocol_test.go index a3111d9b9..bea4442bf 100644 --- a/cmd/unbounded-net-node/gateway_pool_peering_protocol_test.go +++ b/cmd/unbounded-net-node/gateway_pool_peering_protocol_test.go @@ -15,6 +15,7 @@ import ( unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/internal/net/healthcheck" unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" ) @@ -180,7 +181,7 @@ func newPoolPeeringProtocolFixture(t *testing.T, gateway bool) *poolPeeringProto } original := configureWireGuardFunc - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, mesh []meshPeerInfo, gateways []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, mesh []meshPeerInfo, gateways []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { f.configureCalls++ f.gateways = append([]gatewayPeerInfo(nil), gateways...) diff --git a/cmd/unbounded-net-node/main_update_test.go b/cmd/unbounded-net-node/main_update_test.go index fc88bf5ac..5bdb9c792 100644 --- a/cmd/unbounded-net-node/main_update_test.go +++ b/cmd/unbounded-net-node/main_update_test.go @@ -17,6 +17,7 @@ import ( unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/internal/net/healthcheck" unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" ) @@ -140,7 +141,7 @@ func TestUpdateWireGuardFromSlices_SitePodCIDRPoolChanges(t *testing.T) { ) origConfigure := configureWireGuardFunc - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, state *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, state *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { configureCalls++ gotPools = append([]string(nil), state.sitePodCIDRPools...) @@ -334,7 +335,7 @@ func TestUpdateWireGuardFromSlices_GatewayMeshPeersUseOnlyDirectConnectedSites(t var gotPeers []meshPeerInfo origConfigure := configureWireGuardFunc - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { gotPeers = append([]meshPeerInfo(nil), peers...) return nil } @@ -439,7 +440,7 @@ func TestUpdateWireGuardFromSlices_ExternalGatewayIncludesAssignedNonDirectSites var gotPeers []meshPeerInfo origConfigure := configureWireGuardFunc - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { gotPeers = append([]meshPeerInfo(nil), peers...) return nil } @@ -524,7 +525,7 @@ func TestUpdateWireGuardFromSlices_NonGatewayMeshPeersUseOnlyPeeredSites(t *test var gotPeers []meshPeerInfo origConfigure := configureWireGuardFunc - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { gotPeers = append([]meshPeerInfo(nil), peers...) return nil } @@ -610,7 +611,7 @@ func TestUpdateWireGuardFromSlices_ManageCniPluginFalseSkipsPodCIDRRoutesForSame var gotGatewayPeers []gatewayPeerInfo origConfigure := configureWireGuardFunc - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { gotGatewayPeers = append([]gatewayPeerInfo(nil), gatewayPeers...) return nil } @@ -697,7 +698,7 @@ func TestUpdateWireGuardFromSlices_ManageCniPluginFalseSkipsPodCIDRRoutesForSame var gotPeers []meshPeerInfo origConfigure := configureWireGuardFunc - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { gotPeers = append([]meshPeerInfo(nil), peers...) return nil } @@ -791,7 +792,7 @@ func TestUpdateWireGuardFromSlices_ManageCniPluginFalseKeepsRemotePeeredMeshPeer var gotPeers []meshPeerInfo origConfigure := configureWireGuardFunc - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, _ []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { gotPeers = append([]meshPeerInfo(nil), peers...) return nil } diff --git a/cmd/unbounded-net-node/peer_healthcheck.go b/cmd/unbounded-net-node/peer_healthcheck.go index f12110122..43e6c3b0d 100644 --- a/cmd/unbounded-net-node/peer_healthcheck.go +++ b/cmd/unbounded-net-node/peer_healthcheck.go @@ -4,13 +4,38 @@ package main import ( + "errors" + "fmt" "net" + "time" "k8s.io/klog/v2" "github.com/Azure/unbounded/internal/net/healthcheck" ) +// A disabled association must block fallback to less-specific enabled profiles. +const disabledHealthCheckProfile = "disabled" + +var errRegisterHealthChecks = errors.New("health check registration failed") + +func resolvedHealthCheckSettings(name string, profiles map[string]healthcheck.HealthCheckSettings, maxBackoff time.Duration) (healthcheck.HealthCheckSettings, bool, error) { + if name == "" || name == disabledHealthCheckProfile { + return healthcheck.HealthCheckSettings{}, false, nil + } + + settings, ok := profiles[name] + if !ok { + return healthcheck.HealthCheckSettings{}, false, fmt.Errorf("health check profile %q is missing from the current reconciliation", name) + } + + if maxBackoff > 0 { + settings.MaxBackoff = maxBackoff + } + + return settings, true, nil +} + // registerPeersWithHealthCheck registers mesh and gateway peers with the // healthcheck manager, resolving HC profiles for each peer. It sets // state.meshPeerHealthCheckEnabled and state.gatewayPeerHealthCheckEnabled @@ -24,7 +49,8 @@ import ( // when no pool/assignment-level profile is found for a gateway peer. This is // used by GENEVE which has no WireGuard handshake as a liveness signal. // -// Returns the set of peer names that were registered (desiredHCPeers). +// Returns desired peer names and registration errors. On error, names are retained +// so a failed configuration does not remove an existing healthy session. func registerPeersWithHealthCheck( meshPeers []meshPeerInfo, gatewayPeers []gatewayPeerInfo, @@ -35,15 +61,18 @@ func registerPeersWithHealthCheck( assignmentSiteHCProfileNames map[string]string, assignmentPoolHCProfileNames map[string]string, poolHCProfileNames map[string]string, + profiles map[string]healthcheck.HealthCheckSettings, state *wireGuardState, peerIfaceNameFn func(gatewayPeerInfo) string, useSiteFallbackForGateway bool, -) map[string]bool { +) (map[string]bool, error) { desiredHCPeers := make(map[string]bool) if state.healthCheckManager == nil { - return desiredHCPeers + return desiredHCPeers, nil } + var registrationErrors []error + // Mesh peers. for _, peer := range meshPeers { overlayIP := getHealthIPFromPodCIDRs(peer.PodCIDRs) @@ -53,7 +82,16 @@ func registerPeersWithHealthCheck( hcProfileName := resolveMeshPeerHealthCheckProfileName(isGatewayNode, peer, mySiteName, siteHCProfileNames, peeringHCProfileNames, assignmentSiteHCProfileNames) - if hcProfileName == "" { + + settings, enabled, err := resolvedHealthCheckSettings(hcProfileName, profiles, state.healthFlapMaxBackoff) + if err != nil { + desiredHCPeers[peer.Name] = true + registrationErrors = append(registrationErrors, fmt.Errorf("mesh peer %s: %w", peer.Name, err)) + + continue + } + + if !enabled { continue } @@ -64,13 +102,8 @@ func registerPeersWithHealthCheck( state.mu.Unlock() } - settings := healthcheck.DefaultSettings() - if state.healthFlapMaxBackoff > 0 { - settings.MaxBackoff = state.healthFlapMaxBackoff - } - if err := state.healthCheckManager.AddPeer(peer.Name, net.ParseIP(overlayIP), settings); err != nil { - klog.V(2).Infof("Healthcheck: failed to register mesh peer %s at %s: %v", peer.Name, overlayIP, err) + registrationErrors = append(registrationErrors, fmt.Errorf("register mesh peer %s at %s: %w", peer.Name, overlayIP, err)) } else { klog.V(4).Infof("Healthcheck: registered mesh peer %s at %s", peer.Name, overlayIP) } @@ -94,7 +127,15 @@ func registerPeersWithHealthCheck( hcProfileName = siteHCProfileNames[mySiteName] } - if hcProfileName == "" { + settings, enabled, err := resolvedHealthCheckSettings(hcProfileName, profiles, state.healthFlapMaxBackoff) + if err != nil { + desiredHCPeers[gwPeer.Name] = true + registrationErrors = append(registrationErrors, fmt.Errorf("gateway peer %s: %w", gwPeer.Name, err)) + + continue + } + + if !enabled { continue } @@ -104,19 +145,18 @@ func registerPeersWithHealthCheck( state.gatewayPeerHealthCheckEnabled[ifName] = true state.mu.Unlock() - settings := healthcheck.DefaultSettings() - if state.healthFlapMaxBackoff > 0 { - settings.MaxBackoff = state.healthFlapMaxBackoff - } - if err := state.healthCheckManager.AddPeer(gwPeer.Name, net.ParseIP(overlayIP), settings); err != nil { - klog.V(2).Infof("Healthcheck: failed to register gateway peer %s at %s: %v", gwPeer.Name, overlayIP, err) + registrationErrors = append(registrationErrors, fmt.Errorf("register gateway peer %s at %s: %w", gwPeer.Name, overlayIP, err)) } else { klog.V(4).Infof("Healthcheck: registered gateway peer %s at %s (iface %s)", gwPeer.Name, overlayIP, ifName) } } - return desiredHCPeers + if len(registrationErrors) > 0 { + return desiredHCPeers, fmt.Errorf("%w: %w", errRegisterHealthChecks, errors.Join(registrationErrors...)) + } + + return desiredHCPeers, nil } // peerIfaceNameWireGuard maps a gateway peer to its WireGuard interface name diff --git a/cmd/unbounded-net-node/peer_healthcheck_test.go b/cmd/unbounded-net-node/peer_healthcheck_test.go new file mode 100644 index 000000000..9ced587c6 --- /dev/null +++ b/cmd/unbounded-net-node/peer_healthcheck_test.go @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes/fake" + + unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/internal/net/healthcheck" + unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" +) + +func TestRegisterHealthProfilesAllTunnelModes(t *testing.T) { + for _, protocol := range []string{"WireGuard", "GENEVE", "VXLAN", "IPIP", "None"} { + for _, tc := range []struct { + name string + gatewayPeer, gatewayNode bool + override, site, peering, assignmentSite, assignmentLocal, assignmentRemote, pool, want string + }{ + {name: "mesh default", site: "site", want: "site"}, + {name: "mesh peering", site: "site", peering: "peering", want: "peering"}, + {name: "mesh assignment", site: "site", peering: "peering", assignmentSite: "assignment", want: "assignment"}, + {name: "mesh gateway role", gatewayNode: true, site: "site", peering: "peering", assignmentSite: "assignment", want: "assignment"}, + {name: "mesh explicit pool", override: "pool", assignmentSite: "assignment", want: "pool"}, + {name: "mesh explicit pool peering", override: "pool-peering", assignmentSite: "assignment", want: "pool-peering"}, + {name: "mesh disabled assignment", site: "site", assignmentSite: disabledHealthCheckProfile}, + {name: "mesh disabled override", override: disabledHealthCheckProfile, site: "site"}, + {name: "gateway assignment", gatewayPeer: true, assignmentLocal: "assignment", assignmentRemote: "remote", pool: "pool", want: "assignment"}, + {name: "gateway pool", gatewayPeer: true, gatewayNode: true, assignmentRemote: "remote", pool: "pool", want: "pool"}, + {name: "gateway remote assignment fallback", gatewayPeer: true, gatewayNode: true, assignmentRemote: "remote", want: "remote"}, + {name: "gateway explicit", gatewayPeer: true, override: "pool-peering", assignmentLocal: "assignment", want: "pool-peering"}, + {name: "gateway disabled assignment", gatewayPeer: true, site: "site", assignmentLocal: disabledHealthCheckProfile}, + {name: "gateway disabled pool", gatewayPeer: true, gatewayNode: true, site: "site", pool: disabledHealthCheckProfile, assignmentRemote: "remote"}, + {name: "gateway disabled override", gatewayPeer: true, override: disabledHealthCheckProfile, site: "site", assignmentLocal: "assignment"}, + } { + t.Run(protocol+"/"+tc.name, func(t *testing.T) { + manager, err := healthcheck.NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + defer manager.Stop() + + profiles := make(map[string]healthcheck.HealthCheckSettings) + + for i, name := range []string{"site", "peering", "assignment", "remote", "pool", "pool-peering"} { + _, profile := healthCheckProfileFromSettings(&unboundednetv1alpha1.HealthCheckSettings{ + TransmitInterval: ptrIntOrString(intstr.FromInt(1000 + i)), + }, name) + profiles[name] = profile + } + + state := &wireGuardState{ + healthCheckManager: manager, healthFlapMaxBackoff: 300 * time.Second, + healthCheckProfiles: map[string]healthcheck.HealthCheckSettings{"site": {TransmitInterval: time.Millisecond}}, + meshPeerHealthCheckEnabled: make(map[string]bool), gatewayPeerHealthCheckEnabled: make(map[string]bool), + } + + var ( + mesh []meshPeerInfo + gateways []gatewayPeerInfo + ) + if tc.gatewayPeer { + gateways = []gatewayPeerInfo{{Name: "peer", SiteName: "remote", PoolName: "pool", PodCIDRs: []string{"10.244.1.0/24"}, HealthCheckProfileName: tc.override, TunnelProtocol: protocol}} + } else { + mesh = []meshPeerInfo{{Name: "peer", SiteName: "remote", WireGuardPublicKey: "pub", PodCIDRs: []string{"10.244.1.0/24"}, HealthCheckProfileName: tc.override, TunnelProtocol: protocol}} + } + + desired, err := registerPeersWithHealthCheck(mesh, gateways, "local", tc.gatewayNode, + map[string]string{"local": tc.site, "remote": tc.site}, map[string]string{"remote": tc.peering}, + map[string]string{"remote": tc.assignmentSite}, map[string]string{"local|pool": tc.assignmentLocal, "remote|pool": tc.assignmentRemote}, + map[string]string{"pool": tc.pool}, profiles, state, func(gatewayPeerInfo) string { return "iface" }, protocol != "WireGuard") + if err != nil { + t.Fatal(err) + } + + if tc.want == "" { + if desired["peer"] || len(manager.GetAllPeerStatuses()) != 0 || state.meshPeerHealthCheckEnabled["pub"] || state.gatewayPeerHealthCheckEnabled["iface"] { + t.Fatal("disabled profile fell through to enabled lower-precedence profile") + } + + return + } + + want := profiles[tc.want] + want.MaxBackoff = 300 * time.Second + + got, err := manager.GetPeerSettings("peer") + if err != nil || got != want || !desired["peer"] { + t.Fatalf("got %+v, %v; want %+v", got, err, want) + } + + if got.ReceiveInterval != 15*time.Second { + t.Fatal("partial profile lost 15s receive default") + } + }) + } + } +} + +func TestRegisterHealthProfileFallbackAndMissing(t *testing.T) { + for _, siteFallback := range []bool{false, true} { + manager, err := healthcheck.NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + + state := &wireGuardState{healthCheckManager: manager, meshPeerHealthCheckEnabled: make(map[string]bool), gatewayPeerHealthCheckEnabled: make(map[string]bool)} + gateways := []gatewayPeerInfo{{Name: "peer", PoolName: "pool", PodCIDRs: []string{"10.244.1.0/24"}}} + + desired, err := registerPeersWithHealthCheck(nil, gateways, "local", false, map[string]string{"local": "site"}, nil, nil, nil, nil, + map[string]healthcheck.HealthCheckSettings{"site": healthcheck.DefaultSettings()}, state, func(gatewayPeerInfo) string { return "iface" }, siteFallback) + if err != nil || desired["peer"] != siteFallback { + t.Fatalf("site fallback %t: desired=%v err=%v", siteFallback, desired, err) + } + + gateways[0].HealthCheckProfileName = "missing" + + desired, err = registerPeersWithHealthCheck(nil, gateways, "local", false, nil, nil, nil, nil, nil, nil, state, func(gatewayPeerInfo) string { return "iface" }, siteFallback) + if !errors.Is(err, errRegisterHealthChecks) || !desired["peer"] { + t.Fatal("missing fresh profile must fail without deleting existing session") + } + + manager.Stop() + } +} + +func TestHealthProfilePartialUpdateResetsUnspecifiedValues(t *testing.T) { + manager, err := healthcheck.NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + defer manager.Stop() + + state := &wireGuardState{healthCheckManager: manager, healthFlapMaxBackoff: 240 * time.Second, meshPeerHealthCheckEnabled: make(map[string]bool)} + peer := meshPeerInfo{Name: "peer", SiteName: "local", WireGuardPublicKey: "pub", PodCIDRs: []string{"10.244.1.0/24"}} + profile := healthcheck.DefaultSettings() + profile.TransmitInterval, profile.ReceiveInterval = time.Second, time.Second + profiles := map[string]healthcheck.HealthCheckSettings{"site": profile} + register := func() { + t.Helper() + + if _, err := registerPeersWithHealthCheck([]meshPeerInfo{peer}, nil, "local", false, map[string]string{"local": "site"}, nil, nil, nil, nil, profiles, state, nil, false); err != nil { + t.Fatal(err) + } + } + register() + + _, profiles["site"] = healthCheckProfileFromSettings(&unboundednetv1alpha1.HealthCheckSettings{ + TransmitInterval: ptrIntOrString(intstr.FromString("60s")), + }, "site") + + register() + + got, err := manager.GetPeerSettings("peer") + if err != nil || got.TransmitInterval != 60*time.Second || got.ReceiveInterval != 15*time.Second || got.MaxBackoff != 240*time.Second { + t.Fatalf("partial update reused stale values: %+v %v", got, err) + } +} + +func TestDisabledAssignmentBlocksLowerPrecedence(t *testing.T) { + assignment := unboundednetv1alpha1.SiteGatewayPoolAssignment{ + ObjectMeta: metav1.ObjectMeta{Name: "assignment"}, + Spec: unboundednetv1alpha1.SiteGatewayPoolAssignmentSpec{ + Sites: []string{"local", "remote"}, GatewayPools: []string{"pool"}, + HealthCheckSettings: &unboundednetv1alpha1.HealthCheckSettings{Enabled: ptrBool(false)}, + }, + } + profiles := make(map[string]healthcheck.HealthCheckSettings) + pools, sites := make(map[string]string), make(map[string]string) + mergeAssignmentHealthCheckState(assignment, "local", nil, profiles, nil, pools, make(map[string]string), sites, make(map[string]string)) + + if len(profiles) != 0 || pools["local|pool"] != disabledHealthCheckProfile || sites["remote"] != disabledHealthCheckProfile { + t.Fatalf("disabled association was discarded: pools=%v sites=%v", pools, sites) + } +} + +func TestReconciliationPassesFreshHealthProfilesBeforeStateCommit(t *testing.T) { + manager, err := healthcheck.NewManager("node-self", 0, nil) + if err != nil { + t.Fatal(err) + } + defer manager.Stop() + + site := &unboundedv1alpha3.Site{ + ObjectMeta: metav1.ObjectMeta{Name: "site"}, + Spec: unboundedv1alpha3.SiteSpec{HealthCheckSettings: &unboundednetv1alpha1.HealthCheckSettings{ + TransmitInterval: ptrIntOrString(intstr.FromString("60s")), + }}, + } + siteInformer := newInformerWithObjects(toUnstructured(t, site)) + slices := newInformerWithObjects(toUnstructured(t, &unboundednetv1alpha1.SiteNodeSlice{ + ObjectMeta: metav1.ObjectMeta{Name: "slice"}, SiteName: "site", + Nodes: []unboundednetv1alpha1.NodeInfo{{Name: "peer", WireGuardPublicKey: "pub-peer", InternalIPs: []string{"10.0.0.2"}, PodCIDRs: []string{"10.244.1.0/24"}}}, + })) + stale := healthcheck.DefaultSettings() + stale.TransmitInterval = time.Second + state := &wireGuardState{ + clientset: fake.NewClientset(&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-self"}}), + nodeName: "node-self", healthCheckManager: manager, + healthCheckProfiles: map[string]healthcheck.HealthCheckSettings{"s-site": stale}, + } + + original := configureWireGuardFunc + defer func() { configureWireGuardFunc = original }() + + called := false + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, peers []meshPeerInfo, gateways []gatewayPeerInfo, siteName string, _, _, _ map[string]bool, + siteNames, peeringNames, assignmentSites, assignmentPools, poolNames map[string]string, _, _, _, _, _ map[string]int, + _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, s *wireGuardState, profiles map[string]healthcheck.HealthCheckSettings, + ) error { + called = true + + if s.healthCheckProfiles["s-site"] != stale { + t.Fatal("test must observe uncommitted state") + } + + if profiles["s-site"].TransmitInterval != 60*time.Second { + t.Fatal("fresh profile was not passed") + } + + _, err := registerPeersWithHealthCheck(peers, gateways, siteName, false, siteNames, peeringNames, assignmentSites, assignmentPools, poolNames, profiles, s, func(gatewayPeerInfo) string { return "" }, false) + + return err + } + empty := newInformerWithObjects() + + err = updateWireGuardFromSlices(context.Background(), nil, siteInformer, slices, empty, empty, empty, empty, empty, + &config{NodeName: "node-self", WireGuardPort: 51820}, "site", "private", "pub-self", true, state) + if err != nil || !called { + t.Fatalf("reconciliation: called=%t err=%v", called, err) + } + + got, err := manager.GetPeerSettings("peer") + if err != nil || got.TransmitInterval != 60*time.Second || got.ReceiveInterval != 15*time.Second { + t.Fatalf("registration used stale state: %+v %v", got, err) + } +} diff --git a/cmd/unbounded-net-node/reconciliation_helpers.go b/cmd/unbounded-net-node/reconciliation_helpers.go index 0d9f5381f..091872623 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers.go +++ b/cmd/unbounded-net-node/reconciliation_helpers.go @@ -104,9 +104,7 @@ func healthCheckProfileNameForGatewayPoolPeering(name string) string { } func healthCheckProfilesEqual(a, b healthcheck.HealthCheckSettings) bool { - return a.DetectMultiplier == b.DetectMultiplier && - a.ReceiveInterval == b.ReceiveInterval && - a.TransmitInterval == b.TransmitInterval + return a == b } // mergeAssignmentHealthCheckState merges SiteGatewayPoolAssignment health check settings into active maps. @@ -122,7 +120,7 @@ func mergeAssignmentHealthCheckState( assignmentSiteHealthCheckProfileNames map[string]string, assignmentSiteHealthCheckSourceAssignment map[string]string, ) { - assignmentHealthCheckProfileName := "" + assignmentHealthCheckProfileName := disabledHealthCheckProfile assignmentScope := healthCheckLogScope(siteGatewayPoolAssignmentGVR, assignment.Name) if enabled, profile := healthCheckProfileFromSettings(assignment.Spec.HealthCheckSettings, assignmentScope); enabled { @@ -137,10 +135,6 @@ func mergeAssignmentHealthCheckState( } } - if assignmentHealthCheckProfileName == "" { - return - } - for _, poolName := range assignment.Spec.GatewayPools { poolName = strings.TrimSpace(poolName) if poolName == "" { diff --git a/cmd/unbounded-net-node/reconciliation_helpers_test.go b/cmd/unbounded-net-node/reconciliation_helpers_test.go index 0bab1020b..617a69d84 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers_test.go +++ b/cmd/unbounded-net-node/reconciliation_helpers_test.go @@ -72,6 +72,7 @@ func TestHealthCheckProfileSettingsHelpers(t *testing.T) { } want := healthcheck.HealthCheckSettings{ + MaxBackoff: 120 * time.Second, DetectMultiplier: 5, ReceiveInterval: 150 * time.Millisecond, TransmitInterval: 275 * time.Millisecond, @@ -99,6 +100,7 @@ func TestHealthCheckProfileSettingsHelpers(t *testing.T) { } siteWant := healthcheck.HealthCheckSettings{ + MaxBackoff: 120 * time.Second, DetectMultiplier: 7, ReceiveInterval: 200 * time.Millisecond, TransmitInterval: 400 * time.Millisecond, diff --git a/cmd/unbounded-net-node/route_annotations.go b/cmd/unbounded-net-node/route_annotations.go index e72ff6ea4..c5d78ed79 100644 --- a/cmd/unbounded-net-node/route_annotations.go +++ b/cmd/unbounded-net-node/route_annotations.go @@ -354,11 +354,12 @@ func annotateNodeRoutes( // Build expected routes and annotate expectedIPv4, expectedIPv6 := buildExpectedRoutes(cfg, status, actx, localSiteName) + classificationPeers := buildRouteClassificationPeers(status.Peers) ipv4Routes, ipv6Routes := splitRoutesByFamily(status.RoutingTable.Routes) ipv4Routes = annotateFamilyRoutes(cfg, ipv4Routes, expectedIPv4) ipv6Routes = annotateFamilyRoutes(cfg, ipv6Routes, expectedIPv6) - ipv4Routes = annotateRoutePeerDestinationsForFamily(cfg, ipv4Routes, status.Peers, actx, localSiteName) - ipv6Routes = annotateRoutePeerDestinationsForFamily(cfg, ipv6Routes, status.Peers, actx, localSiteName) + ipv4Routes = annotateRoutePeerDestinationsForFamily(cfg, ipv4Routes, status.Peers, classificationPeers, actx, localSiteName) + ipv6Routes = annotateRoutePeerDestinationsForFamily(cfg, ipv6Routes, status.Peers, classificationPeers, actx, localSiteName) // Mark supernet routes on unbounded0 as expected. These are managed // routes that don't correspond to individual peers but exist in the FIB @@ -836,6 +837,7 @@ func annotateRoutePeerDestinationsForFamily( cfg *config, routes []RouteEntry, peers []WireGuardPeerStatus, + classificationPeers map[string]routeplan.Peer, actx *annotationContext, localSiteName string, ) []RouteEntry { @@ -843,16 +845,6 @@ func annotateRoutePeerDestinationsForFamily( return routes } - peerByName := make(map[string]WireGuardPeerStatus, len(peers)) - for _, peer := range peers { - peerName := strings.TrimSpace(peer.Name) - if peerName == "" { - continue - } - - peerByName[peerName] = peer - } - expectations := make([]peerRouteExpectation, 0, len(peers)) for _, peer := range peers { peerName := strings.TrimSpace(peer.Name) @@ -957,23 +949,23 @@ func annotateRoutePeerDestinationsForFamily( sort.Strings(peerNames) hop.PeerDestinations = peerNames - hop.Info = routeInfoForNextHop(normalizedDestination, peerNames, peerByName, actx) + hop.Info = routeInfoForNextHop(normalizedDestination, peerNames, classificationPeers, actx) } } return routes } -// routeInfoForNextHop classifies a next-hop destination to determine its -// object name, object type, and route type for display purposes. -func routeInfoForNextHop( - normalizedDestination string, - peerNames []string, - peerByName map[string]WireGuardPeerStatus, - actx *annotationContext, -) *NextHopInfo { - routePeers := make(map[string]routeplan.Peer, len(peerByName)) - for peerName, peer := range peerByName { +// buildRouteClassificationPeers creates one read-only index for both address +// families in a status snapshot, rather than rebuilding it for each next-hop. +func buildRouteClassificationPeers(peers []WireGuardPeerStatus) map[string]routeplan.Peer { + routePeers := make(map[string]routeplan.Peer, len(peers)) + for _, peer := range peers { + peerName := strings.TrimSpace(peer.Name) + if peerName == "" { + continue + } + routePeers[peerName] = routeplan.Peer{ Name: peer.Name, PeerType: peer.PeerType, @@ -985,6 +977,17 @@ func routeInfoForNextHop( } } + return routePeers +} + +// routeInfoForNextHop classifies a next-hop destination to determine its +// object name, object type, and route type for display purposes. +func routeInfoForNextHop( + normalizedDestination string, + peerNames []string, + routePeers map[string]routeplan.Peer, + actx *annotationContext, +) *NextHopInfo { sharedInfo := routeplan.ClassifyRouteInfoForPeerDestination(normalizedDestination, peerNames, routePeers, actx.routeNodes, actx.sitePodCIDRs, actx.siteNodeCIDRs, actx.gatewayPoolRoutedCIDRs) if sharedInfo == nil { return nil diff --git a/cmd/unbounded-net-node/route_annotations_memory_test.go b/cmd/unbounded-net-node/route_annotations_memory_test.go new file mode 100644 index 000000000..0d5037b51 --- /dev/null +++ b/cmd/unbounded-net-node/route_annotations_memory_test.go @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/Azure/unbounded/internal/net/routeplan" +) + +func TestRouteClassificationSnapshot(t *testing.T) { + peers := []WireGuardPeerStatus{ + {Name: " peer-a ", PeerType: "site", SiteName: "site-a", Tunnel: PeerTunnelStatus{Endpoint: "old"}}, + {Name: "peer-a", PeerType: "site", SiteName: "site-a", PodCIDRGateways: []string{"10.244.1.1", "fd00:1::1"}}, + {Name: "peer-b", PeerType: "site", SiteName: "site-a"}, + {Name: " peer-c ", PeerType: "site", SiteName: "site-b"}, + {Name: "gw", PeerType: "gateway", SiteName: "site-a", Tunnel: PeerTunnelStatus{ + Endpoint: "203.0.113.1:51820", AllowedIPs: []string{"100.64.0.0/16"}, + }}, + {Name: " \t", PeerType: "site"}, + } + ctx := &annotationContext{ + routeNodes: map[string]routeplan.Node{ + "peer-a": {Name: "peer-a", SiteName: "site-a", PodCIDRs: []string{"10.244.1.0/24", "fd00:1::/64"}}, + "peer-b": {Name: "peer-b", SiteName: "site-a", PodCIDRs: []string{"10.244.2.0/24"}}, + "gw": {Name: "gw", SiteName: "site-a", InternalIPs: []string{"172.20.0.1"}, ExternalIPs: []string{"203.0.113.1"}}, + }, + sitePodCIDRs: map[string]map[string]struct{}{ + "site-a": {"10.244.1.0/24": {}, "10.244.2.0/24": {}}, + }, + gatewayPoolRoutedCIDRs: map[string]map[string]struct{}{ + "pool-a": {"100.64.0.0/16": {}}, + }, + } + + index := buildRouteClassificationPeers(peers) + if len(index) != 4 || index["peer-a"].Endpoint != "" || index["peer-c"].Name != " peer-c " { + t.Fatalf("blank-name filtering or last-duplicate precedence changed: %+v", index) + } + + before, err := json.Marshal(struct { + Peers []WireGuardPeerStatus + Index map[string]routeplan.Peer + }{peers, index}) + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name string + destination string + peers []string + want *NextHopInfo + }{ + {"node-v4", "10.244.1.1/32", []string{"peer-a"}, &NextHopInfo{ObjectName: "peer-a", ObjectType: "node", RouteType: "podCidr"}}, + {"node-v6", "fd00:1::1/128", []string{"peer-a"}, &NextHopInfo{ObjectName: "peer-a", ObjectType: "node", RouteType: "podCidr"}}, + {"site-supernet", "10.244.0.0/16", []string{"peer-a", "peer-b"}, &NextHopInfo{ObjectName: "site-a", ObjectType: "site", RouteType: "podCidr"}}, + {"pool", "100.64.0.0/16", []string{"gw"}, &NextHopInfo{ObjectName: "pool-a", ObjectType: "gatewayPool", RouteType: "routedCidr"}}, + {"gateway-host", "172.20.0.1/32", []string{"gw"}, &NextHopInfo{ObjectName: "gw", ObjectType: "gateway", RouteType: "nodeCidr"}}, + {"unknown-peer", "10.1.0.0/24", []string{"missing"}, nil}, + {"empty-destination", "", []string{"peer-a"}, nil}, + {"empty-peers", "10.244.1.0/24", nil, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := routeInfoForNextHop(tc.destination, tc.peers, index, ctx); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("classification: got %+v, want %+v", got, tc.want) + } + }) + } + + after, err := json.Marshal(struct { + Peers []WireGuardPeerStatus + Index map[string]routeplan.Peer + }{peers, index}) + if err != nil { + t.Fatal(err) + } + + if string(before) != string(after) { + t.Fatal("classification mutated snapshot data") + } + + peers[1].SiteName = "site-new" + + next := buildRouteClassificationPeers(peers) + if next["peer-a"].SiteName != "site-new" || index["peer-a"].SiteName != "site-a" { + t.Fatal("peer metadata leaked between snapshot indexes") + } + + if len(buildRouteClassificationPeers(nil)) != 0 { + t.Fatal("empty snapshot created peer entries") + } +} + +func routeClassificationFixture(count int) ([]WireGuardPeerStatus, []string, *annotationContext) { + peers := make([]WireGuardPeerStatus, count) + names := make([]string, count) + ctx := &annotationContext{routeNodes: make(map[string]routeplan.Node, count)} + + for i := range peers { + name := fmt.Sprintf("peer-%d", i) + cidr := fmt.Sprintf("10.%d.%d.0/24", i/256, i%256) + names[i] = name + peers[i] = WireGuardPeerStatus{ + Name: name, PeerType: "site", SiteName: "site-a", + Tunnel: PeerTunnelStatus{Interface: "wg51820", AllowedIPs: []string{cidr}}, + } + ctx.routeNodes[name] = routeplan.Node{Name: name, PodCIDRs: []string{cidr}} + } + + return peers, names, ctx +} + +// legacyRouteClassification retains the original per-hop projection as a benchmark baseline. +func legacyRouteClassification(destination string, names []string, peers map[string]WireGuardPeerStatus, ctx *annotationContext) *NextHopInfo { + index := make(map[string]routeplan.Peer, len(peers)) + for name, peer := range peers { + index[name] = routeplan.Peer{ + Name: peer.Name, PeerType: peer.PeerType, SiteName: peer.SiteName, + SkipPodCIDRRoutes: peer.SkipPodCIDRRoutes, Endpoint: peer.Tunnel.Endpoint, + PodCIDRGateways: peer.PodCIDRGateways, AllowedIPs: peer.Tunnel.AllowedIPs, + } + } + + return routeInfoForNextHop(destination, names, index, ctx) +} + +func TestRouteClassificationAllocationBound(t *testing.T) { + peers, names, ctx := routeClassificationFixture(2000) + index := buildRouteClassificationPeers(peers) + + result := testing.Benchmark(func(b *testing.B) { + for b.Loop() { + info := routeInfoForNextHop("10.0.0.0/24", names[:1], index, ctx) + if info == nil || info.ObjectName != names[0] { + b.Fatal("unexpected classification") + } + } + }) + if result.AllocedBytesPerOp() > 4096 { + t.Fatalf("classification allocated %d B/op with 2000 indexed peers; limit 4096", result.AllocedBytesPerOp()) + } +} + +func TestRouteAnnotationFamiliesShareReadOnlyIndex(t *testing.T) { + cfg := &config{WireGuardInterfacePrefix: "wg"} + peers := []WireGuardPeerStatus{{ + Name: "node-b", PeerType: "site", SiteName: "site-a", + Tunnel: PeerTunnelStatus{Interface: "wg51820", AllowedIPs: []string{"10.1.0.0/24", "fd00:1::/64"}}, + }} + ctx := &annotationContext{routeNodes: map[string]routeplan.Node{ + "node-b": {Name: "node-b", SiteName: "site-a", PodCIDRs: []string{"10.1.0.0/24", "fd00:1::/64"}}, + }} + index := buildRouteClassificationPeers(peers) + + for _, destination := range []string{"10.1.0.0/24", "fd00:1::/64"} { + routes := []RouteEntry{{Destination: destination, NextHops: []NextHop{{ + Device: "wg51820", PeerDestinations: []string{" node-b ", "node-b", ""}, + }}}} + got := annotateRoutePeerDestinationsForFamily(cfg, routes, peers, index, ctx, "site-a") + + hop := got[0].NextHops[0] + if !reflect.DeepEqual(hop.PeerDestinations, []string{"node-b"}) || + !reflect.DeepEqual(hop.Info, &NextHopInfo{ObjectName: "node-b", ObjectType: "node", RouteType: "podCidr"}) { + t.Fatalf("%s: unexpected annotation %+v", destination, hop) + } + + for _, tc := range []struct{ destination, device string }{ + {"invalid", "wg51820"}, + {"10.1.0.0/24", "eth0"}, + {"10.2.0.0/24", "wg51820"}, + } { + routes := []RouteEntry{{Destination: tc.destination, NextHops: []NextHop{{ + Device: tc.device, Info: &NextHopInfo{ObjectName: "stale"}, + }}}} + + got := annotateRoutePeerDestinationsForFamily(cfg, routes, peers, index, ctx, "site-a") + if hop := got[0].NextHops[0]; hop.Info != nil || len(hop.PeerDestinations) != 0 { + t.Fatalf("unmatched route retained classification: %+v", hop) + } + } + } +} + +func BenchmarkRouteClassificationSnapshot(b *testing.B) { + for _, count := range []int{10, 100, 2000} { + peers, names, ctx := routeClassificationFixture(count) + + legacy := make(map[string]WireGuardPeerStatus, count) + for _, peer := range peers { + legacy[strings.TrimSpace(peer.Name)] = peer + } + + for _, rebuild := range []bool{true, false} { + b.Run(fmt.Sprintf("peers-%d/rebuild-per-hop-%t", count, rebuild), func(b *testing.B) { + b.ReportAllocs() + + for b.Loop() { + var index map[string]routeplan.Peer + if !rebuild { + index = buildRouteClassificationPeers(peers) + } + + for i, name := range names { + destination := ctx.routeNodes[name].PodCIDRs[0] + + var info *NextHopInfo + if rebuild { + info = legacyRouteClassification(destination, names[i:i+1], legacy, ctx) + } else { + info = routeInfoForNextHop(destination, names[i:i+1], index, ctx) + } + + if info == nil || info.ObjectName != name { + b.Fatal("unexpected classification") + } + } + } + }) + } + } +} diff --git a/cmd/unbounded-net-node/site_routing_reconcile_test.go b/cmd/unbounded-net-node/site_routing_reconcile_test.go index 0acbe6696..12a4b17c4 100644 --- a/cmd/unbounded-net-node/site_routing_reconcile_test.go +++ b/cmd/unbounded-net-node/site_routing_reconcile_test.go @@ -15,6 +15,7 @@ import ( unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/internal/net/healthcheck" unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" ) @@ -49,7 +50,7 @@ func TestUpdateWireGuardFromSlices_LocalGatewayExclusions(t *testing.T) { var gotGatewayPeers []gatewayPeerInfo original := configureWireGuardFunc - configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState) error { + configureWireGuardFunc = func(_ context.Context, _ *config, _ string, _ []meshPeerInfo, gatewayPeers []gatewayPeerInfo, _ string, _, _, _ map[string]bool, _, _, _, _, _ map[string]string, _, _, _, _, _ map[string]int, _ []unboundednetnetlink.DesiredRoute, _ map[string]bool, _ *wireGuardState, _ map[string]healthcheck.HealthCheckSettings) error { calls++ gotGatewayPeers = gatewayPeers @@ -208,7 +209,7 @@ func TestUpdateWireGuardFromSlices_GatewayRoutingChanges(t *testing.T) { var configureErr error original := configureWireGuardFunc - configureWireGuardFunc = func(context.Context, *config, string, []meshPeerInfo, []gatewayPeerInfo, string, map[string]bool, map[string]bool, map[string]bool, map[string]string, map[string]string, map[string]string, map[string]string, map[string]string, map[string]int, map[string]int, map[string]int, map[string]int, map[string]int, []unboundednetnetlink.DesiredRoute, map[string]bool, *wireGuardState) error { + configureWireGuardFunc = func(context.Context, *config, string, []meshPeerInfo, []gatewayPeerInfo, string, map[string]bool, map[string]bool, map[string]bool, map[string]string, map[string]string, map[string]string, map[string]string, map[string]string, map[string]int, map[string]int, map[string]int, map[string]int, map[string]int, []unboundednetnetlink.DesiredRoute, map[string]bool, *wireGuardState, map[string]healthcheck.HealthCheckSettings) error { calls++ return configureErr } diff --git a/cmd/unbounded-net-node/site_watch_reconcile.go b/cmd/unbounded-net-node/site_watch_reconcile.go index 6a62a0aa5..8abc5ff5c 100644 --- a/cmd/unbounded-net-node/site_watch_reconcile.go +++ b/cmd/unbounded-net-node/site_watch_reconcile.go @@ -989,6 +989,7 @@ func updateWireGuardFromSlices(ctx context.Context, dynamicClient dynamic.Interf enabled, profile := healthCheckProfileFromSettings(site.Spec.HealthCheckSettings, siteScope) if !enabled { + siteHealthCheckProfileNames[siteName] = disabledHealthCheckProfile continue } @@ -1109,7 +1110,7 @@ func updateWireGuardFromSlices(ctx context.Context, dynamicClient dynamic.Interf // If our site is in this peering, add all other sites if mySiteInPeering { remoteSites := make([]string, 0, len(peering.Spec.Sites)) - peeringHealthCheckProfileName := "" + peeringHealthCheckProfileName := disabledHealthCheckProfile peeringScope := healthCheckLogScope(sitePeeringGVR, peering.Name) if enabled, profile := healthCheckProfileFromSettings(peering.Spec.HealthCheckSettings, peeringScope); enabled { @@ -1287,6 +1288,8 @@ func updateWireGuardFromSlices(ctx context.Context, dynamicClient dynamic.Interf poolHealthCheckProfileNames[pool.Name] = profileName healthCheckProfileSources[profileName] = poolScope } + } else { + poolHealthCheckProfileNames[pool.Name] = disabledHealthCheckProfile } // Collect pool-level tunnelMTU override. if v := tunnelMTUFromSpec(pool.Spec.TunnelMTU); v > 0 { @@ -1407,7 +1410,7 @@ func updateWireGuardFromSlices(ctx context.Context, dynamicClient dynamic.Interf continue } - peeringHealthCheckProfileName := "" + peeringHealthCheckProfileName := disabledHealthCheckProfile peeringScope := healthCheckLogScope(gatewayPoolPeeringGVR, peering.Name) if enabled, profile := healthCheckProfileFromSettings(peering.Spec.HealthCheckSettings, peeringScope); enabled { @@ -2100,7 +2103,7 @@ func updateWireGuardFromSlices(ctx context.Context, dynamicClient dynamic.Interf assignmentSiteHealthCheckProfileNames, assignmentPoolHealthCheckProfileNames, poolHealthCheckProfileNames, siteTunnelMTUs, peeringSiteTunnelMTUs, assignmentSiteTunnelMTUs, - assignmentPoolTunnelMTUs, poolTunnelMTUs, state) + assignmentPoolTunnelMTUs, poolTunnelMTUs, state, healthCheckProfiles) if sharedTunnelErr != nil { klog.Warningf("Tunnel configuration failed (WireGuard will still be configured): %v", sharedTunnelErr) } @@ -2133,11 +2136,15 @@ func updateWireGuardFromSlices(ctx context.Context, dynamicClient dynamic.Interf // Configure WireGuard with WG peers, merging tunnel routes into // the unified route manager's SyncRoutes call. - if err := configureWireGuardFunc(ctx, cfg, privKey, wgMeshPeers, wgGatewayPeers, mySiteName, peeredSites, networkPeeredSites, gatewayNodePubKeys, siteHealthCheckProfileNames, peeringSiteHealthCheckProfileNames, assignmentSiteHealthCheckProfileNames, assignmentPoolHealthCheckProfileNames, poolHealthCheckProfileNames, siteTunnelMTUs, peeringSiteTunnelMTUs, assignmentSiteTunnelMTUs, assignmentPoolTunnelMTUs, poolTunnelMTUs, tunnelRoutes, tunnelHCPeers, state); err != nil { + if err := configureWireGuardFunc(ctx, cfg, privKey, wgMeshPeers, wgGatewayPeers, mySiteName, peeredSites, networkPeeredSites, gatewayNodePubKeys, siteHealthCheckProfileNames, peeringSiteHealthCheckProfileNames, assignmentSiteHealthCheckProfileNames, assignmentPoolHealthCheckProfileNames, poolHealthCheckProfileNames, siteTunnelMTUs, peeringSiteTunnelMTUs, assignmentSiteTunnelMTUs, assignmentPoolTunnelMTUs, poolTunnelMTUs, tunnelRoutes, tunnelHCPeers, state, healthCheckProfiles); err != nil { return err } - if sharedTunnelErr != nil && fabricMTUIncreased { + if sharedTunnelErr != nil && (fabricMTUIncreased || errors.Is(sharedTunnelErr, errRegisterHealthChecks)) { + if errors.Is(sharedTunnelErr, errRegisterHealthChecks) { + return sharedTunnelErr + } + return fmt.Errorf("cannot raise fabric MTU while tunnel reconciliation is incomplete: %w", sharedTunnelErr) } diff --git a/cmd/unbounded-net-node/status_ack.go b/cmd/unbounded-net-node/status_ack.go new file mode 100644 index 000000000..b81a85770 --- /dev/null +++ b/cmd/unbounded-net-node/status_ack.go @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "sync/atomic" + + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +// statusAckState is created fresh for every connection. One outstanding message +// keeps the sender's snapshot and the controller's acknowledged revision aligned. +type statusAckState struct { + revision atomic.Uint64 + resync atomic.Bool + pending atomic.Bool + compact atomic.Bool +} + +func (s *statusAckState) accept(data []byte) bool { + var ack statusproto.NodeStatusAck + if err := proto.Unmarshal(data, &ack); err != nil { + var envelope struct { + Type string `json:"type"` + Data nodeStatusPushAck `json:"data"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return false + } + + switch envelope.Type { + case "node_status_ack": + ack.Status = "ok" + case "node_status_resync": + ack.Status = "resync_required" + default: + return false + } + + ack.Revision = envelope.Data.Revision + } + + switch ack.Status { + case "ok": + s.compact.Store(ack.PeerMeasurements && ack.Revision > 0) + case "resync_required": + s.compact.Store(false) + s.resync.Store(true) + default: + return false + } + + if ack.Revision > 0 { + s.revision.Store(ack.Revision) + } + + s.pending.Store(false) + + return true +} diff --git a/cmd/unbounded-net-node/status_delta.go b/cmd/unbounded-net-node/status_delta.go new file mode 100644 index 000000000..8e9b1fe47 --- /dev/null +++ b/cmd/unbounded-net-node/status_delta.go @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "cmp" + "reflect" + "slices" + "time" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +// sortStatusPeers is called only on freshly collected, exclusively owned peers. +// Gateway map iteration must not turn unchanged topology into peer replacements. +func sortStatusPeers(peers []WireGuardPeerStatus) { + slices.SortFunc(peers, func(a, b WireGuardPeerStatus) int { + return cmp.Or( + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Tunnel.Protocol, b.Tunnel.Protocol), + cmp.Compare(a.Tunnel.Interface, b.Tunnel.Interface), + cmp.Compare(a.Tunnel.PublicKey, b.Tunnel.PublicKey), + ) + }) +} + +// typedStatusDelta avoids serializing both snapshots to JSON and decoding again. +// refresh emits a timestamp even when values are unchanged, preserving liveness. +func typedStatusDelta(prev, curr *NodeStatusResponse, compact, refresh bool) *statusproto.NodeStatusDelta { + if prev == nil || curr == nil { + return nil + } + + pb := &statusproto.NodeStatusDelta{} + + add := func(field string) { pb.UpdatedFields = append(pb.UpdatedFields, field) } + if refresh || !prev.Timestamp.Equal(curr.Timestamp) { + add("timestamp") + + if !curr.Timestamp.IsZero() { + pb.TimestampUnixNs = curr.Timestamp.UnixNano() + } + } + + if !reflect.DeepEqual(prev.NodeInfo, curr.NodeInfo) { + add("nodeInfo") + + pb.NodeInfo = nodeInfoToProto(&curr.NodeInfo) + } + + if !reflect.DeepEqual(prev.Peers, curr.Peers) { + metadataEqual := compact && len(prev.Peers) == len(curr.Peers) + if metadataEqual { + for i := range curr.Peers { + if !netstatus.PeerMetadataEqual(prev.Peers[i], curr.Peers[i]) { + metadataEqual = false + break + } + } + } + + if metadataEqual { + // Ambiguous identities must use the full replacement path. + measurements, err := netstatus.PeerMeasurementsToProto(curr.Peers) + if err == nil { + pb.PeerMeasurements = measurements + } + } + + if pb.PeerMeasurements != nil { + add("peerMeasurements") + } else { + add("peers") + + pb.Peers = peersToProto(curr.Peers) + } + } + + if !reflect.DeepEqual(prev.RoutingTable, curr.RoutingTable) { + add("routingTable") + + pb.RoutingTable = routingTableToProto(&curr.RoutingTable) + } + + if !reflect.DeepEqual(prev.HealthCheck, curr.HealthCheck) { + add("healthCheck") + + pb.HealthCheck = healthCheckStatusToProto(curr.HealthCheck) + } + + if !reflect.DeepEqual(prev.NodeErrors, curr.NodeErrors) { + add("nodeErrors") + + pb.NodeErrors = nodeErrorsToProto(curr.NodeErrors) + } + + if !reflect.DeepEqual(prev.BpfEntries, curr.BpfEntries) { + add("bpfEntries") + + pb.BpfEntries = bpfEntriesToProto(curr.BpfEntries) + } + + if prev.FetchError != curr.FetchError { + add("fetchError") + + pb.FetchError = curr.FetchError + } + + if !reflect.DeepEqual(prev.LastPushTime, curr.LastPushTime) { + add("lastPushTime") + + if curr.LastPushTime != nil && !curr.LastPushTime.IsZero() { + pb.LastPushTimeUnixNs = curr.LastPushTime.UnixNano() + } + } + + if prev.StatusSource != curr.StatusSource { + add("statusSource") + + pb.StatusSource = curr.StatusSource + } + + if !reflect.DeepEqual(prev.NodePodInfo, curr.NodePodInfo) { + add("nodePodInfo") + + pb.NodePodInfo = nodePodInfoToProto(curr.NodePodInfo) + } + + if len(pb.UpdatedFields) == 0 { + return nil + } + + return pb +} + +// criticalStatus preserves the last published measurements while applying current +// metadata. Never modify shared snapshots or their health pointers. +func criticalStatus(prev, curr *NodeStatusResponse) *NodeStatusResponse { + result := *curr + result.Timestamp = prev.Timestamp + + if curr.HealthCheck != nil { + health := *curr.HealthCheck + + health.CheckedAt = time.Time{} + if prev.HealthCheck != nil { + health.CheckedAt = prev.HealthCheck.CheckedAt + } + + result.HealthCheck = &health + } + + result.Peers = append([]WireGuardPeerStatus(nil), curr.Peers...) + + type identity struct{ name, protocol, iface, key string } + + key := func(peer WireGuardPeerStatus) identity { + return identity{peer.Name, peer.Tunnel.Protocol, peer.Tunnel.Interface, peer.Tunnel.PublicKey} + } + + previous := make(map[identity]WireGuardPeerStatus, len(prev.Peers)) + for _, peer := range prev.Peers { + previous[key(peer)] = peer + } + + for i := range result.Peers { + peer := &result.Peers[i] + + old, ok := previous[key(*peer)] + if !ok { + continue + } + + peer.Tunnel.RxBytes = old.Tunnel.RxBytes + peer.Tunnel.TxBytes = old.Tunnel.TxBytes + + peer.Tunnel.LastHandshake = old.Tunnel.LastHandshake + if peer.HealthCheck != nil { + health := *peer.HealthCheck + + health.Uptime, health.RTT = "", "" + if old.HealthCheck != nil { + health.Uptime, health.RTT = old.HealthCheck.Uptime, old.HealthCheck.RTT + } + + peer.HealthCheck = &health + } + } + + return &result +} diff --git a/cmd/unbounded-net-node/status_delta_test.go b/cmd/unbounded-net-node/status_delta_test.go new file mode 100644 index 000000000..f1ff4ec89 --- /dev/null +++ b/cmd/unbounded-net-node/status_delta_test.go @@ -0,0 +1,438 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func measuredNodeStatus() *NodeStatusResponse { + status := testNodeStatus("old") + status.HealthCheck = &HealthCheckStatus{Healthy: true, Summary: "ok", PeerCount: 1, CheckedAt: testStatusTime} + status.Peers[0].HealthCheck = &HealthCheckPeerStatus{Enabled: true, Status: "up", Uptime: "1m", RTT: "1ms"} + + return status +} + +func TestCriticalStatusIgnoresOnlyMeasurements(t *testing.T) { + measurements := map[string]func(*NodeStatusResponse){ + "timestamp": func(s *NodeStatusResponse) { s.Timestamp = s.Timestamp.Add(time.Second) }, + "checkedAt": func(s *NodeStatusResponse) { s.HealthCheck.CheckedAt = s.HealthCheck.CheckedAt.Add(time.Second) }, + "rx": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.RxBytes++ }, + "tx": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.TxBytes++ }, + "handshake": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.LastHandshake = time.Time{} }, + "uptime": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.Uptime = "2m" }, + "rtt": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.RTT = "2ms" }, + } + for name, change := range measurements { + t.Run(name, func(t *testing.T) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + change(curr) + + before := nodeStatusToProto(curr) + if !reflect.DeepEqual(stripPeerStats(prev), stripPeerStats(curr)) { + t.Fatal("measurement triggered critical comparison") + } + + if delta := typedStatusDelta(prev, criticalStatus(prev, curr), true, false); delta != nil { + t.Fatalf("measurement leaked into critical delta: %v", delta) + } + + if typedStatusDelta(prev, curr, true, true) == nil { + t.Fatal("statistics refresh lost measurement") + } + + if !proto.Equal(before, nodeStatusToProto(curr)) { + t.Fatal("shared current snapshot mutated") + } + }) + } + + critical := map[string]func(*NodeStatusResponse){ + "nodeInfo": func(s *NodeStatusResponse) { s.NodeInfo.SiteName = "new" }, + "healthy": func(s *NodeStatusResponse) { s.HealthCheck.Healthy = false }, + "summary": func(s *NodeStatusResponse) { s.HealthCheck.Summary = "failed" }, + "count": func(s *NodeStatusResponse) { s.HealthCheck.PeerCount++ }, + "health removal": func(s *NodeStatusResponse) { s.HealthCheck = nil }, + "peer status": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.Status = "down" }, + "peer enabled": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.Enabled = false }, + "peer health removal": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck = nil }, + "endpoint": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.Endpoint = "new" }, + "error": func(s *NodeStatusResponse) { s.NodeErrors = []NodeError{{Type: "test", Message: "failed"}} }, + "error clear": func(s *NodeStatusResponse) { s.FetchError = "" }, + "bpf": func(s *NodeStatusResponse) { s.BpfEntries = []BpfEntry{{CIDR: "10.0.0.0/24"}} }, + "route": func(s *NodeStatusResponse) { s.RoutingTable.ManagedRouteCount++ }, + } + for name, change := range critical { + t.Run(name, func(t *testing.T) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + change(curr) + curr.Timestamp = curr.Timestamp.Add(time.Second) + + curr.Peers[0].Tunnel.RxBytes++ + if reflect.DeepEqual(stripPeerStats(prev), stripPeerStats(curr)) { + t.Fatal("critical change filtered") + } + + published := criticalStatus(prev, curr) + + delta := typedStatusDelta(prev, published, true, false) + if delta == nil || slices.Contains(delta.UpdatedFields, "timestamp") || delta.PeerMeasurements != nil { + t.Fatalf("invalid critical delta: %v", delta) + } + + if published.Peers[0].Tunnel.RxBytes != prev.Peers[0].Tunnel.RxBytes { + t.Fatal("critical change advanced statistics baseline") + } + }) + } +} + +func TestTypedStatusDeltaCompactCompatibility(t *testing.T) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + curr.Peers[0].Tunnel.RxBytes = 0 + curr.Peers[0].Tunnel.TxBytes = 0 + curr.Peers[0].Tunnel.LastHandshake = time.Time{} + curr.Peers[0].HealthCheck.Uptime = "" + + curr.Peers[0].HealthCheck.RTT = "" + for _, compact := range []bool{false, true} { + delta := typedStatusDelta(prev, curr, compact, false) + if compact { + if delta.PeerMeasurements == nil || len(delta.Peers) != 0 || !slices.Contains(delta.UpdatedFields, "peerMeasurements") { + t.Fatalf("missing compact measurements: %v", delta) + } + + m := delta.PeerMeasurements + if m.RxBytes[0] != 0 || m.TxBytes[0] != 0 || m.LastHandshakeUnixNs[0] != 0 || m.Uptime[0] != "" || m.Rtt[0] != "" { + t.Fatal("zero measurements did not clear") + } + } else if delta.PeerMeasurements != nil || len(delta.Peers) != 1 { + t.Fatalf("old controller requires peers replacement: %v", delta) + } + + data, err := proto.Marshal(delta) + if err != nil { + t.Fatal(err) + } + + var decoded statusproto.NodeStatusDelta + if err := proto.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + + if !proto.Equal(delta, &decoded) { + t.Fatal("delta did not round trip") + } + } + + for name, change := range map[string]func(*NodeStatusResponse){ + "added": func(s *NodeStatusResponse) { p := s.Peers[0]; p.Name = "peer-b"; s.Peers = append(s.Peers, p) }, + "deleted": func(s *NodeStatusResponse) { s.Peers = nil }, + "identity": func(s *NodeStatusResponse) { s.Peers[0].Name = "other" }, + "public key": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.PublicKey = "other" }, + "interface": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.Interface = "other" }, + "protocol": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.Protocol = "other" }, + "metadata": func(s *NodeStatusResponse) { s.Peers[0].SiteName = "other" }, + "health": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.Status = "down" }, + "unnamed": func(s *NodeStatusResponse) { s.Peers[0].Name = "" }, + } { + t.Run(name, func(t *testing.T) { + next := measuredNodeStatus() + change(next) + + delta := typedStatusDelta(prev, next, true, false) + if delta.PeerMeasurements != nil || !slices.Contains(delta.UpdatedFields, "peers") { + t.Fatalf("topology change requires replacement: %v", delta) + } + }) + } + + if typedStatusDelta(prev, prev, true, false) != nil { + t.Fatal("no-op produced delta") + } + + if typedStatusDelta(nil, curr, true, true) != nil { + t.Fatal("first update must be full") + } + + if got := typedStatusDelta(prev, prev, true, true); !slices.Equal(got.UpdatedFields, []string{"timestamp"}) { + t.Fatalf("periodic freshness: %v", got) + } +} + +func TestTypedStatusDeltaFieldClearings(t *testing.T) { + prev := measuredNodeStatus() + prev.NodeErrors = []NodeError{{Type: "test"}} + prev.LastPushTime = &prev.Timestamp + prev.StatusSource = "push" + prev.NodePodInfo = &statusv1alpha1.NodePodInfo{PodName: "old"} + prev.BpfEntries = []BpfEntry{{CIDR: "10.0.0.0/24"}} + prev.RoutingTable.ManagedRouteCount = 1 + curr := &NodeStatusResponse{} + delta := typedStatusDelta(prev, curr, true, false) + + want := []string{"timestamp", "nodeInfo", "peers", "routingTable", "healthCheck", "nodeErrors", "bpfEntries", "fetchError", "lastPushTime", "statusSource", "nodePodInfo"} + if !slices.Equal(delta.UpdatedFields, want) { + t.Fatalf("clearings %v, want %v", delta.UpdatedFields, want) + } +} + +func TestTypedStatusDeltaReorderingAndDuplicateFallback(t *testing.T) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + other := prev.Peers[0] + other.Name = "peer-b" + prev.Peers = append(prev.Peers, other) + + curr.Peers = []WireGuardPeerStatus{other, curr.Peers[0]} + if delta := typedStatusDelta(prev, curr, true, false); delta.PeerMeasurements != nil || len(delta.Peers) != 2 { + t.Fatal("reordered base must be replaced") + } + + sortStatusPeers(curr.Peers) + + if delta := typedStatusDelta(prev, curr, true, false); delta != nil { + t.Fatal("canonical ordering must eliminate iteration-only changes") + } + + prev.Peers[1] = prev.Peers[0] + curr.Peers = append([]WireGuardPeerStatus(nil), prev.Peers...) + + curr.Peers[0].Tunnel.RxBytes++ + if delta := typedStatusDelta(prev, curr, true, false); delta.PeerMeasurements != nil || len(delta.Peers) != 2 { + t.Fatal("duplicate identities must use legacy replacement") + } +} + +func TestWebSocketCriticalNoopStatsAndResync(t *testing.T) { + for _, mode := range []string{"critical", "stats-resync", "full-refresh"} { + t.Run(mode, func(t *testing.T) { + received := make(chan *statusproto.NodeStatusMessage, 32) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Errorf("accept: %v", err) + return + } + defer func() { _ = conn.Close(websocket.StatusNormalClosure, "done") }() + + var revision uint64 + + for { + _, data, err := conn.Read(ctx) + if err != nil { + return + } + + var message statusproto.NodeStatusMessage + if err := proto.Unmarshal(data, &message); err != nil { + t.Errorf("decode: %v", err) + return + } + + select { + case received <- &message: + case <-ctx.Done(): + return + } + + revision++ + + ack := &statusproto.NodeStatusAck{Status: "ok", Revision: revision, PeerMeasurements: true} + if mode == "stats-resync" && revision == 2 { + ack.Status = "resync_required" + } + + payload, err := proto.Marshal(ack) + if err != nil { + t.Errorf("encode ACK: %v", err) + return + } + + if err := conn.Write(ctx, websocket.MessageBinary, payload); err != nil { + return + } + } + })) + defer server.Close() + + cfg := &config{ + NodeName: "node-a", StatusWSEnabled: true, + StatusWSURL: "ws" + strings.TrimPrefix(server.URL, "http"), StatusWSAPIServerMode: statusWSAPIServerModeNever, + CriticalDeltaEvery: time.Hour, StatsDeltaEvery: time.Hour, FullSyncEvery: time.Hour, + } + + switch mode { + case "critical": + cfg.CriticalDeltaEvery = 10 * time.Millisecond + case "stats-resync": + cfg.StatsDeltaEvery = 15 * time.Millisecond + case "full-refresh": + cfg.FullSyncEvery = 15 * time.Millisecond + } + + health := blockedBootstrapHealthState() + + startStatusPublishers(ctx, cfg, health) + defer health.stopStatusPublishers() + + next := func() *statusproto.NodeStatusMessage { + t.Helper() + + select { + case message := <-received: + return message + case <-time.After(3 * time.Second): + t.Fatal("publisher did not send expected message") + return nil + } + } + if message := next(); message.Status == nil || message.Delta != nil { + t.Fatal("first frame must be full") + } + + if mode == "critical" { + select { + case message := <-received: + t.Fatalf("timestamp-only critical publication: %v", message) + case <-time.After(100 * time.Millisecond): + } + + health.setCNIReady("cbr0", []string{"10.244.7.0/24"}) + + if message := next(); !isCNIGuardClearingDelta(message) { + t.Fatal("critical error clearing lost") + } + } else if mode == "stats-resync" { + if message := next(); message.Delta == nil || !slices.Contains(message.Delta.UpdatedFields, "timestamp") { + t.Fatal("statistics interval did not publish freshness") + } + + if message := next(); message.Status == nil || message.Delta != nil { + t.Fatal("statistics interval must honor resync with full status") + } + } else if message := next(); message.Status == nil || message.Delta != nil { + t.Fatal("periodic full refresh lost") + } + + cancel() + }) + } +} + +func TestStatusAckNegotiation(t *testing.T) { + var state statusAckState + if state.compact.Load() || state.revision.Load() != 0 { + t.Fatal("new connection negotiated without ACK") + } + + for _, tc := range []struct { + name string + data []byte + compact, resync, valid bool + }{ + {"new controller", mustStatusAck(t, &statusproto.NodeStatusAck{Status: "ok", Revision: 1, PeerMeasurements: true}), true, false, true}, + {"old controller", mustStatusAck(t, &statusproto.NodeStatusAck{Status: "ok", Revision: 2}), false, false, true}, + {"JSON", []byte(`{"type":"node_status_ack","data":{"revision":3}}`), false, false, true}, + {"no base", mustStatusAck(t, &statusproto.NodeStatusAck{Status: "ok", PeerMeasurements: true}), false, false, true}, + {"resync", mustStatusAck(t, &statusproto.NodeStatusAck{Status: "resync_required", Revision: 4, PeerMeasurements: true}), false, true, true}, + {"invalid", []byte("invalid"), false, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + state.pending.Store(true) + + if state.accept(tc.data) != tc.valid || state.compact.Load() != tc.compact || state.resync.Load() != tc.resync { + t.Fatal("incorrect ACK negotiation") + } + + if state.pending.Load() == tc.valid { + t.Fatal("invalid pending-message state") + } + }) + } + + state.resync.Store(false) + + if !state.accept(mustStatusAck(t, &statusproto.NodeStatusAck{Status: "ok", Revision: 5, PeerMeasurements: true})) || !state.compact.Load() { + t.Fatal("full resync ACK did not renegotiate") + } + + reconnected := &statusAckState{} + if reconnected.compact.Load() || reconnected.revision.Load() != 0 { + t.Fatal("capability leaked across connections") + } +} + +func mustStatusAck(t *testing.T, ack *statusproto.NodeStatusAck) []byte { + t.Helper() + + data, err := proto.Marshal(ack) + if err != nil { + t.Fatal(err) + } + + return data +} + +func BenchmarkStatusDelta2000Peers(b *testing.B) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + + prev.Peers, curr.Peers = nil, nil + for i := range 2000 { + peer := measuredNodeStatus().Peers[0] + peer.Name = fmt.Sprintf("peer-%d", i) + peer.Tunnel.AllowedIPs = []string{"10.244.0.0/24"} + peer.PodCIDRGateways = []string{"10.244.0.1"} + peer.RouteDistances = map[string]int{"10.244.0.0/24": 1} + prev.Peers = append(prev.Peers, peer) + peer.Tunnel.RxBytes++ + curr.Peers = append(curr.Peers, peer) + } + + for _, mode := range []string{"legacy-json-full-peers", "typed-full-peers", "compact"} { + b.Run(mode, func(b *testing.B) { + b.ReportAllocs() + + for b.Loop() { + var delta *statusproto.NodeStatusDelta + + if mode == "legacy-json-full-peers" { + raw, err := computeStatusDelta(prev, curr) + if err != nil { + b.Fatal(err) + } + + delta = nodeStatusDeltaToProto(raw) + } else { + delta = typedStatusDelta(prev, curr, mode == "compact", false) + } + + data, err := proto.Marshal(delta) + if err != nil { + b.Fatal(err) + } + + b.ReportMetric(float64(len(data)), "wire-B/op") + } + }) + } +} diff --git a/cmd/unbounded-net-node/status_legacy_delta_test.go b/cmd/unbounded-net-node/status_legacy_delta_test.go new file mode 100644 index 000000000..1aaf39508 --- /dev/null +++ b/cmd/unbounded-net-node/status_legacy_delta_test.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// These legacy JSON helpers are retained only for existing compatibility tests +// and baseline benchmarks. Production publishers exclusively use typedStatusDelta, +// whose tests enforce critical metadata changes and explicit field clearings. +func computeStatusDelta(prev, curr *NodeStatusResponse) (map[string]json.RawMessage, error) { + if prev == nil { + return nil, nil + } + + prevRaw, err := json.Marshal(prev) + if err != nil { + return nil, err + } + + currRaw, err := json.Marshal(curr) + if err != nil { + return nil, err + } + + var prevMap map[string]json.RawMessage + if err := json.Unmarshal(prevRaw, &prevMap); err != nil { + return nil, err + } + + var currMap map[string]json.RawMessage + if err := json.Unmarshal(currRaw, &currMap); err != nil { + return nil, err + } + + delta := make(map[string]json.RawMessage) + if nodeInfo, ok := currMap["nodeInfo"]; ok { + delta["nodeInfo"] = nodeInfo + } + + for key, value := range currMap { + if key == "nodeInfo" { + continue + } + + prevValue, exists := prevMap[key] + if !exists || !bytes.Equal(prevValue, value) { + delta[key] = value + } + } + + if _, previouslyPresent := prevMap["nodeErrors"]; previouslyPresent { + if _, currentlyPresent := currMap["nodeErrors"]; !currentlyPresent { + delta["nodeErrors"] = json.RawMessage("[]") + } + } + // Preserve the old omission behavior for baseline comparisons only. + if len(delta) == 1 { + return nil, nil + } + + return delta, nil +} + +func nodeStatusDeltaToProto(delta map[string]json.RawMessage) *statusproto.NodeStatusDelta { + if len(delta) == 0 { + return nil + } + + pb := &statusproto.NodeStatusDelta{ + UpdatedFields: make([]string, 0, len(delta)), + } + for key, raw := range delta { + pb.UpdatedFields = append(pb.UpdatedFields, key) + + switch key { + case "nodeInfo": + var ni NodeInfo + if json.Unmarshal(raw, &ni) == nil { + pb.NodeInfo = nodeInfoToProto(&ni) + } + case "peers": + var peers []statusv1alpha1.PeerStatus + if json.Unmarshal(raw, &peers) == nil { + pb.Peers = peersToProto(peers) + } + case "routingTable": + var rt RoutingTableInfo + if json.Unmarshal(raw, &rt) == nil { + pb.RoutingTable = routingTableToProto(&rt) + } + case "healthCheck": + var hc HealthCheckStatus + if json.Unmarshal(raw, &hc) == nil { + pb.HealthCheck = healthCheckStatusToProto(&hc) + } + case "nodeErrors": + var errs []NodeError + if json.Unmarshal(raw, &errs) == nil { + pb.NodeErrors = nodeErrorsToProto(errs) + } + case "bpfEntries": + var entries []BpfEntry + if json.Unmarshal(raw, &entries) == nil { + pb.BpfEntries = bpfEntriesToProto(entries) + } + } + } + + return pb +} diff --git a/cmd/unbounded-net-node/status_proto.go b/cmd/unbounded-net-node/status_proto.go index b6d03f265..256a64711 100644 --- a/cmd/unbounded-net-node/status_proto.go +++ b/cmd/unbounded-net-node/status_proto.go @@ -4,12 +4,20 @@ package main import ( - "encoding/json" + "time" statusproto "github.com/Azure/unbounded/internal/net/status/proto" statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) +func statusUnixNano(t time.Time) int64 { + if t.IsZero() { + return 0 + } + + return t.UnixNano() +} + // nodeStatusToProto converts a Go NodeStatusResponse to the protobuf NodeStatusFull message. func nodeStatusToProto(status *NodeStatusResponse) *statusproto.NodeStatusFull { if status == nil { @@ -17,7 +25,7 @@ func nodeStatusToProto(status *NodeStatusResponse) *statusproto.NodeStatusFull { } full := &statusproto.NodeStatusFull{ - TimestampUnixNs: status.Timestamp.UnixNano(), + TimestampUnixNs: statusUnixNano(status.Timestamp), NodeInfo: nodeInfoToProto(&status.NodeInfo), Peers: peersToProto(status.Peers), RoutingTable: routingTableToProto(&status.RoutingTable), @@ -35,58 +43,6 @@ func nodeStatusToProto(status *NodeStatusResponse) *statusproto.NodeStatusFull { return full } -// nodeStatusDeltaToProto converts a JSON delta map to the protobuf NodeStatusDelta message. -// Each key in the map represents an updated top-level field; the values are the -// JSON-encoded field contents (already produced by computeStatusDelta). -func nodeStatusDeltaToProto(delta map[string]json.RawMessage) *statusproto.NodeStatusDelta { - if len(delta) == 0 { - return nil - } - - pb := &statusproto.NodeStatusDelta{ - UpdatedFields: make([]string, 0, len(delta)), - } - - for key, raw := range delta { - pb.UpdatedFields = append(pb.UpdatedFields, key) - - switch key { - case "nodeInfo": - var ni NodeInfo - if json.Unmarshal(raw, &ni) == nil { - pb.NodeInfo = nodeInfoToProto(&ni) - } - case "peers": - var peers []statusv1alpha1.PeerStatus - if json.Unmarshal(raw, &peers) == nil { - pb.Peers = peersToProto(peers) - } - case "routingTable": - var rt RoutingTableInfo - if json.Unmarshal(raw, &rt) == nil { - pb.RoutingTable = routingTableToProto(&rt) - } - case "healthCheck": - var hc HealthCheckStatus - if json.Unmarshal(raw, &hc) == nil { - pb.HealthCheck = healthCheckStatusToProto(&hc) - } - case "nodeErrors": - var errs []NodeError - if json.Unmarshal(raw, &errs) == nil { - pb.NodeErrors = nodeErrorsToProto(errs) - } - case "bpfEntries": - var entries []BpfEntry - if json.Unmarshal(raw, &entries) == nil { - pb.BpfEntries = bpfEntriesToProto(entries) - } - } - } - - return pb -} - // nodeInfoToProto converts a Go NodeInfo to its protobuf equivalent. func nodeInfoToProto(ni *NodeInfo) *statusproto.NodeInfo { if ni == nil { @@ -316,7 +272,7 @@ func healthCheckStatusToProto(hc *HealthCheckStatus) *statusproto.HealthCheckSta Healthy: hc.Healthy, Summary: hc.Summary, PeerCount: int32(hc.PeerCount), - CheckedAtUnixNs: hc.CheckedAt.UnixNano(), + CheckedAtUnixNs: statusUnixNano(hc.CheckedAt), } } @@ -364,7 +320,7 @@ func nodePodInfoToProto(npi *statusv1alpha1.NodePodInfo) *statusproto.NodePodInf return &statusproto.NodePodInfo{ PodName: npi.PodName, - StartTimeUnixNs: npi.StartTime.UnixNano(), + StartTimeUnixNs: statusUnixNano(npi.StartTime), Restarts: npi.Restarts, } } diff --git a/cmd/unbounded-net-node/status_server.go b/cmd/unbounded-net-node/status_server.go index de8671bf8..08a453828 100644 --- a/cmd/unbounded-net-node/status_server.go +++ b/cmd/unbounded-net-node/status_server.go @@ -845,66 +845,15 @@ func resolveFallbackStatusWebSocketURL(cfg *config, directWSURL string) string { return "" } -func computeStatusDelta(prev, curr *NodeStatusResponse) (map[string]json.RawMessage, error) { - if prev == nil { - return nil, nil - } - - prevRaw, err := json.Marshal(prev) - if err != nil { - return nil, err - } - - currRaw, err := json.Marshal(curr) - if err != nil { - return nil, err - } - - var prevMap map[string]json.RawMessage - if err := json.Unmarshal(prevRaw, &prevMap); err != nil { - return nil, err - } - - var currMap map[string]json.RawMessage - if err := json.Unmarshal(currRaw, &currMap); err != nil { - return nil, err - } - - delta := make(map[string]json.RawMessage) - if nodeInfo, ok := currMap["nodeInfo"]; ok { - delta["nodeInfo"] = nodeInfo - } - - for key, value := range currMap { - if key == "nodeInfo" { - continue - } - - prevValue, exists := prevMap[key] - if !exists || !bytes.Equal(prevValue, value) { - delta[key] = value - } - } - - if _, previouslyPresent := prevMap["nodeErrors"]; previouslyPresent { - if _, currentlyPresent := currMap["nodeErrors"]; !currentlyPresent { - delta["nodeErrors"] = json.RawMessage("[]") - } - } - // NOTE: don't emit "null" for keys present in prev but missing in curr. - // Go json.Marshal omits nil slices/pointers with omitempty, so a missing - // key in currMap usually means the field is nil/empty, not intentionally - // cleared. Emitting null would wipe out the controller's cached data. - - if len(delta) == 1 { - return nil, nil - } - - return delta, nil -} - func stripPeerStats(status *NodeStatusResponse) *NodeStatusResponse { clone := *status + clone.Timestamp = time.Time{} + + if status.HealthCheck != nil { + health := *status.HealthCheck + health.CheckedAt = time.Time{} + clone.HealthCheck = &health + } clone.Peers = make([]WireGuardPeerStatus, 0, len(status.Peers)) for _, peer := range status.Peers { @@ -912,6 +861,13 @@ func stripPeerStats(status *NodeStatusResponse) *NodeStatusResponse { peerCopy.Tunnel.RxBytes = 0 peerCopy.Tunnel.TxBytes = 0 peerCopy.Tunnel.LastHandshake = time.Time{} + + if peer.HealthCheck != nil { + health := *peer.HealthCheck + health.Uptime, health.RTT = "", "" + peerCopy.HealthCheck = &health + } + clone.Peers = append(clone.Peers, peerCopy) } @@ -1416,13 +1372,19 @@ func runStatusWebSocketPusher( var ( lastSentStatus *NodeStatusResponse lastCriticalSnapshot *NodeStatusResponse - revision atomic.Uint64 - resyncRequired atomic.Bool + acks statusAckState lastAckTimeNs atomic.Int64 + lastWriteTime time.Time ) lastAckTimeNs.Store(time.Now().UnixNano()) + if initialStatus != nil { + acks.pending.Store(true) + + lastWriteTime = time.Now() + } + readCtx, readCancel := context.WithCancel(connCtx) go func() { @@ -1435,47 +1397,8 @@ func runStatusWebSocketPusher( return } - lastAckTimeNs.Store(time.Now().UnixNano()) - - var ack statusproto.NodeStatusAck - if err := proto.Unmarshal(data, &ack); err != nil { - klog.V(4).Infof("Status websocket: failed to unmarshal protobuf ack, trying JSON fallback: %v", err) - // Fallback: try JSON for backward compatibility during rollout. - var envelope struct { - Type string `json:"type"` - Data nodeStatusPushAck `json:"data"` - } - if jsonErr := json.Unmarshal(data, &envelope); jsonErr != nil { - continue - } - - switch envelope.Type { - case "node_status_ack": - if envelope.Data.Revision > 0 { - revision.Store(envelope.Data.Revision) - } - case "node_status_resync": - if envelope.Data.Revision > 0 { - revision.Store(envelope.Data.Revision) - } - - resyncRequired.Store(true) - } - - continue - } - - switch ack.Status { - case "ok": - if ack.Revision > 0 { - revision.Store(ack.Revision) - } - case "resync_required": - if ack.Revision > 0 { - revision.Store(ack.Revision) - } - - resyncRequired.Store(true) + if acks.accept(data) { + lastAckTimeNs.Store(time.Now().UnixNano()) } } }() @@ -1486,6 +1409,11 @@ func runStatusWebSocketPusher( return err } + acks.resync.Store(false) + acks.pending.Store(true) + + lastWriteTime = time.Now() + if err := conn.Write(connCtx, websocket.MessageBinary, payload); err != nil { return err } @@ -1495,8 +1423,6 @@ func runStatusWebSocketPusher( lastSentStatus = status lastCriticalSnapshot = stripPeerStats(status) - resyncRequired.Store(false) - return nil } @@ -1611,7 +1537,11 @@ func runStatusWebSocketPusher( case <-readCtx.Done(): break loop case <-criticalTicker.C: - if resyncRequired.Load() || lastSentStatus == nil { + if acks.pending.Load() { + continue + } + + if acks.resync.Load() || lastSentStatus == nil { if err := sendFull(); err != nil { klog.V(2).Infof("Status websocket: resync full send failed: %v", err) break loop @@ -1627,16 +1557,18 @@ func runStatusWebSocketPusher( continue } - delta, err := computeStatusDelta(lastSentStatus, current) - if err != nil || len(delta) == 0 { + published := criticalStatus(lastSentStatus, current) + + delta := typedStatusDelta(lastSentStatus, published, acks.compact.Load(), false) + if delta == nil { continue } message := &statusproto.NodeStatusMessage{ Type: "node_status_delta", NodeName: current.NodeInfo.Name, - BaseRevision: revision.Load(), - Delta: nodeStatusDeltaToProto(delta), + BaseRevision: acks.revision.Load(), + Delta: delta, } payload, err := proto.Marshal(message) @@ -1644,20 +1576,36 @@ func runStatusWebSocketPusher( continue } + acks.pending.Store(true) + + lastWriteTime = time.Now() + if err := conn.Write(connCtx, websocket.MessageBinary, payload); err != nil { klog.V(2).Infof("Status websocket: critical delta write failed: %v", err) break loop } - lastSentStatus = current + lastSentStatus = published lastCriticalSnapshot = criticalSnapshot case <-statsTicker.C: + if acks.pending.Load() { + continue + } + + if acks.resync.Load() || lastSentStatus == nil { + if err := sendFull(); err != nil { + klog.V(2).Infof("Status websocket: stats resync failed: %v", err) + break loop + } + + continue + } // Always send a delta on the stats interval even if stats // appear unchanged, so the controller sees fresh timestamps. current := healthState.getStatusSnapshot() - delta, err := computeStatusDelta(lastSentStatus, current) - if err != nil || len(delta) == 0 { + delta := typedStatusDelta(lastSentStatus, current, acks.compact.Load(), true) + if delta == nil { // No computable delta -- fall back to full send. if err := sendFull(); err != nil { klog.V(2).Infof("Status websocket: stats fallback full send failed: %v", err) @@ -1670,8 +1618,8 @@ func runStatusWebSocketPusher( wsMsg := &statusproto.NodeStatusMessage{ Type: "node_status_delta", NodeName: current.NodeInfo.Name, - BaseRevision: revision.Load(), - Delta: nodeStatusDeltaToProto(delta), + BaseRevision: acks.revision.Load(), + Delta: delta, } payload, err := proto.Marshal(wsMsg) @@ -1679,6 +1627,10 @@ func runStatusWebSocketPusher( continue } + acks.pending.Store(true) + + lastWriteTime = time.Now() + if err := conn.Write(connCtx, websocket.MessageBinary, payload); err != nil { klog.V(2).Infof("Status websocket: stats delta write failed: %v", err) break loop @@ -1687,6 +1639,9 @@ func runStatusWebSocketPusher( lastSentStatus = current lastCriticalSnapshot = stripPeerStats(current) case <-fullSyncTicker.C: + if acks.pending.Load() { + continue + } // Forced full status sync to ensure the controller has // complete status regardless of delta accumulation. if err := sendFull(); err != nil { @@ -1734,6 +1689,11 @@ func runStatusWebSocketPusher( directRecoveryTimer.Reset(directRecoveryBackoff) } case <-fallbackCloseTicker.C: + if acks.pending.Load() && time.Since(lastWriteTime) > 30*time.Second { + klog.V(2).Info("Status websocket: status acknowledgment timed out") + break loop + } + if wsURL == fallbackWSURL && closeFallbackWS != nil && closeFallbackWS.Load() { closeFallbackWS.Store(false) @@ -2090,21 +2050,22 @@ func startStatusPusher( protoMsg := &statusproto.NodeStatusMessage{ Type: "node_status_full", NodeName: nodeStatus.NodeInfo.Name, - Status: nodeStatusToProto(nodeStatus), } if cfg.StatusPushDelta && !currentForceFull { - delta, deltaErr := computeStatusDelta(previousStatus, nodeStatus) - if deltaErr != nil { - klog.V(3).Infof("Status push: failed to compute delta: %v", deltaErr) - } else if len(delta) > 0 { + delta := typedStatusDelta(previousStatus, nodeStatus, false, true) + if delta != nil { mode = "delta" protoMsg.Type = "node_status_delta" protoMsg.BaseRevision = currentRevision protoMsg.Status = nil - protoMsg.Delta = nodeStatusDeltaToProto(delta) + protoMsg.Delta = delta } } + if protoMsg.Delta == nil { + protoMsg.Status = nodeStatusToProto(nodeStatus) + } + marshalStart := time.Now() data, err := proto.Marshal(protoMsg) @@ -2907,6 +2868,8 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } s.state.mu.Unlock() + sortStatusPeers(status.Peers) + // Collect routing table from kernel via netlink status.RoutingTable = s.collectRoutingTableFromKernel() diff --git a/cmd/unbounded-net-node/status_websocket_auth_test.go b/cmd/unbounded-net-node/status_websocket_auth_test.go index 0c42eb346..3e6033258 100644 --- a/cmd/unbounded-net-node/status_websocket_auth_test.go +++ b/cmd/unbounded-net-node/status_websocket_auth_test.go @@ -102,6 +102,16 @@ func TestWebSocketEstablishedFailureFallsBack(t *testing.T) { initialStatus <- data + ack, err := proto.Marshal(&statusproto.NodeStatusAck{Status: "ok", Revision: 1}) + if err != nil { + t.Error(err) + return + } + + if err := conn.Write(ctx, websocket.MessageBinary, ack); err != nil { + return + } + if failure == "read" { select { case <-dropDirect: @@ -249,6 +259,7 @@ func TestWebSocketRecoveryPromotesInitializedConnection(t *testing.T) { directFrames, fallbackFrames atomic.Int32 connected, failWrites, fallbackClosed atomic.Bool closeFallback atomic.Bool + initialAckSent atomic.Bool mode atomic.Int32 ) @@ -290,6 +301,10 @@ func TestWebSocketRecoveryPromotesInitializedConnection(t *testing.T) { var revision int32 if direct { revision = directFrames.Add(1) + if revision > 1 && !initialAckSent.Load() { + t.Error("promoted connection published before its initial ACK") + return + } } else { revision = fallbackFrames.Add(1) } @@ -300,6 +315,24 @@ func TestWebSocketRecoveryPromotesInitializedConnection(t *testing.T) { return } + if direct && revision == 1 { + go func() { + select { + case <-r.Context().Done(): + return + case <-time.After(150 * time.Millisecond): + } + + initialAckSent.Store(true) + + if err := conn.Write(r.Context(), websocket.MessageBinary, ack); err != nil && r.Context().Err() == nil { + t.Errorf("initial recovery ACK failed: %v", err) + } + }() + + continue + } + if err := conn.Write(r.Context(), websocket.MessageBinary, ack); err != nil { return } diff --git a/cmd/unbounded-net-node/tunnel_config.go b/cmd/unbounded-net-node/tunnel_config.go index c390ec924..8e0551479 100644 --- a/cmd/unbounded-net-node/tunnel_config.go +++ b/cmd/unbounded-net-node/tunnel_config.go @@ -16,6 +16,7 @@ import ( unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" ebpfpkg "github.com/Azure/unbounded/internal/net/ebpf" + "github.com/Azure/unbounded/internal/net/healthcheck" unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" ) @@ -204,6 +205,7 @@ func configureTunnelPeers( assignmentPoolTunnelMTUs map[string]int, poolTunnelMTUs map[string]int, state *wireGuardState, + profiles map[string]healthcheck.HealthCheckSettings, ) ([]unboundednetnetlink.DesiredRoute, map[string]bool, error) { // Do NOT early-return when both peer lists are empty. Even when // there are no tunnel-protocol peers (e.g. WG-only gateway @@ -518,12 +520,19 @@ func configureTunnelPeers( // TC egress BPF intercepts and redirects to geneve0. var routes []unboundednetnetlink.DesiredRoute - hcPeers := registerPeersWithHealthCheck(meshPeers, gatewayPeers, mySiteName, false, + state.mu.Lock() + isGatewayNode := state.isGatewayNode + state.mu.Unlock() + + hcPeers, healthErr := registerPeersWithHealthCheck(meshPeers, gatewayPeers, mySiteName, isGatewayNode, siteHealthCheckProfileNames, peeringSiteHealthCheckProfileNames, assignmentSiteHealthCheckProfileNames, assignmentPoolHealthCheckProfileNames, - poolHealthCheckProfileNames, state, + poolHealthCheckProfileNames, profiles, state, func(gw gatewayPeerInfo) string { return peerIfaceName(cfg, gw) }, true) + if healthErr != nil { + return routes, hcPeers, fmt.Errorf("register shared-tunnel health checks: %w", healthErr) + } klog.V(2).Infof("eBPF tunnel: configured %d mesh + %d gateway peers, %d BPF entries, %d supernet routes on %s", len(meshPeers), len(gatewayPeers), len(bpfEntries), len(routes), ifName) diff --git a/cmd/unbounded-net-node/wireguard_config.go b/cmd/unbounded-net-node/wireguard_config.go index bf9dbcde6..4c3e486b5 100644 --- a/cmd/unbounded-net-node/wireguard_config.go +++ b/cmd/unbounded-net-node/wireguard_config.go @@ -12,6 +12,7 @@ import ( "k8s.io/klog/v2" unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/internal/net/healthcheck" unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" "github.com/Azure/unbounded/internal/net/routeplan" ) @@ -20,7 +21,7 @@ import ( // - wg: Main mesh interface for all mesh peers (intra-site, remote, same-pool gateways) // - wg: Separate interfaces for each gateway peer (for ECMP routing) // Endpoint and routing decisions are driven by peer.SiteName and peeredSites membership. -func configureWireGuard(ctx context.Context, cfg *config, privKey string, peers []meshPeerInfo, gatewayPeers []gatewayPeerInfo, mySiteName string, peeredSites, networkPeeredSites, gatewayNodePubKeys map[string]bool, siteHealthCheckProfileNames, peeringSiteHealthCheckProfileNames, assignmentSiteHealthCheckProfileNames, assignmentPoolHealthCheckProfileNames, poolHealthCheckProfileNames map[string]string, siteTunnelMTUs, peeringSiteTunnelMTUs, assignmentSiteTunnelMTUs, assignmentPoolTunnelMTUs, poolTunnelMTUs map[string]int, additionalRoutes []unboundednetnetlink.DesiredRoute, geneveHCPeers map[string]bool, state *wireGuardState) error { +func configureWireGuard(ctx context.Context, cfg *config, privKey string, peers []meshPeerInfo, gatewayPeers []gatewayPeerInfo, mySiteName string, peeredSites, networkPeeredSites, gatewayNodePubKeys map[string]bool, siteHealthCheckProfileNames, peeringSiteHealthCheckProfileNames, assignmentSiteHealthCheckProfileNames, assignmentPoolHealthCheckProfileNames, poolHealthCheckProfileNames map[string]string, siteTunnelMTUs, peeringSiteTunnelMTUs, assignmentSiteTunnelMTUs, assignmentPoolTunnelMTUs, poolTunnelMTUs map[string]int, additionalRoutes []unboundednetnetlink.DesiredRoute, geneveHCPeers map[string]bool, state *wireGuardState, profiles map[string]healthcheck.HealthCheckSettings) error { nodePodCIDRs := state.nodePodCIDRs isGatewayNode := state.isGatewayNode myGatewayPort := state.myGatewayPort @@ -619,12 +620,15 @@ func configureWireGuard(ctx context.Context, cfg *config, privKey string, peers len(peers), len(gatewayPeers), len(allDesiredRoutes)) // === Register healthcheck peers via shared HC registration === - wgHCPeers := registerPeersWithHealthCheck(peers, gatewayPeers, mySiteName, isGatewayNode, + wgHCPeers, err := registerPeersWithHealthCheck(peers, gatewayPeers, mySiteName, isGatewayNode, siteHealthCheckProfileNames, peeringSiteHealthCheckProfileNames, assignmentSiteHealthCheckProfileNames, assignmentPoolHealthCheckProfileNames, - poolHealthCheckProfileNames, state, + poolHealthCheckProfileNames, profiles, state, func(gw gatewayPeerInfo) string { return peerIfaceNameWireGuard(cfg, gw) }, false) + if err != nil { + return fmt.Errorf("register WireGuard health checks: %w", err) + } // Remove peers that are no longer desired (preserve GENEVE HC peers) if state.healthCheckManager != nil { diff --git a/deploy/machina/crd/unbounded-cloud.io_sites.yaml b/deploy/machina/crd/unbounded-cloud.io_sites.yaml index dcb2d2897..4161a7b45 100644 --- a/deploy/machina/crd/unbounded-cloud.io_sites.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_sites.yaml @@ -174,6 +174,7 @@ spec: description: |- ReceiveInterval is the minimum interval between received health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true transmitInterval: anyOf: @@ -182,6 +183,7 @@ spec: description: |- TransmitInterval is the minimum interval between transmitted health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true type: object localCidrs: diff --git a/deploy/net/controller/02-rbac.yaml.tmpl b/deploy/net/controller/02-rbac.yaml.tmpl index e1210be04..d479483aa 100644 --- a/deploy/net/controller/02-rbac.yaml.tmpl +++ b/deploy/net/controller/02-rbac.yaml.tmpl @@ -186,10 +186,14 @@ rules: resources: ["services"] resourceNames: ["unbounded-net-controller"] verbs: ["get"] - # Pods: read pods for node agent status in dashboard (controller namespace only) + # Pods: dashboard status and local OIDC bound-object validation. - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"] + # ServiceAccounts: local OIDC bound-object validation (controller namespace only). + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/net/controller/rbac_test.go b/deploy/net/controller/rbac_test.go new file mode 100644 index 000000000..45f251af0 --- /dev/null +++ b/deploy/net/controller/rbac_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "bytes" + "io" + "os" + "path/filepath" + "slices" + "testing" + + rbacv1 "k8s.io/api/rbac/v1" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + + "github.com/Azure/unbounded/hack/cmd/render-manifests/render" +) + +func TestBoundObjectCacheRBAC(t *testing.T) { + for _, namespace := range []string{"unbounded-system", "custom-system"} { + t.Run(namespace, func(t *testing.T) { + output := t.TempDir() + if err := render.Render(".", output, map[string]string{"Namespace": namespace}); err != nil { + t.Fatal(err) + } + + raw, err := os.ReadFile(filepath.Join(output, "02-rbac.yaml")) + if err != nil { + t.Fatal(err) + } + + decoder := utilyaml.NewYAMLOrJSONDecoder(bytes.NewReader(raw), 4096) + found := false + + for { + var role rbacv1.Role + if err := decoder.Decode(&role); err != nil { + if err == io.EOF { + break + } + + t.Fatal(err) + } + + for _, rule := range role.Rules { + if slices.Contains(rule.Resources, "serviceaccounts") || slices.Contains(rule.Resources, "*") { + if role.Kind != "Role" || role.Namespace != namespace || + !slices.Equal(rule.APIGroups, []string{""}) || + !slices.Equal(rule.Verbs, []string{"list", "watch"}) || + len(rule.ResourceNames) != 0 { + t.Fatalf("unexpected service account permission: %+v %+v", role.ObjectMeta, rule) + } + + found = true + } + } + + if role.Kind == "Role" && role.Namespace == namespace { + for _, verb := range []string{"get", "list", "watch"} { + if !grantsPodRead(role.Rules, verb) { + t.Fatalf("missing Pod %s permission in controller namespace", verb) + } + } + } + } + + if !found { + t.Fatal("missing namespaced service account list/watch permissions") + } + }) + } +} + +func grantsPodRead(rules []rbacv1.PolicyRule, verb string) bool { + for _, rule := range rules { + if slices.Contains(rule.APIGroups, "") && slices.Contains(rule.Resources, "pods") && slices.Contains(rule.Verbs, verb) { + return true + } + } + + return false +} diff --git a/deploy/net/crd/net.unbounded-cloud.io_gatewaypoolpeerings.yaml b/deploy/net/crd/net.unbounded-cloud.io_gatewaypoolpeerings.yaml index cdfcdf1dd..20c215143 100644 --- a/deploy/net/crd/net.unbounded-cloud.io_gatewaypoolpeerings.yaml +++ b/deploy/net/crd/net.unbounded-cloud.io_gatewaypoolpeerings.yaml @@ -82,6 +82,7 @@ spec: description: |- ReceiveInterval is the minimum interval between received health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true transmitInterval: anyOf: @@ -90,6 +91,7 @@ spec: description: |- TransmitInterval is the minimum interval between transmitted health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true type: object tunnelMTU: diff --git a/deploy/net/crd/net.unbounded-cloud.io_gatewaypools.yaml b/deploy/net/crd/net.unbounded-cloud.io_gatewaypools.yaml index 5bac17b5a..b5eaf4baf 100644 --- a/deploy/net/crd/net.unbounded-cloud.io_gatewaypools.yaml +++ b/deploy/net/crd/net.unbounded-cloud.io_gatewaypools.yaml @@ -76,6 +76,7 @@ spec: description: |- ReceiveInterval is the minimum interval between received health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true transmitInterval: anyOf: @@ -84,6 +85,7 @@ spec: description: |- TransmitInterval is the minimum interval between transmitted health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true type: object nodeSelector: diff --git a/deploy/net/crd/net.unbounded-cloud.io_sitegatewaypoolassignments.yaml b/deploy/net/crd/net.unbounded-cloud.io_sitegatewaypoolassignments.yaml index e84653908..240b1d1ae 100644 --- a/deploy/net/crd/net.unbounded-cloud.io_sitegatewaypoolassignments.yaml +++ b/deploy/net/crd/net.unbounded-cloud.io_sitegatewaypoolassignments.yaml @@ -82,6 +82,7 @@ spec: description: |- ReceiveInterval is the minimum interval between received health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true transmitInterval: anyOf: @@ -90,6 +91,7 @@ spec: description: |- TransmitInterval is the minimum interval between transmitted health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true type: object sites: diff --git a/deploy/net/crd/net.unbounded-cloud.io_sitepeerings.yaml b/deploy/net/crd/net.unbounded-cloud.io_sitepeerings.yaml index 10796e16d..272bbcbff 100644 --- a/deploy/net/crd/net.unbounded-cloud.io_sitepeerings.yaml +++ b/deploy/net/crd/net.unbounded-cloud.io_sitepeerings.yaml @@ -84,6 +84,7 @@ spec: description: |- ReceiveInterval is the minimum interval between received health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true transmitInterval: anyOf: @@ -92,6 +93,7 @@ spec: description: |- TransmitInterval is the minimum interval between transmitted health check packets. Accepts either a duration string (e.g. "300ms") or an integer interpreted as milliseconds. + Defaults to 15s when omitted from the selected health check scope. x-kubernetes-int-or-string: true type: object meshNodes: diff --git a/docs/net/architecture.md b/docs/net/architecture.md index e8a9a33bb..6a90368ee 100644 --- a/docs/net/architecture.md +++ b/docs/net/architecture.md @@ -474,7 +474,7 @@ sequenceDiagram participant Routes as Netlink Route Table Agent->>HC: Start health check sessions for peers - loop Every transmitInterval (default 1s) + loop Every transmitInterval (default 15s) HC->>Peer: UDP probe (over WireGuard tunnel) alt Healthy Peer-->>HC: UDP response @@ -488,8 +488,9 @@ sequenceDiagram ``` **Key Design Decisions:** -- Probes are sent and received over WireGuard tunnels at configurable intervals (default 1s) +- Probes are sent and received over supported tunnel types at configurable intervals (default 15s) - Failure detection uses `detectMultiplier * max(transmitInterval, receiveInterval)` to determine when a peer is down +- The 15s defaults reduce probe traffic versus the previous 1s defaults, trading a nominal 3s detection timeout for 45s; explicit intervals retain their requested cadence - On failure, route metrics are increased to deprioritize unhealthy paths rather than removing routes entirely - On recovery, route metrics are restored to their base values diff --git a/docs/net/configuration.md b/docs/net/configuration.md index dae351fb4..1cc49ffe2 100644 --- a/docs/net/configuration.md +++ b/docs/net/configuration.md @@ -356,24 +356,47 @@ The **security-wins rule** ensures that if any scope in the hierarchy explicitly ### Health Check (UDP Probe over Tunnel) -The health check protocol provides sub-second failure detection for overlay peers using a custom UDP probe protocol (similar to SBFD) running over all tunnel types. Sessions are automatically created for all routes with nexthops (supernet/RoutedCidrs routes, podCIDR routes, and internal IP routes). Bootstrap routes (/32 and /128 host routes for peer nexthops) do not use health checks to avoid a chicken-and-egg dependency. +The health check protocol monitors overlay peers using a custom UDP probe protocol (similar to SBFD) running over all tunnel types. +The node registers per-peer sessions using the peer's overlay health IP when the resolved health-check profile is enabled. +Transmit and receive intervals default to **15s**, with detect multiplier **3** and maximum flap backoff **120s**. +The detection timeout is `detectMultiplier * max(transmitInterval, receiveInterval)`, or **45s** with defaults. +Compared with the previous 1s default, this sends one-fifteenth as many probes per peer, trading a nominal 3s detection timeout for 45s to reduce steady-state traffic and CPU. +Existing explicit intervals are unchanged; set both intervals to `1s` to retain the previous cadence and nominal timeout. +Timeouts are checked every half-timeout (at least 100ms), so an unresponsive established session can take up to another check interval to be marked down. +Shorter explicit intervals can provide faster detection at the cost of additional probe traffic and CPU. **Health Check Behavior:** -- Health check sessions are managed automatically for all routed traffic +- Health check sessions are managed automatically for peers with enabled profiles - Session status is displayed on the `/status` endpoint - Health checks replace the legacy gateway health checking mechanism -- route metric adjustment on health check failure provides faster and more reliable failover -- No additional configuration flags are needed -- health checks are always active for routed traffic +- Profiles are enabled by default; `healthCheckSettings.enabled: false` disables the selected association without falling back to a less-specific enabled profile **Health Check Settings Precedence (CRDs):** - `Site.spec.healthCheckSettings` applies to node-to-node routes within the same site. - `SitePeering.spec.healthCheckSettings` applies to node-to-node routes between sites in that peering. -- `GatewayPool.spec.healthCheckSettings` applies to routes from nodes to peers in that gateway pool. +- `GatewayPool.spec.healthCheckSettings` governs same-pool gateway peers. - `GatewayPoolPeering.spec.healthCheckSettings` applies to routes between gateway pools in that peering. -For gateway-pool routes, precedence is: -1. `SiteGatewayPoolAssignment.spec.healthCheckSettings` -2. `GatewayPool.spec.healthCheckSettings` -3. `Site.spec.healthCheckSettings` +For mesh peers, an explicit gateway-pool or pool-peering profile wins, followed by a `SiteGatewayPoolAssignment`, a `SitePeering`, and then the peer's `Site`. +For node-to-gateway peers, the node's site/pool assignment governs; gateway nodes use their explicit peer profile or the peer's pool profile, with a remote-site/pool assignment as fallback. +Shared-tunnel gateway peers may fall back to the local site's profile when no governing association exists; WireGuard gateway peers do not use that site fallback. +An explicitly disabled governing profile blocks every fallback. + +The selected scope is merged with fresh defaults, not the last applied profile or values from lower-priority scopes. +For example, specifying only `transmitInterval: 60s` uses a 15s receive interval and detect multiplier 3. +Both duration strings and integer milliseconds are accepted; explicit settings and examples retain their requested intervals. +The node's global maximum flap-backoff setting overrides the selected profile's backoff. + +Reconciliation passes the freshly resolved settings to every supported transport before committing its cached profile maps. +Settings-only changes for an existing peer and overlay IP update the live session in place, preserving health state, uptime, RTT, packet counters, and flap history. +Probe and detection timers wake promptly rather than waiting for the previous interval to elapse. +Each directed `(local node, peer)` identity has a deterministic transmit phase strictly greater than zero and no larger than the configured transmit interval. +Initial probes and transmit-interval changes use this phase to spread fleet-wide starts and updates; subsequent probes use the exact configured interval, without per-probe jitter or extra retries. +Receive-only, backoff-only, and unchanged settings do not reset the transmit phase. +When an established healthy session's detection timeout is shortened, one bounded transition window lets the first probe under the new cadence receive a reply before applying the shorter timeout to old-cadence data. +The window is at most one new transmit interval plus one new nominal detection timeout; a fresh reply immediately restores ordinary detection. +The nominal timeout remains `detectMultiplier * max(transmitInterval, receiveInterval)`, and the transition does not fabricate a reply or reset counters. +Changing a peer's overlay IP still replaces and cancels the old session; newly created sessions begin down until sufficient replies arrive. If multiple peerings define conflicting health check settings for the same target site or gateway pool, the controller processes peerings in deterministic name order and keeps the first profile, @@ -383,7 +406,7 @@ logging both the kept and ignored peering/profile details. The node agent uses configurable API server mode for websocket and push behavior: -1. WebSocket transport (`/status/nodews` and aggregated API path) with JSON full+delta messages and compression. +1. WebSocket transport (`/status/nodews` and aggregated API path) with protobuf full+delta messages and compression. The controller also accepts JSON messages for compatibility. 2. Periodic HTTP push (`/status/push` and aggregated API path) when websocket is unavailable or configured for periodic reconciliation. 3. Controller pull fallback when push data is stale/unavailable. @@ -392,7 +415,35 @@ Direct controller routes, including `/token/node`, `/status/nodews`, and `/statu When TokenReview is selected at startup, each aggregated HTTP status upload and new WebSocket handshake requires a TokenReview API call. Positive authentication results are not cached, preserving API-server bound-object revocation checks. Large deployments should use local OIDC with a suitable explicit audience when automatic discovery is ambiguous, prefer direct HMAC transport, and size the aggregated HTTP push interval for outage load. `node.criticalDeltaEvery` (default `1s`) and `node.statsDeltaEvery` (default `15s`) are maximum publish frequencies. -The node only sends a delta when fields changed, and changed fields are queued up to each interval to batch related updates. +Critical updates are change-only: root collection timestamps, aggregate health-check timestamps, tunnel counters/handshakes, and peer health RTT/uptime do not trigger them. +Health state, health enablement, topology, metadata, routes, BPF entries, and errors (including clearing errors) remain critical. +When critical changes are published, existing peers retain their last published measurements; unrelated statistics wait for the statistics interval. +Statistics updates include a timestamp even when measurements are unchanged, so the controller continues to see fresh status. +Periodic full synchronization and full resynchronization after a rejected delta remain in place. + +New nodes use compact protobuf peer-measurement deltas only after the controller positively advertises `peer_measurements` in a successful WebSocket ACK with a nonzero revision. +The capability and revision are reset on every connection; the first message and any resynchronization are full snapshots. +Old controllers receive full peer replacements, and new controllers continue to accept legacy protobuf and JSON full/top-level deltas. +HTTP fallback uses typed top-level deltas without compact measurements and does not rely on a capability learned on a different connection. + +Compact measurements carry packed RX/TX/handshake columns and RTT/uptime string columns instead of repeated static peer metadata and nested health objects. +Each column replaces all measurements for the ordered base peer list, including zero and empty values. +A SHA-256 digest over ordered, length-prefixed `(name, protocol, interface, public key)` identities guards snapshot indices; duplicate or unnamed identities cannot use the compact path. +The controller requires a nonzero matching base revision, matching identities, matching column lengths, and an explicit `peerMeasurements` field mask. +Peer replacement and measurement updates cannot coexist in one message. +Invalid batches request a full resynchronization without partially updating the cache. +Metadata or topology changes, including reordered peers, still replace the peer list. +Fresh node snapshots sort peers by identity so map iteration does not cause artificial topology changes. +Cache updates copy peer values and health measurements while retaining immutable static metadata. +This reduces wire/decode work but does not eliminate the controller's per-node peer cache or full-refresh costs. + +Only one status message is outstanding per WebSocket connection; updates batch while its ACK is pending. +Missing status ACKs reconnect after 30 seconds rather than advancing an unacknowledged base. +The bounded-cardinality controller metric `unbounded_cni_controller_peer_measurement_updates_total{outcome="applied|resync|error"}` counts compact batches. +An increase in `outcome="applied"` confirms use of the negotiated compact path; `resync` denotes a missing/stale base and `error` denotes an invalid batch. +The metric deliberately has no node or peer labels. + +The controller decodes each protobuf WebSocket frame once, reusing the validated message for node-bound authorization and cache updates. Identity checks, full/delta revisions, and resynchronization behavior are unchanged. HTTP push also supports delta mode (`node.statusPushDelta`). If the controller cannot apply a delta (missing/mismatched base state), it returns `429` and the node immediately resends a full state on the next push. @@ -419,7 +470,7 @@ HTTP push also supports delta mode (`node.statusPushDelta`). If the controller c | `--status-ws-keepalive-interval` | duration | `10s` | Interval between node websocket keepalive pings (`0s` disables pings). | | `--status-ws-keepalive-failure-count` | int | `2` | Sequential websocket keepalive ping failures before the node reconnects. | | `--status-critical-interval` | duration | `1s` | Maximum critical-delta publish frequency; changed fields are batched and sent at most once per interval. | -| `--status-stats-interval` | duration | `15s` | Maximum statistics-delta publish frequency; changed fields are batched and sent at most once per interval. | +| `--status-stats-interval` | duration | `15s` | Statistics refresh interval; includes a freshness timestamp even when measurements are unchanged. | | `--shutdown-remove-wireguard-configuration` | bool | `false` | Remove WireGuard interfaces on node-agent shutdown. | | `--shutdown-cleanup-netlink` | bool | `false` | Remove managed netlink routes and policy routing rules on node-agent shutdown. | | `--enable-policy-routing` | bool | `false` | **Deprecated.** Enable connmark/fwmark/ip-rule policy-based routing on gateway WireGuard interfaces. Replaced by per-interface iptables FORWARD ACCEPT rules that are added when tunnel/WG gateway interfaces are created and removed on deletion. Set to `true` only for backward compatibility with pre-1.0.2 deployments. | @@ -435,6 +486,12 @@ HTTP push also supports delta mode (`node.statusPushDelta`). If the controller c - Signing keys refresh on demand when the cache is at least 15 minutes old or a token names an unknown key. Concurrent refreshes share one fetch, with a 30-second cooldown after completion, including failures. Runtime fetches have a 10-second timeout and are not canceled when an individual caller disconnects. A newly rotated key may therefore require a retry after the cooldown. - TokenReview uses the Kubernetes API server audience by default. `controller.oidcAudience` is used only by the OIDC verifier and does not affect TokenReview. - Both local OIDC validation and TokenReview require the expected `unbounded-net-node` service account in the controller's namespace and authorization for the matching node name. +- Local OIDC node authentication supports only Pod-bound tokens with nonempty Pod name/UID and service account UID claims. After signature, issuer, audience, and expiry validation, every authentication checks the Pod and service account in controller-namespace informer caches. Missing objects and same-name replacements with different UIDs are rejected. Both objects use Kubernetes' 60-second deletion grace: a deletion timestamp strictly older than 60 seconds is rejected; the exact boundary is accepted. +- As an additional node-agent consistency check, the Pod's `spec.serviceAccountName` and `spec.nodeName` must match the authenticated claims. Pod phase and readiness are not authentication requirements. The node claim in a Pod-bound Kubernetes token is informational; this local path does not check Node object existence or UID and does not support node-bound-only tokens. +- Pod and service account informers run on every serving controller replica, not just the leader. OIDC requests fail closed until both initial caches sync, when caches stop, or when the request or process is canceled. Health endpoints can still serve during initial sync or RBAC failure. When local OIDC is selected, `/readyz` returns 503 until both caches sync and after either cache stops or the process is canceled; `/healthz` retains its API-connectivity check. Readiness does not wait for leadership or Site CRD caches. Cache validation failures never trigger a per-request fallback to TokenReview or unvalidated OIDC. TokenReview, when selected at startup, keeps its API-server object validation and does not depend on these caches for authentication or readiness. +- Informer validation is eventually consistent, not instantaneous API-server/TokenReview equivalence. A watch interruption after initial sync can leave stale objects usable until the watch reconnects and observes changes; initial sync does not establish ongoing freshness. No per-authentication object API calls are made. Removing a node Pod's `app.kubernetes.io/name=unbounded-net-node` label also removes it from the watched cache and prevents new local authentications once observed. +- Apply the updated controller-namespace Role before rolling out controllers: it requires Pod `get/list/watch` (also used for diagnostics) and service account `list/watch`, without cluster-wide service account grants. Missing initial list permissions keep local OIDC authentication unavailable; existing watch-outage limitations still apply after initial sync. +- Object revocation prevents new exchanges and new aggregated HTTP/WebSocket authentication only after the cache observes it (and deletion grace expires, if applicable). Already issued HMAC tokens retain their existing four-hour default lifetime, and established WebSocket connections are not revoked by these checks. - All nonempty node names in JSON/protobuf envelopes, full status, and deltas must agree. Conflicting identities are rejected before updating the status cache or registering a WebSocket connection. - The node agent uses its mounted service account token and does not request a custom-audience projected token. - Dashboard viewer authorization continues to use SubjectAccessReview. @@ -671,12 +728,17 @@ resources: ### Scaling Considerations -| Cluster Size | Controller Memory | Informer Load | -|--------------|-------------------|---------------| -| < 100 nodes | 64Mi | Low | -| 100-500 nodes | 128Mi | Medium | -| 500-1000 nodes | 256Mi | High | -| > 1000 nodes | 512Mi+ | Very High | +Controller memory depends on diagnostic payload size and update frequency, not +just node count or informer load. The status cache retains each node's peer, +BPF, and routing details. In dense all-to-all topologies, the total peer and +BPF data can grow quadratically with node count. + +Decoding WebSocket frames once and avoiding connectivity-matrix peer copies +reduces transient allocations, but does not eliminate the cached status data. +Size controller resources using measured RSS and Go heap usage under the +expected topology and update rates, with headroom for garbage collection and +in-flight messages. The resource examples above are not sizing recommendations +for large clusters. --- diff --git a/docs/net/operations.md b/docs/net/operations.md index 1e0a2aa13..735d7cfd1 100644 --- a/docs/net/operations.md +++ b/docs/net/operations.md @@ -212,7 +212,6 @@ kubectl unbounded-system controller proxy The dashboard displays: - **Overview**: Cluster health summary with node counts, site counts, and gateway status - **Sites**: All configured sites with node counts and health indicators -- **Connectivity Matrix**: Visual representation of node-to-node connectivity (pingmesh results) - **Nodes**: Detailed list of all nodes with filtering, sorting, and pagination - Tunnel peer status (WireGuard peers or eBPF tunnel endpoints) - Gateway health for each node @@ -225,9 +224,11 @@ The dashboard uses **WebSocket** for real-time updates with delta compression, f - Filtering nodes by name, site, or role (gateway/worker) - Sorting by any column - Auto-sizing pagination based on screen height -- Expandable connectivity matrix with zoom and labels - Dark/light theme toggle +The dashboard does not render a site connectivity graph or connectivity matrix. +Use the Site summaries and filtered node list to inspect individual resources. + ### Health Endpoints #### Controller Health @@ -684,6 +685,37 @@ curl -s http://:9998/metrics | grep status_push curl -s http://:9999/status/json | python3 -m json.tool | head -40 ``` +#### Status Collection Efficiency + +The node agent builds a read-only peer classification index once per status +snapshot and shares it across IPv4 and IPv6 route annotations. It does not rebuild +the entire peer map for every next-hop. The index is rebuilt for the next snapshot, +so peer and topology changes remain visible. + +BPF status collection also reuses successful interface-name and MTU lookups within +one collection, rather than issuing a netlink lookup for every BPF next-hop. +Failed lookups retain the `if` placeholder and zero MTU and are retried on +the next entry. A new collection starts with an empty interface cache, so interface +renames, replacements, and MTU changes are not persistently cached. These changes +do not alter routes, health-check intervals, or status-publishing intervals. + +Controller compact updates retain their revision and identity checks but reuse +successful duplicate-identity validation while the ordered identity digest is +unchanged. A fixed-size per-node memo is invalidated when peers are replaced. +The digest is still recomputed for each update, so changed identities cannot +reuse stale validation. + +The controller also reuses up to four idle gzip response writers. Writers are +closed and detached from the completed response before reuse; failed or +interrupted responses discard their writers. This reduces discovery and error +response allocation without caching response contents or changing compression +negotiation. + +WebSocket input buffers are reused by size class and released after each frame +is processed, not retained for an idle connection's lifetime. The idle pool is +bounded below 64 MiB across connections; active frames still require their own +storage. The existing 2 MiB per-message limit remains enforced. + ### Unused Device Cleanup When the tunnel protocol changes for a peer (e.g., from GENEVE to VXLAN, or from diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 63bb576f4..fcf712c6c 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4,7 +4,6 @@ import * as React from 'react'; import { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import NodeTable from './components/nodes/NodesTable'; -import NetworkCard from './components/dashboard/NetworkCard'; import SitesCard from './components/network/SitesCard'; import StatusJsonModal from './components/status/StatusJsonModal'; import ErrorBoundary from './components/common/ErrorBoundary'; @@ -23,7 +22,6 @@ export default function App() { } = useClusterStatus(); const nodes = status?.nodes || []; const sites = summary?.sites || status?.sites || []; - const peerings = summary?.peerings || status?.peerings || []; const gatewayPools = summary?.gatewayPools || status?.gatewayPools || []; const nodeSummaries = summary?.nodeSummaries || []; const [hiddenSites, setHiddenSites] = useState>(new Set()); @@ -32,8 +30,7 @@ export default function App() { const [selectedNodeDetailTab, setSelectedNodeDetailTab] = useState<'peerings' | 'routes' | 'bpf'>('peerings'); const [pullEnabledOptimistic, setPullEnabledOptimistic] = useState(null); const [selectedNodeTypesFilter, setSelectedNodeTypesFilter] = useState>(new Set(['Gateway', 'Worker'])); - const [networkTab, setNetworkTab] = useState<'siteTopology' | 'matrix'>('siteTopology'); - const [maximizedPanel, setMaximizedPanel] = useState<'nodes' | 'siteTopology' | 'matrix' | null>(null); + const [maximizedPanel, setMaximizedPanel] = useState<'nodes' | null>(null); const [infoOpen, setInfoOpen] = useState(false); const [statusJsonOpen, setStatusJsonOpen] = useState(false); const [errorsDismissed, setErrorsDismissed] = useState(false); @@ -52,12 +49,6 @@ export default function App() { window.localStorage.setItem('theme', theme); }, [theme]); - useEffect(() => { - if (maximizedPanel === 'siteTopology' || maximizedPanel === 'matrix') { - setNetworkTab(maximizedPanel); - } - }, [maximizedPanel]); - useEffect(() => { if (!maximizedPanel) return; const onKeyDown = (event: KeyboardEvent) => { @@ -217,9 +208,7 @@ export default function App() { }, []); const { - activeNetworkTab, activeSelectedNode, - edgeHealthCheckCounts, effectivePullEnabled, gatewayByNode, nodeK8sStatusMap, @@ -227,7 +216,6 @@ export default function App() { nodeTotalCount, peerHealth, poolCounts, - poolToSite, siteCounts, visibleNodeSummaries } = useDashboardData({ @@ -239,8 +227,6 @@ export default function App() { gatewayPoolHiddenNames: hiddenGatewayPools, hiddenSites, selectedNodeTypesFilter, - networkTab, - maximizedPanel, pullEnabledOptimistic, selectedNodeName, nodeDetail @@ -272,18 +258,6 @@ export default function App() { ? 'Polling only' : 'No data'; - const onSelectNetworkTab = (tab: 'siteTopology' | 'matrix') => { - if (maximizedPanel === 'siteTopology' || maximizedPanel === 'matrix') { - setMaximizedPanel(tab); - return; - } - setNetworkTab(tab); - }; - - const onToggleNetworkMaximize = (isMaximized: boolean) => { - setMaximizedPanel(isMaximized ? null : activeNetworkTab); - }; - const renderNodesCard = (isMaximized: boolean) => { const content = (
setMaximizedPanel(null)}>
- {maximizedPanel === 'nodes' ? renderNodesCard(true) : ( - onToggleNetworkMaximize(true)} - /> - )} + {renderNodesCard(true)} - onToggleNetworkMaximize(false)} - />
{loading &&
Loading...
} diff --git a/frontend/src/components/common/topologyIcons.tsx b/frontend/src/components/common/topologyIcons.tsx deleted file mode 100644 index 0c81c0156..000000000 --- a/frontend/src/components/common/topologyIcons.tsx +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import * as React from 'react'; -import { useEffect, useMemo } from 'react'; -import * as THREE from 'three'; - -const gatewayPoolSvgIcon = ` - - - - - - - - - - - - - - -`; - -const workerSiteSvgIcon = ` - - - - - - - - - - - - - - - - - - - - - - - - -`; - -const maskFromSVG = (svg: string): string => { - return svg - .split('#00bbf1').join('#ffffff') - .split('#c5c5c5').join('#ffffff') - .split('#e6f8fe').join('#ffffff') - .split('#ccf1fc').join('#ffffff') - .split('#80ddf8').join('#ffffff') - .split('#919191').join('#ffffff'); -}; - -type TopologyNodeIconTextures = { - gatewayPoolIconTexture: THREE.Texture; - workerSiteIconTexture: THREE.Texture; - gatewayPoolMaskTexture: THREE.Texture; - workerSiteMaskTexture: THREE.Texture; -}; - -type RenderTopologyNodeGlyphArgs = { - glyph: 'router' | 'vm' | 'default'; - size: number; - color: string; - textures: TopologyNodeIconTextures; - workerOffsetScale?: number; -}; - -export function getTopologyNodeGlyph(group?: string): 'router' | 'vm' | 'default' { - if (group === 'pool' || group === 'gateway-node') { - return 'router'; - } - if (group === 'site' || group === 'worker-node') { - return 'vm'; - } - return 'default'; -} - -export function useTopologyNodeIconTextures(): TopologyNodeIconTextures { - const gatewayPoolIconTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(gatewayPoolSvgIcon)}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - const workerSiteIconTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(workerSiteSvgIcon)}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - const gatewayPoolMaskTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(maskFromSVG(gatewayPoolSvgIcon))}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - const workerSiteMaskTexture = useMemo(() => { - const texture = new THREE.TextureLoader().load(`data:image/svg+xml;utf8,${encodeURIComponent(maskFromSVG(workerSiteSvgIcon))}`); - texture.colorSpace = THREE.SRGBColorSpace; - texture.needsUpdate = true; - return texture; - }, []); - - useEffect(() => { - return () => { - gatewayPoolIconTexture.dispose(); - workerSiteIconTexture.dispose(); - gatewayPoolMaskTexture.dispose(); - workerSiteMaskTexture.dispose(); - }; - }, [gatewayPoolIconTexture, workerSiteIconTexture, gatewayPoolMaskTexture, workerSiteMaskTexture]); - - return { - gatewayPoolIconTexture, - workerSiteIconTexture, - gatewayPoolMaskTexture, - workerSiteMaskTexture - }; -} - -export function renderTopologyNodeGlyph({ - glyph, - size, - color, - textures, - workerOffsetScale = -0.16 -}: RenderTopologyNodeGlyphArgs): React.ReactNode { - if (glyph === 'router') { - return ( - - - - - - - - - - - ); - } - - if (glyph === 'vm') { - return ( - - - - - - - - - - - ); - } - - return null; -} diff --git a/frontend/src/components/dashboard/NetworkCard.tsx b/frontend/src/components/dashboard/NetworkCard.tsx deleted file mode 100644 index aa599c22e..000000000 --- a/frontend/src/components/dashboard/NetworkCard.tsx +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import React, { Suspense } from 'react'; -import { CloseXIcon, MagnifyPlusIcon } from '../nodes/shared/index'; -import ConnectivityHeatmap from '../network/ConnectivityHeatmap'; -const Topology = React.lazy(() => import('../network/Topology')); -import { GatewayPoolStatus, NodeStatus, PeeringStatus, SiteMatrix, SiteStatus } from '../../types'; - -type NetworkTab = 'siteTopology' | 'matrix'; - -type NetworkCardProps = { - activeNetworkTab: NetworkTab; - edgeHealthCheckCounts: Map; - gatewayByNode: Map; - gatewayPools: GatewayPoolStatus[]; - hiddenGatewayPools: Set; - hiddenSites: Set; - isMaximized: boolean; - nodeStatuses: NodeStatus[]; - peerings: PeeringStatus[]; - poolCounts: Map; - poolToSite: Map; - siteCounts: Map; - sites: SiteStatus[]; - statusMatrix?: Record; - theme: 'dark' | 'light'; - onSelectTab: (tab: NetworkTab) => void; - onToggleMaximize: () => void; -}; - -function NetworkCard({ - activeNetworkTab, - edgeHealthCheckCounts, - gatewayByNode, - gatewayPools, - hiddenGatewayPools, - hiddenSites, - isMaximized, - nodeStatuses, - peerings, - poolCounts, - poolToSite, - siteCounts, - sites, - statusMatrix, - theme, - onSelectTab, - onToggleMaximize -}: NetworkCardProps) { - return ( -
-
-
- - -
-
- -
-
- {activeNetworkTab === 'siteTopology' && ( -
- Loading topology...
}> - - -
- )} - {activeNetworkTab === 'matrix' && ( -
- -
- )} -
- ); -} - -export default NetworkCard; diff --git a/frontend/src/components/network/ConnectivityHeatmap.tsx b/frontend/src/components/network/ConnectivityHeatmap.tsx deleted file mode 100644 index 0e6d24a4c..000000000 --- a/frontend/src/components/network/ConnectivityHeatmap.tsx +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import { useEffect, useMemo, useRef, useState } from 'react'; -import { GatewayPoolStatus, NodeStatus, SiteMatrix, SiteStatus } from '../../types'; -import { getCniStatus } from '../nodes/shared/index'; - -function getGatewayPoolNodeNames(pool: GatewayPoolStatus): string[] { - const nodeNames = (pool.nodes || []) - .map((node) => node.name) - .filter((name): name is string => Boolean(name)); - if (nodeNames.length > 0) { - return nodeNames; - } - return (pool.gateways || []).filter((name): name is string => Boolean(name)); -} - -function ConnectivityHeatmap({ - matrix, - hiddenSites, - hiddenGatewayPools, - sites, - gatewayPools, - siteCounts, - nodeStatuses -}: { - matrix?: Record; - hiddenSites: Set; - hiddenGatewayPools: Set; - sites: SiteStatus[]; - gatewayPools: GatewayPoolStatus[]; - siteCounts: Map; - nodeStatuses: NodeStatus[]; -}) { - const [activeScope, setActiveScope] = useState('all'); - const matrixRef = useRef(null); - const [matrixSize, setMatrixSize] = useState({ width: 420, height: 420 }); - const [tooltip, setTooltip] = useState<{ - x: number; - y: number; - src: string; - dst: string; - value: number; - } | null>(null); - const siteNames = useMemo(() => { - return Object.keys(matrix || {}) - .filter((name) => !name.startsWith('pool:')) - .filter((name) => !hiddenSites.has(name)) - .sort(); - }, [matrix, hiddenSites]); - const gatewayPoolNames = useMemo(() => { - return (gatewayPools || []) - .map((pool) => pool.name || '') - .filter((name) => Boolean(name) && !hiddenGatewayPools.has(name)) - .sort(); - }, [gatewayPools, hiddenGatewayPools]); - const selectorOptions = useMemo( - () => [ - { key: 'all', label: 'All', kind: 'all' as const }, - ...siteNames.map((site) => ({ key: `site:${site}`, label: site, kind: 'site' as const })), - ...gatewayPoolNames.map((pool) => ({ key: `pool:${pool}`, label: pool, kind: 'pool' as const })) - ], - [siteNames, gatewayPoolNames] - ); - const siteLookup = useMemo(() => { - const map = new Map(); - for (const site of sites) { - if (site.name) { - map.set(site.name, site); - } - } - return map; - }, [sites]); - const nodeStatusByName = useMemo(() => { - const map = new Map(); - for (const node of nodeStatuses) { - const name = node.nodeInfo?.name; - if (name) { - map.set(name, node); - } - } - return map; - }, [nodeStatuses]); - const visibleNodeNames = useMemo(() => { - return nodeStatuses - .filter((node) => { - const nodeName = node.nodeInfo?.name || ''; - if (!nodeName) return false; - const siteName = node.nodeInfo?.siteName; - if (siteName && hiddenSites.has(siteName)) { - return false; - } - const poolName = (gatewayPools || []).find((pool) => { - const poolNameValue = pool.name || ''; - if (!poolNameValue) return false; - return getGatewayPoolNodeNames(pool).includes(nodeName); - })?.name; - if (poolName && hiddenGatewayPools.has(poolName)) { - return false; - } - return true; - }) - .map((node) => node.nodeInfo?.name || '') - .filter((name) => Boolean(name)); - }, [nodeStatuses, hiddenSites, hiddenGatewayPools, gatewayPools]); - - const adjacency = useMemo(() => { - const map = new Map>(); - const allVisible = new Set(visibleNodeNames); - for (const node of nodeStatuses) { - const src = node.nodeInfo?.name; - if (!src || !allVisible.has(src)) continue; - if (!map.has(src)) map.set(src, new Set()); - for (const peer of node.peers || []) { - const dst = peer.name; - if (!dst || !allVisible.has(dst) || dst === src) continue; - map.get(src)?.add(dst); - if (!map.has(dst)) map.set(dst, new Set()); - map.get(dst)?.add(src); - } - } - return map; - }, [nodeStatuses, visibleNodeNames]); - - const healthCheckStatusByPair = useMemo(() => { - const map = new Map(); - for (const siteMatrix of Object.values(matrix || {})) { - const results = siteMatrix?.results || {}; - for (const [src, row] of Object.entries(results)) { - for (const [dst, cell] of Object.entries(row || {})) { - const key = src < dst ? `${src}|${dst}` : `${dst}|${src}`; - if (!map.has(key) && cell) { - map.set(key, typeof cell === 'string' ? cell : (cell as { healthCheckStatus?: string })?.healthCheckStatus || ''); - } - } - } - } - // Also extract health check status from per-node peer data (covers cross-site links) - for (const node of nodeStatuses) { - const src = node.nodeInfo?.name; - if (!src) continue; - for (const peer of node.peers || []) { - const dst = peer.name; - if (!dst || dst === src) continue; - const key = src < dst ? `${src}|${dst}` : `${dst}|${src}`; - if (map.has(key)) continue; // matrix data takes priority - const status = peer.healthCheck?.status; - if (status) { - map.set(key, status); - } - } - } - return map; - }, [matrix, nodeStatuses]); - - const selectedNodeNames = useMemo(() => { - const visibleSet = new Set(visibleNodeNames); - if (activeScope === 'all') { - return [...visibleNodeNames].sort(); - } - - if (activeScope.startsWith('site:')) { - const siteName = activeScope.slice(5); - const fromMatrix = (matrix?.[siteName]?.nodes || []) - .map((name) => String(name)) - .filter((name) => visibleSet.has(name)); - if (fromMatrix.length > 0) { - return Array.from(new Set(fromMatrix)).sort(); - } - return nodeStatuses - .map((node) => node.nodeInfo?.name || '') - .filter((name) => { - if (!name || !visibleSet.has(name)) return false; - const site = nodeStatusByName.get(name)?.nodeInfo?.siteName; - return site === siteName; - }) - .sort(); - } - - if (activeScope.startsWith('pool:')) { - const poolName = activeScope.slice(5); - const poolMatrixNodes = (matrix?.[`pool:${poolName}`]?.nodes || []) - .map((name) => String(name)) - .filter((name) => visibleSet.has(name)); - if (poolMatrixNodes.length > 0) { - return Array.from(new Set(poolMatrixNodes)).sort(); - } - const pool = gatewayPools.find((item) => item.name === poolName); - if (!pool) return []; - const members = new Set(); - for (const nodeName of getGatewayPoolNodeNames(pool)) { - if (!visibleSet.has(nodeName)) continue; - members.add(nodeName); - for (const peerName of adjacency.get(nodeName) || []) { - if (visibleSet.has(peerName)) { - members.add(peerName); - } - } - } - return Array.from(members).sort(); - } - - return []; - }, [activeScope, visibleNodeNames, matrix, nodeStatuses, nodeStatusByName, gatewayPools, adjacency]); - - useEffect(() => { - if (!selectorOptions.some((option) => option.key === activeScope)) { - setActiveScope('all'); - } - }, [activeScope, selectorOptions]); - - useEffect(() => { - if (!matrixRef.current) return; - const updateSize = () => { - const rect = matrixRef.current?.getBoundingClientRect(); - if (!rect) return; - const width = Math.max(0, Math.floor(rect.width)); - const height = Math.max(0, Math.floor(rect.height)); - if (width > 0 && height > 0) { - setMatrixSize({ width, height }); - } - }; - updateSize(); - const observer = new ResizeObserver(updateSize); - observer.observe(matrixRef.current); - return () => observer.disconnect(); - }, []); - - if ((!matrix || siteNames.length === 0) && selectedNodeNames.length === 0) { - return
No connectivity data available.
; - } - - const matrixNodes = selectedNodeNames; - if (matrixNodes.length === 0) { - return
No connectivity data available.
; - } - const cells: Array<{ row: number; col: number; value: number; src: string; dst: string }> = []; - const hasExpectedLink = (src: string, dst: string) => { - if (src === dst) return true; - return adjacency.get(src)?.has(dst) || adjacency.get(dst)?.has(src) || false; - }; - - const selfCellValueFromCniStatus = (node?: NodeStatus) => { - if (!node) { - return 0; - } - if (node.statusSource === 'apiserver-push' || node.statusSource === 'apiserver-ws') { - return 2; - } - const cni = getCniStatus(node); - if (cni.tone === 'success') return 1; - if (cni.tone === 'warning') return 2; - return 0; - }; - - for (let i = 0; i < matrixNodes.length; i++) { - const src = matrixNodes[i]; - for (let j = 0; j < matrixNodes.length; j++) { - const dst = matrixNodes[j]; - const pairKey = src < dst ? `${src}|${dst}` : `${dst}|${src}`; - const hcStatus = (healthCheckStatusByPair.get(pairKey) || '').trim().toLowerCase(); - let value = hcStatus === 'up' ? 1 : hcStatus === 'mixed' ? 2 : hcStatus ? 0 : -1; - if (src === dst) { - const node = nodeStatusByName.get(src); - value = selfCellValueFromCniStatus(node); - } else if (!hasExpectedLink(src, dst)) { - value = -2; - } - cells.push({ row: i, col: j, value, src, dst }); - } - } - - const { width, height } = matrixSize; - - const getColor = (value: number) => { - if (value === 1) return '#4ade80'; - if (value === 2) return '#facc15'; - if (value === 0) return '#f87171'; - if (value === -2) return 'transparent'; - return '#6b7280'; - }; - - const renderMatrix = (width: number, height: number) => { - if (width <= 0 || height <= 0) { - return
; - } - const showAxisLabels = false; - const margin = { top: 12, left: 12, right: 12, bottom: 12 }; - const innerWidth = width - margin.left - margin.right; - const innerHeight = height - margin.top - margin.bottom; - const minInner = Math.min(innerWidth, innerHeight); - const gap = Math.max(1, Math.round(minInner * 0.005)); - const cellSize = Math.floor((minInner - gap * (matrixNodes.length - 1)) / matrixNodes.length); - const gridWidth = cellSize * matrixNodes.length + gap * (matrixNodes.length - 1); - const gridHeight = cellSize * matrixNodes.length + gap * (matrixNodes.length - 1); - const offsetX = Math.max(0, Math.floor((innerWidth - gridWidth) / 2)); - const offsetY = Math.max(0, Math.floor((innerHeight - gridHeight) / 2)); - const scale = 1; - - return ( -
-
- - - {cells.map((cell) => { - const x = cell.col * (cellSize + gap); - const y = cell.row * (cellSize + gap); - const w = cellSize; - const h = cellSize; - return ( - { - setTooltip({ - x: event.clientX + 12, - y: event.clientY + 12, - src: cell.src, - dst: cell.dst, - value: cell.value - }); - }} - onMouseLeave={() => setTooltip(null)} - /> - ); - })} - {showAxisLabels && - matrixNodes.map((label, index) => ( - - {label} - - ))} - {showAxisLabels && - matrixNodes.map((label, index) => ( - - {label} - - ))} - - -
- {tooltip && ( -
-
{tooltip.src} {'->'} {tooltip.dst}
-
- {tooltip.src === tooltip.dst - ? (tooltip.value === 1 - ? 'CNI Healthy' - : tooltip.value === 2 - ? `CNI Warning: ${getCniStatus(nodeStatusByName.get(tooltip.src))?.label || 'Warning'}` - : 'CNI No Data') - : tooltip.value === 1 - ? 'HC Up' - : tooltip.value === 2 - ? 'HC Mixed' - : tooltip.value === 0 - ? 'HC Down' - : tooltip.value === -2 - ? 'No Link Expected' - : 'No Data'} -
-
- )} -
- ); - }; - - return ( -
-
- {selectorOptions.map((option) => { - const isActive = option.key === activeScope; - const isSite = option.kind === 'site'; - const isPool = option.kind === 'pool'; - const siteInfo = isSite ? siteLookup.get(option.label) : undefined; - const counts = isSite ? siteCounts.get(option.label) : undefined; - const online = counts?.online ?? siteInfo?.onlineCount ?? 0; - const total = counts?.total ?? siteInfo?.nodeCount ?? 0; - const status = isSite - ? (online === 0 && total > 0 ? 'danger' : online < total ? 'warning' : 'success') - : isPool - ? 'info' - : 'all'; - return ( - - ); - })} -
- {renderMatrix(width, height)} -
- ); -} - - -export default ConnectivityHeatmap; diff --git a/frontend/src/components/network/Topology.tsx b/frontend/src/components/network/Topology.tsx deleted file mode 100644 index 9107d77af..000000000 --- a/frontend/src/components/network/Topology.tsx +++ /dev/null @@ -1,949 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import * as React from 'react'; -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { createPortal } from 'react-dom'; -import { - getTopologyNodeGlyph, - renderTopologyNodeGlyph, - useTopologyNodeIconTextures -} from '../common/topologyIcons'; -import { GatewayPoolStatus, NodeStatus, PeeringStatus, SiteStatus } from '../../types'; -import { getNodeStatus, ReagraphModule } from '../nodes/shared/index'; - -function buildGraph( - allSiteNames: string[], - allPoolNames: string[], - peerings: PeeringStatus[], - hiddenSites: Set, - hiddenGatewayPools: Set, - existingGatewayPools: Set, - palette: { site: string; siteEmpty: string; siteWarn: string; siteDanger: string; pool: string; poolEmpty: string; poolWarn: string; poolDanger: string; edge: string; edgeDim: string; edgeUp: string; edgeWarn: string; edgeDanger: string }, - siteCounts?: Map, - poolCounts?: Map, - edgeHealthCheckCounts?: Map, - poolToSite?: Map -) { - // Dim a hex color by reducing its opacity (blend toward background) - const dimColor = (hex: string) => { - const r = parseInt(hex.slice(1, 3), 16); - const g = parseInt(hex.slice(3, 5), 16); - const b = parseInt(hex.slice(5, 7), 16); - const mix = (c: number) => Math.round(c * 0.3 + 30 * 0.7); - return `#${mix(r).toString(16).padStart(2, '0')}${mix(g).toString(16).padStart(2, '0')}${mix(b).toString(16).padStart(2, '0')}`; - }; - const nodes: { id: string; label: string; fill: string; activeFill: string; data: { level: number; group: string; peerings: string[] } }[] = []; - const edges: { id: string; source: string; target: string; fill?: string; size?: number; data?: { peerings: string[]; hcUp: number; hcTotal: number } }[] = []; - const nodeSet = new Set(); - const edgeSet = new Set(); - const edgePeerings = new Map>(); - const sitePeerings = new Map>(); - const poolPeerings = new Map>(); - - for (const peering of peerings) { - const peeringName = peering.name || 'peering'; - for (const site of peering.sites || []) { - if (!sitePeerings.has(site)) { - sitePeerings.set(site, new Set()); - } - sitePeerings.get(site)?.add(peeringName); - } - for (const pool of peering.gatewayPools || []) { - if (!existingGatewayPools.has(pool)) { - continue; - } - if (!poolPeerings.has(pool)) { - poolPeerings.set(pool, new Set()); - } - poolPeerings.get(pool)?.add(peeringName); - } - } - - const addNode = (id: string, label: string, group: string, hidden: boolean) => { - if (!nodeSet.has(id)) { - nodeSet.add(id); - let fill = group === 'pool' ? palette.pool : palette.site; - if (group === 'site' && siteCounts) { - const counts = siteCounts.get(label); - if (counts) { - if (counts.total === 0) { - fill = palette.siteEmpty; - } else if ((counts.danger || 0) > 0) { - fill = palette.siteDanger; - } else if ((counts.warning || 0) > 0) { - fill = palette.siteWarn; - } - } - } - if (group === 'pool' && poolCounts) { - const counts = poolCounts.get(label); - if (counts) { - // No matching node entries for this pool in current cluster status: - // render gray (empty) instead of green. - if (counts.total === 0) { - fill = palette.poolEmpty; - } else if ((counts.danger || 0) > 0) { - fill = palette.poolDanger; - } else if ((counts.warning || 0) > 0) { - fill = palette.poolWarn; - } else if (counts.online === 0 && counts.total > 0) { - fill = palette.poolDanger; - } else if (counts.online < counts.total) { - fill = palette.poolWarn; - } - } - } - if (hidden) { - fill = dimColor(fill); - } - const peeringsForNode = group === 'pool' - ? Array.from(poolPeerings.get(label) || []) - : Array.from(sitePeerings.get(label) || []); - nodes.push({ id, label, fill, activeFill: fill, data: { level: group === 'pool' ? 0 : 1, group, peerings: peeringsForNode } }); - } - }; - - const addEdge = (a: string, b: string) => { - if (a === b) return; - const key = a < b ? `${a}|${b}` : `${b}|${a}`; - if (edgeSet.has(key)) return; - edgeSet.add(key); - const isHidden = (nodeId: string) => { - if (nodeId.startsWith('site:')) return hiddenSites.has(nodeId.slice(5)); - if (nodeId.startsWith('pool:')) return hiddenGatewayPools.has(nodeId.slice(5)); - return false; - }; - let edgeFill = palette.edge; - if (isHidden(a) || isHidden(b)) { - edgeFill = palette.edgeDim; - } else if (edgeHealthCheckCounts) { - const counts = edgeHealthCheckCounts.get(key); - // No matching health check entries for this edge (missing map entry or total=0): - // keep default gray edge color. Only color healthy/warn/danger with real data. - if (!counts || counts.total === 0) { - edgeFill = palette.edge; - } else { - if (counts.up === counts.total) { - edgeFill = palette.edgeUp; - } else if (counts.total - counts.up > counts.total / 2) { - edgeFill = palette.edgeDanger; - } else { - edgeFill = palette.edgeWarn; - } - } - } - edges.push({ id: key, source: a, target: b, fill: edgeFill, size: 3.5 }); - }; - - // Seed all known sites and gateway pools so isolated entities still render without edges. - for (const site of allSiteNames) { - addNode(`site:${site}`, site, 'site', hiddenSites.has(site)); - } - for (const pool of allPoolNames) { - addNode(`pool:${pool}`, pool, 'pool', hiddenGatewayPools.has(pool)); - } - - for (const peering of peerings) { - const sites = peering.sites || []; - const pools = (peering.gatewayPools || []).filter((pool) => existingGatewayPools.has(pool)); - const pName = peering.name || 'peering'; - const isPoolPeering = pName.startsWith('poolpeering/'); - - for (const site of sites) { - addNode(`site:${site}`, site, 'site', hiddenSites.has(site)); - } - for (const pool of pools) { - addNode(`pool:${pool}`, pool, 'pool', hiddenGatewayPools.has(pool)); - } - - // Track which peerings each edge belongs to - const trackEdgePeering = (a: string, b: string) => { - const key = a < b ? `${a}|${b}` : `${b}|${a}`; - if (!edgePeerings.has(key)) edgePeerings.set(key, new Set()); - edgePeerings.get(key)?.add(pName); - }; - - if (pools.length > 0) { - for (const site of sites) { - for (const pool of pools) { - trackEdgePeering(`site:${site}`, `pool:${pool}`); - addEdge(`site:${site}`, `pool:${pool}`); - } - } - if (isPoolPeering && pools.length > 1) { - for (let i = 0; i < pools.length; i++) { - for (let j = i + 1; j < pools.length; j++) { - trackEdgePeering(`pool:${pools[i]}`, `pool:${pools[j]}`); - addEdge(`pool:${pools[i]}`, `pool:${pools[j]}`); - } - } - } - } else if (sites.length > 1) { - for (let i = 0; i < sites.length; i++) { - for (let j = i + 1; j < sites.length; j++) { - trackEdgePeering(`site:${sites[i]}`, `site:${sites[j]}`); - addEdge(`site:${sites[i]}`, `site:${sites[j]}`); - } - } - } - } - - // Attach peering names and health check counts to edge data - for (const edge of edges) { - const pNames = edgePeerings.get(edge.id); - const hc = edgeHealthCheckCounts?.get(edge.id); - edge.data = { - peerings: pNames ? Array.from(pNames) : [], - hcUp: hc?.up ?? 0, - hcTotal: hc?.total ?? 0 - }; - } - - // Sort nodes lexicographically by label so layout is deterministic - nodes.sort((a, b) => a.label.localeCompare(b.label)); - - return { nodes, edges }; -} - -function buildNodeGraph( - nodes: NodeStatus[], - gatewayByNode: Map, - palette: { - nodeHealthy: string; - nodeWarn: string; - nodeDanger: string; - edge: string; - edgeUp: string; - edgeWarn: string; - edgeDanger: string; - }, - options?: { - edgeSize?: number; - } -) { - const graphNodes: { - id: string; - label: string; - fill: string; - activeFill: string; - data: { level: number; group: string; peerings: string[]; lines: string[] }; - }[] = []; - const graphEdges: { - id: string; - source: string; - target: string; - fill?: string; - size?: number; - data?: { - peerings: string[]; - hcUp: number; - hcTotal: number; - sourceLabel: string; - targetLabel: string; - }; - }[] = []; - - const nodeByName = new Map(); - for (const node of nodes) { - const nodeName = node.nodeInfo?.name; - if (!nodeName) continue; - nodeByName.set(nodeName, node); - } - - for (const [nodeName, node] of nodeByName.entries()) { - const isGateway = node.nodeInfo?.isGateway || gatewayByNode.has(nodeName); - const status = getNodeStatus(node); - let fill = palette.nodeHealthy; - if (status === 'warning') { - fill = palette.nodeWarn; - } else if (status === 'danger') { - fill = palette.nodeDanger; - } - - const siteName = node.nodeInfo?.siteName || '-'; - const poolName = gatewayByNode.get(nodeName) || '-'; - const lines = isGateway - ? [`Gateway Pool: ${poolName}`, `Site: ${siteName}`] - : [`Site: ${siteName}`]; - - graphNodes.push({ - id: `node:${nodeName}`, - label: nodeName, - fill, - activeFill: fill, - data: { - level: isGateway ? 0 : 1, - group: isGateway ? 'gateway-node' : 'worker-node', - peerings: [], - lines - } - }); - } - - const edgeCounts = new Map(); - for (const [nodeName, node] of nodeByName.entries()) { - for (const peer of node.peers || []) { - const peerName = peer.name; - if (!peerName || !nodeByName.has(peerName) || peerName === nodeName) { - continue; - } - const srcId = `node:${nodeName}`; - const dstId = `node:${peerName}`; - const key = srcId < dstId ? `${srcId}|${dstId}` : `${dstId}|${srcId}`; - const current = edgeCounts.get(key) || { up: 0, total: 0 }; - if (peer.healthCheck?.enabled || peer.healthCheck) { - current.total += 1; - const rawStatus = (peer.healthCheck?.status || '').trim().toLowerCase(); - if (rawStatus === 'up') { - current.up += 1; - } - } - edgeCounts.set(key, current); - } - } - - for (const [edgeId, counts] of edgeCounts.entries()) { - const [source, target] = edgeId.split('|'); - let edgeFill = palette.edge; - if (counts.total > 0) { - if (counts.up === counts.total) { - edgeFill = palette.edgeUp; - } else if (counts.total - counts.up > counts.total / 2) { - edgeFill = palette.edgeDanger; - } else { - edgeFill = palette.edgeWarn; - } - } - - graphEdges.push({ - id: edgeId, - source, - target, - fill: edgeFill, - size: options?.edgeSize ?? 2, - data: { - peerings: [], - hcUp: counts.up, - hcTotal: counts.total, - sourceLabel: source.startsWith('node:') ? source.slice(5) : source, - targetLabel: target.startsWith('node:') ? target.slice(5) : target - } - }); - } - - graphNodes.sort((a, b) => a.label.localeCompare(b.label)); - graphEdges.sort((a, b) => a.id.localeCompare(b.id)); - - return { nodes: graphNodes, edges: graphEdges }; -} - -function Topology({ - mode, - isMaximized, - sites, - peerings, - nodes, - gatewayPools, - gatewayByNode, - hiddenSites, - hiddenGatewayPools, - theme, - siteCounts, - poolCounts, - edgeHealthCheckCounts, - poolToSite -}: { - mode: 'sitesAndPools' | 'nodes'; - isMaximized: boolean; - sites: SiteStatus[]; - peerings: PeeringStatus[]; - nodes: NodeStatus[]; - gatewayPools: GatewayPoolStatus[]; - gatewayByNode: Map; - hiddenSites: Set; - hiddenGatewayPools: Set; - theme: 'dark' | 'light'; - siteCounts: Map; - poolCounts: Map; - edgeHealthCheckCounts: Map; - poolToSite: Map; -}) { - const graphRef = useRef(null); - const topologyWrapperRef = useRef(null); - const [reagraphModule, setReagraphModule] = useState(null); - - useEffect(() => { - let canceled = false; - import('reagraph') - .then((module) => { - if (canceled) return; - setReagraphModule({ - GraphCanvas: module.GraphCanvas as React.ComponentType, - darkTheme: module.darkTheme as Record, - lightTheme: module.lightTheme as Record - }); - }) - .catch((error) => { - console.error('Failed to load topology renderer', error); - }); - - return () => { - canceled = true; - }; - }, []); - - const palette = useMemo( - () => - theme === 'light' - ? { - site: '#16a34a', - siteEmpty: '#6b7280', - siteWarn: '#facc15', - siteDanger: '#dc2626', - pool: '#15803d', - poolEmpty: '#6b7280', - poolWarn: '#facc15', - poolDanger: '#dc2626', - edge: '#94a3b8', - edgeDim: '#cbd5e1', - edgeUp: '#16a34a', - edgeWarn: '#facc15', - edgeDanger: '#dc2626', - nodeHealthy: '#16a34a', - nodeWarn: '#facc15', - nodeDanger: '#dc2626', - label: '#1f2937', - background: '#ffffff' - } - : { - site: '#1DE9AC', - siteEmpty: '#6b7280', - siteWarn: '#facc15', - siteDanger: '#ef4444', - pool: '#1DE9AC', - poolEmpty: '#6b7280', - poolWarn: '#facc15', - poolDanger: '#ef4444', - edge: '#4b5563', - edgeDim: '#334155', - edgeUp: '#1DE9AC', - edgeWarn: '#facc15', - edgeDanger: '#ef4444', - nodeHealthy: '#1DE9AC', - nodeWarn: '#facc15', - nodeDanger: '#ef4444', - label: '#e0e0e0', - background: '#1a1a1a' - }, - [theme] - ); - const graphTheme = useMemo(() => { - if (!reagraphModule) return null; - const base = theme === 'light' ? reagraphModule.lightTheme : reagraphModule.darkTheme; - return { - ...base, - canvas: { - ...base.canvas, - background: palette.background, - fog: null - }, - edge: { - ...base.edge, - fill: palette.edge, - activeFill: palette.edge, - opacity: 1, - inactiveOpacity: 0.25 - }, - node: { - ...base.node, - activeFill: 'rgba(0,0,0,0)', - inactiveOpacity: 1, - hoverOpacity: 0, - label: { - ...base.node.label, - color: palette.label, - activeColor: palette.label, - fontSize: 16, - stroke: palette.background, - strokeColor: palette.background - } - } - }; - }, [palette, reagraphModule, theme]); - - const [hovered, setHovered] = useState<{ - x: number; - y: number; - label: string; - group: string; - lines: string[]; - } | null>(null); - const [edgeHovered, setEdgeHovered] = useState<{ - x: number; - y: number; - title: string; - detail: string; - } | null>(null); - const [hoveredNodeId, setHoveredNodeId] = useState(null); - const [showZoomHint, setShowZoomHint] = useState(false); - const [zoomHintRect, setZoomHintRect] = useState<{ left: number; top: number; width: number; height: number } | null>(null); - const zoomHintTimerRef = useRef(null); - const topologyIconTextures = useTopologyNodeIconTextures(); - const allSiteNames = useMemo( - () => sites - .map((site) => (site.name || '').trim()) - .filter((name): name is string => name.length > 0), - [sites] - ); - const allPoolNames = useMemo( - () => gatewayPools - .map((pool) => (pool.name || '').trim()) - .filter((name): name is string => name.length > 0), - [gatewayPools] - ); - const topologySiteCounts = useMemo(() => { - const counts = new Map(); - for (const siteName of allSiteNames) { - counts.set(siteName, { online: 0, total: 0, warning: 0, danger: 0 }); - } - - if (nodes.length > 0) { - for (const node of nodes) { - const siteName = node.nodeInfo?.siteName; - if (!siteName) continue; - const nodeName = node.nodeInfo?.name; - const isGatewayNode = node.nodeInfo?.isGateway || (nodeName ? gatewayByNode.has(nodeName) : false); - if (isGatewayNode) continue; - - const current = counts.get(siteName) || { online: 0, total: 0, warning: 0, danger: 0 }; - current.total += 1; - - const status = getNodeStatus(node); - if (status === 'success') { - current.online += 1; - } else if (status === 'warning') { - current.warning += 1; - } else { - current.danger += 1; - } - - counts.set(siteName, current); - } - } else { - // Summary mode: use siteCounts prop (online/total only). - for (const [siteName, sc] of siteCounts.entries()) { - const offline = Math.max(0, sc.total - sc.online); - counts.set(siteName, { online: sc.online, total: sc.total, warning: 0, danger: offline }); - } - } - - return counts; - }, [allSiteNames, gatewayByNode, nodes, siteCounts]); - const topologyPoolCounts = useMemo(() => { - const counts = new Map(); - - if (nodes.length > 0) { - const nodeByName = new Map(); - for (const node of nodes) { - const nodeName = node.nodeInfo?.name; - if (!nodeName) continue; - nodeByName.set(nodeName, node); - } - - for (const poolName of allPoolNames) { - const baseline = poolCounts.get(poolName) || { online: 0, total: 0 }; - const current = { online: 0, total: baseline.total, warning: 0, danger: 0 }; - - for (const [nodeName, nodePoolName] of gatewayByNode.entries()) { - if (nodePoolName !== poolName) continue; - const node = nodeByName.get(nodeName); - if (!node) continue; - const status = getNodeStatus(node); - if (status === 'success') { - current.online += 1; - } else if (status === 'warning') { - current.warning += 1; - } else { - current.danger += 1; - } - } - - const accounted = current.online + current.warning + current.danger; - if (current.total < accounted) { - current.total = accounted; - } else if (current.total > accounted) { - current.danger += current.total - accounted; - } - - counts.set(poolName, current); - } - } else { - // Summary mode: use poolCounts prop (online/total only). - for (const poolName of allPoolNames) { - const pc = poolCounts.get(poolName) || { online: 0, total: 0 }; - const offline = Math.max(0, pc.total - pc.online); - counts.set(poolName, { online: pc.online, total: pc.total, warning: 0, danger: offline }); - } - } - - return counts; - }, [allPoolNames, gatewayByNode, nodes, poolCounts]); - - const sitePoolGraph = useMemo( - () => buildGraph( - allSiteNames, - allPoolNames, - peerings, - hiddenSites, - hiddenGatewayPools, - new Set(gatewayPools.map((pool) => pool.name).filter((name): name is string => Boolean(name))), - palette, - topologySiteCounts, - topologyPoolCounts, - edgeHealthCheckCounts, - poolToSite - ), - [allPoolNames, allSiteNames, peerings, gatewayPools, hiddenSites, hiddenGatewayPools, palette, topologySiteCounts, topologyPoolCounts, edgeHealthCheckCounts, poolToSite] - ); - const nodeGraph = useMemo( - () => buildNodeGraph(nodes, gatewayByNode, palette, { edgeSize: 1.4 }), - [nodes, gatewayByNode, palette] - ); - const graph = mode === 'nodes' ? nodeGraph : sitePoolGraph; - const dimNodeColor = useCallback((color: string) => { - const hex = (color || '').trim(); - const match = /^#([0-9a-fA-F]{6})$/.exec(hex); - if (!match) return palette.edgeDim; - const value = match[1]; - const r = parseInt(value.slice(0, 2), 16); - const g = parseInt(value.slice(2, 4), 16); - const b = parseInt(value.slice(4, 6), 16); - const mix = (channel: number) => Math.round(channel * 0.35 + 40 * 0.65); - return `#${mix(r).toString(16).padStart(2, '0')}${mix(g).toString(16).padStart(2, '0')}${mix(b).toString(16).padStart(2, '0')}`; - }, [palette.edgeDim]); - - const graphNodesForRender = useMemo(() => { - const baseNodes = mode === 'nodes' - ? graph.nodes.map((node) => ({ ...node, label: '' })) - : graph.nodes; - - if (!hoveredNodeId) return baseNodes; - const connectedNodeIds = new Set([hoveredNodeId]); - for (const edge of graph.edges) { - if (edge.source === hoveredNodeId) { - connectedNodeIds.add(edge.target); - } else if (edge.target === hoveredNodeId) { - connectedNodeIds.add(edge.source); - } - } - - return baseNodes.map((node) => { - if (connectedNodeIds.has(node.id)) { - return node; - } - const dimmed = dimNodeColor(node.fill || palette.site); - return { - ...node, - fill: dimmed, - activeFill: dimmed - }; - }); - }, [graph.edges, graph.nodes, hoveredNodeId, dimNodeColor, mode, palette.site]); - - const graphEdgesForRender = useMemo(() => { - if (!hoveredNodeId) return graph.edges; - return graph.edges.map((edge) => { - const connected = edge.source === hoveredNodeId || edge.target === hoveredNodeId; - if (connected) return edge; - return { - ...edge, - fill: palette.edgeDim - }; - }); - }, [graph.edges, hoveredNodeId, palette.edgeDim]); - - const topologyConfig = useMemo(() => { - const nodeCount = graph.nodes.length; - - const siteTopologyConfig = { - minCameraDistance: 2, - nodeSize: nodeCount <= 6 ? 68 : 50, - layoutOverrides: { - radius: 25, - concentricSpacing: 25 - } - }; - - const nodeTopologyConfig = { - ...siteTopologyConfig, - minCameraDistance: 6, - - layoutOverrides: { - ...siteTopologyConfig.layoutOverrides, - radius: 25, - concentricSpacing: 50 - } - }; - - return mode === 'nodes' ? nodeTopologyConfig : siteTopologyConfig; - }, [graph.nodes.length, mode]); - const topologyLayoutType = mode === 'nodes' ? 'concentric2d' : 'concentric2d'; - - useEffect(() => { - if (mode !== 'sitesAndPools') return; - if (!graphRef.current) return; - - const fit = () => graphRef.current?.fitNodesInView(); - const first = requestAnimationFrame(() => { - const second = requestAnimationFrame(fit); - (fit as unknown as { _second?: number })._second = second; - }); - - return () => { - cancelAnimationFrame(first); - const second = (fit as unknown as { _second?: number })._second; - if (typeof second === 'number') { - cancelAnimationFrame(second); - } - }; - }, [mode, graph.nodes.length, graph.edges.length]); - - useEffect(() => () => { - if (zoomHintTimerRef.current !== null) { - window.clearTimeout(zoomHintTimerRef.current); - zoomHintTimerRef.current = null; - } - }, []); - - const updateZoomHintRect = useCallback(() => { - const rect = topologyWrapperRef.current?.getBoundingClientRect(); - if (!rect) { - setZoomHintRect(null); - return; - } - setZoomHintRect({ left: rect.left, top: rect.top, width: rect.width, height: rect.height }); - }, []); - - const showCtrlZoomHint = useCallback(() => { - updateZoomHintRect(); - setShowZoomHint(true); - if (zoomHintTimerRef.current !== null) { - window.clearTimeout(zoomHintTimerRef.current); - } - zoomHintTimerRef.current = window.setTimeout(() => { - setShowZoomHint(false); - setZoomHintRect(null); - zoomHintTimerRef.current = null; - }, 1400); - }, [updateZoomHintRect]); - - useEffect(() => { - if (!showZoomHint) { - return; - } - const update = () => updateZoomHintRect(); - window.addEventListener('resize', update, { passive: true }); - window.addEventListener('scroll', update, { passive: true, capture: true }); - document.addEventListener('scroll', update, { passive: true, capture: true }); - return () => { - window.removeEventListener('resize', update); - window.removeEventListener('scroll', update, true); - document.removeEventListener('scroll', update, true); - }; - }, [showZoomHint, updateZoomHintRect]); - - const handleWheelCapture = useCallback((event: React.WheelEvent) => { - if (event.ctrlKey) { - return; - } - event.stopPropagation(); - showCtrlZoomHint(); - }, [showCtrlZoomHint]); - - if (graph.nodes.length === 0) { - return
No peering data available.
; - } - - if (!reagraphModule || !graphTheme) { - return
Loading topology renderer...
; - } - - const GraphCanvas = reagraphModule.GraphCanvas; - const zoomHintStyle = (() => { - if (!zoomHintRect) { - return { left: '50vw', top: '50vh' } as React.CSSProperties; - } - return { left: zoomHintRect.left + zoomHintRect.width / 2, top: zoomHintRect.top + zoomHintRect.height / 2 }; - })(); - const zoomHintOverlayStyle = (() => { - if (!zoomHintRect) { - return null; - } - return { left: zoomHintRect.left, top: zoomHintRect.top, width: zoomHintRect.width, height: zoomHintRect.height }; - })(); - - return ( -
-
- - { - const color = n.fill || palette.site; - const group = (n.data as { group?: string } | undefined)?.group; - const glyph = getTopologyNodeGlyph(group); - const material = ( - - ); - const iconNode = renderTopologyNodeGlyph({ - glyph, - size, - color, - textures: topologyIconTextures, - workerOffsetScale: -0.16 - }); - if (iconNode) return iconNode; - - return ( - - - {material} - - ); - }} - onNodePointerOver={(node, event) => { - const tooltipLabel = (node.label || '').trim() - || (node.id.startsWith('node:') - ? node.id.slice(5) - : node.id.startsWith('site:') - ? node.id.slice(5) - : node.id.startsWith('pool:') - ? node.id.slice(5) - : node.id); - setHoveredNodeId(node.id); - setHovered({ - x: event.clientX + 12, - y: event.clientY + 12, - label: tooltipLabel, - group: (node.data as { group?: string })?.group || 'site', - lines: (() => { - const data = node.data as { lines?: string[]; peerings?: string[] } | undefined; - if (data?.lines && data.lines.length > 0) { - return data.lines; - } - const peerings = data?.peerings || []; - return [`Peerings: ${peerings.length > 0 ? peerings.join(', ') : '-'}`]; - })() - }); - }} - onNodePointerOut={() => { - setHovered(null); - setHoveredNodeId(null); - }} - onEdgePointerOver={(edge, event) => { - if (!event) return; - setHovered(null); - setHoveredNodeId(null); - const data = edge.data as { - peerings?: string[]; - hcUp?: number; - hcTotal?: number; - sourceLabel?: string; - targetLabel?: string; - } | undefined; - const hcDetail = data?.hcTotal && data.hcTotal > 0 - ? `${data.hcUp ?? 0}/${data.hcTotal} links up` - : 'No health check data'; - const title = mode === 'nodes' - ? `${data?.sourceLabel || edge.source} <-> ${data?.targetLabel || edge.target}` - : (data?.peerings?.join(', ') || 'Unknown peering'); - setEdgeHovered({ - x: event.clientX + 12, - y: event.clientY + 12, - title, - detail: hcDetail - }); - }} - onEdgePointerOut={() => setEdgeHovered(null)} - onCanvasPointerOut={() => { - setHovered(null); - setHoveredNodeId(null); - setEdgeHovered(null); - }} - onCanvasClick={() => { - setHovered(null); - setHoveredNodeId(null); - setEdgeHovered(null); - }} - /> - {hovered && createPortal( -
-
{hovered.label}
-
Type: { - hovered.group === 'pool' - ? 'Gateway Pool' - : hovered.group === 'site' - ? 'Site' - : hovered.group === 'gateway-node' - ? 'Gateway Node' - : 'Node' - }
- {hovered.lines.map((line, index) => ( -
{line}
- ))} -
, - document.body - )} - {edgeHovered && createPortal( -
-
{edgeHovered.title}
-
{edgeHovered.detail}
-
, - document.body - )} - {showZoomHint && zoomHintOverlayStyle && createPortal( -
, - document.body - )} - {showZoomHint && createPortal( -
- Hold Ctrl and scroll to zoom graph -
, - document.body - )} -
-
- ); -} - - -export default Topology; diff --git a/frontend/src/components/nodes/shared/index.ts b/frontend/src/components/nodes/shared/index.ts index 6e8a4b51b..e57457d0f 100644 --- a/frontend/src/components/nodes/shared/index.ts +++ b/frontend/src/components/nodes/shared/index.ts @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 -export type { ReagraphModule } from './types'; - export { uiDiag } from './uiDiag'; export { CloseXIcon, MagnifyPlusIcon, TableFilterButton, useDismissOnOutside } from './tableUi'; export { diff --git a/frontend/src/components/nodes/shared/types.ts b/frontend/src/components/nodes/shared/types.ts deleted file mode 100644 index f4a4994bc..000000000 --- a/frontend/src/components/nodes/shared/types.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -import * as React from 'react'; - -type ReagraphModule = { - GraphCanvas: React.ComponentType; - darkTheme: Record; - lightTheme: Record; -}; - -export type { ReagraphModule }; diff --git a/frontend/src/hooks/useDashboardData.ts b/frontend/src/hooks/useDashboardData.ts index f604a6766..1e4fde281 100644 --- a/frontend/src/hooks/useDashboardData.ts +++ b/frontend/src/hooks/useDashboardData.ts @@ -14,8 +14,6 @@ type DashboardDataParams = { gatewayPoolHiddenNames: Set; hiddenSites: Set; selectedNodeTypesFilter: Set; - networkTab: 'siteTopology' | 'matrix'; - maximizedPanel: 'nodes' | 'siteTopology' | 'matrix' | null; pullEnabledOptimistic: boolean | null; selectedNodeName: string | null; nodeDetail: (name: string) => NodeStatus | undefined; @@ -30,8 +28,6 @@ function useDashboardData({ gatewayPoolHiddenNames, hiddenSites, selectedNodeTypesFilter, - networkTab, - maximizedPanel, pullEnabledOptimistic, selectedNodeName, nodeDetail @@ -163,96 +159,7 @@ function useDashboardData({ return counts; }, [gatewayPools, nodes, nodeSummaries]); - const edgeHealthCheckCounts = useMemo(() => { - const nodeToEntity = new Map(); - const nodeNameSet = new Set(); - - // Build node-to-entity mapping from full nodes or summaries - const nodeSource = nodes.length > 0 - ? nodes.map((n) => ({ name: n.nodeInfo?.name, siteName: n.nodeInfo?.siteName })) - : nodeSummaries.map((ns) => ({ name: ns.name, siteName: ns.siteName })); - for (const node of nodeSource) { - const name = node.name; - if (!name) continue; - nodeNameSet.add(name); - const poolName = gatewayByNode.get(name); - if (poolName) { - nodeToEntity.set(name, `pool:${poolName}`); - } else if (node.siteName) { - nodeToEntity.set(name, `site:${node.siteName}`); - } - } - - const counts = new Map(); - - if (nodes.length > 0) { - // Full nodes available: use peer-level health check data - for (const node of nodes) { - const name = node.nodeInfo?.name; - if (!name) continue; - const srcEntity = nodeToEntity.get(name); - if (!srcEntity) continue; - for (const peer of node.peers || []) { - if (!peer.healthCheck?.enabled && !peer.healthCheck) continue; - const peerSite = peer.siteName; - if (!peerSite) continue; - let dstEntity: string | undefined; - if (peer.name) { - if (!nodeNameSet.has(peer.name)) continue; - dstEntity = nodeToEntity.get(peer.name); - } - if (!dstEntity) dstEntity = `site:${peerSite}`; - if (srcEntity === dstEntity) continue; - const edgeKey = srcEntity < dstEntity - ? `${srcEntity}|${dstEntity}` - : `${dstEntity}|${srcEntity}`; - const current = counts.get(edgeKey) || { up: 0, total: 0 }; - current.total++; - const rawStatus = (peer.healthCheck?.status || '').trim().toLowerCase(); - if (rawStatus === 'up') current.up++; - counts.set(edgeKey, current); - } - } - } else { - // Summary mode: derive counts from connectivity matrix - const matrix = summary?.connectivityMatrix; - if (matrix) { - for (const [, siteMatrix] of Object.entries(matrix)) { - const results = siteMatrix?.results || {}; - for (const [src, row] of Object.entries(results)) { - const srcEntity = nodeToEntity.get(src); - if (!srcEntity) continue; - for (const [dst, cellStatus] of Object.entries(row || {})) { - if (src >= dst) continue; // count each pair once - const dstEntity = nodeToEntity.get(dst); - if (!dstEntity || srcEntity === dstEntity) continue; - const edgeKey = srcEntity < dstEntity - ? `${srcEntity}|${dstEntity}` - : `${dstEntity}|${srcEntity}`; - const current = counts.get(edgeKey) || { up: 0, total: 0 }; - current.total++; - const status = (typeof cellStatus === 'string' ? cellStatus : '').trim().toLowerCase(); - if (status === 'up') current.up++; - counts.set(edgeKey, current); - } - } - } - } - } - return counts; - }, [nodes, nodeSummaries, gatewayByNode, summary]); - - const poolToSite = useMemo(() => { - const map = new Map(); - for (const pool of gatewayPools) { - if (pool.name && pool.siteName) { - map.set(pool.name, pool.siteName); - } - } - return map; - }, [gatewayPools]); - - // Visible full nodes (for NetworkCard and other components needing full NodeStatus) + // Visible full nodes provide backward compatibility for summary filtering. const visibleNodes = useMemo(() => { return nodes.filter((node) => { const nodeName = node.nodeInfo?.name || ''; @@ -324,10 +231,6 @@ function useDashboardData({ return { healthy, total }; }, [nodes, nodeSummaries]); - const activeNetworkTab = maximizedPanel === 'siteTopology' || maximizedPanel === 'matrix' - ? maximizedPanel - : networkTab; - const effectivePullEnabled = pullEnabledOptimistic ?? Boolean(summary?.pullEnabled ?? status?.pullEnabled); // Active selected node detail from the cache @@ -340,9 +243,7 @@ function useDashboardData({ }, [selectedNodeName, nodeDetail, nodes]); return { - activeNetworkTab, activeSelectedNode, - edgeHealthCheckCounts, effectivePullEnabled, gatewayByNode, nodeK8sStatusMap, @@ -350,7 +251,6 @@ function useDashboardData({ nodeTotalCount, peerHealth, poolCounts, - poolToSite, siteCounts, visibleNodes, visibleNodeSummaries diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 9a28582bd..9bf92c9b4 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -28,9 +28,7 @@ export default defineConfig(({ mode }) => { output: { codeSplitting: { groups: [ - { name: 'tanstack', test: /node_modules[\\/]@tanstack[\\/](react-table|table-core)[\\/]/ }, - { name: 'reagraph', test: /node_modules[\\/]reagraph[\\/]/ }, - { name: 'three', test: /node_modules[\\/]three[\\/]/ } + { name: 'tanstack', test: /node_modules[\\/]@tanstack[\\/](react-table|table-core)[\\/]/ } ] } } diff --git a/internal/net/authn/kubernetes.go b/internal/net/authn/kubernetes.go index 03e2e6f9d..3ee1450ef 100644 --- a/internal/net/authn/kubernetes.go +++ b/internal/net/authn/kubernetes.go @@ -20,7 +20,12 @@ type kubernetesTokenClaims struct { Namespace string `json:"namespace"` ServiceAccount struct { Name string `json:"name"` + UID string `json:"uid"` } `json:"serviceaccount"` + Pod struct { + Name string `json:"name"` + UID string `json:"uid"` + } `json:"pod"` Node struct { Name string `json:"name"` } `json:"node"` @@ -40,6 +45,9 @@ func DecodeKubernetesServiceAccountIdentity(token string) (*KubernetesServiceAcc Subject: claims.Subject, Namespace: claims.Kubernetes.Namespace, ServiceAccountName: claims.Kubernetes.ServiceAccount.Name, + ServiceAccountUID: claims.Kubernetes.ServiceAccount.UID, + PodName: claims.Kubernetes.Pod.Name, + PodUID: claims.Kubernetes.Pod.UID, NodeName: claims.Kubernetes.Node.Name, }, nil } diff --git a/internal/net/authn/oidc.go b/internal/net/authn/oidc.go index def97611d..17b9be050 100644 --- a/internal/net/authn/oidc.go +++ b/internal/net/authn/oidc.go @@ -39,6 +39,9 @@ type KubernetesServiceAccountIdentity struct { Subject string Namespace string ServiceAccountName string + ServiceAccountUID string + PodName string + PodUID string NodeName string } @@ -48,7 +51,12 @@ type kubernetesServiceAccountClaims struct { Namespace string `json:"namespace"` ServiceAccount struct { Name string `json:"name"` + UID string `json:"uid"` } `json:"serviceaccount"` + Pod struct { + Name string `json:"name"` + UID string `json:"uid"` + } `json:"pod"` Node struct { Name string `json:"name"` } `json:"node"` @@ -268,6 +276,9 @@ func (v *KubernetesOIDCVerifier) Verify(ctx context.Context, tokenString string) Subject: claims.Subject, Namespace: claims.Kubernetes.Namespace, ServiceAccountName: claims.Kubernetes.ServiceAccount.Name, + ServiceAccountUID: claims.Kubernetes.ServiceAccount.UID, + PodName: claims.Kubernetes.Pod.Name, + PodUID: claims.Kubernetes.Pod.UID, NodeName: claims.Kubernetes.Node.Name, }, nil } diff --git a/internal/net/authn/oidc_test.go b/internal/net/authn/oidc_test.go index e247a1b76..627ade18b 100644 --- a/internal/net/authn/oidc_test.go +++ b/internal/net/authn/oidc_test.go @@ -86,6 +86,9 @@ func TestKubernetesOIDCVerifier(t *testing.T) { } claims.Kubernetes.Namespace = "unbounded-system" claims.Kubernetes.ServiceAccount.Name = "unbounded-net-node" + claims.Kubernetes.ServiceAccount.UID = "sa-uid" + claims.Kubernetes.Pod.Name = "node-agent" + claims.Kubernetes.Pod.UID = "pod-uid" claims.Kubernetes.Node.Name = "node-a" token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) @@ -160,6 +163,15 @@ func TestKubernetesOIDCVerifier(t *testing.T) { } }) + if identity.PodName != "node-agent" || identity.PodUID != "pod-uid" || identity.ServiceAccountUID != "sa-uid" { + t.Fatalf("missing bound object claims: %#v", identity) + } + + decoded, err := DecodeKubernetesServiceAccountIdentity(tokenString) + if err != nil || decoded == nil || *decoded != *identity { + t.Fatalf("unverified decoding lost authenticated claims: %#v, %v", decoded, err) + } + claims.Audience = jwt.ClaimStrings{"other-service"} wrongAudienceToken := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) wrongAudienceToken.Header["kid"] = keyID diff --git a/internal/net/authn/pod_binding.go b/internal/net/authn/pod_binding.go new file mode 100644 index 000000000..77982b6ef --- /dev/null +++ b/internal/net/authn/pod_binding.go @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package authn + +import ( + "context" + "fmt" + "time" + + corelisters "k8s.io/client-go/listers/core/v1" +) + +// PodBoundTokenVerifierOptions supplies process-lifetime informer caches. +// Ready must reject unsynced or stopped caches. Informer reads are eventually +// consistent, not a strongly consistent substitute for TokenReview. +type PodBoundTokenVerifierOptions struct { + Namespace string + ServiceAccount string + Pods corelisters.PodLister + ServiceAccounts corelisters.ServiceAccountLister + Ready func() bool +} + +// PodBoundTokenVerifier checks bound objects after cryptographic verification. +// It deliberately supports only Pod-bound node agents, not every Kubernetes +// service account token type, and never caches an authorization decision. +type PodBoundTokenVerifier struct { + verifier interface { + Verify(context.Context, string) (*KubernetesServiceAccountIdentity, error) + } + options PodBoundTokenVerifierOptions + now func() time.Time +} + +// NewPodBoundTokenVerifier decorates an OIDC verifier, not a TokenReview verifier. +func NewPodBoundTokenVerifier(verifier interface { + Verify(context.Context, string) (*KubernetesServiceAccountIdentity, error) +}, options PodBoundTokenVerifierOptions, +) *PodBoundTokenVerifier { + return &PodBoundTokenVerifier{verifier: verifier, options: options, now: time.Now} +} + +// Verify authenticates the token and rechecks its Pod and service account in the +// caches on every request, including when the signing keys are already cached. +func (v *PodBoundTokenVerifier) Verify(ctx context.Context, token string) (*KubernetesServiceAccountIdentity, error) { + if err := v.checkReady(ctx); err != nil { + return nil, err + } + + identity, err := v.verifier.Verify(ctx, token) + if err != nil { + return nil, err + } + + if identity == nil || identity.Namespace != v.options.Namespace || + identity.ServiceAccountName != v.options.ServiceAccount || + identity.Namespace == "" || identity.ServiceAccountName == "" || + identity.Subject != "system:serviceaccount:"+identity.Namespace+":"+identity.ServiceAccountName { + return nil, fmt.Errorf("unexpected node service account identity") + } + + if identity.PodName == "" || identity.PodUID == "" || identity.ServiceAccountUID == "" || identity.NodeName == "" { + return nil, fmt.Errorf("node token requires complete Pod, service account UID, and node claims") + } + + sa, err := v.options.ServiceAccounts.ServiceAccounts(identity.Namespace).Get(identity.ServiceAccountName) + if err != nil { + return nil, fmt.Errorf("get bound service account from cache: %w", err) + } + + // Kubernetes accepts deletion timestamps exactly at the 60-second boundary. + cutoff := v.now().Add(-60 * time.Second) + if string(sa.UID) != identity.ServiceAccountUID || + (sa.DeletionTimestamp != nil && sa.DeletionTimestamp.Time.Before(cutoff)) { + return nil, fmt.Errorf("bound service account has been deleted or replaced") + } + + pod, err := v.options.Pods.Pods(identity.Namespace).Get(identity.PodName) + if err != nil { + return nil, fmt.Errorf("get bound Pod from cache: %w", err) + } + + if string(pod.UID) != identity.PodUID || + (pod.DeletionTimestamp != nil && pod.DeletionTimestamp.Time.Before(cutoff)) { + return nil, fmt.Errorf("bound Pod has been deleted or replaced") + } + + // For our node-only scope, also tie the informational node claim and service + // account identity to the Pod spec. Node readiness is not an auth gate. + if pod.Spec.ServiceAccountName != identity.ServiceAccountName || pod.Spec.NodeName != identity.NodeName { + return nil, fmt.Errorf("bound Pod service account or node does not match token") + } + + if err := v.checkReady(ctx); err != nil { + return nil, err + } + + return identity, nil +} + +func (v *PodBoundTokenVerifier) checkReady(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + + if v.options.Ready == nil || !v.options.Ready() { + return fmt.Errorf("node authentication object caches are not ready") + } + + return nil +} diff --git a/internal/net/authn/pod_binding_test.go b/internal/net/authn/pod_binding_test.go new file mode 100644 index 000000000..3bdb10b95 --- /dev/null +++ b/internal/net/authn/pod_binding_test.go @@ -0,0 +1,287 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package authn + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "errors" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" +) + +type podBindingFixture struct { + verifier *PodBoundTokenVerifier + claims *kubernetesServiceAccountClaims + key *ecdsa.PrivateKey + pod *corev1.Pod + sa *corev1.ServiceAccount + pods cache.Indexer + sas cache.Indexer + ready bool + now time.Time +} + +type podBindingVerifierFunc func(context.Context, string) (*KubernetesServiceAccountIdentity, error) + +func (f podBindingVerifierFunc) Verify(ctx context.Context, token string) (*KubernetesServiceAccountIdentity, error) { + return f(ctx, token) +} + +func TestPodBoundTokenVerifierRejectsChangesDuringVerification(t *testing.T) { + for _, change := range []string{"request canceled", "caches stopped", "nil identity", "identity with error"} { + t.Run(change, func(t *testing.T) { + f := newPodBindingFixture(t) + f.populate(t) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + oidc := f.verifier.verifier + + f.verifier.verifier = podBindingVerifierFunc(func(ctx context.Context, token string) (*KubernetesServiceAccountIdentity, error) { + identity, err := oidc.Verify(ctx, token) + if err != nil { + t.Fatalf("signed test token failed: %v", err) + } + + switch change { + case "request canceled": + cancel() + case "caches stopped": + f.ready = false + case "nil identity": + return nil, nil + case "identity with error": + return identity, errors.New("verification failed") + } + + return identity, nil + }) + if identity, err := f.verifier.Verify(ctx, f.token(t)); err == nil || identity != nil { + t.Fatalf("unsafe verifier result accepted: %+v, %v", identity, err) + } + }) + } +} + +func newPodBindingFixture(t *testing.T) *podBindingFixture { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + + f := &podBindingFixture{ + key: key, ready: true, now: time.Now(), + pods: cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}), + sas: cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}), + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "unbounded-system", Name: "agent", UID: "pod-uid"}, + Spec: corev1.PodSpec{ServiceAccountName: "unbounded-net-node", NodeName: "node-a"}, + }, + sa: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: "unbounded-system", Name: "unbounded-net-node", UID: "sa-uid"}}, + } + f.claims = &kubernetesServiceAccountClaims{RegisteredClaims: jwt.RegisteredClaims{ + Issuer: "https://issuer.example", Subject: "system:serviceaccount:unbounded-system:unbounded-net-node", + Audience: jwt.ClaimStrings{"api"}, ExpiresAt: jwt.NewNumericDate(f.now.Add(time.Hour)), + }} + f.claims.Kubernetes.Namespace = f.sa.Namespace + f.claims.Kubernetes.ServiceAccount.Name = f.sa.Name + f.claims.Kubernetes.ServiceAccount.UID = string(f.sa.UID) + f.claims.Kubernetes.Pod.Name = f.pod.Name + f.claims.Kubernetes.Pod.UID = string(f.pod.UID) + f.claims.Kubernetes.Node.Name = f.pod.Spec.NodeName + oidc := &KubernetesOIDCVerifier{ + issuer: f.claims.Issuer, audience: "api", now: func() time.Time { return f.now }, + loadedAt: f.now, + keys: map[string]oidcSigningKey{"key": {key: &key.PublicKey, algorithm: "ES256"}}, + } + f.verifier = NewPodBoundTokenVerifier(oidc, PodBoundTokenVerifierOptions{ + Namespace: f.sa.Namespace, ServiceAccount: f.sa.Name, + Pods: corelisters.NewPodLister(f.pods), ServiceAccounts: corelisters.NewServiceAccountLister(f.sas), + Ready: func() bool { return f.ready }, + }) + f.verifier.now = func() time.Time { return f.now } + + return f +} + +func (f *podBindingFixture) token(t *testing.T) string { + t.Helper() + + token := jwt.NewWithClaims(jwt.SigningMethodES256, f.claims) + token.Header["kid"] = "key" + + signed, err := token.SignedString(f.key) + if err != nil { + t.Fatal(err) + } + + return signed +} + +func (f *podBindingFixture) populate(t *testing.T) { + t.Helper() + + if f.pod != nil { + if err := f.pods.Add(f.pod); err != nil { + t.Fatal(err) + } + } + + if f.sa != nil { + if err := f.sas.Add(f.sa); err != nil { + t.Fatal(err) + } + } +} + +func TestPodBoundTokenVerifier(t *testing.T) { + wrongKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name string + mutate func(*podBindingFixture) + allow bool + }{ + {"live objects", func(*podBindingFixture) {}, true}, + {"missing Pod", func(f *podBindingFixture) { f.pod = nil }, false}, + {"missing service account", func(f *podBindingFixture) { f.sa = nil }, false}, + {"replaced Pod", func(f *podBindingFixture) { f.pod.UID = "new-pod" }, false}, + {"replaced service account", func(f *podBindingFixture) { f.sa.UID = "new-sa" }, false}, + {"no Pod claims", func(f *podBindingFixture) { f.claims.Kubernetes.Pod.Name = ""; f.claims.Kubernetes.Pod.UID = "" }, false}, + {"missing Pod name", func(f *podBindingFixture) { f.claims.Kubernetes.Pod.Name = "" }, false}, + {"missing Pod UID", func(f *podBindingFixture) { f.claims.Kubernetes.Pod.UID = "" }, false}, + {"missing service account UID", func(f *podBindingFixture) { f.claims.Kubernetes.ServiceAccount.UID = "" }, false}, + {"wrong Pod name", func(f *podBindingFixture) { f.claims.Kubernetes.Pod.Name = "other" }, false}, + {"wrong namespace", func(f *podBindingFixture) { + f.claims.Kubernetes.Namespace = "other" + f.claims.Subject = "system:serviceaccount:other:unbounded-net-node" + f.pod.Namespace = "other" + f.sa.Namespace = "other" + }, false}, + {"wrong service account", func(f *podBindingFixture) { + f.claims.Kubernetes.ServiceAccount.Name = "other" + f.claims.Subject = "system:serviceaccount:unbounded-system:other" + f.pod.Spec.ServiceAccountName = "other" + f.sa.Name = "other" + }, false}, + {"Pod belongs to different service account", func(f *podBindingFixture) { f.pod.Spec.ServiceAccountName = "other" }, false}, + {"Pod belongs to different node", func(f *podBindingFixture) { f.pod.Spec.NodeName = "node-b" }, false}, + {"unscheduled Pod", func(f *podBindingFixture) { f.pod.Spec.NodeName = "" }, false}, + {"missing node claim", func(f *podBindingFixture) { f.claims.Kubernetes.Node.Name = "" }, false}, + {"subject mismatch", func(f *podBindingFixture) { f.claims.Subject = "system:serviceaccount:other:other" }, false}, + {"unsynced caches", func(f *podBindingFixture) { f.ready = false }, false}, + {"missing readiness guard", func(f *podBindingFixture) { f.verifier.options.Ready = nil }, false}, + {"wrong audience", func(f *podBindingFixture) { f.claims.Audience = jwt.ClaimStrings{"other"} }, false}, + {"wrong issuer", func(f *podBindingFixture) { f.claims.Issuer = "https://other.example" }, false}, + {"expired token", func(f *podBindingFixture) { f.claims.ExpiresAt = jwt.NewNumericDate(f.now.Add(-time.Hour)) }, false}, + {"invalid signature", func(f *podBindingFixture) { + f.key = wrongKey + }, false}, + {"not ready Pod without Node object", func(f *podBindingFixture) { + f.pod.Status.Phase = corev1.PodFailed + f.pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}} + }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newPodBindingFixture(t) + tc.mutate(f) + f.populate(t) + + identity, err := f.verifier.Verify(t.Context(), f.token(t)) + if (err == nil) != tc.allow || (identity != nil) != tc.allow { + t.Fatalf("identity=%+v error=%v, want allowed=%v", identity, err, tc.allow) + } + }) + } +} + +func TestPodBoundTokenVerifierDeletionGrace(t *testing.T) { + for _, object := range []string{"Pod", "service account"} { + for _, offset := range []time.Duration{-time.Nanosecond, 0, time.Nanosecond} { + t.Run(object+"/"+offset.String(), func(t *testing.T) { + f := newPodBindingFixture(t) + + deletedAt := metav1.NewTime(f.now.Add(-60*time.Second + offset)) + if object == "Pod" { + f.pod.DeletionTimestamp = &deletedAt + } else { + f.sa.DeletionTimestamp = &deletedAt + } + + f.populate(t) + + identity, err := f.verifier.Verify(t.Context(), f.token(t)) + if (err == nil) != (offset >= 0) || (identity != nil) != (offset >= 0) { + t.Fatalf("identity=%+v error=%v, deletion offset=%v", identity, err, offset) + } + }) + } + } +} + +func TestPodBoundTokenVerifierRechecksCachedKeys(t *testing.T) { + for _, object := range []string{"Pod", "service account", "stopped caches", "canceled request", "grace elapsed"} { + t.Run(object, func(t *testing.T) { + f := newPodBindingFixture(t) + f.populate(t) + + token := f.token(t) + if _, err := f.verifier.Verify(t.Context(), token); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + switch object { + case "Pod": + if err := f.pods.Delete(f.pod); err != nil { + t.Fatal(err) + } + case "service account": + if err := f.sas.Delete(f.sa); err != nil { + t.Fatal(err) + } + case "stopped caches": + f.ready = false + case "canceled request": + cancel() + case "grace elapsed": + pod := f.pod.DeepCopy() + deletedAt := metav1.NewTime(f.now) + + pod.DeletionTimestamp = &deletedAt + if err := f.pods.Update(pod); err != nil { + t.Fatal(err) + } + + f.now = f.now.Add(60 * time.Second) + if _, err := f.verifier.Verify(ctx, token); err != nil { + t.Fatalf("exact grace boundary rejected: %v", err) + } + + f.now = f.now.Add(time.Nanosecond) + } + + if identity, err := f.verifier.Verify(ctx, token); err == nil || identity != nil { + t.Fatalf("cached-key authentication bypassed revocation: %+v, %v", identity, err) + } + }) + } +} diff --git a/internal/net/healthcheck/manager.go b/internal/net/healthcheck/manager.go index f511d9d9a..5bc291f58 100644 --- a/internal/net/healthcheck/manager.go +++ b/internal/net/healthcheck/manager.go @@ -26,6 +26,7 @@ type Manager struct { conn net.PacketConn mu sync.RWMutex + peerMu sync.Mutex sessions map[string]*session ctx context.Context @@ -90,15 +91,26 @@ func (m *Manager) Start(ctx context.Context) error { // Stop gracefully shuts down the listener and all sessions. func (m *Manager) Stop() { + m.peerMu.Lock() + defer m.peerMu.Unlock() + if m.cancel != nil { m.cancel() } - m.mu.Lock() + m.mu.RLock() + + sessions := make([]*session, 0, len(m.sessions)) for _, s := range m.sessions { + sessions = append(sessions, s) + } + + m.mu.RUnlock() + + for _, s := range sessions { s.stop() } - m.mu.Unlock() + m.listener.Stop() klog.Info("healthcheck manager stopped") } @@ -106,19 +118,30 @@ func (m *Manager) Stop() { // AddPeer registers a new peer for health checking. If the manager is // already running, the session starts immediately. func (m *Manager) AddPeer(peerHostname string, overlayIP net.IP, settings HealthCheckSettings) error { - m.mu.Lock() - defer m.mu.Unlock() + if err := settings.validate(); err != nil { + return err + } + m.peerMu.Lock() + defer m.peerMu.Unlock() + + m.mu.Lock() if existing, exists := m.sessions[peerHostname]; exists { - // Peer already registered -- update settings/IP if changed, otherwise no-op - if existing.overlayIP.Equal(overlayIP) && existing.settings == settings { + if existing.overlayIP.Equal(overlayIP) { + existing.updateSettings(settings) + m.mu.Unlock() + return nil } - // Settings or IP changed -- stop old session and replace + + delete(m.sessions, peerHostname) + m.mu.Unlock() + // Callbacks may query the manager while stop waits for their completion. existing.stop() klog.V(4).Infof("healthcheck: updating peer %s (%s -> %s)", peerHostname, existing.overlayIP, overlayIP) - delete(m.sessions, peerHostname) + m.mu.Lock() } + defer m.mu.Unlock() s := newSession(sessionConfig{ peerHostname: peerHostname, @@ -145,6 +168,9 @@ func (m *Manager) AddPeer(peerHostname string, overlayIP net.IP, settings Health // RemovePeer stops and removes a peer session. func (m *Manager) RemovePeer(peerHostname string) error { + m.peerMu.Lock() + defer m.peerMu.Unlock() + m.mu.Lock() s, exists := m.sessions[peerHostname] @@ -165,9 +191,14 @@ func (m *Manager) RemovePeer(peerHostname string) error { // UpdatePeerSettings modifies the health check parameters for an existing peer. func (m *Manager) UpdatePeerSettings(peerHostname string, settings HealthCheckSettings) error { + if err := settings.validate(); err != nil { + return err + } + m.mu.RLock() + defer m.mu.RUnlock() + s, exists := m.sessions[peerHostname] - m.mu.RUnlock() if !exists { return fmt.Errorf("peer %q not found", peerHostname) @@ -178,6 +209,22 @@ func (m *Manager) UpdatePeerSettings(peerHostname string, settings HealthCheckSe return nil } +// GetPeerSettings returns a copy of the currently applied session settings. +func (m *Manager) GetPeerSettings(peerHostname string) (HealthCheckSettings, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + s, exists := m.sessions[peerHostname] + if !exists { + return HealthCheckSettings{}, fmt.Errorf("peer %q not found", peerHostname) + } + + s.mu.Lock() + defer s.mu.Unlock() + + return s.settings, nil +} + // GetPeerStatus returns the current health status for a single peer. func (m *Manager) GetPeerStatus(peerHostname string) (*PeerStatus, error) { m.mu.RLock() diff --git a/internal/net/healthcheck/probe_phase.go b/internal/net/healthcheck/probe_phase.go new file mode 100644 index 000000000..6f047bc9e --- /dev/null +++ b/internal/net/healthcheck/probe_phase.go @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package healthcheck + +import ( + "crypto/sha256" + "encoding/binary" + "math/bits" + "time" +) + +// Directional identities spread both a node's outgoing probes and the probes +// arriving at one peer from different nodes. No process-global RNG is needed. +func peerProbePhaseSeed(local, remote string) uint64 { + key := make([]byte, 0, 16+len(local)+len(remote)) + key = binary.BigEndian.AppendUint64(key, uint64(len(local))) + key = append(key, local...) + key = binary.BigEndian.AppendUint64(key, uint64(len(remote))) + key = append(key, remote...) + hash := sha256.Sum256(key) + + return binary.BigEndian.Uint64(hash[:8]) +} + +// probePhase scales a stable fraction into [1ns, interval], without overflow or +// floating-point rounding. Only the first probe is offset; steady ticks retain +// the configured interval. Manager validation guarantees interval is positive. +func (s *session) probePhase(interval time.Duration) time.Duration { + high, _ := bits.Mul64(s.probePhaseSeed, uint64(interval)) + return time.Duration(high + 1) +} diff --git a/internal/net/healthcheck/probe_phase_test.go b/internal/net/healthcheck/probe_phase_test.go new file mode 100644 index 000000000..fa3e7a248 --- /dev/null +++ b/internal/net/healthcheck/probe_phase_test.go @@ -0,0 +1,327 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package healthcheck + +import ( + "context" + "fmt" + "net" + "sync" + "testing" + "testing/synctest" + "time" + + "google.golang.org/protobuf/proto" + + pb "github.com/Azure/unbounded/internal/net/healthcheck/proto" +) + +var intervalEndProbePhaseSeed = ^uint64(0) + +func TestProbePhaseBoundsAndIdentitySpread(t *testing.T) { + for _, interval := range []time.Duration{1, 2, 15 * time.Second, 60 * time.Second, time.Duration(1<<63 - 1)} { + for _, seed := range []uint64{0, 1, 1 << 63, ^uint64(0)} { + s := newSession(sessionConfig{probePhaseSeed: &seed}) + + phase := s.probePhase(interval) + if phase <= 0 || phase > interval { + t.Fatalf("seed=%d interval=%v phase=%v", seed, interval, phase) + } + + if seed == 0 && phase != time.Nanosecond { + t.Fatal("minimum phase must be positive") + } + + if seed == ^uint64(0) && phase != interval { + t.Fatal("maximum injected fraction must reach the interval boundary") + } + } + } + + if peerProbePhaseSeed("ab", "c") == peerProbePhaseSeed("a", "bc") || + peerProbePhaseSeed("node-a", "node-b") == peerProbePhaseSeed("node-b", "node-a") { + t.Fatal("phase identities are ambiguous or not directional") + } + + for _, incoming := range []bool{false, true} { + buckets := make([]int, 60) + + for i := range 2000 { + local, remote := "node-fixed", fmt.Sprintf("node-%d", i) + if incoming { + local, remote = remote, local + } + + s := newSession(sessionConfig{localHostname: local, peerHostname: remote}) + phase := s.probePhase(time.Minute) + + buckets[int((phase-1)/time.Second)]++ + if phase != newSession(sessionConfig{localHostname: local, peerHostname: remote}).probePhase(time.Minute) { + t.Fatal("identity-derived phase is not deterministic") + } + } + + for second, count := range buckets { + if count == 0 || count > 80 { + t.Fatalf("incoming=%t second=%d count=%d: phases concentrate peers", incoming, second, count) + } + } + + minimum, maximum := 2000, 0 + for _, count := range buckets { + minimum, maximum = min(minimum, count), max(maximum, count) + } + + t.Logf("incoming=%t: 2000 peers span all 60 one-second buckets; min=%d max=%d", incoming, minimum, maximum) + } +} + +type phaseRecordingConn struct { + discardProbeConn + mu sync.Mutex + writes map[string][]time.Time +} + +func (c *phaseRecordingConn) WriteTo(data []byte, _ net.Addr) (int, error) { + var packet pb.HealthCheckPacket + if err := proto.Unmarshal(data, &packet); err != nil { + return 0, err + } + + c.mu.Lock() + defer c.mu.Unlock() + + key := packet.SourceHostname + "|" + packet.DestinationHostname + c.writes[key] = append(c.writes[key], time.Now()) + + return len(data), nil +} + +func TestProbePhasesSpreadFleetUpdatesAndPreserveRate(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + conn := &phaseRecordingConn{writes: make(map[string][]time.Time)} + settings := DefaultSettings() + settings.TransmitInterval, settings.ReceiveInterval = time.Minute, time.Minute + start := time.Now() + + var sessions []*session + + for i := range 128 { + s := newSession(sessionConfig{localHostname: "local", peerHostname: fmt.Sprintf("peer-%d", i), settings: settings, conn: conn}) + s.start(ctx) + sessions = append(sessions, s) + } + + defer func() { + cancel() + + for _, s := range sessions { + s.stop() + } + }() + + synctest.Wait() + time.Sleep(2 * time.Minute) + synctest.Wait() + + buckets := make(map[int]int) + + for _, writes := range conn.writes { + if len(writes) != 2 || writes[1].Sub(writes[0]) != time.Minute { + t.Fatalf("steady cadence changed: %v", writes) + } + + buckets[int(writes[0].Sub(start)/time.Second)]++ + } + + if len(conn.writes) != len(sessions) || len(buckets) < 30 { + t.Fatalf("initial phases concentrated: peers=%d buckets=%d", len(conn.writes), len(buckets)) + } + + for _, count := range buckets { + if count > 10 { + t.Fatalf("initial one-second burst contains %d/128 peers", count) + } + } + + t.Logf("128 initial sessions: %d one-second buckets, exactly one probe per 60s interval", len(buckets)) + + changedAt := time.Now() + + for _, s := range sessions { + for _, interval := range []time.Duration{15 * time.Second, time.Minute, 15 * time.Second} { + settings.TransmitInterval, settings.ReceiveInterval = interval, interval + s.updateSettings(settings) + } + } + + synctest.Wait() + time.Sleep(30 * time.Second) + synctest.Wait() + + buckets = make(map[int]int) + + for _, writes := range conn.writes { + if len(writes) != 4 || writes[3].Sub(writes[2]) != 15*time.Second { + t.Fatalf("coalesced update changed steady cadence: %v", writes) + } + + phase := writes[2].Sub(changedAt) + if phase <= 0 || phase > 15*time.Second { + t.Fatalf("updated phase out of bounds: %v", phase) + } + + buckets[int(phase/time.Second)]++ + } + + if len(buckets) < 12 { + t.Fatalf("reconfiguration phases concentrated in %d seconds", len(buckets)) + } + + t.Logf("128 reconfigured sessions: %d one-second buckets, exactly one probe per 15s interval", len(buckets)) + cancel() + + for _, s := range sessions { + s.stop() + } + + time.Sleep(time.Minute) + + for _, writes := range conn.writes { + if len(writes) != 4 { + t.Fatal("stopped sessions leaked probes") + } + } + }) +} + +func TestProbePhaseCancellationAndNonTransmitUpdates(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + settings := DefaultSettings() + settings.TransmitInterval = time.Minute + s := newSession(sessionConfig{settings: settings, conn: &discardProbeConn{}, probePhaseSeed: &intervalEndProbePhaseSeed}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s.start(ctx) + defer s.stop() + + synctest.Wait() + time.Sleep(10 * time.Second) + + settings.ReceiveInterval = 2 * time.Minute + s.updateSettings(settings) + settings.MaxBackoff = 240 * time.Second + s.updateSettings(settings) + s.updateSettings(settings) + synctest.Wait() + time.Sleep(50 * time.Second) + synctest.Wait() + + if s.status().PacketsSent != 1 { + t.Fatal("receive/backoff/unchanged settings disturbed the original transmit phase") + } + + settings.TransmitInterval = time.Hour + s.updateSettings(settings) + synctest.Wait() + cancel() + s.stop() + time.Sleep(2 * time.Hour) + + if s.status().PacketsSent != 1 { + t.Fatal("canceled initial phase sent probes") + } + }) +} + +func TestShortenedIntervalsAllowFirstProbeNominalTimeout(t *testing.T) { + for _, reply := range []bool{false, true} { + t.Run(fmt.Sprintf("reply-%t", reply), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + settings := DefaultSettings() + settings.TransmitInterval, settings.ReceiveInterval = time.Minute, time.Minute + s := newSession(sessionConfig{settings: settings, conn: &discardProbeConn{}, probePhaseSeed: &intervalEndProbePhaseSeed}) + s.state, s.stateSince = StateUp, time.Now().Add(-time.Hour) + s.lastReceived = time.Now().Add(-50 * time.Second) + oldReply := s.lastReceived + s.packetsReceived = 10 + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s.start(ctx) + defer s.stop() + + synctest.Wait() + + changedAt := time.Now() + + s.updateSettings(DefaultSettings()) + synctest.Wait() + + if s.detectTimeout() != 45*time.Second || s.detectGraceUntil.Sub(changedAt) != time.Minute { + t.Fatal("nominal timeout or bounded transition window changed") + } + + time.Sleep(23 * time.Second) + synctest.Wait() + + if s.status().State != StateUp || s.packetsReceived != 10 || !s.lastReceived.Equal(oldReply) { + t.Fatal("shortening applied the new timeout to an old-cadence reply or fabricated liveness") + } + + if reply { + s.receiveReply(&pb.HealthCheckPacket{TimestampNs: time.Now().UnixNano()}) + synctest.Wait() + + if !s.detectGraceUntil.IsZero() { + t.Fatal("fresh reply did not restore ordinary detection") + } + + time.Sleep(23 * time.Second) + synctest.Wait() + + if s.status().State != StateUp { + t.Fatal("healthy peer flapped during interval shortening") + } + + time.Sleep(45 * time.Second) + } else { + time.Sleep(45 * time.Second) + } + + synctest.Wait() + + if s.status().State != StateDown { + t.Fatal("transition protection silently disabled failure detection") + } + }) + }) + } +} + +func TestTimeoutTransitionRechecksFreshReply(t *testing.T) { + settings := DefaultSettings() + s := newSession(sessionConfig{settings: settings}) + s.state = StateUp + s.lastReceived = time.Now().Add(-time.Minute) + s.mu.Lock() + expired := s.replyTimedOut(time.Now()) + s.mu.Unlock() + + if !expired { + t.Fatal("test requires expired old snapshot") + } + + s.receiveReply(&pb.HealthCheckPacket{TimestampNs: time.Now().UnixNano()}) + + if s.setStateIf(StateDown, func() bool { return s.state == StateUp && s.replyTimedOut(time.Now()) }) { + t.Fatal("stale timeout snapshot overrode a fresh reply") + } +} diff --git a/internal/net/healthcheck/reconfiguration_test.go b/internal/net/healthcheck/reconfiguration_test.go new file mode 100644 index 000000000..e081a643a --- /dev/null +++ b/internal/net/healthcheck/reconfiguration_test.go @@ -0,0 +1,390 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package healthcheck + +import ( + "context" + "io" + "net" + "reflect" + "sync" + "testing" + "testing/synctest" + "time" + + pb "github.com/Azure/unbounded/internal/net/healthcheck/proto" +) + +type discardProbeConn struct{} + +func (*discardProbeConn) ReadFrom([]byte) (int, net.Addr, error) { return 0, nil, io.EOF } +func (*discardProbeConn) WriteTo(b []byte, _ net.Addr) (int, error) { return len(b), nil } +func (*discardProbeConn) Close() error { return nil } +func (*discardProbeConn) LocalAddr() net.Addr { return &net.UDPAddr{} } +func (*discardProbeConn) SetDeadline(time.Time) error { return nil } +func (*discardProbeConn) SetReadDeadline(time.Time) error { return nil } +func (*discardProbeConn) SetWriteDeadline(time.Time) error { return nil } + +func TestDefaultHealthCheckIntervals(t *testing.T) { + settings := DefaultSettings() + if settings.TransmitInterval != 15*time.Second || settings.ReceiveInterval != 15*time.Second || + settings.DetectMultiplier != 3 || settings.MaxBackoff != 120*time.Second { + t.Fatalf("unexpected defaults: %+v", settings) + } + + s := newSession(sessionConfig{settings: settings}) + if s.detectTimeout() != 45*time.Second { + t.Fatal("unexpected default detection timeout") + } +} + +func TestManagerLiveSettingsPreserveHealthySession(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + m, err := NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + m.ctx, m.conn = ctx, &discardProbeConn{} + defer m.Stop() + + ip := net.ParseIP("10.0.0.1") + if err := m.AddPeer("peer", ip, DefaultSettings()); err != nil { + t.Fatal(err) + } + + synctest.Wait() + + s := m.sessions["peer"] + + time.Sleep(15 * time.Second) + synctest.Wait() + + for range 3 { + s.receiveReply(&pb.HealthCheckPacket{TimestampNs: time.Now().Add(-17 * time.Millisecond).UnixNano()}) + } + + synctest.Wait() + + before := s.status() + if before.State != StateUp || before.PacketsSent != 1 || before.PacketsReceived != 3 || before.LastRTT != 17*time.Millisecond { + t.Fatalf("session must start healthy with measurements: %+v", before) + } + + for _, interval := range []time.Duration{60 * time.Second, 15 * time.Second, time.Second} { + settings := DefaultSettings() + + settings.TransmitInterval, settings.ReceiveInterval = interval, interval + if err := m.AddPeer("peer", ip, settings); err != nil { + t.Fatal(err) + } + + synctest.Wait() + + if m.sessions["peer"] != s { + t.Fatal("settings-only AddPeer replaced session") + } + + got, err := m.GetPeerSettings("peer") + if err != nil || got != settings { + t.Fatalf("applied settings: %+v, %v", got, err) + } + + after := s.status() + + after.RequiredReplies = before.RequiredReplies + if !reflect.DeepEqual(before, after) { + t.Fatalf("health history reset: before=%+v after=%+v", before, after) + } + + settings.MaxBackoff = 300 * time.Second + if err := m.UpdatePeerSettings("peer", settings); err != nil { + t.Fatal(err) + } + + synctest.Wait() + + after = s.status() + + after.RequiredReplies = before.RequiredReplies + if !reflect.DeepEqual(before, after) { + t.Fatal("UpdatePeerSettings reset health history") + } + } + }) +} + +func TestSessionSettingsWakeProbeTimer(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + settings := DefaultSettings() + settings.TransmitInterval, settings.ReceiveInterval = time.Hour, time.Hour + s := newSession(sessionConfig{settings: settings, conn: &discardProbeConn{}, probePhaseSeed: &intervalEndProbePhaseSeed}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s.start(ctx) + defer s.stop() + + synctest.Wait() + + settings.TransmitInterval = 20 * time.Millisecond + s.updateSettings(settings) + synctest.Wait() + time.Sleep(21 * time.Millisecond) + synctest.Wait() + + if s.status().PacketsSent != 1 { + t.Fatal("shortened interval waited for old one-hour timer") + } + + settings.TransmitInterval = time.Hour + s.updateSettings(settings) + synctest.Wait() + time.Sleep(time.Second) + synctest.Wait() + + if s.status().PacketsSent != 1 { + t.Fatal("lengthened interval allowed old fast probes") + } + + for _, interval := range []time.Duration{10 * time.Millisecond, time.Hour, 20 * time.Millisecond} { + settings.TransmitInterval = interval + s.updateSettings(settings) + } + + synctest.Wait() + time.Sleep(21 * time.Millisecond) + synctest.Wait() + + if s.status().PacketsSent != 2 { + t.Fatal("coalesced updates did not use latest interval") + } + + cancel() + s.stop() + count := s.status().PacketsSent + s.updateSettings(DefaultSettings()) + time.Sleep(time.Minute) + + if s.status().PacketsSent != count { + t.Fatal("stopped session resumed probes") + } + }) +} + +func TestSessionSettingsWakeDetectionTimer(t *testing.T) { + for _, increase := range []bool{false, true} { + t.Run(map[bool]string{false: "shorten", true: "lengthen"}[increase], func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + settings := DefaultSettings() + + settings.TransmitInterval, settings.ReceiveInterval = time.Hour, time.Hour + if increase { + settings.TransmitInterval, settings.ReceiveInterval = 20*time.Millisecond, 20*time.Millisecond + } + + s := newSession(sessionConfig{settings: settings, conn: &discardProbeConn{}, probePhaseSeed: &intervalEndProbePhaseSeed}) + s.state, s.lastReceived = StateUp, time.Now() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s.start(ctx) + defer s.stop() + + synctest.Wait() + + settings.TransmitInterval, settings.ReceiveInterval = 20*time.Millisecond, 20*time.Millisecond + if increase { + settings.TransmitInterval, settings.ReceiveInterval = time.Hour, time.Hour + } + + s.updateSettings(settings) + synctest.Wait() + time.Sleep(201 * time.Millisecond) + synctest.Wait() + + want := StateDown + if increase { + want = StateUp + } + + if got := s.status().State; got != want { + t.Fatalf("state=%v want %v after timer update", got, want) + } + }) + }) + } +} + +func TestSessionIdenticalSettingsKeepProbeDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + settings := DefaultSettings() + settings.TransmitInterval = 50 * time.Millisecond + s := newSession(sessionConfig{settings: settings, conn: &discardProbeConn{}}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s.start(ctx) + defer s.stop() + + synctest.Wait() + time.Sleep(20 * time.Millisecond) + s.updateSettings(settings) + synctest.Wait() + time.Sleep(31 * time.Millisecond) + synctest.Wait() + + if s.status().PacketsSent != 1 { + t.Fatal("identical settings reset probe deadline") + } + }) +} + +func TestManagerIPReplacementJoinsSessionOutsideLookupLock(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + + var ( + m *Manager + err error + ) + + m, err = NewManager("local", 0, func(string, SessionState, SessionState) { + close(entered) + <-release + m.GetAllPeerStatuses() + }) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + m.ctx, m.conn = ctx, &discardProbeConn{} + defer m.Stop() + + if err := m.AddPeer("peer", net.ParseIP("10.0.0.1"), DefaultSettings()); err != nil { + t.Fatal(err) + } + + old := m.sessions["peer"] + old.setState(StateUp) + <-entered + + done := make(chan error, 1) + go func() { done <- m.AddPeer("peer", net.ParseIP("10.0.0.2"), DefaultSettings()) }() + + synctest.Wait() + close(release) + + if err := <-done; err != nil { + t.Fatal(err) + } + + if m.sessions["peer"] == old || m.sessions["peer"].status().State != StateDown { + t.Fatal("IP replacement retained old session") + } + + synctest.Wait() + + before := old.status().PacketsSent + + time.Sleep(16 * time.Second) + synctest.Wait() + + if old.status().PacketsSent != before { + t.Fatal("old IP session still sends probes") + } + }) +} + +func TestManagerConcurrentSettingsAndStatus(t *testing.T) { + m, err := NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + + ip := net.ParseIP("10.0.0.1") + if err := m.AddPeer("peer", ip, DefaultSettings()); err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + for worker := range 4 { + wg.Go(func() { + for i := range 100 { + settings := DefaultSettings() + settings.TransmitInterval += time.Duration(i) * time.Millisecond + + var err error + + switch worker { + case 0: + err = m.AddPeer("peer", ip, settings) + case 1: + err = m.UpdatePeerSettings("peer", settings) + case 2: + _, err = m.GetPeerSettings("peer") + case 3: + _, err = m.GetPeerStatus("peer") + } + + if err != nil { + t.Errorf("concurrent operation: %v", err) + return + } + } + }) + } + + wg.Wait() +} + +func TestManagerRejectsInvalidSettings(t *testing.T) { + for name, change := range map[string]func(*HealthCheckSettings){ + "tx": func(s *HealthCheckSettings) { s.TransmitInterval = 0 }, + "rx": func(s *HealthCheckSettings) { s.ReceiveInterval = -1 }, + "multiplier": func(s *HealthCheckSettings) { s.DetectMultiplier = 0 }, + "large multiplier": func(s *HealthCheckSettings) { s.DetectMultiplier = 256 }, + "backoff": func(s *HealthCheckSettings) { s.MaxBackoff = -1 }, + "overflow": func(s *HealthCheckSettings) { s.ReceiveInterval = time.Duration(1<<63 - 1) }, + } { + t.Run(name, func(t *testing.T) { + m, err := NewManager("local", 0, nil) + if err != nil { + t.Fatal(err) + } + + settings := DefaultSettings() + change(&settings) + + if err := m.AddPeer("peer", net.ParseIP("10.0.0.1"), settings); err == nil { + t.Fatal("invalid settings accepted") + } + + if len(m.sessions) != 0 { + t.Fatal("invalid AddPeer mutated sessions") + } + + if err := m.AddPeer("peer", net.ParseIP("10.0.0.1"), DefaultSettings()); err != nil { + t.Fatal(err) + } + + if err := m.UpdatePeerSettings("peer", settings); err == nil { + t.Fatal("invalid update accepted") + } + + if got, _ := m.GetPeerSettings("peer"); got != DefaultSettings() { + t.Fatal("invalid update changed settings") + } + }) + } +} diff --git a/internal/net/healthcheck/session.go b/internal/net/healthcheck/session.go index 49b30e26a..3c5a6403b 100644 --- a/internal/net/healthcheck/session.go +++ b/internal/net/healthcheck/session.go @@ -31,16 +31,19 @@ const flapWindowDuration = 5 * time.Minute // It sends periodic probes and monitors for replies to determine // if the peer is up. type session struct { - peerHostname string - overlayIP net.IP - port int - localHost string + peerHostname string + overlayIP net.IP + port int + localHost string + probePhaseSeed uint64 mu sync.Mutex settings HealthCheckSettings + probeRevision uint64 state SessionState stateSince time.Time lastReceived time.Time + detectGraceUntil time.Time lastRTT time.Duration packetsSent uint64 packetsReceived uint64 @@ -49,11 +52,13 @@ type session struct { seqNum atomic.Uint64 onStateChange StateChangeFunc - conn net.PacketConn - cancel context.CancelFunc - callbackCh chan stateEvent - wg sync.WaitGroup - started bool + conn net.PacketConn + cancel context.CancelFunc + callbackCh chan stateEvent + probeSettingsCh chan struct{} + detectSettingsCh chan struct{} + wg sync.WaitGroup + started bool } // sessionConfig holds the parameters needed to create a new session. @@ -65,23 +70,33 @@ type sessionConfig struct { settings HealthCheckSettings onChange StateChangeFunc conn net.PacketConn + // Tests can inject the phase fraction, not an unchecked timer duration. + probePhaseSeed *uint64 } // newSession creates a new health check session for a remote peer. func newSession(cfg sessionConfig) *session { now := time.Now() + seed := peerProbePhaseSeed(cfg.localHostname, cfg.peerHostname) + if cfg.probePhaseSeed != nil { + seed = *cfg.probePhaseSeed + } + return &session{ - peerHostname: cfg.peerHostname, - overlayIP: cfg.overlayIP, - port: cfg.port, - localHost: cfg.localHostname, - settings: cfg.settings, - state: StateDown, - stateSince: now, - onStateChange: cfg.onChange, - conn: cfg.conn, - callbackCh: make(chan stateEvent, 8), + peerHostname: cfg.peerHostname, + overlayIP: cfg.overlayIP, + port: cfg.port, + localHost: cfg.localHostname, + probePhaseSeed: seed, + settings: cfg.settings, + state: StateDown, + stateSince: now, + onStateChange: cfg.onChange, + conn: cfg.conn, + callbackCh: make(chan stateEvent, 8), + probeSettingsCh: make(chan struct{}, 1), + detectSettingsCh: make(chan struct{}, 1), } } @@ -118,6 +133,7 @@ func (s *session) receiveReply(pkt *pb.HealthCheckPacket) { s.mu.Lock() s.lastReceived = now + s.detectGraceUntil = time.Time{} s.lastRTT = rtt s.packetsReceived++ @@ -158,9 +174,48 @@ func (s *session) status() *PeerStatus { // updateSettings applies new health check settings. func (s *session) updateSettings(settings HealthCheckSettings) { s.mu.Lock() - defer s.mu.Unlock() + if s.settings == settings { + s.mu.Unlock() + return + } + + oldTimeout := s.detectTimeout() + probeChanged := s.settings.TransmitInterval != settings.TransmitInterval s.settings = settings + if probeChanged { + s.probeRevision++ + } + + newTimeout := s.detectTimeout() + if s.state == StateUp && newTimeout < oldTimeout { + // A reply from the old cadence may already exceed the new timeout. + // Allow the first newly scheduled probe one nominal reply timeout. + // Receive-only changes retain the existing phase, at most one TX away. + firstProbe := settings.TransmitInterval + if probeChanged { + firstProbe = s.probePhase(settings.TransmitInterval) + } + + s.detectGraceUntil = time.Now().Add(firstProbe).Add(newTimeout) + } + s.mu.Unlock() + + // Coalesced wakeups read the latest settings. Receive/backoff changes do not + // disturb a running transmit phase. + if probeChanged { + select { + case s.probeSettingsCh <- struct{}{}: + default: + } + } + + if newTimeout != oldTimeout { + select { + case s.detectSettingsCh <- struct{}{}: + default: + } + } } func (s *session) probeLoop(ctx context.Context) { @@ -168,26 +223,68 @@ func (s *session) probeLoop(ctx context.Context) { s.mu.Lock() interval := s.settings.TransmitInterval + revision := s.probeRevision s.mu.Unlock() - ticker := time.NewTicker(interval) - defer ticker.Stop() + phaseTimer := time.NewTimer(s.probePhase(interval)) + defer phaseTimer.Stop() + + var ( + ticker *time.Ticker + ticks <-chan time.Time + ) + + defer func() { + if ticker != nil { + ticker.Stop() + } + }() + + resetPhase := func() bool { + s.mu.Lock() + newInterval := s.settings.TransmitInterval + newRevision := s.probeRevision + s.mu.Unlock() + + if newRevision == revision { + return false + } + + interval = newInterval + revision = newRevision + + if ticker != nil { + ticker.Stop() + ticker = nil + ticks = nil + } + + phaseTimer.Reset(s.probePhase(interval)) + + return true + } for { select { case <-ctx.Done(): return - case <-ticker.C: - s.sendProbe() - // Check if interval changed. - s.mu.Lock() - newInterval := s.settings.TransmitInterval - s.mu.Unlock() + case <-s.probeSettingsCh: + resetPhase() + case <-phaseTimer.C: + if resetPhase() { + continue + } + + ticker = time.NewTicker(interval) + ticks = ticker.C - if newInterval != interval { - interval = newInterval - ticker.Reset(interval) + s.sendProbe() + case <-ticks: + if resetPhase() { + continue } + + s.sendProbe() } } } @@ -211,15 +308,22 @@ func (s *session) detectLoop(ctx context.Context) { select { case <-ctx.Done(): return + case <-s.detectSettingsCh: + s.mu.Lock() + timeout = s.detectTimeout() + s.mu.Unlock() + + checkInterval = max(timeout/2, 100*time.Millisecond) + ticker.Reset(checkInterval) case <-ticker.C: s.mu.Lock() state := s.state - lastRecv := s.lastReceived timeout = s.detectTimeout() + expired := s.replyTimedOut(time.Now()) // Reset consecutive replies counter when we detect a timeout, // whether currently Up (transitioning to Down) or already Down // (stale counter from a partial reply burst). - if !lastRecv.IsZero() && time.Since(lastRecv) > timeout { + if expired { s.consecutiveReplies = 0 } s.mu.Unlock() @@ -228,9 +332,13 @@ func (s *session) detectLoop(ctx context.Context) { continue } - if state == StateUp && !lastRecv.IsZero() && time.Since(lastRecv) > timeout { - metricPacketsTimeout.Inc() - s.setState(StateDown) + if state == StateUp && expired { + // A reply or settings update may have arrived after the snapshot. + if s.setStateIf(StateDown, func() bool { + return s.state == StateUp && s.replyTimedOut(time.Now()) + }) { + metricPacketsTimeout.Inc() + } } // Update check interval if settings changed. @@ -247,6 +355,11 @@ func (s *session) detectLoop(ctx context.Context) { } } +// replyTimedOut requires s.mu and leaves the configured timeout unchanged. +func (s *session) replyTimedOut(now time.Time) bool { + return !s.lastReceived.IsZero() && now.Sub(s.lastReceived) > s.detectTimeout() && !now.Before(s.detectGraceUntil) +} + func (s *session) detectTimeout() time.Duration { tx := s.settings.TransmitInterval rx := s.settings.ReceiveInterval @@ -293,12 +406,16 @@ func (s *session) sendProbe() { } func (s *session) setState(newState SessionState) { + s.setStateIf(newState, nil) +} + +func (s *session) setStateIf(newState SessionState, ready func() bool) bool { s.mu.Lock() oldState := s.state - if oldState == newState { + if oldState == newState || (ready != nil && !ready()) { s.mu.Unlock() - return + return false } s.state = newState @@ -322,6 +439,8 @@ func (s *session) setState(newState SessionState) { klog.V(2).Infof("healthcheck: callback channel full for peer %s, dropping %s -> %s", s.peerHostname, oldState, newState) } + + return true } // trimFlapTimestamps removes flap timestamps older than flapWindowDuration. diff --git a/internal/net/healthcheck/types.go b/internal/net/healthcheck/types.go index 47d8c348d..4ebd30a6a 100644 --- a/internal/net/healthcheck/types.go +++ b/internal/net/healthcheck/types.go @@ -3,7 +3,10 @@ package healthcheck -import "time" +import ( + "fmt" + "time" +) // SessionState represents the health state of a peer session. type SessionState int @@ -42,13 +45,34 @@ type HealthCheckSettings struct { // DefaultSettings returns the default health check settings. func DefaultSettings() HealthCheckSettings { return HealthCheckSettings{ - TransmitInterval: 1000 * time.Millisecond, - ReceiveInterval: 1000 * time.Millisecond, + TransmitInterval: 15 * time.Second, + ReceiveInterval: 15 * time.Second, DetectMultiplier: 3, MaxBackoff: 120 * time.Second, } } +func (s HealthCheckSettings) validate() error { + if s.TransmitInterval <= 0 || s.ReceiveInterval <= 0 { + return fmt.Errorf("health check intervals must be positive") + } + + if s.DetectMultiplier < 1 || s.DetectMultiplier > 255 { + return fmt.Errorf("health check detect multiplier must be between 1 and 255") + } + + const maxDuration = time.Duration(1<<63 - 1) + if max(s.TransmitInterval, s.ReceiveInterval) > maxDuration/time.Duration(s.DetectMultiplier) { + return fmt.Errorf("health check detection timeout overflows time.Duration") + } + + if s.MaxBackoff < 0 { + return fmt.Errorf("health check maximum backoff must not be negative") + } + + return nil +} + // PeerStatus contains the current health status of a peer. type PeerStatus struct { State SessionState diff --git a/internal/net/status/measurements.go b/internal/net/status/measurements.go new file mode 100644 index 000000000..0a3607fc1 --- /dev/null +++ b/internal/net/status/measurements.go @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" + "maps" + "slices" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// PeerIdentityDigest guards snapshot indices against reordered or replaced peers. +// Length prefixes avoid ambiguous concatenations; duplicate identities are invalid. +func PeerIdentityDigest(peers []statusv1alpha1.PeerStatus) ([]byte, error) { + digest := sha256.New() + seen := make(map[[4]string]struct{}, len(peers)) + + var size [8]byte + + for _, peer := range peers { + identity := [4]string{peer.Name, peer.Tunnel.Protocol, peer.Tunnel.Interface, peer.Tunnel.PublicKey} + if peer.Name == "" { + return nil, fmt.Errorf("peer identity has no name") + } + + if _, exists := seen[identity]; exists { + return nil, fmt.Errorf("duplicate peer identity") + } + + seen[identity] = struct{}{} + for _, value := range identity { + binary.BigEndian.PutUint64(size[:], uint64(len(value))) + _, _ = digest.Write(size[:]) + _, _ = digest.Write([]byte(value)) + } + } + + return digest.Sum(nil), nil +} + +// PeerMetadataEqual excludes only measurements, not health state or topology. +func PeerMetadataEqual(a, b statusv1alpha1.PeerStatus) bool { + ah, bh := a.HealthCheck, b.HealthCheck + if a.Name != b.Name || a.PeerType != b.PeerType || a.SiteName != b.SiteName || + a.SkipPodCIDRRoutes != b.SkipPodCIDRRoutes || + !slices.Equal(a.PodCIDRGateways, b.PodCIDRGateways) || + !slices.Equal(a.RouteDestinations, b.RouteDestinations) || + !maps.Equal(a.RouteDistances, b.RouteDistances) || + a.Tunnel.Protocol != b.Tunnel.Protocol || a.Tunnel.Interface != b.Tunnel.Interface || + a.Tunnel.PublicKey != b.Tunnel.PublicKey || a.Tunnel.Endpoint != b.Tunnel.Endpoint || + !slices.Equal(a.Tunnel.AllowedIPs, b.Tunnel.AllowedIPs) { + return false + } + + if ah == nil || bh == nil { + return ah == bh + } + + return ah.Enabled == bh.Enabled && ah.Status == bh.Status +} + +// PeerMeasurementsToProto uses packed scalar columns instead of nested peer DTOs. +func PeerMeasurementsToProto(peers []statusv1alpha1.PeerStatus) (*statusproto.PeerMeasurements, error) { + digest, err := PeerIdentityDigest(peers) + if err != nil { + return nil, err + } + + count := len(peers) + + pb := &statusproto.PeerMeasurements{ + PeerCount: uint32(count), + IdentityDigest: digest, + RxBytes: make([]int64, count), + TxBytes: make([]int64, count), + LastHandshakeUnixNs: make([]int64, count), + Uptime: make([]string, count), + Rtt: make([]string, count), + } + for i, peer := range peers { + pb.RxBytes[i] = peer.Tunnel.RxBytes + + pb.TxBytes[i] = peer.Tunnel.TxBytes + if !peer.Tunnel.LastHandshake.IsZero() { + pb.LastHandshakeUnixNs[i] = peer.Tunnel.LastHandshake.UnixNano() + } + + if peer.HealthCheck != nil { + pb.Uptime[i] = peer.HealthCheck.Uptime + pb.Rtt[i] = peer.HealthCheck.RTT + } + } + + return pb, nil +} diff --git a/internal/net/status/measurements_test.go b/internal/net/status/measurements_test.go new file mode 100644 index 000000000..d286ee017 --- /dev/null +++ b/internal/net/status/measurements_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "bytes" + "testing" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestPeerIdentityDigest(t *testing.T) { + peer := statusv1alpha1.PeerStatus{Name: "a", Tunnel: statusv1alpha1.PeerTunnelStatus{Protocol: "bc", Interface: "d", PublicKey: "e"}} + + original, err := PeerIdentityDigest([]statusv1alpha1.PeerStatus{peer}) + if err != nil { + t.Fatal(err) + } + + ambiguous := peer + ambiguous.Name, ambiguous.Tunnel.Protocol = "ab", "c" + + digest, err := PeerIdentityDigest([]statusv1alpha1.PeerStatus{ambiguous}) + if err != nil || bytes.Equal(original, digest) { + t.Fatal("identity fields require unambiguous length prefixes") + } + + another := peer + + another.Tunnel.Interface = "other" + if _, err := PeerIdentityDigest([]statusv1alpha1.PeerStatus{peer, another}); err != nil { + t.Fatal("different links to the same named peer must remain distinguishable") + } + + if _, err := PeerIdentityDigest([]statusv1alpha1.PeerStatus{peer, peer}); err == nil { + t.Fatal("duplicate identity accepted") + } + + peer.Name = "" + if _, err := PeerMeasurementsToProto([]statusv1alpha1.PeerStatus{peer}); err == nil { + t.Fatal("unnamed identity accepted") + } + + empty, err := PeerMeasurementsToProto(nil) + if err != nil || empty.PeerCount != 0 || len(empty.IdentityDigest) != 32 { + t.Fatal("empty peer set must have a valid digest") + } +} + +func TestPeerMetadataEqualAllFields(t *testing.T) { + base := statusv1alpha1.PeerStatus{Name: "peer"} + + for field, change := range map[string]func(*statusv1alpha1.PeerStatus){ + "name": func(p *statusv1alpha1.PeerStatus) { p.Name = "other" }, + "type": func(p *statusv1alpha1.PeerStatus) { p.PeerType = "site" }, + "site": func(p *statusv1alpha1.PeerStatus) { p.SiteName = "site-a" }, + "gateway": func(p *statusv1alpha1.PeerStatus) { p.PodCIDRGateways = []string{"10.0.0.1"} }, + "skip": func(p *statusv1alpha1.PeerStatus) { p.SkipPodCIDRRoutes = true }, + "distances": func(p *statusv1alpha1.PeerStatus) { p.RouteDistances = map[string]int{"10.0.0.0/24": 1} }, + "destinations": func(p *statusv1alpha1.PeerStatus) { p.RouteDestinations = []string{"10.0.0.0/24"} }, + "protocol": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.Protocol = "GENEVE" }, + "interface": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.Interface = "geneve0" }, + "key": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.PublicKey = "key" }, + "endpoint": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.Endpoint = "10.0.0.1" }, + "allowedIPs": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.AllowedIPs = []string{"10.0.0.0/24"} }, + "health presence": func(p *statusv1alpha1.PeerStatus) { p.HealthCheck = &statusv1alpha1.HealthCheckPeerStatus{} }, + } { + t.Run(field, func(t *testing.T) { + changed := base + change(&changed) + + if PeerMetadataEqual(base, changed) || PeerMetadataEqual(changed, base) { + t.Fatal("metadata change ignored") + } + }) + } + + changed := base + changed.Tunnel.RxBytes, changed.Tunnel.TxBytes = 1, 2 + + changed.Tunnel.LastHandshake = time.Unix(123, 0) + if !PeerMetadataEqual(base, changed) { + t.Fatal("measurements are not metadata") + } + + base.HealthCheck = &statusv1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "up"} + + changed.HealthCheck = &statusv1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "up", Uptime: "1h", RTT: "5ms"} + if !PeerMetadataEqual(base, changed) { + t.Fatal("health measurements are not metadata") + } +} diff --git a/internal/net/status/proto/status.pb.go b/internal/net/status/proto/status.pb.go index da066505d..3330a065b 100644 --- a/internal/net/status/proto/status.pb.go +++ b/internal/net/status/proto/status.pb.go @@ -103,12 +103,13 @@ func (x *NodeStatusMessage) GetDelta() *NodeStatusDelta { // NodeStatusAck is the acknowledgment returned by the controller for push updates. type NodeStatusAck struct { - state protoimpl.MessageState `protogen:"open.v1"` - Status string `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` // "ok" or "resync_required" - Revision uint64 `protobuf:"varint,2,opt,name=revision,proto3" json:"revision,omitempty"` - Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Status string `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` // "ok" or "resync_required" + Revision uint64 `protobuf:"varint,2,opt,name=revision,proto3" json:"revision,omitempty"` + Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` + PeerMeasurements bool `protobuf:"varint,4,opt,name=peer_measurements,json=peerMeasurements,proto3" json:"peer_measurements,omitempty"` // Positive capability, scoped to this WebSocket connection. + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeStatusAck) Reset() { @@ -162,6 +163,13 @@ func (x *NodeStatusAck) GetReason() string { return "" } +func (x *NodeStatusAck) GetPeerMeasurements() bool { + if x != nil { + return x.PeerMeasurements + } + return false +} + // NodeStatusFull mirrors the complete NodeStatusResponse payload. type NodeStatusFull struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -296,10 +304,16 @@ type NodeStatusDelta struct { HealthCheck *HealthCheckStatus `protobuf:"bytes,4,opt,name=health_check,json=healthCheck,proto3" json:"health_check,omitempty"` NodeErrors []*NodeError `protobuf:"bytes,5,rep,name=node_errors,json=nodeErrors,proto3" json:"node_errors,omitempty"` // Track which top-level fields are present in this delta. - UpdatedFields []string `protobuf:"bytes,15,rep,name=updated_fields,json=updatedFields,proto3" json:"updated_fields,omitempty"` - BpfEntries []*BpfEntry `protobuf:"bytes,6,rep,name=bpf_entries,json=bpfEntries,proto3" json:"bpf_entries,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + UpdatedFields []string `protobuf:"bytes,15,rep,name=updated_fields,json=updatedFields,proto3" json:"updated_fields,omitempty"` + BpfEntries []*BpfEntry `protobuf:"bytes,6,rep,name=bpf_entries,json=bpfEntries,proto3" json:"bpf_entries,omitempty"` + PeerMeasurements *PeerMeasurements `protobuf:"bytes,7,opt,name=peer_measurements,json=peerMeasurements,proto3" json:"peer_measurements,omitempty"` + TimestampUnixNs int64 `protobuf:"varint,8,opt,name=timestamp_unix_ns,json=timestampUnixNs,proto3" json:"timestamp_unix_ns,omitempty"` + FetchError string `protobuf:"bytes,9,opt,name=fetch_error,json=fetchError,proto3" json:"fetch_error,omitempty"` + LastPushTimeUnixNs int64 `protobuf:"varint,10,opt,name=last_push_time_unix_ns,json=lastPushTimeUnixNs,proto3" json:"last_push_time_unix_ns,omitempty"` + StatusSource string `protobuf:"bytes,11,opt,name=status_source,json=statusSource,proto3" json:"status_source,omitempty"` + NodePodInfo *NodePodInfo `protobuf:"bytes,12,opt,name=node_pod_info,json=nodePodInfo,proto3" json:"node_pod_info,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeStatusDelta) Reset() { @@ -381,6 +395,145 @@ func (x *NodeStatusDelta) GetBpfEntries() []*BpfEntry { return nil } +func (x *NodeStatusDelta) GetPeerMeasurements() *PeerMeasurements { + if x != nil { + return x.PeerMeasurements + } + return nil +} + +func (x *NodeStatusDelta) GetTimestampUnixNs() int64 { + if x != nil { + return x.TimestampUnixNs + } + return 0 +} + +func (x *NodeStatusDelta) GetFetchError() string { + if x != nil { + return x.FetchError + } + return "" +} + +func (x *NodeStatusDelta) GetLastPushTimeUnixNs() int64 { + if x != nil { + return x.LastPushTimeUnixNs + } + return 0 +} + +func (x *NodeStatusDelta) GetStatusSource() string { + if x != nil { + return x.StatusSource + } + return "" +} + +func (x *NodeStatusDelta) GetNodePodInfo() *NodePodInfo { + if x != nil { + return x.NodePodInfo + } + return nil +} + +// PeerMeasurements replaces all measurement columns for the ordered base peers. +// Every column must have exactly peer_count entries, including zero/empty values. +// identity_digest is SHA-256 over the ordered, length-prefixed peer identities. +// A nonzero matching base_revision and matching digest are required. This field +// must be listed in updated_fields and cannot accompany a peers replacement. +type PeerMeasurements struct { + state protoimpl.MessageState `protogen:"open.v1"` + PeerCount uint32 `protobuf:"varint,1,opt,name=peer_count,json=peerCount,proto3" json:"peer_count,omitempty"` + IdentityDigest []byte `protobuf:"bytes,2,opt,name=identity_digest,json=identityDigest,proto3" json:"identity_digest,omitempty"` + RxBytes []int64 `protobuf:"varint,3,rep,packed,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` + TxBytes []int64 `protobuf:"varint,4,rep,packed,name=tx_bytes,json=txBytes,proto3" json:"tx_bytes,omitempty"` + LastHandshakeUnixNs []int64 `protobuf:"varint,5,rep,packed,name=last_handshake_unix_ns,json=lastHandshakeUnixNs,proto3" json:"last_handshake_unix_ns,omitempty"` + Uptime []string `protobuf:"bytes,6,rep,name=uptime,proto3" json:"uptime,omitempty"` + Rtt []string `protobuf:"bytes,7,rep,name=rtt,proto3" json:"rtt,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PeerMeasurements) Reset() { + *x = PeerMeasurements{} + mi := &file_status_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PeerMeasurements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PeerMeasurements) ProtoMessage() {} + +func (x *PeerMeasurements) ProtoReflect() protoreflect.Message { + mi := &file_status_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PeerMeasurements.ProtoReflect.Descriptor instead. +func (*PeerMeasurements) Descriptor() ([]byte, []int) { + return file_status_proto_rawDescGZIP(), []int{4} +} + +func (x *PeerMeasurements) GetPeerCount() uint32 { + if x != nil { + return x.PeerCount + } + return 0 +} + +func (x *PeerMeasurements) GetIdentityDigest() []byte { + if x != nil { + return x.IdentityDigest + } + return nil +} + +func (x *PeerMeasurements) GetRxBytes() []int64 { + if x != nil { + return x.RxBytes + } + return nil +} + +func (x *PeerMeasurements) GetTxBytes() []int64 { + if x != nil { + return x.TxBytes + } + return nil +} + +func (x *PeerMeasurements) GetLastHandshakeUnixNs() []int64 { + if x != nil { + return x.LastHandshakeUnixNs + } + return nil +} + +func (x *PeerMeasurements) GetUptime() []string { + if x != nil { + return x.Uptime + } + return nil +} + +func (x *PeerMeasurements) GetRtt() []string { + if x != nil { + return x.Rtt + } + return nil +} + // NodeInfo contains basic node identification and metadata. type NodeInfo struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -407,7 +560,7 @@ type NodeInfo struct { func (x *NodeInfo) Reset() { *x = NodeInfo{} - mi := &file_status_proto_msgTypes[4] + mi := &file_status_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -419,7 +572,7 @@ func (x *NodeInfo) String() string { func (*NodeInfo) ProtoMessage() {} func (x *NodeInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[4] + mi := &file_status_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -432,7 +585,7 @@ func (x *NodeInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use NodeInfo.ProtoReflect.Descriptor instead. func (*NodeInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{4} + return file_status_proto_rawDescGZIP(), []int{5} } func (x *NodeInfo) GetName() string { @@ -566,7 +719,7 @@ type BuildInfo struct { func (x *BuildInfo) Reset() { *x = BuildInfo{} - mi := &file_status_proto_msgTypes[5] + mi := &file_status_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -578,7 +731,7 @@ func (x *BuildInfo) String() string { func (*BuildInfo) ProtoMessage() {} func (x *BuildInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[5] + mi := &file_status_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -591,7 +744,7 @@ func (x *BuildInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use BuildInfo.ProtoReflect.Descriptor instead. func (*BuildInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{5} + return file_status_proto_rawDescGZIP(), []int{6} } func (x *BuildInfo) GetVersion() string { @@ -628,7 +781,7 @@ type WireGuardStatusInfo struct { func (x *WireGuardStatusInfo) Reset() { *x = WireGuardStatusInfo{} - mi := &file_status_proto_msgTypes[6] + mi := &file_status_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -640,7 +793,7 @@ func (x *WireGuardStatusInfo) String() string { func (*WireGuardStatusInfo) ProtoMessage() {} func (x *WireGuardStatusInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[6] + mi := &file_status_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -653,7 +806,7 @@ func (x *WireGuardStatusInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use WireGuardStatusInfo.ProtoReflect.Descriptor instead. func (*WireGuardStatusInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{6} + return file_status_proto_rawDescGZIP(), []int{7} } func (x *WireGuardStatusInfo) GetInterface() string { @@ -702,7 +855,7 @@ type PeerStatus struct { func (x *PeerStatus) Reset() { *x = PeerStatus{} - mi := &file_status_proto_msgTypes[7] + mi := &file_status_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -714,7 +867,7 @@ func (x *PeerStatus) String() string { func (*PeerStatus) ProtoMessage() {} func (x *PeerStatus) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[7] + mi := &file_status_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -727,7 +880,7 @@ func (x *PeerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use PeerStatus.ProtoReflect.Descriptor instead. func (*PeerStatus) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{7} + return file_status_proto_rawDescGZIP(), []int{8} } func (x *PeerStatus) GetName() string { @@ -810,7 +963,7 @@ type PeerTunnelStatus struct { func (x *PeerTunnelStatus) Reset() { *x = PeerTunnelStatus{} - mi := &file_status_proto_msgTypes[8] + mi := &file_status_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -822,7 +975,7 @@ func (x *PeerTunnelStatus) String() string { func (*PeerTunnelStatus) ProtoMessage() {} func (x *PeerTunnelStatus) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[8] + mi := &file_status_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -835,7 +988,7 @@ func (x *PeerTunnelStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use PeerTunnelStatus.ProtoReflect.Descriptor instead. func (*PeerTunnelStatus) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{8} + return file_status_proto_rawDescGZIP(), []int{9} } func (x *PeerTunnelStatus) GetProtocol() string { @@ -906,7 +1059,7 @@ type RoutingTableInfo struct { func (x *RoutingTableInfo) Reset() { *x = RoutingTableInfo{} - mi := &file_status_proto_msgTypes[9] + mi := &file_status_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -918,7 +1071,7 @@ func (x *RoutingTableInfo) String() string { func (*RoutingTableInfo) ProtoMessage() {} func (x *RoutingTableInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[9] + mi := &file_status_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -931,7 +1084,7 @@ func (x *RoutingTableInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use RoutingTableInfo.ProtoReflect.Descriptor instead. func (*RoutingTableInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{9} + return file_status_proto_rawDescGZIP(), []int{10} } func (x *RoutingTableInfo) GetRoutes() []*RouteEntry { @@ -968,7 +1121,7 @@ type RouteEntry struct { func (x *RouteEntry) Reset() { *x = RouteEntry{} - mi := &file_status_proto_msgTypes[10] + mi := &file_status_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -980,7 +1133,7 @@ func (x *RouteEntry) String() string { func (*RouteEntry) ProtoMessage() {} func (x *RouteEntry) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[10] + mi := &file_status_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -993,7 +1146,7 @@ func (x *RouteEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use RouteEntry.ProtoReflect.Descriptor instead. func (*RouteEntry) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{10} + return file_status_proto_rawDescGZIP(), []int{11} } func (x *RouteEntry) GetDestination() string { @@ -1043,7 +1196,7 @@ type NextHop struct { func (x *NextHop) Reset() { *x = NextHop{} - mi := &file_status_proto_msgTypes[11] + mi := &file_status_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1055,7 +1208,7 @@ func (x *NextHop) String() string { func (*NextHop) ProtoMessage() {} func (x *NextHop) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[11] + mi := &file_status_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1068,7 +1221,7 @@ func (x *NextHop) ProtoReflect() protoreflect.Message { // Deprecated: Use NextHop.ProtoReflect.Descriptor instead. func (*NextHop) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{11} + return file_status_proto_rawDescGZIP(), []int{12} } func (x *NextHop) GetGateway() string { @@ -1151,7 +1304,7 @@ type OptionalBool struct { func (x *OptionalBool) Reset() { *x = OptionalBool{} - mi := &file_status_proto_msgTypes[12] + mi := &file_status_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1163,7 +1316,7 @@ func (x *OptionalBool) String() string { func (*OptionalBool) ProtoMessage() {} func (x *OptionalBool) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[12] + mi := &file_status_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1176,7 +1329,7 @@ func (x *OptionalBool) ProtoReflect() protoreflect.Message { // Deprecated: Use OptionalBool.ProtoReflect.Descriptor instead. func (*OptionalBool) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{12} + return file_status_proto_rawDescGZIP(), []int{13} } func (x *OptionalBool) GetValue() bool { @@ -1198,7 +1351,7 @@ type NextHopInfo struct { func (x *NextHopInfo) Reset() { *x = NextHopInfo{} - mi := &file_status_proto_msgTypes[13] + mi := &file_status_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1210,7 +1363,7 @@ func (x *NextHopInfo) String() string { func (*NextHopInfo) ProtoMessage() {} func (x *NextHopInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[13] + mi := &file_status_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1223,7 +1376,7 @@ func (x *NextHopInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use NextHopInfo.ProtoReflect.Descriptor instead. func (*NextHopInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{13} + return file_status_proto_rawDescGZIP(), []int{14} } func (x *NextHopInfo) GetObjectName() string { @@ -1258,7 +1411,7 @@ type RouteType struct { func (x *RouteType) Reset() { *x = RouteType{} - mi := &file_status_proto_msgTypes[14] + mi := &file_status_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1270,7 +1423,7 @@ func (x *RouteType) String() string { func (*RouteType) ProtoMessage() {} func (x *RouteType) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[14] + mi := &file_status_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1283,7 +1436,7 @@ func (x *RouteType) ProtoReflect() protoreflect.Message { // Deprecated: Use RouteType.ProtoReflect.Descriptor instead. func (*RouteType) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{14} + return file_status_proto_rawDescGZIP(), []int{15} } func (x *RouteType) GetType() string { @@ -1313,7 +1466,7 @@ type HealthCheckStatus struct { func (x *HealthCheckStatus) Reset() { *x = HealthCheckStatus{} - mi := &file_status_proto_msgTypes[15] + mi := &file_status_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1325,7 +1478,7 @@ func (x *HealthCheckStatus) String() string { func (*HealthCheckStatus) ProtoMessage() {} func (x *HealthCheckStatus) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[15] + mi := &file_status_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1338,7 +1491,7 @@ func (x *HealthCheckStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use HealthCheckStatus.ProtoReflect.Descriptor instead. func (*HealthCheckStatus) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{15} + return file_status_proto_rawDescGZIP(), []int{16} } func (x *HealthCheckStatus) GetHealthy() bool { @@ -1382,7 +1535,7 @@ type HealthCheckPeerStatus struct { func (x *HealthCheckPeerStatus) Reset() { *x = HealthCheckPeerStatus{} - mi := &file_status_proto_msgTypes[16] + mi := &file_status_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1394,7 +1547,7 @@ func (x *HealthCheckPeerStatus) String() string { func (*HealthCheckPeerStatus) ProtoMessage() {} func (x *HealthCheckPeerStatus) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[16] + mi := &file_status_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1407,7 +1560,7 @@ func (x *HealthCheckPeerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use HealthCheckPeerStatus.ProtoReflect.Descriptor instead. func (*HealthCheckPeerStatus) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{16} + return file_status_proto_rawDescGZIP(), []int{17} } func (x *HealthCheckPeerStatus) GetEnabled() bool { @@ -1450,7 +1603,7 @@ type NodeError struct { func (x *NodeError) Reset() { *x = NodeError{} - mi := &file_status_proto_msgTypes[17] + mi := &file_status_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1462,7 +1615,7 @@ func (x *NodeError) String() string { func (*NodeError) ProtoMessage() {} func (x *NodeError) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[17] + mi := &file_status_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1475,7 +1628,7 @@ func (x *NodeError) ProtoReflect() protoreflect.Message { // Deprecated: Use NodeError.ProtoReflect.Descriptor instead. func (*NodeError) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{17} + return file_status_proto_rawDescGZIP(), []int{18} } func (x *NodeError) GetType() string { @@ -1511,7 +1664,7 @@ type NodePodInfo struct { func (x *NodePodInfo) Reset() { *x = NodePodInfo{} - mi := &file_status_proto_msgTypes[18] + mi := &file_status_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1523,7 +1676,7 @@ func (x *NodePodInfo) String() string { func (*NodePodInfo) ProtoMessage() {} func (x *NodePodInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[18] + mi := &file_status_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1536,7 +1689,7 @@ func (x *NodePodInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use NodePodInfo.ProtoReflect.Descriptor instead. func (*NodePodInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{18} + return file_status_proto_rawDescGZIP(), []int{19} } func (x *NodePodInfo) GetPodName() string { @@ -1578,7 +1731,7 @@ type BpfEntry struct { func (x *BpfEntry) Reset() { *x = BpfEntry{} - mi := &file_status_proto_msgTypes[19] + mi := &file_status_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1590,7 +1743,7 @@ func (x *BpfEntry) String() string { func (*BpfEntry) ProtoMessage() {} func (x *BpfEntry) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[19] + mi := &file_status_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1603,7 +1756,7 @@ func (x *BpfEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use BpfEntry.ProtoReflect.Descriptor instead. func (*BpfEntry) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{19} + return file_status_proto_rawDescGZIP(), []int{20} } func (x *BpfEntry) GetCidr() string { @@ -1679,11 +1832,12 @@ const file_status_proto_rawDesc = "" + "\tnode_name\x18\x02 \x01(\tR\bnodeName\x12#\n" + "\rbase_revision\x18\x03 \x01(\x04R\fbaseRevision\x12>\n" + "\x06status\x18\x04 \x01(\v2&.unboundednet.status.v1.NodeStatusFullR\x06status\x12=\n" + - "\x05delta\x18\x05 \x01(\v2'.unboundednet.status.v1.NodeStatusDeltaR\x05delta\"[\n" + + "\x05delta\x18\x05 \x01(\v2'.unboundednet.status.v1.NodeStatusDeltaR\x05delta\"\x88\x01\n" + "\rNodeStatusAck\x12\x16\n" + "\x06status\x18\x01 \x01(\tR\x06status\x12\x1a\n" + "\brevision\x18\x02 \x01(\x04R\brevision\x12\x16\n" + - "\x06reason\x18\x03 \x01(\tR\x06reason\"\x9c\x05\n" + + "\x06reason\x18\x03 \x01(\tR\x06reason\x12+\n" + + "\x11peer_measurements\x18\x04 \x01(\bR\x10peerMeasurements\"\x9c\x05\n" + "\x0eNodeStatusFull\x12*\n" + "\x11timestamp_unix_ns\x18\x01 \x01(\x03R\x0ftimestampUnixNs\x12=\n" + "\tnode_info\x18\x02 \x01(\v2 .unboundednet.status.v1.NodeInfoR\bnodeInfo\x128\n" + @@ -1699,7 +1853,7 @@ const file_status_proto_rawDesc = "" + "\rnode_pod_info\x18\n" + " \x01(\v2#.unboundednet.status.v1.NodePodInfoR\vnodePodInfo\x12A\n" + "\vbpf_entries\x18\v \x03(\v2 .unboundednet.status.v1.BpfEntryR\n" + - "bpfEntries\"\xd5\x03\n" + + "bpfEntries\"\x9b\x06\n" + "\x0fNodeStatusDelta\x12=\n" + "\tnode_info\x18\x01 \x01(\v2 .unboundednet.status.v1.NodeInfoR\bnodeInfo\x128\n" + "\x05peers\x18\x02 \x03(\v2\".unboundednet.status.v1.PeerStatusR\x05peers\x12M\n" + @@ -1709,7 +1863,24 @@ const file_status_proto_rawDesc = "" + "nodeErrors\x12%\n" + "\x0eupdated_fields\x18\x0f \x03(\tR\rupdatedFields\x12A\n" + "\vbpf_entries\x18\x06 \x03(\v2 .unboundednet.status.v1.BpfEntryR\n" + - "bpfEntries\"\xc5\x05\n" + + "bpfEntries\x12U\n" + + "\x11peer_measurements\x18\a \x01(\v2(.unboundednet.status.v1.PeerMeasurementsR\x10peerMeasurements\x12*\n" + + "\x11timestamp_unix_ns\x18\b \x01(\x03R\x0ftimestampUnixNs\x12\x1f\n" + + "\vfetch_error\x18\t \x01(\tR\n" + + "fetchError\x122\n" + + "\x16last_push_time_unix_ns\x18\n" + + " \x01(\x03R\x12lastPushTimeUnixNs\x12#\n" + + "\rstatus_source\x18\v \x01(\tR\fstatusSource\x12G\n" + + "\rnode_pod_info\x18\f \x01(\v2#.unboundednet.status.v1.NodePodInfoR\vnodePodInfo\"\xef\x01\n" + + "\x10PeerMeasurements\x12\x1d\n" + + "\n" + + "peer_count\x18\x01 \x01(\rR\tpeerCount\x12'\n" + + "\x0fidentity_digest\x18\x02 \x01(\fR\x0eidentityDigest\x12\x19\n" + + "\brx_bytes\x18\x03 \x03(\x03R\arxBytes\x12\x19\n" + + "\btx_bytes\x18\x04 \x03(\x03R\atxBytes\x123\n" + + "\x16last_handshake_unix_ns\x18\x05 \x03(\x03R\x13lastHandshakeUnixNs\x12\x16\n" + + "\x06uptime\x18\x06 \x03(\tR\x06uptime\x12\x10\n" + + "\x03rtt\x18\a \x03(\tR\x03rtt\"\xc5\x05\n" + "\bNodeInfo\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1b\n" + "\tsite_name\x18\x02 \x01(\tR\bsiteName\x12\x1d\n" + @@ -1854,64 +2025,67 @@ func file_status_proto_rawDescGZIP() []byte { return file_status_proto_rawDescData } -var file_status_proto_msgTypes = make([]protoimpl.MessageInfo, 22) +var file_status_proto_msgTypes = make([]protoimpl.MessageInfo, 23) var file_status_proto_goTypes = []any{ (*NodeStatusMessage)(nil), // 0: unboundednet.status.v1.NodeStatusMessage (*NodeStatusAck)(nil), // 1: unboundednet.status.v1.NodeStatusAck (*NodeStatusFull)(nil), // 2: unboundednet.status.v1.NodeStatusFull (*NodeStatusDelta)(nil), // 3: unboundednet.status.v1.NodeStatusDelta - (*NodeInfo)(nil), // 4: unboundednet.status.v1.NodeInfo - (*BuildInfo)(nil), // 5: unboundednet.status.v1.BuildInfo - (*WireGuardStatusInfo)(nil), // 6: unboundednet.status.v1.WireGuardStatusInfo - (*PeerStatus)(nil), // 7: unboundednet.status.v1.PeerStatus - (*PeerTunnelStatus)(nil), // 8: unboundednet.status.v1.PeerTunnelStatus - (*RoutingTableInfo)(nil), // 9: unboundednet.status.v1.RoutingTableInfo - (*RouteEntry)(nil), // 10: unboundednet.status.v1.RouteEntry - (*NextHop)(nil), // 11: unboundednet.status.v1.NextHop - (*OptionalBool)(nil), // 12: unboundednet.status.v1.OptionalBool - (*NextHopInfo)(nil), // 13: unboundednet.status.v1.NextHopInfo - (*RouteType)(nil), // 14: unboundednet.status.v1.RouteType - (*HealthCheckStatus)(nil), // 15: unboundednet.status.v1.HealthCheckStatus - (*HealthCheckPeerStatus)(nil), // 16: unboundednet.status.v1.HealthCheckPeerStatus - (*NodeError)(nil), // 17: unboundednet.status.v1.NodeError - (*NodePodInfo)(nil), // 18: unboundednet.status.v1.NodePodInfo - (*BpfEntry)(nil), // 19: unboundednet.status.v1.BpfEntry - nil, // 20: unboundednet.status.v1.NodeInfo.K8sLabelsEntry - nil, // 21: unboundednet.status.v1.PeerStatus.RouteDistancesEntry + (*PeerMeasurements)(nil), // 4: unboundednet.status.v1.PeerMeasurements + (*NodeInfo)(nil), // 5: unboundednet.status.v1.NodeInfo + (*BuildInfo)(nil), // 6: unboundednet.status.v1.BuildInfo + (*WireGuardStatusInfo)(nil), // 7: unboundednet.status.v1.WireGuardStatusInfo + (*PeerStatus)(nil), // 8: unboundednet.status.v1.PeerStatus + (*PeerTunnelStatus)(nil), // 9: unboundednet.status.v1.PeerTunnelStatus + (*RoutingTableInfo)(nil), // 10: unboundednet.status.v1.RoutingTableInfo + (*RouteEntry)(nil), // 11: unboundednet.status.v1.RouteEntry + (*NextHop)(nil), // 12: unboundednet.status.v1.NextHop + (*OptionalBool)(nil), // 13: unboundednet.status.v1.OptionalBool + (*NextHopInfo)(nil), // 14: unboundednet.status.v1.NextHopInfo + (*RouteType)(nil), // 15: unboundednet.status.v1.RouteType + (*HealthCheckStatus)(nil), // 16: unboundednet.status.v1.HealthCheckStatus + (*HealthCheckPeerStatus)(nil), // 17: unboundednet.status.v1.HealthCheckPeerStatus + (*NodeError)(nil), // 18: unboundednet.status.v1.NodeError + (*NodePodInfo)(nil), // 19: unboundednet.status.v1.NodePodInfo + (*BpfEntry)(nil), // 20: unboundednet.status.v1.BpfEntry + nil, // 21: unboundednet.status.v1.NodeInfo.K8sLabelsEntry + nil, // 22: unboundednet.status.v1.PeerStatus.RouteDistancesEntry } var file_status_proto_depIdxs = []int32{ 2, // 0: unboundednet.status.v1.NodeStatusMessage.status:type_name -> unboundednet.status.v1.NodeStatusFull 3, // 1: unboundednet.status.v1.NodeStatusMessage.delta:type_name -> unboundednet.status.v1.NodeStatusDelta - 4, // 2: unboundednet.status.v1.NodeStatusFull.node_info:type_name -> unboundednet.status.v1.NodeInfo - 7, // 3: unboundednet.status.v1.NodeStatusFull.peers:type_name -> unboundednet.status.v1.PeerStatus - 9, // 4: unboundednet.status.v1.NodeStatusFull.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo - 15, // 5: unboundednet.status.v1.NodeStatusFull.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus - 17, // 6: unboundednet.status.v1.NodeStatusFull.node_errors:type_name -> unboundednet.status.v1.NodeError - 18, // 7: unboundednet.status.v1.NodeStatusFull.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo - 19, // 8: unboundednet.status.v1.NodeStatusFull.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry - 4, // 9: unboundednet.status.v1.NodeStatusDelta.node_info:type_name -> unboundednet.status.v1.NodeInfo - 7, // 10: unboundednet.status.v1.NodeStatusDelta.peers:type_name -> unboundednet.status.v1.PeerStatus - 9, // 11: unboundednet.status.v1.NodeStatusDelta.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo - 15, // 12: unboundednet.status.v1.NodeStatusDelta.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus - 17, // 13: unboundednet.status.v1.NodeStatusDelta.node_errors:type_name -> unboundednet.status.v1.NodeError - 19, // 14: unboundednet.status.v1.NodeStatusDelta.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry - 5, // 15: unboundednet.status.v1.NodeInfo.build_info:type_name -> unboundednet.status.v1.BuildInfo - 6, // 16: unboundednet.status.v1.NodeInfo.wire_guard:type_name -> unboundednet.status.v1.WireGuardStatusInfo - 20, // 17: unboundednet.status.v1.NodeInfo.k8s_labels:type_name -> unboundednet.status.v1.NodeInfo.K8sLabelsEntry - 21, // 18: unboundednet.status.v1.PeerStatus.route_distances:type_name -> unboundednet.status.v1.PeerStatus.RouteDistancesEntry - 8, // 19: unboundednet.status.v1.PeerStatus.tunnel:type_name -> unboundednet.status.v1.PeerTunnelStatus - 16, // 20: unboundednet.status.v1.PeerStatus.health_check:type_name -> unboundednet.status.v1.HealthCheckPeerStatus - 10, // 21: unboundednet.status.v1.RoutingTableInfo.routes:type_name -> unboundednet.status.v1.RouteEntry - 11, // 22: unboundednet.status.v1.RouteEntry.next_hops:type_name -> unboundednet.status.v1.NextHop - 14, // 23: unboundednet.status.v1.NextHop.route_types:type_name -> unboundednet.status.v1.RouteType - 12, // 24: unboundednet.status.v1.NextHop.expected:type_name -> unboundednet.status.v1.OptionalBool - 12, // 25: unboundednet.status.v1.NextHop.present:type_name -> unboundednet.status.v1.OptionalBool - 13, // 26: unboundednet.status.v1.NextHop.info:type_name -> unboundednet.status.v1.NextHopInfo - 27, // [27:27] is the sub-list for method output_type - 27, // [27:27] is the sub-list for method input_type - 27, // [27:27] is the sub-list for extension type_name - 27, // [27:27] is the sub-list for extension extendee - 0, // [0:27] is the sub-list for field type_name + 5, // 2: unboundednet.status.v1.NodeStatusFull.node_info:type_name -> unboundednet.status.v1.NodeInfo + 8, // 3: unboundednet.status.v1.NodeStatusFull.peers:type_name -> unboundednet.status.v1.PeerStatus + 10, // 4: unboundednet.status.v1.NodeStatusFull.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo + 16, // 5: unboundednet.status.v1.NodeStatusFull.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 6: unboundednet.status.v1.NodeStatusFull.node_errors:type_name -> unboundednet.status.v1.NodeError + 19, // 7: unboundednet.status.v1.NodeStatusFull.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 20, // 8: unboundednet.status.v1.NodeStatusFull.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry + 5, // 9: unboundednet.status.v1.NodeStatusDelta.node_info:type_name -> unboundednet.status.v1.NodeInfo + 8, // 10: unboundednet.status.v1.NodeStatusDelta.peers:type_name -> unboundednet.status.v1.PeerStatus + 10, // 11: unboundednet.status.v1.NodeStatusDelta.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo + 16, // 12: unboundednet.status.v1.NodeStatusDelta.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 13: unboundednet.status.v1.NodeStatusDelta.node_errors:type_name -> unboundednet.status.v1.NodeError + 20, // 14: unboundednet.status.v1.NodeStatusDelta.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry + 4, // 15: unboundednet.status.v1.NodeStatusDelta.peer_measurements:type_name -> unboundednet.status.v1.PeerMeasurements + 19, // 16: unboundednet.status.v1.NodeStatusDelta.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 6, // 17: unboundednet.status.v1.NodeInfo.build_info:type_name -> unboundednet.status.v1.BuildInfo + 7, // 18: unboundednet.status.v1.NodeInfo.wire_guard:type_name -> unboundednet.status.v1.WireGuardStatusInfo + 21, // 19: unboundednet.status.v1.NodeInfo.k8s_labels:type_name -> unboundednet.status.v1.NodeInfo.K8sLabelsEntry + 22, // 20: unboundednet.status.v1.PeerStatus.route_distances:type_name -> unboundednet.status.v1.PeerStatus.RouteDistancesEntry + 9, // 21: unboundednet.status.v1.PeerStatus.tunnel:type_name -> unboundednet.status.v1.PeerTunnelStatus + 17, // 22: unboundednet.status.v1.PeerStatus.health_check:type_name -> unboundednet.status.v1.HealthCheckPeerStatus + 11, // 23: unboundednet.status.v1.RoutingTableInfo.routes:type_name -> unboundednet.status.v1.RouteEntry + 12, // 24: unboundednet.status.v1.RouteEntry.next_hops:type_name -> unboundednet.status.v1.NextHop + 15, // 25: unboundednet.status.v1.NextHop.route_types:type_name -> unboundednet.status.v1.RouteType + 13, // 26: unboundednet.status.v1.NextHop.expected:type_name -> unboundednet.status.v1.OptionalBool + 13, // 27: unboundednet.status.v1.NextHop.present:type_name -> unboundednet.status.v1.OptionalBool + 14, // 28: unboundednet.status.v1.NextHop.info:type_name -> unboundednet.status.v1.NextHopInfo + 29, // [29:29] is the sub-list for method output_type + 29, // [29:29] is the sub-list for method input_type + 29, // [29:29] is the sub-list for extension type_name + 29, // [29:29] is the sub-list for extension extendee + 0, // [0:29] is the sub-list for field type_name } func init() { file_status_proto_init() } @@ -1925,7 +2099,7 @@ func file_status_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_status_proto_rawDesc), len(file_status_proto_rawDesc)), NumEnums: 0, - NumMessages: 22, + NumMessages: 23, NumExtensions: 0, NumServices: 0, }, diff --git a/internal/net/status/proto/status.proto b/internal/net/status/proto/status.proto index 11b0d6015..98903e2d8 100644 --- a/internal/net/status/proto/status.proto +++ b/internal/net/status/proto/status.proto @@ -19,6 +19,7 @@ message NodeStatusAck { string status = 1; // "ok" or "resync_required" uint64 revision = 2; string reason = 3; + bool peer_measurements = 4; // Positive capability, scoped to this WebSocket connection. } // NodeStatusFull mirrors the complete NodeStatusResponse payload. @@ -46,6 +47,27 @@ message NodeStatusDelta { // Track which top-level fields are present in this delta. repeated string updated_fields = 15; repeated BpfEntry bpf_entries = 6; + PeerMeasurements peer_measurements = 7; + int64 timestamp_unix_ns = 8; + string fetch_error = 9; + int64 last_push_time_unix_ns = 10; + string status_source = 11; + NodePodInfo node_pod_info = 12; +} + +// PeerMeasurements replaces all measurement columns for the ordered base peers. +// Every column must have exactly peer_count entries, including zero/empty values. +// identity_digest is SHA-256 over the ordered, length-prefixed peer identities. +// A nonzero matching base_revision and matching digest are required. This field +// must be listed in updated_fields and cannot accompany a peers replacement. +message PeerMeasurements { + uint32 peer_count = 1; + bytes identity_digest = 2; + repeated int64 rx_bytes = 3; + repeated int64 tx_bytes = 4; + repeated int64 last_handshake_unix_ns = 5; + repeated string uptime = 6; + repeated string rtt = 7; } // NodeInfo contains basic node identification and metadata.