From f20b5bd3c12cf135e1ffe03195beac7a4afcb74e Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Wed, 16 Sep 2026 19:56:03 +0000 Subject: [PATCH 1/2] perf(net): reduce status transport and controller allocation overhead Use typed deltas and negotiated compact peer measurements with complete controller validation, connection-scoped acknowledgments, and legacy fallbacks. Decode WebSocket frames once, bound and reuse frame buffers and gzip writers, memoize validated peer identities, and avoid connectivity-matrix peer copies. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- .../cluster_status.go | 28 +- .../cni_status_test.go | 2 +- .../gzip_writer_pool.go | 59 +++ .../gzip_writer_pool_test.go | 476 ++++++++++++++++++ .../matrix_memory_test.go | 113 +++++ .../memory_bench_test.go | 104 ++++ cmd/unbounded-net-controller/metrics.go | 5 + cmd/unbounded-net-controller/node_status.go | 91 +++- .../peer_identity_cache_test.go | 457 +++++++++++++++++ .../peer_measurements.go | 117 +++++ .../peer_measurements_test.go | 444 ++++++++++++++++ .../proto_ws_identity_test.go | 181 +++++++ cmd/unbounded-net-controller/server.go | 40 +- cmd/unbounded-net-controller/status_proto.go | 116 +++-- .../status_proto_test.go | 43 +- .../ws_frame_buffer.go | 141 ++++++ .../ws_frame_buffer_test.go | 318 ++++++++++++ cmd/unbounded-net-node/status_ack.go | 64 +++ cmd/unbounded-net-node/status_delta.go | 193 +++++++ cmd/unbounded-net-node/status_delta_test.go | 438 ++++++++++++++++ .../status_legacy_delta_test.go | 117 +++++ cmd/unbounded-net-node/status_proto.go | 68 +-- cmd/unbounded-net-node/status_server.go | 195 +++---- docs/net/configuration.md | 51 +- docs/net/operations.md | 21 + internal/net/status/measurements.go | 100 ++++ internal/net/status/measurements_test.go | 94 ++++ internal/net/status/proto/status.pb.go | 398 ++++++++++----- internal/net/status/proto/status.proto | 22 + 29 files changed, 4117 insertions(+), 379 deletions(-) create mode 100644 cmd/unbounded-net-controller/gzip_writer_pool.go create mode 100644 cmd/unbounded-net-controller/gzip_writer_pool_test.go create mode 100644 cmd/unbounded-net-controller/matrix_memory_test.go create mode 100644 cmd/unbounded-net-controller/memory_bench_test.go create mode 100644 cmd/unbounded-net-controller/peer_identity_cache_test.go create mode 100644 cmd/unbounded-net-controller/peer_measurements.go create mode 100644 cmd/unbounded-net-controller/peer_measurements_test.go create mode 100644 cmd/unbounded-net-controller/proto_ws_identity_test.go create mode 100644 cmd/unbounded-net-controller/ws_frame_buffer.go create mode 100644 cmd/unbounded-net-controller/ws_frame_buffer_test.go create mode 100644 cmd/unbounded-net-node/status_ack.go create mode 100644 cmd/unbounded-net-node/status_delta.go create mode 100644 cmd/unbounded-net-node/status_delta_test.go create mode 100644 cmd/unbounded-net-node/status_legacy_delta_test.go create mode 100644 internal/net/status/measurements.go create mode 100644 internal/net/status/measurements_test.go diff --git a/cmd/unbounded-net-controller/cluster_status.go b/cmd/unbounded-net-controller/cluster_status.go index 1681435af..6185da8c8 100644 --- a/cmd/unbounded-net-controller/cluster_status.go +++ b/cmd/unbounded-net-controller/cluster_status.go @@ -1061,15 +1061,9 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway siteNodes[site][name] = true - var allPeers []WireGuardPeerStatus - - for _, p := range n.Peers { - if p.PeerType == "site" || p.PeerType == "gateway" { - allPeers = append(allPeers, p) - } - } - - nodePeers[name] = allPeers + // Keep the immutable snapshot's slice; filter when reading rather + // than copying every peer, including for scopes above the size limit. + nodePeers[name] = n.Peers for _, p := range n.Peers { if p.PeerType == "gateway" && p.Name != "" && p.SiteName == site { @@ -1111,6 +1105,10 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway } for _, peer := range nodePeers[srcNode] { + if !isConnectivityMatrixPeer(peer) { + continue + } + tgtNode := peer.Name if tgtNode == "" || tgtNode == srcNode || !nodeSet[tgtNode] { continue @@ -1153,6 +1151,10 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway poolNodeSet[name] = true for _, peer := range nodePeers[name] { + if !isConnectivityMatrixPeer(peer) { + continue + } + peerName := strings.TrimSpace(peer.Name) if peerName == "" { continue @@ -1165,6 +1167,10 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway for srcNodeName, peers := range nodePeers { for _, peer := range peers { + if !isConnectivityMatrixPeer(peer) { + continue + } + if strings.TrimSpace(peer.Name) == name { if _, ok := nodeByName[srcNodeName]; ok { poolNodeSet[srcNodeName] = true @@ -1188,3 +1194,7 @@ func buildConnectivityMatrix(nodes []*NodeStatusResponse, gatewayPools []Gateway return result } + +func isConnectivityMatrixPeer(peer WireGuardPeerStatus) bool { + return peer.PeerType == "site" || peer.PeerType == "gateway" +} diff --git a/cmd/unbounded-net-controller/cni_status_test.go b/cmd/unbounded-net-controller/cni_status_test.go index 044163a09..569a41c07 100644 --- a/cmd/unbounded-net-controller/cni_status_test.go +++ b/cmd/unbounded-net-controller/cni_status_test.go @@ -121,7 +121,7 @@ func TestCNIGuardProtoStatusRecovery(t *testing.T) { if source == "ws" || source == "apiserver-ws" { var messageType string - messageType, ack = handleProtoWSMessage(health, data, source) + messageType, ack = handleProtoWSBytes(health, data, source) if messageType != "node_status_ack" { t.Fatalf("unexpected message type %q: %+v", messageType, ack) } diff --git a/cmd/unbounded-net-controller/gzip_writer_pool.go b/cmd/unbounded-net-controller/gzip_writer_pool.go new file mode 100644 index 000000000..c0b90ba09 --- /dev/null +++ b/cmd/unbounded-net-controller/gzip_writer_pool.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "compress/gzip" + "io" +) + +const gzipIdleWriterLimit = 4 + +type gzipWriterPool struct { + idle chan *gzip.Writer +} + +func newGzipWriterPool() *gzipWriterPool { + return &gzipWriterPool{idle: make(chan *gzip.Writer, gzipIdleWriterLimit)} +} + +func (p *gzipWriterPool) get(output io.Writer) (*gzip.Writer, error) { + var writer *gzip.Writer + + select { + case writer = <-p.idle: + default: + var err error + + writer, err = gzip.NewWriterLevel(io.Discard, gzip.BestSpeed) + if err != nil { + return nil, err + } + } + + writer.Reset(output) + + return writer, nil +} + +func (p *gzipWriterPool) put(writer *gzip.Writer, completed bool) { + closed := false + + defer func() { + // Detach even if Close panics, and reset before another request can + // acquire the writer. Failed or interrupted responses are not pooled. + writer.Reset(io.Discard) + + if !completed || !closed { + return + } + + select { + case p.idle <- writer: + default: + } + }() + + closed = writer.Close() == nil +} diff --git a/cmd/unbounded-net-controller/gzip_writer_pool_test.go b/cmd/unbounded-net-controller/gzip_writer_pool_test.go new file mode 100644 index 000000000..0149c3be4 --- /dev/null +++ b/cmd/unbounded-net-controller/gzip_writer_pool_test.go @@ -0,0 +1,476 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// freshGzipHandler preserves the original handler for behavior and allocation +// comparisons against the reusable-writer path. +func freshGzipHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") || + strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + next.ServeHTTP(w, r) + return + } + + gz, err := gzip.NewWriterLevel(w, gzip.BestSpeed) + if err != nil { + next.ServeHTTP(w, r) + return + } + + defer func() { _ = gz.Close() }() + + w.Header().Set("Content-Encoding", "gzip") + w.Header().Del("Content-Length") + next.ServeHTTP(&gzipResponseWriter{ResponseWriter: w, Writer: gz}, r) + }) +} + +func TestGzipHandlerPreservesBehavior(t *testing.T) { + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Response", r.URL.Path) + + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + + switch r.URL.Path { + case "/empty": + w.WriteHeader(http.StatusAccepted) + case "/no-content": + w.WriteHeader(http.StatusNoContent) + case "/error": + http.Error(w, "not found", http.StatusNotFound) + default: + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusCreated) + + for _, chunk := range []string{"response:", r.URL.Path} { + if _, err := io.WriteString(w, chunk); err != nil { + t.Error(err) + } + } + } + }) + fresh, pooled := freshGzipHandler(next), gzipHandler(next) + + for _, encoding := range []string{"", "br", "gzip", "br, gzip", "gzip;q=0", "GZIP", "xgzip"} { + for _, upgrade := range []string{"", "WebSocket"} { + for _, path := range []string{"/first", "/empty", "/no-content", "/error", "/second"} { + t.Run(encoding+"/"+upgrade+path, func(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.Header.Set("Accept-Encoding", encoding) + request.Header.Set("Upgrade", upgrade) + + want, got := httptest.NewRecorder(), httptest.NewRecorder() + + for _, recorder := range []*httptest.ResponseRecorder{want, got} { + recorder.Header().Set("Content-Length", "123") + recorder.Header().Set("Content-Encoding", "original") + } + + fresh.ServeHTTP(want, request) + pooled.ServeHTTP(got, request) + + if got.Code != want.Code || got.Flushed != want.Flushed || + !reflect.DeepEqual(got.Header(), want.Header()) || !bytes.Equal(got.Body.Bytes(), want.Body.Bytes()) { + t.Fatalf("response behavior changed: status=%d/%d flush=%t/%t headers=%v/%v", got.Code, want.Code, got.Flushed, want.Flushed, got.Header(), want.Header()) + } + }) + } + } + } +} + +func TestGzipHandlerHTTPRoundTrips(t *testing.T) { + handler := gzipHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Response", r.URL.Path) + + if r.URL.Path == "/error" { + http.Error(w, "not found", http.StatusNotFound) + return + } + + if r.URL.Path == "/no-content" { + w.WriteHeader(http.StatusNoContent) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusCreated) + + if r.URL.Path != "/empty" { + if _, err := io.WriteString(w, strings.Repeat(r.URL.Path, 100)); err != nil { + t.Error(err) + } + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := server.Client() + client.Timeout = 10 * time.Second + + check := func(method, path string) { + t.Helper() + + request, err := http.NewRequestWithContext(t.Context(), method, server.URL+path, nil) + if err != nil { + t.Error(err) + return + } + + request.Header.Set("Accept-Encoding", "gzip") + + response, err := client.Do(request) + if err != nil { + t.Error(err) + return + } + defer response.Body.Close() + + var reader io.Reader = response.Body + + if method != http.MethodHead && path != "/no-content" { + compressed, err := gzip.NewReader(response.Body) + if err != nil { + t.Error(err) + return + } + defer compressed.Close() + + reader = compressed + } + + body, err := io.ReadAll(reader) + if err != nil { + t.Error(err) + return + } + + wantStatus, wantBody := http.StatusCreated, strings.Repeat(path, 100) + switch path { + case "/empty": + wantBody = "" + case "/no-content": + wantStatus, wantBody = http.StatusNoContent, "" + case "/error": + wantStatus, wantBody = http.StatusNotFound, "not found\n" + } + + if method == http.MethodHead { + wantBody = "" + } + + if response.StatusCode != wantStatus || string(body) != wantBody || + response.Header.Get("Content-Encoding") != "gzip" || response.Header.Get("X-Response") != path { + t.Errorf("round trip %s: status=%d body=%q headers=%v", path, response.StatusCode, body, response.Header) + } + } + + for _, path := range []string{"/first", "/different", "/empty", "/no-content", "/error", "/last"} { + check(http.MethodGet, path) + } + + check(http.MethodHead, "/head") + check(http.MethodGet, "/after-head") + + var workers sync.WaitGroup + + for worker := range 12 { + workers.Go(func() { + for request := range 4 { + check(http.MethodGet, fmt.Sprintf("/worker-%d-request-%d", worker, request)) + } + }) + } + + workers.Wait() +} + +func TestGzipWriterPoolBoundedAndDetached(t *testing.T) { + pool := newGzipWriterPool() + writers := make([]*gzip.Writer, gzipIdleWriterLimit+3) + outputs := make([]bytes.Buffer, len(writers)) + + for i := range writers { + writer, err := pool.get(&outputs[i]) + if err != nil { + t.Fatal(err) + } + + writers[i] = writer + + if _, err := io.WriteString(writer, strconv.Itoa(i)); err != nil { + t.Fatal(err) + } + } + + for _, writer := range writers { + pool.put(writer, true) + } + + if len(pool.idle) != gzipIdleWriterLimit { + t.Fatalf("idle pool length=%d, want %d", len(pool.idle), gzipIdleWriterLimit) + } + + // Take ownership of all idle writers before inspecting their detached + // outputs. Overflow writers are already outside the pool. + for len(pool.idle) > 0 { + <-pool.idle + } + + for i, writer := range writers { + before := outputs[i].Len() + + if _, err := io.WriteString(writer, "discard this"); err != nil { + t.Fatal(err) + } + + if err := writer.Close(); err != nil { + t.Fatal(err) + } + + if outputs[i].Len() != before { + t.Fatal("released writer retained its response destination") + } + } +} + +func TestGzipWriterPoolResetsHeadersAndSurvivesGC(t *testing.T) { + pool := newGzipWriterPool() + + var first, second bytes.Buffer + + writer, err := pool.get(&first) + if err != nil { + t.Fatal(err) + } + + writer.Name, writer.Comment, writer.Extra, writer.OS = "private-name", "private-comment", []byte("private-extra"), 7 + + if _, err := io.WriteString(writer, "first body"); err != nil { + t.Fatal(err) + } + + pool.put(writer, true) + runtime.GC() + runtime.GC() + + reused, err := pool.get(&second) + if err != nil { + t.Fatal(err) + } + + if reused != writer { + t.Fatal("idle writer was not reused") + } + + if _, err := io.WriteString(reused, "second body"); err != nil { + t.Fatal(err) + } + + pool.put(reused, true) + + reader, err := gzip.NewReader(&second) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + + body, err := io.ReadAll(reader) + if err != nil || string(body) != "second body" { + t.Fatalf("reused body: %q %v", body, err) + } + + if reader.Name != "" || reader.Comment != "" || len(reader.Extra) != 0 || reader.OS != 255 { + t.Fatalf("reused writer retained gzip headers: %+v", reader.Header) + } +} + +type gzipFailureResponseWriter struct { + *httptest.ResponseRecorder + failAt int + panicAt int + writes int +} + +func (w *gzipFailureResponseWriter) Write(data []byte) (int, error) { + w.writes++ + if w.writes == w.panicAt { + panic("gzip test output panic") + } + + if w.writes == w.failAt { + return 0, io.ErrClosedPipe + } + + return w.ResponseRecorder.Write(data) +} + +func invokeGzipHandler(handler http.Handler, writer http.ResponseWriter, request *http.Request) (panicValue any) { + defer func() { panicValue = recover() }() + + handler.ServeHTTP(writer, request) + + return nil +} + +func TestGzipHandlerWriteFailuresAndPanics(t *testing.T) { + for _, test := range []struct { + name string + failAt int + panicAt int + handlerPanic bool + beforeWrite bool + }{ + {name: "write failure", failAt: 1}, + {name: "close failure", failAt: 2}, + {name: "close panic", panicAt: 2}, + {name: "handler panic", handlerPanic: true}, + {name: "handler panic before write", handlerPanic: true, beforeWrite: true}, + } { + t.Run(test.name, func(t *testing.T) { + var used *gzip.Writer + + var writeErr error + + handler := gzipHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + used = w.(*gzipResponseWriter).Writer.(*gzip.Writer) + + if test.beforeWrite && r.URL.Path == "/failure" { + panic("gzip test handler panic") + } + + _, writeErr = io.WriteString(w, r.URL.Path) + + if test.handlerPanic && r.URL.Path == "/failure" { + panic("gzip test handler panic") + } + })) + failed := &gzipFailureResponseWriter{ + ResponseRecorder: httptest.NewRecorder(), failAt: test.failAt, panicAt: test.panicAt, + } + request := httptest.NewRequest(http.MethodGet, "/failure", nil) + request.Header.Set("Accept-Encoding", "gzip") + panicValue := invokeGzipHandler(handler, failed, request) + + wantPanic := "" + if test.panicAt != 0 { + wantPanic = "gzip test output panic" + } else if test.handlerPanic { + wantPanic = "gzip test handler panic" + } + + if wantPanic != "" && panicValue != wantPanic || wantPanic == "" && panicValue != nil { + t.Fatalf("panic behavior changed: got %v, want %q", panicValue, wantPanic) + } + + if test.failAt == 1 && !errors.Is(writeErr, io.ErrClosedPipe) { + t.Fatalf("write error was suppressed: %v", writeErr) + } + + oldWriter, oldWrites := used, failed.writes + if _, err := io.WriteString(oldWriter, "discard this"); err != nil { + t.Fatalf("failed writer was not reset: %v", err) + } + + if err := oldWriter.Close(); err != nil { + t.Fatal(err) + } + + if failed.writes != oldWrites { + t.Fatal("failed or panicking writer retained its response destination") + } + + recorder := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodGet, "/healthy", nil) + request.Header.Set("Accept-Encoding", "gzip") + handler.ServeHTTP(recorder, request) + + if used == oldWriter || writeErr != nil { + t.Fatal("failed or panicking writer was returned to the idle pool") + } + + reader, err := gzip.NewReader(recorder.Body) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + + body, err := io.ReadAll(reader) + if err != nil || string(body) != "/healthy" { + t.Fatalf("failure contaminated the next response: %q %v", body, err) + } + }) + } +} + +func BenchmarkGzipResponseHandler(b *testing.B) { + for _, response := range []struct { + name string + status int + body string + }{ + { + name: "discovery", status: http.StatusOK, + body: `{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"status.net.unbounded-cloud.io/v1alpha1","resources":[{"name":"status/push","singularName":"","namespaced":false,"kind":"NodeStatusPush","verbs":["create"]},{"name":"status/nodews","singularName":"","namespaced":false,"kind":"NodeStatusStream","verbs":["get"]},{"name":"status/json","singularName":"","namespaced":false,"kind":"ClusterStatus","verbs":["get"]},{"name":"token/node","singularName":"","namespaced":false,"kind":"TokenRequest","verbs":["create"]},{"name":"token/viewer","singularName":"","namespaced":false,"kind":"TokenRequest","verbs":["create"]}]}`, + }, + {name: "not-found", status: http.StatusNotFound, body: "404 page not found\n"}, + {name: "empty", status: http.StatusOK}, + } { + for _, mode := range []struct { + name string + wrap func(http.Handler) http.Handler + }{ + {name: "fresh", wrap: freshGzipHandler}, + {name: "pooled", wrap: gzipHandler}, + } { + b.Run(response.name+"/"+mode.name, func(b *testing.B) { + body := []byte(response.body) + handler := mode.wrap(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(response.status) + + if _, err := w.Write(body); err != nil { + b.Fatal(err) + } + })) + request := httptest.NewRequest(http.MethodGet, "/", nil) + request.Header.Set("Accept-Encoding", "gzip") + handler.ServeHTTP(httptest.NewRecorder(), request) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + + if recorder.Code != response.status { + b.Fatal(recorder.Code) + } + } + }) + } + } +} diff --git a/cmd/unbounded-net-controller/matrix_memory_test.go b/cmd/unbounded-net-controller/matrix_memory_test.go new file mode 100644 index 000000000..e7ed3f048 --- /dev/null +++ b/cmd/unbounded-net-controller/matrix_memory_test.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "slices" + "testing" + "time" +) + +func TestConnectivityMatrixMixedScopesPreservesPeers(t *testing.T) { + nodes := make([]*NodeStatusResponse, 0, 102) + for i := range 101 { + nodes = append(nodes, &NodeStatusResponse{NodeInfo: NodeInfo{ + Name: fmt.Sprintf("node-%d", i), SiteName: "large", + }}) + } + + nodes[0].Peers = []WireGuardPeerStatus{{ + Name: "gateway", PeerType: "gateway", SiteName: "small", + Tunnel: PeerTunnelStatus{LastHandshake: time.Unix(1, 0)}, + }} + nodes[1].Peers = []WireGuardPeerStatus{{Name: "gateway", PeerType: "ignored"}} + nodes = append(nodes, &NodeStatusResponse{ + NodeInfo: NodeInfo{Name: "gateway", SiteName: "small"}, + Peers: []WireGuardPeerStatus{ + {Name: "node-0", PeerType: "site", HealthCheck: &HealthCheckPeerStatus{Status: "up"}}, + {Name: "node-0", PeerType: "ignored", HealthCheck: &HealthCheckPeerStatus{Status: "down"}}, + {Name: "node-2", PeerType: "ignored"}, + {Name: "gateway", PeerType: "ignored", HealthCheck: &HealthCheckPeerStatus{Status: "down"}}, + }, + }) + + before, err := json.Marshal(nodes) + if err != nil { + t.Fatal(err) + } + + matrix := buildConnectivityMatrix(nodes, []GatewayPoolStatus{{Name: "pool", Gateways: []string{"gateway"}}}) + if _, ok := matrix["large"]; ok { + t.Fatal("oversized site produced a matrix") + } + + if matrix["small"] == nil || !slices.Equal(matrix["small"].Nodes, []string{"gateway"}) { + t.Fatalf("small site lost its matrix: %+v", matrix["small"]) + } + + pool := matrix["pool:pool"] + if pool == nil || !slices.Equal(pool.Nodes, []string{"gateway", "node-0"}) { + t.Fatalf("small pool crossing a large site has wrong membership: %+v", pool) + } + + if pool.Results["gateway"]["node-0"] != "up" || pool.Results["node-0"]["gateway"] != "up" { + t.Fatalf("pool connectivity changed: %+v", pool.Results) + } + + after, err := json.Marshal(nodes) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(before, after) { + t.Fatal("matrix construction mutated the shared node snapshots") + } +} + +func TestConnectivityMatrixDoesNotCopyPeerSlices(t *testing.T) { + nodes := matrixBenchmarkNodes(200) + + peers := matrixBenchmarkNodes(2000)[0].Peers + for _, node := range nodes { + node.Peers = peers + } + + var matrix map[string]*SiteMatrix + + result := testing.Benchmark(func(b *testing.B) { + for b.Loop() { + matrix = buildConnectivityMatrix(nodes, nil) + } + }) + + if matrix != nil { + t.Fatal("oversized site produced a matrix") + } + // Allow map bookkeeping, but not storage proportional to every peer. + if allocated := result.AllocedBytesPerOp(); allocated > 512*1024 { + t.Fatalf("matrix copied peer data: %d bytes per call", allocated) + } +} + +func TestConnectivityMatrixSizeBoundary(t *testing.T) { + for _, count := range []int{0, 100, 101} { + t.Run(fmt.Sprintf("nodes-%d", count), func(t *testing.T) { + matrix := buildConnectivityMatrix(matrixBenchmarkNodes(count), nil) + if count != 100 { + if matrix != nil { + t.Fatal("empty or oversized site produced a matrix") + } + + return + } + + if matrix["site-a"] == nil || len(matrix["site-a"].Nodes) != count { + t.Fatal("site at the size limit lost its matrix") + } + }) + } +} diff --git a/cmd/unbounded-net-controller/memory_bench_test.go b/cmd/unbounded-net-controller/memory_bench_test.go new file mode 100644 index 000000000..ca548d0a6 --- /dev/null +++ b/cmd/unbounded-net-controller/memory_bench_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "testing" + + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func benchmarkProtoStatus(b *testing.B, peerCount int) []byte { + b.Helper() + + status := &statusproto.NodeStatusFull{ + NodeInfo: &statusproto.NodeInfo{Name: "node-a", SiteName: "site-a"}, + } + + for i := range peerCount { + name := fmt.Sprintf("peer-%d", i) + status.Peers = append(status.Peers, &statusproto.PeerStatus{ + Name: name, PeerType: "site", SiteName: "site-a", + Tunnel: &statusproto.PeerTunnelStatus{ + Interface: "wg0", PublicKey: name, + Endpoint: "10.224.0.1:51820", AllowedIps: []string{"10.244.0.0/24"}, + }, + HealthCheck: &statusproto.HealthCheckPeerStatus{Enabled: true, Status: "up"}, + }) + status.BpfEntries = append(status.BpfEntries, &statusproto.BpfEntry{ + Cidr: "10.244.0.0/24", Remote: "10.224.0.1", Node: name, + InterfaceName: "wg0", Protocol: "WireGuard", + }) + } + + data, err := proto.Marshal(&statusproto.NodeStatusMessage{ + Type: "node_status_full", NodeName: "node-a", Status: status, + }) + if err != nil { + b.Fatal(err) + } + + return data +} + +func BenchmarkProtoWSStatusFrame(b *testing.B) { + for _, peerCount := range []int{100, 2000} { + b.Run(fmt.Sprintf("peers-%d", peerCount), func(b *testing.B) { + data := benchmarkProtoStatus(b, peerCount) + health := &healthState{statusCache: NewNodeStatusCache()} + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + decoded, err := decodeProtoWSMessage(data) + if err != nil || decoded.nodeName != "node-a" { + b.Fatalf("unexpected decoded message: %+v, err=%v", decoded, err) + } + + _, ack := handleProtoWSMessage(health, decoded, "ws") + if ack.Status != "ok" { + b.Fatalf("unexpected ack: %+v", ack) + } + } + }) + } +} + +func matrixBenchmarkNodes(count int) []*NodeStatusResponse { + peers := make([]WireGuardPeerStatus, count) + + nodes := make([]*NodeStatusResponse, count) + for i := range count { + name := fmt.Sprintf("node-%d", i) + peers[i] = WireGuardPeerStatus{Name: name, PeerType: "site", SiteName: "site-a"} + nodes[i] = &NodeStatusResponse{ + NodeInfo: NodeInfo{Name: name, SiteName: "site-a"}, + Peers: peers, + } + } + + return nodes +} + +func BenchmarkBuildConnectivityMatrix(b *testing.B) { + for _, count := range []int{100, 101, 2000} { + b.Run(fmt.Sprintf("nodes-%d", count), func(b *testing.B) { + nodes := matrixBenchmarkNodes(count) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + matrix := buildConnectivityMatrix(nodes, nil) + if count > 100 && matrix != nil { + b.Fatal("oversized site produced a matrix") + } + } + }) + } +} diff --git a/cmd/unbounded-net-controller/metrics.go b/cmd/unbounded-net-controller/metrics.go index cc5f7999d..c05469630 100644 --- a/cmd/unbounded-net-controller/metrics.go +++ b/cmd/unbounded-net-controller/metrics.go @@ -14,6 +14,11 @@ const controllerMetricsNamespace = "unbounded_cni_controller" // Status/push metrics. var ( + peerMeasurementUpdatesTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: controllerMetricsNamespace, + Name: "peer_measurement_updates_total", + Help: "Compact peer measurement batches by outcome (applied, resync, error).", + }, []string{"outcome"}) nodeStatusPushesTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: controllerMetricsNamespace, Name: "node_status_pushes_total", diff --git a/cmd/unbounded-net-controller/node_status.go b/cmd/unbounded-net-controller/node_status.go index 3f7317184..7fb9b09b1 100644 --- a/cmd/unbounded-net-controller/node_status.go +++ b/cmd/unbounded-net-controller/node_status.go @@ -10,6 +10,8 @@ import ( "net/http" "sync" "time" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" ) // CachedNodeStatus stores a node's pushed status with timestamp and revision. @@ -18,6 +20,8 @@ type CachedNodeStatus struct { ReceivedAt time.Time Source string Revision uint64 + + peerIdentity *peerIdentityDigest } // NodeStatusCache is a thread-safe cache of node status data pushed from node agents. @@ -73,17 +77,19 @@ func (c *NodeStatusCache) StoreFull(nodeName string, status NodeStatusResponse, // parsedDelta holds pre-deserialized delta fields parsed outside the lock. type parsedDelta struct { - timestamp *time.Time - nodeInfo *NodeInfo - peers []WireGuardPeerStatus - routingTable *RoutingTableInfo - healthCheck *HealthCheckStatus - nodeErrors []NodeError - fetchError *string - lastPushTime *time.Time - statusSource *string - nodePodInfo *NodePodInfo - bpfEntries []BpfEntry + peerMeasurements *statusproto.PeerMeasurements + parseError error + timestamp *time.Time + nodeInfo *NodeInfo + peers []WireGuardPeerStatus + routingTable *RoutingTableInfo + healthCheck *HealthCheckStatus + nodeErrors []NodeError + fetchError *string + lastPushTime *time.Time + statusSource *string + nodePodInfo *NodePodInfo + bpfEntries []BpfEntry // nullFields tracks fields explicitly set to null for clearing. nullFields map[string]bool @@ -193,13 +199,28 @@ func (c *NodeStatusCache) ApplyParsedDelta(nodeName string, baseRevision uint64, source = "push" } - return c.applyParsedDelta(nodeName, baseRevision, pd, source) + rev, conflict, err := c.applyParsedDelta(nodeName, baseRevision, pd, source) + if pd.peerMeasurements != nil || pd.parseError != nil { + outcome := "applied" + if err != nil { + outcome = "error" + } else if conflict { + outcome = "resync" + } + + peerMeasurementUpdatesTotal.WithLabelValues(outcome).Inc() + } + + return rev, conflict, err } // applyParsedDelta merges pre-parsed delta fields into a cached node status // under the lock. Both ApplyDelta (JSON) and ApplyParsedDelta (protobuf) // converge here. func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, pd parsedDelta, source string) (uint64, bool, error) { + if pd.parseError != nil { + return 0, false, pd.parseError + } // Phase 1: Read entry under lock, copy it, release lock. c.mu.RLock() @@ -209,7 +230,7 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, return 0, true, nil } - if baseRevision != 0 && entry.Revision != baseRevision { + if (pd.peerMeasurements != nil && baseRevision == 0) || (baseRevision != 0 && entry.Revision != baseRevision) { rev := entry.Revision c.mu.RUnlock() @@ -219,6 +240,7 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, // Snapshot values we need under lock prevStatus := entry.Status prevRevision := entry.Revision + peerIdentity := entry.peerIdentity c.mu.RUnlock() @@ -226,6 +248,20 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, // part (~1MB copy per node) and must not block other goroutines. merged := *prevStatus + if pd.peerMeasurements != nil { + if pd.peers != nil || pd.nullFields["peers"] { + return prevRevision, false, fmt.Errorf("peer replacement conflicts with measurements") + } + + peers, identity, err := applyPeerMeasurementsWithIdentity(prevStatus.Peers, pd.peerMeasurements, peerIdentity) + if err != nil { + return prevRevision, false, err + } + + merged.Peers = peers + peerIdentity = identity + } + if pd.timestamp != nil { merged.Timestamp = *pd.timestamp } else if pd.nullFields["timestamp"] { @@ -240,10 +276,16 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, if pd.peers != nil { merged.Peers = pd.peers + peerIdentity = nil + } else if pd.nullFields["peers"] { + merged.Peers = nil + peerIdentity = nil } if pd.routingTable != nil { merged.RoutingTable = *pd.routingTable + } else if pd.nullFields["routingTable"] { + merged.RoutingTable = RoutingTableInfo{} } if pd.healthCheck != nil { @@ -292,16 +334,21 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, merged.NodeInfo.Name = nodeName } + return c.commitParsedDelta(nodeName, entry, &merged, peerIdentity, source) +} + +func (c *NodeStatusCache) commitParsedDelta(nodeName string, previous *CachedNodeStatus, merged *NodeStatusResponse, peerIdentity *peerIdentityDigest, source string) (uint64, bool, error) { // Phase 3: Write lock for the brief pointer swap. c.mu.Lock() - // Re-check entry still exists and revision hasn't changed - entry, ok = c.entries[nodeName] + // Deletion and recreation can reuse a revision. The identity memo and + // merged status must still belong to the same entry, not just its number. + entry, ok := c.entries[nodeName] if !ok { c.mu.Unlock() return 0, true, nil } - if entry.Revision != prevRevision { + if entry != previous { // Another goroutine updated this node while we were merging. // Our merge is stale; signal resync. rev := entry.Revision @@ -312,10 +359,11 @@ func (c *NodeStatusCache) applyParsedDelta(nodeName string, baseRevision uint64, revision := entry.Revision + 1 c.entries[nodeName] = &CachedNodeStatus{ - Status: &merged, - ReceivedAt: time.Now(), - Source: source, - Revision: revision, + Status: merged, + ReceivedAt: time.Now(), + Source: source, + Revision: revision, + peerIdentity: peerIdentity, } fn := c.onChange mergedPtr := c.entries[nodeName].Status @@ -336,7 +384,8 @@ func (c *NodeStatusCache) SetOnChange(fn func(nodeName string, status *NodeStatu c.onChange = fn } -// Get returns a copy of the cached status for a node when present. +// Get returns shallow copies of the cached entry and status when present. +// Nested slices and maps remain shared. func (c *NodeStatusCache) Get(nodeName string) (*CachedNodeStatus, bool) { c.mu.RLock() defer c.mu.RUnlock() diff --git a/cmd/unbounded-net-controller/peer_identity_cache_test.go b/cmd/unbounded-net-controller/peer_identity_cache_test.go new file mode 100644 index 000000000..ad2c3634d --- /dev/null +++ b/cmd/unbounded-net-controller/peer_identity_cache_test.go @@ -0,0 +1,457 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" + "sync" + "testing" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func TestPeerIdentityHashMatchesProtocol(t *testing.T) { + for _, count := range []int{0, 1, 2000} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + peers := protoToNodeStatus(measurementTestStatus(count)).Peers + if count > 0 { + peers[0].Name = "peer\x00雪" + peers[0].Tunnel.Protocol = "" + peers[0].Tunnel.Interface = "ab" + peers[0].Tunnel.PublicKey = "c" + } + + digest, err := netstatus.PeerIdentityDigest(peers) + if err != nil { + t.Fatal(err) + } + + got := hashPeerIdentities(peers) + if !bytes.Equal(got[:], digest) { + t.Fatal("cached identity encoding differs from the wire protocol") + } + + if count > 0 { + peers[0].Tunnel.Interface = "a" + peers[0].Tunnel.PublicKey = "bc" + + if hashPeerIdentities(peers) == got { + t.Fatal("identity boundaries are ambiguous") + } + } + }) + } +} + +func warmPeerIdentityCache(t *testing.T, cache *NodeStatusCache) *CachedNodeStatus { + t.Helper() + + entry, ok := cache.Get("node-a") + if !ok { + t.Fatal("missing base") + } + + message := measurementMessage(t, *entry.Status, entry.Revision) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "ok" { + t.Fatalf("compact update: %+v", ack) + } + + result, ok := cache.Get("node-a") + if !ok || result.peerIdentity == nil { + t.Fatal("successful compact update did not memoize identity validation") + } + + return result +} + +func TestPeerIdentityCacheReuse(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(2)) + cache := NewNodeStatusCache() + cache.StoreFull("node-a", status, "ws") + first := warmPeerIdentityCache(t, cache) + + before, err := json.Marshal(first.Status) + if err != nil { + t.Fatal(err) + } + + // The compact copy owns its peer value fields, independently of StoreFull's + // original caller. It still shares immutable static maps and slices. + status.Peers[0].Name = "changed original input" + + for range 3 { + next := warmPeerIdentityCache(t, cache) + if next.peerIdentity != first.peerIdentity { + t.Fatal("unchanged identities rebuilt their validation memo") + } + } + + _, conflict, err := cache.ApplyDelta("node-a", 0, map[string]json.RawMessage{"fetchError": []byte(`"unavailable"`)}, "push") + if err != nil || conflict { + t.Fatalf("non-peer delta: %t %v", conflict, err) + } + + cache.UpdateSource("node-a", "ws") + + next := warmPeerIdentityCache(t, cache) + if next.peerIdentity != first.peerIdentity { + t.Fatal("non-peer update invalidated identity validation") + } + + after, err := json.Marshal(first.Status) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("later updates mutated a retained snapshot") + } + + allocations := testing.AllocsPerRun(10, func() { + identity, err := validatePeerIdentity(next.Status.Peers, next.peerIdentity) + if err != nil || identity != next.peerIdentity { + t.Fatalf("reuse: %v", err) + } + }) + if allocations != 0 { + t.Fatalf("identity reuse allocated %g objects", allocations) + } +} + +func TestPeerIdentityCacheInvalidatesReplacementPaths(t *testing.T) { + changes := map[string]func(*statusproto.NodeStatusFull){ + "identical": func(_ *statusproto.NodeStatusFull) {}, + "metadata": func(s *statusproto.NodeStatusFull) { s.Peers[0].SiteName = "another-site" }, + "renamed": func(s *statusproto.NodeStatusFull) { s.Peers[0].Name = "another-peer" }, + "added": func(s *statusproto.NodeStatusFull) { s.Peers = measurementTestStatus(3).Peers }, + "removed": func(s *statusproto.NodeStatusFull) { s.Peers = s.Peers[:1] }, + "reordered": func(s *statusproto.NodeStatusFull) { + s.Peers[0], s.Peers[1] = s.Peers[1], s.Peers[0] + }, + "duplicate": func(s *statusproto.NodeStatusFull) { s.Peers[1] = s.Peers[0] }, + "unnamed": func(s *statusproto.NodeStatusFull) { s.Peers[0].Name = "" }, + "nil": func(s *statusproto.NodeStatusFull) { s.Peers = nil }, + "empty": func(s *statusproto.NodeStatusFull) { s.Peers = []*statusproto.PeerStatus{} }, + } + for _, path := range []string{"StoreFull", "protobuf full", "protobuf delta", "JSON delta"} { + for name, change := range changes { + t.Run(path+"/"+name, func(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "ws") + first := warmPeerIdentityCache(t, cache) + full := measurementTestStatus(2) + change(full) + + status := protoToNodeStatus(full) + if full.Peers != nil && len(full.Peers) == 0 { + status.Peers = []WireGuardPeerStatus{} + } + + var conflict bool + + var err error + + switch path { + case "StoreFull": + cache.StoreFull("node-a", status, "push") + case "protobuf full": + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, &statusproto.NodeStatusMessage{ + Type: "node_status_full", NodeName: "node-a", Status: full, + }) + if ack.Status != "ok" { + t.Fatal(ack) + } + case "protobuf delta": + delta := protoToParsedDelta(&statusproto.NodeStatusDelta{UpdatedFields: []string{"peers"}, Peers: full.Peers}) + _, conflict, err = cache.ApplyParsedDelta("node-a", first.Revision, delta, "ws") + case "JSON delta": + var raw []byte + + raw, err = json.Marshal(status.Peers) + if err == nil { + _, conflict, err = cache.ApplyDelta("node-a", first.Revision, map[string]json.RawMessage{"peers": raw}, "push") + } + } + + if err != nil || conflict { + t.Fatalf("replacement changed legacy behavior: %t %v", conflict, err) + } + + replaced := cache.entries["node-a"] + if replaced.peerIdentity != nil { + t.Fatal("peer replacement retained the old identity memo") + } + + if name == "duplicate" || name == "unnamed" { + message := measurementMessage(t, *first.Status, replaced.Revision) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "resync_required" || cache.entries["node-a"] != replaced { + t.Fatal("invalid replacement accepted compact measurements or changed the cache") + } + + return + } + + if name != "identical" && name != "metadata" { + message := measurementMessage(t, *first.Status, replaced.Revision) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "resync_required" || cache.entries["node-a"] != replaced { + t.Fatal("replacement accepted old-topology measurements") + } + } + + next := warmPeerIdentityCache(t, cache) + if next.peerIdentity == first.peerIdentity { + t.Fatal("replacement did not independently validate its identities") + } + }) + } + } +} + +func TestPeerIdentityCacheDetectsBorrowedIdentityChanges(t *testing.T) { + changes := map[string]func([]WireGuardPeerStatus){ + "name": func(p []WireGuardPeerStatus) { p[0].Name = "another-peer" }, + "protocol": func(p []WireGuardPeerStatus) { p[0].Tunnel.Protocol = "another-protocol" }, + "interface": func(p []WireGuardPeerStatus) { p[0].Tunnel.Interface = "another-interface" }, + "key": func(p []WireGuardPeerStatus) { p[0].Tunnel.PublicKey = "another-key" }, + "reorder": func(p []WireGuardPeerStatus) { p[0], p[1] = p[1], p[0] }, + "duplicate": func(p []WireGuardPeerStatus) { p[1] = p[0] }, + "unnamed": func(p []WireGuardPeerStatus) { p[0].Name = "" }, + } + for name, change := range changes { + t.Run(name, func(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "ws") + first := warmPeerIdentityCache(t, cache) + message := measurementMessage(t, *first.Status, first.Revision) + digest := *first.peerIdentity + change(first.Status.Peers) + + before, err := json.Marshal(first.Status) + if err != nil { + t.Fatal(err) + } + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + + after, marshalErr := json.Marshal(cache.entries["node-a"].Status) + if ack.Status != "resync_required" || marshalErr != nil || !bytes.Equal(before, after) { + t.Fatal("borrowed identity mutation used a stale memo or partially updated the cache") + } + + if *first.peerIdentity != digest || cache.entries["node-a"].Revision != first.Revision { + t.Fatal("rejected update mutated the old memo or revision") + } + + if name != "duplicate" && name != "unnamed" { + next := warmPeerIdentityCache(t, cache) + if next.peerIdentity == first.peerIdentity { + t.Fatal("changed borrowed identity did not force revalidation") + } + } + }) + } +} + +func TestPeerMeasurementsCachedValidationMatchesDirect(t *testing.T) { + changes := map[string]func([]WireGuardPeerStatus, *statusproto.PeerMeasurements){ + "valid": func(_ []WireGuardPeerStatus, _ *statusproto.PeerMeasurements) {}, + "count": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.PeerCount++ }, + "rx": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.RxBytes = nil }, + "tx": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.TxBytes = nil }, + "handshake": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.LastHandshakeUnixNs = nil }, + "uptime": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.Uptime = nil }, + "rtt": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.Rtt = nil }, + "digest": func(_ []WireGuardPeerStatus, m *statusproto.PeerMeasurements) { m.IdentityDigest[0] ^= 1 }, + "duplicate": func(p []WireGuardPeerStatus, _ *statusproto.PeerMeasurements) { p[1] = p[0] }, + "unnamed": func(p []WireGuardPeerStatus, _ *statusproto.PeerMeasurements) { p[0].Name = "" }, + "health": func(p []WireGuardPeerStatus, _ *statusproto.PeerMeasurements) { p[0].HealthCheck = nil }, + } + for name, change := range changes { + t.Run(name, func(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(2)) + message := measurementMessage(t, status, 1) + m := message.Delta.PeerMeasurements + + identity, err := validatePeerIdentity(status.Peers, nil) + if err != nil { + t.Fatal(err) + } + + change(status.Peers, m) + direct, directErr := applyPeerMeasurements(status.Peers, m) + + cached, _, cachedErr := applyPeerMeasurementsWithIdentity(status.Peers, m, identity) + if fmt.Sprint(directErr) != fmt.Sprint(cachedErr) || !reflect.DeepEqual(direct, cached) { + t.Fatalf("direct and memoized validation disagree: %v / %v", directErr, cachedErr) + } + + if name != "valid" && directErr == nil { + t.Fatal("invalid measurements were accepted") + } + }) + } +} + +func TestPeerIdentityCacheRevisionAndRemoval(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "ws") + first := warmPeerIdentityCache(t, cache) + fetchError := "must not apply" + delta := parsedDelta{peerMeasurements: &statusproto.PeerMeasurements{}, fetchError: &fetchError} + + for _, revision := range []uint64{0, first.Revision - 1, first.Revision + 1} { + _, conflict, err := cache.ApplyParsedDelta("node-a", revision, delta, "ws") + if !conflict || err != nil { + t.Fatalf("revision %d did not precede malformed identity/column validation: %t %v", revision, conflict, err) + } + } + + if current := cache.entries["node-a"]; current.Revision != first.Revision || current.peerIdentity != first.peerIdentity || current.Status.FetchError != "" { + t.Fatal("revision conflict changed the cache") + } + + for _, remove := range []func(){func() { cache.Delete("node-a") }, func() { cache.CleanupStaleEntries(nil) }} { + remove() + + _, conflict, err := cache.ApplyParsedDelta("node-a", first.Revision, delta, "ws") + if !conflict || err != nil || cache.Len() != 0 { + t.Fatal("missing entry did not request resync before validation") + } + + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "push") + + if cache.entries["node-a"].peerIdentity != nil { + t.Fatal("recreated cache entry inherited an identity memo") + } + + warmPeerIdentityCache(t, cache) + } +} + +func TestPeerIdentityCacheConcurrentUpdates(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2000)), "ws") + first := warmPeerIdentityCache(t, cache) + delta := protoToParsedDelta(measurementMessage(t, *first.Status, first.Revision).Delta) + start := make(chan struct{}) + results := make(chan bool, 8) + + var workers sync.WaitGroup + + for range cap(results) { + workers.Go(func() { + <-start + + _, conflict, err := cache.ApplyParsedDelta("node-a", first.Revision, delta, "ws") + if err != nil { + t.Errorf("concurrent update: %v", err) + } + + results <- !conflict && err == nil + }) + } + + close(start) + workers.Wait() + close(results) + + applied := 0 + + for success := range results { + if success { + applied++ + } + } + + if current := cache.entries["node-a"]; applied != 1 || current.Revision != first.Revision+1 || current.peerIdentity != first.peerIdentity { + t.Fatalf("concurrent update lost revision or identity invariants: successes=%d", applied) + } +} + +func TestPeerIdentityCacheRejectsCommitAfterRecreation(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", protoToNodeStatus(measurementTestStatus(2)), "ws") + warmPeerIdentityCache(t, cache) + previous := cache.entries["node-a"] + merged := *previous.Status + merged.FetchError = "must not apply" + + cache.Delete("node-a") + + replacement := protoToNodeStatus(measurementTestStatus(1)) + for range previous.Revision { + cache.StoreFull("node-a", replacement, "push") + } + + current := cache.entries["node-a"] + if current.Revision != previous.Revision { + t.Fatal("test requires a recreated entry with the same revision") + } + + called := false + + cache.SetOnChange(func(_ string, _ *NodeStatusResponse) { called = true }) + + revision, conflict, err := cache.commitParsedDelta("node-a", previous, &merged, previous.peerIdentity, "ws") + if err != nil || !conflict || revision != current.Revision || called { + t.Fatalf("stale commit after recreation: revision=%d conflict=%t err=%v callback=%t", revision, conflict, err, called) + } + + if cache.entries["node-a"] != current || current.peerIdentity != nil { + t.Fatal("stale commit overwrote the recreated entry or restored its old memo") + } +} + +func TestPeerIdentityValidationRejectsDuplicatesAfterMemo(t *testing.T) { + peers := protoToNodeStatus(measurementTestStatus(2)).Peers + + identity, err := validatePeerIdentity(peers, nil) + if err != nil { + t.Fatal(err) + } + + peers[1] = peers[0] + + _, err = validatePeerIdentity(peers, identity) + if err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatalf("memo skipped duplicate validation: %v", err) + } +} + +func BenchmarkCompactIdentityCache2000(b *testing.B) { + status := protoToNodeStatus(measurementTestStatus(2000)) + + measurements, err := netstatus.PeerMeasurementsToProto(status.Peers) + if err != nil { + b.Fatal(err) + } + + cache := NewNodeStatusCache() + revision := cache.StoreFull("node-a", status, "ws") + delta := parsedDelta{peerMeasurements: measurements} + + revision, conflict, err := cache.ApplyParsedDelta("node-a", revision, delta, "ws") + if err != nil || conflict { + b.Fatalf("warm cache: conflict=%t err=%v", conflict, err) + } + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + next, conflict, err := cache.ApplyParsedDelta("node-a", revision, delta, "ws") + if err != nil || conflict { + b.Fatalf("apply: conflict=%t err=%v", conflict, err) + } + + revision = next + } +} diff --git a/cmd/unbounded-net-controller/peer_measurements.go b/cmd/unbounded-net-controller/peer_measurements.go new file mode 100644 index 000000000..6b4fba93d --- /dev/null +++ b/cmd/unbounded-net-controller/peer_measurements.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "fmt" + "time" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// peerIdentityDigest records successful name and uniqueness validation. It is +// immutable and bounded to one digest per cached peer topology. +type peerIdentityDigest [sha256.Size]byte + +// hashPeerIdentities uses the same ordered, length-prefixed identity encoding as +// status.PeerIdentityDigest, without constructing its duplicate-detection map. +func hashPeerIdentities(peers []WireGuardPeerStatus) peerIdentityDigest { + digest := sha256.New() + + var size [8]byte + + for _, peer := range peers { + for _, value := range [4]string{peer.Name, peer.Tunnel.Protocol, peer.Tunnel.Interface, peer.Tunnel.PublicKey} { + binary.BigEndian.PutUint64(size[:], uint64(len(value))) + _, _ = digest.Write(size[:]) + _, _ = digest.Write([]byte(value)) + } + } + + var result peerIdentityDigest + + digest.Sum(result[:0]) + + return result +} + +func validatePeerIdentity(peers []WireGuardPeerStatus, previous *peerIdentityDigest) (*peerIdentityDigest, error) { + // StoreFull and Get share nested slices with callers. Re-hashing preserves + // that ownership contract: even an out-of-band identity change must not use + // a stale uniqueness result. A matching SHA-256 avoids rebuilding the map. + if previous != nil && hashPeerIdentities(peers) == *previous { + return previous, nil + } + + digest, err := netstatus.PeerIdentityDigest(peers) + if err != nil { + return nil, err + } + + result := peerIdentityDigest(digest) + + return &result, nil +} + +// applyPeerMeasurements validates every column and identity before creating a +// replacement slice. Static maps/slices remain shared and immutable. +func applyPeerMeasurements(peers []WireGuardPeerStatus, pb *statusproto.PeerMeasurements) ([]WireGuardPeerStatus, error) { + result, _, err := applyPeerMeasurementsWithIdentity(peers, pb, nil) + + return result, err +} + +func applyPeerMeasurementsWithIdentity(peers []WireGuardPeerStatus, pb *statusproto.PeerMeasurements, previous *peerIdentityDigest) ([]WireGuardPeerStatus, *peerIdentityDigest, error) { + count := len(peers) + if uint64(pb.PeerCount) != uint64(count) || + len(pb.RxBytes) != count || len(pb.TxBytes) != count || + len(pb.LastHandshakeUnixNs) != count || len(pb.Uptime) != count || len(pb.Rtt) != count { + return nil, nil, fmt.Errorf("peer measurement column length mismatch") + } + + digest, err := validatePeerIdentity(peers, previous) + if err != nil { + return nil, nil, err + } + + if !bytes.Equal(digest[:], pb.IdentityDigest) { + return nil, nil, fmt.Errorf("peer measurement identity mismatch") + } + + for i, peer := range peers { + if peer.HealthCheck == nil && (pb.Uptime[i] != "" || pb.Rtt[i] != "") { + return nil, nil, fmt.Errorf("peer measurements require existing health metadata") + } + } + + result := make([]WireGuardPeerStatus, count) + copy(result, peers) + + health := make([]statusv1alpha1.HealthCheckPeerStatus, count) + + for i := range result { + peer := &result[i] + peer.Tunnel.RxBytes = pb.RxBytes[i] + peer.Tunnel.TxBytes = pb.TxBytes[i] + + peer.Tunnel.LastHandshake = time.Time{} + if pb.LastHandshakeUnixNs[i] != 0 { + peer.Tunnel.LastHandshake = time.Unix(0, pb.LastHandshakeUnixNs[i]) + } + + if peer.HealthCheck != nil { + health[i] = *peer.HealthCheck + health[i].Uptime = pb.Uptime[i] + health[i].RTT = pb.Rtt[i] + peer.HealthCheck = &health[i] + } + } + + return result, digest, nil +} diff --git a/cmd/unbounded-net-controller/peer_measurements_test.go b/cmd/unbounded-net-controller/peer_measurements_test.go new file mode 100644 index 000000000..5953ae1ad --- /dev/null +++ b/cmd/unbounded-net-controller/peer_measurements_test.go @@ -0,0 +1,444 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "google.golang.org/protobuf/proto" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func measurementTestStatus(count int) *statusproto.NodeStatusFull { + status := &statusproto.NodeStatusFull{NodeInfo: &statusproto.NodeInfo{Name: "node-a"}} + for i := range count { + status.Peers = append(status.Peers, &statusproto.PeerStatus{ + Name: fmt.Sprintf("peer-%d", i), PeerType: "site", SiteName: "site-a", + PodCidrGateways: []string{"10.244.0.1"}, SkipPodCidrRoutes: true, + RouteDistances: map[string]int32{"10.244.0.0/24": 1}, RouteDestinations: []string{"10.244.0.0/24"}, + Tunnel: &statusproto.PeerTunnelStatus{ + Interface: "wg0", Protocol: "wireguard", PublicKey: fmt.Sprintf("key-%d", i), + Endpoint: "10.224.0.1:51820", AllowedIps: []string{"10.244.0.0/24"}, + RxBytes: 100, TxBytes: 200, LastHandshakeUnixNs: 123456789, + }, + HealthCheck: &statusproto.HealthCheckPeerStatus{Enabled: true, Status: "up", Uptime: "1m", Rtt: "1ms"}, + }) + } + + return status +} + +func measurementMessage(t *testing.T, status NodeStatusResponse, rev uint64) *statusproto.NodeStatusMessage { + t.Helper() + + pb, err := netstatus.PeerMeasurementsToProto(status.Peers) + if err != nil { + t.Fatal(err) + } + + return &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: rev, + Delta: &statusproto.NodeStatusDelta{UpdatedFields: []string{"peerMeasurements"}, PeerMeasurements: pb}, + } +} + +func applyMeasurementMessage(t *testing.T, health *healthState, message *statusproto.NodeStatusMessage) NodeStatusPushAck { + t.Helper() + + data, err := proto.Marshal(message) + if err != nil { + t.Fatal(err) + } + + decoded, err := decodeProtoWSMessage(data) + if err != nil { + t.Fatal(err) + } + + _, ack := handleProtoWSMessage(health, decoded, "ws") + + return ack +} + +func TestPeerMeasurementsCacheOwnsDecodedFrameData(t *testing.T) { + cache := NewNodeStatusCache() + health := &healthState{statusCache: cache} + + var buffer []byte + + applyReusedFrame := func(message *statusproto.NodeStatusMessage) NodeStatusPushAck { + t.Helper() + + var err error + + buffer, err = proto.MarshalOptions{}.MarshalAppend(buffer[:0], message) + if err != nil { + t.Fatal(err) + } + + decoded, err := decodeProtoWSMessage(buffer) + if err != nil { + t.Fatal(err) + } + + _, ack := handleProtoWSMessage(health, decoded, "ws") + if ack.Status != "ok" { + t.Fatalf("apply reused frame: %+v", ack) + } + + for i := range buffer { + buffer[i] = 0xa5 + } + + return ack + } + + full := measurementTestStatus(2) + wantFull := protoToNodeStatus(full) + ack := applyReusedFrame(&statusproto.NodeStatusMessage{ + Type: "node_status_full", NodeName: "node-a", Status: full, + }) + oldSnapshot := cache.entries["node-a"].Status + + if !reflect.DeepEqual(*oldSnapshot, wantFull) { + t.Fatal("full status retained overwritten frame data") + } + + message := measurementMessage(t, wantFull, ack.Revision) + measurements := message.Delta.PeerMeasurements + wantNext := protoToNodeStatus(proto.Clone(full).(*statusproto.NodeStatusFull)) + + for i := range wantNext.Peers { + measurements.RxBytes[i], measurements.TxBytes[i] = 321, 654 + measurements.LastHandshakeUnixNs[i] = 987654321 + measurements.Uptime[i], measurements.Rtt[i] = "2h", "7ms" + wantNext.Peers[i].Tunnel.RxBytes, wantNext.Peers[i].Tunnel.TxBytes = 321, 654 + wantNext.Peers[i].Tunnel.LastHandshake = time.Unix(0, 987654321) + wantNext.Peers[i].HealthCheck.Uptime, wantNext.Peers[i].HealthCheck.RTT = "2h", "7ms" + } + + applyReusedFrame(message) + + if !reflect.DeepEqual(*cache.entries["node-a"].Status, wantNext) { + t.Fatal("compact update retained overwritten frame data or lost static metadata") + } + + if !reflect.DeepEqual(*oldSnapshot, wantFull) { + t.Fatal("reusing frame storage or applying measurements mutated the old snapshot") + } + + identity := cache.entries["node-a"].peerIdentity + compactSnapshot := cache.entries["node-a"].Status + message.BaseRevision = cache.entries["node-a"].Revision + applyReusedFrame(message) + + if cache.entries["node-a"].peerIdentity != identity || + !reflect.DeepEqual(*cache.entries["node-a"].Status, wantNext) || + !reflect.DeepEqual(*compactSnapshot, wantNext) { + t.Fatal("warm identity memo retained frame data or mutated a compact snapshot") + } +} + +func TestPeerMeasurementsApplyAndClearImmutable(t *testing.T) { + for _, count := range []int{0, 3} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(count)) + cache := NewNodeStatusCache() + rev := cache.StoreFull("node-a", status, "ws") + old := cache.entries["node-a"] + + before, err := json.Marshal(old.Status) + if err != nil { + t.Fatal(err) + } + + message := measurementMessage(t, status, rev) + + m := message.Delta.PeerMeasurements + for i := range count { + m.RxBytes[i], m.TxBytes[i], m.LastHandshakeUnixNs[i] = 0, 0, 0 + m.Uptime[i], m.Rtt[i] = "", "" + } + + counter := peerMeasurementUpdatesTotal.WithLabelValues("applied") + start := testutil.ToFloat64(counter) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "ok" || ack.Revision != rev+1 { + t.Fatalf("ack: %+v", ack) + } + + if testutil.ToFloat64(counter) != start+1 { + t.Fatal("missing applied metric") + } + + next := cache.entries["node-a"].Status + for i, peer := range next.Peers { + if !netstatus.PeerMetadataEqual(status.Peers[i], peer) { + t.Fatal("static metadata lost") + } + + if peer.Tunnel.RxBytes != 0 || peer.Tunnel.TxBytes != 0 || !peer.Tunnel.LastHandshake.IsZero() || + peer.HealthCheck.Uptime != "" || peer.HealthCheck.RTT != "" { + t.Fatal("measurements not cleared") + } + + if peer.HealthCheck == old.Status.Peers[i].HealthCheck { + t.Fatal("health pointer was reused") + } + } + + after, err := json.Marshal(old.Status) + if err != nil { + t.Fatal(err) + } + + if string(before) != string(after) || cache.entries["node-a"] == old { + t.Fatal("cached snapshot mutated") + } + }) + } +} + +func TestPeerMeasurementsRejectWithoutMutation(t *testing.T) { + tests := map[string]func(*statusproto.NodeStatusMessage, *NodeStatusResponse){ + "zero revision": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.BaseRevision = 0 }, + "stale revision": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.BaseRevision++ }, + "unknown identity": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.PeerMeasurements.IdentityDigest[0] ^= 1 + }, + "missing identity": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.PeerMeasurements.IdentityDigest = nil + }, + "count": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements.PeerCount++ }, + "rx": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements.RxBytes = nil }, + "tx": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements.TxBytes = nil }, + "handshake": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.PeerMeasurements.LastHandshakeUnixNs = nil + }, + "uptime": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements.Uptime = nil }, + "rtt": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.PeerMeasurements.Rtt = append(m.Delta.PeerMeasurements.Rtt, "") + }, + "reordered": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { + s.Peers[0], s.Peers[1] = s.Peers[1], s.Peers[0] + }, + "deleted": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers = s.Peers[:1] }, + "added": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { + s.Peers = append(s.Peers, WireGuardPeerStatus{Name: "new"}) + }, + "duplicate": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1] = s.Peers[0] }, + "renamed": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].Name = "other" }, + "key": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].Tunnel.PublicKey = "other" }, + "interface": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].Tunnel.Interface = "other" }, + "protocol": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].Tunnel.Protocol = "other" }, + "absent health": func(_ *statusproto.NodeStatusMessage, s *NodeStatusResponse) { s.Peers[1].HealthCheck = nil }, + "missing mask": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.UpdatedFields = nil }, + "missing payload": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Delta.PeerMeasurements = nil }, + "duplicate mask": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.UpdatedFields = append(m.Delta.UpdatedFields, "peerMeasurements") + }, + "peers mask conflict": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.UpdatedFields = append(m.Delta.UpdatedFields, "peers") + }, + "peers payload conflict": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Delta.Peers = []*statusproto.PeerStatus{{Name: "other"}} + }, + "full conflict": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { m.Status = measurementTestStatus(2) }, + "full type conflict": func(m *statusproto.NodeStatusMessage, _ *NodeStatusResponse) { + m.Type = "node_status_full" + m.Status = measurementTestStatus(2) + }, + } + for name, change := range tests { + t.Run(name, func(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(2)) + message := measurementMessage(t, status, 1) + change(message, &status) + message.Delta.UpdatedFields = append(message.Delta.UpdatedFields, "fetchError") + message.Delta.FetchError = "must not apply" + cache := NewNodeStatusCache() + cache.StoreFull("node-a", status, "ws") + old := cache.entries["node-a"] + + before, err := json.Marshal(old.Status) + if err != nil { + t.Fatal(err) + } + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, message) + if ack.Status != "resync_required" { + t.Fatalf("accepted malformed batch: %+v", ack) + } + + after, err := json.Marshal(old.Status) + if err != nil { + t.Fatal(err) + } + + if old != cache.entries["node-a"] || string(before) != string(after) { + t.Fatal("partially mutated cache") + } + }) + } + + t.Run("missing cache", func(t *testing.T) { + status := protoToNodeStatus(measurementTestStatus(1)) + cache := NewNodeStatusCache() + counter := peerMeasurementUpdatesTotal.WithLabelValues("resync") + before := testutil.ToFloat64(counter) + + ack := applyMeasurementMessage(t, &healthState{statusCache: cache}, measurementMessage(t, status, 1)) + if ack.Status != "resync_required" || cache.Len() != 0 || testutil.ToFloat64(counter) != before+1 { + t.Fatalf("missing-base behavior: %+v", ack) + } + }) +} + +func TestPeerMeasurementsLegacyAndFullResync(t *testing.T) { + cache := NewNodeStatusCache() + health := &healthState{statusCache: cache} + full := measurementTestStatus(2) + + ack := applyMeasurementMessage(t, health, &statusproto.NodeStatusMessage{Type: "node_status_full", NodeName: "node-a", Status: full}) + if ack.Status != "ok" { + t.Fatal(ack) + } + + full.Peers[0].Tunnel.RxBytes++ + + ack = applyMeasurementMessage(t, health, &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: ack.Revision, + Delta: &statusproto.NodeStatusDelta{UpdatedFields: []string{"peers"}, Peers: full.Peers}, + }) + if ack.Status != "ok" || cache.entries["node-a"].Status.Peers[0].Tunnel.RxBytes != 101 { + t.Fatal("old protobuf node failed") + } + + rev, conflict, err := cache.ApplyDelta("node-a", ack.Revision, map[string]json.RawMessage{"peers": []byte("[]")}, "ws") + if err != nil || conflict || len(cache.entries["node-a"].Status.Peers) != 0 { + t.Fatal("legacy JSON clear failed") + } + + stale := measurementMessage(t, protoToNodeStatus(full), rev) + if got := applyMeasurementMessage(t, health, stale); got.Status != "resync_required" { + t.Fatal("missing peers accepted") + } + + ack = applyMeasurementMessage(t, health, &statusproto.NodeStatusMessage{Type: "node_status_full", NodeName: "node-a", Status: full}) + if got := applyMeasurementMessage(t, health, measurementMessage(t, *cache.entries["node-a"].Status, ack.Revision)); got.Status != "ok" { + t.Fatal("full resync did not restore compact path") + } + + data, err := marshalProtoAck("node_status_ack", ack) + if err != nil { + t.Fatal(err) + } + + var pbAck statusproto.NodeStatusAck + if err := proto.Unmarshal(data, &pbAck); err != nil { + t.Fatal(err) + } + + if !pbAck.PeerMeasurements || pbAck.Revision != ack.Revision { + t.Fatal("capability ACK missing") + } +} + +func TestTypedProtoDeltaAllFieldsAndClearings(t *testing.T) { + cache := NewNodeStatusCache() + cache.StoreFull("node-a", NodeStatusResponse{NodeInfo: NodeInfo{Name: "node-a"}}, "ws") + delta := &statusproto.NodeStatusDelta{ + UpdatedFields: []string{"timestamp", "nodeInfo", "peers", "routingTable", "healthCheck", "nodeErrors", "bpfEntries", "fetchError", "lastPushTime", "statusSource", "nodePodInfo"}, + TimestampUnixNs: 123, NodeInfo: &statusproto.NodeInfo{Name: "node-a", SiteName: "new"}, + Peers: measurementTestStatus(1).Peers, RoutingTable: &statusproto.RoutingTableInfo{ManagedRouteCount: 2}, + HealthCheck: &statusproto.HealthCheckStatus{Healthy: true, CheckedAtUnixNs: 456}, + NodeErrors: []*statusproto.NodeError{{Type: "test", Message: "error"}}, + BpfEntries: []*statusproto.BpfEntry{{Cidr: "10.0.0.0/24"}}, + FetchError: "failed", LastPushTimeUnixNs: 789, StatusSource: "push", NodePodInfo: &statusproto.NodePodInfo{PodName: "pod"}, + } + + rev, conflict, err := cache.ApplyParsedDelta("node-a", 1, protoToParsedDelta(delta), "ws") + if err != nil || conflict { + t.Fatalf("apply: %v %v", err, conflict) + } + + got := cache.entries["node-a"].Status + if !got.Timestamp.Equal(time.Unix(0, 123)) || got.NodeInfo.SiteName != "new" || len(got.Peers) != 1 || + got.RoutingTable.ManagedRouteCount != 2 || got.HealthCheck == nil || !got.HealthCheck.CheckedAt.Equal(time.Unix(0, 456)) || + len(got.NodeErrors) != 1 || len(got.BpfEntries) != 1 || got.FetchError != "failed" || + got.LastPushTime == nil || !got.LastPushTime.Equal(time.Unix(0, 789)) || got.StatusSource != "push" || got.NodePodInfo.PodName != "pod" { + t.Fatalf("typed fields not applied: %+v", got) + } + + clearDelta := &statusproto.NodeStatusDelta{UpdatedFields: delta.UpdatedFields} + + _, conflict, err = cache.ApplyParsedDelta("node-a", rev, protoToParsedDelta(clearDelta), "ws") + if err != nil || conflict { + t.Fatal("clear failed") + } + + got = cache.entries["node-a"].Status + if !got.Timestamp.IsZero() || got.NodeInfo.Name != "node-a" || got.NodeInfo.SiteName != "" || len(got.Peers) != 0 || + !reflect.DeepEqual(got.RoutingTable, RoutingTableInfo{}) || got.HealthCheck != nil || len(got.NodeErrors) != 0 || + len(got.BpfEntries) != 0 || got.FetchError != "" || got.LastPushTime != nil || got.StatusSource != "" || got.NodePodInfo != nil { + t.Fatalf("typed fields not cleared: %+v", got) + } +} + +func BenchmarkApplyPeerDelta2000(b *testing.B) { + full := measurementTestStatus(2000) + status := protoToNodeStatus(full) + + measurements, err := netstatus.PeerMeasurementsToProto(status.Peers) + if err != nil { + b.Fatal(err) + } + + for _, compact := range []bool{false, true} { + name := "full-peers" + delta := &statusproto.NodeStatusDelta{UpdatedFields: []string{"peers"}, Peers: full.Peers} + + if compact { + name = "compact" + delta = &statusproto.NodeStatusDelta{UpdatedFields: []string{"peerMeasurements"}, PeerMeasurements: measurements} + } + + b.Run(name, func(b *testing.B) { + data, err := proto.Marshal(delta) + if err != nil { + b.Fatal(err) + } + + cache := NewNodeStatusCache() + rev := cache.StoreFull("node-a", status, "ws") + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + var decoded statusproto.NodeStatusDelta + if err := proto.Unmarshal(data, &decoded); err != nil { + b.Fatal(err) + } + + next, conflict, err := cache.ApplyParsedDelta("node-a", rev, protoToParsedDelta(&decoded), "ws") + if err != nil || conflict { + b.Fatalf("apply: %v %v", err, conflict) + } + + rev = next + } + + b.ReportMetric(float64(len(data)), "wire-B/op") + }) + } +} diff --git a/cmd/unbounded-net-controller/proto_ws_identity_test.go b/cmd/unbounded-net-controller/proto_ws_identity_test.go new file mode 100644 index 000000000..d90112867 --- /dev/null +++ b/cmd/unbounded-net-controller/proto_ws_identity_test.go @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +func TestProtoWSIdentityBeforeMutation(t *testing.T) { + full := func(envelope, nested string) *statusproto.NodeStatusMessage { + return &statusproto.NodeStatusMessage{ + Type: "node_status_full", NodeName: envelope, + Status: &statusproto.NodeStatusFull{ + NodeInfo: &statusproto.NodeInfo{Name: nested, SiteName: "updated"}, + }, + } + } + cases := []struct { + name string + message *statusproto.NodeStatusMessage + corrupt bool + reject bool + wantClosed bool + }{ + {name: "matching full", message: full("node-a", "node-a")}, + {name: "nested identity", message: full("", "node-a")}, + {name: "matching delta", message: &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: 1, + Delta: &statusproto.NodeStatusDelta{ + UpdatedFields: []string{"nodeInfo"}, + NodeInfo: &statusproto.NodeInfo{Name: "node-a", SiteName: "updated"}, + }, + }}, + {name: "wrong authenticated node", message: full("node-b", "node-b"), reject: true, wantClosed: true}, + {name: "conflicting full", message: full("node-a", "node-b"), reject: true}, + {name: "missing identity", message: full("", ""), reject: true}, + {name: "malformed after valid identity", message: full("node-a", "node-a"), corrupt: true, reject: true}, + {name: "conflicting delta", reject: true, message: &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: 1, + Delta: &statusproto.NodeStatusDelta{ + UpdatedFields: []string{"nodeInfo"}, NodeInfo: &statusproto.NodeInfo{Name: "node-b"}, + }, + }}, + {name: "conflicting unused full field", reject: true, message: &statusproto.NodeStatusMessage{ + Type: "node_status_delta", NodeName: "node-a", BaseRevision: 1, + Status: &statusproto.NodeStatusFull{NodeInfo: &statusproto.NodeInfo{Name: "node-b"}}, + Delta: &statusproto.NodeStatusDelta{ + UpdatedFields: []string{"nodeInfo"}, NodeInfo: &statusproto.NodeInfo{Name: "node-a"}, + }, + }}, + } + proxy, clientTLS := testNodeTokenFrontProxy(t) + issuer := testTokenIssuer(t) + token := testNodeToken(t, issuer) + + for _, path := range []string{"/status/nodews", aggregatedNodeStatusWebSocketPath} { + for _, tc := range cases { + t.Run(path+"/"+tc.name, func(t *testing.T) { + health := newJSONIdentityHealth() + before := health.statusCache.GetAll() + + var evicted atomic.Bool + + health.registerNodeWS("node-a", func() { evicted.Store(true) }) + health.registerNodeWS("node-b", func() { evicted.Store(true) }) + + mux := http.NewServeMux() + registerPushHandlers(mux, health, proxy, make(chan struct{}, maxConcurrentNodeWS), issuer) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.TLS = clientTLS + mux.ServeHTTP(w, r) + })) + defer server.Close() + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + conn, _, err := websocket.Dial(ctx, server.URL+path, &websocket.DialOptions{HTTPHeader: http.Header{ + "Authorization": []string{"Bearer " + token}, + "X-Remote-User": []string{"system:serviceaccount:unbounded-system:unbounded-net-node"}, + nodeIdentityTokenHeader: []string{"service-account-token"}, + }}) + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := conn.CloseNow(); err != nil { + t.Logf("close websocket: %v", err) + } + }() + + data, err := proto.Marshal(tc.message) + if err != nil { + t.Fatal(err) + } + + if tc.corrupt { + data = append(data, 0xff) + } + + if err := conn.Write(ctx, websocket.MessageBinary, data); err != nil { + t.Fatal(err) + } + + frameType, reply, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + + if frameType != websocket.MessageBinary { + t.Fatal("expected a binary acknowledgment") + } + + var ack statusproto.NodeStatusAck + if err := proto.Unmarshal(reply, &ack); err != nil { + t.Fatal(err) + } + + assertJSONIdentityCache(t, health, before, tc.reject) + + if !tc.reject { + if ack.Status != "ok" || !evicted.Load() { + t.Fatalf("valid frame was not applied and registered: %v", &ack) + } + + return + } + + if ack.Status != "resync_required" || ack.Reason == "" || evicted.Load() { + t.Fatalf("rejected identity mutated connection state or lost its error: %v", &ack) + } + + if tc.wantClosed { + if _, _, err := conn.Read(ctx); err == nil || ctx.Err() != nil { + t.Fatal("authorization rejection did not close the connection") + } + + return + } + + // Invalid binary payloads retain the existing resync-and-retry behavior. + valid, err := proto.Marshal(full("node-a", "node-a")) + if err != nil { + t.Fatal(err) + } + + if err := conn.Write(ctx, websocket.MessageBinary, valid); err != nil { + t.Fatal(err) + } + + _, reply, err = conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + + if err := proto.Unmarshal(reply, &ack); err != nil { + t.Fatal(err) + } + + if ack.Status != "ok" || !evicted.Load() { + t.Fatalf("valid retry failed: %v", &ack) + } + + assertJSONIdentityCache(t, health, before, false) + }) + } + } +} diff --git a/cmd/unbounded-net-controller/server.go b/cmd/unbounded-net-controller/server.go index 31af3830b..c706ca95e 100644 --- a/cmd/unbounded-net-controller/server.go +++ b/cmd/unbounded-net-controller/server.go @@ -635,7 +635,7 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer return } - conn.SetReadLimit(2 * 1024 * 1024) // 2 MiB -- status payloads grow with cluster size + conn.SetReadLimit(maxNodeWSFrameBytes) websocketConnections.Inc() defer func() { @@ -685,16 +685,19 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer defer close(recvCh) for { - msgType, data, readErr := conn.Read(wsCtx) + frame, readErr := nodeWSBuffers.readFrame(wsCtx, conn) if readErr != nil { errCh <- readErr return } select { - case recvCh <- wsFrame{msgType: msgType, data: data}: + case recvCh <- frame: case <-wsCtx.Done(): + nodeWSBuffers.put(frame.data) + errCh <- wsCtx.Err() + return } } @@ -746,7 +749,13 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer pingInFlight := false lastActivity := time.Now() + var frameData []byte + defer func() { nodeWSBuffers.put(frameData) }() + for { + nodeWSBuffers.put(frameData) + frameData = nil + select { case <-wsCtx.Done(): return @@ -781,10 +790,21 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer return } + frameData = frame.data lastActivity = time.Now() if frame.msgType == websocket.MessageBinary { - nodeName := extractNodeNameFromProtoMessage(frame.data) + decoded, decodeErr := decodeProtoWSMessage(frame.data) + if decodeErr != nil { + send(websocket.MessageBinary, "node_status_resync", NodeStatusPushAck{ + Status: "resync_required", + Reason: decodeErr.Error(), + }) + + continue + } + + nodeName := decoded.nodeName if expectedNode := authorizedNodeName(r); expectedNode != "" && nodeName != "" && nodeName != expectedNode { send(websocket.MessageBinary, "node_status_resync", NodeStatusPushAck{ Status: "resync_required", @@ -803,7 +823,7 @@ func registerPushHandlers(mux *http.ServeMux, health *healthState, webhookServer lastWSNodeName = nodeName } - ackType, ack := handleProtoWSMessage(health, frame.data, source) + ackType, ack := handleProtoWSMessage(health, decoded, source) send(websocket.MessageBinary, ackType, ack) } else { nodeName, identityErr := extractNodeNameFromWSMessage(frame.data) @@ -1387,6 +1407,8 @@ func serveUnifiedServer(ctx context.Context, port int, mux *http.ServeMux, certM // advertises gzip support via Accept-Encoding. WebSocket upgrades and // requests without gzip support are passed through unmodified. func gzipHandler(next http.Handler) http.Handler { + pool := newGzipWriterPool() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") || strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { @@ -1394,17 +1416,21 @@ func gzipHandler(next http.Handler) http.Handler { return } - gz, err := gzip.NewWriterLevel(w, gzip.BestSpeed) + gz, err := pool.get(w) if err != nil { next.ServeHTTP(w, r) return } - defer func() { _ = gz.Close() }() //nolint:errcheck + completed := false + + defer func() { pool.put(gz, completed) }() w.Header().Set("Content-Encoding", "gzip") w.Header().Del("Content-Length") next.ServeHTTP(&gzipResponseWriter{ResponseWriter: w, Writer: gz}, r) + + completed = true }) } diff --git a/cmd/unbounded-net-controller/status_proto.go b/cmd/unbounded-net-controller/status_proto.go index a1ce3bfcc..cb10196b6 100644 --- a/cmd/unbounded-net-controller/status_proto.go +++ b/cmd/unbounded-net-controller/status_proto.go @@ -330,13 +330,63 @@ func protoToParsedDelta(msg *statusproto.NodeStatusDelta) parsedDelta { updatedSet := make(map[string]bool, len(msg.UpdatedFields)) for _, f := range msg.UpdatedFields { + if f == "peerMeasurements" && updatedSet[f] { + pd.parseError = fmt.Errorf("duplicate peerMeasurements field mask") + } + updatedSet[f] = true } + if updatedSet["peerMeasurements"] || msg.PeerMeasurements != nil { + pd.peerMeasurements = msg.PeerMeasurements + if !updatedSet["peerMeasurements"] || msg.PeerMeasurements == nil { + pd.parseError = fmt.Errorf("peerMeasurements payload and field mask must agree") + } else if updatedSet["peers"] || len(msg.Peers) != 0 { + pd.parseError = fmt.Errorf("peer replacement conflicts with measurements") + } + } + + if pd.parseError != nil { + return pd + } + + if updatedSet["timestamp"] { + t := time.Time{} + if msg.TimestampUnixNs != 0 { + t = time.Unix(0, msg.TimestampUnixNs) + } + + pd.timestamp = &t + } + + if updatedSet["fetchError"] { + pd.fetchError = &msg.FetchError + } + + if updatedSet["statusSource"] { + pd.statusSource = &msg.StatusSource + } + + if updatedSet["lastPushTime"] { + if msg.LastPushTimeUnixNs == 0 { + pd.nullFields["lastPushTime"] = true + } else { + t := time.Unix(0, msg.LastPushTimeUnixNs) + pd.lastPushTime = &t + } + } + + if updatedSet["nodePodInfo"] { + pd.nodePodInfo = protoToNodePodInfo(msg.NodePodInfo) + pd.nullFields["nodePodInfo"] = msg.NodePodInfo == nil + } + if updatedSet["nodeInfo"] { if msg.NodeInfo != nil { ni := protoToNodeInfo(msg.NodeInfo) pd.nodeInfo = &ni + } else { + pd.nullFields["nodeInfo"] = true } } @@ -352,6 +402,8 @@ func protoToParsedDelta(msg *statusproto.NodeStatusDelta) parsedDelta { if msg.RoutingTable != nil { rt := protoToRoutingTable(msg.RoutingTable) pd.routingTable = &rt + } else { + pd.nullFields["routingTable"] = true } } @@ -380,20 +432,30 @@ func protoToParsedDelta(msg *statusproto.NodeStatusDelta) parsedDelta { return pd } -// extractNodeNameFromProtoMessage extracts the node name from a protobuf -// NodeStatusMessage for early identification on WebSocket connections. -func extractNodeNameFromProtoMessage(data []byte) string { - var msg statusproto.NodeStatusMessage - if err := proto.Unmarshal(data, &msg); err != nil { - return "" +type decodedProtoWSMessage struct { + message statusproto.NodeStatusMessage + nodeName string +} + +// Decode and validate identity before connection registration or cache mutation. +func decodeProtoWSMessage(data []byte) (*decodedProtoWSMessage, error) { + decoded := &decodedProtoWSMessage{} + if err := proto.Unmarshal(data, &decoded.message); err != nil { + return nil, fmt.Errorf("invalid protobuf message") } - nodeName, err := validatedProtoNodeName(&msg) + nodeName, err := validatedProtoNodeName(&decoded.message) if err != nil { - return "" + return nil, err + } + + if nodeName == "" { + return nil, fmt.Errorf("nodeName is required") } - return nodeName + decoded.nodeName = nodeName + + return decoded, nil } func validatedProtoNodeName(msg *statusproto.NodeStatusMessage) (string, error) { @@ -409,21 +471,14 @@ func validatedProtoNodeName(msg *statusproto.NodeStatusMessage) (string, error) return validatedNodeNames(nodeNames) } -// handleProtoWSMessage processes a binary (protobuf) WebSocket message and -// returns the ack type string and ack struct, identical to the JSON path. -func handleProtoWSMessage(health *healthState, data []byte, source string) (string, NodeStatusPushAck) { - var msg statusproto.NodeStatusMessage - if err := proto.Unmarshal(data, &msg); err != nil { - return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "invalid protobuf message"} - } +// handleProtoWSMessage applies the same decoded message used for authorization. +func handleProtoWSMessage(health *healthState, decoded *decodedProtoWSMessage, source string) (string, NodeStatusPushAck) { + msg := &decoded.message + nodeName := decoded.nodeName - nodeName, err := validatedProtoNodeName(&msg) - if err != nil { - return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} - } - - if nodeName == "" { - return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "nodeName is required"} + if msg.Delta.GetPeerMeasurements() != nil && (msg.Type != "node_status_delta" || msg.Status != nil) { + peerMeasurementUpdatesTotal.WithLabelValues("error").Inc() + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "full status conflicts with measurements"} } switch msg.Type { @@ -441,7 +496,7 @@ func handleProtoWSMessage(health *healthState, data []byte, source string) (stri return "node_status_ack", NodeStatusPushAck{Status: "ok", Revision: rev} case "node_status_delta": - if msg.Delta == nil || len(msg.Delta.UpdatedFields) == 0 { + if msg.Delta == nil || (len(msg.Delta.UpdatedFields) == 0 && msg.Delta.PeerMeasurements == nil) { return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: "delta message missing delta"} } @@ -479,6 +534,10 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string } ack := NodeStatusPushAck{Status: "ok"} + if msg.Delta.GetPeerMeasurements() != nil && (msg.Type != "node_status_delta" || msg.Status != nil) { + peerMeasurementUpdatesTotal.WithLabelValues("error").Inc() + return NodeStatusPushAck{Status: "resync_required", Reason: "full status conflicts with measurements"}, 429, nil + } switch msg.Type { case "node_status_full": @@ -495,7 +554,7 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string return ack, 200, nil case "node_status_delta": - if msg.Delta == nil || len(msg.Delta.UpdatedFields) == 0 { + if msg.Delta == nil || (len(msg.Delta.UpdatedFields) == 0 && msg.Delta.PeerMeasurements == nil) { return NodeStatusPushAck{}, 400, fmt.Errorf("delta is required for delta mode") } @@ -521,9 +580,10 @@ func handleProtoPushRequest(health *healthState, bodyBytes []byte, source string // marshalProtoAck serializes a NodeStatusPushAck into a protobuf NodeStatusAck. func marshalProtoAck(ackType string, ack NodeStatusPushAck) ([]byte, error) { pbAck := &statusproto.NodeStatusAck{ - Status: ack.Status, - Revision: ack.Revision, - Reason: ack.Reason, + PeerMeasurements: true, + Status: ack.Status, + Revision: ack.Revision, + Reason: ack.Reason, } return proto.Marshal(pbAck) diff --git a/cmd/unbounded-net-controller/status_proto_test.go b/cmd/unbounded-net-controller/status_proto_test.go index b4160a6f1..d90ebc873 100644 --- a/cmd/unbounded-net-controller/status_proto_test.go +++ b/cmd/unbounded-net-controller/status_proto_test.go @@ -216,11 +216,20 @@ func TestProtoToParsedDeltaEmptyPeers(t *testing.T) { } } +func handleProtoWSBytes(health *healthState, data []byte, source string) (string, NodeStatusPushAck) { + decoded, err := decodeProtoWSMessage(data) + if err != nil { + return "node_status_resync", NodeStatusPushAck{Status: "resync_required", Reason: err.Error()} + } + + return handleProtoWSMessage(health, decoded, source) +} + func TestHandleProtoWSMessage(t *testing.T) { health := &healthState{statusCache: NewNodeStatusCache()} t.Run("invalid proto", func(t *testing.T) { - msgType, ack := handleProtoWSMessage(health, []byte("not-proto"), "ws") + msgType, ack := handleProtoWSBytes(health, []byte("not-proto"), "ws") if msgType != "node_status_resync" || ack.Status != "resync_required" { t.Fatalf("expected resync on invalid proto, got type=%q ack=%+v", msgType, ack) } @@ -230,7 +239,7 @@ func TestHandleProtoWSMessage(t *testing.T) { msg := &statusproto.NodeStatusMessage{Type: "node_status_full"} data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "nodeName is required" { t.Fatalf("expected nodeName required, got type=%q ack=%+v", msgType, ack) } @@ -246,7 +255,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_ack" || ack.Status != "ok" || ack.Revision == 0 { t.Fatalf("expected full ack success, got type=%q ack=%+v", msgType, ack) } @@ -267,7 +276,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Status != "resync_required" { t.Fatalf("expected conflicting node names to require resync, got type=%q ack=%+v", msgType, ack) } @@ -280,7 +289,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "full message missing status" { t.Fatalf("expected missing status resync, got type=%q ack=%+v", msgType, ack) } @@ -298,7 +307,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_ack" || ack.Status != "ok" || ack.Revision < 2 { t.Fatalf("expected delta ack success, got type=%q ack=%+v", msgType, ack) } @@ -321,7 +330,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "base revision mismatch" { t.Fatalf("expected conflict resync, got type=%q ack=%+v", msgType, ack) } @@ -334,7 +343,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "delta message missing delta" { t.Fatalf("expected missing delta resync, got type=%q ack=%+v", msgType, ack) } @@ -347,7 +356,7 @@ func TestHandleProtoWSMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - msgType, ack := handleProtoWSMessage(health, data, "ws") + msgType, ack := handleProtoWSBytes(health, data, "ws") if msgType != "node_status_resync" || ack.Reason != "unsupported message type" { t.Fatalf("expected unsupported type resync, got type=%q ack=%+v", msgType, ack) } @@ -471,8 +480,8 @@ func TestExtractNodeNameFromProtoMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - if got := extractNodeNameFromProtoMessage(data); got != "node-x" { - t.Fatalf("expected node-x, got %q", got) + if got, err := decodeProtoWSMessage(data); err != nil || got.nodeName != "node-x" { + t.Fatalf("expected node-x, got %+v, err=%v", got, err) } }) @@ -483,14 +492,14 @@ func TestExtractNodeNameFromProtoMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - if got := extractNodeNameFromProtoMessage(data); got != "node-y" { - t.Fatalf("expected node-y, got %q", got) + if got, err := decodeProtoWSMessage(data); err != nil || got.nodeName != "node-y" { + t.Fatalf("expected node-y, got %+v, err=%v", got, err) } }) t.Run("invalid data", func(t *testing.T) { - if got := extractNodeNameFromProtoMessage([]byte("bad")); got != "" { - t.Fatalf("expected empty, got %q", got) + if got, err := decodeProtoWSMessage([]byte("bad")); err == nil || got != nil { + t.Fatalf("expected invalid protobuf error, got %+v, err=%v", got, err) } }) @@ -501,8 +510,8 @@ func TestExtractNodeNameFromProtoMessage(t *testing.T) { } data, _ := proto.Marshal(msg) - if got := extractNodeNameFromProtoMessage(data); got != "" { - t.Fatalf("expected empty for conflicting node names, got %q", got) + if got, err := decodeProtoWSMessage(data); err == nil || got != nil { + t.Fatalf("expected conflicting node names error, got %+v, err=%v", got, err) } }) } diff --git a/cmd/unbounded-net-controller/ws_frame_buffer.go b/cmd/unbounded-net-controller/ws_frame_buffer.go new file mode 100644 index 000000000..ba7c0fec3 --- /dev/null +++ b/cmd/unbounded-net-controller/ws_frame_buffer.go @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "io" + + "github.com/coder/websocket" + "k8s.io/klog/v2" +) + +const ( + maxNodeWSFrameBytes = 2 * 1024 * 1024 + nodeWSBufferClasses = 9 + nodeWSMinBufferSize = 8 * 1024 + nodeWSBuffersPerClass = 16 +) + +// Idle buffers are capped below 64 MiB across all connections. Active frames +// retain exclusive ownership until processing finishes, not for a connection's lifetime. +type nodeWSBufferPool struct { + buffers [nodeWSBufferClasses]chan []byte +} + +var nodeWSBuffers = newNodeWSBufferPool(nodeWSBuffersPerClass) + +func newNodeWSBufferPool(perClass int) *nodeWSBufferPool { + pool := &nodeWSBufferPool{} + for i := range pool.buffers { + pool.buffers[i] = make(chan []byte, perClass) + } + + return pool +} + +func (p *nodeWSBufferPool) get(size int) []byte { + for i, buffers := range p.buffers { + classSize := nodeWSMinBufferSize << i + if classSize < size { + continue + } + + select { + case data := <-buffers: + return data[:0] + default: + return make([]byte, 0, classSize) + } + } + + panic("node WebSocket buffer request exceeds the frame limit") +} + +func (p *nodeWSBufferPool) put(data []byte) { + for i, buffers := range p.buffers { + if cap(data) != nodeWSMinBufferSize< 0 { + emptyReads = 0 + continue + } + } + + emptyReads++ + if emptyReads >= 100 { + p.put(data) + return nil, io.ErrNoProgress + } + } +} + +func (p *nodeWSBufferPool) readFrame(ctx context.Context, conn *websocket.Conn) (wsFrame, error) { + msgType, reader, err := conn.Reader(ctx) + if err != nil { + return wsFrame{}, err + } + + data, err := p.read(reader) + if errors.Is(err, websocket.ErrMessageTooBig) { + if closeErr := conn.Close(websocket.StatusMessageTooBig, "node status exceeds 2 MiB"); closeErr != nil { + klog.V(4).Infof("Node WebSocket oversized message close failed: %v", closeErr) + } + } + + return wsFrame{msgType: msgType, data: data}, err +} diff --git a/cmd/unbounded-net-controller/ws_frame_buffer_test.go b/cmd/unbounded-net-controller/ws_frame_buffer_test.go new file mode 100644 index 000000000..1723fdef1 --- /dev/null +++ b/cmd/unbounded-net-controller/ws_frame_buffer_test.go @@ -0,0 +1,318 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/coder/websocket" +) + +func TestNodeWSBufferSizes(t *testing.T) { + for _, size := range []int{0, 1, nodeWSMinBufferSize - 1, nodeWSMinBufferSize, nodeWSMinBufferSize + 1, 512 * 1024, maxNodeWSFrameBytes, maxNodeWSFrameBytes + 1} { + t.Run(fmt.Sprint(size), func(t *testing.T) { + pool := newNodeWSBufferPool(2) + payload := bytes.Repeat([]byte{0x5a}, size) + + data, err := pool.read(bytes.NewReader(payload)) + defer pool.put(data) + + if size > maxNodeWSFrameBytes { + if !errors.Is(err, websocket.ErrMessageTooBig) || data != nil { + t.Fatalf("oversized frame returned %d bytes, %v", len(data), err) + } + + return + } + + if err != nil || !bytes.Equal(data, payload) { + t.Fatalf("frame size %d: got %d bytes, %v", size, len(data), err) + } + }) + } +} + +func TestNodeWSBufferOwnershipAndBound(t *testing.T) { + pool := newNodeWSBufferPool(2) + + first, err := pool.read(bytes.NewReader([]byte("first"))) + if err != nil { + t.Fatal(err) + } + + second, err := pool.read(bytes.NewReader([]byte("second"))) + if err != nil { + t.Fatal(err) + } + + pool.put(second) + + reused := pool.get(nodeWSMinBufferSize) + copy(reused[:cap(reused)], []byte("overwrite")) + + if string(first) != "first" { + t.Fatal("reusing another frame mutated an in-flight frame") + } + + pool.put(first) + pool.put(reused) + + retained := 0 + + for i, buffers := range pool.buffers { + size := nodeWSMinBufferSize << i + for range 4 { + pool.put(make([]byte, size)) + } + + if len(buffers) != 2 { + t.Fatalf("class %d retained %d buffers, want 2", i, len(buffers)) + } + + retained += len(buffers) * size + } + + if retained != 2*(2*maxNodeWSFrameBytes-nodeWSMinBufferSize) { + t.Fatalf("unexpected retention: %d bytes", retained) + } +} + +type failingWSReader struct { + err error +} + +func (r failingWSReader) Read([]byte) (int, error) { + return 0, r.err +} + +func TestNodeWSBufferReadErrors(t *testing.T) { + readFailure := errors.New("read failure") + for _, tc := range []struct { + name string + reader io.Reader + want error + }{ + {"failure", failingWSReader{readFailure}, readFailure}, + {"cancellation", failingWSReader{context.Canceled}, context.Canceled}, + {"wrapped-eof", failingWSReader{fmt.Errorf("closed mid-frame: %w", io.EOF)}, io.EOF}, + {"no-progress", failingWSReader{}, io.ErrNoProgress}, + {"failure-after-growth", io.MultiReader(bytes.NewReader(make([]byte, 2*nodeWSMinBufferSize)), failingWSReader{readFailure}), readFailure}, + } { + t.Run(tc.name, func(t *testing.T) { + pool := newNodeWSBufferPool(2) + + data, err := pool.read(tc.reader) + if data != nil || !errors.Is(err, tc.want) { + t.Fatalf("read returned %d bytes, %v", len(data), err) + } + + if len(pool.buffers[0]) == 0 { + t.Fatal("failed read did not release its buffer") + } + }) + } +} + +func TestNodeWSBufferAllocations(t *testing.T) { + pool := newNodeWSBufferPool(2) + reader := bytes.NewReader(make([]byte, 512*1024)) + + allocs := testing.AllocsPerRun(100, func() { + if _, err := reader.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + + data, err := pool.read(reader) + if err != nil { + t.Fatal(err) + } + + pool.put(data) + }) + if allocs > 1 { + t.Fatalf("warmed frame reader allocated %.1f objects; want at most 1", allocs) + } +} + +func TestNodeWSFrameTransport(t *testing.T) { + for _, compression := range []websocket.CompressionMode{websocket.CompressionDisabled, websocket.CompressionContextTakeover} { + for _, size := range []int{0, 32769, maxNodeWSFrameBytes, maxNodeWSFrameBytes + 1} { + t.Run(fmt.Sprintf("compression-%d/bytes-%d", compression, size), func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + pool := newNodeWSBufferPool(2) + result := make(chan error, 1) + payload := bytes.Repeat([]byte("x"), size) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{CompressionMode: compression}) + if err != nil { + result <- err + return + } + defer func() { + if err := conn.CloseNow(); err != nil && !errors.Is(err, net.ErrClosed) { + t.Errorf("server close: %v", err) + } + }() + + conn.SetReadLimit(maxNodeWSFrameBytes) + + frame, err := pool.readFrame(ctx, conn) + defer pool.put(frame.data) + + if err == nil && (frame.msgType != websocket.MessageBinary || !bytes.Equal(frame.data, payload)) { + err = errors.New("frame type or payload changed") + } + + result <- err + })) + defer server.Close() + + conn, _, err := websocket.Dial(ctx, server.URL, &websocket.DialOptions{CompressionMode: compression}) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := conn.CloseNow(); err != nil && !errors.Is(err, net.ErrClosed) { + t.Errorf("client close: %v", err) + } + }() + + if err := conn.Write(ctx, websocket.MessageBinary, payload); err != nil && size <= maxNodeWSFrameBytes { + t.Fatal(err) + } + + if size > maxNodeWSFrameBytes { + _, _, err := conn.Read(ctx) + if websocket.CloseStatus(err) != websocket.StatusMessageTooBig { + t.Fatalf("oversize close: %v", err) + } + } + + select { + case err := <-result: + if size > maxNodeWSFrameBytes { + if !errors.Is(err, websocket.ErrMessageTooBig) { + t.Fatalf("expected size rejection, got %v", err) + } + } else if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + }) + } + } +} + +func TestNodeWSFrameCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + pool := newNodeWSBufferPool(2) + started := make(chan struct{}) + result := make(chan error, 1) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + result <- err + return + } + defer func() { + if err := conn.CloseNow(); err != nil && !errors.Is(err, net.ErrClosed) { + t.Errorf("server close: %v", err) + } + }() + + close(started) + + frame, err := pool.readFrame(ctx, conn) + pool.put(frame.data) + + result <- err + })) + defer server.Close() + + conn, _, err := websocket.Dial(t.Context(), server.URL, nil) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := conn.CloseNow(); err != nil && !errors.Is(err, net.ErrClosed) { + t.Errorf("client close: %v", err) + } + }() + + <-started + cancel() + + select { + case err := <-result: + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation, got %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("canceled reader did not return") + } +} + +func BenchmarkNodeWSFrameBuffer(b *testing.B) { + for _, size := range []int{8 * 1024, 512 * 1024, maxNodeWSFrameBytes} { + for _, pooled := range []bool{false, true} { + b.Run(fmt.Sprintf("bytes-%d/pooled-%t", size, pooled), func(b *testing.B) { + pool := newNodeWSBufferPool(2) + + reader := bytes.NewReader(make([]byte, size)) + if pooled { + data, err := pool.read(reader) + if err != nil { + b.Fatal(err) + } + + pool.put(data) + } + + b.ReportAllocs() + b.SetBytes(int64(size)) + + for b.Loop() { + if _, err := reader.Seek(0, io.SeekStart); err != nil { + b.Fatal(err) + } + + var ( + data []byte + err error + ) + if pooled { + data, err = pool.read(reader) + } else { + data, err = io.ReadAll(reader) + } + + if err != nil || len(data) != size { + b.Fatalf("read returned %d bytes, %v", len(data), err) + } + + if pooled { + pool.put(data) + } + } + }) + } + } +} diff --git a/cmd/unbounded-net-node/status_ack.go b/cmd/unbounded-net-node/status_ack.go new file mode 100644 index 000000000..b81a85770 --- /dev/null +++ b/cmd/unbounded-net-node/status_ack.go @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "sync/atomic" + + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +// statusAckState is created fresh for every connection. One outstanding message +// keeps the sender's snapshot and the controller's acknowledged revision aligned. +type statusAckState struct { + revision atomic.Uint64 + resync atomic.Bool + pending atomic.Bool + compact atomic.Bool +} + +func (s *statusAckState) accept(data []byte) bool { + var ack statusproto.NodeStatusAck + if err := proto.Unmarshal(data, &ack); err != nil { + var envelope struct { + Type string `json:"type"` + Data nodeStatusPushAck `json:"data"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return false + } + + switch envelope.Type { + case "node_status_ack": + ack.Status = "ok" + case "node_status_resync": + ack.Status = "resync_required" + default: + return false + } + + ack.Revision = envelope.Data.Revision + } + + switch ack.Status { + case "ok": + s.compact.Store(ack.PeerMeasurements && ack.Revision > 0) + case "resync_required": + s.compact.Store(false) + s.resync.Store(true) + default: + return false + } + + if ack.Revision > 0 { + s.revision.Store(ack.Revision) + } + + s.pending.Store(false) + + return true +} diff --git a/cmd/unbounded-net-node/status_delta.go b/cmd/unbounded-net-node/status_delta.go new file mode 100644 index 000000000..8e9b1fe47 --- /dev/null +++ b/cmd/unbounded-net-node/status_delta.go @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "cmp" + "reflect" + "slices" + "time" + + netstatus "github.com/Azure/unbounded/internal/net/status" + statusproto "github.com/Azure/unbounded/internal/net/status/proto" +) + +// sortStatusPeers is called only on freshly collected, exclusively owned peers. +// Gateway map iteration must not turn unchanged topology into peer replacements. +func sortStatusPeers(peers []WireGuardPeerStatus) { + slices.SortFunc(peers, func(a, b WireGuardPeerStatus) int { + return cmp.Or( + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Tunnel.Protocol, b.Tunnel.Protocol), + cmp.Compare(a.Tunnel.Interface, b.Tunnel.Interface), + cmp.Compare(a.Tunnel.PublicKey, b.Tunnel.PublicKey), + ) + }) +} + +// typedStatusDelta avoids serializing both snapshots to JSON and decoding again. +// refresh emits a timestamp even when values are unchanged, preserving liveness. +func typedStatusDelta(prev, curr *NodeStatusResponse, compact, refresh bool) *statusproto.NodeStatusDelta { + if prev == nil || curr == nil { + return nil + } + + pb := &statusproto.NodeStatusDelta{} + + add := func(field string) { pb.UpdatedFields = append(pb.UpdatedFields, field) } + if refresh || !prev.Timestamp.Equal(curr.Timestamp) { + add("timestamp") + + if !curr.Timestamp.IsZero() { + pb.TimestampUnixNs = curr.Timestamp.UnixNano() + } + } + + if !reflect.DeepEqual(prev.NodeInfo, curr.NodeInfo) { + add("nodeInfo") + + pb.NodeInfo = nodeInfoToProto(&curr.NodeInfo) + } + + if !reflect.DeepEqual(prev.Peers, curr.Peers) { + metadataEqual := compact && len(prev.Peers) == len(curr.Peers) + if metadataEqual { + for i := range curr.Peers { + if !netstatus.PeerMetadataEqual(prev.Peers[i], curr.Peers[i]) { + metadataEqual = false + break + } + } + } + + if metadataEqual { + // Ambiguous identities must use the full replacement path. + measurements, err := netstatus.PeerMeasurementsToProto(curr.Peers) + if err == nil { + pb.PeerMeasurements = measurements + } + } + + if pb.PeerMeasurements != nil { + add("peerMeasurements") + } else { + add("peers") + + pb.Peers = peersToProto(curr.Peers) + } + } + + if !reflect.DeepEqual(prev.RoutingTable, curr.RoutingTable) { + add("routingTable") + + pb.RoutingTable = routingTableToProto(&curr.RoutingTable) + } + + if !reflect.DeepEqual(prev.HealthCheck, curr.HealthCheck) { + add("healthCheck") + + pb.HealthCheck = healthCheckStatusToProto(curr.HealthCheck) + } + + if !reflect.DeepEqual(prev.NodeErrors, curr.NodeErrors) { + add("nodeErrors") + + pb.NodeErrors = nodeErrorsToProto(curr.NodeErrors) + } + + if !reflect.DeepEqual(prev.BpfEntries, curr.BpfEntries) { + add("bpfEntries") + + pb.BpfEntries = bpfEntriesToProto(curr.BpfEntries) + } + + if prev.FetchError != curr.FetchError { + add("fetchError") + + pb.FetchError = curr.FetchError + } + + if !reflect.DeepEqual(prev.LastPushTime, curr.LastPushTime) { + add("lastPushTime") + + if curr.LastPushTime != nil && !curr.LastPushTime.IsZero() { + pb.LastPushTimeUnixNs = curr.LastPushTime.UnixNano() + } + } + + if prev.StatusSource != curr.StatusSource { + add("statusSource") + + pb.StatusSource = curr.StatusSource + } + + if !reflect.DeepEqual(prev.NodePodInfo, curr.NodePodInfo) { + add("nodePodInfo") + + pb.NodePodInfo = nodePodInfoToProto(curr.NodePodInfo) + } + + if len(pb.UpdatedFields) == 0 { + return nil + } + + return pb +} + +// criticalStatus preserves the last published measurements while applying current +// metadata. Never modify shared snapshots or their health pointers. +func criticalStatus(prev, curr *NodeStatusResponse) *NodeStatusResponse { + result := *curr + result.Timestamp = prev.Timestamp + + if curr.HealthCheck != nil { + health := *curr.HealthCheck + + health.CheckedAt = time.Time{} + if prev.HealthCheck != nil { + health.CheckedAt = prev.HealthCheck.CheckedAt + } + + result.HealthCheck = &health + } + + result.Peers = append([]WireGuardPeerStatus(nil), curr.Peers...) + + type identity struct{ name, protocol, iface, key string } + + key := func(peer WireGuardPeerStatus) identity { + return identity{peer.Name, peer.Tunnel.Protocol, peer.Tunnel.Interface, peer.Tunnel.PublicKey} + } + + previous := make(map[identity]WireGuardPeerStatus, len(prev.Peers)) + for _, peer := range prev.Peers { + previous[key(peer)] = peer + } + + for i := range result.Peers { + peer := &result.Peers[i] + + old, ok := previous[key(*peer)] + if !ok { + continue + } + + peer.Tunnel.RxBytes = old.Tunnel.RxBytes + peer.Tunnel.TxBytes = old.Tunnel.TxBytes + + peer.Tunnel.LastHandshake = old.Tunnel.LastHandshake + if peer.HealthCheck != nil { + health := *peer.HealthCheck + + health.Uptime, health.RTT = "", "" + if old.HealthCheck != nil { + health.Uptime, health.RTT = old.HealthCheck.Uptime, old.HealthCheck.RTT + } + + peer.HealthCheck = &health + } + } + + return &result +} diff --git a/cmd/unbounded-net-node/status_delta_test.go b/cmd/unbounded-net-node/status_delta_test.go new file mode 100644 index 000000000..f1ff4ec89 --- /dev/null +++ b/cmd/unbounded-net-node/status_delta_test.go @@ -0,0 +1,438 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + "google.golang.org/protobuf/proto" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func measuredNodeStatus() *NodeStatusResponse { + status := testNodeStatus("old") + status.HealthCheck = &HealthCheckStatus{Healthy: true, Summary: "ok", PeerCount: 1, CheckedAt: testStatusTime} + status.Peers[0].HealthCheck = &HealthCheckPeerStatus{Enabled: true, Status: "up", Uptime: "1m", RTT: "1ms"} + + return status +} + +func TestCriticalStatusIgnoresOnlyMeasurements(t *testing.T) { + measurements := map[string]func(*NodeStatusResponse){ + "timestamp": func(s *NodeStatusResponse) { s.Timestamp = s.Timestamp.Add(time.Second) }, + "checkedAt": func(s *NodeStatusResponse) { s.HealthCheck.CheckedAt = s.HealthCheck.CheckedAt.Add(time.Second) }, + "rx": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.RxBytes++ }, + "tx": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.TxBytes++ }, + "handshake": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.LastHandshake = time.Time{} }, + "uptime": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.Uptime = "2m" }, + "rtt": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.RTT = "2ms" }, + } + for name, change := range measurements { + t.Run(name, func(t *testing.T) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + change(curr) + + before := nodeStatusToProto(curr) + if !reflect.DeepEqual(stripPeerStats(prev), stripPeerStats(curr)) { + t.Fatal("measurement triggered critical comparison") + } + + if delta := typedStatusDelta(prev, criticalStatus(prev, curr), true, false); delta != nil { + t.Fatalf("measurement leaked into critical delta: %v", delta) + } + + if typedStatusDelta(prev, curr, true, true) == nil { + t.Fatal("statistics refresh lost measurement") + } + + if !proto.Equal(before, nodeStatusToProto(curr)) { + t.Fatal("shared current snapshot mutated") + } + }) + } + + critical := map[string]func(*NodeStatusResponse){ + "nodeInfo": func(s *NodeStatusResponse) { s.NodeInfo.SiteName = "new" }, + "healthy": func(s *NodeStatusResponse) { s.HealthCheck.Healthy = false }, + "summary": func(s *NodeStatusResponse) { s.HealthCheck.Summary = "failed" }, + "count": func(s *NodeStatusResponse) { s.HealthCheck.PeerCount++ }, + "health removal": func(s *NodeStatusResponse) { s.HealthCheck = nil }, + "peer status": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.Status = "down" }, + "peer enabled": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.Enabled = false }, + "peer health removal": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck = nil }, + "endpoint": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.Endpoint = "new" }, + "error": func(s *NodeStatusResponse) { s.NodeErrors = []NodeError{{Type: "test", Message: "failed"}} }, + "error clear": func(s *NodeStatusResponse) { s.FetchError = "" }, + "bpf": func(s *NodeStatusResponse) { s.BpfEntries = []BpfEntry{{CIDR: "10.0.0.0/24"}} }, + "route": func(s *NodeStatusResponse) { s.RoutingTable.ManagedRouteCount++ }, + } + for name, change := range critical { + t.Run(name, func(t *testing.T) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + change(curr) + curr.Timestamp = curr.Timestamp.Add(time.Second) + + curr.Peers[0].Tunnel.RxBytes++ + if reflect.DeepEqual(stripPeerStats(prev), stripPeerStats(curr)) { + t.Fatal("critical change filtered") + } + + published := criticalStatus(prev, curr) + + delta := typedStatusDelta(prev, published, true, false) + if delta == nil || slices.Contains(delta.UpdatedFields, "timestamp") || delta.PeerMeasurements != nil { + t.Fatalf("invalid critical delta: %v", delta) + } + + if published.Peers[0].Tunnel.RxBytes != prev.Peers[0].Tunnel.RxBytes { + t.Fatal("critical change advanced statistics baseline") + } + }) + } +} + +func TestTypedStatusDeltaCompactCompatibility(t *testing.T) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + curr.Peers[0].Tunnel.RxBytes = 0 + curr.Peers[0].Tunnel.TxBytes = 0 + curr.Peers[0].Tunnel.LastHandshake = time.Time{} + curr.Peers[0].HealthCheck.Uptime = "" + + curr.Peers[0].HealthCheck.RTT = "" + for _, compact := range []bool{false, true} { + delta := typedStatusDelta(prev, curr, compact, false) + if compact { + if delta.PeerMeasurements == nil || len(delta.Peers) != 0 || !slices.Contains(delta.UpdatedFields, "peerMeasurements") { + t.Fatalf("missing compact measurements: %v", delta) + } + + m := delta.PeerMeasurements + if m.RxBytes[0] != 0 || m.TxBytes[0] != 0 || m.LastHandshakeUnixNs[0] != 0 || m.Uptime[0] != "" || m.Rtt[0] != "" { + t.Fatal("zero measurements did not clear") + } + } else if delta.PeerMeasurements != nil || len(delta.Peers) != 1 { + t.Fatalf("old controller requires peers replacement: %v", delta) + } + + data, err := proto.Marshal(delta) + if err != nil { + t.Fatal(err) + } + + var decoded statusproto.NodeStatusDelta + if err := proto.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + + if !proto.Equal(delta, &decoded) { + t.Fatal("delta did not round trip") + } + } + + for name, change := range map[string]func(*NodeStatusResponse){ + "added": func(s *NodeStatusResponse) { p := s.Peers[0]; p.Name = "peer-b"; s.Peers = append(s.Peers, p) }, + "deleted": func(s *NodeStatusResponse) { s.Peers = nil }, + "identity": func(s *NodeStatusResponse) { s.Peers[0].Name = "other" }, + "public key": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.PublicKey = "other" }, + "interface": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.Interface = "other" }, + "protocol": func(s *NodeStatusResponse) { s.Peers[0].Tunnel.Protocol = "other" }, + "metadata": func(s *NodeStatusResponse) { s.Peers[0].SiteName = "other" }, + "health": func(s *NodeStatusResponse) { s.Peers[0].HealthCheck.Status = "down" }, + "unnamed": func(s *NodeStatusResponse) { s.Peers[0].Name = "" }, + } { + t.Run(name, func(t *testing.T) { + next := measuredNodeStatus() + change(next) + + delta := typedStatusDelta(prev, next, true, false) + if delta.PeerMeasurements != nil || !slices.Contains(delta.UpdatedFields, "peers") { + t.Fatalf("topology change requires replacement: %v", delta) + } + }) + } + + if typedStatusDelta(prev, prev, true, false) != nil { + t.Fatal("no-op produced delta") + } + + if typedStatusDelta(nil, curr, true, true) != nil { + t.Fatal("first update must be full") + } + + if got := typedStatusDelta(prev, prev, true, true); !slices.Equal(got.UpdatedFields, []string{"timestamp"}) { + t.Fatalf("periodic freshness: %v", got) + } +} + +func TestTypedStatusDeltaFieldClearings(t *testing.T) { + prev := measuredNodeStatus() + prev.NodeErrors = []NodeError{{Type: "test"}} + prev.LastPushTime = &prev.Timestamp + prev.StatusSource = "push" + prev.NodePodInfo = &statusv1alpha1.NodePodInfo{PodName: "old"} + prev.BpfEntries = []BpfEntry{{CIDR: "10.0.0.0/24"}} + prev.RoutingTable.ManagedRouteCount = 1 + curr := &NodeStatusResponse{} + delta := typedStatusDelta(prev, curr, true, false) + + want := []string{"timestamp", "nodeInfo", "peers", "routingTable", "healthCheck", "nodeErrors", "bpfEntries", "fetchError", "lastPushTime", "statusSource", "nodePodInfo"} + if !slices.Equal(delta.UpdatedFields, want) { + t.Fatalf("clearings %v, want %v", delta.UpdatedFields, want) + } +} + +func TestTypedStatusDeltaReorderingAndDuplicateFallback(t *testing.T) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + other := prev.Peers[0] + other.Name = "peer-b" + prev.Peers = append(prev.Peers, other) + + curr.Peers = []WireGuardPeerStatus{other, curr.Peers[0]} + if delta := typedStatusDelta(prev, curr, true, false); delta.PeerMeasurements != nil || len(delta.Peers) != 2 { + t.Fatal("reordered base must be replaced") + } + + sortStatusPeers(curr.Peers) + + if delta := typedStatusDelta(prev, curr, true, false); delta != nil { + t.Fatal("canonical ordering must eliminate iteration-only changes") + } + + prev.Peers[1] = prev.Peers[0] + curr.Peers = append([]WireGuardPeerStatus(nil), prev.Peers...) + + curr.Peers[0].Tunnel.RxBytes++ + if delta := typedStatusDelta(prev, curr, true, false); delta.PeerMeasurements != nil || len(delta.Peers) != 2 { + t.Fatal("duplicate identities must use legacy replacement") + } +} + +func TestWebSocketCriticalNoopStatsAndResync(t *testing.T) { + for _, mode := range []string{"critical", "stats-resync", "full-refresh"} { + t.Run(mode, func(t *testing.T) { + received := make(chan *statusproto.NodeStatusMessage, 32) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Errorf("accept: %v", err) + return + } + defer func() { _ = conn.Close(websocket.StatusNormalClosure, "done") }() + + var revision uint64 + + for { + _, data, err := conn.Read(ctx) + if err != nil { + return + } + + var message statusproto.NodeStatusMessage + if err := proto.Unmarshal(data, &message); err != nil { + t.Errorf("decode: %v", err) + return + } + + select { + case received <- &message: + case <-ctx.Done(): + return + } + + revision++ + + ack := &statusproto.NodeStatusAck{Status: "ok", Revision: revision, PeerMeasurements: true} + if mode == "stats-resync" && revision == 2 { + ack.Status = "resync_required" + } + + payload, err := proto.Marshal(ack) + if err != nil { + t.Errorf("encode ACK: %v", err) + return + } + + if err := conn.Write(ctx, websocket.MessageBinary, payload); err != nil { + return + } + } + })) + defer server.Close() + + cfg := &config{ + NodeName: "node-a", StatusWSEnabled: true, + StatusWSURL: "ws" + strings.TrimPrefix(server.URL, "http"), StatusWSAPIServerMode: statusWSAPIServerModeNever, + CriticalDeltaEvery: time.Hour, StatsDeltaEvery: time.Hour, FullSyncEvery: time.Hour, + } + + switch mode { + case "critical": + cfg.CriticalDeltaEvery = 10 * time.Millisecond + case "stats-resync": + cfg.StatsDeltaEvery = 15 * time.Millisecond + case "full-refresh": + cfg.FullSyncEvery = 15 * time.Millisecond + } + + health := blockedBootstrapHealthState() + + startStatusPublishers(ctx, cfg, health) + defer health.stopStatusPublishers() + + next := func() *statusproto.NodeStatusMessage { + t.Helper() + + select { + case message := <-received: + return message + case <-time.After(3 * time.Second): + t.Fatal("publisher did not send expected message") + return nil + } + } + if message := next(); message.Status == nil || message.Delta != nil { + t.Fatal("first frame must be full") + } + + if mode == "critical" { + select { + case message := <-received: + t.Fatalf("timestamp-only critical publication: %v", message) + case <-time.After(100 * time.Millisecond): + } + + health.setCNIReady("cbr0", []string{"10.244.7.0/24"}) + + if message := next(); !isCNIGuardClearingDelta(message) { + t.Fatal("critical error clearing lost") + } + } else if mode == "stats-resync" { + if message := next(); message.Delta == nil || !slices.Contains(message.Delta.UpdatedFields, "timestamp") { + t.Fatal("statistics interval did not publish freshness") + } + + if message := next(); message.Status == nil || message.Delta != nil { + t.Fatal("statistics interval must honor resync with full status") + } + } else if message := next(); message.Status == nil || message.Delta != nil { + t.Fatal("periodic full refresh lost") + } + + cancel() + }) + } +} + +func TestStatusAckNegotiation(t *testing.T) { + var state statusAckState + if state.compact.Load() || state.revision.Load() != 0 { + t.Fatal("new connection negotiated without ACK") + } + + for _, tc := range []struct { + name string + data []byte + compact, resync, valid bool + }{ + {"new controller", mustStatusAck(t, &statusproto.NodeStatusAck{Status: "ok", Revision: 1, PeerMeasurements: true}), true, false, true}, + {"old controller", mustStatusAck(t, &statusproto.NodeStatusAck{Status: "ok", Revision: 2}), false, false, true}, + {"JSON", []byte(`{"type":"node_status_ack","data":{"revision":3}}`), false, false, true}, + {"no base", mustStatusAck(t, &statusproto.NodeStatusAck{Status: "ok", PeerMeasurements: true}), false, false, true}, + {"resync", mustStatusAck(t, &statusproto.NodeStatusAck{Status: "resync_required", Revision: 4, PeerMeasurements: true}), false, true, true}, + {"invalid", []byte("invalid"), false, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + state.pending.Store(true) + + if state.accept(tc.data) != tc.valid || state.compact.Load() != tc.compact || state.resync.Load() != tc.resync { + t.Fatal("incorrect ACK negotiation") + } + + if state.pending.Load() == tc.valid { + t.Fatal("invalid pending-message state") + } + }) + } + + state.resync.Store(false) + + if !state.accept(mustStatusAck(t, &statusproto.NodeStatusAck{Status: "ok", Revision: 5, PeerMeasurements: true})) || !state.compact.Load() { + t.Fatal("full resync ACK did not renegotiate") + } + + reconnected := &statusAckState{} + if reconnected.compact.Load() || reconnected.revision.Load() != 0 { + t.Fatal("capability leaked across connections") + } +} + +func mustStatusAck(t *testing.T, ack *statusproto.NodeStatusAck) []byte { + t.Helper() + + data, err := proto.Marshal(ack) + if err != nil { + t.Fatal(err) + } + + return data +} + +func BenchmarkStatusDelta2000Peers(b *testing.B) { + prev, curr := measuredNodeStatus(), measuredNodeStatus() + + prev.Peers, curr.Peers = nil, nil + for i := range 2000 { + peer := measuredNodeStatus().Peers[0] + peer.Name = fmt.Sprintf("peer-%d", i) + peer.Tunnel.AllowedIPs = []string{"10.244.0.0/24"} + peer.PodCIDRGateways = []string{"10.244.0.1"} + peer.RouteDistances = map[string]int{"10.244.0.0/24": 1} + prev.Peers = append(prev.Peers, peer) + peer.Tunnel.RxBytes++ + curr.Peers = append(curr.Peers, peer) + } + + for _, mode := range []string{"legacy-json-full-peers", "typed-full-peers", "compact"} { + b.Run(mode, func(b *testing.B) { + b.ReportAllocs() + + for b.Loop() { + var delta *statusproto.NodeStatusDelta + + if mode == "legacy-json-full-peers" { + raw, err := computeStatusDelta(prev, curr) + if err != nil { + b.Fatal(err) + } + + delta = nodeStatusDeltaToProto(raw) + } else { + delta = typedStatusDelta(prev, curr, mode == "compact", false) + } + + data, err := proto.Marshal(delta) + if err != nil { + b.Fatal(err) + } + + b.ReportMetric(float64(len(data)), "wire-B/op") + } + }) + } +} diff --git a/cmd/unbounded-net-node/status_legacy_delta_test.go b/cmd/unbounded-net-node/status_legacy_delta_test.go new file mode 100644 index 000000000..1aaf39508 --- /dev/null +++ b/cmd/unbounded-net-node/status_legacy_delta_test.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// These legacy JSON helpers are retained only for existing compatibility tests +// and baseline benchmarks. Production publishers exclusively use typedStatusDelta, +// whose tests enforce critical metadata changes and explicit field clearings. +func computeStatusDelta(prev, curr *NodeStatusResponse) (map[string]json.RawMessage, error) { + if prev == nil { + return nil, nil + } + + prevRaw, err := json.Marshal(prev) + if err != nil { + return nil, err + } + + currRaw, err := json.Marshal(curr) + if err != nil { + return nil, err + } + + var prevMap map[string]json.RawMessage + if err := json.Unmarshal(prevRaw, &prevMap); err != nil { + return nil, err + } + + var currMap map[string]json.RawMessage + if err := json.Unmarshal(currRaw, &currMap); err != nil { + return nil, err + } + + delta := make(map[string]json.RawMessage) + if nodeInfo, ok := currMap["nodeInfo"]; ok { + delta["nodeInfo"] = nodeInfo + } + + for key, value := range currMap { + if key == "nodeInfo" { + continue + } + + prevValue, exists := prevMap[key] + if !exists || !bytes.Equal(prevValue, value) { + delta[key] = value + } + } + + if _, previouslyPresent := prevMap["nodeErrors"]; previouslyPresent { + if _, currentlyPresent := currMap["nodeErrors"]; !currentlyPresent { + delta["nodeErrors"] = json.RawMessage("[]") + } + } + // Preserve the old omission behavior for baseline comparisons only. + if len(delta) == 1 { + return nil, nil + } + + return delta, nil +} + +func nodeStatusDeltaToProto(delta map[string]json.RawMessage) *statusproto.NodeStatusDelta { + if len(delta) == 0 { + return nil + } + + pb := &statusproto.NodeStatusDelta{ + UpdatedFields: make([]string, 0, len(delta)), + } + for key, raw := range delta { + pb.UpdatedFields = append(pb.UpdatedFields, key) + + switch key { + case "nodeInfo": + var ni NodeInfo + if json.Unmarshal(raw, &ni) == nil { + pb.NodeInfo = nodeInfoToProto(&ni) + } + case "peers": + var peers []statusv1alpha1.PeerStatus + if json.Unmarshal(raw, &peers) == nil { + pb.Peers = peersToProto(peers) + } + case "routingTable": + var rt RoutingTableInfo + if json.Unmarshal(raw, &rt) == nil { + pb.RoutingTable = routingTableToProto(&rt) + } + case "healthCheck": + var hc HealthCheckStatus + if json.Unmarshal(raw, &hc) == nil { + pb.HealthCheck = healthCheckStatusToProto(&hc) + } + case "nodeErrors": + var errs []NodeError + if json.Unmarshal(raw, &errs) == nil { + pb.NodeErrors = nodeErrorsToProto(errs) + } + case "bpfEntries": + var entries []BpfEntry + if json.Unmarshal(raw, &entries) == nil { + pb.BpfEntries = bpfEntriesToProto(entries) + } + } + } + + return pb +} diff --git a/cmd/unbounded-net-node/status_proto.go b/cmd/unbounded-net-node/status_proto.go index b6d03f265..256a64711 100644 --- a/cmd/unbounded-net-node/status_proto.go +++ b/cmd/unbounded-net-node/status_proto.go @@ -4,12 +4,20 @@ package main import ( - "encoding/json" + "time" statusproto "github.com/Azure/unbounded/internal/net/status/proto" statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) +func statusUnixNano(t time.Time) int64 { + if t.IsZero() { + return 0 + } + + return t.UnixNano() +} + // nodeStatusToProto converts a Go NodeStatusResponse to the protobuf NodeStatusFull message. func nodeStatusToProto(status *NodeStatusResponse) *statusproto.NodeStatusFull { if status == nil { @@ -17,7 +25,7 @@ func nodeStatusToProto(status *NodeStatusResponse) *statusproto.NodeStatusFull { } full := &statusproto.NodeStatusFull{ - TimestampUnixNs: status.Timestamp.UnixNano(), + TimestampUnixNs: statusUnixNano(status.Timestamp), NodeInfo: nodeInfoToProto(&status.NodeInfo), Peers: peersToProto(status.Peers), RoutingTable: routingTableToProto(&status.RoutingTable), @@ -35,58 +43,6 @@ func nodeStatusToProto(status *NodeStatusResponse) *statusproto.NodeStatusFull { return full } -// nodeStatusDeltaToProto converts a JSON delta map to the protobuf NodeStatusDelta message. -// Each key in the map represents an updated top-level field; the values are the -// JSON-encoded field contents (already produced by computeStatusDelta). -func nodeStatusDeltaToProto(delta map[string]json.RawMessage) *statusproto.NodeStatusDelta { - if len(delta) == 0 { - return nil - } - - pb := &statusproto.NodeStatusDelta{ - UpdatedFields: make([]string, 0, len(delta)), - } - - for key, raw := range delta { - pb.UpdatedFields = append(pb.UpdatedFields, key) - - switch key { - case "nodeInfo": - var ni NodeInfo - if json.Unmarshal(raw, &ni) == nil { - pb.NodeInfo = nodeInfoToProto(&ni) - } - case "peers": - var peers []statusv1alpha1.PeerStatus - if json.Unmarshal(raw, &peers) == nil { - pb.Peers = peersToProto(peers) - } - case "routingTable": - var rt RoutingTableInfo - if json.Unmarshal(raw, &rt) == nil { - pb.RoutingTable = routingTableToProto(&rt) - } - case "healthCheck": - var hc HealthCheckStatus - if json.Unmarshal(raw, &hc) == nil { - pb.HealthCheck = healthCheckStatusToProto(&hc) - } - case "nodeErrors": - var errs []NodeError - if json.Unmarshal(raw, &errs) == nil { - pb.NodeErrors = nodeErrorsToProto(errs) - } - case "bpfEntries": - var entries []BpfEntry - if json.Unmarshal(raw, &entries) == nil { - pb.BpfEntries = bpfEntriesToProto(entries) - } - } - } - - return pb -} - // nodeInfoToProto converts a Go NodeInfo to its protobuf equivalent. func nodeInfoToProto(ni *NodeInfo) *statusproto.NodeInfo { if ni == nil { @@ -316,7 +272,7 @@ func healthCheckStatusToProto(hc *HealthCheckStatus) *statusproto.HealthCheckSta Healthy: hc.Healthy, Summary: hc.Summary, PeerCount: int32(hc.PeerCount), - CheckedAtUnixNs: hc.CheckedAt.UnixNano(), + CheckedAtUnixNs: statusUnixNano(hc.CheckedAt), } } @@ -364,7 +320,7 @@ func nodePodInfoToProto(npi *statusv1alpha1.NodePodInfo) *statusproto.NodePodInf return &statusproto.NodePodInfo{ PodName: npi.PodName, - StartTimeUnixNs: npi.StartTime.UnixNano(), + StartTimeUnixNs: statusUnixNano(npi.StartTime), Restarts: npi.Restarts, } } diff --git a/cmd/unbounded-net-node/status_server.go b/cmd/unbounded-net-node/status_server.go index de8671bf8..b27cd0751 100644 --- a/cmd/unbounded-net-node/status_server.go +++ b/cmd/unbounded-net-node/status_server.go @@ -845,66 +845,15 @@ func resolveFallbackStatusWebSocketURL(cfg *config, directWSURL string) string { return "" } -func computeStatusDelta(prev, curr *NodeStatusResponse) (map[string]json.RawMessage, error) { - if prev == nil { - return nil, nil - } - - prevRaw, err := json.Marshal(prev) - if err != nil { - return nil, err - } - - currRaw, err := json.Marshal(curr) - if err != nil { - return nil, err - } - - var prevMap map[string]json.RawMessage - if err := json.Unmarshal(prevRaw, &prevMap); err != nil { - return nil, err - } - - var currMap map[string]json.RawMessage - if err := json.Unmarshal(currRaw, &currMap); err != nil { - return nil, err - } - - delta := make(map[string]json.RawMessage) - if nodeInfo, ok := currMap["nodeInfo"]; ok { - delta["nodeInfo"] = nodeInfo - } - - for key, value := range currMap { - if key == "nodeInfo" { - continue - } - - prevValue, exists := prevMap[key] - if !exists || !bytes.Equal(prevValue, value) { - delta[key] = value - } - } - - if _, previouslyPresent := prevMap["nodeErrors"]; previouslyPresent { - if _, currentlyPresent := currMap["nodeErrors"]; !currentlyPresent { - delta["nodeErrors"] = json.RawMessage("[]") - } - } - // NOTE: don't emit "null" for keys present in prev but missing in curr. - // Go json.Marshal omits nil slices/pointers with omitempty, so a missing - // key in currMap usually means the field is nil/empty, not intentionally - // cleared. Emitting null would wipe out the controller's cached data. - - if len(delta) == 1 { - return nil, nil - } - - return delta, nil -} - func stripPeerStats(status *NodeStatusResponse) *NodeStatusResponse { clone := *status + clone.Timestamp = time.Time{} + + if status.HealthCheck != nil { + health := *status.HealthCheck + health.CheckedAt = time.Time{} + clone.HealthCheck = &health + } clone.Peers = make([]WireGuardPeerStatus, 0, len(status.Peers)) for _, peer := range status.Peers { @@ -912,6 +861,13 @@ func stripPeerStats(status *NodeStatusResponse) *NodeStatusResponse { peerCopy.Tunnel.RxBytes = 0 peerCopy.Tunnel.TxBytes = 0 peerCopy.Tunnel.LastHandshake = time.Time{} + + if peer.HealthCheck != nil { + health := *peer.HealthCheck + health.Uptime, health.RTT = "", "" + peerCopy.HealthCheck = &health + } + clone.Peers = append(clone.Peers, peerCopy) } @@ -1416,9 +1372,9 @@ func runStatusWebSocketPusher( var ( lastSentStatus *NodeStatusResponse lastCriticalSnapshot *NodeStatusResponse - revision atomic.Uint64 - resyncRequired atomic.Bool + acks statusAckState lastAckTimeNs atomic.Int64 + lastWriteTime time.Time ) lastAckTimeNs.Store(time.Now().UnixNano()) @@ -1435,47 +1391,8 @@ func runStatusWebSocketPusher( return } - lastAckTimeNs.Store(time.Now().UnixNano()) - - var ack statusproto.NodeStatusAck - if err := proto.Unmarshal(data, &ack); err != nil { - klog.V(4).Infof("Status websocket: failed to unmarshal protobuf ack, trying JSON fallback: %v", err) - // Fallback: try JSON for backward compatibility during rollout. - var envelope struct { - Type string `json:"type"` - Data nodeStatusPushAck `json:"data"` - } - if jsonErr := json.Unmarshal(data, &envelope); jsonErr != nil { - continue - } - - switch envelope.Type { - case "node_status_ack": - if envelope.Data.Revision > 0 { - revision.Store(envelope.Data.Revision) - } - case "node_status_resync": - if envelope.Data.Revision > 0 { - revision.Store(envelope.Data.Revision) - } - - resyncRequired.Store(true) - } - - continue - } - - switch ack.Status { - case "ok": - if ack.Revision > 0 { - revision.Store(ack.Revision) - } - case "resync_required": - if ack.Revision > 0 { - revision.Store(ack.Revision) - } - - resyncRequired.Store(true) + if acks.accept(data) { + lastAckTimeNs.Store(time.Now().UnixNano()) } } }() @@ -1486,6 +1403,11 @@ func runStatusWebSocketPusher( return err } + acks.resync.Store(false) + acks.pending.Store(true) + + lastWriteTime = time.Now() + if err := conn.Write(connCtx, websocket.MessageBinary, payload); err != nil { return err } @@ -1495,8 +1417,6 @@ func runStatusWebSocketPusher( lastSentStatus = status lastCriticalSnapshot = stripPeerStats(status) - resyncRequired.Store(false) - return nil } @@ -1611,7 +1531,11 @@ func runStatusWebSocketPusher( case <-readCtx.Done(): break loop case <-criticalTicker.C: - if resyncRequired.Load() || lastSentStatus == nil { + if acks.pending.Load() { + continue + } + + if acks.resync.Load() || lastSentStatus == nil { if err := sendFull(); err != nil { klog.V(2).Infof("Status websocket: resync full send failed: %v", err) break loop @@ -1627,16 +1551,18 @@ func runStatusWebSocketPusher( continue } - delta, err := computeStatusDelta(lastSentStatus, current) - if err != nil || len(delta) == 0 { + published := criticalStatus(lastSentStatus, current) + + delta := typedStatusDelta(lastSentStatus, published, acks.compact.Load(), false) + if delta == nil { continue } message := &statusproto.NodeStatusMessage{ Type: "node_status_delta", NodeName: current.NodeInfo.Name, - BaseRevision: revision.Load(), - Delta: nodeStatusDeltaToProto(delta), + BaseRevision: acks.revision.Load(), + Delta: delta, } payload, err := proto.Marshal(message) @@ -1644,20 +1570,36 @@ func runStatusWebSocketPusher( continue } + acks.pending.Store(true) + + lastWriteTime = time.Now() + if err := conn.Write(connCtx, websocket.MessageBinary, payload); err != nil { klog.V(2).Infof("Status websocket: critical delta write failed: %v", err) break loop } - lastSentStatus = current + lastSentStatus = published lastCriticalSnapshot = criticalSnapshot case <-statsTicker.C: + if acks.pending.Load() { + continue + } + + if acks.resync.Load() || lastSentStatus == nil { + if err := sendFull(); err != nil { + klog.V(2).Infof("Status websocket: stats resync failed: %v", err) + break loop + } + + continue + } // Always send a delta on the stats interval even if stats // appear unchanged, so the controller sees fresh timestamps. current := healthState.getStatusSnapshot() - delta, err := computeStatusDelta(lastSentStatus, current) - if err != nil || len(delta) == 0 { + delta := typedStatusDelta(lastSentStatus, current, acks.compact.Load(), true) + if delta == nil { // No computable delta -- fall back to full send. if err := sendFull(); err != nil { klog.V(2).Infof("Status websocket: stats fallback full send failed: %v", err) @@ -1670,8 +1612,8 @@ func runStatusWebSocketPusher( wsMsg := &statusproto.NodeStatusMessage{ Type: "node_status_delta", NodeName: current.NodeInfo.Name, - BaseRevision: revision.Load(), - Delta: nodeStatusDeltaToProto(delta), + BaseRevision: acks.revision.Load(), + Delta: delta, } payload, err := proto.Marshal(wsMsg) @@ -1679,6 +1621,10 @@ func runStatusWebSocketPusher( continue } + acks.pending.Store(true) + + lastWriteTime = time.Now() + if err := conn.Write(connCtx, websocket.MessageBinary, payload); err != nil { klog.V(2).Infof("Status websocket: stats delta write failed: %v", err) break loop @@ -1687,6 +1633,9 @@ func runStatusWebSocketPusher( lastSentStatus = current lastCriticalSnapshot = stripPeerStats(current) case <-fullSyncTicker.C: + if acks.pending.Load() { + continue + } // Forced full status sync to ensure the controller has // complete status regardless of delta accumulation. if err := sendFull(); err != nil { @@ -1734,6 +1683,11 @@ func runStatusWebSocketPusher( directRecoveryTimer.Reset(directRecoveryBackoff) } case <-fallbackCloseTicker.C: + if acks.pending.Load() && time.Since(lastWriteTime) > 30*time.Second { + klog.V(2).Info("Status websocket: status acknowledgment timed out") + break loop + } + if wsURL == fallbackWSURL && closeFallbackWS != nil && closeFallbackWS.Load() { closeFallbackWS.Store(false) @@ -2090,21 +2044,22 @@ func startStatusPusher( protoMsg := &statusproto.NodeStatusMessage{ Type: "node_status_full", NodeName: nodeStatus.NodeInfo.Name, - Status: nodeStatusToProto(nodeStatus), } if cfg.StatusPushDelta && !currentForceFull { - delta, deltaErr := computeStatusDelta(previousStatus, nodeStatus) - if deltaErr != nil { - klog.V(3).Infof("Status push: failed to compute delta: %v", deltaErr) - } else if len(delta) > 0 { + delta := typedStatusDelta(previousStatus, nodeStatus, false, true) + if delta != nil { mode = "delta" protoMsg.Type = "node_status_delta" protoMsg.BaseRevision = currentRevision protoMsg.Status = nil - protoMsg.Delta = nodeStatusDeltaToProto(delta) + protoMsg.Delta = delta } } + if protoMsg.Delta == nil { + protoMsg.Status = nodeStatusToProto(nodeStatus) + } + marshalStart := time.Now() data, err := proto.Marshal(protoMsg) @@ -2907,6 +2862,8 @@ func (s *nodeStatusServer) getNodeStatus() *NodeStatusResponse { } s.state.mu.Unlock() + sortStatusPeers(status.Peers) + // Collect routing table from kernel via netlink status.RoutingTable = s.collectRoutingTableFromKernel() diff --git a/docs/net/configuration.md b/docs/net/configuration.md index 87af1fbaf..1569b0785 100644 --- a/docs/net/configuration.md +++ b/docs/net/configuration.md @@ -383,7 +383,7 @@ logging both the kept and ignored peering/profile details. The node agent uses configurable API server mode for websocket and push behavior: -1. WebSocket transport (`/status/nodews` and aggregated API path) with JSON full+delta messages and compression. +1. WebSocket transport (`/status/nodews` and aggregated API path) with protobuf full+delta messages and compression. The controller also accepts JSON messages for compatibility. 2. Periodic HTTP push (`/status/push` and aggregated API path) when websocket is unavailable or configured for periodic reconciliation. 3. Controller pull fallback when push data is stale/unavailable. @@ -392,7 +392,35 @@ Direct controller routes, including `/token/node`, `/status/nodews`, and `/statu When TokenReview is selected at startup, each aggregated HTTP status upload and new WebSocket handshake requires a TokenReview API call. Positive authentication results are not cached, preserving API-server bound-object revocation checks. Large deployments should use local OIDC with a suitable explicit audience when automatic discovery is ambiguous, prefer direct HMAC transport, and size the aggregated HTTP push interval for outage load. `node.criticalDeltaEvery` (default `1s`) and `node.statsDeltaEvery` (default `15s`) are maximum publish frequencies. -The node only sends a delta when fields changed, and changed fields are queued up to each interval to batch related updates. +Critical updates are change-only: root collection timestamps, aggregate health-check timestamps, tunnel counters/handshakes, and peer health RTT/uptime do not trigger them. +Health state, health enablement, topology, metadata, routes, BPF entries, and errors (including clearing errors) remain critical. +When critical changes are published, existing peers retain their last published measurements; unrelated statistics wait for the statistics interval. +Statistics updates include a timestamp even when measurements are unchanged, so the controller continues to see fresh status. +Periodic full synchronization and full resynchronization after a rejected delta remain in place. + +New nodes use compact protobuf peer-measurement deltas only after the controller positively advertises `peer_measurements` in a successful WebSocket ACK with a nonzero revision. +The capability and revision are reset on every connection; the first message and any resynchronization are full snapshots. +Old controllers receive full peer replacements, and new controllers continue to accept legacy protobuf and JSON full/top-level deltas. +HTTP fallback uses typed top-level deltas without compact measurements and does not rely on a capability learned on a different connection. + +Compact measurements carry packed RX/TX/handshake columns and RTT/uptime string columns instead of repeated static peer metadata and nested health objects. +Each column replaces all measurements for the ordered base peer list, including zero and empty values. +A SHA-256 digest over ordered, length-prefixed `(name, protocol, interface, public key)` identities guards snapshot indices; duplicate or unnamed identities cannot use the compact path. +The controller requires a nonzero matching base revision, matching identities, matching column lengths, and an explicit `peerMeasurements` field mask. +Peer replacement and measurement updates cannot coexist in one message. +Invalid batches request a full resynchronization without partially updating the cache. +Metadata or topology changes, including reordered peers, still replace the peer list. +Fresh node snapshots sort peers by identity so map iteration does not cause artificial topology changes. +Cache updates copy peer values and health measurements while retaining immutable static metadata. +This reduces wire/decode work but does not eliminate the controller's per-node peer cache or full-refresh costs. + +Only one status message is outstanding per WebSocket connection; updates batch while its ACK is pending. +Missing status ACKs reconnect after 30 seconds rather than advancing an unacknowledged base. +The bounded-cardinality controller metric `unbounded_cni_controller_peer_measurement_updates_total{outcome="applied|resync|error"}` counts compact batches. +An increase in `outcome="applied"` confirms use of the negotiated compact path; `resync` denotes a missing/stale base and `error` denotes an invalid batch. +The metric deliberately has no node or peer labels. + +The controller decodes each protobuf WebSocket frame once, reusing the validated message for node-bound authorization and cache updates. Identity checks, full/delta revisions, and resynchronization behavior are unchanged. HTTP push also supports delta mode (`node.statusPushDelta`). If the controller cannot apply a delta (missing/mismatched base state), it returns `429` and the node immediately resends a full state on the next push. @@ -419,7 +447,7 @@ HTTP push also supports delta mode (`node.statusPushDelta`). If the controller c | `--status-ws-keepalive-interval` | duration | `10s` | Interval between node websocket keepalive pings (`0s` disables pings). | | `--status-ws-keepalive-failure-count` | int | `2` | Sequential websocket keepalive ping failures before the node reconnects. | | `--status-critical-interval` | duration | `1s` | Maximum critical-delta publish frequency; changed fields are batched and sent at most once per interval. | -| `--status-stats-interval` | duration | `15s` | Maximum statistics-delta publish frequency; changed fields are batched and sent at most once per interval. | +| `--status-stats-interval` | duration | `15s` | Statistics refresh interval; includes a freshness timestamp even when measurements are unchanged. | | `--shutdown-remove-wireguard-configuration` | bool | `false` | Remove WireGuard interfaces on node-agent shutdown. | | `--shutdown-cleanup-netlink` | bool | `false` | Remove managed netlink routes and policy routing rules on node-agent shutdown. | | `--enable-policy-routing` | bool | `false` | **Deprecated.** Enable connmark/fwmark/ip-rule policy-based routing on gateway WireGuard interfaces. Replaced by per-interface iptables FORWARD ACCEPT rules that are added when tunnel/WG gateway interfaces are created and removed on deletion. Set to `true` only for backward compatibility with pre-1.0.2 deployments. | @@ -677,12 +705,17 @@ resources: ### Scaling Considerations -| Cluster Size | Controller Memory | Informer Load | -|--------------|-------------------|---------------| -| < 100 nodes | 64Mi | Low | -| 100-500 nodes | 128Mi | Medium | -| 500-1000 nodes | 256Mi | High | -| > 1000 nodes | 512Mi+ | Very High | +Controller memory depends on diagnostic payload size and update frequency, not +just node count or informer load. The status cache retains each node's peer, +BPF, and routing details. In dense all-to-all topologies, the total peer and +BPF data can grow quadratically with node count. + +Decoding WebSocket frames once and avoiding connectivity-matrix peer copies +reduces transient allocations, but does not eliminate the cached status data. +Size controller resources using measured RSS and Go heap usage under the +expected topology and update rates, with headroom for garbage collection and +in-flight messages. The resource examples above are not sizing recommendations +for large clusters. --- diff --git a/docs/net/operations.md b/docs/net/operations.md index 943672319..77d3a073c 100644 --- a/docs/net/operations.md +++ b/docs/net/operations.md @@ -228,6 +228,10 @@ The dashboard uses **WebSocket** for real-time updates with delta compression, f - Expandable connectivity matrix with zoom and labels - Dark/light theme toggle +Connectivity matrices are omitted for site or gateway-pool scopes containing +more than 100 nodes. Smaller scopes remain visible even when other scopes +exceed that limit. + ### Health Endpoints #### Controller Health @@ -698,6 +702,23 @@ the next entry. A new collection starts with an empty interface cache, so interf renames, replacements, and MTU changes are not persistently cached. These changes do not alter routes, health-check intervals, or status-publishing intervals. +Controller compact updates retain their revision and identity checks but reuse +successful duplicate-identity validation while the ordered identity digest is +unchanged. A fixed-size per-node memo is invalidated when peers are replaced. +The digest is still recomputed for each update, so changed identities cannot +reuse stale validation. + +The controller also reuses up to four idle gzip response writers. Writers are +closed and detached from the completed response before reuse; failed or +interrupted responses discard their writers. This reduces discovery and error +response allocation without caching response contents or changing compression +negotiation. + +WebSocket input buffers are reused by size class and released after each frame +is processed, not retained for an idle connection's lifetime. The idle pool is +bounded below 64 MiB across connections; active frames still require their own +storage. The existing 2 MiB per-message limit remains enforced. + ### Unused Device Cleanup When the tunnel protocol changes for a peer (e.g., from GENEVE to VXLAN, or from diff --git a/internal/net/status/measurements.go b/internal/net/status/measurements.go new file mode 100644 index 000000000..0a3607fc1 --- /dev/null +++ b/internal/net/status/measurements.go @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" + "maps" + "slices" + + statusproto "github.com/Azure/unbounded/internal/net/status/proto" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// PeerIdentityDigest guards snapshot indices against reordered or replaced peers. +// Length prefixes avoid ambiguous concatenations; duplicate identities are invalid. +func PeerIdentityDigest(peers []statusv1alpha1.PeerStatus) ([]byte, error) { + digest := sha256.New() + seen := make(map[[4]string]struct{}, len(peers)) + + var size [8]byte + + for _, peer := range peers { + identity := [4]string{peer.Name, peer.Tunnel.Protocol, peer.Tunnel.Interface, peer.Tunnel.PublicKey} + if peer.Name == "" { + return nil, fmt.Errorf("peer identity has no name") + } + + if _, exists := seen[identity]; exists { + return nil, fmt.Errorf("duplicate peer identity") + } + + seen[identity] = struct{}{} + for _, value := range identity { + binary.BigEndian.PutUint64(size[:], uint64(len(value))) + _, _ = digest.Write(size[:]) + _, _ = digest.Write([]byte(value)) + } + } + + return digest.Sum(nil), nil +} + +// PeerMetadataEqual excludes only measurements, not health state or topology. +func PeerMetadataEqual(a, b statusv1alpha1.PeerStatus) bool { + ah, bh := a.HealthCheck, b.HealthCheck + if a.Name != b.Name || a.PeerType != b.PeerType || a.SiteName != b.SiteName || + a.SkipPodCIDRRoutes != b.SkipPodCIDRRoutes || + !slices.Equal(a.PodCIDRGateways, b.PodCIDRGateways) || + !slices.Equal(a.RouteDestinations, b.RouteDestinations) || + !maps.Equal(a.RouteDistances, b.RouteDistances) || + a.Tunnel.Protocol != b.Tunnel.Protocol || a.Tunnel.Interface != b.Tunnel.Interface || + a.Tunnel.PublicKey != b.Tunnel.PublicKey || a.Tunnel.Endpoint != b.Tunnel.Endpoint || + !slices.Equal(a.Tunnel.AllowedIPs, b.Tunnel.AllowedIPs) { + return false + } + + if ah == nil || bh == nil { + return ah == bh + } + + return ah.Enabled == bh.Enabled && ah.Status == bh.Status +} + +// PeerMeasurementsToProto uses packed scalar columns instead of nested peer DTOs. +func PeerMeasurementsToProto(peers []statusv1alpha1.PeerStatus) (*statusproto.PeerMeasurements, error) { + digest, err := PeerIdentityDigest(peers) + if err != nil { + return nil, err + } + + count := len(peers) + + pb := &statusproto.PeerMeasurements{ + PeerCount: uint32(count), + IdentityDigest: digest, + RxBytes: make([]int64, count), + TxBytes: make([]int64, count), + LastHandshakeUnixNs: make([]int64, count), + Uptime: make([]string, count), + Rtt: make([]string, count), + } + for i, peer := range peers { + pb.RxBytes[i] = peer.Tunnel.RxBytes + + pb.TxBytes[i] = peer.Tunnel.TxBytes + if !peer.Tunnel.LastHandshake.IsZero() { + pb.LastHandshakeUnixNs[i] = peer.Tunnel.LastHandshake.UnixNano() + } + + if peer.HealthCheck != nil { + pb.Uptime[i] = peer.HealthCheck.Uptime + pb.Rtt[i] = peer.HealthCheck.RTT + } + } + + return pb, nil +} diff --git a/internal/net/status/measurements_test.go b/internal/net/status/measurements_test.go new file mode 100644 index 000000000..d286ee017 --- /dev/null +++ b/internal/net/status/measurements_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package status + +import ( + "bytes" + "testing" + "time" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func TestPeerIdentityDigest(t *testing.T) { + peer := statusv1alpha1.PeerStatus{Name: "a", Tunnel: statusv1alpha1.PeerTunnelStatus{Protocol: "bc", Interface: "d", PublicKey: "e"}} + + original, err := PeerIdentityDigest([]statusv1alpha1.PeerStatus{peer}) + if err != nil { + t.Fatal(err) + } + + ambiguous := peer + ambiguous.Name, ambiguous.Tunnel.Protocol = "ab", "c" + + digest, err := PeerIdentityDigest([]statusv1alpha1.PeerStatus{ambiguous}) + if err != nil || bytes.Equal(original, digest) { + t.Fatal("identity fields require unambiguous length prefixes") + } + + another := peer + + another.Tunnel.Interface = "other" + if _, err := PeerIdentityDigest([]statusv1alpha1.PeerStatus{peer, another}); err != nil { + t.Fatal("different links to the same named peer must remain distinguishable") + } + + if _, err := PeerIdentityDigest([]statusv1alpha1.PeerStatus{peer, peer}); err == nil { + t.Fatal("duplicate identity accepted") + } + + peer.Name = "" + if _, err := PeerMeasurementsToProto([]statusv1alpha1.PeerStatus{peer}); err == nil { + t.Fatal("unnamed identity accepted") + } + + empty, err := PeerMeasurementsToProto(nil) + if err != nil || empty.PeerCount != 0 || len(empty.IdentityDigest) != 32 { + t.Fatal("empty peer set must have a valid digest") + } +} + +func TestPeerMetadataEqualAllFields(t *testing.T) { + base := statusv1alpha1.PeerStatus{Name: "peer"} + + for field, change := range map[string]func(*statusv1alpha1.PeerStatus){ + "name": func(p *statusv1alpha1.PeerStatus) { p.Name = "other" }, + "type": func(p *statusv1alpha1.PeerStatus) { p.PeerType = "site" }, + "site": func(p *statusv1alpha1.PeerStatus) { p.SiteName = "site-a" }, + "gateway": func(p *statusv1alpha1.PeerStatus) { p.PodCIDRGateways = []string{"10.0.0.1"} }, + "skip": func(p *statusv1alpha1.PeerStatus) { p.SkipPodCIDRRoutes = true }, + "distances": func(p *statusv1alpha1.PeerStatus) { p.RouteDistances = map[string]int{"10.0.0.0/24": 1} }, + "destinations": func(p *statusv1alpha1.PeerStatus) { p.RouteDestinations = []string{"10.0.0.0/24"} }, + "protocol": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.Protocol = "GENEVE" }, + "interface": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.Interface = "geneve0" }, + "key": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.PublicKey = "key" }, + "endpoint": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.Endpoint = "10.0.0.1" }, + "allowedIPs": func(p *statusv1alpha1.PeerStatus) { p.Tunnel.AllowedIPs = []string{"10.0.0.0/24"} }, + "health presence": func(p *statusv1alpha1.PeerStatus) { p.HealthCheck = &statusv1alpha1.HealthCheckPeerStatus{} }, + } { + t.Run(field, func(t *testing.T) { + changed := base + change(&changed) + + if PeerMetadataEqual(base, changed) || PeerMetadataEqual(changed, base) { + t.Fatal("metadata change ignored") + } + }) + } + + changed := base + changed.Tunnel.RxBytes, changed.Tunnel.TxBytes = 1, 2 + + changed.Tunnel.LastHandshake = time.Unix(123, 0) + if !PeerMetadataEqual(base, changed) { + t.Fatal("measurements are not metadata") + } + + base.HealthCheck = &statusv1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "up"} + + changed.HealthCheck = &statusv1alpha1.HealthCheckPeerStatus{Enabled: true, Status: "up", Uptime: "1h", RTT: "5ms"} + if !PeerMetadataEqual(base, changed) { + t.Fatal("health measurements are not metadata") + } +} diff --git a/internal/net/status/proto/status.pb.go b/internal/net/status/proto/status.pb.go index da066505d..3330a065b 100644 --- a/internal/net/status/proto/status.pb.go +++ b/internal/net/status/proto/status.pb.go @@ -103,12 +103,13 @@ func (x *NodeStatusMessage) GetDelta() *NodeStatusDelta { // NodeStatusAck is the acknowledgment returned by the controller for push updates. type NodeStatusAck struct { - state protoimpl.MessageState `protogen:"open.v1"` - Status string `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` // "ok" or "resync_required" - Revision uint64 `protobuf:"varint,2,opt,name=revision,proto3" json:"revision,omitempty"` - Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Status string `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` // "ok" or "resync_required" + Revision uint64 `protobuf:"varint,2,opt,name=revision,proto3" json:"revision,omitempty"` + Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` + PeerMeasurements bool `protobuf:"varint,4,opt,name=peer_measurements,json=peerMeasurements,proto3" json:"peer_measurements,omitempty"` // Positive capability, scoped to this WebSocket connection. + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeStatusAck) Reset() { @@ -162,6 +163,13 @@ func (x *NodeStatusAck) GetReason() string { return "" } +func (x *NodeStatusAck) GetPeerMeasurements() bool { + if x != nil { + return x.PeerMeasurements + } + return false +} + // NodeStatusFull mirrors the complete NodeStatusResponse payload. type NodeStatusFull struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -296,10 +304,16 @@ type NodeStatusDelta struct { HealthCheck *HealthCheckStatus `protobuf:"bytes,4,opt,name=health_check,json=healthCheck,proto3" json:"health_check,omitempty"` NodeErrors []*NodeError `protobuf:"bytes,5,rep,name=node_errors,json=nodeErrors,proto3" json:"node_errors,omitempty"` // Track which top-level fields are present in this delta. - UpdatedFields []string `protobuf:"bytes,15,rep,name=updated_fields,json=updatedFields,proto3" json:"updated_fields,omitempty"` - BpfEntries []*BpfEntry `protobuf:"bytes,6,rep,name=bpf_entries,json=bpfEntries,proto3" json:"bpf_entries,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + UpdatedFields []string `protobuf:"bytes,15,rep,name=updated_fields,json=updatedFields,proto3" json:"updated_fields,omitempty"` + BpfEntries []*BpfEntry `protobuf:"bytes,6,rep,name=bpf_entries,json=bpfEntries,proto3" json:"bpf_entries,omitempty"` + PeerMeasurements *PeerMeasurements `protobuf:"bytes,7,opt,name=peer_measurements,json=peerMeasurements,proto3" json:"peer_measurements,omitempty"` + TimestampUnixNs int64 `protobuf:"varint,8,opt,name=timestamp_unix_ns,json=timestampUnixNs,proto3" json:"timestamp_unix_ns,omitempty"` + FetchError string `protobuf:"bytes,9,opt,name=fetch_error,json=fetchError,proto3" json:"fetch_error,omitempty"` + LastPushTimeUnixNs int64 `protobuf:"varint,10,opt,name=last_push_time_unix_ns,json=lastPushTimeUnixNs,proto3" json:"last_push_time_unix_ns,omitempty"` + StatusSource string `protobuf:"bytes,11,opt,name=status_source,json=statusSource,proto3" json:"status_source,omitempty"` + NodePodInfo *NodePodInfo `protobuf:"bytes,12,opt,name=node_pod_info,json=nodePodInfo,proto3" json:"node_pod_info,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeStatusDelta) Reset() { @@ -381,6 +395,145 @@ func (x *NodeStatusDelta) GetBpfEntries() []*BpfEntry { return nil } +func (x *NodeStatusDelta) GetPeerMeasurements() *PeerMeasurements { + if x != nil { + return x.PeerMeasurements + } + return nil +} + +func (x *NodeStatusDelta) GetTimestampUnixNs() int64 { + if x != nil { + return x.TimestampUnixNs + } + return 0 +} + +func (x *NodeStatusDelta) GetFetchError() string { + if x != nil { + return x.FetchError + } + return "" +} + +func (x *NodeStatusDelta) GetLastPushTimeUnixNs() int64 { + if x != nil { + return x.LastPushTimeUnixNs + } + return 0 +} + +func (x *NodeStatusDelta) GetStatusSource() string { + if x != nil { + return x.StatusSource + } + return "" +} + +func (x *NodeStatusDelta) GetNodePodInfo() *NodePodInfo { + if x != nil { + return x.NodePodInfo + } + return nil +} + +// PeerMeasurements replaces all measurement columns for the ordered base peers. +// Every column must have exactly peer_count entries, including zero/empty values. +// identity_digest is SHA-256 over the ordered, length-prefixed peer identities. +// A nonzero matching base_revision and matching digest are required. This field +// must be listed in updated_fields and cannot accompany a peers replacement. +type PeerMeasurements struct { + state protoimpl.MessageState `protogen:"open.v1"` + PeerCount uint32 `protobuf:"varint,1,opt,name=peer_count,json=peerCount,proto3" json:"peer_count,omitempty"` + IdentityDigest []byte `protobuf:"bytes,2,opt,name=identity_digest,json=identityDigest,proto3" json:"identity_digest,omitempty"` + RxBytes []int64 `protobuf:"varint,3,rep,packed,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` + TxBytes []int64 `protobuf:"varint,4,rep,packed,name=tx_bytes,json=txBytes,proto3" json:"tx_bytes,omitempty"` + LastHandshakeUnixNs []int64 `protobuf:"varint,5,rep,packed,name=last_handshake_unix_ns,json=lastHandshakeUnixNs,proto3" json:"last_handshake_unix_ns,omitempty"` + Uptime []string `protobuf:"bytes,6,rep,name=uptime,proto3" json:"uptime,omitempty"` + Rtt []string `protobuf:"bytes,7,rep,name=rtt,proto3" json:"rtt,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PeerMeasurements) Reset() { + *x = PeerMeasurements{} + mi := &file_status_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PeerMeasurements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PeerMeasurements) ProtoMessage() {} + +func (x *PeerMeasurements) ProtoReflect() protoreflect.Message { + mi := &file_status_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PeerMeasurements.ProtoReflect.Descriptor instead. +func (*PeerMeasurements) Descriptor() ([]byte, []int) { + return file_status_proto_rawDescGZIP(), []int{4} +} + +func (x *PeerMeasurements) GetPeerCount() uint32 { + if x != nil { + return x.PeerCount + } + return 0 +} + +func (x *PeerMeasurements) GetIdentityDigest() []byte { + if x != nil { + return x.IdentityDigest + } + return nil +} + +func (x *PeerMeasurements) GetRxBytes() []int64 { + if x != nil { + return x.RxBytes + } + return nil +} + +func (x *PeerMeasurements) GetTxBytes() []int64 { + if x != nil { + return x.TxBytes + } + return nil +} + +func (x *PeerMeasurements) GetLastHandshakeUnixNs() []int64 { + if x != nil { + return x.LastHandshakeUnixNs + } + return nil +} + +func (x *PeerMeasurements) GetUptime() []string { + if x != nil { + return x.Uptime + } + return nil +} + +func (x *PeerMeasurements) GetRtt() []string { + if x != nil { + return x.Rtt + } + return nil +} + // NodeInfo contains basic node identification and metadata. type NodeInfo struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -407,7 +560,7 @@ type NodeInfo struct { func (x *NodeInfo) Reset() { *x = NodeInfo{} - mi := &file_status_proto_msgTypes[4] + mi := &file_status_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -419,7 +572,7 @@ func (x *NodeInfo) String() string { func (*NodeInfo) ProtoMessage() {} func (x *NodeInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[4] + mi := &file_status_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -432,7 +585,7 @@ func (x *NodeInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use NodeInfo.ProtoReflect.Descriptor instead. func (*NodeInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{4} + return file_status_proto_rawDescGZIP(), []int{5} } func (x *NodeInfo) GetName() string { @@ -566,7 +719,7 @@ type BuildInfo struct { func (x *BuildInfo) Reset() { *x = BuildInfo{} - mi := &file_status_proto_msgTypes[5] + mi := &file_status_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -578,7 +731,7 @@ func (x *BuildInfo) String() string { func (*BuildInfo) ProtoMessage() {} func (x *BuildInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[5] + mi := &file_status_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -591,7 +744,7 @@ func (x *BuildInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use BuildInfo.ProtoReflect.Descriptor instead. func (*BuildInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{5} + return file_status_proto_rawDescGZIP(), []int{6} } func (x *BuildInfo) GetVersion() string { @@ -628,7 +781,7 @@ type WireGuardStatusInfo struct { func (x *WireGuardStatusInfo) Reset() { *x = WireGuardStatusInfo{} - mi := &file_status_proto_msgTypes[6] + mi := &file_status_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -640,7 +793,7 @@ func (x *WireGuardStatusInfo) String() string { func (*WireGuardStatusInfo) ProtoMessage() {} func (x *WireGuardStatusInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[6] + mi := &file_status_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -653,7 +806,7 @@ func (x *WireGuardStatusInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use WireGuardStatusInfo.ProtoReflect.Descriptor instead. func (*WireGuardStatusInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{6} + return file_status_proto_rawDescGZIP(), []int{7} } func (x *WireGuardStatusInfo) GetInterface() string { @@ -702,7 +855,7 @@ type PeerStatus struct { func (x *PeerStatus) Reset() { *x = PeerStatus{} - mi := &file_status_proto_msgTypes[7] + mi := &file_status_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -714,7 +867,7 @@ func (x *PeerStatus) String() string { func (*PeerStatus) ProtoMessage() {} func (x *PeerStatus) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[7] + mi := &file_status_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -727,7 +880,7 @@ func (x *PeerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use PeerStatus.ProtoReflect.Descriptor instead. func (*PeerStatus) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{7} + return file_status_proto_rawDescGZIP(), []int{8} } func (x *PeerStatus) GetName() string { @@ -810,7 +963,7 @@ type PeerTunnelStatus struct { func (x *PeerTunnelStatus) Reset() { *x = PeerTunnelStatus{} - mi := &file_status_proto_msgTypes[8] + mi := &file_status_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -822,7 +975,7 @@ func (x *PeerTunnelStatus) String() string { func (*PeerTunnelStatus) ProtoMessage() {} func (x *PeerTunnelStatus) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[8] + mi := &file_status_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -835,7 +988,7 @@ func (x *PeerTunnelStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use PeerTunnelStatus.ProtoReflect.Descriptor instead. func (*PeerTunnelStatus) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{8} + return file_status_proto_rawDescGZIP(), []int{9} } func (x *PeerTunnelStatus) GetProtocol() string { @@ -906,7 +1059,7 @@ type RoutingTableInfo struct { func (x *RoutingTableInfo) Reset() { *x = RoutingTableInfo{} - mi := &file_status_proto_msgTypes[9] + mi := &file_status_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -918,7 +1071,7 @@ func (x *RoutingTableInfo) String() string { func (*RoutingTableInfo) ProtoMessage() {} func (x *RoutingTableInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[9] + mi := &file_status_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -931,7 +1084,7 @@ func (x *RoutingTableInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use RoutingTableInfo.ProtoReflect.Descriptor instead. func (*RoutingTableInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{9} + return file_status_proto_rawDescGZIP(), []int{10} } func (x *RoutingTableInfo) GetRoutes() []*RouteEntry { @@ -968,7 +1121,7 @@ type RouteEntry struct { func (x *RouteEntry) Reset() { *x = RouteEntry{} - mi := &file_status_proto_msgTypes[10] + mi := &file_status_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -980,7 +1133,7 @@ func (x *RouteEntry) String() string { func (*RouteEntry) ProtoMessage() {} func (x *RouteEntry) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[10] + mi := &file_status_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -993,7 +1146,7 @@ func (x *RouteEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use RouteEntry.ProtoReflect.Descriptor instead. func (*RouteEntry) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{10} + return file_status_proto_rawDescGZIP(), []int{11} } func (x *RouteEntry) GetDestination() string { @@ -1043,7 +1196,7 @@ type NextHop struct { func (x *NextHop) Reset() { *x = NextHop{} - mi := &file_status_proto_msgTypes[11] + mi := &file_status_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1055,7 +1208,7 @@ func (x *NextHop) String() string { func (*NextHop) ProtoMessage() {} func (x *NextHop) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[11] + mi := &file_status_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1068,7 +1221,7 @@ func (x *NextHop) ProtoReflect() protoreflect.Message { // Deprecated: Use NextHop.ProtoReflect.Descriptor instead. func (*NextHop) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{11} + return file_status_proto_rawDescGZIP(), []int{12} } func (x *NextHop) GetGateway() string { @@ -1151,7 +1304,7 @@ type OptionalBool struct { func (x *OptionalBool) Reset() { *x = OptionalBool{} - mi := &file_status_proto_msgTypes[12] + mi := &file_status_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1163,7 +1316,7 @@ func (x *OptionalBool) String() string { func (*OptionalBool) ProtoMessage() {} func (x *OptionalBool) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[12] + mi := &file_status_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1176,7 +1329,7 @@ func (x *OptionalBool) ProtoReflect() protoreflect.Message { // Deprecated: Use OptionalBool.ProtoReflect.Descriptor instead. func (*OptionalBool) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{12} + return file_status_proto_rawDescGZIP(), []int{13} } func (x *OptionalBool) GetValue() bool { @@ -1198,7 +1351,7 @@ type NextHopInfo struct { func (x *NextHopInfo) Reset() { *x = NextHopInfo{} - mi := &file_status_proto_msgTypes[13] + mi := &file_status_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1210,7 +1363,7 @@ func (x *NextHopInfo) String() string { func (*NextHopInfo) ProtoMessage() {} func (x *NextHopInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[13] + mi := &file_status_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1223,7 +1376,7 @@ func (x *NextHopInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use NextHopInfo.ProtoReflect.Descriptor instead. func (*NextHopInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{13} + return file_status_proto_rawDescGZIP(), []int{14} } func (x *NextHopInfo) GetObjectName() string { @@ -1258,7 +1411,7 @@ type RouteType struct { func (x *RouteType) Reset() { *x = RouteType{} - mi := &file_status_proto_msgTypes[14] + mi := &file_status_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1270,7 +1423,7 @@ func (x *RouteType) String() string { func (*RouteType) ProtoMessage() {} func (x *RouteType) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[14] + mi := &file_status_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1283,7 +1436,7 @@ func (x *RouteType) ProtoReflect() protoreflect.Message { // Deprecated: Use RouteType.ProtoReflect.Descriptor instead. func (*RouteType) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{14} + return file_status_proto_rawDescGZIP(), []int{15} } func (x *RouteType) GetType() string { @@ -1313,7 +1466,7 @@ type HealthCheckStatus struct { func (x *HealthCheckStatus) Reset() { *x = HealthCheckStatus{} - mi := &file_status_proto_msgTypes[15] + mi := &file_status_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1325,7 +1478,7 @@ func (x *HealthCheckStatus) String() string { func (*HealthCheckStatus) ProtoMessage() {} func (x *HealthCheckStatus) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[15] + mi := &file_status_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1338,7 +1491,7 @@ func (x *HealthCheckStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use HealthCheckStatus.ProtoReflect.Descriptor instead. func (*HealthCheckStatus) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{15} + return file_status_proto_rawDescGZIP(), []int{16} } func (x *HealthCheckStatus) GetHealthy() bool { @@ -1382,7 +1535,7 @@ type HealthCheckPeerStatus struct { func (x *HealthCheckPeerStatus) Reset() { *x = HealthCheckPeerStatus{} - mi := &file_status_proto_msgTypes[16] + mi := &file_status_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1394,7 +1547,7 @@ func (x *HealthCheckPeerStatus) String() string { func (*HealthCheckPeerStatus) ProtoMessage() {} func (x *HealthCheckPeerStatus) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[16] + mi := &file_status_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1407,7 +1560,7 @@ func (x *HealthCheckPeerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use HealthCheckPeerStatus.ProtoReflect.Descriptor instead. func (*HealthCheckPeerStatus) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{16} + return file_status_proto_rawDescGZIP(), []int{17} } func (x *HealthCheckPeerStatus) GetEnabled() bool { @@ -1450,7 +1603,7 @@ type NodeError struct { func (x *NodeError) Reset() { *x = NodeError{} - mi := &file_status_proto_msgTypes[17] + mi := &file_status_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1462,7 +1615,7 @@ func (x *NodeError) String() string { func (*NodeError) ProtoMessage() {} func (x *NodeError) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[17] + mi := &file_status_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1475,7 +1628,7 @@ func (x *NodeError) ProtoReflect() protoreflect.Message { // Deprecated: Use NodeError.ProtoReflect.Descriptor instead. func (*NodeError) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{17} + return file_status_proto_rawDescGZIP(), []int{18} } func (x *NodeError) GetType() string { @@ -1511,7 +1664,7 @@ type NodePodInfo struct { func (x *NodePodInfo) Reset() { *x = NodePodInfo{} - mi := &file_status_proto_msgTypes[18] + mi := &file_status_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1523,7 +1676,7 @@ func (x *NodePodInfo) String() string { func (*NodePodInfo) ProtoMessage() {} func (x *NodePodInfo) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[18] + mi := &file_status_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1536,7 +1689,7 @@ func (x *NodePodInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use NodePodInfo.ProtoReflect.Descriptor instead. func (*NodePodInfo) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{18} + return file_status_proto_rawDescGZIP(), []int{19} } func (x *NodePodInfo) GetPodName() string { @@ -1578,7 +1731,7 @@ type BpfEntry struct { func (x *BpfEntry) Reset() { *x = BpfEntry{} - mi := &file_status_proto_msgTypes[19] + mi := &file_status_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1590,7 +1743,7 @@ func (x *BpfEntry) String() string { func (*BpfEntry) ProtoMessage() {} func (x *BpfEntry) ProtoReflect() protoreflect.Message { - mi := &file_status_proto_msgTypes[19] + mi := &file_status_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1603,7 +1756,7 @@ func (x *BpfEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use BpfEntry.ProtoReflect.Descriptor instead. func (*BpfEntry) Descriptor() ([]byte, []int) { - return file_status_proto_rawDescGZIP(), []int{19} + return file_status_proto_rawDescGZIP(), []int{20} } func (x *BpfEntry) GetCidr() string { @@ -1679,11 +1832,12 @@ const file_status_proto_rawDesc = "" + "\tnode_name\x18\x02 \x01(\tR\bnodeName\x12#\n" + "\rbase_revision\x18\x03 \x01(\x04R\fbaseRevision\x12>\n" + "\x06status\x18\x04 \x01(\v2&.unboundednet.status.v1.NodeStatusFullR\x06status\x12=\n" + - "\x05delta\x18\x05 \x01(\v2'.unboundednet.status.v1.NodeStatusDeltaR\x05delta\"[\n" + + "\x05delta\x18\x05 \x01(\v2'.unboundednet.status.v1.NodeStatusDeltaR\x05delta\"\x88\x01\n" + "\rNodeStatusAck\x12\x16\n" + "\x06status\x18\x01 \x01(\tR\x06status\x12\x1a\n" + "\brevision\x18\x02 \x01(\x04R\brevision\x12\x16\n" + - "\x06reason\x18\x03 \x01(\tR\x06reason\"\x9c\x05\n" + + "\x06reason\x18\x03 \x01(\tR\x06reason\x12+\n" + + "\x11peer_measurements\x18\x04 \x01(\bR\x10peerMeasurements\"\x9c\x05\n" + "\x0eNodeStatusFull\x12*\n" + "\x11timestamp_unix_ns\x18\x01 \x01(\x03R\x0ftimestampUnixNs\x12=\n" + "\tnode_info\x18\x02 \x01(\v2 .unboundednet.status.v1.NodeInfoR\bnodeInfo\x128\n" + @@ -1699,7 +1853,7 @@ const file_status_proto_rawDesc = "" + "\rnode_pod_info\x18\n" + " \x01(\v2#.unboundednet.status.v1.NodePodInfoR\vnodePodInfo\x12A\n" + "\vbpf_entries\x18\v \x03(\v2 .unboundednet.status.v1.BpfEntryR\n" + - "bpfEntries\"\xd5\x03\n" + + "bpfEntries\"\x9b\x06\n" + "\x0fNodeStatusDelta\x12=\n" + "\tnode_info\x18\x01 \x01(\v2 .unboundednet.status.v1.NodeInfoR\bnodeInfo\x128\n" + "\x05peers\x18\x02 \x03(\v2\".unboundednet.status.v1.PeerStatusR\x05peers\x12M\n" + @@ -1709,7 +1863,24 @@ const file_status_proto_rawDesc = "" + "nodeErrors\x12%\n" + "\x0eupdated_fields\x18\x0f \x03(\tR\rupdatedFields\x12A\n" + "\vbpf_entries\x18\x06 \x03(\v2 .unboundednet.status.v1.BpfEntryR\n" + - "bpfEntries\"\xc5\x05\n" + + "bpfEntries\x12U\n" + + "\x11peer_measurements\x18\a \x01(\v2(.unboundednet.status.v1.PeerMeasurementsR\x10peerMeasurements\x12*\n" + + "\x11timestamp_unix_ns\x18\b \x01(\x03R\x0ftimestampUnixNs\x12\x1f\n" + + "\vfetch_error\x18\t \x01(\tR\n" + + "fetchError\x122\n" + + "\x16last_push_time_unix_ns\x18\n" + + " \x01(\x03R\x12lastPushTimeUnixNs\x12#\n" + + "\rstatus_source\x18\v \x01(\tR\fstatusSource\x12G\n" + + "\rnode_pod_info\x18\f \x01(\v2#.unboundednet.status.v1.NodePodInfoR\vnodePodInfo\"\xef\x01\n" + + "\x10PeerMeasurements\x12\x1d\n" + + "\n" + + "peer_count\x18\x01 \x01(\rR\tpeerCount\x12'\n" + + "\x0fidentity_digest\x18\x02 \x01(\fR\x0eidentityDigest\x12\x19\n" + + "\brx_bytes\x18\x03 \x03(\x03R\arxBytes\x12\x19\n" + + "\btx_bytes\x18\x04 \x03(\x03R\atxBytes\x123\n" + + "\x16last_handshake_unix_ns\x18\x05 \x03(\x03R\x13lastHandshakeUnixNs\x12\x16\n" + + "\x06uptime\x18\x06 \x03(\tR\x06uptime\x12\x10\n" + + "\x03rtt\x18\a \x03(\tR\x03rtt\"\xc5\x05\n" + "\bNodeInfo\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1b\n" + "\tsite_name\x18\x02 \x01(\tR\bsiteName\x12\x1d\n" + @@ -1854,64 +2025,67 @@ func file_status_proto_rawDescGZIP() []byte { return file_status_proto_rawDescData } -var file_status_proto_msgTypes = make([]protoimpl.MessageInfo, 22) +var file_status_proto_msgTypes = make([]protoimpl.MessageInfo, 23) var file_status_proto_goTypes = []any{ (*NodeStatusMessage)(nil), // 0: unboundednet.status.v1.NodeStatusMessage (*NodeStatusAck)(nil), // 1: unboundednet.status.v1.NodeStatusAck (*NodeStatusFull)(nil), // 2: unboundednet.status.v1.NodeStatusFull (*NodeStatusDelta)(nil), // 3: unboundednet.status.v1.NodeStatusDelta - (*NodeInfo)(nil), // 4: unboundednet.status.v1.NodeInfo - (*BuildInfo)(nil), // 5: unboundednet.status.v1.BuildInfo - (*WireGuardStatusInfo)(nil), // 6: unboundednet.status.v1.WireGuardStatusInfo - (*PeerStatus)(nil), // 7: unboundednet.status.v1.PeerStatus - (*PeerTunnelStatus)(nil), // 8: unboundednet.status.v1.PeerTunnelStatus - (*RoutingTableInfo)(nil), // 9: unboundednet.status.v1.RoutingTableInfo - (*RouteEntry)(nil), // 10: unboundednet.status.v1.RouteEntry - (*NextHop)(nil), // 11: unboundednet.status.v1.NextHop - (*OptionalBool)(nil), // 12: unboundednet.status.v1.OptionalBool - (*NextHopInfo)(nil), // 13: unboundednet.status.v1.NextHopInfo - (*RouteType)(nil), // 14: unboundednet.status.v1.RouteType - (*HealthCheckStatus)(nil), // 15: unboundednet.status.v1.HealthCheckStatus - (*HealthCheckPeerStatus)(nil), // 16: unboundednet.status.v1.HealthCheckPeerStatus - (*NodeError)(nil), // 17: unboundednet.status.v1.NodeError - (*NodePodInfo)(nil), // 18: unboundednet.status.v1.NodePodInfo - (*BpfEntry)(nil), // 19: unboundednet.status.v1.BpfEntry - nil, // 20: unboundednet.status.v1.NodeInfo.K8sLabelsEntry - nil, // 21: unboundednet.status.v1.PeerStatus.RouteDistancesEntry + (*PeerMeasurements)(nil), // 4: unboundednet.status.v1.PeerMeasurements + (*NodeInfo)(nil), // 5: unboundednet.status.v1.NodeInfo + (*BuildInfo)(nil), // 6: unboundednet.status.v1.BuildInfo + (*WireGuardStatusInfo)(nil), // 7: unboundednet.status.v1.WireGuardStatusInfo + (*PeerStatus)(nil), // 8: unboundednet.status.v1.PeerStatus + (*PeerTunnelStatus)(nil), // 9: unboundednet.status.v1.PeerTunnelStatus + (*RoutingTableInfo)(nil), // 10: unboundednet.status.v1.RoutingTableInfo + (*RouteEntry)(nil), // 11: unboundednet.status.v1.RouteEntry + (*NextHop)(nil), // 12: unboundednet.status.v1.NextHop + (*OptionalBool)(nil), // 13: unboundednet.status.v1.OptionalBool + (*NextHopInfo)(nil), // 14: unboundednet.status.v1.NextHopInfo + (*RouteType)(nil), // 15: unboundednet.status.v1.RouteType + (*HealthCheckStatus)(nil), // 16: unboundednet.status.v1.HealthCheckStatus + (*HealthCheckPeerStatus)(nil), // 17: unboundednet.status.v1.HealthCheckPeerStatus + (*NodeError)(nil), // 18: unboundednet.status.v1.NodeError + (*NodePodInfo)(nil), // 19: unboundednet.status.v1.NodePodInfo + (*BpfEntry)(nil), // 20: unboundednet.status.v1.BpfEntry + nil, // 21: unboundednet.status.v1.NodeInfo.K8sLabelsEntry + nil, // 22: unboundednet.status.v1.PeerStatus.RouteDistancesEntry } var file_status_proto_depIdxs = []int32{ 2, // 0: unboundednet.status.v1.NodeStatusMessage.status:type_name -> unboundednet.status.v1.NodeStatusFull 3, // 1: unboundednet.status.v1.NodeStatusMessage.delta:type_name -> unboundednet.status.v1.NodeStatusDelta - 4, // 2: unboundednet.status.v1.NodeStatusFull.node_info:type_name -> unboundednet.status.v1.NodeInfo - 7, // 3: unboundednet.status.v1.NodeStatusFull.peers:type_name -> unboundednet.status.v1.PeerStatus - 9, // 4: unboundednet.status.v1.NodeStatusFull.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo - 15, // 5: unboundednet.status.v1.NodeStatusFull.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus - 17, // 6: unboundednet.status.v1.NodeStatusFull.node_errors:type_name -> unboundednet.status.v1.NodeError - 18, // 7: unboundednet.status.v1.NodeStatusFull.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo - 19, // 8: unboundednet.status.v1.NodeStatusFull.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry - 4, // 9: unboundednet.status.v1.NodeStatusDelta.node_info:type_name -> unboundednet.status.v1.NodeInfo - 7, // 10: unboundednet.status.v1.NodeStatusDelta.peers:type_name -> unboundednet.status.v1.PeerStatus - 9, // 11: unboundednet.status.v1.NodeStatusDelta.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo - 15, // 12: unboundednet.status.v1.NodeStatusDelta.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus - 17, // 13: unboundednet.status.v1.NodeStatusDelta.node_errors:type_name -> unboundednet.status.v1.NodeError - 19, // 14: unboundednet.status.v1.NodeStatusDelta.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry - 5, // 15: unboundednet.status.v1.NodeInfo.build_info:type_name -> unboundednet.status.v1.BuildInfo - 6, // 16: unboundednet.status.v1.NodeInfo.wire_guard:type_name -> unboundednet.status.v1.WireGuardStatusInfo - 20, // 17: unboundednet.status.v1.NodeInfo.k8s_labels:type_name -> unboundednet.status.v1.NodeInfo.K8sLabelsEntry - 21, // 18: unboundednet.status.v1.PeerStatus.route_distances:type_name -> unboundednet.status.v1.PeerStatus.RouteDistancesEntry - 8, // 19: unboundednet.status.v1.PeerStatus.tunnel:type_name -> unboundednet.status.v1.PeerTunnelStatus - 16, // 20: unboundednet.status.v1.PeerStatus.health_check:type_name -> unboundednet.status.v1.HealthCheckPeerStatus - 10, // 21: unboundednet.status.v1.RoutingTableInfo.routes:type_name -> unboundednet.status.v1.RouteEntry - 11, // 22: unboundednet.status.v1.RouteEntry.next_hops:type_name -> unboundednet.status.v1.NextHop - 14, // 23: unboundednet.status.v1.NextHop.route_types:type_name -> unboundednet.status.v1.RouteType - 12, // 24: unboundednet.status.v1.NextHop.expected:type_name -> unboundednet.status.v1.OptionalBool - 12, // 25: unboundednet.status.v1.NextHop.present:type_name -> unboundednet.status.v1.OptionalBool - 13, // 26: unboundednet.status.v1.NextHop.info:type_name -> unboundednet.status.v1.NextHopInfo - 27, // [27:27] is the sub-list for method output_type - 27, // [27:27] is the sub-list for method input_type - 27, // [27:27] is the sub-list for extension type_name - 27, // [27:27] is the sub-list for extension extendee - 0, // [0:27] is the sub-list for field type_name + 5, // 2: unboundednet.status.v1.NodeStatusFull.node_info:type_name -> unboundednet.status.v1.NodeInfo + 8, // 3: unboundednet.status.v1.NodeStatusFull.peers:type_name -> unboundednet.status.v1.PeerStatus + 10, // 4: unboundednet.status.v1.NodeStatusFull.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo + 16, // 5: unboundednet.status.v1.NodeStatusFull.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 6: unboundednet.status.v1.NodeStatusFull.node_errors:type_name -> unboundednet.status.v1.NodeError + 19, // 7: unboundednet.status.v1.NodeStatusFull.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 20, // 8: unboundednet.status.v1.NodeStatusFull.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry + 5, // 9: unboundednet.status.v1.NodeStatusDelta.node_info:type_name -> unboundednet.status.v1.NodeInfo + 8, // 10: unboundednet.status.v1.NodeStatusDelta.peers:type_name -> unboundednet.status.v1.PeerStatus + 10, // 11: unboundednet.status.v1.NodeStatusDelta.routing_table:type_name -> unboundednet.status.v1.RoutingTableInfo + 16, // 12: unboundednet.status.v1.NodeStatusDelta.health_check:type_name -> unboundednet.status.v1.HealthCheckStatus + 18, // 13: unboundednet.status.v1.NodeStatusDelta.node_errors:type_name -> unboundednet.status.v1.NodeError + 20, // 14: unboundednet.status.v1.NodeStatusDelta.bpf_entries:type_name -> unboundednet.status.v1.BpfEntry + 4, // 15: unboundednet.status.v1.NodeStatusDelta.peer_measurements:type_name -> unboundednet.status.v1.PeerMeasurements + 19, // 16: unboundednet.status.v1.NodeStatusDelta.node_pod_info:type_name -> unboundednet.status.v1.NodePodInfo + 6, // 17: unboundednet.status.v1.NodeInfo.build_info:type_name -> unboundednet.status.v1.BuildInfo + 7, // 18: unboundednet.status.v1.NodeInfo.wire_guard:type_name -> unboundednet.status.v1.WireGuardStatusInfo + 21, // 19: unboundednet.status.v1.NodeInfo.k8s_labels:type_name -> unboundednet.status.v1.NodeInfo.K8sLabelsEntry + 22, // 20: unboundednet.status.v1.PeerStatus.route_distances:type_name -> unboundednet.status.v1.PeerStatus.RouteDistancesEntry + 9, // 21: unboundednet.status.v1.PeerStatus.tunnel:type_name -> unboundednet.status.v1.PeerTunnelStatus + 17, // 22: unboundednet.status.v1.PeerStatus.health_check:type_name -> unboundednet.status.v1.HealthCheckPeerStatus + 11, // 23: unboundednet.status.v1.RoutingTableInfo.routes:type_name -> unboundednet.status.v1.RouteEntry + 12, // 24: unboundednet.status.v1.RouteEntry.next_hops:type_name -> unboundednet.status.v1.NextHop + 15, // 25: unboundednet.status.v1.NextHop.route_types:type_name -> unboundednet.status.v1.RouteType + 13, // 26: unboundednet.status.v1.NextHop.expected:type_name -> unboundednet.status.v1.OptionalBool + 13, // 27: unboundednet.status.v1.NextHop.present:type_name -> unboundednet.status.v1.OptionalBool + 14, // 28: unboundednet.status.v1.NextHop.info:type_name -> unboundednet.status.v1.NextHopInfo + 29, // [29:29] is the sub-list for method output_type + 29, // [29:29] is the sub-list for method input_type + 29, // [29:29] is the sub-list for extension type_name + 29, // [29:29] is the sub-list for extension extendee + 0, // [0:29] is the sub-list for field type_name } func init() { file_status_proto_init() } @@ -1925,7 +2099,7 @@ func file_status_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_status_proto_rawDesc), len(file_status_proto_rawDesc)), NumEnums: 0, - NumMessages: 22, + NumMessages: 23, NumExtensions: 0, NumServices: 0, }, diff --git a/internal/net/status/proto/status.proto b/internal/net/status/proto/status.proto index 11b0d6015..98903e2d8 100644 --- a/internal/net/status/proto/status.proto +++ b/internal/net/status/proto/status.proto @@ -19,6 +19,7 @@ message NodeStatusAck { string status = 1; // "ok" or "resync_required" uint64 revision = 2; string reason = 3; + bool peer_measurements = 4; // Positive capability, scoped to this WebSocket connection. } // NodeStatusFull mirrors the complete NodeStatusResponse payload. @@ -46,6 +47,27 @@ message NodeStatusDelta { // Track which top-level fields are present in this delta. repeated string updated_fields = 15; repeated BpfEntry bpf_entries = 6; + PeerMeasurements peer_measurements = 7; + int64 timestamp_unix_ns = 8; + string fetch_error = 9; + int64 last_push_time_unix_ns = 10; + string status_source = 11; + NodePodInfo node_pod_info = 12; +} + +// PeerMeasurements replaces all measurement columns for the ordered base peers. +// Every column must have exactly peer_count entries, including zero/empty values. +// identity_digest is SHA-256 over the ordered, length-prefixed peer identities. +// A nonzero matching base_revision and matching digest are required. This field +// must be listed in updated_fields and cannot accompany a peers replacement. +message PeerMeasurements { + uint32 peer_count = 1; + bytes identity_digest = 2; + repeated int64 rx_bytes = 3; + repeated int64 tx_bytes = 4; + repeated int64 last_handshake_unix_ns = 5; + repeated string uptime = 6; + repeated string rtt = 7; } // NodeInfo contains basic node identification and metadata. From 2b31c32d110672f2ab4512a3f88a9b326b9a921f Mon Sep 17 00:00:00 2001 From: "Patrick W. Healy" Date: Thu, 17 Sep 2026 20:56:31 +0000 Subject: [PATCH 2/2] fix(net): retain pending publication ACK when promoting recovery Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2243398-6c36-4c3d-969e-7ed7bfb5b459 --- cmd/unbounded-net-node/status_server.go | 6 ++++ .../status_websocket_auth_test.go | 33 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/cmd/unbounded-net-node/status_server.go b/cmd/unbounded-net-node/status_server.go index b27cd0751..08a453828 100644 --- a/cmd/unbounded-net-node/status_server.go +++ b/cmd/unbounded-net-node/status_server.go @@ -1379,6 +1379,12 @@ func runStatusWebSocketPusher( lastAckTimeNs.Store(time.Now().UnixNano()) + if initialStatus != nil { + acks.pending.Store(true) + + lastWriteTime = time.Now() + } + readCtx, readCancel := context.WithCancel(connCtx) go func() { diff --git a/cmd/unbounded-net-node/status_websocket_auth_test.go b/cmd/unbounded-net-node/status_websocket_auth_test.go index 0c42eb346..3e6033258 100644 --- a/cmd/unbounded-net-node/status_websocket_auth_test.go +++ b/cmd/unbounded-net-node/status_websocket_auth_test.go @@ -102,6 +102,16 @@ func TestWebSocketEstablishedFailureFallsBack(t *testing.T) { initialStatus <- data + ack, err := proto.Marshal(&statusproto.NodeStatusAck{Status: "ok", Revision: 1}) + if err != nil { + t.Error(err) + return + } + + if err := conn.Write(ctx, websocket.MessageBinary, ack); err != nil { + return + } + if failure == "read" { select { case <-dropDirect: @@ -249,6 +259,7 @@ func TestWebSocketRecoveryPromotesInitializedConnection(t *testing.T) { directFrames, fallbackFrames atomic.Int32 connected, failWrites, fallbackClosed atomic.Bool closeFallback atomic.Bool + initialAckSent atomic.Bool mode atomic.Int32 ) @@ -290,6 +301,10 @@ func TestWebSocketRecoveryPromotesInitializedConnection(t *testing.T) { var revision int32 if direct { revision = directFrames.Add(1) + if revision > 1 && !initialAckSent.Load() { + t.Error("promoted connection published before its initial ACK") + return + } } else { revision = fallbackFrames.Add(1) } @@ -300,6 +315,24 @@ func TestWebSocketRecoveryPromotesInitializedConnection(t *testing.T) { return } + if direct && revision == 1 { + go func() { + select { + case <-r.Context().Done(): + return + case <-time.After(150 * time.Millisecond): + } + + initialAckSent.Store(true) + + if err := conn.Write(r.Context(), websocket.MessageBinary, ack); err != nil && r.Context().Err() == nil { + t.Errorf("initial recovery ACK failed: %v", err) + } + }() + + continue + } + if err := conn.Write(r.Context(), websocket.MessageBinary, ack); err != nil { return }