diff --git a/cmd/unbounded-net-controller/detail_aggregated_test.go b/cmd/unbounded-net-controller/detail_aggregated_test.go new file mode 100644 index 000000000..60b323f0d --- /dev/null +++ b/cmd/unbounded-net-controller/detail_aggregated_test.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" + "github.com/Azure/unbounded/internal/net/webhook" +) + +func TestAggregatedDetailsUsesTrustedProxyAndSharedLifecycle(t *testing.T) { + certPEM, _, caPEM, err := webhook.GenerateClientAuthCertificateForTest("front-proxy-client") + if err != nil { + t.Fatal(err) + } + + cert, err := x509.ParseCertificate(mustParseCertPEM(t, certPEM)) + if err != nil { + t.Fatal(err) + } + + server := testWebhookServerForPush(t, caPEM) + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + health := &healthState{detailRequests: manager, registerAggregatedAPIServer: true} + health.isLeader.Store(true) + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, true, server, nil, nil) + + path := "/apis/status.net.unbounded-cloud.io/v1alpha1/nodes/node/details" + + send := func(method, path, body string, trusted bool) *httptest.ResponseRecorder { + t.Helper() + + request := httptest.NewRequest(method, path, strings.NewReader(body)) + request.Header.Set("X-Remote-User", "viewer") + + if trusted { + request.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{cert}} + } + + recorder := httptest.NewRecorder() + mux.ServeHTTP(recorder, request) + + return recorder + } + + for _, method := range []string{http.MethodPost, http.MethodGet} { + if response := send(method, path, "{}", false); response.Code != http.StatusForbidden { + t.Fatalf("spoofed front-proxy header accepted: %d", response.Code) + } + } + + response := send(http.MethodPost, path, `{"forceRefresh":true}`, true) + if response.Code != http.StatusAccepted { + t.Fatalf("aggregated request failed: %d %s", response.Code, response.Body.String()) + } + + var pending statusv1alpha1.NodeDetailResult + if err := json.Unmarshal(response.Body.Bytes(), &pending); err != nil { + t.Fatal(err) + } + + if err := manager.Complete("node", pending.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + for _, resultPath := range []string{path, "/status/node/node/details"} { + response = send(http.MethodGet, resultPath+"?requestId="+pending.RequestID, "", true) + + var result statusv1alpha1.NodeDetailResult + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + + if response.Code != http.StatusOK || result.State != statusv1alpha1.NodeDetailComplete || + result.RequestID != pending.RequestID || result.Details == nil || result.Details.Status.NodeInfo.Name != "node" { + t.Fatalf("paths do not share detail lifecycle: %d %+v", response.Code, result) + } + } + + if response = send(http.MethodDelete, path, "", true); response.Code != http.StatusMethodNotAllowed { + t.Fatalf("unsupported method accepted: %d", response.Code) + } + + disabledMux := http.NewServeMux() + registerStatusHandlers(disabledMux, &healthState{}, false, server, nil, nil) + + disabled := httptest.NewRecorder() + disabledMux.ServeHTTP(disabled, httptest.NewRequest(http.MethodPost, path, strings.NewReader("{}"))) + + if disabled.Code != http.StatusNotFound { + t.Fatalf("aggregated route exposed when disabled: %d", disabled.Code) + } +} diff --git a/cmd/unbounded-net-controller/detail_api.go b/cmd/unbounded-net-controller/detail_api.go new file mode 100644 index 000000000..3517f6e88 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_api.go @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + "k8s.io/klog/v2" + + "github.com/Azure/unbounded/internal/net/authn" + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" + webhookpkg "github.com/Azure/unbounded/internal/net/webhook" +) + +func registerNodeDetailHandlers(mux *http.ServeMux, health *healthState, requireAuth bool, webhookServer *webhookpkg.Server, authorizer *dashboardAuthorizer, issuer *authn.TokenIssuer) { + handler := func(w http.ResponseWriter, r *http.Request) { + if !authorizeDashboardOrAggregated(requireAuth, issuer, authorizer, webhookServer, r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + + return + } + + nodeName := r.PathValue("name") + manager := health.getDetailRequests() + + if !health.isLeader.Load() || manager == nil { + writeNodeDetailResult(w, http.StatusServiceUnavailable, + detailRequestFailure(nodeName, r.URL.Query().Get("requestId"), statusv1alpha1.NodeDetailRetryable, "detail request leader is unavailable")) + + return + } + + var result statusv1alpha1.NodeDetailResult + + switch r.Method { + case http.MethodPost: + var input *struct { + ForceRefresh bool `json:"forceRefresh"` + } + + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + + err := decoder.Decode(&input) + if err == nil { + var extra any + + err = decoder.Decode(&extra) + if errors.Is(err, io.EOF) && input != nil { + err = nil + } else if err == nil || input == nil { + err = errors.New("expected one JSON object") + } + } + + if err != nil { + code := http.StatusBadRequest + + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + code = http.StatusRequestEntityTooLarge + } + + writeNodeDetailResult(w, code, detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailUnavailable, err.Error())) + + return + } + + result = manager.Request(nodeName, input.ForceRefresh) + case http.MethodGet: + requestID := r.URL.Query().Get("requestId") + if requestID == "" { + writeNodeDetailResult(w, http.StatusBadRequest, + detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailUnavailable, "requestId is required")) + + return + } + + result = manager.Result(nodeName, requestID) + default: + w.Header().Set("Allow", "GET, POST") + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + + return + } + + writeNodeDetailResult(w, nodeDetailHTTPStatus(result.State), result) + } + mux.HandleFunc("/status/node/{name}/details", handler) + + if health.registerAggregatedAPIServer { + mux.HandleFunc("/apis/status.net.unbounded-cloud.io/v1alpha1/nodes/{name}/details", func(w http.ResponseWriter, r *http.Request) { + if !webhookServer.IsTrustedAggregatedRequest(r) { + http.Error(w, "Forbidden", http.StatusForbidden) + + return + } + + handler(w, r) + }) + } +} + +func nodeDetailHTTPStatus(state statusv1alpha1.NodeDetailState) int { + switch state { + case statusv1alpha1.NodeDetailPending: + return http.StatusAccepted + case statusv1alpha1.NodeDetailComplete: + return http.StatusOK + case statusv1alpha1.NodeDetailExpired: + return http.StatusGone + case statusv1alpha1.NodeDetailUnavailable: + return http.StatusNotFound + default: + return http.StatusServiceUnavailable + } +} + +func writeNodeDetailResult(w http.ResponseWriter, code int, result statusv1alpha1.NodeDetailResult) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(code) + + if err := json.NewEncoder(w).Encode(result); err != nil { + klog.V(4).Infof("node detail response encode failed: %v", err) + } +} diff --git a/cmd/unbounded-net-controller/detail_api_test.go b/cmd/unbounded-net-controller/detail_api_test.go new file mode 100644 index 000000000..42579d0f7 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_api_test.go @@ -0,0 +1,377 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "testing/synctest" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func serveDetailRequest(t *testing.T, mux *http.ServeMux, method, path, body string) (*httptest.ResponseRecorder, statusv1alpha1.NodeDetailResult) { + t.Helper() + + recorder := httptest.NewRecorder() + mux.ServeHTTP(recorder, httptest.NewRequest(method, path, strings.NewReader(body))) + + var result statusv1alpha1.NodeDetailResult + if recorder.Header().Get("Content-Type") == "application/json" { + if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + } + + return recorder, result +} + +func TestDetailAPIRequestAndResult(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + health := &healthState{detailRequests: manager} + health.isLeader.Store(true) + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, false, nil, nil, nil) + + path := "/status/node/node/details" + response, pending := serveDetailRequest(t, mux, http.MethodPost, path, `{}`) + + if response.Code != http.StatusAccepted || pending.State != statusv1alpha1.NodeDetailPending || pending.RequestID == "" { + t.Fatalf("POST did not return a pending request: %d %s", response.Code, response.Body.String()) + } + + if err := manager.Complete("node", pending.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + response, complete := serveDetailRequest(t, mux, http.MethodGet, path+"?requestId="+pending.RequestID, "") + if response.Code != http.StatusOK || complete.State != statusv1alpha1.NodeDetailComplete || + complete.Details == nil || complete.Details.Status.NodeInfo.Name != "node" || + response.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("GET did not return cached details: %d %s", response.Code, response.Body.String()) + } + + response, reused := serveDetailRequest(t, mux, http.MethodPost, path, `{"forceRefresh":false}`) + if response.Code != http.StatusOK || reused.RequestID != pending.RequestID { + t.Fatal("POST did not reuse existing details") + } + + response, refresh := serveDetailRequest(t, mux, http.MethodPost, path, `{"forceRefresh":true}`) + if response.Code != http.StatusAccepted || refresh.RequestID == pending.RequestID { + t.Fatal("forced refresh did not create a new request") + } + + time.Sleep(manager.timeout) + synctest.Wait() + + response, expired := serveDetailRequest(t, mux, http.MethodGet, path+"?requestId="+refresh.RequestID, "") + + if response.Code != http.StatusGone || expired.State != statusv1alpha1.NodeDetailExpired || expired.Details != nil { + t.Fatal("expired request was not explicit") + } + + health.setLeader(false) + + response, stopped := serveDetailRequest(t, mux, http.MethodGet, path+"?requestId="+pending.RequestID, "") + + if response.Code != http.StatusServiceUnavailable || stopped.State != statusv1alpha1.NodeDetailRetryable || stopped.Details != nil { + t.Fatal("leadership loss did not produce retryable failure") + } + + assertNodeDetailEntries(t, manager.cache, 0) + }) +} + +func TestDetailAPIMethodsAndErrors(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + health := &healthState{detailRequests: manager} + health.isLeader.Store(true) + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, false, nil, nil, nil) + + for _, tc := range []struct { + method string + query string + body string + code int + }{ + {http.MethodDelete, "", "", http.StatusMethodNotAllowed}, + {http.MethodGet, "", "", http.StatusBadRequest}, + {http.MethodGet, "?requestId=unknown", "", http.StatusServiceUnavailable}, + {http.MethodPost, "", "", http.StatusBadRequest}, + {http.MethodPost, "", "null", http.StatusBadRequest}, + {http.MethodPost, "", "{} {}", http.StatusBadRequest}, + {http.MethodPost, "", `{"forceRefresh":"yes"}`, http.StatusBadRequest}, + {http.MethodPost, "", `{"url":"http://caller-controlled"}`, http.StatusBadRequest}, + {http.MethodPost, "", strings.Repeat(" ", 1<<20) + "{}", http.StatusRequestEntityTooLarge}, + } { + response, _ := serveDetailRequest(t, mux, tc.method, "/status/node/node/details"+tc.query, tc.body) + if response.Code != tc.code { + t.Fatalf("%s %s: got %d, want %d: %s", tc.method, tc.query, response.Code, tc.code, response.Body.String()) + } + + if tc.code == http.StatusMethodNotAllowed && response.Header().Get("Allow") != "GET, POST" { + t.Fatal("missing Allow header") + } + } + + manager.mu.Lock() + count := len(manager.requests) + manager.mu.Unlock() + + if count != 0 { + t.Fatal("invalid API requests created work") + } + }) +} + +func TestDetailAPIAuthorization(t *testing.T) { + for _, allowed := range []bool{false, true} { + t.Run(strconv.FormatBool(allowed), func(t *testing.T) { + client := k8sfake.NewClientset() + client.PrependReactor("create", "subjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + review := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SubjectAccessReview) + if review.Spec.ResourceAttributes.Name != "dashboard" || review.Spec.ResourceAttributes.Verb != "get" { + t.Error("detail API changed the existing authorization resource") + } + + return true, &authorizationv1.SubjectAccessReview{Status: authorizationv1.SubjectAccessReviewStatus{Allowed: allowed}}, nil + }) + + issuer := testTokenIssuer(t) + + viewer, _, err := issuer.IssueViewerToken("viewer", nil, time.Hour) + if err != nil { + t.Fatal(err) + } + + health := &healthState{detailRequests: testDetailRequests(t, nodeDetailRequestHooks{})} + health.isLeader.Store(true) + + proxy, trustedTLS := testNodeTokenFrontProxy(t) + mux := http.NewServeMux() + registerStatusHandlers(mux, health, true, proxy, newDashboardAuthorizer(client), issuer) + + for _, token := range []string{"", "invalid", testNodeToken(t, issuer), viewer} { + request := httptest.NewRequest(http.MethodPost, "/status/node/node/details", strings.NewReader("{}")) + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + + want := http.StatusUnauthorized + if token == viewer && allowed { + want = http.StatusAccepted + } + + if response.Code != want { + t.Fatalf("authorization: got %d, want %d", response.Code, want) + } + } + + request := httptest.NewRequest(http.MethodPost, "/status/node/node/details", strings.NewReader("{}")) + request.TLS = trustedTLS + request.Header.Set("X-Remote-User", "aggregated-viewer") + + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + + if response.Code != http.StatusAccepted { + t.Fatalf("trusted aggregated request rejected: %d %s", response.Code, response.Body.String()) + } + }) + } +} + +func testDetailLifecycle(t *testing.T, port int) (*healthState, cache.SharedIndexInformer, *nodeDetailRequests) { + t.Helper() + + health := &healthState{ + statusDetailCacheTTL: 10 * time.Second, statusDetailRequestTimeout: 3 * time.Second, + nodeAgentHealthPort: port, + } + health.isLeader.Store(true) + + factory := informers.NewSharedInformerFactory(k8sfake.NewClientset(), 0) + informer := factory.Core().V1().Nodes().Informer() + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node", UID: "uid"}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "127.0.0.1"}}}, + } + if err := informer.GetIndexer().Add(node); err != nil { + t.Fatal(err) + } + + manager, err := health.startDetailRequests(t.Context(), informer) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(manager.Close) + + return health, informer, manager +} + +func TestDetailAPIHTTPPull(t *testing.T) { + for _, mode := range []string{"success", "failure", "wrong-node", "oversized"} { + t.Run(mode, func(t *testing.T) { + pullSucceeds := mode == "success" || mode == "oversized" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/status/json" || r.Method != http.MethodGet { + t.Error("incorrect node detail pull endpoint") + } + + if mode == "failure" { + http.Error(w, "unreachable", http.StatusServiceUnavailable) + + return + } + + status := testDetailStatus() + if mode == "wrong-node" { + status.NodeInfo.Name = "other" + } + + if mode == "oversized" { + status.NodeInfo.K8sLabels = map[string]string{"large": strings.Repeat("x", 1<<20)} + } + + json.NewEncoder(w).Encode(status) + })) + defer server.Close() + + _, portText, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) + if err != nil { + t.Fatal(err) + } + + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatal(err) + } + + health, _, manager := testDetailLifecycle(t, port) + if health.pullEnabled.Load() { + t.Fatal("test must exercise disabled background pulls") + } + + mux := http.NewServeMux() + registerStatusHandlers(mux, health, false, nil, nil, nil) + _, request := serveDetailRequest(t, mux, http.MethodPost, "/status/node/node/details", "{}") + + deadline := time.NewTimer(time.Second) + defer deadline.Stop() + + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + + for { + response, result := serveDetailRequest(t, mux, http.MethodGet, "/status/node/node/details?requestId="+request.RequestID, "") + if pullSucceeds && result.State == statusv1alpha1.NodeDetailComplete { + if response.Code != http.StatusOK || result.Details == nil || result.Details.Status.NodeInfo.Name != "node" { + t.Fatal("HTTP pull result is incomplete") + } + + // The status POST body limit does not limit legacy HTTP pull responses. + if mode == "oversized" && len(result.Details.Status.NodeInfo.K8sLabels["large"]) != 1<<20 { + t.Fatal("HTTP pull response was truncated to the POST body limit") + } + + break + } + + if !pullSucceeds { + if _, ok := manager.Pending("node"); ok { + result = manager.Result("node", request.RequestID) + + if result.Details != nil || result.Error == "" { + t.Fatal("failed pull returned success-shaped details") + } + + break + } + } + + select { + case <-deadline.C: + t.Fatalf("HTTP pull did not settle: %+v", result) + case <-ticker.C: + } + } + }) + } +} + +func TestDetailLifecycleInvalidationAndShutdown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + health, informer, manager := testDetailLifecycle(t, 0) + + request := manager.Request("node", false) + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + oldNode := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", UID: "uid"}} + newNode := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node", UID: "new"}} + detailNodeEvents(manager).OnUpdate(oldNode, newNode) + assertNodeDetailEntries(t, manager.cache, 0) + + if _, err := health.startDetailRequests(t.Context(), informer); err == nil { + t.Fatal("duplicate lifecycle initialized") + } + + health.setLeader(false) + synctest.Wait() + + if health.getDetailRequests() != nil { + t.Fatal("leadership loss retained manager") + } + + ctx, cancel := context.WithCancel(t.Context()) + + health.isLeader.Store(true) + + restarted, err := health.startDetailRequests(ctx, informer) + if err != nil { + t.Fatal(err) + } + + restarted.Request("node", true) + detailNodeEvents(restarted).OnDelete(cache.DeletedFinalStateUnknown{Obj: oldNode}) + detailNodeEvents(restarted).OnDelete("invalid") + cancel() + restarted.Close() + synctest.Wait() + + if health.getDetailRequests() != nil { + t.Fatal("context cancellation retained manager") + } + + assertNodeDetailEntries(t, restarted.cache, 0) + }) +} diff --git a/cmd/unbounded-net-controller/detail_cache.go b/cmd/unbounded-net-controller/detail_cache.go index bd51416e4..ccd64fe2c 100644 --- a/cmd/unbounded-net-controller/detail_cache.go +++ b/cmd/unbounded-net-controller/detail_cache.go @@ -10,19 +10,14 @@ import ( "time" "k8s.io/utils/clock" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" ) // nodeDetailSnapshot carries immutable details, separate from routine status. // Status and all its nested data must remain read-only, including for callers // retaining a returned snapshot after its cache entry expires. -type nodeDetailSnapshot struct { - NodeName string - RequestID string - CollectedAt time.Time - ReceivedAt time.Time - ExpiresAt time.Time - Status *NodeStatusResponse -} +type nodeDetailSnapshot = statusv1alpha1.NodeDetailSnapshot // nodeDetailCache is a leader-local, TTL-only store. It owns no second result // history or per-entry timers. TTL bounds retention time, not peak memory. diff --git a/cmd/unbounded-net-controller/detail_lifecycle.go b/cmd/unbounded-net-controller/detail_lifecycle.go new file mode 100644 index 000000000..3da73cdb8 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_lifecycle.go @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "fmt" + "net" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +func (h *healthState) getDetailRequests() *nodeDetailRequests { + h.detailMu.Lock() + defer h.detailMu.Unlock() + + return h.detailRequests +} + +func (h *healthState) stopDetailRequests() { + h.detailMu.Lock() + manager := h.detailRequests + h.detailRequests = nil + h.detailMu.Unlock() + + if manager != nil { + manager.Close() + } +} + +// startDetailRequests is called once per leadership term, with that term's +// context and node informer. Neither hooks nor workers use the HTTP caller's +// context, so disconnecting a viewer does not cancel another viewer's request. +func (h *healthState) startDetailRequests(ctx context.Context, nodeInformer cache.SharedIndexInformer) (*nodeDetailRequests, error) { + if nodeInformer == nil { + return nil, errors.New("node detail requests require a node informer") + } + + detailCache, err := newNodeDetailCache(h.statusDetailCacheTTL) + if err != nil { + return nil, err + } + + lister := corev1listers.NewNodeLister(nodeInformer.GetIndexer()) + + manager, err := newNodeDetailRequests(ctx, detailCache, h.statusDetailRequestTimeout, nodeDetailRequestHooks{ + Resolve: func(name string) (types.UID, error) { + node, err := lister.Get(name) + if err != nil { + return "", err + } + + return node.UID, nil + }, + Pull: func(ctx context.Context, name string) (*NodeStatusResponse, error) { + node, err := lister.Get(name) + if err != nil { + return nil, err + } + + for _, address := range node.Status.Addresses { + if address.Type != corev1.NodeInternalIP || net.ParseIP(address.Address) == nil { + continue + } + + host := address.Address + if net.ParseIP(host).To4() == nil { + host = "[" + host + "]" + } + + return fetchNodeStatus(ctx, host, h.nodeAgentHealthPort) + } + + return nil, fmt.Errorf("node %q has no valid InternalIP", name) + }, + }) + if err != nil { + return nil, err + } + + registration, err := nodeInformer.AddEventHandler(detailNodeEvents(manager)) + if err != nil { + manager.Close() + + return nil, fmt.Errorf("register detail node invalidation: %w", err) + } + + h.detailMu.Lock() + if !h.isLeader.Load() || ctx.Err() != nil || h.detailRequests != nil { + h.detailMu.Unlock() + manager.Close() + + if err := nodeInformer.RemoveEventHandler(registration); err != nil { + klog.Warningf("Removing node detail event handler: %v", err) + } + + return nil, errors.New("detail request leadership is unavailable or already initialized") + } + + h.detailRequests = manager + h.detailMu.Unlock() + + go func() { + <-manager.done + + if err := nodeInformer.RemoveEventHandler(registration); err != nil { + klog.Warningf("Removing node detail event handler: %v", err) + } + + h.detailMu.Lock() + if h.detailRequests == manager { + h.detailRequests = nil + } + h.detailMu.Unlock() + }() + + return manager, nil +} + +func detailNodeEvents(manager *nodeDetailRequests) cache.ResourceEventHandlerFuncs { + return cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(oldObj, newObj any) { + oldNode, oldOK := oldObj.(*corev1.Node) + + newNode, newOK := newObj.(*corev1.Node) + if oldOK && newOK && oldNode.UID != newNode.UID { + manager.InvalidateNode(oldNode.Name, oldNode.UID) + } + }, + DeleteFunc: func(obj any) { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + + if node, ok := obj.(*corev1.Node); ok { + manager.InvalidateNode(node.Name, node.UID) + } + }, + } +} diff --git a/cmd/unbounded-net-controller/detail_requests.go b/cmd/unbounded-net-controller/detail_requests.go new file mode 100644 index 000000000..e2cc5bcac --- /dev/null +++ b/cmd/unbounded-net-controller/detail_requests.go @@ -0,0 +1,401 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "sync" + "time" + + "k8s.io/apimachinery/pkg/types" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +// Hooks must honor cancellation. Resolve reads the current informer identity; +// Dispatch returns true only when an active WebSocket accepted the command. +type nodeDetailRequestHooks struct { + Resolve func(string) (types.UID, error) + Dispatch func(context.Context, string, statusv1alpha1.DetailRequest) (bool, error) + Pull func(context.Context, string) (*NodeStatusResponse, error) +} + +// A request owns metadata and cancellation only, never a result payload. +type nodeDetailRequest struct { + nodeName string + uid types.UID + command statusv1alpha1.DetailRequest + state statusv1alpha1.NodeDetailState + message string + wakeAt time.Time + poll bool + cancel context.CancelFunc +} + +type nodeDetailRequests struct { + mu sync.Mutex + ctx context.Context + cancel context.CancelFunc + done chan struct{} + changed chan struct{} + workers sync.WaitGroup + closed bool + timeout time.Duration + cache *nodeDetailCache + hooks nodeDetailRequestHooks + requests map[string]*nodeDetailRequest + active map[string]*nodeDetailRequest +} + +// newNodeDetailRequests takes exclusive lifecycle ownership of cache. All +// snapshots must enter through Complete so their node UID binding is known. +func newNodeDetailRequests(ctx context.Context, cache *nodeDetailCache, timeout time.Duration, hooks nodeDetailRequestHooks) (*nodeDetailRequests, error) { + if cache == nil || timeout <= 0 || hooks.Resolve == nil { + return nil, errors.New("detail requests require a cache, positive timeout, and node resolver") + } + + ctx, cancel := context.WithCancel(ctx) + m := &nodeDetailRequests{ + ctx: ctx, cancel: cancel, done: make(chan struct{}), changed: make(chan struct{}, 1), + timeout: timeout, cache: cache, hooks: hooks, + requests: make(map[string]*nodeDetailRequest), active: make(map[string]*nodeDetailRequest), + } + cache.Clear() + + go m.run() + + return m, nil +} + +// Close cancels dispatches, clears leader-local state, and waits for all workers. +func (m *nodeDetailRequests) Close() { + m.cancel() + <-m.done +} + +func (m *nodeDetailRequests) Request(nodeName string, forceRefresh bool) statusv1alpha1.NodeDetailResult { + m.mu.Lock() + defer m.mu.Unlock() + + if m.ctx.Err() != nil || m.closed { + return detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailRetryable, "detail request leader is unavailable") + } + + m.expireLocked(time.Now()) + + uid, err := m.hooks.Resolve(nodeName) + if err != nil || uid == "" { + return detailRequestFailure(nodeName, "", statusv1alpha1.NodeDetailUnavailable, "node identity is unavailable") + } + + for _, request := range m.requests { + if request.nodeName == nodeName && request.uid != uid { + m.invalidateLocked(request) + } + } + + if !forceRefresh { + if snapshot, ok := m.cache.Get(nodeName); ok { + if request := m.requests[snapshot.RequestID]; request != nil && request.uid == uid { + return m.resultLocked(request) + } + } + } + + if request := m.active[nodeName]; request != nil { + return m.resultLocked(request) + } + + now := time.Now() + request := &nodeDetailRequest{ + nodeName: nodeName, uid: uid, state: statusv1alpha1.NodeDetailPending, + command: statusv1alpha1.DetailRequest{RequestID: rand.Text(), Deadline: now.Add(m.timeout)}, + wakeAt: now.Add(m.timeout), + } + ctx, cancel := context.WithDeadline(m.ctx, request.command.Deadline) + request.cancel = cancel + m.requests[request.command.RequestID] = request + m.active[nodeName] = request + m.notify() + m.workers.Go(func() { m.dispatch(ctx, nodeName, request.command) }) + + return m.resultLocked(request) +} + +func (m *nodeDetailRequests) Result(nodeName, requestID string) statusv1alpha1.NodeDetailResult { + m.mu.Lock() + defer m.mu.Unlock() + + if m.ctx.Err() != nil || m.closed { + return detailRequestFailure(nodeName, requestID, statusv1alpha1.NodeDetailRetryable, "detail request leader is unavailable") + } + + m.expireLocked(time.Now()) + + request := m.requests[requestID] + if request == nil || request.nodeName != nodeName { + return detailRequestFailure(nodeName, requestID, statusv1alpha1.NodeDetailRetryable, "request is no longer known; retry on the current leader") + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + } + + return m.resultLocked(request) +} + +// Complete is idempotent while a completed request is retained. It rejects +// mismatched, late, and expired replies without replacing data or renewing TTL. +func (m *nodeDetailRequests) Complete(nodeName, requestID string, status *NodeStatusResponse) error { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + request := m.requests[requestID] + if m.ctx.Err() != nil || m.closed || request == nil || request.nodeName != nodeName { + return errors.New("detail request is no longer available") + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + + return errors.New("detail request node was deleted or replaced") + } + + if status == nil || status.NodeInfo.Name != nodeName || status.FetchError != "" { + return errors.New("detail response has missing data, a fetch error, or a mismatched node name") + } + + if request.state == statusv1alpha1.NodeDetailComplete { + return nil + } + + if request.state != statusv1alpha1.NodeDetailPending { + return errors.New("detail request is no longer pending") + } + + snapshot, err := m.cache.Store(nodeName, requestID, status.Timestamp, status) + if err != nil { + return err + } + + request.state = statusv1alpha1.NodeDetailComplete + request.message = "" + request.poll = false + request.wakeAt = snapshot.ExpiresAt + request.cancel() + delete(m.active, nodeName) + m.notify() + + return nil +} + +// Pending exposes only a failed-pull fallback command, without refreshing its +// deadline. Returning it repeatedly is safe until a valid reply completes it. +func (m *nodeDetailRequests) Pending(nodeName string) (statusv1alpha1.DetailRequest, bool) { + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + request := m.active[nodeName] + if m.ctx.Err() != nil || request == nil || !request.poll { + return statusv1alpha1.DetailRequest{}, false + } + + if uid, err := m.hooks.Resolve(nodeName); err != nil || uid != request.uid { + m.invalidateLocked(request) + + return statusv1alpha1.DetailRequest{}, false + } + + return request.command, true +} + +// InvalidateNode handles deletion/replacement of a specific UID. A delayed old +// informer event cannot cancel a request for a newer node with the same name. +func (m *nodeDetailRequests) InvalidateNode(nodeName string, uid types.UID) { + m.mu.Lock() + defer m.mu.Unlock() + + for _, request := range m.requests { + if request.nodeName == nodeName && request.uid == uid { + m.invalidateLocked(request) + } + } +} + +func (m *nodeDetailRequests) invalidateLocked(request *nodeDetailRequest) { + if request.state == statusv1alpha1.NodeDetailUnavailable { + return + } + + request.cancel() + request.state = statusv1alpha1.NodeDetailUnavailable + request.message = "node was deleted or replaced" + request.poll = false + request.wakeAt = time.Now().Add(m.timeout) + + if m.active[request.nodeName] == request { + delete(m.active, request.nodeName) + } + + if snapshot, ok := m.cache.Get(request.nodeName); ok && snapshot.RequestID == request.command.RequestID { + m.cache.Delete(request.nodeName) + } + + m.notify() +} + +func (m *nodeDetailRequests) resultLocked(request *nodeDetailRequest) statusv1alpha1.NodeDetailResult { + result := statusv1alpha1.NodeDetailResult{ + NodeName: request.nodeName, RequestID: request.command.RequestID, Deadline: request.command.Deadline, + State: request.state, Error: request.message, + } + if request.state == statusv1alpha1.NodeDetailComplete { + if snapshot, ok := m.cache.Get(request.nodeName); ok && snapshot.RequestID == request.command.RequestID { + result.Details = &snapshot + } else { + result.State = statusv1alpha1.NodeDetailExpired + result.Error = "details expired or were replaced" + } + } + + return result +} + +func detailRequestFailure(nodeName, requestID string, state statusv1alpha1.NodeDetailState, message string) statusv1alpha1.NodeDetailResult { + return statusv1alpha1.NodeDetailResult{NodeName: nodeName, RequestID: requestID, State: state, Error: message} +} + +func (m *nodeDetailRequests) dispatch(ctx context.Context, nodeName string, command statusv1alpha1.DetailRequest) { + if m.hooks.Dispatch != nil { + if sent, err := m.hooks.Dispatch(ctx, nodeName, command); sent && err == nil { + return + } + } + + err := errors.New("node HTTP detail pull is unavailable") + + if m.hooks.Pull != nil && ctx.Err() == nil { + pullCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + status, pullErr := m.hooks.Pull(pullCtx, nodeName) + + cancel() + + err = pullErr + if err == nil { + err = m.Complete(nodeName, command.RequestID, status) + } + + if err == nil { + return + } + } + + m.mu.Lock() + defer m.mu.Unlock() + + m.expireLocked(time.Now()) + + if request := m.active[nodeName]; request != nil && request.command.RequestID == command.RequestID && m.ctx.Err() == nil { + request.poll = true + request.message = fmt.Sprintf("HTTP detail pull failed; waiting for status POST: %v", err) + } +} + +func (m *nodeDetailRequests) expireLocked(now time.Time) time.Time { + var next time.Time + + for id, request := range m.requests { + if !now.Before(request.wakeAt) { + if request.state == statusv1alpha1.NodeDetailPending || request.state == statusv1alpha1.NodeDetailComplete { + request.cancel() + request.state = statusv1alpha1.NodeDetailExpired + request.message = "detail request or snapshot expired" + request.poll = false + request.wakeAt = request.wakeAt.Add(m.timeout) + + if m.active[request.nodeName] == request { + delete(m.active, request.nodeName) + } + } + + if !now.Before(request.wakeAt) { + delete(m.requests, id) + + continue + } + } + + if next.IsZero() || request.wakeAt.Before(next) { + next = request.wakeAt + } + } + + return next +} + +func (m *nodeDetailRequests) notify() { + select { + case m.changed <- struct{}{}: + default: + } +} + +func (m *nodeDetailRequests) run() { + cacheDone := make(chan struct{}) + go func() { + defer close(cacheDone) + + if err := m.cache.Run(m.ctx); err != nil { + m.cancel() + } + }() + + for m.ctx.Err() == nil { + m.mu.Lock() + next := m.expireLocked(time.Now()) + m.mu.Unlock() + + var ( + timer *time.Timer + timerC <-chan time.Time + ) + + if !next.IsZero() { + timer = time.NewTimer(time.Until(next)) + timerC = timer.C + } + + select { + case <-m.ctx.Done(): + case <-m.changed: + case <-timerC: + } + + if timer != nil { + timer.Stop() + } + } + + m.mu.Lock() + m.closed = true + + for _, request := range m.requests { + request.cancel() + } + + clear(m.requests) + clear(m.active) + m.cache.Clear() + m.mu.Unlock() + m.workers.Wait() + <-cacheDone + close(m.done) +} diff --git a/cmd/unbounded-net-controller/detail_requests_test.go b/cmd/unbounded-net-controller/detail_requests_test.go new file mode 100644 index 000000000..af5c38567 --- /dev/null +++ b/cmd/unbounded-net-controller/detail_requests_test.go @@ -0,0 +1,316 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "k8s.io/apimachinery/pkg/types" + + statusv1alpha1 "github.com/Azure/unbounded/internal/net/status/v1alpha1" +) + +func testDetailRequests(t *testing.T, hooks nodeDetailRequestHooks) *nodeDetailRequests { + t.Helper() + + if hooks.Resolve == nil { + hooks.Resolve = func(string) (types.UID, error) { return "uid", nil } + } + + cache, err := newNodeDetailCache(10 * time.Second) + if err != nil { + t.Fatal(err) + } + + manager, err := newNodeDetailRequests(t.Context(), cache, 3*time.Second, hooks) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(manager.Close) + + return manager +} + +func testDetailStatus() *NodeStatusResponse { + return &NodeStatusResponse{Timestamp: time.Now(), NodeInfo: NodeInfo{Name: "node"}} +} + +func TestDetailRequestsValidation(t *testing.T) { + cache, _ := newNodeDetailCache(time.Second) + resolve := func(string) (types.UID, error) { return "uid", nil } + + for _, tc := range []struct { + cache *nodeDetailCache + timeout time.Duration + hooks nodeDetailRequestHooks + }{ + {nil, time.Second, nodeDetailRequestHooks{Resolve: resolve}}, + {cache, 0, nodeDetailRequestHooks{Resolve: resolve}}, + {cache, -time.Second, nodeDetailRequestHooks{Resolve: resolve}}, + {cache, time.Second, nodeDetailRequestHooks{}}, + } { + if _, err := newNodeDetailRequests(t.Context(), tc.cache, tc.timeout, tc.hooks); err == nil { + t.Fatal("invalid constructor accepted") + } + } +} + +func TestDetailRequestsCoalesceAndDispatch(t *testing.T) { + for _, activeWS := range []bool{false, true} { + t.Run(map[bool]string{false: "HTTP", true: "WebSocket"}[activeWS], func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var pulls, sends atomic.Int32 + + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: func(context.Context, string, statusv1alpha1.DetailRequest) (bool, error) { + sends.Add(1) + + return activeWS, nil + }, + Pull: func(ctx context.Context, _ string) (*NodeStatusResponse, error) { + pulls.Add(1) + <-ctx.Done() + + return nil, ctx.Err() + }, + }) + first := manager.Request("node", false) + + var workers sync.WaitGroup + + for range 32 { + workers.Go(func() { + result := manager.Request("node", true) + if result.State != statusv1alpha1.NodeDetailPending || result.RequestID != first.RequestID || result.Deadline != first.Deadline { + t.Error("concurrent refresh did not coalesce") + } + }) + } + + workers.Wait() + synctest.Wait() + + if sends.Load() != 1 || pulls.Load() != map[bool]int32{false: 1, true: 0}[activeWS] { + t.Fatal("unexpected dispatch count") + } + + if _, ok := manager.Pending("node"); ok { + t.Fatal("poll command available before pull failure") + } + }) + }) + } +} + +func TestDetailRequestsCacheRefreshAndDuplicate(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{}) + request := manager.Request("node", false) + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + original := manager.Result("node", request.RequestID) + + time.Sleep(time.Second) + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + duplicate := manager.Request("node", false) + if duplicate.State != statusv1alpha1.NodeDetailComplete || duplicate.Details == nil || + *duplicate.Details != *original.Details { + t.Fatal("duplicate changed data or receipt TTL") + } + + refresh := manager.Request("node", true) + if refresh.RequestID == request.RequestID || refresh.State != statusv1alpha1.NodeDetailPending { + t.Fatal("refresh did not create a fresh request") + } + + if cached := manager.Request("node", false); cached.RequestID != request.RequestID || cached.Details == nil { + t.Fatal("pending refresh prevented reuse of valid data") + } + + if err := manager.Complete("node", refresh.RequestID, testDetailStatus()); err != nil { + t.Fatal(err) + } + + if old := manager.Result("node", request.RequestID); old.State != statusv1alpha1.NodeDetailExpired || old.Details != nil { + t.Fatal("old request retained a second result") + } + + time.Sleep(10 * time.Second) + synctest.Wait() + + if result := manager.Result("node", refresh.RequestID); result.State != statusv1alpha1.NodeDetailExpired || result.Details != nil { + t.Fatal("result still available at TTL boundary") + } + + assertNodeDetailEntries(t, manager.cache, 0) + }) +} + +func TestDetailRequestsFallbackDeadlineAndCleanup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Dispatch: func(context.Context, string, statusv1alpha1.DetailRequest) (bool, error) { + return true, errors.New("socket closed") + }, + Pull: func(ctx context.Context, _ string) (*NodeStatusResponse, error) { + deadline, ok := ctx.Deadline() + if !ok || deadline != time.Now().Add(3*time.Second) { + t.Error("pull did not inherit the overall deadline") + } + + time.Sleep(time.Second) + + return nil, errors.New("unreachable") + }, + }) + request := manager.Request("node", false) + + synctest.Wait() + time.Sleep(time.Second) + synctest.Wait() + + for range 2 { + command, ok := manager.Pending("node") + if !ok || command.RequestID != request.RequestID || command.Deadline != request.Deadline { + t.Fatal("failed pull did not expose the unchanged polling command") + } + } + + time.Sleep(2*time.Second - time.Nanosecond) + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailPending { + t.Fatal("request expired too early") + } + + time.Sleep(time.Nanosecond) + synctest.Wait() + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailExpired { + t.Fatal("request remained pending at deadline") + } + + if _, ok := manager.Pending("node"); ok { + t.Fatal("expired polling command retained") + } + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("late result accepted") + } + + assertNodeDetailEntries(t, manager.cache, 0) + time.Sleep(manager.timeout) + synctest.Wait() + manager.mu.Lock() + count := len(manager.requests) + len(manager.active) + manager.mu.Unlock() + + if count != 0 { + t.Fatal("terminal metadata was not proactively removed") + } + }) +} + +func TestDetailRequestsBindingAndDeletion(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + uid := types.UID("old") + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Resolve: func(string) (types.UID, error) { return uid, nil }, + }) + request := manager.Request("node", false) + + for _, status := range []*NodeStatusResponse{nil, {}, {NodeInfo: NodeInfo{Name: "wrong"}}, {NodeInfo: NodeInfo{Name: "node"}, FetchError: "failed"}} { + if err := manager.Complete("node", request.RequestID, status); err == nil { + t.Fatal("invalid details accepted") + } + } + + if err := manager.Complete("other", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("wrong node binding accepted") + } + + if err := manager.Complete("node", "unknown", testDetailStatus()); err == nil { + t.Fatal("unknown request accepted") + } + + synctest.Wait() + + uid = "replacement" + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("replaced node accepted") + } + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailUnavailable { + t.Fatal("replacement did not invalidate request") + } + + fresh := manager.Request("node", false) + manager.InvalidateNode("node", "old") + + if err := manager.Complete("node", fresh.RequestID, testDetailStatus()); err != nil { + t.Fatalf("old informer event invalidated new node: %v", err) + } + + manager.InvalidateNode("node", "replacement") + assertNodeDetailEntries(t, manager.cache, 0) + + if result := manager.Result("node", fresh.RequestID); result.State != statusv1alpha1.NodeDetailUnavailable { + t.Fatal("node deletion did not invalidate cached details") + } + }) +} + +func TestDetailRequestsHTTPCompletionAndShutdown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := testDetailRequests(t, nodeDetailRequestHooks{ + Pull: func(context.Context, string) (*NodeStatusResponse, error) { return testDetailStatus(), nil }, + }) + request := manager.Request("node", false) + + synctest.Wait() + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailComplete || result.Details == nil { + t.Fatal("HTTP pull did not complete") + } + + manager.Close() + + if result := manager.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailRetryable || result.Details != nil { + t.Fatal("shutdown did not make results retryable") + } + + if result := manager.Request("node", true); result.State != statusv1alpha1.NodeDetailRetryable { + t.Fatal("shutdown accepted a new request") + } + + if err := manager.Complete("node", request.RequestID, testDetailStatus()); err == nil { + t.Fatal("shutdown accepted late details") + } + + assertNodeDetailEntries(t, manager.cache, 0) + restarted := testDetailRequests(t, nodeDetailRequestHooks{}) + + if result := restarted.Result("node", request.RequestID); result.State != statusv1alpha1.NodeDetailRetryable { + t.Fatal("new leader pretended to own an old request") + } + + if result := restarted.Request("node", false); result.RequestID == request.RequestID { + t.Fatal("restart reused an old request ID") + } + }) +} diff --git a/cmd/unbounded-net-controller/health_state.go b/cmd/unbounded-net-controller/health_state.go index fcc3eadb6..1f09d89fd 100644 --- a/cmd/unbounded-net-controller/health_state.go +++ b/cmd/unbounded-net-controller/health_state.go @@ -63,6 +63,11 @@ type healthState struct { nodeTokenVerifier serviceAccountTokenVerifier nodeAuthReady func() bool // Required only by the startup-selected local OIDC verifier. + detailMu sync.Mutex + detailRequests *nodeDetailRequests + statusDetailCacheTTL time.Duration + statusDetailRequestTimeout time.Duration + // Pull fallback toggle (controlled via dashboard WS message; default: disabled). pullEnabled atomic.Bool // registerAggregatedAPIServer controls serving aggregated API status push endpoints. @@ -166,6 +171,10 @@ func (h *healthState) setLeader(leader bool) { h.controllerReady.Store(false) } + if !leader { + h.stopDetailRequests() + } + if leader { leaderIsLeader.Set(1) klog.Info("Health: marked as leader") diff --git a/cmd/unbounded-net-controller/main.go b/cmd/unbounded-net-controller/main.go index b604dfbea..f88cc4804 100644 --- a/cmd/unbounded-net-controller/main.go +++ b/cmd/unbounded-net-controller/main.go @@ -129,8 +129,8 @@ on site configuration, and maintain SiteNodeSlice and GatewayPool status.`, flags.IntVar(&cfg.HealthPort, "health-port", 9999, "Port for health check HTTP server (0 to disable)") flags.IntVar(&cfg.NodeAgentHealthPort, "node-agent-health-port", 9998, "Port where node agents serve their health/status endpoints") flags.DurationVar(&cfg.StatusStaleThreshold, "status-stale-threshold", 90*time.Second, "Duration after which a node's pushed status is considered stale") - flags.DurationVar(&cfg.StatusDetailCacheTTL, "status-detail-cache-ttl", config.DefaultStatusDetailCacheTTL, "Lifetime of received node details (positive duration; preparatory)") - flags.DurationVar(&cfg.StatusDetailRequestTimeout, "status-detail-request-timeout", config.DefaultStatusDetailRequestTimeout, "End-to-end node detail request timeout (positive duration; preparatory)") + flags.DurationVar(&cfg.StatusDetailCacheTTL, "status-detail-cache-ttl", config.DefaultStatusDetailCacheTTL, "Lifetime of received node details (positive duration)") + flags.DurationVar(&cfg.StatusDetailRequestTimeout, "status-detail-request-timeout", config.DefaultStatusDetailRequestTimeout, "End-to-end node detail request timeout (positive duration)") flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "Interval between websocket keepalive pings on controller node status streams (0 to disable)") flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "Sequential websocket keepalive ping failures before closing node status websocket") flags.BoolVar(&cfg.RegisterAggregatedAPIServer, "register-aggregated-apiserver", true, "Serve node status push endpoints via aggregated API server paths") @@ -505,6 +505,8 @@ func run(cfg *config.Config, forceNotLeader bool) error { nodeName: os.Getenv("NODE_NAME"), statusCache: NewNodeStatusCache(), staleThreshold: cfg.StatusStaleThreshold, + statusDetailCacheTTL: cfg.StatusDetailCacheTTL, + statusDetailRequestTimeout: cfg.StatusDetailRequestTimeout, tokenAuth: newTokenAuthenticator(nodeTokenVerifier, []string{fmt.Sprintf("%s:unbounded-net-node", controllerNamespace)}), nodeServiceAccount: fmt.Sprintf("%s:unbounded-net-node", controllerNamespace), nodeTokenVerifier: nodeTokenVerifier, @@ -585,6 +587,14 @@ func run(cfg *config.Config, forceNotLeader bool) error { // Set informers in health state for efficient lookups in status endpoints healthState.setInformers(siteCtrl.GetNodeLister(), podLister, siteCtrl.GetSiteInformer(), gatewayPoolInformer, sitePeeringInformer, assignmentInformer, poolPeeringInformer) + detailRequests, err := healthState.startDetailRequests(ctx, informerFactory.Core().V1().Nodes().Informer()) + if err != nil { + klog.Errorf("Failed to start node detail requests: %v", err) + + return + } + defer detailRequests.Close() + healthState.siteController = siteCtrl if healthState.clusterStatusCache != nil { healthState.clusterStatusCache.MarkFullRebuildNeeded() diff --git a/cmd/unbounded-net-controller/server.go b/cmd/unbounded-net-controller/server.go index c706ca95e..6bd4551db 100644 --- a/cmd/unbounded-net-controller/server.go +++ b/cmd/unbounded-net-controller/server.go @@ -383,6 +383,8 @@ func serveStatusJSON(health *healthState, w http.ResponseWriter, r *http.Request } func registerStatusHandlers(mux *http.ServeMux, health *healthState, requireDashboardAuth bool, webhookServer *webhookpkg.Server, dashAuthorizer *dashboardAuthorizer, tokenIssuer *authn.TokenIssuer) { + registerNodeDetailHandlers(mux, health, requireDashboardAuth, webhookServer, dashAuthorizer, tokenIssuer) + mux.HandleFunc("/status/json", func(w http.ResponseWriter, r *http.Request) { if !authorizeDashboardOrAggregated(requireDashboardAuth, tokenIssuer, dashAuthorizer, webhookServer, r) { http.Error(w, "Unauthorized", http.StatusUnauthorized) diff --git a/deploy/net/controller/10-status-viewer.yaml.tmpl b/deploy/net/controller/10-status-viewer.yaml.tmpl index bb6393b76..c43ca862d 100644 --- a/deploy/net/controller/10-status-viewer.yaml.tmpl +++ b/deploy/net/controller/10-status-viewer.yaml.tmpl @@ -11,6 +11,9 @@ metadata: app.kubernetes.io/component: controller rbac.authorization.k8s.io/aggregate-to-view: "true" rules: + - apiGroups: ["status.net.unbounded-cloud.io"] + resources: ["nodes/details"] + verbs: ["get", "create"] - apiGroups: ["status.net.unbounded-cloud.io"] resources: ["status"] resourceNames: ["dashboard", "json"] diff --git a/internal/net/webhook/server.go b/internal/net/webhook/server.go index 1b7ea03e2..1ad1b336c 100644 --- a/internal/net/webhook/server.go +++ b/internal/net/webhook/server.go @@ -163,7 +163,7 @@ func (s *Server) registerAggregatedDiscoveryHandlers() { } w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"status.net.unbounded-cloud.io/v1alpha1","resources":[{"name":"status/push","singularName":"","namespaced":false,"kind":"NodeStatusPush","verbs":["create"]},{"name":"status/nodews","singularName":"","namespaced":false,"kind":"NodeStatusStream","verbs":["get"]},{"name":"status/json","singularName":"","namespaced":false,"kind":"ClusterStatus","verbs":["get"]},{"name":"token/node","singularName":"","namespaced":false,"kind":"TokenRequest","verbs":["create"]},{"name":"token/viewer","singularName":"","namespaced":false,"kind":"TokenRequest","verbs":["create"]}]}`)) //nolint:errcheck + _, _ = w.Write([]byte(`{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"status.net.unbounded-cloud.io/v1alpha1","resources":[{"name":"status/push","singularName":"","namespaced":false,"kind":"NodeStatusPush","verbs":["create"]},{"name":"status/nodews","singularName":"","namespaced":false,"kind":"NodeStatusStream","verbs":["get"]},{"name":"status/json","singularName":"","namespaced":false,"kind":"ClusterStatus","verbs":["get"]},{"name":"nodes/details","singularName":"","namespaced":false,"kind":"NodeDetails","verbs":["get","create"]},{"name":"token/node","singularName":"","namespaced":false,"kind":"TokenRequest","verbs":["create"]},{"name":"token/viewer","singularName":"","namespaced":false,"kind":"TokenRequest","verbs":["create"]}]}`)) //nolint:errcheck }) }