diff --git a/.gitignore b/.gitignore index 5b57e2dd..9ca32834 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ docs/public docs/resources/_gen/ docs/.hugo_build.lock test/integration/**/clab-* +test/integration/clab-* diff --git a/Makefile b/Makefile index fdcc2b24..8f87ed29 100644 --- a/Makefile +++ b/Makefile @@ -282,7 +282,7 @@ delete-outputs-dev-lab: ## Delete the outputs for the development lab cluster kubectl delete -f lab/dev/resources/outputs .PHONY: apply-pipelines-dev-lab -apply-pipelines-dev-lab: ## Apply the pipelines for the development lab cluster + §apply-pipelines-dev-lab: ## Apply the pipelines for the development lab cluster kubectl apply -f lab/dev/resources/pipelines .PHONY: delete-pipelines-dev-lab @@ -308,9 +308,10 @@ delete-targetsources-dev-lab: ## Delete the target sources for the development l ##@ Testing Lab .PHONY: run-integration-tests -run-integration-tests: docker-build undeploy-test-cluster deploy-test-cluster install-test-cluster-dependencies load-test-image deploy install-kubectl install-gnmic install-containerlab deploy-test-topology apply-test-resources +run-integration-tests: docker-build undeploy-test-cluster deploy-test-cluster install-test-cluster-dependencies load-test-image deploy deploy-test-http-server create-secrets-for-apiserver install-kubectl install-gnmic install-containerlab deploy-test-topology apply-test-resources send-target-to-apiserver kubectl wait --for=condition=Ready cluster --all --timeout=180s kubectl wait --for=condition=Ready pipeline --all --timeout=180s + kubectl wait --for=condition=Ready targetsource --all --timeout=180s kubectl wait --for=jsonpath='{.status.connectionState}'=READY target --all --timeout=180s kubectl get subscriptions -o yaml kubectl get outputs -o yaml diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 26a106f5..733299cd 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -58,18 +58,64 @@ type HTTPConfig struct { // +kubebuilder:validation:Optional URL string `json:"url,omitempty"` + // HTTP method used for the request. + // + // Defaults to GET if not specified. + // + // Supported values: + // - GET (default, no request body) + // - POST (supports request body) + // + // +kubebuilder:validation:Enum=GET;POST + // +kubebuilder:default="GET" + // +kubebuilder:validation:Optional + Method string `json:"method,omitempty"` + + // Optional HTTP headers to include in the request. + // + // These map directly to HTTP headers (key-value pairs). + // + // Example: + // headers: + // Content-Type: application/json + // X-Custom-Header: value + // + // Precedence: + // - Authentication configuration overrides any conflicting headers e.g. Authorization + // + // +kubebuilder:validation:Optional + Headers map[string]string `json:"headers,omitempty"` + + // Optional raw request body. + // + // Typically used with POST requests and contains JSON payload. + // + // Example: + // body: | + // { + // "limit": 100, + // "status": "active" + // } + // + // Notes: + // - Ignored for GET requests + // - User must set appropriate Content-Type header if needed + // + // +kubebuilder:validation:Optional + Body string `json:"body,omitempty"` + // Optional authentication configuration for accessing the HTTP endpoint // +kubebuilder:validation:Optional Authentication *AuthenticationSpec `json:"authentication,omitempty"` // Optional interval for polling the HTTP endpoint for targets // TODO: document about default value - // +kubebuilder:default="6h" + // +kubebuilder:default="30m" // +kubebuilder:validation:Optional Interval *metav1.Duration `json:"interval,omitempty"` // Optional timeout for HTTP requests to the endpoint - // +kubebuilder:default="10s" + // +kubebuilder:default="30s" // +kubebuilder:validation:Optional Timeout *metav1.Duration `json:"timeout,omitempty"` @@ -132,94 +178,238 @@ type TokenAuthSpec struct { TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` } -// PaginationSpec defines the configuration for paginating through responses from providers +// PaginationSpec defines how pagination is handled for HTTP APIs. +// +// The pagination mechanism is fully server-driven. The loader will repeatedly: +// 1. Extract the "next" reference from the response +// 2. Use it to construct the next request +// 3. Continue until no next reference is returned +// +// Supported pagination styles: +// 1. Cursor-based: +// - Response returns a token (e.g. "next_page_token") +// - Client sends it back via a query parameter (e.g. "page_token") +// 2. URL-based (nextLink): +// - Response returns a full URL +// - Client follows it directly without modification +// 3. Expression-based extraction: +// - The next reference is extracted using a CEL expression +// - This allows access to nested fields or special keys +// (e.g. "@odata.nextLink") +// +// Behavior: +// - If the extracted value is a full URL, it will be used as-is +// - Otherwise, it is treated as a token and appended using RequestParam +// - The token is treated as opaque and must not be interpreted +// +// Example: +// +// pagination: +// nextField: "self.next_page_token" +// requestParam: "page_token" +// +// pagination: +// nextField: "self['@odata.nextLink']" type PaginationSpec struct { - // Field name in the JSON response that contains the next page reference. - // The value can be either: - // - a full URL (used directly for the next request), or - // - a pagination token (appended as a query parameter using this field name as the key). + // CEL expression used to extract the next page reference from the response. + // + // The expression is evaluated with: + // self -> full JSON response // - // Must refer to a top-level key in the response object. - // Example: "next" or "nextToken" + // It must evaluate to either: + // - string (full URL OR token), or + // - null (indicates end of pagination) + // + // Examples: + // "self.next" + // "self.next_page_token" + // "self['@odata.nextLink']" + // + // +kubebuilder:validation:Optional NextField string `json:"nextField,omitempty"` + + // Query parameter name used when the extracted value is a token. + // + // Required for token-based pagination. + // Ignored when NextField resolves to a full URL. + // + // Example: + // requestParam: "page_token" + // + // +kubebuilder:validation:Optional + RequestParam string `json:"requestParam,omitempty"` } -// CEL expressions to extract target fields from the response -// and map them to the corresponding Target fields. +// ResponseMappingSpec controls how targets are extracted from an HTTP JSON response. +// +// This allows you to map fields from a JSON API into targets using either: +// - simple direct field access (e.g. item["name"]) +// - or CEL expressions for more advanced logic +// +// General behavior: +// +// 1. Selecting targets: +// - `targetsField` is a CEL expression that selects the list of targets +// - It runs once on the full response (`self`) and MUST return a list +// - If not set, the response itself must be a JSON array +// +// 2. Extracting fields: +// - Each field (name, address, port, labels, etc.) is handled independently +// - If a CEL expression is provided → it is evaluated +// - If not provided → the value is read directly from the target object +// +// 3. Available variables in CEL: +// - item -> the current target object +// - self -> the full HTTP response JSON +// +// Example: +// +// Response: +// { +// "results": [ +// { "name": "device1", "ip": "10.0.0.1", "env": "prod" } +// ], +// "meta": { "region": "eu-west" } +// } +// +// Mapping: +// targetsField: "self.results" +// +// name: "" # direct → item["name"] +// address: "item.ip" # CEL +// +// labels: +// env: "item.env" +// region: "self.meta.region" type ResponseMappingSpec struct { - // Field name in the JSON response that contains the list of items (targets). - // If not specified, the entire response is expected to be a list of items. - // All subsequent fields are specified relative to this field - // Example: "results" if the response is of the form {"results": [ ... list of items ... ]} + // CEL expression that selects the list of target objects from the response. + // + // This is evaluated once using: + // self -> full JSON response + // + // Example: + // targetsField: "self.results" + // + // If not set, the response itself must be a JSON array with the targets. + // // +kubebuilder:validation:Optional TargetsField string `json:"targetsField,omitempty"` - // CEL expression to extract the target name from the response - // If TargetsField is specified, this should be relative to TargetsField + // CEL expression for the target name. + // + // If not set, defaults to: + // item["name"] + // + // Example: + // "item.hostname" + // // +kubebuilder:validation:Optional - Name string `json:"name"` + Name string `json:"name,omitempty"` - // CEL expression to extract the target Address from the response - // If TargetsField is specified, this should be relative to TargetsField + // CEL expression for the target address. + // + // If not set, defaults to: + // item["address"] + // + // Example: + // "item.ip" + // // +kubebuilder:validation:Optional - Address string `json:"address"` + Address string `json:"address,omitempty"` - // CEL expression to extract the target port from the response - // If TargetsField is specified, this should be relative to TargetsField + // CEL expression for the target port. + // + // If not set, defaults to: + // item["port"] + // + // Example: + // "item.port" + // // +kubebuilder:validation:Optional Port string `json:"port,omitempty"` - // CEL expression to extract the target labels from the response + // CEL expression that returns a map of labels. + // The expression must evaluate to an object (map). + // + // Example: + // + // labels: | + // { + // "env": item.environment, + // "region": self.meta.region, + // item.dynamicKey: "value" + // } + // + // If not set, defaults to: + // item["labels"] + // + // The resulting map will be converted into labels. // The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec, // with values from the response taking precedence in case of conflicts. + // // +kubebuilder:validation:Optional - Labels map[string]string `json:"labels,omitempty"` + Labels string `json:"labels,omitempty"` - // CEL expression to extract the target profile from the response - // If TargetsField is specified, this should be relative to TargetsField + // CEL expression for the target profile. + // + // If not set, defaults to: + // item["targetProfile"] + // + // Example: + // "item.type == 'edge' ? 'edge-profile' : 'default'" + // // +kubebuilder:validation:Optional TargetProfile string `json:"targetProfile,omitempty"` } // PushSpec defines the settings for event-based update mechanism (i.e. webhooks sent from the server) type PushSpec struct { + // Enables the applyTargets endpoint; when disabled, incoming requests are rejected. + // +kubebuilder:validation:Required // +kubebuilder:default=false Enabled bool `json:"enabled"` + // Enables Authentication; when empty, authentication is not checked // +kubebuilder:validation:Optional Auth *PushAuthSpec `json:"auth,omitempty"` + + // Enables Signature verification; when empty, signature is not checked + // +kubebuilder:validation:Optional + Signature *PushSignatureSpec `json:"signature,omitempty"` } -// +kubebuilder:validation:ExactlyOneOf:=bearer;signature +// +kubebuilder:validation:Optional type PushAuthSpec struct { - Bearer *PushBearerAuthSpec `json:"bearer,omitempty"` - Signature *PushSignatureAuthSpec `json:"signature,omitempty"` + // Spec for Bearer Token authentication + Bearer *PushBearerAuthSpec `json:"bearer,omitempty"` } // +kubebuilder:validation:Required type PushBearerAuthSpec struct { + // Reference to a secret containing a token that is checked to the Bearer token found in the + // authorization header of the incoming HTTP request. TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` } // +kubebuilder:validation:Required -type PushSignatureAuthSpec struct { - SecretRef *corev1.SecretKeySelector `json:"secretRef"` - - // Header containing the signature - // +kubebuilder:validation:MinLength=1 - Header string `json:"header"` +type PushSignatureSpec struct { + // Reference to a secret containing a signature key. HMAC calculated with secret and HTTP body + // to calculate X-Hook-Signature. Compared with X-Hook-Signature in HTTP header + SecretRef *corev1.SecretKeySelector `json:"secretRef,omitempty"` + // Enum to select hashing algorithm during HMAC calculation // +kubebuilder:default="sha512" - // +kubebuilder:validation:Enum=sha1;sha256;sha512 + // +kubebuilder:validation:Enum=sha256;sha512 Algorithm string `json:"algorithm"` } + // TargetSourceStatus defines the observed state of TargetSource type TargetSourceStatus struct { - Status string `json:"status,omitempty"` - ObservedGeneration int64 `json:"observedGeneration"` - TargetsCount int32 `json:"targetsCount,omitempty"` - LastSync metav1.Time `json:"lastSync,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + TargetsCount int32 `json:"targetsCount,omitempty"` + LastSync metav1.Time `json:"lastSync,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 201a35da..cb0e520b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -323,6 +323,13 @@ func (in *GRPCTunnelConfig) DeepCopy() *GRPCTunnelConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) { *out = *in + if in.Headers != nil { + in, out := &in.Headers, &out.Headers + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Authentication != nil { in, out := &in.Authentication, &out.Authentication *out = new(AuthenticationSpec) @@ -351,7 +358,7 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) { if in.ResponseMapping != nil { in, out := &in.ResponseMapping, &out.ResponseMapping *out = new(ResponseMappingSpec) - (*in).DeepCopyInto(*out) + **out = **in } if in.Push != nil { in, out := &in.Push, &out.Push @@ -946,11 +953,6 @@ func (in *PushAuthSpec) DeepCopyInto(out *PushAuthSpec) { *out = new(PushBearerAuthSpec) (*in).DeepCopyInto(*out) } - if in.Signature != nil { - in, out := &in.Signature, &out.Signature - *out = new(PushSignatureAuthSpec) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushAuthSpec. @@ -984,7 +986,7 @@ func (in *PushBearerAuthSpec) DeepCopy() *PushBearerAuthSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PushSignatureAuthSpec) DeepCopyInto(out *PushSignatureAuthSpec) { +func (in *PushSignatureSpec) DeepCopyInto(out *PushSignatureSpec) { *out = *in if in.SecretRef != nil { in, out := &in.SecretRef, &out.SecretRef @@ -993,12 +995,12 @@ func (in *PushSignatureAuthSpec) DeepCopyInto(out *PushSignatureAuthSpec) { } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushSignatureAuthSpec. -func (in *PushSignatureAuthSpec) DeepCopy() *PushSignatureAuthSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushSignatureSpec. +func (in *PushSignatureSpec) DeepCopy() *PushSignatureSpec { if in == nil { return nil } - out := new(PushSignatureAuthSpec) + out := new(PushSignatureSpec) in.DeepCopyInto(out) return out } @@ -1011,6 +1013,11 @@ func (in *PushSpec) DeepCopyInto(out *PushSpec) { *out = new(PushAuthSpec) (*in).DeepCopyInto(*out) } + if in.Signature != nil { + in, out := &in.Signature, &out.Signature + *out = new(PushSignatureSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushSpec. @@ -1026,13 +1033,6 @@ func (in *PushSpec) DeepCopy() *PushSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) { *out = *in - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResponseMappingSpec. @@ -1517,6 +1517,13 @@ func (in *TargetSourceSpec) DeepCopy() *TargetSourceSpec { func (in *TargetSourceStatus) DeepCopyInto(out *TargetSourceStatus) { *out = *in in.LastSync.DeepCopyInto(&out.LastSync) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetSourceStatus. diff --git a/cmd/main.go b/cmd/main.go index 3bb04f7a..b049d86c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -18,7 +18,9 @@ package main import ( "context" + "errors" "flag" + "net/http" "os" "time" @@ -125,12 +127,22 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Pipeline") os.Exit(1) } + + var api *apiserver.APIServer + if apiAddr != "" { + api, err = apiserver.New(apiAddr, clusterReconciler, discoveryRegistry, discoveryChunkSize, os.Getenv("API_BEARER_TOKEN")) + if err != nil { + setupLog.Error(err, "unable to initialize API server") + os.Exit(1) + } + } if err := (&controller.TargetSourceReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), BufferSize: discoveryBufferSize, ChunkSize: discoveryChunkSize, DiscoveryRegistry: discoveryRegistry, + APIRouter: api.Router(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TargetSource") os.Exit(1) @@ -230,21 +242,27 @@ func main() { os.Exit(1) } - if apiAddr != "" { - apiServer := apiserver.New(apiAddr, clusterReconciler) - apiServer.DiscoveryRegistry = discoveryRegistry + if api != nil { err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) go func() { - errCh <- apiServer.Server.ListenAndServe() + err := api.Server.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- err + } + close(errCh) }() + select { - case err := <-errCh: + case err, ok := <-errCh: + if !ok { + return nil + } return err case <-ctx.Done(): ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - return apiServer.Server.Shutdown(ctx) + return api.Server.Shutdown(ctx) } })) if err != nil { diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 4ecef754..2ed51056 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -127,8 +127,41 @@ spec: be set rule: '[has(self.basic),has(self.token)].filter(x,x==true).size() == 1' + body: + description: |- + Optional raw request body. + + Typically used with POST requests and contains JSON payload. + + Example: + body: | + { + "limit": 100, + "status": "active" + } + + Notes: + - Ignored for GET requests + - User must set appropriate Content-Type header if needed + type: string + headers: + additionalProperties: + type: string + description: |- + Optional HTTP headers to include in the request. + + These map directly to HTTP headers (key-value pairs). + + Example: + headers: + Content-Type: application/json + X-Custom-Header: value + + Precedence: + - Authentication configuration overrides any conflicting headers e.g. Authorization + type: object interval: - default: 6h + default: 30m description: Optional interval for polling the HTTP endpoint for targets type: string @@ -138,65 +171,137 @@ spec: properties: address: description: |- - CEL expression to extract the target Address from the response - If TargetsField is specified, this should be relative to TargetsField + CEL expression for the target address. + + If not set, defaults to: + item["address"] + + Example: + "item.ip" type: string labels: - additionalProperties: - type: string description: |- - CEL expression to extract the target labels from the response + CEL expression that returns a map of labels. + The expression must evaluate to an object (map). + + Example: + + labels: | + { + "env": item.environment, + "region": self.meta.region, + item.dynamicKey: "value" + } + + If not set, defaults to: + item["labels"] + + The resulting map will be converted into labels. The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec, with values from the response taking precedence in case of conflicts. - type: object + type: string name: description: |- - CEL expression to extract the target name from the response - If TargetsField is specified, this should be relative to TargetsField + CEL expression for the target name. + + If not set, defaults to: + item["name"] + + Example: + "item.hostname" type: string port: description: |- - CEL expression to extract the target port from the response - If TargetsField is specified, this should be relative to TargetsField + CEL expression for the target port. + + If not set, defaults to: + item["port"] + + Example: + "item.port" type: string targetProfile: description: |- - CEL expression to extract the target profile from the response - If TargetsField is specified, this should be relative to TargetsField + CEL expression for the target profile. + + If not set, defaults to: + item["targetProfile"] + + Example: + "item.type == 'edge' ? 'edge-profile' : 'default'" type: string targetsField: description: |- - Field name in the JSON response that contains the list of items (targets). - If not specified, the entire response is expected to be a list of items. - All subsequent fields are specified relative to this field - Example: "results" if the response is of the form {"results": [ ... list of items ... ]} + CEL expression that selects the list of target objects from the response. + + This is evaluated once using: + self -> full JSON response + + Example: + targetsField: "self.results" + + If not set, the response itself must be a JSON array with the targets. type: string type: object + method: + default: GET + description: |- + HTTP method used for the request. + + Defaults to GET if not specified. + + Supported values: + - GET (default, no request body) + - POST (supports request body) + enum: + - GET + - POST + type: string pagination: description: Optional pagination configuration for parsing responses from the HTTP endpoint properties: nextField: description: |- - Field name in the JSON response that contains the next page reference. - The value can be either: - - a full URL (used directly for the next request), or - - a pagination token (appended as a query parameter using this field name as the key). + CEL expression used to extract the next page reference from the response. + + The expression is evaluated with: + self -> full JSON response + + It must evaluate to either: + - string (full URL OR token), or + - null (indicates end of pagination) + + Examples: + "self.next" + "self.next_page_token" + "self['@odata.nextLink']" + type: string + requestParam: + description: |- + Query parameter name used when the extracted value is a token. + + Required for token-based pagination. + Ignored when NextField resolves to a full URL. - Must refer to a top-level key in the response object. - Example: "next" or "nextToken" + Example: + requestParam: "page_token" type: string type: object push: description: Optional configuration to enable push properties: auth: + description: Enables Authentication; when empty, authentication + is not checked properties: bearer: + description: Spec for Bearer Token authentication properties: tokenSecretRef: - description: SecretKeySelector selects a key of - a Secret. + description: |- + Reference to a secret containing a token that is checked to the Bearer token found in the + authorization header of the incoming HTTP request. properties: key: description: The key of the secret to select @@ -220,63 +325,58 @@ spec: type: object x-kubernetes-map-type: atomic type: object - signature: + type: object + enabled: + default: false + description: Enables the applyTargets endpoint; when disabled, + incoming requests are rejected. + type: boolean + signature: + description: Enables Signature verification; when empty, + signature is not checked + properties: + algorithm: + default: sha512 + description: Enum to select hashing algorithm during + HMAC calculation + enum: + - sha256 + - sha512 + type: string + secretRef: + description: |- + Reference to a secret containing a signature key. HMAC calculated with secret and HTTP body + to calculate X-Hook-Signature. Compared with X-Hook-Signature in HTTP header properties: - algorithm: - default: sha512 - enum: - - sha1 - - sha256 - - sha512 + key: + description: The key of the secret to select from. Must + be a valid secret key. type: string - header: - description: Header containing the signature - minLength: 1 + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string - secretRef: - description: SecretKeySelector selects a key of - a Secret. - properties: - key: - description: The key of the secret to select - from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or - its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean required: - - algorithm - - header - - secretRef + - key type: object + x-kubernetes-map-type: atomic + required: + - algorithm type: object - x-kubernetes-validations: - - message: exactly one of the fields in [bearer signature] - must be set - rule: '[has(self.bearer),has(self.signature)].filter(x,x==true).size() - == 1' - enabled: - default: false - type: boolean required: - enabled type: object timeout: - default: 10s + default: 30s description: Optional timeout for HTTP requests to the endpoint type: string tls: @@ -350,19 +450,71 @@ spec: status: description: TargetSourceStatus defines the observed state of TargetSource properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array lastSync: format: date-time type: string observedGeneration: format: int64 type: integer - status: - type: string targetsCount: format: int32 type: integer - required: - - observedGeneration type: object type: object served: true diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index c8822678..687edd2d 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -6,4 +6,4 @@ kind: Kustomization images: - name: controller newName: gnmic-operator - newTag: dev + newTag: ci diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 2cd79f09..b377fe45 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -73,6 +73,21 @@ spec: - --leader-elect image: controller:latest name: manager + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/name'] + - name: API_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: gnmic-api-auth + key: bearer-token + optional: true securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/docs/content/docs/Apis/DefaultApi.md b/docs/content/docs/Apis/DefaultApi.md new file mode 100644 index 00000000..513f6da6 --- /dev/null +++ b/docs/content/docs/Apis/DefaultApi.md @@ -0,0 +1,2 @@ +# DefaultApi + diff --git a/docs/content/docs/advanced/openapi-generator-config.yaml b/docs/content/docs/advanced/openapi-generator-config.yaml new file mode 100644 index 00000000..75e95512 --- /dev/null +++ b/docs/content/docs/advanced/openapi-generator-config.yaml @@ -0,0 +1,9 @@ +# docker run --rm -v ${PWD}:/local openapitools/openapi-generator-cli generate -c /local/docs/content/docs/advanced/openapi-generator-config.yaml + +generatorName: markdown +inputSpec: /local/internal/apiserver/openapi.yaml +outputDir: /local/docs/content/docs/advanced/rest-api-documentation +templateDir: /local/docs/content/docs/advanced/openapi-templates +files: + README.mustache: + destinationFilename: _index.md \ No newline at end of file diff --git a/docs/content/docs/advanced/openapi-templates/_index.mustache b/docs/content/docs/advanced/openapi-templates/_index.mustache new file mode 100644 index 00000000..7b3df486 --- /dev/null +++ b/docs/content/docs/advanced/openapi-templates/_index.mustache @@ -0,0 +1,69 @@ +--- +title: "REST API interface" +linkTitle: "REST API interface" +weight: 3 +description: > + This document describes the REST API exposed by the gNMIc Operator, including the available endpoints, request formats, and usage examples. +--- + +{{#generateApiDocs}} + +## Documentation for API Endpoints + +All URIs are relative to *{{{basePath}}}:8082* + +| Class | Method | HTTP request | Description | +|------------ | ------------- | ------------- | -------------| +{{#apiInfo}}{{#apis}}{{#operations}}| {{#operation}}*{{#lambda.lowercase}}{{classname}}{{/lambda.lowercase}}* | [**{{operationId}}**](/docs/advanced/rest-api-documentation/apis/{{#lambda.lowercase}}{{classname}}{{/lambda.lowercase}}) | **{{httpMethod}}** {{path}} | {{{summary}}}{{^summary}}{{{notes}}}{{/summary}} | +{{/operation}}{{/operations}}{{/apis}}{{/apiInfo}} +{{/generateApiDocs}} + +{{#generateModelDocs}} + +## Documentation for Models + +{{#modelPackage}} +{{#models}}{{#model}} - [{{#lambda.lowercase}}{{{classname}}}{{/lambda.lowercase}}](/docs/advanced/rest-api-documentation/models/{{#lambda.lowercase}}{{{classFilename}}}{{/lambda.lowercase}}/) +{{/model}}{{/models}} +{{/modelPackage}} +{{^modelPackage}} +No model defined in this package +{{/modelPackage}} +{{/generateModelDocs}} + +{{! TODO: optional documentation for authorization? }} +## Documentation for Authorization + +For a detailed explanation on how to configure the required secrets within the gNMIc Operator, refer to [TargetSource > Push mode](/docs/user-guide/targetsource/push/). + +{{^authMethods}} +All endpoints do not require authorization. +{{/authMethods}} +{{#authMethods}} +{{#last}} + Authentication schemes defined for the API: +{{/last}} +{{/authMethods}} +{{#authMethods}} + +### {{name}} + +{{#isApiKey}}- **Type**: API key +- **API key parameter name**: {{keyParamName}} +- **Location**: {{#isKeyInQuery}}URL query string{{/isKeyInQuery}}{{#isKeyInHeader}}HTTP header{{/isKeyInHeader}} +{{/isApiKey}} +{{#isBasicBasic}}- **Type**: HTTP basic authentication +{{/isBasicBasic}} +{{#isBasicBearer}}- **Type**: HTTP Bearer Token authentication{{#bearerFormat}} ({{{.}}}){{/bearerFormat}} +{{/isBasicBearer}} +{{#isHttpSignature}}- **Type**: HTTP signature authentication +{{/isHttpSignature}} +{{#isOAuth}}- **Type**: OAuth +- **Flow**: {{flow}} +- **Authorization URL**: {{authorizationUrl}} +- **Scopes**: {{^scopes}}N/A{{/scopes}} +{{#scopes}} - {{scope}}: {{description}} +{{/scopes}} +{{/isOAuth}} + +{{/authMethods}} \ No newline at end of file diff --git a/docs/content/docs/advanced/openapi-templates/apis.mustache b/docs/content/docs/advanced/openapi-templates/apis.mustache new file mode 100644 index 00000000..8a75a8a9 --- /dev/null +++ b/docs/content/docs/advanced/openapi-templates/apis.mustache @@ -0,0 +1,50 @@ +--- +title: "Routes" +linkTitle: "Routes" +weight: 4 +description: > + Available HTTP routes on the gNMIc Operator API interface. +--- + +# {{#lambda.lowercase}}{{classname}}{{/lambda.lowercase}}{{#description}} + {{.}}{{/description}} + +All URIs are relative to *{{basePath}}:8082* + +| Method | HTTP request | Description | +|------------- | ------------- | -------------| +{{#operations}}{{#operation}}| **{{operationId}}** | **{{httpMethod}}** {{path}} | {{summary}} | +{{/operation}}{{/operations}} + +{{#operations}} +{{#operation}} + +# **{{operationId}}** +> {{#returnType}}{{.}} {{/returnType}}{{operationId}}({{#allParams}}{{{paramName}}}{{^-last}}, {{/-last}}{{/allParams}}) + +{{summary}}{{#notes}} + + {{.}}{{/notes}} + +### Parameters +{{^allParams}}This endpoint does not need any parameter.{{/allParams}}{{#allParams}}{{#-last}} +|Name | Type | Description | Notes | +|------------- | ------------- | ------------- | -------------|{{/-last}}{{/allParams}} +{{#allParams}}| **{{paramName}}** | {{#isPrimitiveType}}**{{dataType}}**{{/isPrimitiveType}}{{^isPrimitiveType}}{{#isFile}}**{{dataType}}**{{/isFile}}{{^isFile}}{{#generateModelDocs}}[**{{dataType}}**](/docs/advanced/rest-api-documentation/models/{{#lambda.lowercase}}{{baseType}}{{/lambda.lowercase}}/){{/generateModelDocs}}{{^generateModelDocs}}**{{dataType}}**{{/generateModelDocs}}{{/isFile}}{{/isPrimitiveType}}| {{description}} |{{^required}} [optional]{{/required}}{{#defaultValue}} [default to {{.}}]{{/defaultValue}}{{#allowableValues}} [enum: {{#values}}{{{.}}}{{^-last}}, {{/-last}}{{/values}}]{{/allowableValues}} | +{{/allParams}} + +### Return type + +{{#returnType}}{{#returnTypeIsPrimitive}}**{{returnType}}**{{/returnTypeIsPrimitive}}{{^returnTypeIsPrimitive}}{{#generateModelDocs}}[**{{returnType}}**](/docs/advanced/rest-api-documentation/models/{{#lambda.lowercase}}{{returnBaseType}}{{/lambda.lowercase}}/){{/generateModelDocs}}{{^generateModelDocs}}**{{returnType}}**{{/generateModelDocs}}{{/returnTypeIsPrimitive}}{{/returnType}}{{^returnType}}null (empty response body){{/returnType}} + +### Authorization + +{{^authMethods}}No authorization required{{/authMethods}}{{#authMethods}}[{{name}}](/docs/advanced/rest-api-documentation/#{{name}}){{^-last}}, {{/-last}}{{/authMethods}} + +### HTTP request headers + +- **Content-Type**: {{#consumes}}{{{mediaType}}}{{^-last}}, {{/-last}}{{/consumes}}{{^consumes}}Not defined{{/consumes}} +- **Accept**: {{#produces}}{{{mediaType}}}{{^-last}}, {{/-last}}{{/produces}}{{^produces}}Not defined{{/produces}} + +{{/operation}} +{{/operations}} \ No newline at end of file diff --git a/docs/content/docs/advanced/openapi-templates/models.mustache b/docs/content/docs/advanced/openapi-templates/models.mustache new file mode 100644 index 00000000..a110b934 --- /dev/null +++ b/docs/content/docs/advanced/openapi-templates/models.mustache @@ -0,0 +1,29 @@ +--- +title: "Model" +linkTitle: "Model" +weight: 4 +description: > + Documentation for OpenAPI models and their schema-defined properties. +--- + +{{#models}} +{{#model}} +# {{#lambda}}{{{classname}}}{{/lambda}} +{{#description}} +{{{description}}} +{{/description}} + +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +{{#parent}} +{{#parentVars}} +| **{{name}}** | {{#isPrimitiveType}}**{{dataType}}**{{/isPrimitiveType}}{{^isPrimitiveType}}**{{dataType}}**{{/isPrimitiveType}} | {{{description}}} | {{^required}}[optional] {{/required}}{{#readOnly}}[readonly] {{/readOnly}}{{#defaultValue}}[default to {{{.}}}]{{/defaultValue}} | +{{/parentVars}} +{{/parent}} +{{#vars}}| **{{name}}** | {{#isPrimitiveType}}**{{dataType}}**{{/isPrimitiveType}}{{^isPrimitiveType}}**{{dataType}}**{{/isPrimitiveType}} | {{{description}}} | {{^required}}[optional] {{/required}}{{#readOnly}}[readonly] {{/readOnly}}{{#defaultValue}}[default to {{{.}}}]{{/defaultValue}} | +{{/vars}} + + {{/model}} +{{/models}} \ No newline at end of file diff --git a/docs/content/docs/advanced/rest-api-documentation.md b/docs/content/docs/advanced/rest-api-documentation.md new file mode 100644 index 00000000..edba35f0 --- /dev/null +++ b/docs/content/docs/advanced/rest-api-documentation.md @@ -0,0 +1,33 @@ +--- +title: "Documentation for gNMIc Operator REST API" +linkTitle: "REST API interface" +weight: 3 +description: > + Documentation of REST API interface according to openAPI standard. +--- + + +## Documentation for API Endpoints + +All URIs are relative to *http://localhost* + +| Class | Method | HTTP request | Description | +|------------ | ------------- | ------------- | -------------| +| *DefaultApi* | [**applyTargets**](../Apis/DefaultApi.md#applyTargets) | **POST** /api/v1/:namespace/target-source/:name/applyTargets | Interface for real-time target updates, usually using a webhook. Targets are applied in the gNMIc Operator. | +*DefaultApi* | [**getClusterPlan**](../Apis/DefaultApi.md#getClusterPlan) | **GET** /clusters/:namespace/:name/plan | Get cluster plan. | + + + +## Documentation for Models + + - [Target](../Models/Target.md) + + + +## Documentation for Authorization + + +### bearerAuth + +- **Type**: HTTP Bearer Token authentication + diff --git a/docs/content/docs/advanced/rest-api-documentation/Apis/DefaultApi.md b/docs/content/docs/advanced/rest-api-documentation/Apis/DefaultApi.md new file mode 100644 index 00000000..a5f0f3d4 --- /dev/null +++ b/docs/content/docs/advanced/rest-api-documentation/Apis/DefaultApi.md @@ -0,0 +1,57 @@ +# DefaultApi + +All URIs are relative to *http://localhost* + +| Method | HTTP request | Description | +|------------- | ------------- | -------------| +| [**applyTargets**](DefaultApi.md#applyTargets) | **POST** /api/v1/:namespace/target-source/:name/applyTargets | Interface for real-time target updates, usually using a webhook. Targets are applied in the gNMIc Operator. | +| [**getClusterPlan**](DefaultApi.md#getClusterPlan) | **GET** /clusters/:namespace/:name/plan | Get cluster plan. | + + + +# **applyTargets** +> List applyTargets(Target) + +Interface for real-time target updates, usually using a webhook. Targets are applied in the gNMIc Operator. + +### Parameters + +|Name | Type | Description | Notes | +|------------- | ------------- | ------------- | -------------| +| **Target** | [**List**](../Models/Target.md)| Target must be passed as a list, multiple targets possible. | | + +### Return type + +[**List**](../Models/Target.md) + +### Authorization + +[signature](../README.md#signature), [bearerAuth](../README.md#bearerAuth) + +### HTTP request headers + +- **Content-Type**: application/json +- **Accept**: application/json + + +# **getClusterPlan** +> getClusterPlan() + +Get cluster plan. + +### Parameters +This endpoint does not need any parameter. + +### Return type + +null (empty response body) + +### Authorization + +No authorization required + +### HTTP request headers + +- **Content-Type**: Not defined +- **Accept**: Not defined + diff --git a/docs/content/docs/advanced/rest-api-documentation/Models/Target.md b/docs/content/docs/advanced/rest-api-documentation/Models/Target.md new file mode 100644 index 00000000..18f7031c --- /dev/null +++ b/docs/content/docs/advanced/rest-api-documentation/Models/Target.md @@ -0,0 +1,14 @@ +# Target +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **name** | **String** | Name of device to be monitored. | [default to null] | +| **address** | **String** | IPv4/IPv6 address or hostname. | [default to null] | +| **port** | **Integer** | gNMIc port. | [optional] [default to null] | +| **targetProfile** | **String** | TargetProfile applied to apply to this router. | [optional] [default to null] | +| **labels** | [**List**](map.md) | Labels must be map[string]string. For example vendor:nokia. | [optional] [default to null] | +| **operation** | **String** | Either `created`, `updated` or `deleted`. `created` and `updated` are identical and both apply the target. | [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/content/docs/advanced/rest-api-documentation/_index.md b/docs/content/docs/advanced/rest-api-documentation/_index.md new file mode 100644 index 00000000..a552b56a --- /dev/null +++ b/docs/content/docs/advanced/rest-api-documentation/_index.md @@ -0,0 +1,34 @@ +# Documentation for gNMIc Operator REST API + + +## Documentation for API Endpoints + +All URIs are relative to *http://localhost* + +| Class | Method | HTTP request | Description | +|------------ | ------------- | ------------- | -------------| +| *DefaultApi* | [**applyTargets**](Apis/DefaultApi.md#applyTargets) | **POST** /api/v1/:namespace/target-source/:name/applyTargets | Interface for real-time target updates, usually using a webhook. Targets are applied in the gNMIc Operator. | +*DefaultApi* | [**getClusterPlan**](Apis/DefaultApi.md#getClusterPlan) | **GET** /clusters/:namespace/:name/plan | Get cluster plan. | + + + +## Documentation for Models + + - [Target](./Models/Target.md) + + + +## Documentation for Authorization + + +### bearerAuth + +- **Type**: HTTP Bearer Token authentication + + +### signature + +- **Type**: API key +- **API key parameter name**: X-Hook-Signature +- **Location**: HTTP header + diff --git a/docs/content/docs/examples/NetBox/Export Template/_index.md b/docs/content/docs/examples/NetBox/Export Template/_index.md new file mode 100644 index 00000000..fd481841 --- /dev/null +++ b/docs/content/docs/examples/NetBox/Export Template/_index.md @@ -0,0 +1,399 @@ +--- +title: "Pull with Export Template" +linkTitle: "Pull with Export Template" +weight: 1 +description: > + Discover targets from NetBox using HTTP provider with NetBox Export Template +--- + +This guide shows how to use **NetBox Export Templates** with the HTTP provider to discover and sync targets. + +Export Templates offer powerful filtering, transformation, and formatting directly in NetBox, reducing the load on the operator. + +## Overview + +An **Export Template** is a Jinja2 template that: + +1. **Queries** NetBox's internal database (devices, interfaces, etc.) +2. **Filters** results based on custom criteria +3. **Transforms** data into your desired output format +4. **Returns** the formatted output via REST API endpoint + +When used with gNMIc's HTTP provider, the operator fetches the rendered JSON template and parses the result with no further transformation needed by the gNMIc Operator. + +--- + +## Prerequisites + +- A running Kubernetes cluster with gNMIc Operator installed +- `kubectl` access to your cluster +- A reachable NetBox instance with permissions to create Export Templates +- A NetBox API token +- Familiarity with Jinja2 templates + +--- + +## Step 1: Create a NetBox API Token and Store It Securely + +### Step 1a: Create the API Token in NetBox + +Create a dedicated API token in NetBox for gNMIc Operator access. + +1. Log in to NetBox. +2. Open your user profile or go to **User > API Tokens**. +3. Click **Add** or **Add token**. +4. Enter a descriptive name such as `gNMIc Operator`. +5. Grant the minimum permissions required for read-only device discovery. +6. Copy the token value and store it safely; NetBox will not show it again. + +### Step 1b: Store the Token in a Kubernetes Secret + +Create a [Kubernetes Secret](https://kubernetes.io/docs/concepts/configuration/secret/) containing the token. + +```bash +# Substitute YOUR_NETBOX_API_TOKEN with your actual token +# Bearer Token Format (v2): nbt_. +kubectl create secret generic netbox-api-token \ + --from-literal=token=YOUR_NETBOX_API_TOKEN \ + -n gnmic-system +``` + +Verify the Secret was created: + +```bash +kubectl get secret netbox-api-token -n gnmic-system -o yaml +``` + +--- + +## Step 2: Create an Export Template in NetBox + +Log in to your NetBox instance and navigate to **Customization > Export Templates**. + +### Step 2a: Create a New Template + +Click **Add Export Template** and fill in the details: + +| Field | Value | Notes | +|-------|-------|-------| +| **Name** | `gNMIc Device Export` | Descriptive name for your template | +| **Content Type** | `dcim > device` | Export template applies to Device objects | +| **Template Code** | (see below) | Jinja2 template | +| **File Extension** | `json` | Output format | +| **Mime Type** | `application/json` | Correct MIME type for JSON | + +### Step 2b: Template Code Example + +The following Export Templates only work for devices that have a primary IPv4 address set in NetBox. If primary_ip4 is missing, the expression returns '', so those devices will not yield a valid target address. For NetBox data model details, see the [NetBox Devices Data Model](https://netboxlabs.com/docs/netbox/models/dcim/device/) documentation. + +See the HTTP provider's "Default Response Format" section for the expected JSON structure: [HTTP provider docs](../../user-guide/targetsource/providers/http.md) + +#### Basic Template (All Devices) + +```jinja2 +[ + {% for device in queryset %} + { + "name": "{{ device.name }}", + "address": "{{ device.primary_ip4.address.ip }}", + "labels": { + "site": "{{ device.site.name }}", + "role": "{{ device.role.name }}", + "region": "{{ device.site.region.name }}", + "type": "{{ device.device_type.model }}" + } + }{{ "," if not loop.last }} + {% endfor %} +] +``` + +#### Advanced Template (Filtered by Status and Role) + +```jinja2 +[ + {% for device in queryset.filter(status='active', role__name__in=['leaf', 'spine']) %} + { + "name": "{{ device.name }}", + "address": "{{ device.primary_ip4.address.ip }}", + "labels": { + "site": "{{ device.site.name }}", + "role": "{{ device.role.name }}", + "region": "{{ device.site.region.name }}", + "model": "{{ device.device_type.model }}", + "serial": "{{ device.serial }}", + "asset_tag": "{{ device.asset_tag }}" + } + }{{ "," if not loop.last }} + {% endfor %} +] +``` + +**Key template elements:** + +- `queryset`: The filtered set of devices (all unless you add `.filter()`) +- `device.name`: Device hostname +- `device.primary_ip4.address.ip`: Primary IPv4 address +- `device.site.name`, `device.device_role.name`: NetBox relationships (site, role, etc.) +- `loop.last`: Jinja2 loop variable to avoid trailing comma on last item + +### Step 2c: Save and Access the Template + +Once saved, NetBox exposes the template via: + +``` +http://netbox.example.com:8000/api/dcim/devices/?export=gNMIc+Device+Export +``` + +Or fetch it directly: + +```bash +# Replace with your NetBox URL and template name +# Substitute YOUR_NETBOX_API_TOKEN with your actual token +# Bearer Token Format (v2): nbt_. +curl -H "Authorization: Bearer YOUR_NETBOX_API_TOKEN" \ + "http://netbox.example.com:8000/api/dcim/devices/?export=gNMIc%20Device%20Export" +``` + +The response should be a JSON array of targets ready for the gNMIc Operator. + +Sample JSON output produced by the basic export template: + +```json +[ + + { + "name": "edge-rtr-01.dc1.example.com", + "address": "203.0.113.1", + "labels": { + "site": "DC1", + "role": "edge", + "region": "eu-central-1", + "type": "router" + } + }, + +] +``` + +> Ensure the response is valid JSON and contains no hidden or invalid characters, otherwise the gNMIc Operator will fail to parse it. + +> If you instead return a JSON object with a nested array, add a mapping section such as `targetsField: "self.targets"` to the TargetSource CR. + +--- + +## Step 3: Create a TargetProfile + +Define how discovered targets should be configured. The `TargetProfile` points to a [Kubernetes Secret](https://kubernetes.io/docs/concepts/configuration/secret/) containing device credentials, such as username/password or client certificates. + +Create a credentials Secret first, then reference it from the TargetProfile. + +```yaml +# Replace YOUR_DEVICE_USERNAME and YOUR_DEVICE_PASSWORD with your corresponding default device username and password +apiVersion: v1 +kind: Secret +metadata: + name: device-credentials + namespace: gnmic-system +type: Opaque +stringData: + username: YOUR_DEVICE_USERNAME + password: YOUR_DEVICE_PASSWORD +``` + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetProfile +metadata: + name: netbox-device + namespace: gnmic-system +spec: + credentialsRef: device-credentials + timeout: 10s +``` + +For more TargetProfile options and credential handling, see the operator documentation for `TargetProfile`. + +--- + +## Step 4: Create a TargetSource Using Export Template + +Create a `TargetSource` that references your NetBox export template endpoint: + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: netbox-export-source + namespace: gnmic-system +spec: + targetPort: 57400 + targetProfile: netbox-device + targetLabels: + inventory: netbox + sync-source: export-template + provider: + http: + url: "http://netbox.example.com:8000/api/dcim/devices/?export=gNMIc%20Device%20Export" + method: GET + interval: 30m + timeout: 30s + authentication: + token: + scheme: Token + tokenSecretRef: + name: netbox-api-token + key: token +``` + +--- + +## Step 5: Verify Target Discovery + +Once the `TargetSource` is deployed, verify that targets are being discovered: + +```bash +# List discovered targets +kubectl get targets -n gnmic-system + +# Check TargetSource status and sync details +kubectl describe targetsource netbox-export-source -n gnmic-system +``` + +Successful sync shows: + +- `status.status`: "success" (or similar) +- `status.targetsCount`: number of devices +- `status.lastSync`: recent timestamp + +--- + +## Example: Complete Setup + +Here's a full example combining all components: + +```yaml +--- +# Secret for NetBox API token +apiVersion: v1 +kind: Secret +metadata: + name: netbox-api-token + namespace: gnmic-system +type: Opaque +data: + # base64-encoded token (echo -n "YOUR_TOKEN" | base64) + token: YOUR_BASE64_ENCODED_TOKEN + +--- +# Secret for Target Credential +apiVersion: v1 +kind: Secret +metadata: + name: device-credentials + namespace: gnmic-system +type: Opaque +stringData: + username: YOUR_DEVICE_USERNAME + password: YOUR_DEVICE_PASSWORD + +--- +# TargetProfile +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetProfile +metadata: + name: netbox-device + namespace: gnmic-system +spec: + credentialsRef: device-credentials + timeout: 10s + + +--- +# TargetSource using Export Template +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: netbox-export-source + namespace: gnmic-system +spec: + targetPort: 57400 + targetProfile: netbox-device + targetLabels: + inventory: netbox + sync-source: export-template + provider: + http: + url: "http://netbox.example.com:8000/api/dcim/devices/?export=gNMIc%20Device%20Export" + method: GET + interval: 30m + timeout: 30s + authentication: + token: + scheme: Token + tokenSecretRef: + name: netbox-api-token + key: token +``` + +--- + +## Advantages of Export Templates + +- **Powerful Filtering**: Filter devices by site, status, role, tags, etc. directly in NetBox +- **Reduced Operator Load**: NetBox handles data transformation; operator just fetches JSON +- **Reusability**: One template can serve multiple consumers +- **Maintainability**: Update discovery logic in NetBox without changing Kubernetes manifests +- **Performance**: Avoids REST API pagination for large inventories + +--- + +## Limitations & Considerations + +### 1. Reverse Proxy and URL Path Rewriting + +If NetBox is behind a reverse proxy with URL path rewriting: + +- **Issue**: The export template endpoint uses query parameters that may not survive proxy transformation. +- **Solution**: + - Ensure the proxy preserves query strings exactly. + - Test the export URL directly: + ```bash + curl -H "Authorization: Token YOUR_TOKEN" \ + "http://netbox.example.com:8000/api/dcim/devices/?export=gNMIc%20Device%20Export" + ``` + - If the proxy blocks or modifies parameters, consider using a direct NetBox endpoint without proxying. + +### 2. Large Inventory Rendering + +- Very large device counts can cause NetBox to take time rendering the template. +- **Solution**: + - Use `.filter()` in your template to limit results. + - Create separate export templates for different device groups (e.g., by site or role). + +### 3. Complex Jinja2 Logic + +- NetBox's Jinja2 sandbox restricts some Python functions for security. +- **Solution**: Keep templates simple and use NetBox's built-in filters and objects. Test the URL with curl or similar before deploying. + +--- + +## Template Troubleshooting + +### Missing Targets in Kubernetes + +- **Check**: Are all required fields populated in NetBox? (e.g., `primary_ip4` may be `None` if not set) +- **Solution**: Add conditional checks: + ```jinja2 + {% if device.primary_ip4 %} + "address": "{{ device.primary_ip4.address.ip }}" + {% endif %} + ``` + +### Authorization Fails + +If you get a 403 error: + +- Verify the token is valid and not expired. +- Ensure the API token is enabled. + +--- diff --git a/docs/content/docs/examples/NetBox/REST API/_index.md b/docs/content/docs/examples/NetBox/REST API/_index.md new file mode 100644 index 00000000..fcbd990a --- /dev/null +++ b/docs/content/docs/examples/NetBox/REST API/_index.md @@ -0,0 +1,319 @@ +--- +title: "Pull with REST API" +linkTitle: "Pull with REST API" +weight: 2 +description: > + Discover targets from NetBox using the HTTP provider and NetBox REST API +--- + +This guide shows how to configure the HTTP provider to discover targets from NetBox using its REST API. + +The REST API approach is direct and straightforward — query NetBox's standard API endpoints to retrieve devices that match your criteria. + +## Prerequisites + +- A running Kubernetes cluster with gNMIc Operator installed +- `kubectl` access to your cluster +- A reachable NetBox instance (inside or outside the cluster) +- A NetBox API token + +## Overview + +The HTTP `TargetSource` loader performs these steps: + +1. **Fetch** JSON device data from a NetBox REST API endpoint (`/api/dcim/devices/`) +2. **Transform** each device record into a gNMIc target using CEL expressions +3. **Create** or **update** `Target` resources in Kubernetes with the extracted data + +--- + +## Step 1: Create a NetBox API Token and Store It Securely + +### Step 1a: Create the API Token in NetBox + +Create a dedicated API token in NetBox for gNMIc Operator access. + +1. Log in to NetBox. +2. Open your user profile or go to **User > API Tokens**. +3. Click **Add** or **Add token**. +4. Enter a descriptive name such as `gNMIc Operator`. +5. Grant the minimum permissions required for read-only device discovery. +6. Copy the token value and store it safely; NetBox will not show it again. + +### Step 1b: Store the Token in a Kubernetes Secret + +Create a [Kubernetes Secret](https://kubernetes.io/docs/concepts/configuration/secret/) containing the token so it is not embedded in manifests. + +```bash +# Substitute YOUR_NETBOX_API_TOKEN with your actual token +# Bearer Token Format (v2): nbt_. +kubectl create secret generic netbox-api-token \ + --from-literal=token=YOUR_NETBOX_API_TOKEN \ + -n gnmic-system +``` + +Verify the Secret was created: + +```bash +kubectl get secret netbox-api-token -n gnmic-system -o yaml +``` + +--- + +## Step 2: Create a TargetProfile + +Define how discovered targets should be configured. The `TargetProfile` points to a [Kubernetes Secret](https://kubernetes.io/docs/concepts/configuration/secret/) containing device credentials, such as username/password or client certificates. + +Create a credentials Secret first, then reference it from the profile. + +```yaml +# Replace YOUR_DEVICE_USERNAME and YOUR_DEVICE_PASSWORD with your corresponding default device username and password +apiVersion: v1 +kind: Secret +metadata: + name: device-credentials + namespace: gnmic-system +type: Opaque +stringData: + username: YOUR_DEVICE_USERNAME + password: YOUR_DEVICE_PASSWORD +``` + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetProfile +metadata: + name: netbox-device + namespace: gnmic-system +spec: + credentialsRef: device-credentials + timeout: 10s +``` + +For more TargetProfile options and credential handling, see the operator documentation for `TargetProfile`. + +--- + +## Step 3: Create a TargetSource Using REST API + +The following `TargetSource` queries NetBox's REST API to discover devices: + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: netbox-rest-source + namespace: gnmic-system +spec: + targetPort: 57400 + targetProfile: netbox-device + targetLabels: + inventory: netbox + sync-source: rest-api + provider: + http: + url: "http://netbox.example.com:8000/api/dcim/devices/?limit=1000" + method: GET + interval: 5m + timeout: 30s + authentication: + token: + scheme: Bearer + tokenSecretRef: + name: netbox-api-token + key: token + pagination: + nextField: "next" + mapping: + targetsField: "self.results" + address: "item.primary_ip4 != null ? item.primary_ip4.address.split('/')[0] : ''" + labels: | + { + "site": item.site.name, + "role": item.device_role.name, + "model": item.device_type.model, + "status": item.status.value + } +``` + +> This mapping only works for devices that have a primary IPv4 address set in NetBox. If primary_ip4 is missing, the expression returns '', so those devices will not yield a valid target address. For NetBox API details, see the [NetBox REST API](https://netboxlabs.com/docs/netbox/integrations/rest-api/) documentation. + +The HTTP loader supports `targetsField` and individual CEL expressions for `name`, `address`, `port`, `labels`, and `targetProfile`. See the HTTP Provider docs "Response Mapping via CEL" section for more details: [HTTP provider docs](../../user-guide/targetsource/providers/http.md) + +Use `self` for the full response and `item` for each candidate object. + +--- + +## Step 4: Apply and Verify Target Discovery + +Deploy the `TargetSource` and check that targets are being discovered and synced: + +```bash +# List discovered targets +kubectl apply -f /path/to/targetsource.yaml -n gnmic-system + +# List discovered targets +kubectl get targets -n gnmic-system + +# Check TargetSource status +kubectl describe targetsource netbox-rest-source -n gnmic-system +``` + +Look for: +- `status.status`: "success" (or similar) +- `status.targetsCount`: number of discovered devices +- `status.lastSync`: recent timestamp + +--- + +## Example: Complete Setup + +Here's a complete example combining all resources: + +```yaml +--- +# Secret for NetBox API token +apiVersion: v1 +kind: Secret +metadata: + name: netbox-api-token + namespace: gnmic-system +type: Opaque +data: + # base64-encoded token (echo -n "YOUR_TOKEN" | base64) + token: YOUR_BASE64_ENCODED_TOKEN + +--- +# Secret for Target Credential +apiVersion: v1 +kind: Secret +metadata: + name: device-credentials + namespace: gnmic-system +type: Opaque +stringData: + username: YOUR_DEVICE_USERNAME + password: YOUR_DEVICE_PASSWORD + +--- +# TargetProfile +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetProfile +metadata: + name: netbox-device + namespace: gnmic-system +spec: + credentialsRef: device-credentials + timeout: 10s + +--- +# TargetSource with REST API +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: netbox-rest-source + namespace: gnmic-system +spec: + targetPort: 57400 + targetProfile: netbox-device + targetLabels: + inventory: netbox + sync-source: rest-api + provider: + http: + url: "http://netbox.example.com:8000/api/dcim/devices/?limit=1000" + method: GET + interval: 5m + timeout: 30s + authentication: + token: + scheme: Bearer + tokenSecretRef: + name: netbox-api-token + key: token + pagination: + nextField: "next" + mapping: + targetsField: "self.results" + address: "item.primary_ip4 != null ? item.primary_ip4.address.split('/')[0] : ''" + labels: | + { + "site": item.site.name, + "role": item.device_role.name, + "model": item.device_type.model, + "status": item.status.value + } +``` + +--- + +## Performance Considerations & Limitations + +### REST API Query Limits + +- **Query Size**: The example uses `limit=1000`. Adjust based on your NetBox instance's pagination settings and response size limits. +- **Response Timeout**: Large device lists can take time. Set appropriate timeouts in your `TargetSource`. + +### Reverse Proxy Considerations + +If NetBox is behind a reverse proxy: + +- **Base URL**: Ensure the reverse proxy correctly handles the `/api/dcim/devices/` path. +- **Authentication**: Some proxies may require additional headers; verify with your proxy and NetBox admin. +- **HTTPS**: If using HTTPS, ensure certificates are trusted by the operator or else use the `tls` setting. + +### Large Inventories + +For inventories with thousands of devices: + +- Consider using **Export Templates** (see [NetBox Export Templates]({{< relref "../Export Template" >}})) for better filtering and performance. +- Implement filtering in the REST API URL (e.g., `?site=us-west&status=active`). + +--- + +## Security Considerations + +### Token and Credentials + +- **Never** embed plaintext tokens or credentials in manifests or YAML files. +- Always store tokens in Kubernetes Secrets. +- Restrict RBAC permissions on the Secret to only necessary service accounts. + +### HTTPS and Certificates + +If connecting to NetBox via HTTPS: + +- Ensure cluster DNS resolves the hostname correctly. +- Mount CA certificates if using self-signed certificates. +- Verify the operator's HTTP client configuration for certificate validation. + +--- + +## Troubleshooting + +### Show TargetSource Errors + +```bash +kubectl describe targetsource netbox-rest-source -n gnmic-system +``` + +### Targets Not Appearing + +- Check that the `TargetProfile` exists and is correctly referenced. +- Verify labels and addresses are being extracted correctly from the NetBox response. +- Review operator logs for parsing errors: + ```bash + kubectl logs -l app=gnmic-operator -n gnmic-operator-system + ``` + +### Rate Limiting or Timeouts + +Increase the sync interval in your `TargetSource` or adjust timeouts: + +```yaml +spec: + provider: + http: + interval: 1h + timeout: 1m +``` diff --git a/docs/content/docs/examples/NetBox/_index.md b/docs/content/docs/examples/NetBox/_index.md new file mode 100644 index 00000000..4885d60e --- /dev/null +++ b/docs/content/docs/examples/NetBox/_index.md @@ -0,0 +1,8 @@ +--- +title: "NetBox" +linkTitle: "NetBox" +weight: 6 +# draft: true +description: > + Discover targets from NetBox using the HTTP provider +--- diff --git a/docs/content/docs/examples/NetBox/webhook/_index.md b/docs/content/docs/examples/NetBox/webhook/_index.md new file mode 100644 index 00000000..5cc0d316 --- /dev/null +++ b/docs/content/docs/examples/NetBox/webhook/_index.md @@ -0,0 +1,236 @@ +--- +title: "Push Mode with Webhook" +linkTitle: "Push Mode with Webhook" +weight: 2 +description: > + Configure a webhook in NetBox to update targets in the gNMIc Operator in real time. +--- + +## Netbox Webhook Configuration + +This example walks through configuring a webhook in NetBox to push real-time target updates to the gNMIc Operator. It covers the configuration in the gNMIc Operator (Step 1-3), and the configuration within Netbox (step 4). + +1. Create Targetprofile +2. Create Kubernetes Secrets +3. Apply TargetSource +4. Netbox setup + a: Configure Webhook + b: Create Event Rule +5. Verification + +At the end, the logs will show the incoming POST requests and the targets updates can be verified with `kubectl get targets`. + +## Prerequisites + +- Kubernetes cluster with gNMIc Operator installed +- `kubectl` access to your cluster +- Running NetBox instance +- Network connectivity from NetBox to the gNMIc Operator API endpoint + +--- + +### 1. Create TargetProfile + +Define how discovered targets should be configured. The `TargetProfile` contains device credentials, such as username/password or client certificates. These are either defined inline strings or stored in a [Kubernetes Secret](https://kubernetes.io/docs/concepts/configuration/secret/). + +```yaml +# Replace YOUR_DEVICE_USERNAME and YOUR_DEVICE_PASSWORD with your corresponding default device username and password +apiVersion: v1 +kind: Secret +metadata: + name: device-credentials + namespace: gnmic-system +type: Opaque +stringData: + username: YOUR_DEVICE_USERNAME + password: YOUR_DEVICE_PASSWORD +``` + +When using a secret, create a credentials Secret first, then reference it from the profile. + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetProfile +metadata: + name: netbox-device + namespace: gnmic-system +spec: + credentialsRef: device-credentials + timeout: 10s +``` + +For more TargetProfile options and credential handling, see the operator documentation for `TargetProfile`. + +--- + +### 2. Create Kubernetes Secrets + +Bearer authentication and signature verification both require Kubernetes secrets. Ensure that the secrets: + +- Are created in the same namespace as the TargetSource (`gnmic-system` in this example). +- Use `name` and `key` values that match the TargetSource spec. + +```bash +kubectl create secret generic gnmic-api-auth --from-literal=bearer-token=YOUR_SECRET_TOKEN -n gnmic-system +kubectl create secret generic gnmic-signature --from-literal=signature=YOUR_SECRET_SIGNATURE -n gnmic-system +``` + +--- + +### 3. Apply TargetSource + +The TargetSource has the following settings configured: + +- `spec.provider.http.push.enabled` must be set to `true`, otherwise updates are rejected. +- Bearer authentication and signature verification are enabled, referencing to the secrets created in step 2. + +```yaml +# netbox.yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: netbox + namespace: gnmic-system +spec: + targetPort: 57400 + targetProfile: netbox-device + targetLabels: + inventory: netbox + sync-source: rest-api + provider: + http: + push: + enabled: true + auth: + bearer: + tokenSecretRef: + name: gnmic-api-auth + key: bearer-token + signature: + secretRef: + name: gnmic-signature + key: signature +``` + +> Namespace is `gnmic-system`, the name of the TargetSource is `netbox`. These values will be in the URL in step 4. + +--- + +### 4. Netbox Setup + +Next, configure a webhook in NetBox. The webhook is triggered by device events (for example, updates) and sends an HTTP POST request to the gNMIc Operator. + +#### Configure Webhook + +In NetBox, go to `Operations > Webhooks` and create a webhook with the following settings: + +- *Name*: gNMIc Operator push +- *URL*: `http://gnmic-controller-manager-api.gnmic-system.svc.cluster.local:8082/api/v1/gnmic-system/target-source/netbox/applyTargets` + - URL contains the namespace `gnmic-system` and TargetSource name `netbox`. See section address in [Push Mode](/docs/user-guide/targetsource/push/) for more details on URL construction. + - `gnmic-controller-manager-api.gnmic-system.svc.cluster.local` is only reachable if Netbox is inside the cluster. + - The address may instead be `http://localhost:8082/` or `http://servername:8082/`. +- *HTTP method*: POST +- *HTTP content type*: application/json +- *Additional headers:* `Authorization: Bearer YOUR_SECRET_TOKEN` +- *Body Template*: + + ```json + [ + { + "name": "{{ data.name }}", + "address": "{{ data.primary_ip4.address.split('/')[0] }}", + "operation": "{{ event }}", + "targetProfile": "{{ data.custom_fields.target_profile | default('', true) }}", + "port": {{ data.custom_fields.gnmic_port | default(57400, true) }}, + "labels": [ + {"vendor":"{{ data.device_type.manufacturer.name }}"} + ] + } + ] + ``` + +- *Secret*: `YOUR_SECRET_SIGNATURE` +- *SSL Verification*: true + +#### Create Event Rule + +The webhook requires a trigger, configured as an event rule under `Operations > Event Rules`. + +- *Name*: gNMIc Operator push target change +- *Object types*: `DCIM > Device` +- *Event types*: `Object Created`, `Object Updated` and `Object Deleted` +- *Action type*: Webhook +- *Webhook*: gNMIc Operator push + +--- + +### 5. Verification + +Updating a device in NetBox should now trigger the webhook. Verify this with the following commands: + +```bash +kubectl get targets +kubectl get targets -o yaml + +# Check logs of incoming POST requests: +kubectl logs -n gnmic-system deploy/gnmic-controller-manager -f +``` + +Every incoming POST request is logged, including rejected requests. If no POST requests appear in the logs, the webhook request is not reaching the gNMIc Operator. + +--- + +## Example: Complete Setup + +Here's a complete example combining all resources: + + ```yaml +--- +# Secret for Target Credential +apiVersion: v1 +kind: Secret +metadata: + name: device-credentials + namespace: gnmic-system +type: Opaque +stringData: + username: YOUR_DEVICE_USERNAME + password: YOUR_DEVICE_PASSWORD + +--- +# TargetProfile +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetProfile +metadata: + name: netbox-device + namespace: gnmic-system +spec: + credentialsRef: device-credentials + timeout: 10s +--- +# Apply Targetsource +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: netbox + namespace: gnmic-system +spec: + targetPort: 57400 + targetProfile: netbox-device + targetLabels: + inventory: netbox + sync-source: rest-api + provider: + http: + push: + enabled: true + auth: + bearer: + tokenSecretRef: + name: gnmic-api-auth + key: bearer-token + signature: + secretRef: + name: gnmic-signature + key: signature +``` diff --git a/docs/content/docs/reference/api.md b/docs/content/docs/reference/api.md index 1cfe4fb0..d20cb3e5 100644 --- a/docs/content/docs/reference/api.md +++ b/docs/content/docs/reference/api.md @@ -153,30 +153,111 @@ description: > | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `http` | HTTPConfig | No | - | HTTP endpoint for target discovery | -| `consul` | ConsulConfig | No | - | Consul service discovery config | -| `configMap` | string | No | - | ConfigMap name containing targets | -| `podSelector` | LabelSelector | No | - | Select Kubernetes Pods as targets | -| `serviceSelector` | LabelSelector | No | - | Select Kubernetes Services as targets | -| `labels` | map[string]string | No | - | Labels to apply to discovered targets | +| `provider` | ProviderSpec | Yes | - | Provider-specific discovery configuration | +| `targetPort` | int32 | No | - | Default port used when the discovered target does not provide a port | +| `targetProfile` | string | Yes | - | Reference to `TargetProfile` applied to discovered targets | +| `targetLabels` | map[string]string | No | - | Labels added to all discovered targets | + +### ProviderSpec + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `http` | HTTPConfig | No | HTTP provider configuration | ### HTTPConfig +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `url` | string | No | - | HTTP endpoint used to pull targets. Required unless push is enabled | +| `method` | string | No | GET | HTTP request method | +| `headers` | map[string]string | No | - | HTTP headers to include in requests | +| `body` | string | No | - | Request body for POST requests | +| `authentication` | AuthenticationSpec | No | - | Authentication configuration for the HTTP endpoint | +| `interval` | duration | No | 6h | Polling interval used to refresh targets | +| `timeout` | duration | No | 10s | Timeout for HTTP requests | +| `tls` | ClientTLSConfig | No | - | Client TLS configuration for HTTPS endpoints | +| `pagination` | PaginationSpec | No | - | Pagination settings for parsing responses | +| `mapping` | ResponseMappingSpec | No | - | Response mapping configuration for JSON responses | +| `push` | PushSpec | No | - | Push-based update configuration | + +### ClientTLSConfig + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `insecureSkipVerify` | bool | No | false | Skip verification of the server certificate | +| `caBundleRef` | ConfigMapKeySelector | No | - | Reference to a ConfigMap containing a PEM CA bundle | + +### AuthenticationSpec + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `basic` | BasicAuthSpec | No | Basic authentication configuration | +| `token` | TokenAuthSpec | No | Token authentication configuration | + +### BasicAuthSpec + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `credentialsSecretRef` | SecretKeySelector | Yes | Reference to a Secret containing username/password keys | + +### TokenAuthSpec + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `scheme` | string | Yes | Token scheme, e.g. Bearer | +| `tokenSecretRef` | SecretKeySelector | Yes | Reference to a Secret containing the token | + +### PaginationSpec + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `nextField` | string | No | JSON field containing the next page reference or pagination token | + +### ResponseMappingSpec + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `targetsField` | string | No | CEL expression selecting the list of targets from the response | +| `name` | string | No | CEL expression for the target name | +| `address` | string | No | CEL expression for the target address | +| `port` | string | No | CEL expression for the target port | +| `labels` | string | No | CEL expression returning a map of labels | +| `targetProfile` | string | No | CEL expression for the target profile | + +### PushSpec + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `enabled` | bool | No | Enable push updates | +| `auth` | PushAuthSpec | No | Push authentication configuration | + +### PushAuthSpec + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `bearer` | PushBearerAuthSpec | No | Bearer token authentication configuration | +| `signature` | PushSignatureAuthSpec | No | Signature authentication configuration | + +### PushBearerAuthSpec + | Field | Type | Required | Description | |-------|------|----------|-------------| -| `url` | string | Yes | URL of the HTTP endpoint | +| `tokenSecretRef` | SecretKeySelector | Yes | Reference to a Secret containing the bearer token | -### ConsulConfig +### PushSignatureAuthSpec | Field | Type | Required | Description | |-------|------|----------|-------------| -| `url` | string | Yes | Consul server URL | +| `secretRef` | SecretKeySelector | Yes | Reference to a Secret used to verify request signatures | +| `header` | string | Yes | Header containing the signature | +| `algorithm` | string | No | Signature algorithm | ### TargetSourceStatus | Field | Type | Description | |-------|------|-------------| | `status` | string | Sync status (Synced, Error, Pending) | +| `observedGeneration` | int64 | Observed generation of the spec | | `targetsCount` | int32 | Number of discovered targets | | `lastSync` | Time | Last successful sync timestamp | diff --git a/docs/content/docs/user-guide/targetsource.md b/docs/content/docs/user-guide/targetsource.md deleted file mode 100644 index d1fee0c4..00000000 --- a/docs/content/docs/user-guide/targetsource.md +++ /dev/null @@ -1,242 +0,0 @@ ---- -title: "TargetSource" -linkTitle: "TargetSource" -weight: 4 -description: > - Dynamic target discovery from external sources ---- - -The `TargetSource` resource enables dynamic discovery of network devices from external sources. The operator automatically creates, updates, and deletes `Target` resources based on discovered devices. - -## Discovery Sources - -TargetSource supports multiple discovery backends: - -| Source | Description | -|--------|-------------| -| `http` | Fetch targets from an HTTP endpoint | -| `consul` | Discover targets from Consul service registry | -| `configMap` | Read targets from a Kubernetes ConfigMap | -| `podSelector` | Create targets from Kubernetes Pods | -| `serviceSelector` | Create targets from Kubernetes Services | - -## HTTP Discovery - -Discover targets from an HTTP endpoint that returns a JSON list of targets: - -```yaml -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: http-discovery -spec: - http: - url: http://inventory-service:8080/targets - labels: - source: inventory -``` - -The HTTP endpoint should return a JSON array of target objects. - -## Consul Discovery - -Discover targets from Consul service registry: - -```yaml -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: consul-discovery -spec: - consul: - url: http://consul:8500 - labels: - source: consul - datacenter: dc1 -``` - -## ConfigMap Discovery - -Read targets from a Kubernetes ConfigMap: - -```yaml -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: configmap-targets -spec: - configMap: network-devices - labels: - source: configmap -``` - -The ConfigMap should contain target definitions in a structured format. - -## Kubernetes Pod Discovery - -Create targets from Kubernetes Pods matching a label selector: - -```yaml -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: pod-discovery -spec: - podSelector: - matchLabels: - app: network-simulator - gnmi: enabled - labels: - source: kubernetes - type: simulator -``` - -This is useful for: -- Containerized network simulators -- Virtual network functions (VNFs) -- Development/testing environments - -## Kubernetes Service Discovery - -Create targets from Kubernetes Services matching a label selector: - -```yaml -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: service-discovery -spec: - serviceSelector: - matchLabels: - protocol: gnmi - labels: - source: kubernetes -``` - -## Label Inheritance - -Labels defined in the `TargetSource.spec.labels` field are applied to all discovered targets: - -```yaml -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: datacenter-a -spec: - consul: - url: http://consul-dc-a:8500 - labels: - datacenter: dc-a - environment: production - source: consul -``` - -All targets discovered from this source will have: -- `datacenter: dc-a` -- `environment: production` -- `source: consul` - -This enables using label selectors in Pipelines to select targets by their discovery source. - -## Status - -The TargetSource status shows discovery state: - -```yaml -status: - status: Synced - targetsCount: 42 - lastSync: "2024-01-15T10:30:00Z" -``` - -| Field | Description | -|-------|-------------| -| `status` | Current sync status (Synced, Error, Pending) | -| `targetsCount` | Number of targets discovered | -| `lastSync` | Timestamp of last successful sync | - -## Example: Multi-Source Discovery - -Combine multiple TargetSources for different environments: - -```yaml -# Production devices from Consul -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: prod-consul -spec: - consul: - url: http://consul-prod:8500 - labels: - environment: production - source: consul ---- -# Lab devices from ConfigMap -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: lab-devices -spec: - configMap: lab-network-devices - labels: - environment: lab - source: configmap ---- -# Simulator pods -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: simulators -spec: - podSelector: - matchLabels: - app: srlinux - labels: - environment: dev - source: kubernetes -``` - -Then use label selectors in your Pipeline: - -```yaml -apiVersion: operator.gnmic.dev/v1alpha1 -kind: Pipeline -metadata: - name: production-telemetry -spec: - clusterRef: prod-cluster - enabled: true - targetSelectors: - - matchLabels: - environment: production - # ... subscriptions, outputs -``` - -## Lifecycle - -### Target Creation - -When a TargetSource discovers a new device: -1. A new `Target` resource is created -2. Labels from `spec.labels` are applied -3. Owner reference is set to the TargetSource - -### Target Updates - -When a discovered device's properties change: -1. The corresponding `Target` is updated -2. Clusters using that target are reconciled - -### Target Deletion - -When a device is no longer discovered: -1. The `Target` resource is deleted -2. Clusters stop collecting from that target - -### TargetSource Deletion - -When a TargetSource is deleted: -1. All Targets owned by it are deleted (via owner references) -2. Clusters are reconciled to remove those targets - diff --git a/docs/content/docs/user-guide/targetsource/_index.md b/docs/content/docs/user-guide/targetsource/_index.md new file mode 100644 index 00000000..f9897a71 --- /dev/null +++ b/docs/content/docs/user-guide/targetsource/_index.md @@ -0,0 +1,214 @@ +--- +title: "TargetSource" +linkTitle: "TargetSource" +weight: 4 +description: > + Dynamic target discovery from external sources +--- + +The `TargetSource` resource enables dynamic discovery of network devices from external sources. The operator automatically creates, updates, and deletes `Target` resources based on discovered devices. + +## Basic Configuration + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: targetsource-1 +spec: + provider: + # Configure one of the supported providers + targetPort: 57400 + targetProfile: default + targetLabels: + source: inventory +``` + +The supported TargetSource providers are documented on the [TargetSource Provider](./providers/) page. + +## Spec Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `provider` | object | Yes | Provider-specific discovery configuration. Exactly one provider must be configured | +| `targetPort` | int32 | No | Default port used when the discovered target does not provide a port | +| `targetProfile` | string | No | Reference to default `TargetProfile` applied to all discovered targets if no profile was discovered | +| `targetLabels` | map[string]string | No | Labels added to all discovered targets | + + + + + + + +## Label Inheritance + +Each generated `Target` receives an ownership label identifying the originating `TargetSource`: +```yaml +operator.gnmic.dev/targetsource: targetsource-1 +``` + +This label is automatically managed by the operator and is used to: +- Identify targets owned by a specific `TargetSource` +- Determine which targets should be updated or deleted during reconciliation + +The `operator.gnmic.dev/targetsource` label is reserved and always takes precedence over any provider-supplied labels. + +### TargetSource Labels + +Additional labels can be applied to all generated targets using `spec.targetLabels`: + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: targetsource-1 +spec: + provider: + http: + url: http://targetsource-1:8080/targets + targetLabels: + datacenter: dc-a + environment: production +``` + +All targets discovered from this source will have: +- `datacenter: dc-a` +- `environment: production` + +This enables Pipelines to select targets using label selectors. + +### Labels from Discovery Providers + +Discovery providers may return additional labels for each target. These labels are applied directly to the generated `Target` resource. + +The `gnmic_operator_` label prefix is reserved for operator-specific behavior. Labels using this prefix are interpreted by the operator and are not applied directly to the generated `Target` resource. + +Supported operator labels: + +| Label | Description | +|--------|-------------| +| `gnmic_operator_target_profile` | Overrides the `TargetProfile` configured in the `TargetSource` | + +### Label Precedence + +If the same label key is defined in multiple places, labels are applied in the following order (highest precedence first): + +1. `TargetSource` ownership label (`operator.gnmic.dev/targetsource`) +2. Labels from `TargetSource.spec.targetLabels` +3. Labels returned by the discovery provider + +## Status + +The `TargetSource` status shows discovery state: + +```yaml +status: + status: Synced + observedGeneration: 1 + targetsCount: 42 + lastSync: "2024-01-15T10:30:00Z" +``` + +| Field | Description | +|-------|-------------| +| `status` | Current sync status (Synced, Error, Pending) | +| `observedGeneration` | Generation of the spec last processed by the controller | +| `targetsCount` | Number of targets discovered | +| `lastSync` | Timestamp of last successful sync | + +## Lifecycle + +### Target Creation + +When a `TargetSource` discovers a new device: + +1. A new `Target` resource is created +2. The `TargetProfile` referenced in `spec.targetProfile` is assigned +3. Labels from `spec.targetLabels` are applied +4. The `TargetSource` is set as the owner reference + +### Target Updates + +On each discovery cycle, existing `Target` resources are reconciled with the latest discovered state: + +1. The corresponding `Target` resource is updated and overwritten +2. Clusters consuming the target are reconciled automatically + +> Manual changes to `Target` resources managed by a `TargetSource` are overwritten on every reconciliation cycle. + +### Target Deletion + +When a device is no longer returned by the discovery provider: + +1. The corresponding `Target` resource is deleted +2. Clusters automatically stop using the target + +### TargetSource Deletion + +When a `TargetSource` is deleted: + +1. All `Target` resources owned by it are deleted via owner references +2. Clusters are reconciled and remove the deleted targets + diff --git a/docs/content/docs/user-guide/targetsource/providers/_index.md b/docs/content/docs/user-guide/targetsource/providers/_index.md new file mode 100644 index 00000000..29284d7f --- /dev/null +++ b/docs/content/docs/user-guide/targetsource/providers/_index.md @@ -0,0 +1,7 @@ +--- +title: "TargetSource Provider" +linkTitle: "TargetSourceProvider" +weight: 1 +description: > + Configuring TargetSource discovery providers +--- diff --git a/docs/content/docs/user-guide/targetsource/providers/http.md b/docs/content/docs/user-guide/targetsource/providers/http.md new file mode 100644 index 00000000..64b64262 --- /dev/null +++ b/docs/content/docs/user-guide/targetsource/providers/http.md @@ -0,0 +1,629 @@ +--- +title: "HTTP Provider" +linkTitle: "HTTP" +weight: 2 +description: > + The HTTP provider discovers targets from an HTTP endpoint returning JSON, or receives webhook-based updates when push mode is enabled. +--- + +## Basic Configuration + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: targetsource-1 +spec: + provider: + http: + url: http://inventory-service:8080/targets + authentication: + token: + scheme: Bearer + tokenSecretRef: + name: inventory-token + key: token + # Enable push mode + push: + enabled: true + targetPort: 57400 + targetProfile: default + targetLabels: + source: inventory +``` + +## HTTP Spec Fields + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `url` | string | No | - | HTTP endpoint used to pull targets. Required unless `push.enabled` is enabled | +| `method` | string | No | GET | HTTP method used for requests | +| `headers` | map[string]string | No | - | HTTP headers to include in requests | +| `body` | string | No | - | Request body for POST requests | +| `authentication` | object | No | - | Authentication configuration for the HTTP endpoint | +| `interval` | duration | No | 30m | Polling interval used to refresh targets | +| `timeout` | duration | No | 30s | Timeout for HTTP requests | +| `tls` | object | No | - | Client TLS configuration for HTTPS endpoints | +| `pagination` | object | No | - | Pagination configuration for parsing HTTP responses | +| `mapping` | object | No | - | Response mapping configuration for JSON responses | +| `push` | object | No | - | Push-based update configuration | + +## Pull Mode + +The HTTP provider supports pull-based target discovery by periodically querying a remote HTTP endpoint that returns target data in JSON format. + +```yaml +spec: + provider: + http: + url: http://inventory-service:8080/targets +``` + +In pull mode, the operator sends HTTP requests to the configured url at a fixed interval and updates targets based on the response. The `push.enabled` field is optional when pull mode is enabled, but can still be used for accepting incoming webhook notifications. + +*How Pull Mode Works* +1. The operator sends an HTTP request to the configured url +2. The response is parsed (either directly or via mapping) +3. Targets are created, updated, or removed based on the returned data +4. This process repeats according to the configured interval + + +### Authentication + +The HTTP provider supports authenticated requests to the inventory endpoint. + +Exactly one authentication method can be configured. + +#### Basic Authentication + +Credentials are referenced from a [Kubernetes Secret](https://kubernetes.io/docs/concepts/configuration/secret/). + +```yaml +spec: + provider: + http: + url: https://inventory.example.com/targets + authentication: + basic: + credentialSecretRef: + name: inventory-credentials + key: username +``` + +#### Token Authentication + +Token authentication is configured using a [Kubernetes Secret](https://kubernetes.io/docs/concepts/configuration/secret/) reference. + +```yaml +spec: + provider: + http: + url: https://inventory.example.com/targets + authentication: + token: + scheme: Bearer + tokenSecretRef: + name: inventory-token + key: token +``` + +### TLS + +TLS settings can be configured for HTTPS endpoints. + +```yaml +spec: + provider: + http: + url: https://inventory.example.com/targets + tls: + insecureSkipVerify: false + caBundleRef: + name: inventory-ca + key: ca.crt +``` + +#### TLS Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `insecureSkipVerify` | bool | No | Skip verification of the server certificate. Defaults to `false` | +| `caBundleRef` | object | No | Reference to a [Kubernetes ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) containing a PEM-encoded CA bundle | + +### Pagination + +Pagination enables the operator to retrieve complete result sets from APIs that return data in multiple pages. The operator automatically follows pagination until no further pages are available. + +```yaml +spec: + provider: + http: + url: https://inventory.example.com/devices + pagination: + nextField: "self.next" +``` + +#### Pagination Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `nextField` | string | No | CEL expression used to extract the next page reference from the response | +| `requestParam` | string | No | Query parameter used when the extracted value is a token | + +The `nextField` value may either contain: +- A full URL for the next request +- A pagination token appended as a query parameter to the original URL + +#### How Pagination Works + +The operator handles the following pagination patterns: + +##### 1. Link Header Pagination +If the API provides a Link response header with `rel="next"`, the operator will automatically follow it. + +Example response header: +``` +Link: ; rel="next" +``` + +Behavior: +``` +Request 1: GET /devices?page=1 +Request 2: GET /devices?page=2 +Request 3: GET /devices?page=3 +... +``` + +##### 2. URL-Based Pagination +If the response contains a full URL in the body (e.g. `"next": "https://..."`), it will be used directly. + +Example response: +```json +{ + "devices": [...], + "next": "https://inventory.example.com/devices?offset=50" +} +``` + +##### 3. Token-Based Pagination +If the response contains a pagination token, the operator appends it as a query parameter. + +Example: +```yaml +pagination: + nextField: "self.next_token" + requestParam: "page_token" +``` + +Example: +``` +GET /devices +-> "next_token": "abc123" +GET /devices?page_token=abc123 +``` + +##### CEL-Based Extraction +The nextField is evaluated as a CEL expression using: +- `self` -> entire JSON response + +Example: +```yaml +pagination: + nextField: "self['@odata.nextLink']" +``` + +This allows extracting values from nested or special keys. + +### Response Processing + +The HTTP provider supports two response processing modes: + +- **Default response format**: The endpoint returns a JSON array of target objects. +- **Response mapping**: Custom JSON structures are mapped to target fields using CEL expressions. + +If `mapping` is configured, the custom mapping rules are used. Otherwise, the response itself must be a JSON array. + +#### Default Response Format + +If `mapping` is not configured, the endpoint must return a JSON array of objects with the following structure: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Name of the generated `Target` resource | +| `address` | string | Yes | Device address (FQDN or IP address) | +| `port` | int32 | No | Port used for gNMI connections. If omitted, `spec.targetPort` is used | +| `labels` | map[string]string | No | Labels added to the generated `Target` resource | +| `targetProfile` | string | No | Reference to a `TargetProfile`. If omitted, `spec.targetProfile` is used | + +Example response: + +```json +[ + { + "name": "spine1", + "address": "spine1.local", + "port": 57400, + "labels": { + "role": "spine" + }, + "targetProfile": "spine-profile" + }, + { + "name": "leaf1", + "address": "leaf1.local", + "port": 57400, + "labels": { + "role": "leaf" + } + }, + { + "name": "leaf2", + "address": "leaf2.local", + "port": 57400, + "labels": { + "role": "leaf" + } + } +] +``` + +#### Response Mapping via CEL + +When your inventory API's JSON structure differs from the default format, use CEL (Common Expression Language) mapping to extract target fields. + +```yaml +spec: + provider: + http: + url: https://inventory.example.com/devices + mapping: + targetsField: "self.results" + name: "item.hostname" + address: "item.management.ip" + port: "item.gnmi.port" + targetProfile: "item.profile" + labels: "{'role': item.metadata.role, 'site': item.metadata.site}" +``` + +##### Understanding `targetsField` + +The `targetsField` expression tells the operator where to find the list of target objects in your API response. It's particularly important when your API wraps the target list in a data structure. + +**When to use `targetsField`:** +- Your API returns `{"results": [...]}` -> use `"self.results"` +- Your API returns `{"data": {"devices": [...]}}` -> use `"self.data.devices"` +- Your API returns a plain array `[...]` -> omit `targetsField` (default behavior) + +**Example scenarios:** + +*Custom API response example 1:* +```json +{ + "count": 42, + "next": "https://...", + "results": [ + {"id": 1, "name": "device1", "primary_ip": "10.0.0.1"}, + {"id": 2, "name": "device2", "primary_ip": "10.0.0.2"} + ] +} +``` +Usage: `targetsField: "self.results"` + +*Custom API response example 2:* +```json +{ + "status": "success", + "data": { + "timestamp": "2024-01-01T00:00:00Z", + "devices": [ + {"name": "router1", "mgmt_ip": "192.168.1.1"}, + {"name": "router2", "mgmt_ip": "192.168.1.2"} + ] + } +} +``` +Usage: `targetsField: "self.data.devices"` + +##### Mapping Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `targetsField` | string | No | CEL expression selecting the target list from the response. If omitted, assumes response is a direct JSON array | +| `name` | string | No | CEL expression for the target name | +| `address` | string | No | CEL expression for the target address | +| `port` | string | No | CEL expression for the target port | +| `labels` | string | No | CEL expression returning a map of labels | +| `targetProfile` | string | No | CEL expression for the target profile | + +##### CEL Variables + +The mapping expressions support the following variables: +- `item`: the current target object being processed +- `self`: the complete unprocessed response from the HTTP endpoint + +#### Performance: CEL vs Direct Mapping + +Understanding the performance implications helps optimize your configurations: + +**Direct Mapping (No CEL)** - *Fastest* +- Used when your API response matches the default structure exactly +- No expression compilation or evaluation overhead +- Suitable for high-frequency polling (e.g., every minute) +- Example: API returns `[{"name": "...", "address": "..."}]` + +**CEL Mapping** - *Slight overhead* +- CEL expressions are compiled once at startup (not per request) +- Evaluation is performed per target object during each poll cycle +- At high scale (10,000+ targets), consider the `interval` between polls + +**Best practices:** +- Use direct mapping if your API already returns the correct structure +- For large result sets, increase the interval +- Combine CEL and direct mapping for efficiency (see hybrid mapping below) +- Use CEL extensions (see reference table below) to reduce complexity and improve readability + +#### CEL Extensions + +The operator includes a set of standard CEL extensions from the official [CEL Go library](https://github.com/google/cel-go) to enable more advanced expressions. + +These [extensions](https://pkg.go.dev/github.com/google/cel-go/ext) expand CEL with additional capabilities commonly needed when transforming API responses: + +| Extension | Purpose | +|----------|----------| +| **Strings** | String manipulation such as splitting values, case conversion, and extracting parts of text (e.g. parsing hostnames or IPs) | +| **Math** | Numeric operations and comparisons (e.g. calculations, min/max, type conversions) | +| **Lists** | Working with arrays (e.g. indexing, filtering, joining values) | +| **Sets** | Set-style operations such as membership checks and comparisons | +| **Regex** | Pattern matching and validation using regular expressions | +| **Bindings** | Defining intermediate variables to simplify complex expressions | + +**Examples:** + +```yaml +mapping: + # Extract site from hostname + labels: | + { + 'site': item.name.split('-')[0] + } + + # Conditional profile + targetProfile: "item.type == 'edge' ? 'edge' : 'core'" + + # Pattern-based classification + labels: | + { + 'role': item.name.matches('^spine') ? 'spine' : 'leaf' + } +``` + +#### Combining CEL and Direct Mapping (Hybrid Approach) + +You don't need to map all fields with CEL. The operator supports mixing CEL expressions and direct field lookups for maximum efficiency: + +| Scenario | Behavior | Use Case | +|----------|----------|----------| +| `name`, `address` use CEL; others omitted | Extracts mapped fields via CEL; looks for `port`, `labels`, `targetProfile` directly in item JSON | Simple API where only some fields need transformation | +| Only `labels` uses CEL | Other fields use direct mapping; labels constructed from CEL expression | API returns correct `name`, `address`, `port` but custom labels need extraction | +| Only `address` uses CEL | Direct mapping for other fields; only address requires transformation | Most fields match API exactly except address requires CIDR parsing or format conversion | +| All fields use CEL | Complete transformation via expressions | API structure completely different from expected format | + +This hybrid approach optimizes performance by only compiling and evaluating CEL where needed. + +**Example - Partial CEL mapping (only transform what needs transforming):** +```yaml +mapping: + # Use CEL only when you need to transform a field + name: "item.hostname" + address: "item.primary_ip4 != null ? item.primary_ip4.split('/')[0] : item.primary_ip6.split('/')[0]" # CEL: parse CIDR + + # Fields that already exist should be omitted + # Port already exists as "port" field in item + # port: item.port <- omit this + + # Use CEL for structured or derived values + labels: | + { + "site": item.site.name, + "role": item.device_role.name + } + + # targetProfile can also be omitted if already present or not needed +``` + +In this example, only `address` and `labels` use CEL expressions; `name`, `port`, and `targetProfile` use direct field lookups for efficiency. + +#### Using YAML `|` for Complex CEL Expressions + +When writing more complex CEL expressions, it is recommended to use YAML’s pipe (`|`) literal block instead of inline strings. + +This is especially useful for expressions that span multiple lines or contain nested logic. + +**Labels example:** + +```yaml +mapping: + labels: | + { + "site": item.site.name, + "rack": item.rack != null ? item.rack.name : "", + "role": item.role != null ? item.role : "unknown", + "tags": item.tags.join(',') + } +``` + +**Why use `|` instead of quoted strings:** +- **Readability**: Multi-line expressions are easier to understand +- **Maintainability**: Complex CEL expressions don't require escaping +- **YAML best practice**: Literal blocks handle special characters naturally + +## Push Mode + +The HTTP provider supports webhook-based target updates via `spec.provider.http.push`. + +```yaml +spec: + provider: + http: + push: + enabled: true +``` + +When `push.enabled` is true, the operator accepts incoming webhook notifications and can update targets without polling a remote endpoint. The `url` field is optional when push mode is enabled, but can still be used for polling and fallback behavior. + +See [Push mode](/docs/user-guide/targetsource/push/) for more details. + +## Recommended Production Settings + +When deploying HTTP TargetSource providers in production networks, follow these guidelines to ensure reliable and efficient target discovery: + +### Polling Configuration +| Scenario | Setting | Rationale | +|----------|---------|-----------| +| **Small environment** (< 100 targets) | `interval: 5m` | Frequent updates without excessive load | +| **Medium environment** (100-500 targets) | `interval: 10m` | Balance between freshness and API load | +| **Large environment** (500-2000 targets) | `interval: 15m` | Reduce API polling overhead | +| **Very large environment** (2000+ targets) | `interval: 30m` | Minimize impact on inventory system | +| **High-frequency changes** | Use `push` mode with `interval` | Enables updates via push while periodic polling ensures completeness and consistency | + +**Timeout Configuration:** +```yaml +timeout: 30s # Allows for network latency +``` + +If timeouts consistently occur, increase `interval` instead of timeout (don't poll faster) + +### Authentication & Security + +**Always use TLS in production:** +```yaml +tls: + insecureSkipVerify: false # Never skip verification in production + caBundleRef: + name: inventory-ca-bundle + key: ca.crt +``` + +**For authenticated APIs:** +- Store credentials in [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) +- Rotate credentials periodically +- Use token-based auth when possible (simpler secret rotation) + +Example: +```yaml +authentication: + token: + scheme: Bearer + tokenSecretRef: + name: inventory-api-token + key: token +``` + +### Pagination & Large Result Sets + +**Configuration for APIs returning large result sets:** +```yaml +pagination: + nextField: next # Always configure pagination if your API supports it + +interval: 30m # Increase interval for large datasets (reduces cumulative API load) +timeout: 60s # Increase only if individual requests are slow or responses are large +``` + +Pagination splits large datasets into multiple smaller HTTP requests. This improves reliability and reduces the likelihood of timeouts compared to fetching a single large response. + +**Optimization strategies:** +- Request API filtering (if supported) to reduce result set size (e.g. ?limit=1000 or ?status=active) +- If the API does not support pagination or filtering increase the timeout +- Consider webhook push mode for frequently-changing inventories (if API supports it) + +### Mapping Optimization + +**Use hybrid CEL and direct mapping for performance:** +```yaml +# EFFICIENT - Only CEL-transform what needs it +mapping: + # + name: "item.hostname" # CEL expression + # port: (OMITTED) # Direct: exists as "port" in item + + # Only these need transformation -> use CEL + address: "item.primary_ip.split('/')[0]" # CEL: parse CIDR + labels: | # CEL: construct from nested fields + {'site': item.site.name} +``` + +**Avoid unnecessary CEL complexity:** +```yaml +# GOOD - Simple expressions +mapping: + address: "item.management_ip" + port: "int(item.gnmi_port)" + +# AVOID - Nested ternary logic (hard to debug) +mapping: + name: "item.has_override ? item.override_name : (item.hostname != '' ? item.hostname : 'default-' + string(item.id))" +``` + +**CEL expression best practices:** +- Compile expressions once at startup (not per request), so complexity is paid only once +- Use `ext.Bindings` for repeated expressions to avoid redundant evaluation +- Test CEL expressions thoroughly; they're compiled but errors only appear during evaluation +- Keep expressions under 200 characters for maintainability + +### Example Production Configuration + +```yaml +apiVersion: gnmic.openconfig.net/v1alpha1 +kind: TargetSource +metadata: + name: production-inventory +spec: + provider: + http: + # Security + url: https://inventory.prod.example.com/api/dcim/devices/?limit=100 + tls: + insecureSkipVerify: false + caBundleRef: + name: netbox-ca + key: ca.crt + + # Authentication + authentication: + token: + scheme: Bearer + tokenSecretRef: + name: api-token + key: token + + # Timing + interval: 15m # Balanced update frequency + timeout: 30s # Allow for network latency + + # Pagination + pagination: + nextField: next + + # Mapping for fields + mapping: + targetsField: "self.results" + #name: "item.name" -> already handled with fallback direct mapping + address: "item.primary_ip4 != null ? item.primary_ip4.split('/')[0] : item.primary_ip6.split('/')[0]" + port: "item.custom_fields.gnmi_port" + labels: "{\n 'site': item.site.name,\n 'role': item.device_role.name,\n 'status': item.status.value\n }" + targetProfile: "item.custom_fields.gnmi_profile" + + # Global settings + targetPort: 9339 + targetProfile: default-profile +``` + +This configuration ensures: + +- Secure HTTPS communication with certificate validation +- API authentication with token-based credentials +- Balanced polling interval for stable environments +- Proper pagination handling for large device inventories +- Rich label extraction from custom fields +- Fallback to defaults when fields are missing diff --git a/docs/content/docs/user-guide/targetsource/push.md b/docs/content/docs/user-guide/targetsource/push.md new file mode 100644 index 00000000..ef8d77d3 --- /dev/null +++ b/docs/content/docs/user-guide/targetsource/push.md @@ -0,0 +1,136 @@ +--- +title: "Push Mode" +linkTitle: "Push Mode" +weight: 4 +description: > + Enables REST API interface that accepts real-time target updates. +--- + +## Basic configuration + +This CR enables the push interface with no authentication. + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: targetsource-1 +spec: + provider: + http: # can be changed to a differnet TargetSourceProvider + push: + enabled: true +``` + +> `http` is currently the only TargetSourceProvider implemented, once others are added they can be used instead. Push mode is not coupled to a specific TargetSourceProvider implementation. + +--- + +## Spec Fields + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `push` | object | No | - | Push interface config | +| `enabled` | bool | Yes | False | Whether the push interface is active | +| `auth` | object | No | - | Bearer token authentication | +| `signature` | object | No | - | HTTP body verification using HMAC | +| `algorithm` | string | No | sha512 | Algorithm for signature verification(`sha256`or`sha512`) | + +--- + +## Address + +The REST API endpoint runs on `http://cluster-address:8082/api/v1/:namespace/target-source/:name/applyTargets`. + +- `cluster-address`: Address of your cluster. +- `:namespace`: Namespace the TargetSource is created in. +- `:name`: Name of the TargetSource. + +See [Push mode with webhook](/docs/examples/netbox/webhook) for an example on how to configure the URL. + +### Cluster Address + +The cluster address depends on where the API is accessed from. + +- Use `http://:8082/` when accessing the API from outside the cluster. +- Use `http://localhost:8082/` for local development (requires port-forwarding). +- Use `gnmic-controller-manager-api.gnmic-system.svc.cluster.local` when NetBox (or another source of truth) runs in the same cluster. +- If you use a reverse proxy, run `kubectl get service -n ` and use the returned service address and port in your proxy configuration. + +--- + +## REST API + +Refer to the [REST API documentation](/docs/advanced/rest-api-documentation/) for the expected request schema and payload format. Any system or script capable of sending HTTP POST requests can integrate with this interface. + +--- + +## Security + +The API supports Bearer Token authentication and X-Hook-Signature, both are optional and **turned off by default**. They are enabled by adding them to the specification. They can also be used in combination. + +An example configuration of both is documented in the [Netbox webhook](/docs/examples/netbox/webhook) example. + +--- + +### Bearer Authentication + +Bearer authentication compares a token stored in Kubernetes with the one sent in the HTTP header. The Kubernetes secret is referenced as `tokenSecretRef`. + +```yaml +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: targetsource-1 +spec: + provider: + http: + push: + enabled: true + auth: + bearer: + tokenSecretRef: + name: gnmic-api-auth # secret name + key: bearer-token # secret key +``` + +This requires the [creation](https://kubernetes.ltd/docs/reference/kubectl/generated/kubectl_create/kubectl_create_secret_generic/) of an Opaque Kuberentes secret: + +- Must be in the same namespace the gNMIc controller runs in. +- `name`: refers to the secret name +- `key`: key of the secret +- Example: `kubectl create secret generic gnmic-api-auth --from-literal=bearer-token=YOUR_SECRET_TOKEN` + +#### Authorization Header + +HTTP request must contain the Bearer token in the header in the format: + +```yaml +Authorization: Bearer YOUR_SECRET_TOKEN +``` + +--- + +### Signature + +Signature verification requires an Opaque Kubernetes secret that stores the shared key (see Bearer Authentication). For each request, the HMAC generated from the request body and shared key must be provided in the `X-Hook-Signature` header. + +```yaml +spec: + provider: + http: + push: + enabled: true + auth: + signature: + algorithm: sha512 + secretRef: + name: gnmic-signature + key: signature +``` + +--- + +#### Reverse Proxy + +In order to have a secure setup, the HTTP post requests must be sent using TLS. The REST API interface does not support HTTPS, at least not directly. It is recommended to terminate the TLS connection at the reverse proxy and forward a HTTP request to the gNMIc Operator. diff --git a/go.mod b/go.mod index 9dc2b789..62b3b47d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,10 @@ go 1.25.5 require ( github.com/cert-manager/cert-manager v1.19.3 + github.com/getkin/kin-openapi v0.133.0 github.com/go-logr/logr v1.4.3 + github.com/google/cel-go v0.28.1 + github.com/go-openapi/testify/v2 v2.0.2 github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.28.3 github.com/onsi/gomega v1.40.0 @@ -19,8 +22,48 @@ require ( ) require ( + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 // indirect + github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect + github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/speakeasy-api/jsonpath v0.6.0 // indirect + github.com/speakeasy-api/openapi-overlay v0.10.2 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect + github.com/woodsbury/decimal128 v1.3.0 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/crypto v0.50.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +require ( + cel.dev/expr v0.25.1 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -28,6 +71,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gin-gonic/gin v1.12.0 github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.22.4 // indirect github.com/go-openapi/jsonreference v0.21.4 // indirect @@ -63,6 +107,7 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/net v0.53.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect @@ -73,6 +118,7 @@ require ( golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.44.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect google.golang.org/grpc v1.79.3 // indirect google.golang.org/protobuf v1.36.11 // indirect @@ -83,6 +129,8 @@ require ( sigs.k8s.io/gateway-api v1.4.1 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) + +tool github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen diff --git a/go.sum b/go.sum index 45485f13..81a6fd45 100644 --- a/go.sum +++ b/go.sum @@ -1,27 +1,55 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3xCpOjiX5Obs= github.com/cert-manager/cert-manager v1.19.3/go.mod h1:e9NzLtOKxTw7y99qLyWGmPo6mrC1Nh0EKKcMkRfK+GE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= +github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= +github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/getkin/kin-openapi v0.133.0 h1:pJdmNohVIJ97r4AUFtEXRXwESr8b0bD721u/Tz6k8PQ= +github.com/getkin/kin-openapi v0.133.0/go.mod h1:boAciF6cXk5FhPqe/NQeBTeenbjqU4LhWBf09ILVvWE= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -68,39 +96,83 @@ github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxE github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= -github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM= +github.com/google/cel-go v0.28.1/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -109,14 +181,40 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 h1:4i+F2cvwBFZeplxCssNdLy3MhNzUD87mI3HnayHZkAU= +github.com/oapi-codegen/oapi-codegen/v2 v2.6.0/go.mod h1:eWHeJSohQJIINJZzzQriVynfGsnlQVh0UkN2UYYcw4Q= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.28.3 h1:4JvMdwtFU0imd8fHx25OJXoDMRexnf8v5NHKYSTTji4= github.com/onsi/ginkgo/v2 v2.28.3/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/openconfig/gnmic/pkg/api v0.1.10 h1:zU57bogHrnraDFCYDnxHZB8Hcd53bWx1fDkRTPw/R2w= github.com/openconfig/gnmic/pkg/api v0.1.10/go.mod h1:6PntONfjCMq3XzsDfWMkLeoVuBRbkm2foQO5m6PeYo0= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -130,14 +228,32 @@ github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+L github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/speakeasy-api/jsonpath v0.6.0 h1:IhtFOV9EbXplhyRqsVhHoBmmYjblIRh5D1/g8DHMXJ8= +github.com/speakeasy-api/jsonpath v0.6.0/go.mod h1:ymb2iSkyOycmzKwbEAYPJV/yi2rSmvBCLZJcyD+VVWw= +github.com/speakeasy-api/openapi-overlay v0.10.2 h1:VOdQ03eGKeiHnpb1boZCGm7x8Haj6gST0P3SGTX95GU= +github.com/speakeasy-api/openapi-overlay v0.10.2/go.mod h1:n0iOU7AqKpNFfEt6tq7qYITC4f0yzVVdFw0S7hukemg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -148,8 +264,19 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= +github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= +github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0= +github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= @@ -164,6 +291,8 @@ go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6 go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= @@ -172,43 +301,111 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.35.1 h1:0PO/1FhlK/EQNVK5+txc4FuhQibV25VLSdLMmGpDE/Q= @@ -233,7 +430,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index b85e661f..314c5654 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -41,6 +41,15 @@ spec: {{- if .Values.api.port }} - --api-bind-address=:{{ .Values.api.port }} {{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/name'] ports: {{- if .Values.webhook.enabled }} - name: webhook diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 5eb88b83..ce65b85b 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,50 +1,144 @@ package apiserver +//go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml +// To generate code, install openapi-codegen from https://github.com/oapi-codegen/oapi-codegen) +// Then use: go generate ./internal/apiserver +// To generate documentation +// docker run --rm -v ${PWD}:/local openapitools/openapi-generator-cli generate -i /local/internal/apiserver/openapi.yaml -g markdown -o /local/docs/content/docs/user-guide/rest-api + import ( - "encoding/json" + "context" + "fmt" "net/http" + "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" ) type APIServer struct { Server *http.Server + router *gin.Engine clusterReconciler *controller.ClusterReconciler + DiscoveryRegistry *discovery.Registry[ + types.NamespacedName, + core.DiscoveryRegistryValue, + ] + chunzSize int + logger logr.Logger + bearerToken bool +} - DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue] +type urlStruct struct { + Namespace string `uri:"namespace" binding:"required"` + Name string `uri:"name" binding:"required"` } -func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { - mux := http.NewServeMux() +func New( + addr string, + clusterReconciler *controller.ClusterReconciler, + discoveryRegistry *discovery.Registry[ + types.NamespacedName, + core.DiscoveryRegistryValue, + ], + discoveryChunksize int, + bearerToken string, +) (*APIServer, error) { + router := gin.New() + router.Use(gin.Recovery()) + gin.SetMode(gin.ReleaseMode) + logger := log.Log.WithValues("component", "api-server") + a := &APIServer{ Server: &http.Server{ Addr: addr, - Handler: mux, + Handler: router, }, + router: router, clusterReconciler: clusterReconciler, + DiscoveryRegistry: discoveryRegistry, + chunzSize: discoveryChunksize, + logger: logger, } - a.routes(mux) - return a + RegisterHandlers(router, a) + logger.Info("API server initialized", "addr", addr, "chunkSize", discoveryChunksize) + return a, nil } -func (a *APIServer) routes(mux *http.ServeMux) { - mux.HandleFunc("GET /clusters/{namespace}/{name}/plan", a.getClusterPlan) +func (a *APIServer) Router() *gin.Engine { + return a.router } -func (a *APIServer) getClusterPlan(w http.ResponseWriter, r *http.Request) { - namespace, name := r.PathValue("namespace"), r.PathValue("name") - plan, err := a.clusterReconciler.GetClusterPlan(namespace, name) +// GetClusterPlan returns cluster plan +func (a *APIServer) GetClusterPlan(c *gin.Context) { + uri := parseURI(c) + logger := log.FromContext(c.Request.Context()).WithValues( + "component", "apiserver", + "namespace", uri.Namespace, + "cluster", uri.Name, + ) + logger.Info("Received GET request for GetClusterPlan") + + plan, err := a.clusterReconciler.GetClusterPlan(uri.Namespace, uri.Name) if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) + logger.Error(err, "Failed to get cluster plan") + c.String(404, err.Error()) + return + } + c.JSON(200, plan) +} + +// CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. +func (a *APIServer) ApplyTargets(c *gin.Context) { + uri := parseURI(c) + logger := log.FromContext(c.Request.Context()).WithValues( + "component", "apiserver", + "namespace", uri.Namespace, + "targetsource", uri.Name, + ) + logger.Info("Received POST request for CreateTargets") + + key := getKey(uri) + registry, ok := a.DiscoveryRegistry.Get(key) + if !ok { + err := fmt.Errorf("targetSource %s/%s does not exist", uri.Namespace, uri.Name) + logger.Error(err, "TargetSource lookup failed") + c.JSON(http.StatusBadRequest, gin.H{"error": err}) + return + } + + if registry.CommonLoaderConfig.PushConfig == nil || registry.CommonLoaderConfig.PushConfig.Enabled == false { + err := fmt.Errorf("targetSource %s/%s has the push interface turned off", uri.Namespace, uri.Name) + logger.Error(err, "POST request rejected") + c.JSON(http.StatusBadRequest, gin.H{"error": err}) + return + } + + if authenticated, err := a.verifyAuthentication(c, registry, logger); authenticated == false { + logger.Info("Unauthorized request for CreateTargets", "error", err.Error()) + c.JSON(http.StatusUnauthorized, gin.H{"error": err}) return } - w.Header().Set("Content-Type", "application/json") - err = json.NewEncoder(w).Encode(plan) + + var payloadTargets Targets + if err := c.ShouldBind(&payloadTargets); err != nil { + logger.Error(err, "Failed to bind request payload") + c.JSON(http.StatusBadRequest, gin.H{"error": err}) + return + } + + targets, err := createDiscoveryEvent(payloadTargets) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + logger.Error(err, "failed creating discoveryEvent") + c.JSON(http.StatusBadRequest, gin.H{"error": err}) return } + + utils.SendEvents(context.Background(), registry.Channel, targets, a.chunzSize) + c.JSON(http.StatusOK, payloadTargets) } diff --git a/internal/apiserver/auth.go b/internal/apiserver/auth.go new file mode 100644 index 00000000..657fe0fb --- /dev/null +++ b/internal/apiserver/auth.go @@ -0,0 +1,124 @@ +package apiserver + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "fmt" + "hash" + "io" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" +) + +// verifyAuthentication checks for Bearer Token and/or Signature +func (a *APIServer) verifyAuthentication(ctx *gin.Context, registry core.DiscoveryRegistryValue, logger logr.Logger) (bool, error) { + if registry.CommonLoaderConfig.PushConfig.Auth != nil { + if authenticated, err := a.verifyBearerToken(ctx, registry, logger); authenticated == false { + return false, err + } + } + if registry.CommonLoaderConfig.PushConfig.Signature != nil { + if signatureMatch, err := a.verifySignature(ctx, registry, logger); signatureMatch == false { + return false, err + } + } + return true, nil +} + +// verifySignature verifies x-hook-signature from POST header with hmac from body and a kubernetes secret. +func (a *APIServer) verifySignature(ctx *gin.Context, registry core.DiscoveryRegistryValue, logger logr.Logger) (bool, error) { + signatureHeader := ctx.GetHeader("x-hook-signature") + clc := registry.CommonLoaderConfig + secret, err := getSecret(clc, clc.PushConfig.Signature.SecretRef.Key, clc.PushConfig.Signature.SecretRef.Name) + + if err != nil { + logger.Error(err, "error calling getSecret") + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err}) + return false, err + } + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + logger.Error(err, "failed to read request body") + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid request body"}) + return false, err + } + ctx.Request.Body = io.NopCloser(bytes.NewReader(body)) + + var mac hash.Hash + if registry.CommonLoaderConfig.PushConfig.Signature.Algorithm == "sha256" { + mac = hmac.New(sha256.New, []byte(secret)) + signatureHeader = strings.TrimSpace(strings.TrimPrefix(signatureHeader, "sha256=")) + } else { + mac = hmac.New(sha512.New, []byte(secret)) + signatureHeader = strings.TrimSpace(strings.TrimPrefix(signatureHeader, "sha512=")) + } + mac.Write(body) + signatureCalculated := mac.Sum(nil) + signatureProvided, err := hex.DecodeString(signatureHeader) + if err != nil { + logger.Error(err, "error decoding signatureHeader") + } + + if hmac.Equal(signatureCalculated, signatureProvided) { + return true, nil + } + err = fmt.Errorf("POST request signature does not align with signature calulcated from body and Kubernetes secret") + logger.Error(err, "verifySignature failed") + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err}) + return false, err +} + +// verifyBearerToken verifies bearer token from authorization header with value stored in kubernetes secret. +func (a *APIServer) verifyBearerToken(ctx *gin.Context, registry core.DiscoveryRegistryValue, logger logr.Logger) (bool, error) { + const bearerPrefix = "Bearer " + authHeader := strings.TrimSpace(ctx.GetHeader("Authorization")) + if !strings.HasPrefix(authHeader, bearerPrefix) { + err := fmt.Errorf("POST request has missing or invalid authorization header") + logger.Error(err, "verifyBearerToken failed") + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err}) + return false, err + } + + clc := registry.CommonLoaderConfig + bearerSecret, err := getSecret(clc, clc.PushConfig.Auth.Bearer.TokenSecretRef.Key, clc.PushConfig.Auth.Bearer.TokenSecretRef.Name) + if err != nil { + logger.Error(err, "error calling getSecret") + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err}) + return false, err + } + + bearerHeader := strings.TrimSpace(strings.TrimPrefix(authHeader, bearerPrefix)) + if bearerHeader != bearerSecret { + err := fmt.Errorf("POST request bearer is not equal to bearer stored in Kubernetes secret") + logger.Error(err, "bearer token mismatch") + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err}) + return false, err + } + return true, nil +} + +// getSecret returns Kubernetes Opaque secret as string +func getSecret(clc *core.CommonLoaderConfig, key string, name string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + selector := &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + Key: key, + } + secret, err := clc.ResourceFetcher.GetSecretKey(ctx, clc.TargetsourceNN.Namespace, selector) + if err != nil { + return "", fmt.Errorf("failed to get secret %s/%s key %q: %w", clc.TargetsourceNN.Namespace, name, key, err) + } + return secret, nil +} diff --git a/internal/apiserver/cfg.yaml b/internal/apiserver/cfg.yaml new file mode 100644 index 00000000..4bc7f022 --- /dev/null +++ b/internal/apiserver/cfg.yaml @@ -0,0 +1,6 @@ +package: apiserver +output: gen.go +generate: + gin-server: true + models: true + embedded-spec: true \ No newline at end of file diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go new file mode 100644 index 00000000..ecb2f91e --- /dev/null +++ b/internal/apiserver/gen.go @@ -0,0 +1,248 @@ +// Package apiserver provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.6.0 DO NOT EDIT. +package apiserver + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "net/url" + "path" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/gin-gonic/gin" +) + +const ( + BearerAuthScopes = "bearerAuth.Scopes" + SignatureScopes = "signature.Scopes" +) + +// Defines values for TargetOperation. +const ( + Created TargetOperation = "created" + Deleted TargetOperation = "deleted" + Updated TargetOperation = "updated" +) + +// Valid indicates whether the value is a known member of the TargetOperation enum. +func (e TargetOperation) Valid() bool { + switch e { + case Created: + return true + case Deleted: + return true + case Updated: + return true + default: + return false + } +} + +// Label defines model for Label. +type Label map[string]string + +// Target Network device to be monitored. Properties not marked as optional must be in JSON body. +type Target struct { + // Address IPv4/IPv6 address or hostname. + Address string `json:"address"` + + // Labels Labels must be map[string]string. For example vendor:nokia. + Labels *[]Label `json:"labels,omitempty"` + + // Name Name of device to be monitored. + Name string `json:"name"` + + // Operation Either `created`, `updated` or `deleted`. `created` and `updated` are identical and both apply the target. + Operation TargetOperation `json:"operation"` + + // Port gNMIc port. + Port *int `json:"port,omitempty"` + + // TargetProfile TargetProfile applied to apply to this router. + TargetProfile *string `json:"targetProfile,omitempty"` +} + +// TargetOperation Either `created`, `updated` or `deleted`. `created` and `updated` are identical and both apply the target. +type TargetOperation string + +// Targets defines model for Targets. +type Targets = []Target + +// ApplyTargetsJSONRequestBody defines body for ApplyTargets for application/json ContentType. +type ApplyTargetsJSONRequestBody = Targets + +// ServerInterface represents all server handlers. +type ServerInterface interface { + // Interface for real-time target updates, usually using a webhook. Targets are applied in the gNMIc Operator. + // (POST /api/v1/:namespace/target-source/:name/applyTargets) + ApplyTargets(c *gin.Context) + // Get cluster plan. + // (GET /clusters/:namespace/:name/plan) + GetClusterPlan(c *gin.Context) +} + +// ServerInterfaceWrapper converts contexts to parameters. +type ServerInterfaceWrapper struct { + Handler ServerInterface + HandlerMiddlewares []MiddlewareFunc + ErrorHandler func(*gin.Context, error, int) +} + +type MiddlewareFunc func(c *gin.Context) + +// ApplyTargets operation middleware +func (siw *ServerInterfaceWrapper) ApplyTargets(c *gin.Context) { + + c.Set(BearerAuthScopes, []string{}) + + c.Set(SignatureScopes, []string{}) + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.ApplyTargets(c) +} + +// GetClusterPlan operation middleware +func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetClusterPlan(c) +} + +// GinServerOptions provides options for the Gin server. +type GinServerOptions struct { + BaseURL string + Middlewares []MiddlewareFunc + ErrorHandler func(*gin.Context, error, int) +} + +// RegisterHandlers creates http.Handler with routing matching OpenAPI spec. +func RegisterHandlers(router gin.IRouter, si ServerInterface) { + RegisterHandlersWithOptions(router, si, GinServerOptions{}) +} + +// RegisterHandlersWithOptions creates http.Handler with additional options +func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options GinServerOptions) { + errorHandler := options.ErrorHandler + if errorHandler == nil { + errorHandler = func(c *gin.Context, err error, statusCode int) { + c.JSON(statusCode, gin.H{"msg": err.Error()}) + } + } + + wrapper := ServerInterfaceWrapper{ + Handler: si, + HandlerMiddlewares: options.Middlewares, + ErrorHandler: errorHandler, + } + + router.POST(options.BaseURL+"/api/v1/:namespace/target-source/:name/applyTargets", wrapper.ApplyTargets) + router.GET(options.BaseURL+"/clusters/:namespace/:name/plan", wrapper.GetClusterPlan) +} + +// Base64 encoded, gzipped, json marshaled Swagger object +var swaggerSpec = []string{ + + "H4sIAAAAAAAC/7xVTW8bNxD9KwO2x7XWboIedHOCNHbbOELtQwFDgEfLkZbRLsnOzCoVAv33guRGH5UC", + "5JSTKHI+3rx5M/vFNKGPwZNXMdMvRpqWeszHP3FBXTqgtU5d8NjNOERidZQNdBvJTI0oO78yu+rrRVh8", + "okbTxRPyijTZWpKGXUxhzNQ8kH4OvAZLG9cQaIAFQR+808BkJ3DIAz4o9MhrsoACIRYg0A+iycl5+P3x", + "4wMsgt1OTGXiCUK0lknkHMD9bPO6vp9tfoXRBAJDG0Q99pTinNXWJTYuRMosyR5Pj/G5OM3LzwR+Cwz0", + "L/axI9iQt4GnPqwdpjROqc9Bf2Zamqn5qT60ox57UZdGHPhFZtym/wnsBXKxJwjLb5F7qbjEGRb3/0d7", + "57QlhpeGCZXsSwUvQ7T5mDh7sdRR+jM5mAB6e2SFTOAseXUNdvltEbQFjLHbgrYEmlWSgJEfejN9NmMk", + "U5kxiqnMmMjML+CPgS+obPXw4b6B9HZUtPNKK+JMZ84747B03QUen46fM1xHNtE5Ig+grRPgMCjxBVp3", + "lWH6Z3BMNtWUm1XtJXlM+vybo5O18V0iGUftTCW7ygg1AzvdPibTMhgLQia+HbQ9r/vu6WkGOGhbWpZu", + "YRDnV/Ame4GGNXlTlV2RUpVoBwZa1ZiQiFt51IEvkHv34fYt7N+TXJMSEl8kChG3XcDUdZeMW0Kb4xfB", + "m7+v7kJYXz3uwx+Kju4PSlXvkusy5DXlNLV3lMPHzHpg+Ovd4xPczu5NZTbEUmBdT64nN+NEeIzOTM2r", + "yfXkVdosqG3mrsbo6s1NPU1oJGJDdVHSlYSBGyoPdVbJURdjkKzRfdvvrZma22OrIhgSfRPsNtk2wSv5", + "7Jb1V7pRf5IyqaX336cMKaRckvh+eUUUKXsWoXOiFfRDpy4trlKhQAwibtHlHXlQt/JAWe4Sg5cisV+u", + "b35cCbKfTxmahkSWQ9flHfm6wDh1us02RcjgBHonWeCBwfkNds6eDI6ZPp+OzPP8RNzP8126GPoeeZu+", + "Ll6Jl9gQLAMDE3ZX6vqvJELZaVLBIAN23XacLoTPtGhDWE9gXxQfFo/zeUROVTzJOOumG0SJ5ViTRYWx", + "w8zz+Bk+Fd970rfFc5bMzhp4fc7ckT0w6cCeRq721b8nhREQpPQJ4273XwAAAP//UKR4U2kIAAA=", +} + +// GetSwagger returns the content of the embedded swagger specification file +// or error if failed to decode +func decodeSpec() ([]byte, error) { + zipped, err := base64.StdEncoding.DecodeString(strings.Join(swaggerSpec, "")) + if err != nil { + return nil, fmt.Errorf("error base64 decoding spec: %w", err) + } + zr, err := gzip.NewReader(bytes.NewReader(zipped)) + if err != nil { + return nil, fmt.Errorf("error decompressing spec: %w", err) + } + var buf bytes.Buffer + _, err = buf.ReadFrom(zr) + if err != nil { + return nil, fmt.Errorf("error decompressing spec: %w", err) + } + + return buf.Bytes(), nil +} + +var rawSpec = decodeSpecCached() + +// a naive cached of a decoded swagger spec +func decodeSpecCached() func() ([]byte, error) { + data, err := decodeSpec() + return func() ([]byte, error) { + return data, err + } +} + +// Constructs a synthetic filesystem for resolving external references when loading openapi specifications. +func PathToRawSpec(pathToFile string) map[string]func() ([]byte, error) { + res := make(map[string]func() ([]byte, error)) + if len(pathToFile) > 0 { + res[pathToFile] = rawSpec + } + + return res +} + +// GetSwagger returns the Swagger specification corresponding to the generated code +// in this file. The external references of Swagger specification are resolved. +// The logic of resolving external references is tightly connected to "import-mapping" feature. +// Externally referenced files must be embedded in the corresponding golang packages. +// Urls can be supported but this task was out of the scope. +func GetSwagger() (swagger *openapi3.T, err error) { + resolvePath := PathToRawSpec("") + + loader := openapi3.NewLoader() + loader.IsExternalRefsAllowed = true + loader.ReadFromURIFunc = func(loader *openapi3.Loader, url *url.URL) ([]byte, error) { + pathToFile := url.String() + pathToFile = path.Clean(pathToFile) + getSpec, ok := resolvePath[pathToFile] + if !ok { + err1 := fmt.Errorf("path not found: %s", pathToFile) + return nil, err1 + } + return getSpec() + } + var specData []byte + specData, err = rawSpec() + if err != nil { + return + } + swagger, err = loader.LoadFromData(specData) + if err != nil { + return + } + return +} diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go new file mode 100644 index 00000000..ddf47dc4 --- /dev/null +++ b/internal/apiserver/helpers.go @@ -0,0 +1,96 @@ +package apiserver + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller/discovery/core" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// createDiscoveryEvent creates object of type core.DiscoveryEvent +func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error) { + targets := []core.DiscoveryEvent{} + + if len(payloadTargets) > 0 { + for i, target := range payloadTargets { + if target.Name == "" { + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Name.", i) + } + if target.Address == "" { + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Address.", i) + } + event, err := getEvent(target, i) + if err != nil { + return nil, err + } + + targets = append(targets, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: target.Name, + Address: target.Address, + Port: int32(*target.Port), + Labels: convertTargetLabelsToMap(target), + TargetProfile: *target.TargetProfile, + }, + Event: event, + }) + } + } + return targets, nil +} + +// getKey returns key for used to identify correct channel in DiscoveryRegistry +func getKey(u urlStruct) types.NamespacedName { + key := types.NamespacedName{ + Namespace: u.Namespace, + Name: u.Name, + } + return key +} + +// convertTargetLabelsToMap converts target.Labels to map. +func convertTargetLabelsToMap(target Target) map[string]string { + labelMap := make(map[string]string) + if target.Labels != nil { + for _, tag := range *target.Labels { + for key, value := range tag { + if key == "" { + continue + } + labelMap[key] = value + } + } + } + return labelMap +} + +// getEvent converts target.Operation to core.Operation. +func getEvent(target Target, index int) (core.EventAction, error) { + event := core.EventApply + switch target.Operation { + case Created: + event = core.EventApply + case Updated: + event = core.EventApply + case Deleted: + event = core.EventDelete + default: + return event, fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation", index) + } + return event, nil +} + +// parseURI parses URI to urlStruct. +func parseURI(c *gin.Context) (url urlStruct) { + logger := log.FromContext(c.Request.Context()).WithValues("component", "apiserver", "action", "parse-uri") + var u urlStruct + if err := c.ShouldBindUri(&u); err != nil { + logger.Error(err, "Failed to bind request URI") + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + return u +} diff --git a/internal/apiserver/helpers_test.go b/internal/apiserver/helpers_test.go new file mode 100644 index 00000000..3cda0b8a --- /dev/null +++ b/internal/apiserver/helpers_test.go @@ -0,0 +1,268 @@ +package apiserver + +import ( + "reflect" + "testing" + + "net/http" + "net/http/httptest" + + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller/discovery/core" + "k8s.io/apimachinery/pkg/types" +) + +func stringPtr(value string) *string { + return &value +} + +func TestGetEventApply(t *testing.T) { + port := 22 + target := Target{ + Address: "1.1.1.1", + Port: &port, + Name: "routername", + Labels: &[]Label{}, + Operation: "created", + } + event, err := getEvent(target, 0) + if event != core.EventApply { + t.Errorf("getEvent(target) = %d, want core.EventApply", event) + } + if err != nil { + t.Errorf("getEvent(target) returns err: %s", err) + } +} + +func TestGetEventDelete(t *testing.T) { + port := 22 + target := Target{ + Address: "1.1.1.1", + Port: &port, + Name: "routername", + Labels: &[]Label{}, + Operation: "deleted", + } + event, err := getEvent(target, 0) + if event != core.EventDelete { + t.Errorf("getEvent(target) = %d, want core.EventDelete", event) + } + if err != nil { + t.Errorf("getEvent(target) returns err: %s", err) + } +} + +func TestGetEventEmptyOperation(t *testing.T) { + port := 22 + target := Target{ + Address: "1.1.1.1", + Port: &port, + Name: "routername", + Labels: &[]Label{}, + Operation: "", + } + event, err := getEvent(target, 0) + if err == nil { + t.Errorf("getEvent(target, 0) = %d, want error", event) + } +} + +func TestGetEventUpdate(t *testing.T) { + port := 22 + target := Target{ + Address: "1.1.1.1", + Port: &port, + Name: "routername", + Labels: &[]Label{}, + Operation: "updated", + } + event, err := getEvent(target, 0) + if event != core.EventApply { + t.Errorf("getEvent(target) = %d, want core.EventApply", event) + } + if err != nil { + t.Errorf("getEvent(target) returns err: %s", err) + } +} + +func TestGetKey(t *testing.T) { + u := urlStruct{ + Namespace: "default", + Name: "http-discovery", + } + expected := types.NamespacedName{ + Namespace: "default", + Name: "http-discovery", + } + result := getKey(u) + if result != expected { + t.Errorf("getKey(%v) = %v; want %v", u, result, expected) + } +} + +func TestConvertTargetLabelsToMapEmpty(t *testing.T) { + target := Target{} + result := convertTargetLabelsToMap(target) + if len(result) != 0 { + t.Errorf("convertTargetLabelsToMap(target) = %v; want empty map", result) + } +} + +func TestConvertTargetLabelsToMap(t *testing.T) { + label := Label{"Tag": "TT1, TT2"} + target := Target{ + Labels: &[]Label{label}, + } + expected := map[string]string{ + "Tag": "TT1, TT2", + } + result := convertTargetLabelsToMap(target) + if !reflect.DeepEqual(result, expected) { + t.Errorf("convertTargetLabelsToMap(target) = %v; want %v", result, expected) + } +} + +func TestConvertTargetLabelsToMapEmptyKey(t *testing.T) { + label := Label{"": "TT1, TT2"} + target := Target{ + Labels: &[]Label{label}, + } + result := convertTargetLabelsToMap(target) + if len(result) != 0 { + t.Errorf("convertTargetLabelsToMap(target) = %v; want empty map", result) + } +} + +func TestConvertTargetLabelsToMapTwoEntries(t *testing.T) { + label := Label{"Tag": "TT1, TT2"} + label2 := Label{"Tag1": "TT1"} + target := Target{ + Labels: &[]Label{label, label2}, + } + expected := map[string]string{ + "Tag": "TT1, TT2", + "Tag1": "TT1", + } + result := convertTargetLabelsToMap(target) + if !reflect.DeepEqual(result, expected) { + t.Errorf("convertTargetLabelsToMap(target) = %v; want %v", result, expected) + } +} + +func TestCreateDiscoveryEvent(t *testing.T) { + port := 22 + targetprofile := "" + targets := []Target{{ + Name: "router1", + Address: "1.1.1.1", + Port: &port, + Labels: &[]Label{}, + TargetProfile: &targetprofile, + Operation: "updated"}} + + expected := []core.DiscoveryEvent{ + { + Target: core.DiscoveredTarget{ + Name: "router1", + Address: "1.1.1.1", + Port: 22, + Labels: map[string]string{}, + TargetProfile: "", + }, + Event: core.EventApply, + }, + } + result, _ := createDiscoveryEvent(targets) + if !reflect.DeepEqual(result, expected) { + t.Errorf("createDiscoveryEvent(targets) = %v; want %v", result, expected) + } +} + +func TestCreateDiscoveryEventEmptyName(t *testing.T) { + port := 22 + targets := []Target{{ + Address: "1.1.1.1", + Port: &port, + Labels: &[]Label{}, + Operation: "updated"}} + + result, err := createDiscoveryEvent(targets) + if err == nil { + t.Errorf("createDiscoveryEvent(targets) returns %v, want missing name error", result) + } +} + +func TestCreateDiscoveryEventEmptyIP(t *testing.T) { + port := 22 + targets := []Target{{ + Address: "", + Port: &port, + Name: "routername", + Labels: &[]Label{}, + Operation: "updated"}} + + result, err := createDiscoveryEvent(targets) + if err == nil { + t.Errorf("createDiscoveryEvent(targets) returns %v, want missing address error", result) + } +} + +func TestCreateDiscoveryEventWrongEvent(t *testing.T) { + port := 22 + targets := []Target{{ + Address: "1.1.1.1", + Port: &port, + Name: "", + Labels: &[]Label{}, + Operation: "upWROOONGdated"}} + + result, err := createDiscoveryEvent(targets) + if err == nil { + t.Errorf("createDiscoveryEvent(targets) returns %v, want wrong Operation error", result) + } +} + +func TestParseURI(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + router := gin.New() + var result urlStruct + router.POST("/api/v1/:namespace/target-source/:name/createTargets", func(ctx *gin.Context) { + result = parseURI(ctx) + }) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/default/target-source/http-discovery/createTargets", nil) + router.ServeHTTP(recorder, req) + + expected := urlStruct{ + Namespace: "default", + Name: "http-discovery", + } + + if !reflect.DeepEqual(result, expected) { + t.Errorf("parseURI(ctx) = %v; want %v", result, expected) + } + if recorder.Code != http.StatusOK { + t.Errorf("parseURI(ctx) status code = %d; want %d", recorder.Code, http.StatusOK) + } +} + +func TestParseURIMissingName(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + router := gin.New() + var result urlStruct + router.POST("/api/v1/:namespace/target-source/:name/createTargets", func(ctx *gin.Context) { + result = parseURI(ctx) + }) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/default/target-source//createTargets", nil) + router.ServeHTTP(recorder, req) + + if !reflect.DeepEqual(result, urlStruct{}) { + t.Errorf("parseURI(ctx) = %v; want empty urlStruct", result) + } + if recorder.Code != http.StatusBadRequest { + t.Errorf("parseURI(ctx) status code = %d; want %d", recorder.Code, http.StatusBadRequest) + } +} diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml new file mode 100644 index 00000000..475d7615 --- /dev/null +++ b/internal/apiserver/openapi.yaml @@ -0,0 +1,92 @@ +openapi: 3.0.3 +info: + title: "gNMIc Operator REST API" + version: "0.0.1" +paths: + /clusters/:namespace/:name/plan: + get: + summary: "Get cluster plan." + operationId: "getClusterPlan" + responses: + '200': + description: "ClusterPlan returned" + /api/v1/:namespace/target-source/:name/applyTargets: + post: + summary: "Interface for real-time target updates, usually using a webhook. Targets are applied in the gNMIc Operator." + operationId: "applyTargets" + security: + - bearerAuth: [] + signature: [] + requestBody: + required: true + description: Target must be passed as a list, multiple targets possible. + content: + application/json: + schema: + $ref: '#/components/schemas/Targets' + responses: + '201': + description: "Targets applied successfully" + content: + application/json: + schema: + $ref: '#/components/schemas/Targets' + '401': + description: Access token is missing or invalid + +components: + schemas: + Targets: + type: array + items: + $ref: '#/components/schemas/Target' + + Target: + description: Network device to be monitored. Properties not marked as optional must be in JSON body. + type: object + required: + - name + - address + - operation + properties: + name: + type: string + description: Name of device to be monitored. + address: + type: string + description: IPv4/IPv6 address or hostname. + port: + type: integer + description: gNMIc port. + targetProfile: + type: string + description: TargetProfile applied to apply to this router. + labels: + type: array + description: Labels must be map[string]string. For example vendor:nokia. + items: + $ref: '#/components/schemas/Label' + operation: + type: string + enum: + - created + - updated + - deleted + description: "Either `created`, `updated` or `deleted`. `created` and `updated` are identical and both apply the target." + + Label: + type: object + additionalProperties: + type: string + + securitySchemes: + bearerAuth: + type: http + scheme: bearer + description: HTTP authentication using Bearer token + signature: + name: X-Hook-Signature + type: apiKey + in: header + description: HMAC signature of the request payload + \ No newline at end of file diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go index e5cc5ea0..1767be11 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/client.go @@ -2,12 +2,13 @@ package discovery import ( "context" + "fmt" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -32,7 +33,7 @@ func fetchExistingTargets(ctx context.Context, c client.Client, ts *gnmicv1alpha return targetList.Items, nil } -func applyTarget(ctx context.Context, c client.Client, s *runtime.Scheme, desired *gnmicv1alpha1.Target, ts *gnmicv1alpha1.TargetSource) error { +func applyTarget(ctx context.Context, c client.Client, s *runtime.Scheme, desired *gnmicv1alpha1.Target, ts *gnmicv1alpha1.TargetSource) (controllerutil.OperationResult, error) { existing := &gnmicv1alpha1.Target{ ObjectMeta: metav1.ObjectMeta{ Name: desired.Name, @@ -40,14 +41,14 @@ func applyTarget(ctx context.Context, c client.Client, s *runtime.Scheme, desire }, } - _, err := controllerutil.CreateOrUpdate(ctx, c, existing, func() error { + result, err := controllerutil.CreateOrUpdate(ctx, c, existing, func() error { existing.Spec = desired.Spec existing.Labels = desired.Labels return controllerutil.SetControllerReference(ts, existing, s) }) - return err + return result, err } func deleteTarget(ctx context.Context, c client.Client, name string, namespace string) error { @@ -71,19 +72,43 @@ func deleteTarget(ctx context.Context, c client.Client, name string, namespace s return err } -// updateTargetSourceStatus updates the status of the TargetSource Object ts. The only fields updated are targetCount and LastSync, which takes the current timestamp. -func updateTargetSourceStatus(ctx context.Context, c client.Client, ts *gnmicv1alpha1.TargetSource, targetCount int32) error { - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - latest := &gnmicv1alpha1.TargetSource{} - if err := c.Get(ctx, client.ObjectKeyFromObject(ts), latest); err != nil { - return err - } +// Helper: GetSecretValues returns values from a secret +// If keys are provided -> returns only those keys +// If keys is empty -> returns entire secret data +func GetSecretValues( + ctx context.Context, + c client.Client, + namespace string, + secretRef string, + keys ...string, +) (map[string]string, error) { + var secret corev1.Secret + if err := c.Get(ctx, + client.ObjectKey{ + Name: secretRef, + Namespace: namespace, + }, &secret); err != nil { + return nil, fmt.Errorf("failed to get secret %s/%s: %w", namespace, secretRef, err) + } - latest.Status.TargetsCount = targetCount - latest.Status.LastSync = metav1.Now() + result := make(map[string]string) - return c.Status().Update(ctx, latest) - }) + // Return full secret + if len(keys) == 0 { + for k, v := range secret.Data { + result[k] = string(v) + } + return result, nil + } - return err + // Return specific keys + for _, key := range keys { + val, ok := secret.Data[key] + if !ok { + return nil, fmt.Errorf("key %s missing in secret %s/%s", key, namespace, secretRef) + } + result[key] = string(val) + } + + return result, nil } diff --git a/internal/controller/discovery/const.go b/internal/controller/discovery/const.go index b48331d3..8d37785f 100644 --- a/internal/controller/discovery/const.go +++ b/internal/controller/discovery/const.go @@ -4,3 +4,10 @@ const ( // Kubernetes Side Labels LabelTargetSourceName = "operator.gnmic.dev/targetsource" ) + +const ( + // Prefix and Labels for external systems + ExternalLabelPrefix = "gnmic_operator_" + + ExternalLabelTargetProfile = ExternalLabelPrefix + "target_profile" +) diff --git a/internal/controller/discovery/core/ressource_fetcher_interface.go b/internal/controller/discovery/core/ressource_fetcher_interface.go new file mode 100644 index 00000000..31a82cf0 --- /dev/null +++ b/internal/controller/discovery/core/ressource_fetcher_interface.go @@ -0,0 +1,15 @@ +package core + +import ( + "context" + + corev1 "k8s.io/api/core/v1" +) + +// ResourceFetcher provides read-only access to namespaced Secret and +// ConfigMap data for loaders without requiring each loader to carry a +// Kubernetes client. +type ResourceFetcher interface { + GetSecretKey(ctx context.Context, namespace string, selector *corev1.SecretKeySelector) (string, error) + GetConfigMapKey(ctx context.Context, namespace string, selector *corev1.ConfigMapKeySelector) (string, error) +} diff --git a/internal/controller/discovery/core/status_updater_interface.go b/internal/controller/discovery/core/status_updater_interface.go new file mode 100644 index 00000000..35a7c3dc --- /dev/null +++ b/internal/controller/discovery/core/status_updater_interface.go @@ -0,0 +1,33 @@ +package core + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + ConditionTypeReady = "Ready" + ConditionTypeReconciling = "Reconciling" + ConditionTypeDegraded = "Degraded" + ConditionTypeStalled = "Stalled" + + ReasonWaitingForSync Reason = "WaitingForSync" + ReasonSyncStarted Reason = "SyncStarted" + ReasonSyncSucceeded Reason = "SyncSucceeded" + ReasonSyncCompleted Reason = "SyncCompleted" + ReasonSyncWithErrors Reason = "SyncSucceededWithErrors" + ReasonSyncFailed Reason = "SyncFailed" +) + +type Reason string + +type StatusUpdate struct { + Conditions []metav1.Condition + TargetsCount *int32 +} + +// StatusUpdater defines the interface for TargetLoaders and MessageProcessor to update the status of the TargetSource +type StatusUpdater interface { + UpdateStatus(context.Context, StatusUpdate) error +} diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 8de38c1d..ac9c87c3 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -3,6 +3,8 @@ package core import ( "context" + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/api/v1alpha1" "k8s.io/apimachinery/pkg/types" ) @@ -19,9 +21,12 @@ type DiscoveryRegistryValue struct { } type CommonLoaderConfig struct { - TargetsourceNN types.NamespacedName - ChunkSize int - AcceptPush bool + TargetsourceNN types.NamespacedName + ChunkSize int + PushConfig *v1alpha1.PushSpec + Router *gin.Engine + ResourceFetcher ResourceFetcher + Updater StatusUpdater } // EventAction represents the type of a discovery event diff --git a/internal/controller/discovery/discovery.go b/internal/controller/discovery/discovery.go index 491cdfb3..07c9ceda 100644 --- a/internal/controller/discovery/discovery.go +++ b/internal/controller/discovery/discovery.go @@ -10,6 +10,12 @@ package discovery // - core: message contracts, snapshot/event types, and transport helpers. // - message processor: snapshot + event target state application logic. // - loaders: target discovery providers (HTTP, webhook, etc.). -// - registry: key -> channel registry. +// - registry: generic discovery runtime registry. +// +// The package also contains discovery helpers: +// - client helpers for applying/deleting targets and updating TargetSource status. +// - a loader factory for constructing discovery loaders. +// - target normalization and event generation logic. +// - a resource fetcher for resolving Secret/ConfigMap values used by loaders. // // At the moment, the targetsource controller imports specific subpackages explicitly. diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index e8061d93..7eb49441 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -1,24 +1,28 @@ package discovery import ( + "context" "fmt" + "sigs.k8s.io/controller-runtime/pkg/client" + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" - http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" + "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) // NewLoader creates a loader by name -func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { +func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { switch { case spec.Provider.HTTP != nil: - if spec.Provider.HTTP.Push != nil { - cfg.AcceptPush = spec.Provider.HTTP.Push.Enabled + httpSpec := *spec.Provider.HTTP + if httpSpec.Push != nil { + cfg.PushConfig = httpSpec.Push } - return http.New(*cfg), nil + cfg.ResourceFetcher = newK8sResourceFetcher(c) + return http.New(*cfg, httpSpec), nil default: return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN) } - } diff --git a/internal/controller/discovery/loaders/http/auth.go b/internal/controller/discovery/loaders/http/auth.go new file mode 100644 index 00000000..04f48f7e --- /dev/null +++ b/internal/controller/discovery/loaders/http/auth.go @@ -0,0 +1,78 @@ +package http + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + corev1 "k8s.io/api/core/v1" +) + +// fetchSecret uses the configured ResourceFetcher to resolve secret values. +func (l *Loader) fetchSecret(ctx context.Context, sel *corev1.SecretKeySelector) (string, error) { + if l.loaderCfg.ResourceFetcher == nil { + return "", nil + } + return l.loaderCfg.ResourceFetcher.GetSecretKey(ctx, l.loaderCfg.TargetsourceNN.Namespace, sel) +} + +func (l *Loader) applyAuthentication(req *http.Request) error { + auth := l.spec.Authentication + if auth == nil { + return nil + } + + if auth.Basic != nil { + return l.applyBasicAuth(req, auth.Basic.CredentialSecretRef) + } + + if auth.Token != nil { + return l.applyTokenAuth(req, auth.Token.Scheme, auth.Token.TokenSecretRef) + } + + return fmt.Errorf("no supported authentication method configured") +} + +// applyBasicAuth applies Basic authentication using the provided secret selector. +// Returns an error when credentials are missing or cannot be parsed. +func (l *Loader) applyBasicAuth(req *http.Request, sel *corev1.SecretKeySelector) error { + if sel == nil { + return fmt.Errorf("Basic auth enabled but no valid credentials provided") + } + + val, err := l.fetchSecret(req.Context(), sel) + if err != nil { + return err + } + + var cm map[string]string + if err := json.Unmarshal([]byte(val), &cm); err != nil { + return err + } + + username := cm["username"] + password := cm["password"] + if username == "" && password == "" { + return fmt.Errorf("Basic auth enabled but no valid credentials provided") + } + + req.SetBasicAuth(username, password) + return nil +} + +// applyTokenAuth applies token-based authentication using the provided secret selector +// Returns an error when no valid token is found +func (l *Loader) applyTokenAuth(req *http.Request, scheme string, sel *corev1.SecretKeySelector) error { + if sel == nil { + return fmt.Errorf("Token auth enabled but no valid token secret reference provided") + } + + token, err := l.fetchSecret(req.Context(), sel) + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("%s %s", scheme, token)) + return nil +} diff --git a/internal/controller/discovery/loaders/http/auth_test.go b/internal/controller/discovery/loaders/http/auth_test.go new file mode 100644 index 00000000..24fc821f --- /dev/null +++ b/internal/controller/discovery/loaders/http/auth_test.go @@ -0,0 +1,91 @@ +package http + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" +) + +func TestApplyAuthenticationCases(t *testing.T) { + credsJSON, _ := json.Marshal(map[string]string{"username": "user", "password": "pass"}) + + tests := []struct { + name string + config gnmicv1alpha1.HTTPConfig + fetcher core.ResourceFetcher + check func(t *testing.T, req *http.Request, err error) + }{ + { + name: "basic success", + config: gnmicv1alpha1.HTTPConfig{Authentication: &gnmicv1alpha1.AuthenticationSpec{Basic: &gnmicv1alpha1.BasicAuthSpec{CredentialSecretRef: &corev1.SecretKeySelector{}}}}, + fetcher: fakeResourceFetcher{secretValue: string(credsJSON)}, + check: func(t *testing.T, req *http.Request, err error) { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + user, pass, ok := req.BasicAuth() + if !ok || user != "user" || pass != "pass" { + t.Fatalf("basic auth not set correctly") + } + }, + }, + { + name: "basic invalid json", + config: gnmicv1alpha1.HTTPConfig{Authentication: &gnmicv1alpha1.AuthenticationSpec{Basic: &gnmicv1alpha1.BasicAuthSpec{CredentialSecretRef: &corev1.SecretKeySelector{}}}}, + fetcher: fakeResourceFetcher{secretValue: "invalid-json"}, + check: func(t *testing.T, req *http.Request, err error) { + if err == nil { + t.Fatalf("expected error for invalid json") + } + }, + }, + { + name: "token success", + config: gnmicv1alpha1.HTTPConfig{Authentication: &gnmicv1alpha1.AuthenticationSpec{Token: &gnmicv1alpha1.TokenAuthSpec{Scheme: "Bearer", TokenSecretRef: &corev1.SecretKeySelector{}}}}, + fetcher: fakeResourceFetcher{secretValue: "token-value"}, + check: func(t *testing.T, req *http.Request, err error) { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := req.Header.Get("Authorization"); !strings.Contains(got, "token-value") { + t.Fatalf("token header not set: %q", got) + } + }, + }, + { + name: "token missing secret", + config: gnmicv1alpha1.HTTPConfig{Authentication: &gnmicv1alpha1.AuthenticationSpec{Token: &gnmicv1alpha1.TokenAuthSpec{Scheme: "Bearer"}}}, + fetcher: nil, + check: func(t *testing.T, req *http.Request, err error) { + if err == nil { + t.Fatalf("expected token secret ref error") + } + }, + }, + { + name: "no method configured", + config: gnmicv1alpha1.HTTPConfig{Authentication: &gnmicv1alpha1.AuthenticationSpec{}}, + fetcher: nil, + check: func(t *testing.T, req *http.Request, err error) { + if err == nil { + t.Fatalf("expected unsupported auth error") + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + loader := makeLoader(tt.config, tt.fetcher) + req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil) + err := loader.applyAuthentication(req) + tt.check(t, req, err) + }) + } +} diff --git a/internal/controller/discovery/loaders/http/helpers_test.go b/internal/controller/discovery/loaders/http/helpers_test.go new file mode 100644 index 00000000..240e3d40 --- /dev/null +++ b/internal/controller/discovery/loaders/http/helpers_test.go @@ -0,0 +1,112 @@ +package http + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net/http" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" +) + +// fakeResourceFetcher is a lightweight test double. +type fakeResourceFetcher struct { + secretValue string + configuration string + secretErr error + configMapErr error +} + +type fakeStatusUpdater struct { +} + +func (f fakeResourceFetcher) GetSecretKey(_ context.Context, _ string, _ *corev1.SecretKeySelector) (string, error) { + return f.secretValue, f.secretErr +} + +func (f fakeResourceFetcher) GetConfigMapKey(_ context.Context, _ string, _ *corev1.ConfigMapKeySelector) (string, error) { + return f.configuration, f.configMapErr +} + +func (f fakeStatusUpdater) UpdateStatus(ctx context.Context, update core.StatusUpdate) error { + return nil +} + +func makeLoader(spec gnmicv1alpha1.HTTPConfig, fetcher core.ResourceFetcher) *Loader { + if spec.Method == "" { + spec.Method = http.MethodGet + } + if spec.Interval == nil { + spec.Interval = &metav1.Duration{Duration: 6 * time.Hour} + } + return &Loader{ + loaderCfg: core.CommonLoaderConfig{ + TargetsourceNN: types.NamespacedName{Namespace: "default", Name: "test"}, + ChunkSize: 10, + Updater: newFakeStatusUpdater(), + ResourceFetcher: fetcher, + }, + spec: spec, + } +} + +func mustBuildClient(t *testing.T, loader *Loader) *http.Client { + t.Helper() + client, err := loader.buildHTTPClient(context.Background()) + if err != nil { + t.Fatalf("buildHTTPClient failed: %v", err) + } + return client +} + +func startLoaderRun(loader *Loader) (context.Context, context.CancelFunc, chan []core.DiscoveryMessage, chan error) { + ctx, cancel := context.WithCancel(context.Background()) + out := make(chan []core.DiscoveryMessage, 1) + done := make(chan error, 1) + go func() { done <- loader.Run(ctx, out) }() + return ctx, cancel, out, done +} + +// genSelfSignedCertPEM generates a self-signed certificate PEM used in tests. +func genSelfSignedCertPEM() (string, error) { + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return "", err + } + tmpl := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "test-ca", + }, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv) + if err != nil { + return "", err + } + var buf bytes.Buffer + if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil { + return "", err + } + return buf.String(), nil +} + +func newFakeStatusUpdater() core.StatusUpdater { + return fakeStatusUpdater{} +} diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 6b85a9bb..860a6f74 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -1,49 +1,149 @@ package http import ( + "bytes" "context" + "crypto/tls" + "crypto/x509" + "encoding/json" "fmt" + "net/http" "time" + "github.com/go-logr/logr" + "github.com/google/uuid" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/log" + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" - "github.com/google/uuid" ) +// Loader implements the HTTP pull discovery mechanism +// It periodically polls an HTTP endpoint, extracts targets from the response, +// and emits discovery snapshots downstream type Loader struct { - commonCfg core.CommonLoaderConfig + loaderCfg core.CommonLoaderConfig + spec gnmicv1alpha1.HTTPConfig } -// New instantiates the http loader with the provided config -func New(cfg core.CommonLoaderConfig) core.Loader { - return &Loader{commonCfg: cfg} +// New creates a new HTTP loader instance with the provided configuration. +// The loader is stateless apart from its config and spec +func New(cfg core.CommonLoaderConfig, httpConfig gnmicv1alpha1.HTTPConfig) core.Loader { + return &Loader{loaderCfg: cfg, spec: httpConfig} } +// Name returns the loader's name, used for logging and metrics func (l *Loader) Name() string { return "http" } +// Run starts the HTTP discovery loop +// It performs an immediate fetch and then continues polling at a fixed interval func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) error { + if l.spec.URL == "" { + return nil + } + logger := log.FromContext(ctx).WithValues( "component", "loader", "name", l.Name(), - "targetsource", l.commonCfg.TargetsourceNN, + "targetsource", l.loaderCfg.TargetsourceNN, ) logger.Info( "HTTP loader started", - "targetsource", l.commonCfg.TargetsourceNN.Name, - "namespace", l.commonCfg.TargetsourceNN.Namespace, + "targetsource", l.loaderCfg.TargetsourceNN.Name, + "namespace", l.loaderCfg.TargetsourceNN.Namespace, ) - // Only for debugging: emit a static snapshot every 30 seconds - ticker := time.NewTicker(30 * time.Second) + logger.Info("HTTP loader started") + + client, err := l.buildHTTPClient(ctx) + if err != nil { + return fmt.Errorf("failed to build HTTP client: %w", err) + } + if l.spec.Interval == nil { + return fmt.Errorf("interval must be configured") + } + interval := l.spec.Interval.Duration + ticker := time.NewTicker(interval) defer ticker.Stop() - i := 1 + logger.Info( + "HTTP polling discovery started", + "interval", interval.String(), + "url", l.spec.URL, + ) + + // helper function to fetch targets and emit discovery messages + fetchAndEmit := func() { + // Set TargetSource conditions to "Reconciling" + l.loaderCfg.Updater.UpdateStatus( + ctx, + core.StatusUpdate{ + Conditions: []metav1.Condition{ + { + Type: core.ConditionTypeReconciling, + Status: metav1.ConditionStatus("True"), + Reason: string(core.ReasonSyncStarted), + Message: "Started fetching target source", + }, + }, + }, + ) + + // Fetch targets from HTTP endpoint + targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client, logger) + if err != nil { + logger.Error( + err, + "Failed to fetch targets from HTTP endpoint", + "url", l.spec.URL, + ) + // Set TargetSource conditions to "Stalled" if endpoint is not available + l.loaderCfg.Updater.UpdateStatus( + ctx, + core.StatusUpdate{ + Conditions: []metav1.Condition{ + { + Type: core.ConditionTypeStalled, + Status: metav1.ConditionStatus("True"), + Reason: string(core.ReasonSyncFailed), + Message: "HTTP endpoint not available", + }, + }, + }, + ) + return + } + + // Emit discovery snapshot downstream + snapshotID := fmt.Sprintf("%s-%s-%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name, uuid.NewString()) + if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.loaderCfg.ChunkSize); err != nil { + logger.Error( + err, + "Failed to send discovery snapshot", + "snapshotID", snapshotID, + "targets", len(targets), + ) + return + } + + logger.Info( + "Discovery snapshot sent", + "snapshotID", snapshotID, + "targets", len(targets), + ) + } + + // Immediate fetch on startup + fetchAndEmit() + + // Periodic fetch for { select { case <-ctx.Done(): @@ -51,78 +151,242 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er return nil case <-ticker.C: - // Switch case + i only needed to test behavior for messages with different values. - switch i { - case 1: - snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString()) - targets := []core.DiscoveredTarget{ - { - Name: "spine1", - Address: "clab-t1-spine1", - Port: 57400, - Labels: map[string]string{}, - }, - { - Name: "leaf1", - Address: "clab-leaf1", - Port: 57400, - Labels: map[string]string{}, - }, - } - - if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.commonCfg.ChunkSize); err != nil { - return err - } - case 2: - snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString()) - targets := []core.DiscoveredTarget{ - { - Name: "spine1", - Address: "clab-t1-spine1", - Port: 57400, - Labels: map[string]string{}, - }, - { - Name: "leaf2", - Address: "clab-t1-leaf2", - Port: 57400, - Labels: map[string]string{}, - }, - } + fetchAndEmit() + } + } +} - if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.commonCfg.ChunkSize); err != nil { - return err - } +// buildHTTPClient constructs an HTTP client with optional configuration +func (l *Loader) buildHTTPClient(ctx context.Context) (*http.Client, error) { + if l.spec.Timeout == nil { + return nil, fmt.Errorf("timeout must be configured") + } + timeout := l.spec.Timeout.Duration + transport := &http.Transport{} + // If TLS is configured, build TLS config (may include CA bundle). + if l.spec.TLS != nil { + tlsConfig, err := l.buildTLSConfig(ctx) + if err != nil { + return nil, err + } + transport.TLSClientConfig = tlsConfig + } - default: - snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString()) - targets := []core.DiscoveredTarget{ - { - Name: "spine1", - Address: "clab-t1-spine1", - Port: 57400, - Labels: map[string]string{}, - }, - { - Name: "leaf1", - Address: "clab-t1-leaf1", - Port: 57400, - Labels: map[string]string{}, - }, - { - Name: "leaf2", - Address: "clab-t1-leaf2", - Port: 57400, - Labels: map[string]string{}, - }, - } + // Build the HTTP client with the specified timeout and TLS config + client := &http.Client{ + Timeout: timeout, + Transport: transport, + } + return client, nil +} - if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.commonCfg.ChunkSize); err != nil { - return err - } - } +// buildTLSConfig constructs a tls.Config according to the loader spec, +// fetching and parsing a CA bundle if requested. +func (l *Loader) buildTLSConfig(ctx context.Context) (*tls.Config, error) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: l.spec.TLS.InsecureSkipVerify, + } - i++ + if l.spec.TLS.CABundleRef == nil { + return tlsConfig, nil + } + + if l.loaderCfg.ResourceFetcher == nil { + return nil, fmt.Errorf("resource fetcher is not configured") + } + + ref := l.spec.TLS.CABundleRef + if ref.Name == "" || ref.Key == "" { + return nil, fmt.Errorf("CABundleRef must specify both name and key") + } + + caPEM, err := l.loaderCfg.ResourceFetcher.GetConfigMapKey(ctx, l.loaderCfg.TargetsourceNN.Namespace, l.spec.TLS.CABundleRef) + if err != nil { + return nil, fmt.Errorf("failed to fetch CA bundle from config map ref: %w", err) + } + + certPool := x509.NewCertPool() + if ok := certPool.AppendCertsFromPEM([]byte(caPEM)); !ok { + return nil, fmt.Errorf("failed to parse CA bundle PEM") + } + tlsConfig.RootCAs = certPool + + return tlsConfig, nil +} + +// fetchTargetsFromHTTPEndpoint retrieves targets from the configured HTTP endpoint +func (l *Loader) fetchTargetsFromHTTPEndpoint( + ctx context.Context, + client *http.Client, + logger logr.Logger, +) ([]core.DiscoveredTarget, error) { + var allTargets []core.DiscoveredTarget + currentURL := l.spec.URL + + seen := make(map[string]struct{}) + + for { + if _, exists := seen[currentURL]; exists { + logger.Error(fmt.Errorf("pagination loop detected"), "stopping pagination", "url", currentURL) + break + } + seen[currentURL] = struct{}{} + + raw, headers, err := l.fetchPage(ctx, client, currentURL) + if err != nil { + return allTargets, err // do not silently drop pages + } + + // Extract targets + if targets, err := l.extractTargetsFromResponse(raw, logger); err != nil { + logger.Error(err, "Failed to extract targets", "url", currentURL) + } else { + allTargets = append(allTargets, targets...) } + + // Pagination: next page + nextURL, stop := l.getNextURL(raw, headers, currentURL, logger) + if stop { + break + } + currentURL = nextURL + } + + return allTargets, nil +} + +// fetchPage performs an HTTP GET request to the specified URL and decodes the JSON response +// and returns the raw response +func (l *Loader) fetchPage( + ctx context.Context, + client *http.Client, + url string, +) (any, http.Header, error) { + + method := l.spec.Method + if method == "" { + return nil, nil, fmt.Errorf("method must be configured") + } + + // Build request body (only for POST) + var bodyReader *bytes.Reader + if method == http.MethodPost && l.spec.Body != "" { + bodyReader = bytes.NewReader([]byte(l.spec.Body)) + } else { + bodyReader = bytes.NewReader(nil) + } + + // Build HTTP request + req, err := http.NewRequestWithContext(ctx, method, url, bodyReader) + if err != nil { + return nil, nil, fmt.Errorf("creating HTTP request failed: %w", err) + } + + req.Header.Set("Accept", "application/json") + // Apply user-defined headers + for key, val := range l.spec.Headers { + req.Header.Set(key, val) + } + + if err := l.applyAuthentication(req); err != nil { + return nil, nil, err + } + + // Execute HTTP request + resp, err := client.Do(req) + if err != nil { + return nil, nil, err } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, resp.Header, fmt.Errorf("unexpected HTTP status: %d", resp.StatusCode) + } + + // Decode HTTP response + var raw any + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + return nil, resp.Header, err + } + + return raw, resp.Header, nil +} + +// extractTargetsFromResponse extracts items from the response and maps each item into a DiscoveredTarget +func (l *Loader) extractTargetsFromResponse(raw any, logger logr.Logger) ([]core.DiscoveredTarget, error) { + var items []any + // If ResponseMapping is configured and TargetsField is provided we treat + // it as a CEL expression that evaluates against the whole response and + // must return an array of items. + if l.spec.ResponseMapping != nil && l.spec.ResponseMapping.TargetsField != "" { + prog, err := compileCEL(l.spec.ResponseMapping.TargetsField) + if err != nil { + return nil, fmt.Errorf("invalid TargetsField CEL expression: %w", err) + } + out, _, err := prog.Eval(map[string]any{"self": raw}) + if err != nil { + return nil, fmt.Errorf("evaluating TargetsField CEL expression failed: %w", err) + } + if out == nil { + return nil, fmt.Errorf("TargetsField expression returned nil") + } + array, ok := out.Value().([]any) + if !ok { + return nil, fmt.Errorf("invalid HTTP response: targetsField expression must evaluate to an array of objects") + } + items = array + } else { + //If TargetsField is empty, the raw response is expected to be an array of items. + array, ok := raw.([]any) + if !ok { + return nil, fmt.Errorf("invalid HTTP response: expected a JSON array when itemsField is not set") + } + items = array + } + + // Map items to targets + var targets []core.DiscoveredTarget + targets, err := l.mapItemsToTargets(items, raw, logger) + if err != nil { + return nil, fmt.Errorf("mapping items to targets failed: %w", err) + } + + return targets, nil +} + +// getNextURL determines the next page URL +// Returns: +// - nextURL: next request +// - stop: whether to terminate loop +func (l *Loader) getNextURL( + raw any, + headers http.Header, + currentURL string, + logger logr.Logger, +) (string, bool) { + // Extract pagination info + // Link header + if next := extractNextFromLinkHeader(headers); next != "" { + return next, false + } + + // Body + nextPage, err := l.extractNextPageInfo(raw) + if err != nil { + logger.Error(err, "pagination extraction failed") + return "", true + } + + if nextPage == "" { + return "", true + } + + // Build next page URL + nextURL, err := l.buildNextURL(currentURL, nextPage) + if err != nil { + logger.Error(err, "failed to build next URL") + return "", true + } + + return nextURL, false } diff --git a/internal/controller/discovery/loaders/http/loader_test.go b/internal/controller/discovery/loaders/http/loader_test.go index d02cfda6..2a42e1b2 100644 --- a/internal/controller/discovery/loaders/http/loader_test.go +++ b/internal/controller/discovery/loaders/http/loader_test.go @@ -1 +1,204 @@ package http + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" +) + +func TestBuildHTTPClientCases(t *testing.T) { + caPEM, err := genSelfSignedCertPEM() + if err != nil { + t.Fatalf("failed to generate CA PEM: %v", err) + } + + tests := []struct { + name string + spec gnmicv1alpha1.HTTPConfig + fetcher core.ResourceFetcher + expectsErr bool + }{ + { + name: "valid_CABundle", + spec: gnmicv1alpha1.HTTPConfig{ + TLS: &gnmicv1alpha1.ClientTLSConfig{ + CABundleRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "test-ca"}, + Key: "ca.crt", + }, + }, + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + fetcher: fakeResourceFetcher{configuration: caPEM}, + expectsErr: false, + }, + { + name: "invalid_CABundle_PEM", + spec: gnmicv1alpha1.HTTPConfig{ + TLS: &gnmicv1alpha1.ClientTLSConfig{CABundleRef: &corev1.ConfigMapKeySelector{}}, + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + fetcher: fakeResourceFetcher{configuration: "not-pem"}, + expectsErr: true, + }, + { + name: "CABundle_without_fetcher", + spec: gnmicv1alpha1.HTTPConfig{TLS: &gnmicv1alpha1.ClientTLSConfig{CABundleRef: &corev1.ConfigMapKeySelector{}}, Timeout: &metav1.Duration{Duration: 10 * time.Second}}, + fetcher: nil, + expectsErr: true, + }, + { + name: "timeout_missing", + spec: gnmicv1alpha1.HTTPConfig{}, + fetcher: nil, + expectsErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + loader := makeLoader(tc.spec, tc.fetcher) + client, err := loader.buildHTTPClient(context.Background()) + if tc.expectsErr { + if err == nil { + t.Fatalf("%s: expected error, got nil", tc.name) + } + return + } + if err != nil { + t.Fatalf("%s: unexpected error: %v", tc.name, err) + } + if client == nil { + t.Fatalf("%s: expected client, got nil", tc.name) + } + }) + } +} + +func TestFetchPageErrorsAndJSON(t *testing.T) { + loader := &Loader{ + loaderCfg: core.CommonLoaderConfig{TargetsourceNN: types.NamespacedName{Namespace: "default", Name: "test"}}, + spec: gnmicv1alpha1.HTTPConfig{Timeout: &metav1.Duration{Duration: 10 * time.Second}}, + } + + // method missing + if _, _, err := loader.fetchPage(context.Background(), nil, "http://example.com"); err == nil { + t.Fatalf("expected method configuration error") + } + + // non-200 and invalid JSON + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("boom")) + })) + defer server.Close() + + loader = makeLoader(gnmicv1alpha1.HTTPConfig{ + Method: http.MethodGet, + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, nil) + + client := mustBuildClient(t, loader) + + // non-200 + if _, _, err := loader.fetchPage(context.Background(), client, server.URL); err == nil { + t.Fatalf("expected status code error") + } + + // invalid JSON + server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("not-json")) + }) + + if _, _, err := loader.fetchPage(context.Background(), client, server.URL); err == nil { + t.Fatalf("expected JSON decode error") + } +} + +func TestFetchPagePOSTAndHeaders(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // validate method and headers/body + if r.Method != http.MethodPost { + t.Fatalf("expected POST, got %s", r.Method) + } + if r.Header.Get("X-Custom") != "value" { + t.Fatalf("missing header") + } + + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("body decode failed: %v", err) + } + + json.NewEncoder(w).Encode(map[string]any{"name": "target1"}) + })) + defer server.Close() + + spec := gnmicv1alpha1.HTTPConfig{ + URL: server.URL, + Method: http.MethodPost, + Headers: map[string]string{"X-Custom": "value"}, + Body: `{"query":"status"}`, + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + } + + loader := makeLoader(spec, nil) + client := mustBuildClient(t, loader) + + raw, headers, err := loader.fetchPage(context.Background(), client, server.URL) + if err != nil { + t.Fatalf("fetchPage failed: %v", err) + } + + if headers == nil { + t.Fatalf("expected headers, got nil") + } + + resp, ok := raw.(map[string]any) + if !ok || resp["name"] != "target1" { + t.Fatalf("unexpected response: %#v", raw) + } +} + +func TestRunEmitsSnapshotOnImmediateFetch(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode([]any{map[string]any{"name": "t1", "address": "1.1.1.1", "port": float64(830)}}) + })) + defer server.Close() + + spec := gnmicv1alpha1.HTTPConfig{URL: server.URL, Method: http.MethodGet, Timeout: &metav1.Duration{Duration: 10 * time.Second}, Interval: &metav1.Duration{Duration: time.Hour}} + loader := makeLoader(spec, nil) + + _, cancel, out, done := startLoaderRun(loader) + defer cancel() + + select { + case msgs := <-out: + if len(msgs) == 0 { + t.Fatalf("expected discovery messages") + } + cancel() + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for Run to emit snapshot") + } + + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for Run to return") + } +} diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go new file mode 100644 index 00000000..4bd34585 --- /dev/null +++ b/internal/controller/discovery/loaders/http/mapper.go @@ -0,0 +1,329 @@ +package http + +import ( + "fmt" + "math" + "reflect" + "strconv" + + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/go-logr/logr" + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/ext" +) + +// mapItemsToTargets converts a list of raw JSON items into DiscoveredTargets using the configured mapping rules +func (l *Loader) mapItemsToTargets(items []any, full any, logger logr.Logger) ([]core.DiscoveredTarget, error) { + // Compile CEL expressions once for efficiency + compiled, err := l.compileMapping() + if err != nil { + return nil, fmt.Errorf("compile mapping: %w", err) + } + + // Map items to targets + targets := make([]core.DiscoveredTarget, 0, len(items)) + for _, item := range items { + obj, ok := item.(map[string]any) + if !ok { + logger.Error(fmt.Errorf("invalid target format"), + "failed to convert target to map", + "item", item, + ) + continue + } + target, err := l.mapItemToTarget(obj, full, compiled) + if err != nil { + logger.Error(err, + "failed to map target", + "item", obj, + ) + continue + } + + targets = append(targets, target) + } + + return targets, nil +} + +type compiledMapping struct { + name cel.Program + address cel.Program + port cel.Program + + targetProfile cel.Program + labels cel.Program +} + +func (l *Loader) compileMapping() (*compiledMapping, error) { + rm := l.spec.ResponseMapping + cm := &compiledMapping{} + if rm == nil { + return cm, nil + } + + var err error + if rm.Name != "" { + cm.name, err = compileCEL(rm.Name) + if err != nil { + return nil, fmt.Errorf("name: %w", err) + } + } + if rm.Address != "" { + cm.address, err = compileCEL(rm.Address) + if err != nil { + return nil, fmt.Errorf("address: %w", err) + } + } + if rm.Port != "" { + cm.port, err = compileCEL(rm.Port) + if err != nil { + return nil, fmt.Errorf("port: %w", err) + } + } + if rm.TargetProfile != "" { + cm.targetProfile, err = compileCEL(rm.TargetProfile) + if err != nil { + return nil, fmt.Errorf("targetProfile: %w", err) + } + } + if rm.Labels != "" { + cm.labels, err = compileCEL(rm.Labels) + if err != nil { + return nil, fmt.Errorf("labels: %w", err) + } + } + + return cm, nil +} + +// mapItemToTarget converts a raw JSON object into a DiscoveredTarget +func (l *Loader) mapItemToTarget(item map[string]any, full any, cm *compiledMapping) (core.DiscoveredTarget, error) { + name, err := l.getName(item, full, cm) + if err != nil { + return core.DiscoveredTarget{}, err + } + + address, err := l.getAddress(item, full, cm) + if err != nil { + return core.DiscoveredTarget{}, err + } + + return core.DiscoveredTarget{ + Name: name, + Address: address, + Port: l.getPort(item, full, cm), + Labels: l.getLabels(item, full, cm), + TargetProfile: l.getTargetProfile(item, full, cm), + }, nil +} + +// getName extracts the target name from the item using the compiled CEL expression if provided, +// otherwise it falls back to the default "name" field +func (l *Loader) getName(item map[string]any, full any, cm *compiledMapping) (string, error) { + if cm.name != nil { + val, err := evalCEL(cm.name, item, full) + if err != nil { + return "", err + } + + str, ok := val.(string) + if !ok || str == "" { + return "", fmt.Errorf("name must be non-empty string") + } + return str, nil + } + + val, ok := item["name"].(string) + if !ok || val == "" { + return "", fmt.Errorf("name must be non-empty string") + } + return val, nil +} + +// getAddress extracts the target address from the item using the compiled CEL expression if provided, +// otherwise it falls back to the default "address" field +func (l *Loader) getAddress(item map[string]any, full any, cm *compiledMapping) (string, error) { + if cm.address != nil { + val, err := evalCEL(cm.address, item, full) + if err != nil { + return "", err + } + + str, ok := val.(string) + if !ok || str == "" { + return "", fmt.Errorf("address must be non-empty string") + } + return str, nil + } + + val, ok := item["address"].(string) + if !ok || val == "" { + return "", fmt.Errorf("address must be non-empty string") + } + return val, nil +} + +// getPort extracts the target port from the item using the compiled CEL expression if provided, +// otherwise it falls back to the default "port" field +func (l *Loader) getPort(item map[string]any, full any, cm *compiledMapping) int32 { + if cm.port != nil { + val, err := evalCEL(cm.port, item, full) + if err == nil { + return extractPort(val) + } + return 0 + } + + return extractPort(item["port"]) +} + +// getLabels extracts the target labels from the item using the compiled CEL expressions if provided, +// otherwise it falls back to the default "labels" field +func (l *Loader) getLabels(item map[string]any, full any, cm *compiledMapping) map[string]string { + result := make(map[string]string) + + if cm != nil && cm.labels != nil { + val, err := evalCEL(cm.labels, item, full) + if err != nil { + return result + } + m, ok := val.(map[string]any) + if !ok { + return result + } + for k, v := range m { + result[k] = fmt.Sprintf("%v", v) + } + } + + // fallback: direct + if raw, ok := item["labels"].(map[string]any); ok { + for key, val := range raw { + result[key] = fmt.Sprintf("%v", val) + } + } + return result +} + +// getTargetProfile extracts the target profile from the item using the compiled CEL expression if provided, +// otherwise it falls back to the default "targetProfile" field +func (l *Loader) getTargetProfile(item map[string]any, full any, cm *compiledMapping) string { + if cm.targetProfile != nil { + val, err := evalCEL(cm.targetProfile, item, full) + if err == nil { + if str, ok := val.(string); ok { + return str + } + } + return "" + } + + if val, ok := item["targetProfile"].(string); ok { + return val + } + return "" +} + +var celEnv = mustNewEnv() + +// mustNewEnv creates a CEL environment with the necessary variable declarations for evaluating expressions +func mustNewEnv() *cel.Env { + env, err := cel.NewEnv( + cel.Variable("self", cel.DynType), + cel.Variable("item", cel.DynType), + // Required for ext.Regex + cel.OptionalTypes(), + // Include standard CEL declarations for common operations and types + ext.Strings(), + ext.Math(), + ext.Lists(), + ext.Sets(), + ext.Regex(), + ext.Bindings(), + ) + if err != nil { + panic(err) + } + return env +} + +// compileCEL compiles a CEL expression into a program that can be evaluated against items +func compileCEL(expr string) (cel.Program, error) { + ast, issues := celEnv.Compile(expr) + if issues != nil && issues.Err() != nil { + return nil, issues.Err() + } + return celEnv.Program(ast, cel.EvalOptions(cel.OptOptimize)) +} + +// evalCEL evaluates a compiled CEL program against an item +func evalCEL(p cel.Program, item map[string]any, full any) (any, error) { + out, _, err := p.Eval(map[string]any{ + "self": full, + "item": item, + }) + if err != nil { + return nil, err + } + if out == nil { + return nil, fmt.Errorf("CEL returned nil") + } + + return normalizeCEL(out.Value()), nil +} + +// normalizeCEL recursively converts CEL evaluation results into standard Go types +func normalizeCEL(v any) any { + switch raw := v.(type) { + case ref.Val: + v := raw.Value() + if v == nil { + return nil + } + return normalizeCEL(v) + + case []any: + for i := range raw { + raw[i] = normalizeCEL(raw[i]) + } + return raw + } + + // For maps, keys are converted to strings + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.Map { + out := make(map[string]any) + for _, key := range rv.MapKeys() { + k := fmt.Sprintf("%v", normalizeCEL(key.Interface())) + val := normalizeCEL(rv.MapIndex(key).Interface()) + out[k] = val + } + return out + } + + return v +} + +// extractPort converts a CEL evaluation result into an int32 port number, +// handling both numeric and string representations +func extractPort(val any) int32 { + switch v := val.(type) { + case float64: + if v < 0 || v > math.MaxInt32 { + return 0 + } + return int32(v) + + case string: + p, err := strconv.ParseInt(v, 10, 32) + if err != nil { + return 0 + } + return int32(p) + + default: + return 0 + } +} diff --git a/internal/controller/discovery/loaders/http/mapping_test.go b/internal/controller/discovery/loaders/http/mapping_test.go new file mode 100644 index 00000000..2ba1623f --- /dev/null +++ b/internal/controller/discovery/loaders/http/mapping_test.go @@ -0,0 +1,156 @@ +package http + +import ( + "testing" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/go-logr/logr" +) + +func TestExtractTargetsAndMapping(t *testing.T) { + tests := []struct { + name string + config gnmicv1alpha1.HTTPConfig + raw any + validate func(t *testing.T, targets []core.DiscoveredTarget) + }{ + { + name: "direct mapping all fields", + config: gnmicv1alpha1.HTTPConfig{}, + raw: []any{map[string]any{"name": "t1", "address": "1.1.1.1", "port": "9000", "labels": map[string]any{"env": "prod", "region": "us-east"}, "targetProfile": "edge-profile"}}, + validate: func(t *testing.T, targets []core.DiscoveredTarget) { + if len(targets) != 1 { + t.Fatalf("direct mapping: expected 1 target, got %d", len(targets)) + } + tgt := targets[0] + if tgt.Name != "t1" { + t.Fatalf("direct mapping Name failed: got %q", tgt.Name) + } + if tgt.Address != "1.1.1.1" { + t.Fatalf("direct mapping Address failed: got %q", tgt.Address) + } + if tgt.Port != 9000 { + t.Fatalf("direct mapping Port failed: got %d", tgt.Port) + } + if tgt.Labels["env"] != "prod" || tgt.Labels["region"] != "us-east" { + t.Fatalf("direct mapping Labels failed: %#v", tgt.Labels) + } + if tgt.TargetProfile != "edge-profile" { + t.Fatalf("direct mapping TargetProfile failed: got %q", tgt.TargetProfile) + } + }, + }, + { + name: "CEL TargetsField extraction", + config: gnmicv1alpha1.HTTPConfig{ResponseMapping: &gnmicv1alpha1.ResponseMappingSpec{TargetsField: "self.results"}}, + raw: map[string]any{"results": []any{map[string]any{"name": "t1", "address": "1.1.1.1", "port": float64(22)}}}, + validate: func(t *testing.T, targets []core.DiscoveredTarget) { + if len(targets) != 1 { + t.Fatalf("TargetsField extraction failed: got %d targets", len(targets)) + } + }, + }, + { + name: "CEL Name mapping", + config: gnmicv1alpha1.HTTPConfig{ResponseMapping: &gnmicv1alpha1.ResponseMappingSpec{Name: "item.hostname"}}, + raw: []any{map[string]any{"hostname": "host-1", "address": "10.0.0.1", "port": float64(830)}}, + validate: func(t *testing.T, targets []core.DiscoveredTarget) { + if len(targets) != 1 { + t.Fatalf("Name mapping: expected 1 target, got %d", len(targets)) + } + if targets[0].Name != "host-1" { + t.Fatalf("Name mapping failed: got %q", targets[0].Name) + } + }, + }, + { + name: "CEL Address mapping", + config: gnmicv1alpha1.HTTPConfig{ResponseMapping: &gnmicv1alpha1.ResponseMappingSpec{Address: "item.ip"}}, + raw: []any{map[string]any{"name": "t1", "ip": "192.168.1.1", "port": float64(830)}}, + validate: func(t *testing.T, targets []core.DiscoveredTarget) { + if len(targets) != 1 { + t.Fatalf("Address mapping: expected 1 target, got %d", len(targets)) + } + if targets[0].Address != "192.168.1.1" { + t.Fatalf("Address mapping failed: got %q", targets[0].Address) + } + }, + }, + { + name: "CEL Port mapping", + config: gnmicv1alpha1.HTTPConfig{ResponseMapping: &gnmicv1alpha1.ResponseMappingSpec{Port: "item.mgmt_port"}}, + raw: []any{map[string]any{"name": "t1", "address": "10.0.0.1", "mgmt_port": float64(9000)}}, + validate: func(t *testing.T, targets []core.DiscoveredTarget) { + if len(targets) != 1 { + t.Fatalf("Port mapping: expected 1 target, got %d", len(targets)) + } + if targets[0].Port != 9000 { + t.Fatalf("Port mapping failed: got %d", targets[0].Port) + } + }, + }, + { + name: "CEL Labels mapping", + config: gnmicv1alpha1.HTTPConfig{ResponseMapping: &gnmicv1alpha1.ResponseMappingSpec{Labels: `{"env": item.environment, "type": item.device_type}`}}, + raw: []any{map[string]any{"name": "t1", "address": "10.0.0.1", "port": float64(830), "environment": "prod", "device_type": "router"}}, + validate: func(t *testing.T, targets []core.DiscoveredTarget) { + if len(targets) != 1 { + t.Fatalf("Labels mapping: expected 1 target, got %d", len(targets)) + } + if targets[0].Labels["env"] != "prod" || targets[0].Labels["type"] != "router" { + t.Fatalf("Labels mapping failed: %#v", targets[0].Labels) + } + }, + }, + { + name: "CEL TargetProfile mapping", + config: gnmicv1alpha1.HTTPConfig{ResponseMapping: &gnmicv1alpha1.ResponseMappingSpec{TargetProfile: `item.type == "edge" ? "edge-profile" : "default"`}}, + raw: []any{map[string]any{"name": "t1", "address": "10.0.0.1", "port": float64(830), "type": "edge"}}, + validate: func(t *testing.T, targets []core.DiscoveredTarget) { + if len(targets) != 1 { + t.Fatalf("TargetProfile mapping: expected 1 target, got %d", len(targets)) + } + if targets[0].TargetProfile != "edge-profile" { + t.Fatalf("TargetProfile mapping failed: got %q", targets[0].TargetProfile) + } + }, + }, + { + name: "CEL all mapping options combined", + config: gnmicv1alpha1.HTTPConfig{ResponseMapping: &gnmicv1alpha1.ResponseMappingSpec{TargetsField: "self.results", Name: "item.hostname", Address: "item.ip", Port: "item.port", Labels: `{"env": item.env}`, TargetProfile: `item.type == "edge" ? "edge-profile" : "default"`}}, + raw: map[string]any{"results": []any{map[string]any{"hostname": "host-1", "ip": "10.0.0.1", "port": float64(830), "env": "prod", "type": "edge"}}}, + validate: func(t *testing.T, targets []core.DiscoveredTarget) { + if len(targets) != 1 { + t.Fatalf("combined mapping: expected 1 target, got %d", len(targets)) + } + tgt := targets[0] + if tgt.Name != "host-1" || tgt.Address != "10.0.0.1" || tgt.Port != 830 || tgt.Labels["env"] != "prod" || tgt.TargetProfile != "edge-profile" { + t.Fatalf("combined mapping failed: %#v", tgt) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + loader := makeLoader(tt.config, nil) + targets, err := loader.extractTargetsFromResponse(tt.raw, logr.Discard()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tt.validate(t, targets) + }) + } +} + +func TestMapItemsToTargetsSkipsInvalidItems(t *testing.T) { + loader := makeLoader(gnmicv1alpha1.HTTPConfig{}, nil) + tgts, err := loader.mapItemsToTargets([]any{"not-a-map", map[string]any{"name": "n", "address": "a"}}, nil, logr.Discard()) + if err != nil { + t.Fatalf("mapItemsToTargets failed: %v", err) + } + if len(tgts) != 1 || tgts[0].Name != "n" { + t.Fatalf("unexpected targets: %#v", tgts) + } +} diff --git a/internal/controller/discovery/loaders/http/pagination.go b/internal/controller/discovery/loaders/http/pagination.go new file mode 100644 index 00000000..fc4913e5 --- /dev/null +++ b/internal/controller/discovery/loaders/http/pagination.go @@ -0,0 +1,77 @@ +package http + +import ( + "fmt" + "net/http" + "net/url" + "strings" +) + +// extractNextPageInfo extracts pagination information from a response +func (l *Loader) extractNextPageInfo(raw any) (string, error) { + if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" { + return "", nil + } + + // Extract next value + prog, err := compileCEL(l.spec.Pagination.NextField) + if err != nil { + return "", fmt.Errorf("invalid NextField CEL: %w", err) + } + out, _, err := prog.Eval(map[string]any{"self": raw}) + if err != nil { + return "", fmt.Errorf("CEL eval failed: %w", err) + } + if out == nil || out.Value() == nil { + return "", nil + } + + str, ok := out.Value().(string) + if !ok { + return "", fmt.Errorf("NextField must evaluate to string") + } + + return str, nil +} + +// Link header parsing +func extractNextFromLinkHeader(h http.Header) string { + link := h.Get("Link") + if link == "" { + return "" + } + + parts := strings.Split(link, ",") + for _, p := range parts { + if strings.Contains(p, `rel="next"`) { + start := strings.Index(p, "<") + end := strings.Index(p, ">") + if start != -1 && end != -1 { + return p[start+1 : end] + } + } + } + return "" +} + +// buildNextURL supports token and full URL +func (l *Loader) buildNextURL(currentURL, nextVal string) (string, error) { + if parsed, err := url.Parse(nextVal); err == nil && parsed.Scheme != "" { + return nextVal, nil // full URL + } + + if l.spec.Pagination.RequestParam == "" { + return "", fmt.Errorf("requestParam must be set for token pagination") + } + + parsedURL, err := url.Parse(currentURL) + if err != nil { + return "", err + } + + q := parsedURL.Query() + q.Set(l.spec.Pagination.RequestParam, nextVal) + parsedURL.RawQuery = q.Encode() + + return parsedURL.String(), nil +} diff --git a/internal/controller/discovery/loaders/http/pagination_test.go b/internal/controller/discovery/loaders/http/pagination_test.go new file mode 100644 index 00000000..707518d5 --- /dev/null +++ b/internal/controller/discovery/loaders/http/pagination_test.go @@ -0,0 +1,112 @@ +package http + +import ( + "net/http" + "strings" + "testing" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" +) + +func TestPaginationHelpersAndNextURL(t *testing.T) { + loader := makeLoader( + gnmicv1alpha1.HTTPConfig{ + Pagination: &gnmicv1alpha1.PaginationSpec{ + NextField: "self.next", + RequestParam: "next", + }, + }, + nil, + ) + + next, err := loader.extractNextPageInfo(map[string]any{"next": "token"}) + if err != nil || next != "token" { + t.Fatalf("extractNextPageInfo failed: %v", err) + } + + nextURL, err := loader.buildNextURL("https://example.com/path", "token") + if err != nil || !strings.Contains(nextURL, "next=token") { + t.Fatalf("buildNextURL failed: %v, %s", err, nextURL) + } + + nextURL, err = loader.buildNextURL("https://example.com/path", "https://example.com/other") + if err != nil || nextURL != "https://example.com/other" { + t.Fatalf("buildNextURL absolute failed: %v, %s", err, nextURL) + } +} + +func TestPagination_ArrayNoPagination(t *testing.T) { + raw := []any{ + map[string]any{"name": "a"}, + } + + loader := &Loader{ + spec: gnmicv1alpha1.HTTPConfig{}, + } + + next, err := loader.extractNextPageInfo(raw) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if next != "" { + t.Fatalf("expected empty next, got %s", next) + } +} + +func TestPagination_NextURL(t *testing.T) { + raw := map[string]any{ + "next": "http://example.com/page2", + } + + loader := &Loader{ + spec: gnmicv1alpha1.HTTPConfig{ + Pagination: &gnmicv1alpha1.PaginationSpec{ + NextField: "self.next", + }, + }, + } + + next, err := loader.extractNextPageInfo(raw) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if next != "http://example.com/page2" { + t.Fatalf("unexpected next: %s", next) + } +} + +func TestPagination_Token(t *testing.T) { + raw := map[string]any{ + "next_page_token": "abc", + } + + loader := &Loader{ + spec: gnmicv1alpha1.HTTPConfig{ + Pagination: &gnmicv1alpha1.PaginationSpec{ + NextField: "self.next_page_token", + RequestParam: "page_token", + }, + }, + } + + next, err := loader.extractNextPageInfo(raw) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if next != "abc" { + t.Fatalf("unexpected token: %s", next) + } +} + +func TestPagination_LinkHeader(t *testing.T) { + headers := http.Header{} + headers.Set("Link", `; rel="next"`) + + next := extractNextFromLinkHeader(headers) + + if next != "http://example.com/page2" { + t.Fatalf("unexpected next link: %s", next) + } +} diff --git a/internal/controller/discovery/loaders/utils/endpoint.go b/internal/controller/discovery/loaders/utils/endpoint.go new file mode 100644 index 00000000..ef83f18c --- /dev/null +++ b/internal/controller/discovery/loaders/utils/endpoint.go @@ -0,0 +1,22 @@ +package utils + +import ( + "fmt" + + "github.com/gin-gonic/gin" + "k8s.io/apimachinery/pkg/types" +) + +func CreateTargetsPath( + router *gin.Engine, + nn types.NamespacedName, + handler gin.HandlerFunc, +) { + path := fmt.Sprintf( + "/api/v1/%s/target-source/%s/createTargets", + nn.Namespace, + nn.Name, + ) + + router.POST(path, handler) +} diff --git a/internal/controller/discovery/loaders/utils/send_test.go b/internal/controller/discovery/loaders/utils/send_test.go new file mode 100644 index 00000000..78d1173a --- /dev/null +++ b/internal/controller/discovery/loaders/utils/send_test.go @@ -0,0 +1,81 @@ +package utils + +import ( + "context" + "fmt" + "testing" + + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/go-openapi/testify/v2/require" +) + +// mockChannel returns a buffered channel and a cancelable context +func mockChannel(bufferSize int) (chan []core.DiscoveryMessage, context.Context, context.CancelFunc) { + ch := make(chan []core.DiscoveryMessage, bufferSize) + ctx, cancel := context.WithCancel(context.Background()) + return ch, ctx, cancel +} + +// mockTargets returns a slice of fake DiscoveredTarget objects +func mockTargets(n int) []core.DiscoveredTarget { + targets := make([]core.DiscoveredTarget, n) + for i := range n { + targets[i] = core.DiscoveredTarget{ + Name: fmt.Sprintf("target-%d", i), + } + } + return targets +} + +// mockEvents returns a slice of fake DiscoveryEvent objects +func mockEvents(n int) []core.DiscoveryEvent { + events := make([]core.DiscoveryEvent, n) + for i := 0; i < n; i++ { + events[i] = core.DiscoveryEvent{ + Event: core.EventApply, + Target: core.DiscoveredTarget{ + Name: fmt.Sprintf("event-target-%d", i), + }, + } + } + return events +} + +// drainChannel reads all messages from a channel until it's empty +func drainChannel(ch chan []core.DiscoveryMessage) [][]core.DiscoveryMessage { + var out [][]core.DiscoveryMessage + for { + select { + case msgs := <-ch: + out = append(out, msgs) + default: + return out + } + } +} + +func TestSendSnapshot_Basic(t *testing.T) { + ch, ctx, cancel := mockChannel(5) + defer cancel() + + targets := mockTargets(3) + + err := SendSnapshot(ctx, ch, targets, "snap-1", 2) + require.NoError(t, err) + + msgs := drainChannel(ch) + require.Len(t, msgs, 2) // 2 chunks for 3 targets with chunkSize=2 +} + +func TestSendEvents_Basic(t *testing.T) { + ch, ctx, cancel := mockChannel(5) + defer cancel() + + events := mockEvents(4) + + err := SendEvents(ctx, ch, events, 2) + require.NoError(t, err) + + msgs := drainChannel(ch) + require.Len(t, msgs, 2) // 2 chunks for 4 events with chunkSize=2 +} diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go index ffaad401..79f1b92d 100644 --- a/internal/controller/discovery/mapper_test.go +++ b/internal/controller/discovery/mapper_test.go @@ -1,130 +1,11 @@ package discovery import ( - "fmt" "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" ) -func mockDiscoveredTargetList(len int) []core.DiscoveredTarget { - targets := make([]core.DiscoveredTarget, len) - - if len > 100 { - len = 100 - } - - for i := range len { - targets[i] = core.DiscoveredTarget{ - Address: fmt.Sprintf("192.168.1.%d", i+1), - Name: fmt.Sprintf("router%d", i+1), - } - } - - return targets -} - -func mockDiscoveryTarget(opts ...func(*core.DiscoveredTarget)) core.DiscoveredTarget { - t := core.DiscoveredTarget{ - Name: "target1", - Address: "10.0.0.1", - Labels: map[string]string{}, - } - - for _, opt := range opts { - opt(&t) - } - - return t -} - -func withDiscoveredTargetName(name string) func(*core.DiscoveredTarget) { - return func(t *core.DiscoveredTarget) { - t.Name = name - } -} - -func withDiscoveredTargetAddress(address string) func(*core.DiscoveredTarget) { - return func(t *core.DiscoveredTarget) { - t.Address = address - } -} - -func withDiscoveredTargetLabels(labels map[string]string) func(*core.DiscoveredTarget) { - return func(t *core.DiscoveredTarget) { - t.Labels = labels - } -} - -func mockTargetSource(opts ...func(*gnmicv1alpha1.TargetSource)) gnmicv1alpha1.TargetSource { - ts := gnmicv1alpha1.TargetSource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ts1", - Namespace: "default", - }, - Spec: gnmicv1alpha1.TargetSourceSpec{ - TargetProfile: "default", - TargetLabels: map[string]string{}, - }, - } - - for _, opt := range opts { - opt(&ts) - } - - return ts -} - -func withTargetSourceName(name string) func(*gnmicv1alpha1.TargetSource) { - return func(ts *gnmicv1alpha1.TargetSource) { - ts.ObjectMeta.Name = name - } -} - -func withTargetSourceNamespace(namespace string) func(*gnmicv1alpha1.TargetSource) { - return func(ts *gnmicv1alpha1.TargetSource) { - ts.ObjectMeta.Namespace = namespace - } -} - -func withTargetSourceTargetProfile(profile string) func(*gnmicv1alpha1.TargetSource) { - return func(ts *gnmicv1alpha1.TargetSource) { - ts.Spec.TargetProfile = profile - } -} - -func withTargetSourceTargetLabels(labels map[string]string) func(*gnmicv1alpha1.TargetSource) { - return func(ts *gnmicv1alpha1.TargetSource) { - ts.Spec.TargetLabels = labels - } -} - -func mockGnmicTargetList(len int) []gnmicv1alpha1.Target { - targets := make([]gnmicv1alpha1.Target, len) - - if len > 100 { - len = 100 - } - - for i := range len { - targets[i] = gnmicv1alpha1.Target{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("router%d", i+1), - Namespace: "default", - }, - Spec: gnmicv1alpha1.TargetSpec{ - Address: fmt.Sprintf("192.168.1.%d", i+1), - Profile: "default", - }, - } - } - - return targets -} - func TestGenerateEvents_EmptyLists(t *testing.T) { events := generateEvents( mockGnmicTargetList(0), diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index f573b1bc..5353ba7d 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -4,8 +4,10 @@ import ( "context" "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" @@ -31,15 +33,17 @@ type MessageProcessor struct { // Events are deferred while snapshot is in progress deferredEvents []core.DiscoveryEvent targetCount int32 + updater core.StatusUpdater } // NewMessageProcessor wires a MessageProcessor instance -func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor { +func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage, u core.StatusUpdater) *MessageProcessor { return &MessageProcessor{ client: c, scheme: s, targetSource: ts, in: in, + updater: u, } } @@ -229,16 +233,46 @@ func (m *MessageProcessor) processEvent(ctx context.Context, event core.Discover } // Apply events - err := m.applyEvent(ctx, event, logger) + result, err := m.applyEvent(ctx, event, logger) if err == nil { switch event.Event { case core.EventApply: - m.targetCount++ - m.updateStatus(ctx, logger) + if result == controllerutil.OperationResultCreated { + m.targetCount++ + } + m.updater.UpdateStatus( + ctx, + core.StatusUpdate{ + Conditions: []metav1.Condition{ + { + Type: core.ConditionTypeReady, + Status: metav1.ConditionStatus("True"), + Reason: string(core.ReasonSyncSucceeded), + Message: "Successfully applied target from last update", + }, + }, + TargetsCount: &m.targetCount, + }, + ) case core.EventDelete: m.targetCount-- - m.updateStatus(ctx, logger) + m.updater.UpdateStatus( + ctx, + core.StatusUpdate{ + Conditions: []metav1.Condition{ + { + Type: core.ConditionTypeReady, + Status: metav1.ConditionStatus("True"), + Reason: string(core.ReasonSyncSucceeded), + Message: "Successfully removed target from last update", + }, + }, + TargetsCount: &m.targetCount, + }, + ) } + } else { + // m.updateStatus(ctx, gnmicv1alpha1.SyncStatusError, err) } return err @@ -309,10 +343,58 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot "numOfDelete", nDelete, ) + errCount := 0 for _, e := range events { - m.applyEvent(ctx, e, logger) + _, err = m.applyEvent(ctx, e, logger) + if err != nil { + errCount++ + } + } + if errCount != 0 { + m.targetCount = int32(len(allTargets)) - int32(errCount) + m.updater.UpdateStatus( + ctx, + core.StatusUpdate{ + Conditions: []metav1.Condition{ + { + Type: core.ConditionTypeDegraded, + Status: metav1.ConditionStatus("True"), + Reason: string(core.ReasonSyncWithErrors), + Message: "Some target changes weren't applied correctly", + }, + }, + TargetsCount: &m.targetCount, + }, + ) + } else { + // Because of idempotency, allTargets = desired state = targets existing in Kubernetes. Overwrites the counter to "reset" it. + m.targetCount = int32(len(allTargets)) + m.updater.UpdateStatus( + ctx, + core.StatusUpdate{ + Conditions: []metav1.Condition{ + { + Type: core.ConditionTypeReady, + Status: metav1.ConditionStatus("True"), + Reason: string(core.ReasonSyncSucceeded), + Message: "Successfully synced all targets", + }, + }, + TargetsCount: &m.targetCount, + }, + ) + } + + if err := m.replayDeferredEvents(ctx, logger); err != nil { + logger.Error(err, "error replaying deferred events") } + m.resetSnapshot() + m.deferredEvents = nil + return nil +} + +func (m *MessageProcessor) replayDeferredEvents(ctx context.Context, logger logr.Logger) error { // Replay deferred events for _, event := range m.deferredEvents { select { @@ -325,53 +407,40 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot } } - // Because of idempotency, allTargets = desired state = targets existing in Kubernetes. Overwrites the counter to "reset" it. - m.targetCount = int32(len(allTargets)) - m.updateStatus(ctx, logger) - - m.resetSnapshot() - m.deferredEvents = nil return nil } -func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +// applyEvent applies a DiscoveryEvent to the Kubernetes cluster and returns controllerutil.OperationResult to identify create or update events. Returns controllerutil.OperationResultNone for delete or on errors +func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) (controllerutil.OperationResult, error) { switch event.Event { case core.EventDelete: if err := deleteTarget(ctx, m.client, event.Target.Name, m.targetSource.Namespace); err != nil { logger.Error(err, "error deleting target", "targetName", event.Target.Name, ) - return err + return controllerutil.OperationResultNone, err } else { logger.Info("deleted target object", "name", event.Target.Name, ) + return controllerutil.OperationResultNone, err } case core.EventApply: target := generateTargetResource(event.Target, m.targetSource) - if err := applyTarget(ctx, m.client, m.scheme, target, m.targetSource); err != nil { + if result, err := applyTarget(ctx, m.client, m.scheme, target, m.targetSource); err != nil { logger.Error(err, "error applying target", "targetName", event.Target.Name, ) - return err + return controllerutil.OperationResultNone, err } else { logger.Info("applied target object", "name", event.Target.Name, ) + return result, nil } - } - - return nil -} - -func (m *MessageProcessor) updateStatus(ctx context.Context, logger logr.Logger) { - if err := updateTargetSourceStatus(ctx, m.client, m.targetSource, m.targetCount); err != nil { - logger.Error(err, "error updating TargetSource status") - } else { - logger.Info("updated target source status", - "targetCount", m.targetCount, - ) + default: + return controllerutil.OperationResultNone, fmt.Errorf("unknown event type %s", event.Event) } } diff --git a/internal/controller/discovery/message_processor_test.go b/internal/controller/discovery/message_processor_test.go new file mode 100644 index 00000000..df42d76c --- /dev/null +++ b/internal/controller/discovery/message_processor_test.go @@ -0,0 +1,260 @@ +package discovery + +import ( + "context" + "testing" + "time" + + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" + discoveryTypes "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/go-logr/logr" + "github.com/go-openapi/testify/v2/require" +) + +func mockMessageProcessor(opts ...func(*MessageProcessor)) *MessageProcessor { + bufferSize := 10 + + scheme := runtime.NewScheme() + + // Register built-in k8s types + _ = clientgoscheme.AddToScheme(scheme) + + // Register your CRDs + _ = gnmicv1alpha1.AddToScheme(scheme) + + client := fake.NewClientBuilder(). + WithScheme(scheme). + Build() + + targetSource := mockTargetSource() + targetChannel := make(chan []discoveryTypes.DiscoveryMessage, bufferSize) + + m := NewMessageProcessor( + client, + scheme, + &targetSource, + targetChannel, + fakeStatusUpdater{}, + ) + + for _, opt := range opts { + opt(m) + } + + return m +} + +func withTargetChannel(ch <-chan []discoveryTypes.DiscoveryMessage) func(*MessageProcessor) { + return func(m *MessageProcessor) { + m.in = ch + } +} + +func TestRun_StopsWhenChannelClosed(t *testing.T) { + ch := make(chan []discoveryTypes.DiscoveryMessage, 10) + + m := mockMessageProcessor( + withTargetChannel(ch), + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + errCh := make(chan error, 1) + + go func() { + errCh <- m.Run(ctx) + }() + + close(ch) + + select { + case err := <-errCh: + require.NoError(t, err) + + case <-time.After(2 * time.Second): + t.Fatal("Run did not exit after channel close") + } +} + +func TestRun_StopsWhenContextCanceled(t *testing.T) { + m := mockMessageProcessor() + + ctx, cancel := context.WithCancel(context.Background()) + + errCh := make(chan error, 1) + + go func() { + errCh <- m.Run(ctx) + }() + + cancel() + + select { + case err := <-errCh: + require.NoError(t, err) + + case <-time.After(2 * time.Second): + t.Fatal("Run did not exit after context cancel") + } +} + +func TestProcessEvent_DefersDuringSnapshot(t *testing.T) { + m := mockMessageProcessor() + + m.activeSnapshot = &snapshotBuffer{ + snapshotID: "snap-1", + totalChunks: 1, + received: map[int][]core.DiscoveredTarget{}, + } + + event := core.DiscoveryEvent{ + Event: core.EventApply, + Target: core.DiscoveredTarget{ + Name: "router-1", + }, + } + + err := m.processEvent( + context.Background(), + event, + logr.Discard(), + ) + + require.NoError(t, err) + + require.Len(t, m.deferredEvents, 1) + require.Equal(t, "router-1", m.deferredEvents[0].Target.Name) +} + +func TestStartNewSnapshot_ResetsDeferredEvents(t *testing.T) { + m := mockMessageProcessor() + + m.deferredEvents = []core.DiscoveryEvent{ + { + Event: core.EventApply, + }, + } + + chunk := core.DiscoverySnapshot{ + SnapshotID: "snap-1", + TotalChunks: 1, + ChunkIndex: 0, + Targets: []core.DiscoveredTarget{ + { + Name: "router-1", + }, + }, + } + + err := m.startNewSnapshot( + context.Background(), + chunk, + logr.Discard(), + ) + + require.NoError(t, err) + + require.Nil(t, m.deferredEvents) + require.Nil(t, m.activeSnapshot) +} + +func TestCollectSnapshot_DuplicateChunkFails(t *testing.T) { + m := mockMessageProcessor() + + m.activeSnapshot = &snapshotBuffer{ + snapshotID: "snap-1", + totalChunks: 2, + received: map[int][]core.DiscoveredTarget{ + 0: {}, + }, + } + + chunk := core.DiscoverySnapshot{ + SnapshotID: "snap-1", + TotalChunks: 2, + ChunkIndex: 0, + } + + err := m.collectSnapshot( + context.Background(), + chunk, + logr.Discard(), + ) + + require.Error(t, err) + require.Contains(t, err.Error(), "duplicate snapshot chunk") + + require.Nil(t, m.activeSnapshot) +} + +func TestCollectSnapshot_InvalidChunkIndexFails(t *testing.T) { + m := mockMessageProcessor() + + m.activeSnapshot = &snapshotBuffer{ + snapshotID: "snap-1", + totalChunks: 1, + received: map[int][]core.DiscoveredTarget{}, + } + + chunk := core.DiscoverySnapshot{ + SnapshotID: "snap-1", + TotalChunks: 1, + ChunkIndex: 99, + } + + err := m.collectSnapshot( + context.Background(), + chunk, + logr.Discard(), + ) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid chunk index") + + require.Nil(t, m.activeSnapshot) +} + +func TestProcessSnapshot_StartsNewSnapshot(t *testing.T) { + mp := mockMessageProcessor() + + chunk := core.DiscoverySnapshot{ + SnapshotID: "snap-1", + TotalChunks: 2, + ChunkIndex: 0, + Targets: []core.DiscoveredTarget{ + { + Name: "router-1", + }, + }, + } + + err := mp.processSnapshot( + context.Background(), + chunk, + logr.Discard(), + ) + + require.NoError(t, err) + + require.NotNil(t, mp.activeSnapshot) + require.Equal(t, "snap-1", mp.activeSnapshot.snapshotID) + require.Len(t, mp.activeSnapshot.received, 1) +} + +func TestResetSnapshot(t *testing.T) { + mp := mockMessageProcessor() + + mp.activeSnapshot = &snapshotBuffer{ + snapshotID: "snap-1", + } + + mp.resetSnapshot() + + require.Nil(t, mp.activeSnapshot) +} diff --git a/internal/controller/discovery/mocks_test.go b/internal/controller/discovery/mocks_test.go new file mode 100644 index 00000000..1020c714 --- /dev/null +++ b/internal/controller/discovery/mocks_test.go @@ -0,0 +1,133 @@ +package discovery + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" +) + +type fakeStatusUpdater struct { +} + +func (f fakeStatusUpdater) UpdateStatus(ctx context.Context, update core.StatusUpdate) error { + return nil +} + +func mockDiscoveredTargetList(len int) []core.DiscoveredTarget { + targets := make([]core.DiscoveredTarget, len) + + if len > 100 { + len = 100 + } + + for i := range len { + targets[i] = core.DiscoveredTarget{ + Address: fmt.Sprintf("192.168.1.%d", i+1), + Name: fmt.Sprintf("router%d", i+1), + } + } + + return targets +} + +func mockDiscoveryTarget(opts ...func(*core.DiscoveredTarget)) core.DiscoveredTarget { + t := core.DiscoveredTarget{ + Name: "target1", + Address: "10.0.0.1", + Labels: map[string]string{}, + } + + for _, opt := range opts { + opt(&t) + } + + return t +} + +func withDiscoveredTargetName(name string) func(*core.DiscoveredTarget) { + return func(t *core.DiscoveredTarget) { + t.Name = name + } +} + +func withDiscoveredTargetAddress(address string) func(*core.DiscoveredTarget) { + return func(t *core.DiscoveredTarget) { + t.Address = address + } +} + +func withDiscoveredTargetLabels(labels map[string]string) func(*core.DiscoveredTarget) { + return func(t *core.DiscoveredTarget) { + t.Labels = labels + } +} + +func mockTargetSource(opts ...func(*gnmicv1alpha1.TargetSource)) gnmicv1alpha1.TargetSource { + ts := gnmicv1alpha1.TargetSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ts1", + Namespace: "default", + }, + Spec: gnmicv1alpha1.TargetSourceSpec{ + TargetProfile: "default", + TargetLabels: map[string]string{}, + }, + } + + for _, opt := range opts { + opt(&ts) + } + + return ts +} + +func withTargetSourceName(name string) func(*gnmicv1alpha1.TargetSource) { + return func(ts *gnmicv1alpha1.TargetSource) { + ts.ObjectMeta.Name = name + } +} + +func withTargetSourceNamespace(namespace string) func(*gnmicv1alpha1.TargetSource) { + return func(ts *gnmicv1alpha1.TargetSource) { + ts.ObjectMeta.Namespace = namespace + } +} + +func withTargetSourceTargetProfile(profile string) func(*gnmicv1alpha1.TargetSource) { + return func(ts *gnmicv1alpha1.TargetSource) { + ts.Spec.TargetProfile = profile + } +} + +func withTargetSourceTargetLabels(labels map[string]string) func(*gnmicv1alpha1.TargetSource) { + return func(ts *gnmicv1alpha1.TargetSource) { + ts.Spec.TargetLabels = labels + } +} + +func mockGnmicTargetList(len int) []gnmicv1alpha1.Target { + targets := make([]gnmicv1alpha1.Target, len) + + if len > 100 { + len = 100 + } + + for i := range len { + targets[i] = gnmicv1alpha1.Target{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("router%d", i+1), + Namespace: "default", + }, + Spec: gnmicv1alpha1.TargetSpec{ + Address: fmt.Sprintf("192.168.1.%d", i+1), + Profile: "default", + }, + } + } + + return targets +} diff --git a/internal/controller/discovery/ressource_fetcher_client.go b/internal/controller/discovery/ressource_fetcher_client.go new file mode 100644 index 00000000..c544b30a --- /dev/null +++ b/internal/controller/discovery/ressource_fetcher_client.go @@ -0,0 +1,60 @@ +package discovery + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/gnmic/operator/internal/controller/discovery/core" +) + +// k8sResourceFetcher implements core.ResourceFetcher using a controller runtime client +type k8sResourceFetcher struct { + client client.Client +} + +// GetSecretKey retrieves the value of a specific key from a Kubernetes Secret +func (f *k8sResourceFetcher) GetSecretKey(ctx context.Context, namespace string, selector *corev1.SecretKeySelector) (string, error) { + if selector == nil { + return "", nil + } + var secret corev1.Secret + key := client.ObjectKey{Namespace: namespace, Name: selector.Name} + if err := f.client.Get(ctx, key, &secret); err != nil { + return "", err + } + if selector.Key == "" { + return "", fmt.Errorf("secret key selector has empty key for secret %s/%s", namespace, selector.Name) + } + val, ok := secret.Data[selector.Key] + if !ok { + return "", fmt.Errorf("secret %s/%s does not contain key %s", namespace, selector.Name, selector.Key) + } + return string(val), nil +} + +// GetConfigMapKey retrieves the value of a specific key from a Kubernetes ConfigMap +func (f *k8sResourceFetcher) GetConfigMapKey(ctx context.Context, namespace string, selector *corev1.ConfigMapKeySelector) (string, error) { + if selector == nil { + return "", nil + } + var cm corev1.ConfigMap + key := client.ObjectKey{Namespace: namespace, Name: selector.Name} + if err := f.client.Get(ctx, key, &cm); err != nil { + return "", err + } + if selector.Key == "" { + return "", fmt.Errorf("config map key selector has empty key for config map %s/%s", namespace, selector.Name) + } + val, ok := cm.Data[selector.Key] + if !ok { + return "", fmt.Errorf("config map %s/%s does not contain key %s", namespace, selector.Name, selector.Key) + } + return val, nil +} + +func newK8sResourceFetcher(c client.Client) core.ResourceFetcher { + return &k8sResourceFetcher{client: c} +} diff --git a/internal/controller/discovery/status_updater.go b/internal/controller/discovery/status_updater.go new file mode 100644 index 00000000..2b747a0c --- /dev/null +++ b/internal/controller/discovery/status_updater.go @@ -0,0 +1,231 @@ +package discovery + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" +) + +const ( + ConditionReady = "Ready" + ConditionReconciling = "Reconciling" + ConditionDegraded = "Degraded" + ConditionStalled = "Stalled" + + ReasonWaitingForSync = "WaitingForSync" + ReasonSyncStarted = "SyncStarted" + ReasonSyncSucceeded = "SyncSucceeded" + ReasonSyncCompleted = "SyncCompleted" + ReasonSyncWithErrors = "SyncSucceededWithErrors" + ReasonSyncFailed = "SyncFailed" +) + +type TargetSourceStatusUpdater struct { + client client.Client + targetSource *gnmicv1alpha1.TargetSource +} + +func NewTargetSourceStatusUpdater(c client.Client, ts *gnmicv1alpha1.TargetSource) *TargetSourceStatusUpdater { + return &TargetSourceStatusUpdater{ + client: c, + targetSource: ts, + } +} + +func (u *TargetSourceStatusUpdater) SetPending(ctx context.Context) error { + + return u.patchStatus(ctx, func( + ts *gnmicv1alpha1.TargetSource, + ) { + now := metav1.Now() + + // Ready=True + meta.SetStatusCondition(&ts.Status.Conditions, metav1.Condition{ + Type: ConditionReady, + Status: metav1.ConditionFalse, + Reason: ReasonWaitingForSync, + Message: "Waiting for the TargetLoader to start the sync", + LastTransitionTime: now, + }) + + // Remove other status conditions + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionReconciling, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionStalled, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionDegraded, + ) + }) +} + +func (u *TargetSourceStatusUpdater) SetFetching(ctx context.Context) error { + + return u.patchStatus(ctx, func( + ts *gnmicv1alpha1.TargetSource, + ) { + now := metav1.Now() + + // Reconciling=True + meta.SetStatusCondition(&ts.Status.Conditions, metav1.Condition{ + Type: ConditionReconciling, + Status: metav1.ConditionTrue, + Reason: ReasonSyncStarted, + Message: "Started fetching targets", + LastTransitionTime: now, + }) + + // Remove other status conditions + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionReady, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionStalled, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionDegraded, + ) + }) +} + +func (u *TargetSourceStatusUpdater) SetFetchFailed(ctx context.Context, err error) error { + + return u.patchStatus(ctx, func( + ts *gnmicv1alpha1.TargetSource, + ) { + now := metav1.Now() + + // Reconciling=True + meta.SetStatusCondition(&ts.Status.Conditions, metav1.Condition{ + Type: ConditionStalled, + Status: metav1.ConditionTrue, + Reason: ReasonSyncFailed, + Message: err.Error(), + LastTransitionTime: now, + }) + + // Remove other status conditions + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionReady, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionReconciling, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionDegraded, + ) + }) +} + +func (u *TargetSourceStatusUpdater) SetSuccessfulSync(ctx context.Context, targetsCount int32) error { + + return u.patchStatus(ctx, func( + ts *gnmicv1alpha1.TargetSource, + ) { + now := metav1.Now() + + // Ready=True + meta.SetStatusCondition(&ts.Status.Conditions, metav1.Condition{ + Type: ConditionReady, + Status: metav1.ConditionTrue, + Reason: ReasonSyncSucceeded, + Message: "Targets synchronized successfully", + LastTransitionTime: now, + }) + + // Remove other status conditions + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionReconciling, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionStalled, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionDegraded, + ) + + // Update status fields + ts.Status.TargetsCount = targetsCount + ts.Status.LastSync = now + }) +} + +func (u *TargetSourceStatusUpdater) SetSyncWithErrors(ctx context.Context, targetsCount int32, err error) error { + + return u.patchStatus(ctx, func( + ts *gnmicv1alpha1.TargetSource, + ) { + now := metav1.Now() + + // Ready=True + meta.SetStatusCondition(&ts.Status.Conditions, metav1.Condition{ + Type: ConditionReady, + Status: metav1.ConditionTrue, + Reason: ReasonSyncSucceeded, + Message: "Targets synchronized", + LastTransitionTime: now, + }) + + // Degraded=True + meta.SetStatusCondition(&ts.Status.Conditions, metav1.Condition{ + Type: ConditionDegraded, + Status: metav1.ConditionTrue, + Reason: ReasonSyncWithErrors, + Message: err.Error(), + LastTransitionTime: now, + }) + + // Remove other status conditions + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionReady, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionReconciling, + ) + meta.RemoveStatusCondition( + &ts.Status.Conditions, + ConditionStalled, + ) + + // Update status fields + ts.Status.TargetsCount = targetsCount + ts.Status.LastSync = now + }) +} + +func (u *TargetSourceStatusUpdater) patchStatus(ctx context.Context, mutate func(*gnmicv1alpha1.TargetSource)) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &gnmicv1alpha1.TargetSource{} + if err := u.client.Get(ctx, client.ObjectKeyFromObject(u.targetSource), latest); err != nil { + return err + } + + patch := client.MergeFrom(latest.DeepCopy()) + mutate(latest) + + return u.client.Status().Patch(ctx, latest, patch) + }) + + return err +} diff --git a/internal/controller/discovery/status_updater_client.go b/internal/controller/discovery/status_updater_client.go new file mode 100644 index 00000000..16425361 --- /dev/null +++ b/internal/controller/discovery/status_updater_client.go @@ -0,0 +1,69 @@ +package discovery + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" +) + +// k8sStatusUpdater is a client which fulfills the StatusUpdater interface +type k8sStatusUpdater struct { + client client.Client + scheme *runtime.Scheme + targetSource *gnmicv1alpha1.TargetSource +} + +// Returns an instance of k8sStatusUpdater +func NewK8sStatusUpdater(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource) *k8sStatusUpdater { + return &k8sStatusUpdater{ + client: c, + scheme: s, + targetSource: ts, + } +} + +// UpdateStatus takes a StatusUpdate holding Conditions and a pointer referencing the TargetsCount. +// If TargetsCount is set, the LastSync time gets set to metav1.Now(). +// Replaces LastTransitionTime of each Condition with metav1.Now(). +func (c *k8sStatusUpdater) UpdateStatus(ctx context.Context, update core.StatusUpdate) error { + + return c.patchStatus(ctx, func( + ts *gnmicv1alpha1.TargetSource, + ) { + now := metav1.Now() + + // Update status fields: Replace all Conditions and set TargetsCount and LastSync if pointer != nil + for i := range update.Conditions { + update.Conditions[i].LastTransitionTime = now + } + ts.Status.Conditions = update.Conditions + + if update.TargetsCount != nil { + ts.Status.TargetsCount = *update.TargetsCount + ts.Status.LastSync = now + } + }) +} + +// patchStatus is an internal function to update the Kubernetes object +func (c *k8sStatusUpdater) patchStatus(ctx context.Context, mutate func(*gnmicv1alpha1.TargetSource)) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &gnmicv1alpha1.TargetSource{} + if err := c.client.Get(ctx, client.ObjectKeyFromObject(c.targetSource), latest); err != nil { + return err + } + + patch := client.MergeFrom(latest.DeepCopy()) + mutate(latest) + + return c.client.Status().Patch(ctx, latest, patch) + }) + + return err +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 7f30fc85..5b9a38bc 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -20,8 +20,10 @@ import ( "context" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + "github.com/gin-gonic/gin" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" discoveryTypes "github.com/gnmic/operator/internal/controller/discovery/core" @@ -53,6 +56,8 @@ type TargetSourceReconciler struct { types.NamespacedName, discoveryTypes.DiscoveryRegistryValue, ] + + APIRouter *gin.Engine } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -84,7 +89,9 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } if !targetSource.DeletionTimestamp.IsZero() { - return r.reconcileDeletion(ctx, req.NamespacedName, targetSource) + if err := r.reconcileDeletion(ctx, req.NamespacedName, targetSource); err != nil { + return ctrl.Result{}, err + } } if err := r.ensureFinalizer(ctx, targetSource); err != nil { @@ -93,19 +100,21 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request if r.DiscoveryRegistry.Exists(req.NamespacedName) { if targetSource.Generation != targetSource.Status.ObservedGeneration { - return r.reconcileDeletion(ctx, req.NamespacedName, targetSource) + if err := r.reconcileDeletion(ctx, req.NamespacedName, targetSource); err != nil { + return ctrl.Result{}, err + } } else { logger.Info("Discovery runtime already running; reconciliation completed") return ctrl.Result{}, nil } } - if err := r.startDiscovery(req.NamespacedName, targetSource, logger); err != nil { + if err := r.startDiscovery(ctx, req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } - targetSource.Status.ObservedGeneration = targetSource.Generation - if err := r.Status().Update(ctx, targetSource); err != nil { + // Update TargetSource Status for new generation + if err := r.updateObservedGeneration(ctx, targetSource); err != nil { return ctrl.Result{}, err } @@ -123,7 +132,7 @@ func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key type } // reconcileDeletion stops the discovery runtime and removes the finalizer -func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { +func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) error { logger := log.FromContext(ctx).WithValues( "targetsource", key.Name, "namespace", key.Namespace, @@ -138,13 +147,13 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) { controllerutil.RemoveFinalizer(targetSource, LabelTargetSourceFinalizer) if err := r.Update(ctx, targetSource); err != nil { - return ctrl.Result{}, err + return err } logger.Info("Removed TargetSource finalizer") } - return ctrl.Result{}, nil + return nil } // ensureFinalizer adds the finalizer if not present and updates the TargetSource @@ -173,15 +182,32 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // - MessageProcessor and Loader must run for the lifetime of the TargetSource // - Any unexpected exit is treated as a bug and triggers full shutdown func (r *TargetSourceReconciler) startDiscovery( + reconcileCtx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger, ) error { targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) ctx, cancel := context.WithCancel(context.Background()) + + statusUpdater := discovery.NewK8sStatusUpdater(r.Client, r.Scheme, targetSource) + if err := statusUpdater.UpdateStatus(ctx, discoveryTypes.StatusUpdate{ + Conditions: []metav1.Condition{ + { + Type: discoveryTypes.ConditionTypeReady, + Status: metav1.ConditionFalse, + Reason: string(discoveryTypes.ReasonWaitingForSync), + Message: "Waiting for initial sync", + }, + }, + }); err != nil { + logger.Error(err, "updating targetsource status failed") + } + loaderConfig := discoveryTypes.CommonLoaderConfig{ TargetsourceNN: key, ChunkSize: r.ChunkSize, + Updater: statusUpdater, } // Cleanup function to cleanup discovery runtime of targetsource @@ -195,8 +221,9 @@ func (r *TargetSourceReconciler) startDiscovery( r.Scheme, targetSource, targetChannel, + statusUpdater, ) - loader, err := discovery.NewLoader(&loaderConfig, targetSource.Spec) + loader, err := discovery.NewLoader(reconcileCtx, r.Client, &loaderConfig, targetSource.Spec) if err != nil { logger.Error(err, "Target loader could not be created") cleanup() @@ -238,6 +265,22 @@ func (r *TargetSourceReconciler) startDiscovery( return nil } +func (r *TargetSourceReconciler) updateObservedGeneration(ctx context.Context, ts *gnmicv1alpha1.TargetSource) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &gnmicv1alpha1.TargetSource{} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(ts), latest); err != nil { + return err + } + + patch := client.MergeFrom(latest.DeepCopy()) + latest.Status.ObservedGeneration = ts.Generation + + return r.Client.Status().Patch(ctx, latest, patch) + }) + + return err +} + // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/lab/dev/netbox/initializers/ip-addresses.yaml b/lab/dev/netbox/initializers/ip-addresses.yaml index c474fa1a..b3cf7f01 100644 --- a/lab/dev/netbox/initializers/ip-addresses.yaml +++ b/lab/dev/netbox/initializers/ip-addresses.yaml @@ -16,7 +16,7 @@ name: spine1 name: system0 status: active -- address: 172.18.0.4/32 +- address: 172.18.0.3/32 assigned_object: device: name: spine1 @@ -32,7 +32,7 @@ status: active primary: true dns_name: clab-3-nodes-leaf1 -- address: 172.18.0.3/32 +- address: 172.18.0.6/32 assigned_object: device: name: leaf2 @@ -40,7 +40,7 @@ status: active primary: true dns_name: clab-3-nodes-leaf2 -- address: 172.18.0.6/32 +- address: 172.18.0.4/32 assigned_object: device: name: ceos1 diff --git a/lab/dev/netbox/netbox-values.yaml b/lab/dev/netbox/netbox-values.yaml index eb89f14b..325017f0 100644 --- a/lab/dev/netbox/netbox-values.yaml +++ b/lab/dev/netbox/netbox-values.yaml @@ -27,4 +27,9 @@ extraVolumes: extraVolumeMounts: - name: peppers mountPath: /run/config/extra/peppers - readOnly: true \ No newline at end of file + readOnly: true + +# Version pinned to keep "make netbox-sync-data" working +image: + tag: v4.5.7 + \ No newline at end of file diff --git a/lab/dev/resources/targetsource/ts.yaml b/lab/dev/resources/targetsource/ts.yaml new file mode 100644 index 00000000..115ac8fc --- /dev/null +++ b/lab/dev/resources/targetsource/ts.yaml @@ -0,0 +1,28 @@ +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: netbox + namespace: default +spec: + provider: + http: + url: http://srbsci-152:8081/api/dcim/devices/?export=GNMIc_operator_pull + authorization: + token: + scheme: Bearer + tokenSecretRef: + name: netbox-api-token-demo + key: token + push: + enabled: true + auth: + bearer: + tokenSecretRef: + name: gnmic-api-auth + key: bearer-token + + interval: 12h + targetPort: 57400 + targetProfile: default + targetLabels: + site: lab \ No newline at end of file diff --git a/test.mk b/test.mk index 23c59834..1c8362ec 100644 --- a/test.mk +++ b/test.mk @@ -95,6 +95,23 @@ deploy-test-http-server: ## Deploy a test http pod with a static file inventory undeploy-test-http-server: ## Undeploy the http pod for testing kubectl delete -f test/integration/http/resources/ +.PHONY: create-secrets-for-apiserver +create-secrets-for-apiserver: + kubectl create secret generic gnmic-api-auth --from-literal=bearer-token=secureSecret + kubectl create secret generic gnmic-signature --from-literal=signature=1879 + +.PHONY: send-target-to-apiserver +send-target-to-apiserver: + @BEARER_TOKEN=$$(kubectl get secret gnmic-api-auth \ + -o jsonpath='{.data.bearer-token}' | base64 --decode); \ + kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 >/dev/null 2>&1 & \ + sleep 3; \ + curl --retry 3 --retry-delay 1 --retry-connrefused -X POST "http://localhost:8082/api/v1/default/target-source/http-ts/applyTargets" \ + -H "Authorization: Bearer $$BEARER_TOKEN" \ + -H "Content-Type: application/json" \ + -H "x-hook-signature: cec95fd6d3a350ebcf9b25d2d715384ca673ee3a3cd67ed22e212179d9ee20abe724cbed7f93028c5b0e12e5ce6dd791482f2a1045d47253e8cddd637f0f8d7d" \ + -d '[{"address":"clab-t1-leaf2","port":57400,"name":"leaf2","operation":"created","targetProfile":"default","labels":[{"key":"vendor","value":"nokia_srlinux"},{"key":"role","value":"leaf"}]}]'; \ + .PHONY: deploy-test-netbox-instance deploy-test-netbox-instance: NETBOX_CLUSTER_NAME=$(TEST_CLUSTER_NAME) ## Deploy the test netbox instance for testing deploy-test-netbox-instance: NETBOX_PASSWORD=Netbox123 @@ -153,5 +170,5 @@ apply-test-clusters: ## Apply the test clusters for testing kubectl apply -f test/integration/resources/clusters .PHONY: apply-test-resources -apply-test-resources: apply-test-targets apply-test-subscriptions apply-test-outputs apply-test-pipelines apply-test-clusters +apply-test-resources: apply-test-targets apply-test-targetsources apply-test-subscriptions apply-test-outputs apply-test-pipelines apply-test-clusters diff --git a/test/integration/http/resources/configmap.yaml b/test/integration/http/resources/configmap.yaml index be7091f7..e0c9a9f8 100644 --- a/test/integration/http/resources/configmap.yaml +++ b/test/integration/http/resources/configmap.yaml @@ -22,14 +22,5 @@ data: "vendor": "nokia_srlinux", "role": "leaf" } - }, - { - "address": "clab-t1-leaf2", - "port": 57400, - "name": "leaf2", - "labels": { - "vendor": "nokia_srlinux", - "role": "leaf" - } } - ] \ No newline at end of file + ] diff --git a/test/integration/resources/targetsources/http.yaml b/test/integration/resources/targetsources/http.yaml index 422cfdcd..f86a02e0 100644 --- a/test/integration/resources/targetsources/http.yaml +++ b/test/integration/resources/targetsources/http.yaml @@ -6,6 +6,18 @@ spec: provider: http: url: http://http-svc.default.svc/targets.json + interval: 30s + push: + enabled: true + auth: + bearer: + tokenSecretRef: + name: gnmic-api-auth + key: bearer-token + signature: + secretRef: + name: gnmic-signature + key: signature targetLabels: integrationtest: http - targetProfile: default \ No newline at end of file + targetProfile: default