From 3eda2ea94f0c8de748e4c713737b104272951cf3 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Mon, 13 Jul 2026 13:43:05 -0400 Subject: [PATCH 1/4] fix: install downstream WAF data plane in e2e CI The E2E Tests workflow is red because the chainsaw test trafficprotectionpolicy-enforce asserts Gateway waf-gw reaches Programmed=True, but the test requests gatewayClassName datum-downstream-gateway and nothing in CI programs that class. The stock prepare-infra-cluster installs only the vanilla Envoy Gateway and the NSO CRDs, never the Coraza WAF data plane the test needs. Wire the WAF data plane into the infra-cluster setup so the datum-downstream-gateway class is served end to end on kind-nso-infra. Key changes: - Add make target downstream-waf-dataplane (a prepare-infra-cluster dependency) that installs the downstream Envoy Gateway instance, the extension server, and the datum-downstream-gateway GatewayClass plus its Coraza EnvoyProxy, loading the operator image into nso-infra. - Add config/e2e-downstream overlay: reuses the config/extension-server base and resolves its mTLS placeholders with a self-signed CA chain (cert-manager Issuer issues both server and client certs; the CA Secret is mounted directly, avoiding trust-manager). Drops the ServiceMonitor and pins replicas to one for the single-node cluster. - Resolve the envoy-gateway-downstream extensionManager placeholders to the extension server FQDN and the self-signed CA Secret. - Serve the datum-downstream-gateway GatewayClass with the downstream controllerName so the test's hardcoded class matches the installed data plane; config/e2e/config.yaml is left untouched since its datum-downstream-gateway-e2e value drives the separate gateway tests. The test's stale "skipped by CI" note is corrected to reflect that prepare-infra-cluster now installs the data plane. --- Makefile | 11 ++- .../extension-server-config.yaml | 12 +++ config/e2e-downstream/kustomization.yaml | 76 +++++++++++++++++++ config/e2e-downstream/namespace.yaml | 4 + config/e2e-downstream/pki.yaml | 31 ++++++++ config/e2e-downstream/waf-gateway.yaml | 59 ++++++++++++++ .../kustomization.yaml | 11 +-- .../chainsaw-test.yaml | 5 +- 8 files changed, 200 insertions(+), 9 deletions(-) create mode 100644 config/e2e-downstream/extension-server-config.yaml create mode 100644 config/e2e-downstream/kustomization.yaml create mode 100644 config/e2e-downstream/namespace.yaml create mode 100644 config/e2e-downstream/pki.yaml create mode 100644 config/e2e-downstream/waf-gateway.yaml diff --git a/Makefile b/Makefile index fb65611d..64301f0f 100644 --- a/Makefile +++ b/Makefile @@ -153,7 +153,7 @@ set-image-controller: manifests kustomize cd config/manager && $(KUSTOMIZE) edit set image ghcr.io/datum-cloud/network-services-operator=${IMG} .PHONY: prepare-infra-cluster -prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds +prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds downstream-waf-dataplane .PHONY: downstream-crds downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the replicator mirrors into it. @@ -161,6 +161,11 @@ downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml +.PHONY: downstream-waf-dataplane +downstream-waf-dataplane: kustomize load-image-infra ## Install the Coraza WAF data plane (downstream EG instance + extension server + datum-downstream-gateway GatewayClass) on the infra cluster. + $(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway-downstream | kubectl apply --server-side=true --force-conflicts -f - + $(KUSTOMIZE) build config/e2e-downstream | kubectl apply --server-side=true --force-conflicts -f - + .PHONY: prepare-e2e prepare-e2e: chainsaw set-image-controller cert-manager load-image-all deploy-e2e @@ -174,6 +179,10 @@ load-image-all: load-image-operator load-image-operator: docker-build kind $(KIND) load docker-image $(IMG) -n nso-standard +.PHONY: load-image-infra +load-image-infra: docker-build kind + $(KIND) load docker-image $(IMG) -n nso-infra + .PHONY: cert-manager cert-manager: cmctl $(KUSTOMIZE) build --enable-helm config/tools/cert-manager | kubectl apply --server-side=true --force-conflicts -f - diff --git a/config/e2e-downstream/extension-server-config.yaml b/config/e2e-downstream/extension-server-config.yaml new file mode 100644 index 00000000..f4c00b6d --- /dev/null +++ b/config/e2e-downstream/extension-server-config.yaml @@ -0,0 +1,12 @@ +apiVersion: apiserver.config.datumapis.com/v1alpha1 +kind: NetworkServicesOperator +gateway: + coraza: + routeBaseDirectives: + - Include @crs-setup-conf + - Include @recommended-conf + - SecResponseBodyLimitAction ProcessPartial + - SecResponseBodyLimit 32000 + errorPage: + enabled: true + bodyPath: /etc/datum/error-pages/error-5xx.html diff --git a/config/e2e-downstream/kustomization.yaml b/config/e2e-downstream/kustomization.yaml new file mode 100644 index 00000000..b00e83e0 --- /dev/null +++ b/config/e2e-downstream/kustomization.yaml @@ -0,0 +1,76 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: datum-downstream-gateway + +resources: + - namespace.yaml + - ../extension-server + - pki.yaml + - waf-gateway.yaml + +configMapGenerator: + - name: nso-extension-server-config + files: + - config.yaml=extension-server-config.yaml + options: + disableNameSuffixHash: true + +patches: + - target: + kind: ServiceMonitor + name: envoy-gateway-extension-server-metrics + patch: | + $patch: delete + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: envoy-gateway-extension-server-metrics + - target: + kind: Certificate + name: envoy-gateway-extension-server-eg-client-tls + patch: | + - op: replace + path: /spec/issuerRef/kind + value: Issuer + - op: replace + path: /spec/issuerRef/name + value: envoy-gateway-extension-server + - target: + kind: Deployment + name: envoy-gateway-extension-server + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: envoy-gateway-extension-server + spec: + replicas: 1 + template: + spec: + containers: + - name: envoy-gateway-extension-server + env: + - name: SERVER_CONFIG + value: /config/config.yaml + volumeMounts: + - name: server-config + mountPath: /config + readOnly: true + volumes: + - name: tls + csi: + volumeAttributes: + csi.cert-manager.io/issuer-kind: Issuer + csi.cert-manager.io/issuer-name: envoy-gateway-extension-server + csi.cert-manager.io/dns-names: "envoy-gateway-extension-server.datum-downstream-gateway.svc,envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local" + - name: tls-ca + configMap: null + secret: + secretName: envoy-gateway-extension-server-ca + items: + - key: ca.crt + path: ca.crt + - name: server-config + configMap: + name: nso-extension-server-config diff --git a/config/e2e-downstream/namespace.yaml b/config/e2e-downstream/namespace.yaml new file mode 100644 index 00000000..6cecd275 --- /dev/null +++ b/config/e2e-downstream/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: datum-downstream-gateway diff --git a/config/e2e-downstream/pki.yaml b/config/e2e-downstream/pki.yaml new file mode 100644 index 00000000..666bb0b3 --- /dev/null +++ b/config/e2e-downstream/pki.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: envoy-gateway-extension-server-selfsigned +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: envoy-gateway-extension-server-ca +spec: + isCA: true + commonName: envoy-gateway-extension-server-ca + secretName: envoy-gateway-extension-server-ca + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: envoy-gateway-extension-server-selfsigned + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: envoy-gateway-extension-server +spec: + ca: + secretName: envoy-gateway-extension-server-ca diff --git a/config/e2e-downstream/waf-gateway.yaml b/config/e2e-downstream/waf-gateway.yaml new file mode 100644 index 00000000..5edeaaff --- /dev/null +++ b/config/e2e-downstream/waf-gateway.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: datum-downstream-gateway +spec: + mergeGateways: true + ipFamily: DualStack + provider: + type: Kubernetes + kubernetes: + envoyDeployment: + pod: + volumes: + - name: coraza-waf + emptyDir: {} + container: + image: envoyproxy/envoy:contrib-v1.35.0 + volumeMounts: + - name: coraza-waf + mountPath: /opt/coraza-waf + initContainers: + - name: coraza-waf + image: ghcr.io/datum-labs/coraza-envoy-go-filter/coraza-waf:v1.3.0-alpha10 + imagePullPolicy: IfNotPresent + command: + - cp + - /app/coraza-waf.so + - /opt/coraza-waf/ + volumeMounts: + - name: coraza-waf + mountPath: /opt/coraza-waf + envoyService: + externalTrafficPolicy: Cluster + type: NodePort + patch: + type: StrategicMerge + value: + spec: + ipFamilyPolicy: RequireDualStack + ports: + - name: http-80 + nodePort: 30080 + port: 80 + - name: https-443 + nodePort: 30443 + port: 443 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: datum-downstream-gateway +spec: + controllerName: gateway.envoyproxy.io/datum-downstream-gateway + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: datum-downstream-gateway + namespace: datum-downstream-gateway diff --git a/config/tools/envoy-gateway-downstream/kustomization.yaml b/config/tools/envoy-gateway-downstream/kustomization.yaml index d6cce819..18420a6e 100644 --- a/config/tools/envoy-gateway-downstream/kustomization.yaml +++ b/config/tools/envoy-gateway-downstream/kustomization.yaml @@ -39,16 +39,17 @@ helmCharts: kind: Connector service: fqdn: - hostname: network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local + hostname: envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local port: 5005 tls: + # EG reads the CA from tls.crt of this Secret; the self-signed CA + # Certificate's Secret carries the CA cert there. certificateRef: - # Placeholder — an overlay must patch this to the Secret holding the CA that issued the server cert. - name: placeholder-ca - namespace: placeholder-namespace + name: envoy-gateway-extension-server-ca + namespace: datum-downstream-gateway clientCertificateRef: name: envoy-gateway-extension-server-eg-client-tls - namespace: network-services-operator-system + namespace: datum-downstream-gateway retry: maxAttempts: 4 initialBackoff: 100ms diff --git a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml index 5d684ff7..11cd6875 100644 --- a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml +++ b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml @@ -12,9 +12,8 @@ metadata: # # Precondition: the downstream (nso-infra) must have the WAF data plane wired up # (extension-server + the extensionManager Envoy Gateway registered on the -# `datum-downstream-gateway` GatewayClass, with the Coraza filter). The stock -# `make prepare-infra-cluster` does NOT install this; this test is skipped by -# CI environments without it (the Gateway never reaches Programmed=True). +# `datum-downstream-gateway` GatewayClass, with the Coraza filter). +# `make prepare-infra-cluster` installs this via `make downstream-waf-dataplane`. spec: cluster: nso-infra # EG only reconciles namespaces carrying this label. From 69d07c5c6f281997775131913aeb2021c24838ce Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Mon, 13 Jul 2026 15:37:04 -0400 Subject: [PATCH 2/4] fix: program the downstream WAF gateway in e2e to match prod The trafficprotectionpolicy-enforce-serves-traffic e2e test never let waf-gw reach Programmed=True. Two independent defects, both diverging from the production downstream WAF wiring, kept the gateway offline. First, the downstream Envoy Gateway watches only namespaces labelled meta.datumapis.com/upstream-cluster-name, and its GatewayClass parametersRef points at the EnvoyProxy in the datum-downstream-gateway namespace. That namespace was created unlabelled, so Envoy Gateway could not see the EnvoyProxy and left the GatewayClass unaccepted, giving every Gateway an empty status. Production labels this namespace 0-datum-internal-anchor for exactly this reason. Second, the coraza-waf init container copied the WAF library from /app/coraza-waf.so, which does not exist in coraza-waf:v1.3.0-alpha10; the image ships it at /coraza-waf.so. The bad path made cp exit 1, the init container CrashLoopBackOff'd, the merged Envoy pod stayed Pending, and the Gateway reported Programmed=False with reason NoResources ("Envoy replicas unavailable"). Production copies from /coraza-waf.so. Key changes: - Label the datum-downstream-gateway namespace (tools and e2e-downstream copies) so the downstream Envoy Gateway watches it and accepts the class. - Copy the WAF library from /coraza-waf.so, matching production. - Pin the downstream Envoy Gateway chart to v1.7.4, the version production runs for this control plane; CRDs still come from the base envoy-gateway install at v1.8.1. - Set extensionManager.maxMessageSize to 256Mi, matching production and the extension server, so the WAF-augmented xDS response is not rejected. --- config/e2e-downstream/namespace.yaml | 2 ++ config/e2e-downstream/waf-gateway.yaml | 2 +- .../tools/envoy-gateway-downstream/kustomization.yaml | 10 +++++++++- config/tools/envoy-gateway-downstream/namespace.yaml | 2 ++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/config/e2e-downstream/namespace.yaml b/config/e2e-downstream/namespace.yaml index 6cecd275..89c7d150 100644 --- a/config/e2e-downstream/namespace.yaml +++ b/config/e2e-downstream/namespace.yaml @@ -2,3 +2,5 @@ apiVersion: v1 kind: Namespace metadata: name: datum-downstream-gateway + labels: + meta.datumapis.com/upstream-cluster-name: 0-datum-internal-anchor diff --git a/config/e2e-downstream/waf-gateway.yaml b/config/e2e-downstream/waf-gateway.yaml index 5edeaaff..ef6a6265 100644 --- a/config/e2e-downstream/waf-gateway.yaml +++ b/config/e2e-downstream/waf-gateway.yaml @@ -25,7 +25,7 @@ spec: imagePullPolicy: IfNotPresent command: - cp - - /app/coraza-waf.so + - /coraza-waf.so - /opt/coraza-waf/ volumeMounts: - name: coraza-waf diff --git a/config/tools/envoy-gateway-downstream/kustomization.yaml b/config/tools/envoy-gateway-downstream/kustomization.yaml index 18420a6e..f95b40f2 100644 --- a/config/tools/envoy-gateway-downstream/kustomization.yaml +++ b/config/tools/envoy-gateway-downstream/kustomization.yaml @@ -6,7 +6,11 @@ helmCharts: includeCRDs: false namespace: datum-downstream-gateway releaseName: envoy-datum-downstream-gateway - version: v1.8.1 + # Pinned to match production: v1.8.x does not program merged gateways served + # through the extension manager (waf-gw stays with an empty status), so the + # downstream WAF control plane runs v1.7.4. CRDs come from the base + # envoy-gateway install (v1.8.1), which is forward-compatible. + version: v1.7.4 repo: oci://docker.io/envoyproxy valuesInline: config: @@ -29,6 +33,10 @@ helmCharts: - key: meta.datumapis.com/upstream-cluster-name operator: Exists extensionManager: + # The WAF-augmented merged xDS response exceeds gRPC's 4 MiB default, + # so EG rejects it and stops publishing config with failOpen:false. + # Match the ext-server's 256 MiB limit; both sides must agree. + maxMessageSize: 256Mi policyResources: - group: networking.datumapis.com version: v1alpha diff --git a/config/tools/envoy-gateway-downstream/namespace.yaml b/config/tools/envoy-gateway-downstream/namespace.yaml index 6cecd275..89c7d150 100644 --- a/config/tools/envoy-gateway-downstream/namespace.yaml +++ b/config/tools/envoy-gateway-downstream/namespace.yaml @@ -2,3 +2,5 @@ apiVersion: v1 kind: Namespace metadata: name: datum-downstream-gateway + labels: + meta.datumapis.com/upstream-cluster-name: 0-datum-internal-anchor From 1722a3088b889536137466a00f167f773886cdba Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Mon, 13 Jul 2026 15:37:13 -0400 Subject: [PATCH 3/4] test: harden and instrument the WAF serves-traffic e2e Give the waf-gw Programmed assert a 3m timeout so the first cold pull of the contrib Envoy and Coraza WAF images on the kind node cannot flake it, and add failure diagnostics for the downstream WAF data plane. Key changes: - Raise the Programmed assert timeout to 3m, matching the longer timeouts the gateway-invalid-certificate suite already uses for this data plane. - Add a chainsaw catch that dumps the live Gateway, GatewayClass, and downstream Envoy Gateway and dataplane logs before cleanup tears them down. - Dump the downstream WAF data plane (GatewayClass, EnvoyProxy, controller and extension server state, certificates) from the workflow on e2e failure. --- .github/workflows/test-e2e.yml | 26 +++++++++++++++++++ .../chainsaw-test.yaml | 17 ++++++++++++ 2 files changed, 43 insertions(+) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 6e86e4e9..badc3109 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -64,3 +64,29 @@ jobs: - name: Running Test e2e run: make test-e2e + + - name: Dump downstream WAF data plane on failure + if: failure() + run: | + kubectl config use-context kind-nso-infra + echo "::group::GatewayClasses" + kubectl get gatewayclass -o wide || true + kubectl describe gatewayclass datum-downstream-gateway || true + echo "::endgroup::" + echo "::group::EnvoyProxy" + kubectl -n datum-downstream-gateway get envoyproxy -o yaml || true + echo "::endgroup::" + echo "::group::datum-downstream-gateway namespace + pods" + kubectl get ns datum-downstream-gateway --show-labels || true + kubectl -n datum-downstream-gateway get pods -o wide || true + echo "::endgroup::" + echo "::group::Downstream Envoy Gateway logs" + kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200 || true + echo "::endgroup::" + echo "::group::Extension server" + kubectl -n datum-downstream-gateway describe deploy envoy-gateway-extension-server || true + kubectl -n datum-downstream-gateway logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200 || true + echo "::endgroup::" + echo "::group::Certificates" + kubectl -n datum-downstream-gateway get certificate,secret || true + echo "::endgroup::" diff --git a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml index 11cd6875..9e42f2e2 100644 --- a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml +++ b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml @@ -123,6 +123,11 @@ spec: - type: OWASPCoreRuleSet owaspCoreRuleSet: {} - assert: + # The downstream Envoy fleet pulls the contrib Envoy and Coraza WAF + # init images cold on the kind node before its endpoints exist, and + # Envoy Gateway only writes Programmed once they do. That first-pull + # window overruns chainsaw's 30s assert default, so allow more time. + timeout: 3m resource: apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -131,6 +136,18 @@ spec: status: (conditions[?type == 'Programmed']): - status: "True" + catch: + - script: + timeout: 60s + content: | + set -x + kubectl get gateway -A -o yaml + kubectl describe gateway -A + kubectl get gatewayclass datum-downstream-gateway -o yaml + kubectl -n datum-downstream-gateway get pods -o wide + kubectl -n datum-downstream-gateway describe pods + kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=-1 + kubectl -n datum-downstream-gateway logs -l gateway.envoyproxy.io/owning-gateway-namespace --all-containers --tail=200 || true - name: Benign traffic must reach the backend (not an empty/5xx reply) description: > From 09610bf321481f8a6fced4b886e72ba87f0b953e Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Mon, 13 Jul 2026 15:43:56 -0400 Subject: [PATCH 4/4] fix: correct coraza-waf.so path in dev downstream manifest Match the e2e overlay and production: coraza-waf:v1.3.0-alpha10 ships the WAF library at /coraza-waf.so, not /app/coraza-waf.so. The stale path made the init container's cp exit 1 and CrashLoopBackOff wherever this manifest is applied. This is the config/dev copy of the same defect fixed in config/e2e-downstream; it is currently orphaned (referenced by no Makefile/Taskfile target) but is corrected here to remove the latent trap and keep the two copies consistent. --- config/dev/downstream_resources/downstream-gateway.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/dev/downstream_resources/downstream-gateway.yaml b/config/dev/downstream_resources/downstream-gateway.yaml index d6905672..3629e5a7 100644 --- a/config/dev/downstream_resources/downstream-gateway.yaml +++ b/config/dev/downstream_resources/downstream-gateway.yaml @@ -67,7 +67,7 @@ spec: imagePullPolicy: IfNotPresent command: - cp - - /app/coraza-waf.so + - /coraza-waf.so - /opt/coraza-waf/ volumeMounts: - name: coraza-waf