fix: install downstream WAF data plane in e2e CI (#263)

.github/workflows/test-e2e.yml

@@ -64,3 +64,29 @@ jobs:
 
       - name: Running Test e2e
         run: make test-e2e
+
+      - name: Dump downstream WAF data plane on failure
+        if: failure()
+        run: |
+          kubectl config use-context kind-nso-infra
+          echo "::group::GatewayClasses"
+          kubectl get gatewayclass -o wide || true
+          kubectl describe gatewayclass datum-downstream-gateway || true
+          echo "::endgroup::"
+          echo "::group::EnvoyProxy"
+          kubectl -n datum-downstream-gateway get envoyproxy -o yaml || true
+          echo "::endgroup::"
+          echo "::group::datum-downstream-gateway namespace + pods"
+          kubectl get ns datum-downstream-gateway --show-labels || true
+          kubectl -n datum-downstream-gateway get pods -o wide || true
+          echo "::endgroup::"
+          echo "::group::Downstream Envoy Gateway logs"
+          kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200 || true
+          echo "::endgroup::"
+          echo "::group::Extension server"
+          kubectl -n datum-downstream-gateway describe deploy envoy-gateway-extension-server || true
+          kubectl -n datum-downstream-gateway logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200 || true
+          echo "::endgroup::"
+          echo "::group::Certificates"
+          kubectl -n datum-downstream-gateway get certificate,secret || true
+          echo "::endgroup::"

Makefile

@@ -153,14 +153,19 @@ set-image-controller: manifests kustomize
 	cd config/manager && $(KUSTOMIZE) edit set image ghcr.io/datum-cloud/network-services-operator=${IMG}
 
 .PHONY: prepare-infra-cluster
-prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds
+prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds downstream-waf-dataplane
 
 .PHONY: downstream-crds
 downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the replicator mirrors into it.
 	$(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_connectors.yaml
 	$(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml
 	$(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml
 
+.PHONY: downstream-waf-dataplane
+downstream-waf-dataplane: kustomize load-image-infra ## Install the Coraza WAF data plane (downstream EG instance + extension server + datum-downstream-gateway GatewayClass) on the infra cluster.
+	$(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway-downstream | kubectl apply --server-side=true --force-conflicts -f -
+	$(KUSTOMIZE) build config/e2e-downstream | kubectl apply --server-side=true --force-conflicts -f -
+
 .PHONY: prepare-e2e
 prepare-e2e: chainsaw set-image-controller cert-manager load-image-all deploy-e2e
 
@@ -174,6 +179,10 @@ load-image-all: load-image-operator
 load-image-operator: docker-build kind
 	$(KIND) load docker-image $(IMG) -n nso-standard
 
+.PHONY: load-image-infra
+load-image-infra: docker-build kind
+	$(KIND) load docker-image $(IMG) -n nso-infra
+
 .PHONY: cert-manager
 cert-manager: cmctl
 	$(KUSTOMIZE) build --enable-helm config/tools/cert-manager | kubectl apply --server-side=true --force-conflicts -f -

config/dev/downstream_resources/downstream-gateway.yaml

@@ -67,7 +67,7 @@ spec:
             imagePullPolicy: IfNotPresent
             command:
               - cp
-              - /app/coraza-waf.so
+              - /coraza-waf.so
               - /opt/coraza-waf/
             volumeMounts:
               - name: coraza-waf

config/e2e-downstream/extension-server-config.yaml

@@ -0,0 +1,12 @@
+apiVersion: apiserver.config.datumapis.com/v1alpha1
+kind: NetworkServicesOperator
+gateway:
+  coraza:
+    routeBaseDirectives:
+      - Include @crs-setup-conf
+      - Include @recommended-conf
+      - SecResponseBodyLimitAction ProcessPartial
+      - SecResponseBodyLimit 32000
+  errorPage:
+    enabled: true
+    bodyPath: /etc/datum/error-pages/error-5xx.html

config/e2e-downstream/kustomization.yaml

@@ -0,0 +1,76 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: datum-downstream-gateway
+
+resources:
+  - namespace.yaml
+  - ../extension-server
+  - pki.yaml
+  - waf-gateway.yaml
+
+configMapGenerator:
+  - name: nso-extension-server-config
+    files:
+      - config.yaml=extension-server-config.yaml
+    options:
+      disableNameSuffixHash: true
+
+patches:
+  - target:
+      kind: ServiceMonitor
+      name: envoy-gateway-extension-server-metrics
+    patch: |
+      $patch: delete
+      apiVersion: monitoring.coreos.com/v1
+      kind: ServiceMonitor
+      metadata:
+        name: envoy-gateway-extension-server-metrics
+  - target:
+      kind: Certificate
+      name: envoy-gateway-extension-server-eg-client-tls
+    patch: |
+      - op: replace
+        path: /spec/issuerRef/kind
+        value: Issuer
+      - op: replace
+        path: /spec/issuerRef/name
+        value: envoy-gateway-extension-server
+  - target:
+      kind: Deployment
+      name: envoy-gateway-extension-server
+    patch: |
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: envoy-gateway-extension-server
+      spec:
+        replicas: 1
+        template:
+          spec:
+            containers:
+              - name: envoy-gateway-extension-server
+                env:
+                  - name: SERVER_CONFIG
+                    value: /config/config.yaml
+                volumeMounts:
+                  - name: server-config
+                    mountPath: /config
+                    readOnly: true
+            volumes:
+              - name: tls
+                csi:
+                  volumeAttributes:
+                    csi.cert-manager.io/issuer-kind: Issuer
+                    csi.cert-manager.io/issuer-name: envoy-gateway-extension-server
+                    csi.cert-manager.io/dns-names: "envoy-gateway-extension-server.datum-downstream-gateway.svc,envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local"
+              - name: tls-ca
+                configMap: null
+                secret:
+                  secretName: envoy-gateway-extension-server-ca
+                  items:
+                    - key: ca.crt
+                      path: ca.crt
+              - name: server-config
+                configMap:
+                  name: nso-extension-server-config

config/e2e-downstream/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: datum-downstream-gateway
+  labels:
+    meta.datumapis.com/upstream-cluster-name: 0-datum-internal-anchor

config/e2e-downstream/pki.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: envoy-gateway-extension-server-selfsigned
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: envoy-gateway-extension-server-ca
+spec:
+  isCA: true
+  commonName: envoy-gateway-extension-server-ca
+  secretName: envoy-gateway-extension-server-ca
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: envoy-gateway-extension-server-selfsigned
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: envoy-gateway-extension-server
+spec:
+  ca:
+    secretName: envoy-gateway-extension-server-ca

config/e2e-downstream/waf-gateway.yaml

@@ -0,0 +1,59 @@
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: datum-downstream-gateway
+spec:
+  mergeGateways: true
+  ipFamily: DualStack
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyDeployment:
+        pod:
+          volumes:
+            - name: coraza-waf
+              emptyDir: {}
+        container:
+          image: envoyproxy/envoy:contrib-v1.35.0
+          volumeMounts:
+            - name: coraza-waf
+              mountPath: /opt/coraza-waf
+        initContainers:
+          - name: coraza-waf
+            image: ghcr.io/datum-labs/coraza-envoy-go-filter/coraza-waf:v1.3.0-alpha10
+            imagePullPolicy: IfNotPresent
+            command:
+              - cp
+              - /coraza-waf.so
+              - /opt/coraza-waf/
+            volumeMounts:
+              - name: coraza-waf
+                mountPath: /opt/coraza-waf
+      envoyService:
+        externalTrafficPolicy: Cluster
+        type: NodePort
+        patch:
+          type: StrategicMerge
+          value:
+            spec:
+              ipFamilyPolicy: RequireDualStack
+              ports:
+                - name: http-80
+                  nodePort: 30080
+                  port: 80
+                - name: https-443
+                  nodePort: 30443
+                  port: 443
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: GatewayClass
+metadata:
+  name: datum-downstream-gateway
+spec:
+  controllerName: gateway.envoyproxy.io/datum-downstream-gateway
+  parametersRef:
+    group: gateway.envoyproxy.io
+    kind: EnvoyProxy
+    name: datum-downstream-gateway
+    namespace: datum-downstream-gateway

config/tools/envoy-gateway-downstream/kustomization.yaml

@@ -6,7 +6,11 @@ helmCharts:
     includeCRDs: false
     namespace: datum-downstream-gateway
     releaseName: envoy-datum-downstream-gateway
-    version: v1.8.1
+    # Pinned to match production: v1.8.x does not program merged gateways served
+    # through the extension manager (waf-gw stays with an empty status), so the
+    # downstream WAF control plane runs v1.7.4. CRDs come from the base
+    # envoy-gateway install (v1.8.1), which is forward-compatible.
+    version: v1.7.4
     repo: oci://docker.io/envoyproxy
     valuesInline:
       config:
@@ -29,6 +33,10 @@ helmCharts:
                     - key: meta.datumapis.com/upstream-cluster-name
                       operator: Exists
           extensionManager:
+            # The WAF-augmented merged xDS response exceeds gRPC's 4 MiB default,
+            # so EG rejects it and stops publishing config with failOpen:false.
+            # Match the ext-server's 256 MiB limit; both sides must agree.
+            maxMessageSize: 256Mi
             policyResources:
               - group: networking.datumapis.com
                 version: v1alpha
@@ -39,16 +47,17 @@ helmCharts:
                 kind: Connector
             service:
               fqdn:
-                hostname: network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local
+                hostname: envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local
                 port: 5005
               tls:
+                # EG reads the CA from tls.crt of this Secret; the self-signed CA
+                # Certificate's Secret carries the CA cert there.
                 certificateRef:
-                  # Placeholder — an overlay must patch this to the Secret holding the CA that issued the server cert.
-                  name: placeholder-ca
-                  namespace: placeholder-namespace
+                  name: envoy-gateway-extension-server-ca
+                  namespace: datum-downstream-gateway
                 clientCertificateRef:
                   name: envoy-gateway-extension-server-eg-client-tls
-                  namespace: network-services-operator-system
+                  namespace: datum-downstream-gateway
               retry:
                 maxAttempts: 4
                 initialBackoff: 100ms

config/tools/envoy-gateway-downstream/namespace.yaml

@@ -2,3 +2,5 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: datum-downstream-gateway
+  labels:
+    meta.datumapis.com/upstream-cluster-name: 0-datum-internal-anchor

test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml

@@ -12,9 +12,8 @@ metadata:
 #
 # Precondition: the downstream (nso-infra) must have the WAF data plane wired up
 # (extension-server + the extensionManager Envoy Gateway registered on the
-# `datum-downstream-gateway` GatewayClass, with the Coraza filter). The stock
-# `make prepare-infra-cluster` does NOT install this; this test is skipped by
-# CI environments without it (the Gateway never reaches Programmed=True).
+# `datum-downstream-gateway` GatewayClass, with the Coraza filter).
+# `make prepare-infra-cluster` installs this via `make downstream-waf-dataplane`.
 spec:
   cluster: nso-infra
   # EG only reconciles namespaces carrying this label.
@@ -124,6 +123,11 @@ spec:
                   - type: OWASPCoreRuleSet
                     owaspCoreRuleSet: {}
         - assert:
+            # The downstream Envoy fleet pulls the contrib Envoy and Coraza WAF
+            # init images cold on the kind node before its endpoints exist, and
+            # Envoy Gateway only writes Programmed once they do. That first-pull
+            # window overruns chainsaw's 30s assert default, so allow more time.
+            timeout: 3m
             resource:
               apiVersion: gateway.networking.k8s.io/v1
               kind: Gateway
@@ -132,6 +136,18 @@ spec:
               status:
                 (conditions[?type == 'Programmed']):
                   - status: "True"
+      catch:
+        - script:
+            timeout: 60s
+            content: |
+              set -x
+              kubectl get gateway -A -o yaml
+              kubectl describe gateway -A
+              kubectl get gatewayclass datum-downstream-gateway -o yaml
+              kubectl -n datum-downstream-gateway get pods -o wide
+              kubectl -n datum-downstream-gateway describe pods
+              kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=-1
+              kubectl -n datum-downstream-gateway logs -l gateway.envoyproxy.io/owning-gateway-namespace --all-containers --tail=200 || true
 
     - name: Benign traffic must reach the backend (not an empty/5xx reply)
       description: >