diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 6e86e4e9..badc3109 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -64,3 +64,29 @@ jobs: - name: Running Test e2e run: make test-e2e + + - name: Dump downstream WAF data plane on failure + if: failure() + run: | + kubectl config use-context kind-nso-infra + echo "::group::GatewayClasses" + kubectl get gatewayclass -o wide || true + kubectl describe gatewayclass datum-downstream-gateway || true + echo "::endgroup::" + echo "::group::EnvoyProxy" + kubectl -n datum-downstream-gateway get envoyproxy -o yaml || true + echo "::endgroup::" + echo "::group::datum-downstream-gateway namespace + pods" + kubectl get ns datum-downstream-gateway --show-labels || true + kubectl -n datum-downstream-gateway get pods -o wide || true + echo "::endgroup::" + echo "::group::Downstream Envoy Gateway logs" + kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200 || true + echo "::endgroup::" + echo "::group::Extension server" + kubectl -n datum-downstream-gateway describe deploy envoy-gateway-extension-server || true + kubectl -n datum-downstream-gateway logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200 || true + echo "::endgroup::" + echo "::group::Certificates" + kubectl -n datum-downstream-gateway get certificate,secret || true + echo "::endgroup::" diff --git a/Makefile b/Makefile index fb65611d..64301f0f 100644 --- a/Makefile +++ b/Makefile @@ -153,7 +153,7 @@ set-image-controller: manifests kustomize cd config/manager && $(KUSTOMIZE) edit set image ghcr.io/datum-cloud/network-services-operator=${IMG} .PHONY: prepare-infra-cluster -prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds +prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds downstream-waf-dataplane .PHONY: downstream-crds downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the replicator mirrors into it. @@ -161,6 +161,11 @@ downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml +.PHONY: downstream-waf-dataplane +downstream-waf-dataplane: kustomize load-image-infra ## Install the Coraza WAF data plane (downstream EG instance + extension server + datum-downstream-gateway GatewayClass) on the infra cluster. + $(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway-downstream | kubectl apply --server-side=true --force-conflicts -f - + $(KUSTOMIZE) build config/e2e-downstream | kubectl apply --server-side=true --force-conflicts -f - + .PHONY: prepare-e2e prepare-e2e: chainsaw set-image-controller cert-manager load-image-all deploy-e2e @@ -174,6 +179,10 @@ load-image-all: load-image-operator load-image-operator: docker-build kind $(KIND) load docker-image $(IMG) -n nso-standard +.PHONY: load-image-infra +load-image-infra: docker-build kind + $(KIND) load docker-image $(IMG) -n nso-infra + .PHONY: cert-manager cert-manager: cmctl $(KUSTOMIZE) build --enable-helm config/tools/cert-manager | kubectl apply --server-side=true --force-conflicts -f - diff --git a/config/dev/downstream_resources/downstream-gateway.yaml b/config/dev/downstream_resources/downstream-gateway.yaml index d6905672..3629e5a7 100644 --- a/config/dev/downstream_resources/downstream-gateway.yaml +++ b/config/dev/downstream_resources/downstream-gateway.yaml @@ -67,7 +67,7 @@ spec: imagePullPolicy: IfNotPresent command: - cp - - /app/coraza-waf.so + - /coraza-waf.so - /opt/coraza-waf/ volumeMounts: - name: coraza-waf diff --git a/config/e2e-downstream/extension-server-config.yaml b/config/e2e-downstream/extension-server-config.yaml new file mode 100644 index 00000000..f4c00b6d --- /dev/null +++ b/config/e2e-downstream/extension-server-config.yaml @@ -0,0 +1,12 @@ +apiVersion: apiserver.config.datumapis.com/v1alpha1 +kind: NetworkServicesOperator +gateway: + coraza: + routeBaseDirectives: + - Include @crs-setup-conf + - Include @recommended-conf + - SecResponseBodyLimitAction ProcessPartial + - SecResponseBodyLimit 32000 + errorPage: + enabled: true + bodyPath: /etc/datum/error-pages/error-5xx.html diff --git a/config/e2e-downstream/kustomization.yaml b/config/e2e-downstream/kustomization.yaml new file mode 100644 index 00000000..b00e83e0 --- /dev/null +++ b/config/e2e-downstream/kustomization.yaml @@ -0,0 +1,76 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: datum-downstream-gateway + +resources: + - namespace.yaml + - ../extension-server + - pki.yaml + - waf-gateway.yaml + +configMapGenerator: + - name: nso-extension-server-config + files: + - config.yaml=extension-server-config.yaml + options: + disableNameSuffixHash: true + +patches: + - target: + kind: ServiceMonitor + name: envoy-gateway-extension-server-metrics + patch: | + $patch: delete + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: envoy-gateway-extension-server-metrics + - target: + kind: Certificate + name: envoy-gateway-extension-server-eg-client-tls + patch: | + - op: replace + path: /spec/issuerRef/kind + value: Issuer + - op: replace + path: /spec/issuerRef/name + value: envoy-gateway-extension-server + - target: + kind: Deployment + name: envoy-gateway-extension-server + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: envoy-gateway-extension-server + spec: + replicas: 1 + template: + spec: + containers: + - name: envoy-gateway-extension-server + env: + - name: SERVER_CONFIG + value: /config/config.yaml + volumeMounts: + - name: server-config + mountPath: /config + readOnly: true + volumes: + - name: tls + csi: + volumeAttributes: + csi.cert-manager.io/issuer-kind: Issuer + csi.cert-manager.io/issuer-name: envoy-gateway-extension-server + csi.cert-manager.io/dns-names: "envoy-gateway-extension-server.datum-downstream-gateway.svc,envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local" + - name: tls-ca + configMap: null + secret: + secretName: envoy-gateway-extension-server-ca + items: + - key: ca.crt + path: ca.crt + - name: server-config + configMap: + name: nso-extension-server-config diff --git a/config/e2e-downstream/namespace.yaml b/config/e2e-downstream/namespace.yaml new file mode 100644 index 00000000..89c7d150 --- /dev/null +++ b/config/e2e-downstream/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: datum-downstream-gateway + labels: + meta.datumapis.com/upstream-cluster-name: 0-datum-internal-anchor diff --git a/config/e2e-downstream/pki.yaml b/config/e2e-downstream/pki.yaml new file mode 100644 index 00000000..666bb0b3 --- /dev/null +++ b/config/e2e-downstream/pki.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: envoy-gateway-extension-server-selfsigned +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: envoy-gateway-extension-server-ca +spec: + isCA: true + commonName: envoy-gateway-extension-server-ca + secretName: envoy-gateway-extension-server-ca + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: envoy-gateway-extension-server-selfsigned + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: envoy-gateway-extension-server +spec: + ca: + secretName: envoy-gateway-extension-server-ca diff --git a/config/e2e-downstream/waf-gateway.yaml b/config/e2e-downstream/waf-gateway.yaml new file mode 100644 index 00000000..ef6a6265 --- /dev/null +++ b/config/e2e-downstream/waf-gateway.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: datum-downstream-gateway +spec: + mergeGateways: true + ipFamily: DualStack + provider: + type: Kubernetes + kubernetes: + envoyDeployment: + pod: + volumes: + - name: coraza-waf + emptyDir: {} + container: + image: envoyproxy/envoy:contrib-v1.35.0 + volumeMounts: + - name: coraza-waf + mountPath: /opt/coraza-waf + initContainers: + - name: coraza-waf + image: ghcr.io/datum-labs/coraza-envoy-go-filter/coraza-waf:v1.3.0-alpha10 + imagePullPolicy: IfNotPresent + command: + - cp + - /coraza-waf.so + - /opt/coraza-waf/ + volumeMounts: + - name: coraza-waf + mountPath: /opt/coraza-waf + envoyService: + externalTrafficPolicy: Cluster + type: NodePort + patch: + type: StrategicMerge + value: + spec: + ipFamilyPolicy: RequireDualStack + ports: + - name: http-80 + nodePort: 30080 + port: 80 + - name: https-443 + nodePort: 30443 + port: 443 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: datum-downstream-gateway +spec: + controllerName: gateway.envoyproxy.io/datum-downstream-gateway + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: datum-downstream-gateway + namespace: datum-downstream-gateway diff --git a/config/tools/envoy-gateway-downstream/kustomization.yaml b/config/tools/envoy-gateway-downstream/kustomization.yaml index d6cce819..f95b40f2 100644 --- a/config/tools/envoy-gateway-downstream/kustomization.yaml +++ b/config/tools/envoy-gateway-downstream/kustomization.yaml @@ -6,7 +6,11 @@ helmCharts: includeCRDs: false namespace: datum-downstream-gateway releaseName: envoy-datum-downstream-gateway - version: v1.8.1 + # Pinned to match production: v1.8.x does not program merged gateways served + # through the extension manager (waf-gw stays with an empty status), so the + # downstream WAF control plane runs v1.7.4. CRDs come from the base + # envoy-gateway install (v1.8.1), which is forward-compatible. + version: v1.7.4 repo: oci://docker.io/envoyproxy valuesInline: config: @@ -29,6 +33,10 @@ helmCharts: - key: meta.datumapis.com/upstream-cluster-name operator: Exists extensionManager: + # The WAF-augmented merged xDS response exceeds gRPC's 4 MiB default, + # so EG rejects it and stops publishing config with failOpen:false. + # Match the ext-server's 256 MiB limit; both sides must agree. + maxMessageSize: 256Mi policyResources: - group: networking.datumapis.com version: v1alpha @@ -39,16 +47,17 @@ helmCharts: kind: Connector service: fqdn: - hostname: network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local + hostname: envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local port: 5005 tls: + # EG reads the CA from tls.crt of this Secret; the self-signed CA + # Certificate's Secret carries the CA cert there. certificateRef: - # Placeholder — an overlay must patch this to the Secret holding the CA that issued the server cert. - name: placeholder-ca - namespace: placeholder-namespace + name: envoy-gateway-extension-server-ca + namespace: datum-downstream-gateway clientCertificateRef: name: envoy-gateway-extension-server-eg-client-tls - namespace: network-services-operator-system + namespace: datum-downstream-gateway retry: maxAttempts: 4 initialBackoff: 100ms diff --git a/config/tools/envoy-gateway-downstream/namespace.yaml b/config/tools/envoy-gateway-downstream/namespace.yaml index 6cecd275..89c7d150 100644 --- a/config/tools/envoy-gateway-downstream/namespace.yaml +++ b/config/tools/envoy-gateway-downstream/namespace.yaml @@ -2,3 +2,5 @@ apiVersion: v1 kind: Namespace metadata: name: datum-downstream-gateway + labels: + meta.datumapis.com/upstream-cluster-name: 0-datum-internal-anchor diff --git a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml index 5d684ff7..9e42f2e2 100644 --- a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml +++ b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml @@ -12,9 +12,8 @@ metadata: # # Precondition: the downstream (nso-infra) must have the WAF data plane wired up # (extension-server + the extensionManager Envoy Gateway registered on the -# `datum-downstream-gateway` GatewayClass, with the Coraza filter). The stock -# `make prepare-infra-cluster` does NOT install this; this test is skipped by -# CI environments without it (the Gateway never reaches Programmed=True). +# `datum-downstream-gateway` GatewayClass, with the Coraza filter). +# `make prepare-infra-cluster` installs this via `make downstream-waf-dataplane`. spec: cluster: nso-infra # EG only reconciles namespaces carrying this label. @@ -124,6 +123,11 @@ spec: - type: OWASPCoreRuleSet owaspCoreRuleSet: {} - assert: + # The downstream Envoy fleet pulls the contrib Envoy and Coraza WAF + # init images cold on the kind node before its endpoints exist, and + # Envoy Gateway only writes Programmed once they do. That first-pull + # window overruns chainsaw's 30s assert default, so allow more time. + timeout: 3m resource: apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -132,6 +136,18 @@ spec: status: (conditions[?type == 'Programmed']): - status: "True" + catch: + - script: + timeout: 60s + content: | + set -x + kubectl get gateway -A -o yaml + kubectl describe gateway -A + kubectl get gatewayclass datum-downstream-gateway -o yaml + kubectl -n datum-downstream-gateway get pods -o wide + kubectl -n datum-downstream-gateway describe pods + kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=-1 + kubectl -n datum-downstream-gateway logs -l gateway.envoyproxy.io/owning-gateway-namespace --all-containers --tail=200 || true - name: Benign traffic must reach the backend (not an empty/5xx reply) description: >