diff --git a/.github/workflows/test-e2e-federated.yml b/.github/workflows/test-e2e-federated.yml new file mode 100644 index 00000000..03966f2b --- /dev/null +++ b/.github/workflows/test-e2e-federated.yml @@ -0,0 +1,128 @@ +name: Federated E2E + +# Exercises the production-fidelity edge: two kind clusters (control plane + edge) +# wired with real Karmada federation, the production Envoy Gateway version, the +# Coraza WAF data plane, and the NSO extension server — then runs the edge e2e +# scenarios against real traffic. This is the path the older single-stack +# `make test-e2e` never covered. +on: + pull_request: + branches: + - main + paths: + - 'Taskfile.test-infra.yml' + - 'config/e2e-downstream/**' + - 'config/federation/**' + - 'config/tools/**' + - 'config/extension-server/**' + - 'internal/extensionserver/**' + - 'cmd/**' + - 'test/e2e-edge/**' + - 'test/parity/**' + - '.github/workflows/test-e2e-federated.yml' + push: + branches: + - main + workflow_dispatch: + +# One federated run per ref; a newer push cancels an in-flight one. +concurrency: + group: federated-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + federated-e2e: + name: Two-cluster federated edge (Karmada) + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + TMPDIR: /tmp + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '~1.26' + + # Two kind clusters plus a full Karmada control plane and the edge data + # plane need more room than the runner ships with. Reclaim disk from + # toolchains this job never uses. + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL + docker system prune -af || true + df -h / + + # Karmada plus two clusters open a large number of file watches; the + # default runner limits crashloop a component with "too many open files". + - name: Raise inotify limits + run: | + sudo sysctl -w fs.inotify.max_user_instances=8192 + sudo sysctl -w fs.inotify.max_user_watches=1048576 + + - name: Install Task + run: | + go install github.com/go-task/task/v3/cmd/task@v3.49.1 + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + + - name: Install pinned tools (kind, karmadactl, kustomize, cmctl, chainsaw) + run: task test-infra:tools + + - name: Bring up the two-cluster prod-fidelity env + run: task test-infra:up + + - name: Stand up Karmada federation + run: task test-infra:karmada-up + + - name: Run edge e2e scenarios + run: task test-infra:e2e + + # When something fails, capture enough cross-cluster state to diagnose it + # from the logs alone — the runner is torn down immediately after. + - name: Diagnostics on failure + if: failure() + run: | + set +e + echo "::group::upstream pods" + kubectl --context kind-nso-upstream get pods -A -o wide + echo "::endgroup::" + echo "::group::downstream pods" + kubectl --context kind-nso-downstream get pods -A -o wide + echo "::endgroup::" + echo "::group::not-ready pods — describe (both clusters)" + for ctx in kind-nso-upstream kind-nso-downstream; do + echo "### $ctx" + kubectl --context "$ctx" get pods -A \ + --field-selector=status.phase!=Running,status.phase!=Succeeded \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' \ + | while read -r ns name; do + [ -n "$name" ] || continue + kubectl --context "$ctx" -n "$ns" describe pod "$name" 2>/dev/null | tail -30 + done + done + echo "::endgroup::" + echo "::group::cert-manager health (upstream) — restarts + issuance" + kubectl --context kind-nso-upstream -n cert-manager get pods -o wide + kubectl --context kind-nso-upstream -n cert-manager \ + logs -l app.kubernetes.io/name=cert-manager --tail=80 --previous 2>/dev/null || true + kubectl --context kind-nso-upstream get certificate,certificaterequest,issuer,clusterissuer -A + echo "::endgroup::" + echo "::group::karmada member clusters" + KCFG=/tmp/karmada-nso-upstream/karmada-apiserver.config + kubectl --kubeconfig "$KCFG" --context karmada-apiserver get clusters -o wide + kubectl --kubeconfig "$KCFG" --context karmada-apiserver get clusterpropagationpolicy,work -A + echo "::endgroup::" + echo "::group::NSO manager logs (upstream)" + kubectl --context kind-nso-upstream -n network-services-operator-system \ + logs deploy/network-services-operator-controller-manager --tail=200 + echo "::endgroup::" + echo "::group::extension server logs (downstream)" + kubectl --context kind-nso-downstream -n network-services-operator-system \ + logs deploy/network-services-operator-envoy-gateway-extension-server --tail=200 + echo "::endgroup::" + echo "::group::downstream Envoy Gateway logs" + kubectl --context kind-nso-downstream -n datum-downstream-gateway \ + logs deploy/envoy-gateway --tail=200 + echo "::endgroup::" diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml new file mode 100644 index 00000000..37c56658 --- /dev/null +++ b/Taskfile.test-infra.yml @@ -0,0 +1,481 @@ +version: '3' + +# Production-fidelity test environment. Two kind clusters: nso-upstream (the +# control plane) and nso-downstream (the edge). Consumed by both the e2e and perf +# suites. +# +# Bring up: task test-infra:up +# Tear down: task test-infra:down +# Smoke: task test-infra:smoke + +# kustomize fetches some CRDs over https. A developer's global git config can +# rewrite https to ssh, which fails in a non-interactive fetch. Neutralize the +# global git config for kustomize's child git so the https fetches succeed; the +# repo-local config is untouched. +env: + GIT_CONFIG_GLOBAL: /dev/null + +vars: + TMP_DIR: + sh: echo "${TMPDIR:-/tmp}" + REPO: + sh: pwd + # Pinned-to-prod tooling. We use an explicit kind v0.32.0 binary because it + # ships the node image production runs, which the checked-in bin/kind predates. + KIND: '{{.REPO}}/bin/kind-v0.32.0' + KUSTOMIZE: '{{.REPO}}/bin/kustomize' + CMCTL: '{{.REPO}}/bin/cmctl' + KARMADACTL: '{{.REPO}}/bin/karmadactl' + CHAINSAW: '{{.REPO}}/bin/chainsaw' + + UPSTREAM_CLUSTER: nso-upstream + DOWNSTREAM_CLUSTER: nso-downstream + UPSTREAM_CTX: kind-nso-upstream + DOWNSTREAM_CTX: kind-nso-downstream + + # The exact node image the production edge runs. + K8S_NODE_IMAGE: kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95 + # Gateway control-plane version, matching production. The extension-server SDK + # version is paired with it deliberately, also matching production. + ENVOY_GATEWAY_VERSION: v1.7.4 + EG_SDK_VERSION: v1.8.1 + # Multi-arch WAF image — the same filter as the production edge, but it also + # loads natively on arm64 dev hosts. The data plane references this image. + CORAZA_WAF_IMAGE: ghcr.io/datum-labs/coraza-envoy-go-filter/coraza-waf:v1.3.0-multiarch.1 + # CONNECT-proxy stand-in for the connector-tunnel scenario, built from + # test/e2e-edge/_fixtures/connector-tunnel and loaded into the downstream cluster. + CONNECT_PROXY_IMAGE: connect-proxy:e2e + # The WAF is enabled by default since the multi-arch image loads on every + # architecture. Set CORAZA_DISABLED=true only to deliberately skip the WAF. + CORAZA_DISABLED: '{{.CORAZA_DISABLED | default "false"}}' + # Karmada-bundled kube-apiserver, pinned to production. + KARMADA_APISERVER_VERSION: v1.33.2 + # NSO image pinned by git SHA so the loaded image is used deterministically. + IMG: + sh: echo "ghcr.io/datum-cloud/network-services-operator:$(git rev-parse --short HEAD)" + +tasks: + + tools: + desc: "Install the pinned tool binaries this env needs into bin/ (kind v0.32.0, karmadactl, kustomize, cmctl, chainsaw). Idempotent — skips any that are already present. Runs on CI; a local dev can run it once instead of hand-placing the binaries." + vars: + # kind is pinned to the version that ships the production edge node image; + # the others match the versions the Makefile installs. + KIND_TOOL_VERSION: v0.32.0 + KARMADACTL_VERSION: v1.15.2 + KUSTOMIZE_VERSION: v5.5.0 + CMCTL_VERSION: v2.1.1 + CHAINSAW_VERSION: v0.2.15 + cmds: + - mkdir -p {{.REPO}}/bin + # go install writes bin/kind; rename it to the versioned path the env pins. + - | + if [ ! -x {{.REPO}}/bin/kind-{{.KIND_TOOL_VERSION}} ]; then + echo "installing kind {{.KIND_TOOL_VERSION}}" + GOBIN={{.REPO}}/bin go install sigs.k8s.io/kind@{{.KIND_TOOL_VERSION}} + mv {{.REPO}}/bin/kind {{.REPO}}/bin/kind-{{.KIND_TOOL_VERSION}} + fi + # kustomize / cmctl / chainsaw go-install directly under their plain names, + # which is exactly what this Taskfile references. + - '[ -x {{.REPO}}/bin/kustomize ] || GOBIN={{.REPO}}/bin go install sigs.k8s.io/kustomize/kustomize/v5@{{.KUSTOMIZE_VERSION}}' + - '[ -x {{.REPO}}/bin/cmctl ] || GOBIN={{.REPO}}/bin go install github.com/cert-manager/cmctl/v2@{{.CMCTL_VERSION}}' + - '[ -x {{.REPO}}/bin/chainsaw ] || GOBIN={{.REPO}}/bin go install github.com/kyverno/chainsaw@{{.CHAINSAW_VERSION}}' + # karmadactl carries replace directives in its go.mod, so `go install` can't + # build it; pull the published release binary for this OS/arch instead. + - | + if [ ! -x {{.REPO}}/bin/karmadactl ]; then + url="https://github.com/karmada-io/karmada/releases/download/{{.KARMADACTL_VERSION}}/karmadactl-{{OS}}-{{ARCH}}.tgz" + echo "downloading karmadactl {{.KARMADACTL_VERSION}} ({{OS}}/{{ARCH}})" + curl -fsSL "$url" | tar -xz -C {{.REPO}}/bin karmadactl + fi + - echo "✅ tools ready in {{.REPO}}/bin" + + up: + desc: "Bring the canonical two-cluster prod-fidelity test env ONLINE (clusters + EG v1.7.4 + ext-server + Coraza data plane + NSO manager)." + cmds: + - echo "🚀 test-infra:up — building nso-upstream + nso-downstream" + - task: clusters + # The gateway CRDs live inside the Envoy Gateway helm chart, which is + # gitignored (config/**/charts). Pull it first so eg-crds can apply them — + # on a dev machine it is already vendored, on CI it is fetched fresh. + - task: eg-chart + # The Gateway-API and gateway CRDs must land on the downstream cluster + # before cert-manager (it probes the Gateway-API CRDs at startup) and before + # the downstream gateway (it needs its own CRDs present). + - task: eg-crds + - task: downstream-crds + - task: downstream-namespaces + # cert-manager runs with enableGatewayAPI:true and crashloops at startup if + # the Gateway-API CRDs are absent. The downstream gets them from eg-crds; the + # upstream otherwise wouldn't until prepare-upstream (config/e2e), far too + # late — the crashloop delays the manager's webhook cert and stalls bring-up. + - task: upstream-gateway-crds + - task: cert-manager-upstream + - task: cert-manager-downstream + - task: nso-image + - task: load-fixtures + - task: prepare-upstream + - task: eg-downstream + - task: extension-server + - task: external-dns + - task: link-clusters + - task: wait-ready + - echo "🎉 test-infra:up complete. Run 'task test-infra:smoke' to confirm M2." + + down: + desc: "Tear down the test-infra clusters (and any Karmada host)." + cmds: + - echo "💥 test-infra:down" + # Deleting both kind clusters removes all federation state, so this unjoin + # is only a best-effort courtesy. + - '{{.KARMADACTL}} deinit --context {{.UPSTREAM_CTX}} --force --purge-namespace 2>/dev/null || true' + - rm -rf {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} + - '{{.KIND}} delete cluster --name {{.UPSTREAM_CLUSTER}} || true' + - '{{.KIND}} delete cluster --name {{.DOWNSTREAM_CLUSTER}} || true' + - rm -f {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml {{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + - echo "✨ done." + + clusters: + desc: "Create the upstream + downstream kind clusters pinned to the prod edge node image." + cmds: + - echo "🏗️ creating clusters (node {{.K8S_NODE_IMAGE}})" + - '{{.KIND}} delete cluster --name {{.UPSTREAM_CLUSTER}} 2>/dev/null || true' + - '{{.KIND}} delete cluster --name {{.DOWNSTREAM_CLUSTER}} 2>/dev/null || true' + - '{{.KIND}} create cluster --image {{.K8S_NODE_IMAGE}} --config=config/tools/kind/upstream-cluster.yaml' + - '{{.KIND}} create cluster --image {{.K8S_NODE_IMAGE}} --config=config/tools/kind/downstream-cluster.yaml' + # Running the full federation control plane alongside the rest of the stack + # in one kind node exhausts the default file-watch limits on the macOS + # Docker VM, which crashloops a component with "too many open files" and + # blocks the member join. Raise the limits on both nodes. + - docker exec {{.UPSTREAM_CLUSTER}}-control-plane sysctl -w fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 + - docker exec {{.DOWNSTREAM_CLUSTER}}-control-plane sysctl -w fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 + + cert-manager-upstream: + desc: "Install cert-manager (+ CSI driver) on the upstream cluster." + cmds: + - '{{.KUSTOMIZE}} build --enable-helm config/tools/cert-manager | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + - '{{.CMCTL}} check api --context {{.UPSTREAM_CTX}} --wait=5m' + + cert-manager-downstream: + desc: "Install cert-manager (+ CSI driver) on the downstream cluster." + cmds: + - '{{.KUSTOMIZE}} build --enable-helm config/tools/cert-manager | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + - '{{.CMCTL}} check api --context {{.DOWNSTREAM_CTX}} --wait=5m' + + nso-image: + desc: "Build the NSO operator image (git-SHA tag) and load it into both clusters." + cmds: + - echo "🔨 building {{.IMG}}" + - docker build -t {{.IMG}} . + - cd config/manager && {{.KUSTOMIZE}} edit set image ghcr.io/datum-cloud/network-services-operator={{.IMG}} + - '{{.KIND}} load docker-image {{.IMG}} --name {{.UPSTREAM_CLUSTER}}' + - '{{.KIND}} load docker-image {{.IMG}} --name {{.DOWNSTREAM_CLUSTER}}' + + prepare-upstream: + desc: "Deploy the NSO manager + webhook (config/e2e) on the upstream cluster, with the prod-base memory profile." + cmds: + - echo "🔧 deploying NSO manager (upstream)" + - '{{.KUSTOMIZE}} build config/e2e | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + # Match production: with the extension server handling proxy configuration, + # NSO must not also emit its own patch policies. The shared config file is + # also used by an older path that still relies on them, so we patch the live + # config here rather than editing that file. + - | + CFG=$(kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system get cm network-services-operator-config -o jsonpath='{.data.config\.yaml}') + if ! echo "$CFG" | grep -q "eppEmissionEnabled"; then + NEW=$(echo "$CFG" | sed 's|^ downstreamGatewayClassName: .*|&\n eppEmissionEnabled: false|') + kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system create cm network-services-operator-config \ + --from-literal=config.yaml="$NEW" --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + fi + # Raise the manager's memory limit to the production base so perf sweeps and + # webhook-under-load reflect production headroom. + - | + kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system patch deploy network-services-operator-controller-manager \ + --type=json -p '[{"op":"replace","path":"/spec/template/spec/containers/0/resources/limits/memory","value":"2Gi"},{"op":"replace","path":"/spec/template/spec/containers/0/resources/requests/memory","value":"512Mi"}]' || true + + eg-chart: + desc: "Vendor the Envoy Gateway helm chart into config/tools/envoy-gateway-downstream/charts so eg-crds can apply the CRDs bundled inside it. The chart dir is gitignored (config/**/charts); this pulls it if absent. Idempotent — a no-op when it is already vendored (dev machine) and a fresh pull on CI." + vars: + CHART_DIR: config/tools/envoy-gateway-downstream/charts/gateway-helm-{{.ENVOY_GATEWAY_VERSION}} + cmds: + - | + if [ ! -d "{{.CHART_DIR}}/gateway-helm/crds" ]; then + echo "📦 pulling gateway-helm {{.ENVOY_GATEWAY_VERSION}} from oci://docker.io/envoyproxy" + mkdir -p "{{.CHART_DIR}}" + helm pull oci://docker.io/envoyproxy/gateway-helm --version {{.ENVOY_GATEWAY_VERSION}} --untar --untardir "{{.CHART_DIR}}" + fi + + eg-crds: + desc: "Install the full Gateway-API + Envoy Gateway CRD set (incl ReferenceGrant + EnvoyProxy) on the downstream cluster. The downstream EG chart sets includeCRDs:false, so CRDs are applied separately (matches prod, where CRDs are managed out-of-band)." + vars: + EG_CRD_DIR: config/tools/envoy-gateway-downstream/charts/gateway-helm-{{.ENVOY_GATEWAY_VERSION}}/gateway-helm/crds + cmds: + - kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f {{.EG_CRD_DIR}}/gatewayapi-crds.yaml + - kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f {{.EG_CRD_DIR}}/generated/ + + upstream-gateway-crds: + desc: "Install the Gateway-API CRDs on the upstream cluster so cert-manager (enableGatewayAPI:true) starts cleanly instead of crashlooping until config/e2e lands them. Uses the same chart-bundled CRDs as eg-crds; requires eg-chart to have pulled the chart." + vars: + EG_CRD_DIR: config/tools/envoy-gateway-downstream/charts/gateway-helm-{{.ENVOY_GATEWAY_VERSION}}/gateway-helm/crds + cmds: + - kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f {{.EG_CRD_DIR}}/gatewayapi-crds.yaml + + downstream-namespaces: + desc: "Create the downstream gateway + hostname-accounting namespaces, plus the EG-watched e2e-direct namespace for hand-delivered fixtures (D1/D2). Runs BEFORE eg-downstream so e2e-direct carries its watch label before EG establishes its informer." + cmds: + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/dev/downstream_resources/namespaces.yaml + # e2e-direct carries the gateway watch label from creation so gateways + # applied directly here reconcile deterministically. The gateway only + # reliably watches a namespace labeled at creation time, so the label must + # be in the manifest, not added afterward. + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/e2e-downstream/direct-namespace.yaml + + eg-downstream: + desc: "Install the dedicated downstream Envoy Gateway (v1.7.4) with the ext-server extensionManager wiring on the downstream cluster." + cmds: + - echo "🔧 installing downstream EG {{.ENVOY_GATEWAY_VERSION}} (+ extensionManager, failOpen:false, maxMessageSize:256Mi)" + # Build from the untracked config/e2e-downstream/eg-downstream copy, which + # bakes in the e2e pins, so a hard reset on the shared branch cannot revert + # them — it kept clobbering the tracked copy. + - '{{.KUSTOMIZE}} build --enable-helm config/e2e-downstream/eg-downstream | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + + load-coraza-waf: + desc: "Fallback: pull the Coraza WAF image and kind-load it into the downstream cluster (offline-CI escape hatch)." + cmds: + - docker pull {{.CORAZA_WAF_IMAGE}} + - '{{.KIND}} load docker-image {{.CORAZA_WAF_IMAGE}} --name {{.DOWNSTREAM_CLUSTER}}' + + load-fixtures: + desc: "Build + load the e2e fixture images into the downstream cluster (CONNECT-proxy stand-in for the connector-tunnel scenario)." + cmds: + - echo "🔧 building + loading fixture image {{.CONNECT_PROXY_IMAGE}}" + - docker build -t {{.CONNECT_PROXY_IMAGE}} test/e2e-edge/_fixtures/connector-tunnel + - '{{.KIND}} load docker-image {{.CONNECT_PROXY_IMAGE}} --name {{.DOWNSTREAM_CLUSTER}}' + + d1-mint-expired-secret: + desc: "D1 helper: mint an ALREADY-EXPIRED kubernetes.io/tls Secret and apply it to the downstream cluster, bypassing the upstream #212 cert-health gate so the ext-server prune backstop can be tested in isolation. Vars: NAMESPACE, SECRET, HOSTNAME." + vars: + # Default to the pre-provisioned, gateway-watched namespace so a gateway + # applied alongside this secret reconciles deterministically. + NAMESPACE: '{{.NAMESPACE | default "e2e-direct"}}' + SECRET: '{{.SECRET | default "d1-expired-tls"}}' + HOSTNAME: '{{.HOSTNAME | default "d1-bad.e2e.env.datum.net"}}' + cmds: + - | + config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh \ + "{{.NAMESPACE}}" "{{.SECRET}}" "{{.HOSTNAME}}" \ + | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + - echo "✅ applied expired TLS Secret {{.NAMESPACE}}/{{.SECRET}} (host {{.HOSTNAME}}) to {{.DOWNSTREAM_CLUSTER}}" + + extension-server: + desc: "Deploy the ext-server (2 replicas + PDB + mTLS) + e2e cert chain + Coraza/branded-page config + test EnvoyProxy (Coraza+admin:19000) on the downstream cluster." + cmds: + - echo "🔧 deploying ext-server + e2e issuer chain + test EnvoyProxy (config/e2e-downstream)" + # Pin the ext-server image to the git-SHA build (same image as the manager). + - cd config/e2e-downstream && {{.KUSTOMIZE}} edit set image ghcr.io/datum-cloud/network-services-operator={{.IMG}} 2>/dev/null || true + - '{{.KUSTOMIZE}} build config/e2e-downstream | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + # WAF disable toggle. The overlay ships the WAF enabled for production + # fidelity; when CORAZA_DISABLED is set we flip the live config so listeners + # program without it. + - | + if [ "{{.CORAZA_DISABLED}}" = "true" ]; then + echo "⚠️ Coraza WAF disabled (host arch {{OS}}/$(uname -m); WAF .so is amd64-only)" + kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system get cm extension-server-config -o yaml \ + | sed 's/disabled: false/disabled: true/' | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + fi + # The extension server's server cert and the gateway's client cert are both + # issued from the e2e certificate authority. Wait for that authority's cert, + # then publish it where each side reads it so they can verify each other. + - task: extserver-ca-bundle + + extserver-ca-bundle: + desc: "Publish the e2e CA's ca.crt into the ext-server CA-bundle ConfigMap and the EG certificateRef Secret (both in network-services-operator-system)." + cmds: + - echo "⏳ waiting for the e2e CA certificate to be issued" + - | + kubectl --context {{.DOWNSTREAM_CTX}} -n cert-manager wait certificate e2e-extension-server-ca \ + --for=condition=Ready --timeout=120s + - kubectl --context {{.DOWNSTREAM_CTX}} create namespace network-services-operator-system --dry-run=client -o yaml | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + - | + CA_CRT=$(kubectl --context {{.DOWNSTREAM_CTX}} -n cert-manager get secret e2e-extension-server-ca -o jsonpath='{.data.ca\.crt}' | base64 -d) + # The certificate-authority bundle the extension server mounts. + kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system create configmap extension-server-ca-bundle \ + --from-literal=ca.crt="$CA_CRT" --dry-run=client -o yaml | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + # The same authority cert the downstream gateway references. The gateway + # reads it from the tls.crt key, not ca.crt. + kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system create secret generic e2e-extension-server-ca \ + --from-literal=tls.crt="$CA_CRT" --dry-run=client -o yaml | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + # Restart the extension server so it picks up the freshly-published bundle. + - kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system rollout restart deploy network-services-operator-envoy-gateway-extension-server || true + + downstream-crds: + desc: "Install the NSO CRDs the replicator mirrors into the downstream cluster (the Gateway-API/EG CRDs come from eg-crds)." + cmds: + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_connectors.yaml + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml + + external-dns: + desc: "Install external-dns CRDs (DNSEndpoint) on the downstream cluster." + cmds: + - '{{.KUSTOMIZE}} build --enable-helm config/tools/external-dns | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + + link-clusters: + desc: "Wire the NSO manager's downstream client to the downstream cluster via the downstream-cluster-kubeconfig secret on the upstream cluster." + cmds: + - echo "🔗 linking upstream -> downstream" + - '{{.KIND}} get kubeconfig --name {{.DOWNSTREAM_CLUSTER}} --internal > {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml' + - kubectl --context {{.UPSTREAM_CTX}} create namespace network-services-operator-system --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + - | + kubectl --context {{.UPSTREAM_CTX}} create secret -n network-services-operator-system \ + generic downstream-cluster-kubeconfig --save-config --dry-run=client -o yaml \ + --from-file=kubeconfig={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + + wait-ready: + desc: "Wait for the core components (NSO manager, downstream EG, ext-server) to be Available." + cmds: + - echo "⏳ waiting for NSO manager (upstream)" + - kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system wait deploy network-services-operator-controller-manager --for=condition=Available --timeout=300s + - echo "⏳ waiting for downstream EG" + # The gateway control-plane Deployment is named `envoy-gateway` by its chart, + # not after the release name. + - kubectl --context {{.DOWNSTREAM_CTX}} -n datum-downstream-gateway wait deploy envoy-gateway --for=condition=Available --timeout=240s + - echo "⏳ waiting for ext-server" + - kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system wait deploy network-services-operator-envoy-gateway-extension-server --for=condition=Available --timeout=240s + - echo "✅ core components ready." + + # ---- Karmada ------------------------------------------------------------ + + karmada-up: + desc: "Stand up a real Karmada host on the upstream cluster (apiserver v1.33.2), join the downstream member, and apply the prod federation artifacts." + vars: + KARMADA_KUBECONFIG: '{{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}}/karmada-apiserver.config' + # The upstream node's docker-network IP, reachable from the downstream node + # on the same network, used to sign the Karmada apiserver cert. + HOST_IP: + sh: docker inspect {{.UPSTREAM_CLUSTER}}-control-plane -f '{{"{{"}}.NetworkSettings.Networks.kind.IPAddress{{"}}"}}' + # The downstream member's docker-network IP — reachable from the upstream + # control-plane pods on the same network, but not from the macOS host. + MEMBER_IP: + sh: docker inspect {{.DOWNSTREAM_CLUSTER}}-control-plane -f '{{"{{"}}.NetworkSettings.Networks.kind.IPAddress{{"}}"}}' + cmds: + # Advertise 127.0.0.1: karmadactl runs on the macOS host, and init's own + # post-deploy steps dial the advertise address. The docker-network IP is not + # routable from the macOS host, so advertising it would make init time out; + # the upstream cluster maps a host port through to the apiserver, so + # 127.0.0.1 reaches it. We still sign the cert for the docker IP for any + # in-cluster path. The member is joined in push mode, so it never needs to + # reach the apiserver address — advertising 127.0.0.1 is safe for the join. + - echo "🌐 karmada init (apiserver {{.KARMADA_APISERVER_VERSION}}, advertise 127.0.0.1:32443, cert-ip incl {{.HOST_IP}}) on {{.UPSTREAM_CLUSTER}}" + - mkdir -p {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} + - | + {{.KARMADACTL}} init --context {{.UPSTREAM_CTX}} \ + --kube-image-tag {{.KARMADA_APISERVER_VERSION}} \ + --karmada-apiserver-advertise-address 127.0.0.1 \ + --cert-external-ip "127.0.0.1,{{.HOST_IP}}" \ + --etcd-storage-mode hostPath \ + --karmada-data {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} \ + --karmada-pki {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}}/pki + - echo "🔗 joining {{.DOWNSTREAM_CLUSTER}} as a Karmada member" + # karmadactl join runs on the host and connects to the member to install its + # agent, so it needs a host-reachable member kubeconfig, not the internal + # docker-hostname form the host cannot resolve. + - '{{.KIND}} get kubeconfig --name {{.DOWNSTREAM_CLUSTER}} > {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-host.yaml' + - | + {{.KARMADACTL}} join {{.DOWNSTREAM_CLUSTER}} \ + --karmada-context karmada-apiserver \ + --kubeconfig {{.KARMADA_KUBECONFIG}} \ + --cluster-kubeconfig {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-host.yaml \ + --cluster-context {{.DOWNSTREAM_CTX}} + # The join stores the host address as the member's endpoint, which the + # control-plane pods running inside the upstream cluster cannot reach. + # Repoint it to the member's docker IP, which they can reach on the shared + # network, and the member then goes Ready. + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver \ + patch cluster {{.DOWNSTREAM_CLUSTER}} --type=merge \ + -p '{"spec":{"apiEndpoint":"https://{{.MEMBER_IP}}:6443"}}' + # Label the member so the production propagation policy places resources onto + # it. + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver \ + label cluster {{.DOWNSTREAM_CLUSTER}} infra.datum.net/gateways=enabled --overwrite + - echo "⏳ waiting for the member cluster to become Ready" + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver \ + wait cluster {{.DOWNSTREAM_CLUSTER}} --for=condition=Ready --timeout=120s + - echo "📜 applying federation artifacts (config/federation)" + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver apply -f config/federation/ + - echo "✅ karmada-up complete. Karmada apiserver kubeconfig at {{.KARMADA_KUBECONFIG}}" + + smoke: + desc: "M2 functional confirmation: drive an upstream Gateway+HTTPRoute through the ext-server path and curl a real 200." + cmds: + - '{{.CHAINSAW}} test ./test/e2e-edge/extension-server-smoke --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml' + + e2e: + desc: "Run chainsaw e2e scenarios against the live two-cluster env. Pass a scenario name or path after -- (e.g. `task test-infra:e2e -- waf-enforcement`); with no arg, runs every ext-server-path scenario that targets nso-upstream/nso-downstream. Use SCENARIOS=... to override the default set." + vars: + # Scenarios authored against this env's cluster names and downstream path. + # Older fixtures targeting the previous cluster names are excluded here. + DEFAULT_SCENARIOS: extension-server-smoke waf-enforcement trafficprotectionpolicy-enforce branded-error-page connector-offline-503 atomic-reject-isolation + # CLI_ARGS (after --) wins; else SCENARIOS env; else the default set. + SELECTED: '{{.CLI_ARGS | default .SCENARIOS | default .DEFAULT_SCENARIOS}}' + deps: + - kubeconfigs + cmds: + - | + set -e + for s in {{.SELECTED}}; do + # Accept either a bare scenario name or a full/relative path. + case "$s" in + test/e2e-edge/*|./test/e2e-edge/*) dir="$s" ;; + */*) dir="$s" ;; + *) dir="./test/e2e-edge/$s" ;; + esac + echo "🧪 chainsaw: $dir" + {{.CHAINSAW}} test "$dir" \ + --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml \ + --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + done + + kubeconfigs: + desc: "Export per-cluster kubeconfigs to TMPDIR for chainsaw." + cmds: + - '{{.KIND}} get kubeconfig --name {{.UPSTREAM_CLUSTER}} > {{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml' + - '{{.KIND}} get kubeconfig --name {{.DOWNSTREAM_CLUSTER}} > {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml' + + parity:check: + desc: "Run the config-dump parity gate against the live downstream Envoy + ext-server. Pass CLI flags after -- (PARITY owns the CLI). Exit 0 PASS / 1 parity FAIL / 2 tool error." + cmds: + - go build -o bin/parity-check ./cmd/parity-check + - ./bin/parity-check {{.CLI_ARGS}} + + parity:check-live: + desc: "Convenience: resolve the live data-plane Envoy pod from this env's labels and run parity:check in kubectl-exec mode (ext-server via selector across replicas). Extra flags pass through after --." + vars: + # In this env the data-plane proxy and the extension server live in these + # namespaces. + DP_NS: datum-downstream-gateway + DP_SELECTOR: gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e + EXT_NS: network-services-operator-system + EXT_SELECTOR: app.kubernetes.io/component=envoy-gateway-extension-server + # The admin side wants a single exact proxy pod name, resolved here. The + # extension-server side takes a selector and picks the authoritative replica + # itself. + DP_POD: + sh: kubectl --context {{.DOWNSTREAM_CTX}} -n datum-downstream-gateway get pods -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -o jsonpath='{.items[0].metadata.name}' + cmds: + - go build -o bin/parity-check ./cmd/parity-check + - | + if [ -z "{{.DP_POD}}" ]; then + echo "no data-plane Envoy pod found (label {{.DP_SELECTOR}} in {{.DP_NS}}); a Gateway must exist in a watched namespace for the merged data plane to be provisioned" >&2 + exit 2 + fi + - | + ./bin/parity-check \ + --coraza-filter=coraza-waf \ + --admin-exec-pod={{.DP_POD}} --admin-exec-namespace={{.DP_NS}} --admin-exec-container=envoy --admin-exec-context={{.DOWNSTREAM_CTX}} \ + --ext-exec-selector={{.EXT_SELECTOR}} --ext-exec-namespace={{.EXT_NS}} --ext-exec-context={{.DOWNSTREAM_CTX}} \ + {{.CLI_ARGS}} diff --git a/Taskfile.yaml b/Taskfile.yaml index a34a3994..890a88f7 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -7,6 +7,8 @@ includes: dir: ./docs dev: taskfile: ./Taskfile.dev.yaml + test-infra: + taskfile: ./Taskfile.test-infra.yml tasks: validate-kustomizations: diff --git a/cmd/parity-check/main.go b/cmd/parity-check/main.go new file mode 100644 index 00000000..b145cdbc --- /dev/null +++ b/cmd/parity-check/main.go @@ -0,0 +1,342 @@ +// Command parity-check is the command-line wrapper around the parity package. It +// reads the configuration the extension server intended to program, reads the +// proxy's live configuration, compares them, prints the result as JSON, and +// exits non-zero on any mismatch. +// +// The proxy and extension server can be reached either by a direct URL or, when +// their containers ship without any shell or HTTP client, by forwarding a port +// to the pod and fetching through it from this process. The flags below select +// which. +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "strings" + "time" + + "go.datum.net/network-services-operator/test/parity" +) + +func main() { + cfg := parseFlags() + if err := run(cfg); err != nil { + fmt.Fprintf(os.Stderr, "parity-check: %v\n", err) + os.Exit(2) // 2 = could not fetch or parse; 1 = mismatch found. + } +} + +type config struct { + corazaFilter string + timeout time.Duration + + adminURL string + admin execTarget + + extURL string + ext execTarget + // extSelector matches all extension server replicas. Only one replica + // actually builds configuration at a time, so the check must query every + // replica and use the authoritative one. Mutually exclusive with + // --ext-exec-pod / --ext-url. + extSelector string + + // expectMinBuildID, when set, requires the build count to be at least this + // value, proving a fresh build happened. Capture the count before a change, + // then require a higher one after. + expectMinBuildID uint64 + jsonOut bool + + // printBuildID, when true, resolves the authoritative replica and prints only + // its build count, then exits without comparing anything. Used to capture the + // count before a change and to wait for configuration to settle. + printBuildID bool +} + +type execTarget struct { + pod string + namespace string + container string + kubeCtx string +} + +func (e execTarget) set() bool { return e.pod != "" } + +func parseFlags() config { + var c config + flag.StringVar(&c.corazaFilter, "coraza-filter", "coraza-waf", + "Coraza HTTP filter name (identifies WAF HCM injection)") + flag.DurationVar(&c.timeout, "timeout", 30*time.Second, "overall fetch timeout") + + flag.StringVar(&c.adminURL, "admin-url", "", + "Envoy admin base URL (e.g. http://127.0.0.1:19000); use this OR --admin-exec-pod") + flag.StringVar(&c.admin.pod, "admin-exec-pod", "", "proxy pod to kubectl-exec into for admin access") + flag.StringVar(&c.admin.namespace, "admin-exec-namespace", "envoy-gateway-system", "namespace of the proxy pod") + flag.StringVar(&c.admin.container, "admin-exec-container", "envoy", "container in the proxy pod") + flag.StringVar(&c.admin.kubeCtx, "admin-exec-context", "", "kubeconfig context for admin exec (optional)") + + flag.StringVar(&c.extURL, "ext-url", "", + "ext-server health base URL (e.g. http://127.0.0.1:8080); use this OR --ext-exec-pod / --ext-exec-selector") + flag.StringVar(&c.ext.pod, "ext-exec-pod", "", "single ext-server pod to kubectl-exec into") + flag.StringVar(&c.extSelector, "ext-exec-selector", "", + "label selector for ALL ext-server replicas; picks the authoritative one (preferred over --ext-exec-pod)") + flag.StringVar(&c.ext.namespace, "ext-exec-namespace", "", "namespace of the ext-server pod(s)") + flag.StringVar(&c.ext.container, "ext-exec-container", "", "container in the ext-server pod") + flag.StringVar(&c.ext.kubeCtx, "ext-exec-context", "", "kubeconfig context for ext exec (optional)") + + flag.Uint64Var(&c.expectMinBuildID, "expect-min-build-id", 0, "require programmed-set BuildID >= this (STALE oracle)") + flag.BoolVar(&c.jsonOut, "json", true, "emit the ParityReport as JSON") + flag.BoolVar(&c.printBuildID, "print-build-id", false, + "resolve the authoritative ext-server replica and print ONLY its BuildID, then exit") + flag.Parse() + return c +} + +func run(c config) error { + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + + // Resolve the authoritative replica and emit only its build count, without + // fetching or comparing the proxy's configuration. + if c.printBuildID { + expected, _, err := resolveExpected(ctx, c) + if err != nil { + return err + } + fmt.Printf("%d\n", expected.BuildID) + return nil + } + + adminFetcher, err := buildFetcher(c.adminURL, c.admin, true) + if err != nil { + return fmt.Errorf("admin fetcher: %w", err) + } + + // Resolve the intended configuration and the fetcher for the authoritative + // replica. With a selector this queries every replica and picks the active + // one; otherwise it uses the single configured endpoint. + expected, extFetcher, err := resolveExpected(ctx, c) + if err != nil { + return err + } + if c.expectMinBuildID > 0 && expected.BuildID < c.expectMinBuildID { + return fmt.Errorf("STALE: programmed-set BuildID %d < required %d (ext-server did not re-translate)", + expected.BuildID, c.expectMinBuildID) + } + + droppedSecrets := expected.Keys[parity.FamilyTLSPrune] + actual, _, err := parity.FetchActual(ctx, adminFetcher, extFetcher, c.corazaFilter, droppedSecrets) + if err != nil { + return err + } + + report := parity.Compare(expected, actual) + + if c.jsonOut { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + _ = enc.Encode(report) + } + fmt.Fprintln(os.Stderr, report.String()) + + if !report.OK() { + os.Exit(1) // mismatch found — distinct from a fetch or parse failure (exit 2). + } + return nil +} + +// resolveExpected returns the intended configuration and the fetcher for the +// authoritative replica. With a selector it queries every replica and picks the +// active one; otherwise it uses the single configured endpoint. The chosen +// fetcher is reused later so other signals also come from the active replica. +func resolveExpected(ctx context.Context, c config) (parity.Expected, parity.Fetcher, error) { + if c.extSelector != "" { + pods, err := resolvePods(ctx, c.ext.kubeCtx, c.ext.namespace, c.extSelector) + if err != nil { + return parity.Expected{}, nil, fmt.Errorf("resolve ext-server pods: %w", err) + } + if len(pods) == 0 { + return parity.Expected{}, nil, fmt.Errorf( + "no ext-server pods matched selector %q in namespace %q", c.extSelector, c.ext.namespace) + } + fetchers := make(map[string]parity.Fetcher, len(pods)) + for _, pod := range pods { + et := c.ext + et.pod = pod + fetchers[pod] = &pfFetcher{target: et, remotePort: "8080"} + } + src, perReplicaErrs, err := parity.FetchExpectedFromMany(ctx, fetchers) + if err != nil { + return parity.Expected{}, nil, err + } + // Report per-replica fetch errors and the chosen replica for diagnosis; + // they are not fatal as long as one replica answered. + for pod, e := range perReplicaErrs { + fmt.Fprintf(os.Stderr, "parity-check: ext replica %s unreachable: %v\n", pod, e) + } + fmt.Fprintf(os.Stderr, "parity-check: authoritative ext replica %s (BuildID %d) of %d\n", + src.Replica, src.Expected.BuildID, len(pods)) + return src.Expected, fetchers[src.Replica], nil + } + + // Single-endpoint fallback. + extFetcher, err := buildFetcher(c.extURL, c.ext, false) + if err != nil { + return parity.Expected{}, nil, fmt.Errorf("ext fetcher: %w", err) + } + exp, err := parity.FetchExpected(ctx, extFetcher) + if err != nil { + return parity.Expected{}, nil, err + } + return exp, extFetcher, nil +} + +// resolvePods returns the names of the running pods matching selector, used to +// enumerate the extension server replicas. +func resolvePods(ctx context.Context, kubeCtx, namespace, selector string) ([]string, error) { + args := []string{} + if kubeCtx != "" { + args = append(args, "--context", kubeCtx) + } + if namespace != "" { + args = append(args, "-n", namespace) + } + args = append(args, "get", "pods", "-l", selector, + "--field-selector=status.phase=Running", + "-o", "jsonpath={.items[*].metadata.name}") + + cmd := exec.CommandContext(ctx, "kubectl", args...) + var stderr strings.Builder + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("kubectl get pods -l %s: %w: %s", selector, err, strings.TrimSpace(stderr.String())) + } + return strings.Fields(string(out)), nil +} + +// buildFetcher returns a direct HTTP fetcher when a URL is given, otherwise a +// port-forward fetcher to the given pod. admin selects which remote port to +// reach. Port-forwarding is used rather than running a command inside the pod +// because the containers ship without a shell or HTTP client. +func buildFetcher(url string, et execTarget, admin bool) (parity.Fetcher, error) { + switch { + case url != "": + return parity.HTTPFetcher{BaseURL: url}, nil + case et.set(): + port := "8080" + if admin { + port = "19000" + } + return &pfFetcher{target: et, remotePort: port}, nil + default: + return nil, fmt.Errorf("either a URL or an exec pod must be provided") + } +} + +// pfFetcher fetches a path by forwarding a local port to the pod and making the +// request from this process. The forward is set up and torn down per call, on a +// fresh local port each time so concurrent fetches don't collide. +type pfFetcher struct { + target execTarget + remotePort string +} + +func (p *pfFetcher) Fetch(ctx context.Context, path string) ([]byte, error) { + localPort, err := freeLocalPort() + if err != nil { + return nil, fmt.Errorf("allocate local port: %w", err) + } + + args := []string{} + if p.target.kubeCtx != "" { + args = append(args, "--context", p.target.kubeCtx) + } + if p.target.namespace != "" { + args = append(args, "-n", p.target.namespace) + } + args = append(args, "port-forward", "pod/"+p.target.pod, + fmt.Sprintf("%d:%s", localPort, p.remotePort)) + + // The forward must outlive the single request but be torn down right after. + pfCtx, cancel := context.WithCancel(ctx) + defer cancel() + cmd := exec.CommandContext(pfCtx, "kubectl", args...) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + var stderr strings.Builder + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start port-forward: %w", err) + } + defer func() { cancel(); _ = cmd.Wait() }() + + // Wait until the forward is listening before fetching. + if err := waitForwardReady(pfCtx, stdout); err != nil { + return nil, fmt.Errorf("port-forward not ready: %w (%s)", err, strings.TrimSpace(stderr.String())) + } + + url := fmt.Sprintf("http://127.0.0.1:%d%s", localPort, path) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("GET %s via port-forward: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read %s: %w", url, err) + } + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("GET %s: status %d", url, resp.StatusCode) + } + return body, nil +} + +// freeLocalPort asks the operating system for an unused port. Another process +// could claim it in the brief gap before we bind, but that window is small and +// the caller retries. +func freeLocalPort() (int, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer func() { _ = l.Close() }() + return l.Addr().(*net.TCPAddr).Port, nil +} + +// waitForwardReady blocks until the forward reports that its local socket is +// listening, or the context is done. +func waitForwardReady(ctx context.Context, stdout io.Reader) error { + ready := make(chan struct{}, 1) + go func() { + sc := bufio.NewScanner(stdout) + for sc.Scan() { + if strings.Contains(sc.Text(), "Forwarding from") { + ready <- struct{}{} + return + } + } + }() + select { + case <-ready: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-time.After(15 * time.Second): + return fmt.Errorf("timed out waiting for port-forward") + } +} diff --git a/cmd/parity-check/main_test.go b/cmd/parity-check/main_test.go new file mode 100644 index 00000000..6f94911c --- /dev/null +++ b/cmd/parity-check/main_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.datum.net/network-services-operator/test/parity" +) + +func TestBuildFetcher_URL(t *testing.T) { + f, err := buildFetcher("http://127.0.0.1:19000", execTarget{}, true) + require.NoError(t, err) + h, ok := f.(parity.HTTPFetcher) + require.True(t, ok) + assert.Equal(t, "http://127.0.0.1:19000", h.BaseURL) +} + +func TestBuildFetcher_PortForward(t *testing.T) { + f, err := buildFetcher("", execTarget{pod: "envoy-abc", namespace: "envoy-gateway-system", container: "envoy"}, true) + require.NoError(t, err) + e, ok := f.(*pfFetcher) + require.True(t, ok) + assert.Equal(t, "19000", e.remotePort, "admin fetcher must port-forward the admin port") + + fExt, err := buildFetcher("", execTarget{pod: "ext-abc", namespace: "nso"}, false) + require.NoError(t, err) + eExt := fExt.(*pfFetcher) + assert.Equal(t, "8080", eExt.remotePort, "ext fetcher must port-forward the health port") +} + +func TestFreeLocalPort(t *testing.T) { + p, err := freeLocalPort() + require.NoError(t, err) + assert.Greater(t, p, 0) +} + +func TestBuildFetcher_NeitherErrors(t *testing.T) { + _, err := buildFetcher("", execTarget{}, true) + assert.Error(t, err) +} diff --git a/config/e2e-downstream/d1-cert-bypass/README.md b/config/e2e-downstream/d1-cert-bypass/README.md new file mode 100644 index 00000000..1923679c --- /dev/null +++ b/config/e2e-downstream/d1-cert-bypass/README.md @@ -0,0 +1,28 @@ +# Expired-certificate isolation fixture (test-env-only) + +In production, an expired or otherwise unusable TLS certificate is caught early: +the platform withholds that listener from the edge before it is ever delivered. +The extension server *also* removes unusable certificates at the edge, as a +second line of defense — but because the earlier check normally catches the +problem first, that edge-side removal rarely gets exercised on the normal path. + +This fixture lets a test exercise it directly, by handing the edge a genuinely +expired certificate and bypassing the earlier check. + +1. `mint-expired-secret.sh ` writes a + self-signed, already-expired certificate as a TLS Secret to stdout. Apply it + into the `e2e-direct` namespace on the edge cluster. +2. The test then applies a gateway directly to the edge whose HTTPS listener + uses that certificate. The extension server removes the bad listener while a + healthy sibling keeps serving — which is what the test asserts. + +> **Use the `e2e-direct` namespace.** The gateway controller only watches +> namespaces that already carry the `meta.datumapis.com/upstream-cluster-name` +> label when they are created; a label added afterward is not reliably picked +> up, and a gateway there can stay unprogrammed. The `e2e-direct` namespace is +> created with the label up front for exactly this reason. If you must create a +> namespace inline, set the label at creation time. + +`task -t Taskfile.test-infra.yml d1-mint-expired-secret` is a thin wrapper around +the script (defaults: `NAMESPACE=e2e-direct`, `SECRET=d1-expired-tls`, +`HOSTNAME=d1-bad.e2e.env.datum.net`). diff --git a/config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh b/config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh new file mode 100755 index 00000000..8ea5e705 --- /dev/null +++ b/config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Mint an already-expired self-signed TLS certificate and emit it as a +# kubernetes.io/tls Secret on stdout. Test-env-only: it hands the gateway an +# expired certificate directly, so the extension server's removal of unusable +# certificates can be exercised on its own, without the earlier check rejecting +# it first. +# +# Usage: mint-expired-secret.sh +set -euo pipefail + +NS="${1:?namespace required}" +SECRET="${2:?secret name required}" +HOST="${3:?hostname required}" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# Generate a key + a self-signed cert dated entirely in the past so it is expired +# the moment it is created. openssl's -not_before/-not_after (LibreSSL/OpenSSL 3) +# set an explicit validity window; fall back to a 1-second window via -days 0 if +# the flags are unavailable. +openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$WORK/tls.key" -out "$WORK/tls.crt" \ + -subj "/CN=${HOST}" \ + -addext "subjectAltName=DNS:${HOST}" \ + -not_before 20200101000000Z -not_after 20200102000000Z 2>/dev/null \ + || openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$WORK/tls.key" -out "$WORK/tls.crt" \ + -subj "/CN=${HOST}" -addext "subjectAltName=DNS:${HOST}" -days 1 2>/dev/null + +CRT_B64="$(base64 < "$WORK/tls.crt" | tr -d '\n')" +KEY_B64="$(base64 < "$WORK/tls.key" | tr -d '\n')" + +cat < + + + Service Unavailable +

This service is temporarily unavailable.

+ diff --git a/config/e2e-downstream/extserver-base/kustomization.yaml b/config/e2e-downstream/extserver-base/kustomization.yaml new file mode 100644 index 00000000..14e55681 --- /dev/null +++ b/config/e2e-downstream/extserver-base/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Ext-server, namespaced + prefixed so the Service FQDN matches the EG +# extensionManager fqdn and the CSI dns-names baked into the base deployment. +namespace: network-services-operator-system +namePrefix: network-services-operator- + +resources: + - ../../extension-server + +patches: + - path: patches/extserver-tls.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server + - path: patches/extserver-serverconfig.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server + - path: patches/extserver-clientcert-issuer.yaml + target: + kind: Certificate + name: envoy-gateway-extension-server-eg-client-tls + - path: patches/extserver-ca-bundle.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server + - path: patches/extserver-programmed-set.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server diff --git a/config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml b/config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml new file mode 100644 index 00000000..6e85fc0f --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml @@ -0,0 +1,18 @@ +# Point the ext-server CA bundle volume at the e2e CA ConfigMap (carrying the +# ca.crt that signed the EG client cert), replacing placeholder-ca-bundle. The +# ConfigMap is published by the bring-up (test-infra:extserver-ca-bundle) from +# the e2e-extension-server-ca cert-manager secret. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + volumes: + - name: tls-ca + configMap: + name: extension-server-ca-bundle + items: + - key: ca.crt + path: ca.crt diff --git a/config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml b/config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml new file mode 100644 index 00000000..9710265e --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml @@ -0,0 +1,11 @@ +# Point the EG client cert (CN=envoy-gateway) at the e2e CA ClusterIssuer, +# replacing the base certificate's placeholder-issuer. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: envoy-gateway-extension-server-eg-client-tls +spec: + issuerRef: + name: e2e-extension-server-ca + kind: ClusterIssuer + group: cert-manager.io diff --git a/config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml b/config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml new file mode 100644 index 00000000..aa4ebfc4 --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml @@ -0,0 +1,17 @@ +# Turn on the read-only /debug/programmed-set endpoint so the parity test can +# confirm the proxy is running exactly the set the build intended. The base +# deployment reads --enable-programmed-set from this env var, defaulting off in +# production; flip it to "true" here for the test environment. Strategic-merge +# on env (matched by name). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + containers: + - name: envoy-gateway-extension-server + env: + - name: ENABLE_PROGRAMMED_SET + value: "true" diff --git a/config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml b/config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml new file mode 100644 index 00000000..2c9701d9 --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml @@ -0,0 +1,26 @@ +# Mount the operator config ConfigMap and set SERVER_CONFIG to its path so the +# ext-server loads Coraza + branded-error-page settings. Strategic-merge on env +# (matched by name) and on volumes/volumeMounts. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + containers: + - name: envoy-gateway-extension-server + env: + - name: SERVER_CONFIG + value: /etc/datum/server-config/config.yaml + volumeMounts: + - name: server-config + mountPath: /etc/datum/server-config + readOnly: true + volumes: + - name: server-config + configMap: + name: extension-server-config + items: + - key: config.yaml + path: config.yaml diff --git a/config/e2e-downstream/extserver-base/patches/extserver-tls.yaml b/config/e2e-downstream/extserver-base/patches/extserver-tls.yaml new file mode 100644 index 00000000..c190ac81 --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-tls.yaml @@ -0,0 +1,21 @@ +# Point the ext-server server-cert CSI volume at the e2e CA-backed ClusterIssuer, +# replacing the base deployment's placeholder-issuer. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + volumes: + - name: tls + csi: + driver: csi.cert-manager.io + readOnly: true + volumeAttributes: + csi.cert-manager.io/issuer-kind: ClusterIssuer + csi.cert-manager.io/issuer-name: e2e-extension-server-ca + csi.cert-manager.io/common-name: envoy-gateway-extension-server + csi.cert-manager.io/dns-names: "network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc,network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local" + csi.cert-manager.io/key-usages: server auth + csi.cert-manager.io/fs-group: "65532" diff --git a/config/e2e-downstream/extserver-config.yaml b/config/e2e-downstream/extserver-config.yaml new file mode 100644 index 00000000..f5828a0c --- /dev/null +++ b/config/e2e-downstream/extserver-config.yaml @@ -0,0 +1,21 @@ +# Extension server config for the e2e edge. +# +# The WAF is enabled so its rules reach the proxy, and the branded 5xx page is +# pointed at the mounted error-pages volume so the suite can assert the branded +# body by content. The connector-tunnel listener name is left at its default so +# the connector fixtures find the listener by the same name production uses. +apiVersion: v1 +kind: ConfigMap +metadata: + name: extension-server-config + namespace: network-services-operator-system +data: + config.yaml: | + apiVersion: apiserver.config.datumapis.com/v1alpha1 + kind: NetworkServicesOperator + gateway: + coraza: + disabled: false + errorPage: + enabled: true + bodyPath: /etc/datum/error-pages/error-5xx.html diff --git a/config/e2e-downstream/issuer.yaml b/config/e2e-downstream/issuer.yaml new file mode 100644 index 00000000..35fc7a62 --- /dev/null +++ b/config/e2e-downstream/issuer.yaml @@ -0,0 +1,41 @@ +# Self-signed root + CA-backed ClusterIssuer for the e2e ext-server mTLS chain. +# +# This issues both sides of the EG <-> ext-server handshake: +# - the ext-server SERVER cert (via the CSI driver, see patches/extserver-tls.yaml) +# - the EG CLIENT cert (envoy-gateway-extension-server-eg-client-tls, CN=envoy-gateway) +# The CA's ca.crt is also published into the ext-server CA bundle ConfigMap +# (so the ext-server can verify the EG client) and into the EG certificateRef +# secret (so EG can verify the ext-server server cert). +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: e2e-extension-server-selfsigned +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: e2e-extension-server-ca + namespace: cert-manager +spec: + isCA: true + commonName: e2e-extension-server-ca + secretName: e2e-extension-server-ca + duration: 8760h + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: e2e-extension-server-selfsigned + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: e2e-extension-server-ca +spec: + ca: + secretName: e2e-extension-server-ca diff --git a/config/e2e-downstream/kustomization.yaml b/config/e2e-downstream/kustomization.yaml new file mode 100644 index 00000000..6ab183ae --- /dev/null +++ b/config/e2e-downstream/kustomization.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The e2e edge overlay applied to nso-downstream. Composes the ext-server +# (prefixed/namespaced via extserver-base) with the e2e mTLS issuer chain, the +# Coraza/branded-page server-config, the branded error page, and the test +# EnvoyProxy (real Coraza WAF image + admin :19000) and its GatewayClass. +# +# The ConfigMaps / issuer / EnvoyProxy / GatewayClass are kept OUT of the +# name-prefix so the deployment's literal references (envoy-error-pages, +# extension-server-config) and the EG fqdn resolve unchanged. +resources: +- namespace.yaml +- extserver-base +- issuer.yaml +- extserver-config.yaml +- error-pages.yaml +- envoyproxy.yaml + +# The NSO image tag is set by the bring-up (test-infra:extension-server) to the +# git-SHA-built image. +images: +- name: ghcr.io/datum-cloud/network-services-operator + newName: ghcr.io/datum-cloud/network-services-operator + newTag: 63fa912 + +# The default e2e path has no Prometheus operator; drop the ServiceMonitor so +# the apply doesn't fail on the missing monitoring.coreos.com CRD. (Re-add via +# OBSERVABILITY=1 when the Flux observability stack is enabled.) +patches: +- patch: | + $patch: delete + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: envoy-gateway-extension-server-metrics + target: + kind: ServiceMonitor diff --git a/config/e2e-downstream/namespace.yaml b/config/e2e-downstream/namespace.yaml new file mode 100644 index 00000000..b27c33c1 --- /dev/null +++ b/config/e2e-downstream/namespace.yaml @@ -0,0 +1,6 @@ +# The ext-server namespace on the edge cluster. The base ext-server resources +# target this namespace but do not create it. +apiVersion: v1 +kind: Namespace +metadata: + name: network-services-operator-system diff --git a/config/extension-server/deployment.yaml b/config/extension-server/deployment.yaml index e9e81fd9..6e4f01d0 100644 --- a/config/extension-server/deployment.yaml +++ b/config/extension-server/deployment.yaml @@ -58,6 +58,7 @@ spec: - --tls-key=$(TLS_KEY) - --tls-client-ca=$(TLS_CLIENT_CA) - --server-config=$(SERVER_CONFIG) + - --enable-programmed-set=$(ENABLE_PROGRAMMED_SET) env: - name: GRPC_ADDR value: ":5005" @@ -73,6 +74,11 @@ spec: # mounted path to apply Coraza directives. - name: SERVER_CONFIG value: "" + # Off in production. The test-environment overlay sets this to "true" + # to serve the read-only /debug/programmed-set endpoint the parity + # test reads. + - name: ENABLE_PROGRAMMED_SET + value: "false" ports: - name: grpc containerPort: 5005 diff --git a/config/federation/README.md b/config/federation/README.md new file mode 100644 index 00000000..eb464295 --- /dev/null +++ b/config/federation/README.md @@ -0,0 +1,41 @@ +# Federation + +The Datum edge runs across many clusters in many regions. Customers, though, +work against a single control plane: they create a Gateway, a route, or a +firewall policy in one place. **Federation is what carries that intent out to +the edge clusters that actually serve traffic.** + +This directory holds the federation configuration the test environment applies, +mirrored from production so the test edge fans configuration out the same way +the real one does. + +## Why it's tested as its own concern + +For most of this system's history, the test environment copied configuration +between clusters with a simple direct mechanism — nothing like production. But +several real incidents lived specifically in the federation layer: some +information (a backend's online/offline status) is intentionally *not* carried +to the edge, and the timing of cross-cluster delivery created races. None of +that is visible unless the test edge federates the way production does. + +So the production-fidelity environment stands up real federation and proves the +thing customers depend on: **configuration created in the control plane actually +arrives at the edge.** The test confirms a change made centrally shows up on a +downstream cluster within seconds. + +## What's here + +- A propagation policy describing *which* resources travel to the edge. +- Interpreter rules describing *how* each resource type is carried — including + the deliberate choice to propagate configuration but not live status, which is + the behavior that caused real "false offline" incidents and is now exercised + directly. + +## Implementation + +Federation is implemented with [Karmada](https://karmada.io/). The directory is +named for the responsibility — fanning configuration out to the edge — rather +than the tool, so the intent stays clear even if the underlying mechanism +changes. The environment that applies these artifacts is described in +[`Taskfile.test-infra.yml`](../../Taskfile.test-infra.yml) (`task +test-infra:karmada-up`). diff --git a/config/federation/clusterpropagationpolicy.yaml b/config/federation/clusterpropagationpolicy.yaml new file mode 100644 index 00000000..5299bb3b --- /dev/null +++ b/config/federation/clusterpropagationpolicy.yaml @@ -0,0 +1,158 @@ +apiVersion: policy.karmada.io/v1alpha1 +kind: ClusterPropagationPolicy +metadata: + name: nso-resources +spec: + conflictResolution: Overwrite + placement: + clusterAffinities: + - affinityName: gateway-enabled + labelSelector: + matchExpressions: + - key: infra.datum.net/gateways + operator: In + values: + - enabled + resourceSelectors: + - apiVersion: v1 + kind: Namespace + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-namespace + operator: Exists + - apiVersion: v1 + kind: ConfigMap + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-namespace + operator: Exists + - apiVersion: v1 + kind: Secret + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-namespace + operator: Exists + # TODO(jreese) clean up dupe secret policies + - apiVersion: v1 + kind: Secret + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: v1 + kind: Secret + labelSelector: + matchExpressions: + - key: cert-manager.io/issuer-name + operator: In + values: + - nso-gateway + - apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: v1 + kind: Service + # TODO(jreese) get labels on these patch policies + # labelSelector: + # matchExpressions: + # - key: meta.datumapis.com/upstream-cluster-name + # operator: Exists + + # Gateway API + - apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + labelSelector: + matchExpressions: + - key: meta.datumapis.com/http01-solver + operator: Exists + - apiVersion: gateway.networking.k8s.io/v1 + kind: BackendTLSPolicy + # TODO(jreese) get labels on these when they are created by the httpproxy + # controller + # labelSelector: + # matchExpressions: + # - key: meta.datumapis.com/upstream-cluster-name + # operator: Exists + + # Envoy Gateway API Extensions + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: HTTPRouteFilter + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: HTTPRouteFilter + labelSelector: + matchExpressions: + - key: meta.datumapis.com/http01-solver + operator: Exists + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: SecurityPolicy + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyPatchPolicy + # TODO(jreese) get labels on these patch policies + # labelSelector: + # matchExpressions: + # - key: meta.datumapis.com/upstream-cluster-name + # operator: Exists + + # Network Services Operator CRDs (replicated for the extension server) + - apiVersion: networking.datumapis.com/v1alpha1 + kind: Connector + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: networking.datumapis.com/v1alpha + kind: TrafficProtectionPolicy + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + - apiVersion: networking.datumapis.com/v1alpha + kind: HTTPProxy + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists + + # External DNS + - apiVersion: externaldns.k8s.io/v1alpha1 + kind: DNSEndpoint + # TODO(jreese) get labels on these + # labelSelector: + # matchExpressions: + # - key: meta.datumapis.com/upstream-cluster-name + # operator: Exists diff --git a/config/federation/resourceinterpreters.yaml b/config/federation/resourceinterpreters.yaml new file mode 100644 index 00000000..452a19ad --- /dev/null +++ b/config/federation/resourceinterpreters.yaml @@ -0,0 +1,234 @@ +# Future test coverage for resource interpreters can leverage the test framework +# once it's merged. +# +# See: https://github.com/karmada-io/karmada/pull/6938 +--- +apiVersion: config.karmada.io/v1alpha1 +kind: ResourceInterpreterCustomization +metadata: + name: gateway.networking.k8s.io-gateway +spec: + target: + apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + customizations: + statusAggregation: + luaScript: > + function AggregateStatus(desiredObj, statusItems) + if statusItems == nil or #statusItems == 0 then + return desiredObj + end + if desiredObj.status == nil then + desiredObj.status = {} + end + + local item = statusItems[1] + if item == nil or item.status == nil then + return desiredObj + end + + -- TODO(jreese) implement proper aggregation logic. Would be good to + -- think through how to represent propagation status across clusters. + if item.status.addresses ~= nil then + desiredObj.status.addresses = item.status.addresses + end + if item.status.conditions ~= nil then + desiredObj.status.conditions = item.status.conditions + end + if item.status.listeners ~= nil then + desiredObj.status.listeners = item.status.listeners + end + return desiredObj + end +--- +apiVersion: config.karmada.io/v1alpha1 +kind: ResourceInterpreterCustomization +metadata: + name: gateway.networking.k8s.io-httproute +spec: + target: + apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + customizations: + statusAggregation: + luaScript: > + function AggregateStatus(desiredObj, statusItems) + if statusItems == nil or #statusItems == 0 then + return desiredObj + end + if desiredObj.status == nil then + desiredObj.status = {} + end + + local item = statusItems[1] + if item == nil or item.status == nil then + return desiredObj + end + + -- TODO(jreese) implement proper aggregation logic. Would be good to + -- think through how to represent propagation status across clusters. + if item.status.parents ~= nil then + desiredObj.status.parents = item.status.parents + end + return desiredObj + end +--- +apiVersion: config.karmada.io/v1alpha1 +kind: ResourceInterpreterCustomization +metadata: + name: gateway.networking.k8s.io-backendtlspolicy +spec: + target: + apiVersion: gateway.networking.k8s.io/v1 + kind: BackendTLSPolicy + customizations: + statusAggregation: + luaScript: > + function AggregateStatus(desiredObj, statusItems) + if statusItems == nil or #statusItems == 0 then + return desiredObj + end + if desiredObj.status == nil then + desiredObj.status = {} + end + + local item = statusItems[1] + if item == nil or item.status == nil then + return desiredObj + end + + -- TODO(jreese) implement proper aggregation logic. Would be good to + -- think through how to represent propagation status across clusters. + if item.status.ancestors ~= nil then + desiredObj.status.ancestors = item.status.ancestors + end + return desiredObj + end +--- +apiVersion: config.karmada.io/v1alpha1 +kind: ResourceInterpreterCustomization +metadata: + name: gateway.envoyproxy.io-backend +spec: + target: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + customizations: + statusAggregation: + luaScript: > + function AggregateStatus(desiredObj, statusItems) + if statusItems == nil or #statusItems == 0 then + return desiredObj + end + if desiredObj.status == nil then + desiredObj.status = {} + end + + local item = statusItems[1] + if item == nil or item.status == nil then + return desiredObj + end + + -- TODO(jreese) implement proper aggregation logic. Would be good to + -- think through how to represent propagation status across clusters. + if item.status.conditions ~= nil then + desiredObj.status.conditions = item.status.conditions + end + return desiredObj + end +--- +apiVersion: config.karmada.io/v1alpha1 +kind: ResourceInterpreterCustomization +metadata: + name: gateway.envoyproxy.io-backendtrafficpolicy +spec: + target: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + customizations: + statusAggregation: + luaScript: > + function AggregateStatus(desiredObj, statusItems) + if statusItems == nil or #statusItems == 0 then + return desiredObj + end + if desiredObj.status == nil then + desiredObj.status = {} + end + + local item = statusItems[1] + if item == nil or item.status == nil then + return desiredObj + end + + -- TODO(jreese) implement proper aggregation logic. Would be good to + -- think through how to represent propagation status across clusters. + if item.status.ancestors ~= nil then + desiredObj.status.ancestors = item.status.ancestors + end + return desiredObj + end +--- +apiVersion: config.karmada.io/v1alpha1 +kind: ResourceInterpreterCustomization +metadata: + name: gateway.envoyproxy.io-envoypatchpolicy +spec: + target: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyPatchPolicy + customizations: + statusAggregation: + luaScript: > + function AggregateStatus(desiredObj, statusItems) + if statusItems == nil or #statusItems == 0 then + return desiredObj + end + if desiredObj.status == nil then + desiredObj.status = {} + end + + local item = statusItems[1] + if item == nil or item.status == nil then + return desiredObj + end + + -- TODO(jreese) implement proper aggregation logic. Would be good to + -- think through how to represent propagation status across clusters. + if item.status.ancestors ~= nil then + desiredObj.status.ancestors = item.status.ancestors + end + return desiredObj + end +--- +apiVersion: config.karmada.io/v1alpha1 +kind: ResourceInterpreterCustomization +metadata: + name: externaldns.k8s.io-dnsendpoint +spec: + target: + apiVersion: externaldns.k8s.io/v1alpha1 + kind: DNSEndpoint + customizations: + statusAggregation: + luaScript: > + function AggregateStatus(desiredObj, statusItems) + if statusItems == nil or #statusItems == 0 then + return desiredObj + end + if desiredObj.status == nil then + desiredObj.status = {} + end + + local item = statusItems[1] + if item == nil or item.status == nil then + return desiredObj + end + + -- TODO(jreese) implement proper aggregation logic. Would be good to + -- think through how to represent propagation status across clusters. + if item.status.observedGeneration ~= nil then + desiredObj.status.observedGeneration = item.status.observedGeneration + end + return desiredObj + end diff --git a/config/tools/envoy-gateway-downstream/kustomization.yaml b/config/tools/envoy-gateway-downstream/kustomization.yaml index d6cce819..9f5a2a92 100644 --- a/config/tools/envoy-gateway-downstream/kustomization.yaml +++ b/config/tools/envoy-gateway-downstream/kustomization.yaml @@ -6,7 +6,7 @@ helmCharts: includeCRDs: false namespace: datum-downstream-gateway releaseName: envoy-datum-downstream-gateway - version: v1.8.1 + version: v1.7.4 repo: oci://docker.io/envoyproxy valuesInline: config: @@ -29,6 +29,10 @@ helmCharts: - key: meta.datumapis.com/upstream-cluster-name operator: Exists extensionManager: + # Match the ext-server's compiled-in 256 MiB gRPC ceiling. Without + # this the EG side defaults to ~4 MiB and silently freezes xDS once a + # translated snapshot exceeds it (~540 gateways in prod). + maxMessageSize: 256Mi policyResources: - group: networking.datumapis.com version: v1alpha @@ -43,9 +47,11 @@ helmCharts: port: 5005 tls: certificateRef: - # Placeholder — an overlay must patch this to the Secret holding the CA that issued the server cert. - name: placeholder-ca - namespace: placeholder-namespace + # e2e: the Secret holding the CA that issued the ext-server + # server cert. Published by the bring-up (test-infra:extserver-ca-bundle) + # from the e2e-extension-server-ca cert-manager secret. + name: e2e-extension-server-ca + namespace: network-services-operator-system clientCertificateRef: name: envoy-gateway-extension-server-eg-client-tls namespace: network-services-operator-system diff --git a/config/tools/kind/downstream-cluster.yaml b/config/tools/kind/downstream-cluster.yaml new file mode 100644 index 00000000..6339242a --- /dev/null +++ b/config/tools/kind/downstream-cluster.yaml @@ -0,0 +1,14 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: nso-downstream +networking: + ipFamily: dual +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30080 + hostPort: 30080 + protocol: TCP + - containerPort: 30443 + hostPort: 30443 + protocol: TCP diff --git a/config/tools/kind/upstream-cluster.yaml b/config/tools/kind/upstream-cluster.yaml new file mode 100644 index 00000000..be723669 --- /dev/null +++ b/config/tools/kind/upstream-cluster.yaml @@ -0,0 +1,15 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: nso-upstream +networking: + ipFamily: dual +nodes: +- role: control-plane + # Expose the Karmada apiserver NodePort (32443) to the host so karmadactl — + # which runs on the host — can reach the Karmada apiserver during init/join. + # On macOS the kind docker-network IP is not host-routable, so without this + # mapping karmadactl times out dialing the advertised node IP:32443. + extraPortMappings: + - containerPort: 32443 + hostPort: 32443 + protocol: TCP diff --git a/docs/testing/README.md b/docs/testing/README.md new file mode 100644 index 00000000..8de1abba --- /dev/null +++ b/docs/testing/README.md @@ -0,0 +1,87 @@ +# Testing the Datum edge + +This is the map for how we test the network-services-operator (NSO) — what we +prove, and why it's built the way it is. It's written for anyone who wants to +understand the safety net, not just the people who maintain it. + +## Why this exists + +NSO programs the edge: when a customer creates a Gateway, a route, a web-app +firewall policy, or a connector, NSO turns that intent into live configuration +on the Envoy proxies that actually serve customer traffic. The hard part isn't +producing the configuration — it's making sure the configuration that lands on +the running proxy *does what the customer asked*. + +Almost every production incident in this system has shared one shape: + +> **The configuration was logically correct, the platform reported success, but +> the running proxy behaved differently than intended** — a firewall rule that +> protected nothing, an offline backend that still returned a blank error, a +> branded error page that never appeared, a single bad certificate that froze a +> whole shared listener. + +These failures are invisible to ordinary tests. The Kubernetes resources report +"Programmed = True," unit tests pass, and the gap only shows up when real +traffic — often a real *attack* — arrives at the edge. So our testing is built +around one principle: + +**Prove behavior at the edge with real traffic, against an environment that +looks like production — not against the platform's own report of success.** + +## The two ideas everything rests on + +**1. Production fidelity.** The recent incidents lived precisely on the axes +where our old test environment differed from production: the proxy version, the +"fail closed" availability coupling, the firewall data plane, and multi-cluster +replication. So the test environment stands up the *real* shape of the edge — +the same proxy version, the same extension server that rewrites configuration, +the same firewall image, and the same federation mechanism that fans +configuration out to edge clusters. If a test passes here, it passes against +something production actually runs. This environment is brought online with a +single command set (`task test-infra:up`); see +[`Taskfile.test-infra.yml`](../../Taskfile.test-infra.yml). + +**2. Traffic-first, with a tie-breaker.** Every test's verdict is the +*observed behavior* of real traffic through the real proxy — a blocked attack, +a 503 for an offline backend, the branded page in the response body. We never +let "the platform says it worked" stand in for "the edge did the right thing." + +Real traffic alone has one blind spot, though: a firewall that protects nothing +and a firewall that's simply not being attacked produce the *same* successful +response. So traffic is backed by a **parity check** — a comparison of what the +edge was told to do against what the running proxy is actually doing — which +turns a surprising result into a diagnosis instead of a guess. Traffic is always +the verdict; parity is the tie-breaker that catches the silently-inert case. +See [`test/parity/README.md`](../../test/parity/README.md). + +## What we guarantee + +The end-to-end suite ([`test/e2e-edge/README.md`](../../test/e2e-edge/README.md)) turns +each past incident into a standing guarantee, checked against real traffic: + +- **A web-app firewall actually blocks attacks** — a malicious request is + refused while a legitimate one still succeeds. +- **An offline backend fails cleanly** — the customer path returns a real 503, + not a hang or a blank. +- **Branded error pages reach the customer** — the styled page appears in the + actual response, not just in configuration. +- **One bad certificate can't take down its neighbors** — an invalid listener + is isolated while the healthy listeners on the same proxy keep serving. + +And the federation layer ([`config/federation/README.md`](../../config/federation/README.md)) +proves that configuration created in the control plane genuinely arrives at the +edge clusters that serve traffic. + +## Where things live + +| Area | What it covers | +|---|---| +| [`Taskfile.test-infra.yml`](../../Taskfile.test-infra.yml) | Brings the production-fidelity edge online and runs the suites | +| [`test/e2e-edge/README.md`](../../test/e2e-edge/README.md) | The real-traffic guarantees, scenario by scenario | +| [`test/parity/README.md`](../../test/parity/README.md) | The parity check that catches silently-inert configuration | +| [`config/federation/README.md`](../../config/federation/README.md) | Fanning configuration out to edge clusters | + +> The design rationale that led here — the original audit of test-vs-production +> gaps and the proposals for closing them — lives in the pull requests that +> introduced this work, not in the repository, because it describes a plan +> rather than the system as it stands today. diff --git a/internal/extensionserver/cmd/run.go b/internal/extensionserver/cmd/run.go index cac6e9f9..116d2847 100644 --- a/internal/extensionserver/cmd/run.go +++ b/internal/extensionserver/cmd/run.go @@ -68,13 +68,15 @@ func envBool(key string, def bool) bool { // --tls-key=/tls/tls.key // --tls-client-ca=/tls/ca.crt // --server-config=/config/config.yaml (optional; provides Coraza WAF config) +// --enable-programmed-set=false (default off; test environments set true) type options struct { - grpcAddr string - healthAddr string - tlsCert string - tlsKey string - tlsClientCA string - serverCfgFile string + grpcAddr string + healthAddr string + tlsCert string + tlsKey string + tlsClientCA string + serverCfgFile string + enableProgrammedSet bool } // NewCommand returns the "extension-server" subcommand. @@ -99,6 +101,8 @@ func NewCommand() *cobra.Command { fs.StringVar(&o.tlsKey, "tls-key", "", "Path to server TLS key (PEM)") fs.StringVar(&o.tlsClientCA, "tls-client-ca", "", "Path to client CA certificate (PEM) for mTLS") fs.StringVar(&o.serverCfgFile, "server-config", "", "Path to operator config file (optional; provides Coraza WAF settings)") + fs.BoolVar(&o.enableProgrammedSet, "enable-programmed-set", false, + "Serve the read-only /debug/programmed-set endpoint used by the parity test (test environments only)") cmd := &cobra.Command{ Use: "envoy-gateway-extension-server", @@ -278,6 +282,9 @@ func run(o options) { ConnectorInternalListener: serverConfig.Gateway.ConnectorTunnelListenerName(), CorazaRouteBaseDirectives: coraza.RouteBaseDirectives, LocalReply: buildLocalReplyConfig(serverConfig.Gateway.ErrorPage, log), + // The programmed-set debug endpoint is for the test environment only; it + // stays off in production unless --enable-programmed-set is passed. + EnableProgrammedSet: o.enableProgrammedSet, } // --- gRPC panic recovery interceptor --- @@ -470,6 +477,12 @@ func run(o options) { } }) mux.Handle("/metrics", promhttp.Handler()) + // Read-only debug endpoint that reports what the last build changed, so a test + // can confirm the proxy is running exactly that. Test environment only; it is + // not served in production. Keeps only the last build. + if srvCfg.EnableProgrammedSet { + mux.HandleFunc(extserver.ProgrammedSetEndpointPath, extSrv.ProgrammedSetHandler()) + } healthServer := &http.Server{ Addr: healthAddr, Handler: mux, diff --git a/internal/extensionserver/server/programmedset.go b/internal/extensionserver/server/programmedset.go new file mode 100644 index 00000000..e4ebdf7a --- /dev/null +++ b/internal/extensionserver/server/programmedset.go @@ -0,0 +1,192 @@ +package server + +import ( + "encoding/json" + "net/http" + "sync" + "time" + + clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" +) + +// programmedSetEndpointPath is a read-only debug endpoint that reports the exact +// set of changes the last build intended to make, so a test can confirm the +// proxy is running that set and not merely the right number of things. +const programmedSetEndpointPath = "/debug/programmed-set" + +// Marker strings, kept in sync with where the configuration is written, used to +// recognize each change when reading the configuration back. +const ( + datumGatewayMetaKey = "datum-gateway" + connectorInternalTransport = "envoy.transport_sockets.internal_upstream" + hcmNetworkFilterName = "envoy.filters.network.http_connection_manager" +) + +// Family names a kind of change recorded in a snapshot. The strings are stable +// JSON keys the parity test depends on; do not rename. +type Family string + +const ( + FamilyWAFRoute Family = "waf_route" + FamilyWAFHCM Family = "waf_hcm" + FamilyLocalReply Family = "local_reply" + FamilyConnectorCluster Family = "connector_cluster" + FamilyConnectorRoute Family = "connector_route" + FamilyConnectorOffline Family = "connector_offline" + FamilyTLSPrune Family = "tls_prune" +) + +// ProgrammedSet is a snapshot of one build's intended changes. It is served as +// JSON and read back by the parity test. +type ProgrammedSet struct { + // BuildID increments once per build so a caller can tell two successive + // snapshots apart and know a fresh build has happened. + BuildID uint64 `json:"buildID"` + // CapturedAt is when the snapshot was recorded. + CapturedAt time.Time `json:"capturedAt"` + // Keys is, per kind of change, the identity of each thing changed. Each value + // is sorted and de-duplicated. The identity format is described on the + // record helpers below and mirrored by the parity test. + Keys map[Family][]string `json:"keys"` + // Counts holds the size of each set in Keys, plus the removed-certificate + // outcomes below that have no per-item identity. Used as a secondary check. + Counts map[Family]int `json:"counts"` + // Removing invalid TLS certificates is confirmed by counting outcomes rather + // than by identity, so the raw numbers are carried here. + TLSPrunedChains int `json:"tlsPrunedChains"` + TLSPrunedSecrets int `json:"tlsPrunedSecrets"` + TLSListenersLeftIntact int `json:"tlsListenersLeftIntact"` +} + +// programmedRecorder holds the most recent snapshot under a lock, shared between +// the build that writes it and the endpoint that reads it. Only the last build +// is kept; each recording replaces the previous one. +type programmedRecorder struct { + mu sync.RWMutex + buildID uint64 + last *ProgrammedSet +} + +// newProgrammedRecorder returns an empty recorder. Before the first build it +// reports a valid empty snapshot with a zero build count, so callers can tell +// "no build yet" from a real build. +func newProgrammedRecorder() *programmedRecorder { + return &programmedRecorder{} +} + +// snapshot returns the last recorded set, or an empty one if no build has run. +// The returned value is a copy safe to serialize without the lock. +func (r *programmedRecorder) snapshot() ProgrammedSet { + r.mu.RLock() + defer r.mu.RUnlock() + if r.last == nil { + return ProgrammedSet{Keys: map[Family][]string{}, Counts: map[Family]int{}} + } + return *r.last +} + +// record reads the configuration the build just produced and stores a snapshot +// of it. It only reads, never changes the configuration, and holds the write +// lock just long enough to store the result so it doesn't slow the build. +func (r *programmedRecorder) record( + listeners []*listenerv3.Listener, + routes []*routev3.RouteConfiguration, + clusters []*clusterv3.Cluster, + corazaFilterName string, + tlsPrunedChains, tlsPrunedSecrets, tlsListenersLeftIntact int, +) { + ps := buildProgrammedSet(listeners, routes, clusters, corazaFilterName, + tlsPrunedChains, tlsPrunedSecrets, tlsListenersLeftIntact) + + r.mu.Lock() + r.buildID++ + ps.BuildID = r.buildID + r.last = &ps + r.mu.Unlock() +} + +// buildProgrammedSet walks the produced configuration and extracts an identity +// for each change. It is a pure function so it can be unit tested directly. The +// identity formats must match what the parity test looks for; keep both sides +// in lockstep. +func buildProgrammedSet( + listeners []*listenerv3.Listener, + routes []*routev3.RouteConfiguration, + clusters []*clusterv3.Cluster, + corazaFilterName string, + tlsPrunedChains, tlsPrunedSecrets, tlsListenersLeftIntact int, +) ProgrammedSet { + keys := map[Family][]string{} + add := func(f Family, k string) { keys[f] = append(keys[f], k) } + + for _, rc := range routes { + rcName := rc.GetName() + for _, vh := range rc.GetVirtualHosts() { + vhName := vh.GetName() + for _, rt := range vh.GetRoutes() { + // The identity includes the protection policy governing the route, + // so a route protected by the wrong policy shows up as a mismatch. + if ns, name, mode, ok := datumGatewayTPP(rt); ok { + add(FamilyWAFRoute, wafRouteKey(rcName, vhName, rt.GetName(), ns, name, mode)) + } + if isConnectRoute(rt) { + add(FamilyConnectorRoute, connectorRouteKey(rcName, vhName, rt.GetName())) + } + if isOfflineDirectResponse(rt) { + add(FamilyConnectorOffline, connectorRouteKey(rcName, vhName, rt.GetName())) + } + } + } + } + + for _, cl := range clusters { + if isReplacedConnectorCluster(cl) { + add(FamilyConnectorCluster, cl.GetName()) + } + } + + for _, l := range listeners { + lName := l.GetName() + eachHCM(l, func(fcName string, hcm *hcmv3.HttpConnectionManager) { + if corazaFilterName != "" && hcmHasFilterAtZero(hcm, corazaFilterName) { + add(FamilyWAFHCM, listenerChainKey(lName, fcName)) + } + if hcm.GetLocalReplyConfig() != nil { + add(FamilyLocalReply, listenerChainKey(lName, fcName)) + } + }) + } + + // Sort and de-duplicate so the output can be compared as a set. + counts := map[Family]int{} + for f := range keys { + keys[f] = sortDedup(keys[f]) + counts[f] = len(keys[f]) + } + // Removed certificates are recorded by count, since they have no identity. + counts[FamilyTLSPrune] = tlsPrunedChains + + return ProgrammedSet{ + CapturedAt: time.Now().UTC(), + Keys: keys, + Counts: counts, + TLSPrunedChains: tlsPrunedChains, + TLSPrunedSecrets: tlsPrunedSecrets, + TLSListenersLeftIntact: tlsListenersLeftIntact, + } +} + +// programmedSetHandler serves the latest snapshot as JSON. It is a read-only, +// test-only debug endpoint and returns an empty snapshot before the first build. +func (r *programmedRecorder) programmedSetHandler() http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + ps := r.snapshot() + w.Header().Set("Content-Type", "application/json") + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + _ = enc.Encode(ps) + } +} diff --git a/internal/extensionserver/server/programmedset_scan.go b/internal/extensionserver/server/programmedset_scan.go new file mode 100644 index 00000000..ad41abf2 --- /dev/null +++ b/internal/extensionserver/server/programmedset_scan.go @@ -0,0 +1,160 @@ +package server + +import ( + "sort" + "strings" + + clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" +) + +// These read-only scanners extract an identity for each change the build made. +// They mirror, in reverse, how the configuration is written, and they define +// the exact identity format the parity test must reproduce. The "##" separator +// is used because it cannot appear in any of the names it joins. + +const keySep = "##" + +// wafRouteKey is the identity of a route protected by a firewall: +// +// ######// +// +// The governing policy is part of the identity, so a route protected by the +// wrong policy produces a different identity and is caught as a mismatch. +func wafRouteKey(rc, vh, rt, tppNS, tppName, mode string) string { + return strings.Join([]string{rc, vh, rt, tppNS + "/" + tppName + "/" + mode}, keySep) +} + +// connectorRouteKey is the identity of a connector route (online or offline): +// +// #### +func connectorRouteKey(rc, vh, rt string) string { + return strings.Join([]string{rc, vh, rt}, keySep) +} + +// listenerChainKey is the identity of a changed listener filter chain: +// +// ## +func listenerChainKey(listener, chain string) string { + return strings.Join([]string{listener, chain}, keySep) +} + +// datumGatewayTPP returns the protection policy governing a route (namespace, +// name, mode). ok is false when the route carries no such marker, meaning it is +// not protected. +func datumGatewayTPP(rt *routev3.Route) (ns, name, mode string, ok bool) { + md := rt.GetMetadata() + if md == nil { + return "", "", "", false + } + dg := md.GetFilterMetadata()[datumGatewayMetaKey] + if dg == nil { + return "", "", "", false + } + res := dg.GetFields()["resources"].GetListValue() + if res == nil || len(res.GetValues()) == 0 { + return "", "", "", false + } + f := res.GetValues()[0].GetStructValue().GetFields() + return f["namespace"].GetStringValue(), + f["name"].GetStringValue(), + f["mode"].GetStringValue(), + true +} + +// isConnectRoute reports whether a route is an online connector tunnel route. +func isConnectRoute(rt *routev3.Route) bool { + ra := rt.GetRoute() + if ra == nil { + return false + } + for _, uc := range ra.GetUpgradeConfigs() { + if strings.EqualFold(uc.GetUpgradeType(), "CONNECT") { + return true + } + } + return false +} + +// isOfflineDirectResponse reports whether a route directly returns the +// tunnel-offline 503 response, covering both the dedicated offline route and +// user-facing routes rewritten to it. +func isOfflineDirectResponse(rt *routev3.Route) bool { + dr := rt.GetDirectResponse() + if dr == nil { + return false + } + if dr.GetStatus() != 503 { + return false + } + return dr.GetBody().GetInlineString() == offlineBodyMarker +} + +// offlineBodyMarker is the response body the connector offline path writes, +// duplicated here so the scanner needs no import dependency. +const offlineBodyMarker = "Tunnel not online" + +// isReplacedConnectorCluster reports whether a connector cluster has been +// replaced with its tunnel form. A cluster that has not been replaced means the +// substitution failed. +func isReplacedConnectorCluster(cl *clusterv3.Cluster) bool { + if cl.GetType() != clusterv3.Cluster_STATIC { + return false + } + return cl.GetTransportSocket().GetName() == connectorInternalTransport +} + +// eachHCM invokes fn for every connection manager across all of the listener's +// filter chains, including the default chain. One that can't be decoded is +// skipped, since recording must never fail the build. +func eachHCM(l *listenerv3.Listener, fn func(chainName string, hcm *hcmv3.HttpConnectionManager)) { + chains := make([]*listenerv3.FilterChain, 0, len(l.GetFilterChains())+1) + chains = append(chains, l.GetFilterChains()...) + if dfc := l.GetDefaultFilterChain(); dfc != nil { + chains = append(chains, dfc) + } + for _, fc := range chains { + for _, f := range fc.GetFilters() { + if f.GetName() != hcmNetworkFilterName { + continue + } + tc := f.GetTypedConfig() + if tc == nil { + continue + } + hcm := &hcmv3.HttpConnectionManager{} + if err := tc.UnmarshalTo(hcm); err != nil { + continue + } + fn(fc.GetName(), hcm) + } + } +} + +// hcmHasFilterAtZero reports whether the first filter is named filterName. The +// firewall is always inserted first, so checking the first position is the +// precise signal that it was injected. +func hcmHasFilterAtZero(hcm *hcmv3.HttpConnectionManager, filterName string) bool { + fs := hcm.GetHttpFilters() + return len(fs) > 0 && fs[0].GetName() == filterName +} + +// sortDedup returns a sorted, de-duplicated copy of in. +func sortDedup(in []string) []string { + if len(in) == 0 { + return in + } + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + sort.Strings(out) + return out +} diff --git a/internal/extensionserver/server/programmedset_test.go b/internal/extensionserver/server/programmedset_test.go new file mode 100644 index 00000000..a18cc253 --- /dev/null +++ b/internal/extensionserver/server/programmedset_test.go @@ -0,0 +1,363 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + pb "github.com/envoyproxy/gateway/proto/extension" + clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + internalupstreamv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/internal_upstream/v3" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + networkingv1alpha1 "go.datum.net/network-services-operator/api/v1alpha1" +) + +// --- buildProgrammedSet (pure function) --- + +// mkWAFRoute builds a route already carrying the datum-gateway metadata and a +// coraza typed_per_filter_config, simulating a route after ApplyTPPRouteConfig. +func mkWAFRoute(t *testing.T, tppNS, tppName string) *routev3.Route { + t.Helper() + meta, err := structpb.NewStruct(map[string]any{ + "resources": []any{ + map[string]any{ + "kind": "TrafficProtectionPolicy", + "namespace": tppNS, + "name": tppName, + "mode": "Observe", + }, + }, + }) + require.NoError(t, err) + return &routev3.Route{ + Name: "fwd", + Metadata: &corev3.Metadata{ + FilterMetadata: map[string]*structpb.Struct{datumGatewayMetaKey: meta}, + }, + } +} + +// mkCorazaHCMListener builds a listener whose HCM has the coraza filter at +// position 0 and a local_reply_config, simulating a fully-mutated listener. +func mkCorazaHCMListener(t *testing.T, name, corazaFilter string, withLocalReply bool) *listenerv3.Listener { + t.Helper() + hcm := &hcmv3.HttpConnectionManager{ + RouteSpecifier: &hcmv3.HttpConnectionManager_Rds{Rds: &hcmv3.Rds{RouteConfigName: name}}, + HttpFilters: []*hcmv3.HttpFilter{ + {Name: corazaFilter, Disabled: true}, + {Name: "envoy.filters.http.router"}, + }, + } + if withLocalReply { + hcm.LocalReplyConfig = &hcmv3.LocalReplyConfig{} + } + hcmAny, err := anypb.New(hcm) + require.NoError(t, err) + return &listenerv3.Listener{ + Name: name, + FilterChains: []*listenerv3.FilterChain{{ + Name: "fc", + Filters: []*listenerv3.Filter{{ + Name: hcmNetworkFilterName, + ConfigType: &listenerv3.Filter_TypedConfig{TypedConfig: hcmAny}, + }}, + }}, + } +} + +// mkReplacedConnectorCluster builds a STATIC internal-upstream cluster matching +// what buildConnectorCluster produces. +func mkReplacedConnectorCluster(t *testing.T, name string) *clusterv3.Cluster { + t.Helper() + ts, err := anypb.New(&internalupstreamv3.InternalUpstreamTransport{}) + require.NoError(t, err) + return &clusterv3.Cluster{ + Name: name, + ClusterDiscoveryType: &clusterv3.Cluster_Type{Type: clusterv3.Cluster_STATIC}, + LoadAssignment: &endpointv3.ClusterLoadAssignment{ClusterName: name}, + TransportSocket: &corev3.TransportSocket{ + Name: connectorInternalTransport, + ConfigType: &corev3.TransportSocket_TypedConfig{TypedConfig: ts}, + }, + } +} + +func TestBuildProgrammedSet_AllFamilies(t *testing.T) { + const corazaFilter = "coraza-waf" + + listeners := []*listenerv3.Listener{ + mkCorazaHCMListener(t, "gw/https", corazaFilter, true), + } + routes := []*routev3.RouteConfiguration{{ + Name: "gw/https", + VirtualHosts: []*routev3.VirtualHost{{ + Name: "vh", + Routes: []*routev3.Route{ + // online CONNECT route + { + Name: "connector-connect-vh", + Action: &routev3.Route_Route{Route: &routev3.RouteAction{ + UpgradeConfigs: []*routev3.RouteAction_UpgradeConfig{{UpgradeType: "CONNECT"}}, + }}, + }, + // WAF-governed forwarding route + mkWAFRoute(t, "proj-ns", "test-tpp"), + }, + }}, + }} + clusters := []*clusterv3.Cluster{ + mkReplacedConnectorCluster(t, "httproute/proj-ns/proxy/rule/0"), + {Name: "infra-cluster"}, // not a connector cluster + } + + ps := buildProgrammedSet(listeners, routes, clusters, corazaFilter, 2, 1, 0) + + assert.Equal(t, []string{wafRouteKey("gw/https", "vh", "fwd", "proj-ns", "test-tpp", "Observe")}, + ps.Keys[FamilyWAFRoute]) + assert.Equal(t, 1, ps.Counts[FamilyWAFRoute]) + + assert.Equal(t, []string{listenerChainKey("gw/https", "fc")}, ps.Keys[FamilyWAFHCM]) + assert.Equal(t, []string{listenerChainKey("gw/https", "fc")}, ps.Keys[FamilyLocalReply]) + + assert.Equal(t, []string{"httproute/proj-ns/proxy/rule/0"}, ps.Keys[FamilyConnectorCluster]) + assert.Equal(t, []string{connectorRouteKey("gw/https", "vh", "connector-connect-vh")}, + ps.Keys[FamilyConnectorRoute]) + + // TLS prune outcome carried directly. + assert.Equal(t, 2, ps.TLSPrunedChains) + assert.Equal(t, 1, ps.TLSPrunedSecrets) + assert.Equal(t, 0, ps.TLSListenersLeftIntact) + assert.Equal(t, 2, ps.Counts[FamilyTLSPrune]) +} + +func TestBuildProgrammedSet_OfflineConnector(t *testing.T) { + routes := []*routev3.RouteConfiguration{{ + Name: "gw/https", + VirtualHosts: []*routev3.VirtualHost{{ + Name: "vh", + Routes: []*routev3.Route{ + { + Name: "connector-offline-vh", + Action: &routev3.Route_DirectResponse{DirectResponse: &routev3.DirectResponseAction{ + Status: 503, + Body: &corev3.DataSource{Specifier: &corev3.DataSource_InlineString{InlineString: offlineBodyMarker}}, + }}, + }, + }, + }}, + }} + + ps := buildProgrammedSet(nil, routes, nil, "coraza-waf", 0, 0, 0) + assert.Equal(t, []string{connectorRouteKey("gw/https", "vh", "connector-offline-vh")}, + ps.Keys[FamilyConnectorOffline]) + // A 503 with a different body is NOT an offline connector marker. + assert.Empty(t, ps.Keys[FamilyConnectorRoute]) +} + +// TestBuildProgrammedSet_WrongKeyedOracle proves the WAF route key changes when +// the governing TPP changes even though the route count is identical — the +// wrong-keyed class the gate exists to catch. +func TestBuildProgrammedSet_WrongKeyedOracle(t *testing.T) { + routesRight := []*routev3.RouteConfiguration{{ + Name: "gw/https", + VirtualHosts: []*routev3.VirtualHost{{Name: "vh", Routes: []*routev3.Route{mkWAFRoute(t, "proj-a", "tpp-a")}}}, + }} + routesWrong := []*routev3.RouteConfiguration{{ + Name: "gw/https", + VirtualHosts: []*routev3.VirtualHost{{Name: "vh", Routes: []*routev3.Route{mkWAFRoute(t, "proj-b", "tpp-b")}}}, + }} + + right := buildProgrammedSet(nil, routesRight, nil, "coraza-waf", 0, 0, 0) + wrong := buildProgrammedSet(nil, routesWrong, nil, "coraza-waf", 0, 0, 0) + + assert.Equal(t, right.Counts[FamilyWAFRoute], wrong.Counts[FamilyWAFRoute], + "counts identical — count-only gate would pass") + assert.NotEqual(t, right.Keys[FamilyWAFRoute], wrong.Keys[FamilyWAFRoute], + "key set differs — wrong-keyed must be detectable") +} + +func TestBuildProgrammedSet_NoCorazaFilterName(t *testing.T) { + // With an empty filter name the WAF HCM family must not be recorded even + // though the HCM happens to carry a position-0 filter. + listeners := []*listenerv3.Listener{mkCorazaHCMListener(t, "gw/https", "coraza-waf", false)} + ps := buildProgrammedSet(listeners, nil, nil, "", 0, 0, 0) + assert.Empty(t, ps.Keys[FamilyWAFHCM]) +} + +// --- recorder + endpoint --- + +func TestProgrammedRecorder_EmptyBeforeFirstBuild(t *testing.T) { + r := newProgrammedRecorder() + ps := r.snapshot() + assert.Equal(t, uint64(0), ps.BuildID) + assert.NotNil(t, ps.Keys) + assert.NotNil(t, ps.Counts) +} + +func TestProgrammedRecorder_BuildIDIncrements(t *testing.T) { + r := newProgrammedRecorder() + r.record(nil, nil, nil, "coraza-waf", 0, 0, 0) + first := r.snapshot() + r.record(nil, nil, nil, "coraza-waf", 0, 0, 0) + second := r.snapshot() + assert.Equal(t, uint64(1), first.BuildID) + assert.Equal(t, uint64(2), second.BuildID) +} + +func TestProgrammedSetHandler_ServesJSON(t *testing.T) { + r := newProgrammedRecorder() + routes := []*routev3.RouteConfiguration{{ + Name: "gw/https", + VirtualHosts: []*routev3.VirtualHost{{Name: "vh", Routes: []*routev3.Route{mkWAFRoute(t, "proj-a", "tpp-a")}}}, + }} + r.record(nil, routes, nil, "coraza-waf", 0, 0, 0) + + req := httptest.NewRequest(http.MethodGet, programmedSetEndpointPath, nil) + w := httptest.NewRecorder() + r.programmedSetHandler()(w, req) + + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "application/json", w.Header().Get("Content-Type")) + + var got ProgrammedSet + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &got)) + assert.Equal(t, uint64(1), got.BuildID) + assert.Equal(t, []string{wafRouteKey("gw/https", "vh", "fwd", "proj-a", "tpp-a", "Observe")}, + got.Keys[FamilyWAFRoute]) +} + +// TestPostTranslateModify_RecordsProgrammedSet drives the full hook with the +// same fixture as TestPostTranslateModify_FullSnapshot and asserts the +// programmed-set endpoint reflects the build (1 WAF route, 1 WAF HCM, 1 +// connector cluster, 1 connector route). +func TestPostTranslateModify_RecordsProgrammedSet(t *testing.T) { + const ( + upstreamNS = "test-project" + dsNS = upstreamNS + gwName = "pset-gw" + proxyName = "test-proxy" + connectorName = "test-connector" + nodeID = "test-node-id" + connCluster = "httproute/" + dsNS + "/" + proxyName + "/rule/0" + ) + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: upstreamNS}} + tpp := &networkingv1alpha.TrafficProtectionPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-tpp", Namespace: upstreamNS}, + Spec: networkingv1alpha.TrafficProtectionPolicySpec{ + Mode: networkingv1alpha.TrafficProtectionPolicyObserve, + TargetRefs: []gatewayv1alpha2.LocalPolicyTargetReferenceWithSectionName{{ + LocalPolicyTargetReference: gatewayv1.LocalPolicyTargetReference{ + Kind: "Gateway", Name: gatewayv1.ObjectName(gwName), + }, + }}, + RuleSets: []networkingv1alpha.TrafficProtectionPolicyRuleSet{{ + Type: networkingv1alpha.TrafficProtectionPolicyOWASPCoreRuleSet, + OWASPCoreRuleSet: networkingv1alpha.OWASPCRS{ + ParanoiaLevels: networkingv1alpha.ParanoiaLevels{Blocking: 1, Detection: 1}, + ScoreThresholds: networkingv1alpha.OWASPScoreThresholds{Inbound: 5, Outbound: 4}, + }, + }}, + }, + } + proxy := &networkingv1alpha.HTTPProxy{ + ObjectMeta: metav1.ObjectMeta{Name: proxyName, Namespace: upstreamNS}, + Spec: networkingv1alpha.HTTPProxySpec{ + Rules: []networkingv1alpha.HTTPProxyRule{{ + Backends: []networkingv1alpha.HTTPProxyRuleBackend{{ + Endpoint: "http://backend.example.com:9000", + Connector: &networkingv1alpha.ConnectorReference{Name: connectorName}, + }}, + }}, + }, + } + connector := &networkingv1alpha1.Connector{ + ObjectMeta: metav1.ObjectMeta{Name: connectorName, Namespace: upstreamNS}, + Status: networkingv1alpha1.ConnectorStatus{ + Conditions: []metav1.Condition{{ + Type: networkingv1alpha1.ConnectorConditionReady, Status: metav1.ConditionTrue, + Reason: "Ready", LastTransitionTime: metav1.Now(), + }}, + ConnectionDetails: &networkingv1alpha1.ConnectorConnectionDetails{ + Type: networkingv1alpha1.PublicKeyConnectorConnectionType, + PublicKey: &networkingv1alpha1.ConnectorConnectionDetailsPublicKey{Id: nodeID}, + }, + }, + } + + scheme := testServerScheme(t) + cl := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(ns, tpp, proxy). + WithStatusSubresource(connector).WithObjects(connector).Build() + // Recording only happens when the test-environment endpoint is enabled. + cfg := testServerConfig() + cfg.EnableProgrammedSet = true + srv := New(cl, cfg, discardLogger()) + + req := &pb.PostTranslateModifyRequest{ + Clusters: []*clusterv3.Cluster{{Name: connCluster}, {Name: "infra-cluster"}}, + Listeners: []*listenerv3.Listener{ + mkListenerWithHCM(t), + mkListenerWithStaticRoute(t, "envoy-gateway-proxy-ready-0.0.0.0-19003"), + }, + Routes: []*routev3.RouteConfiguration{{ + Name: "consumer-gw/pset-gw/https", + VirtualHosts: []*routev3.VirtualHost{{ + Name: "vh", Domains: []string{"app.example.com"}, Metadata: egGatewayMeta(t, dsNS, gwName), + Routes: []*routev3.Route{{ + Name: "fwd", + Action: &routev3.Route_Route{Route: &routev3.RouteAction{ + ClusterSpecifier: &routev3.RouteAction_Cluster{Cluster: connCluster}, + }}, + }}, + }}, + }}, + } + + _, err := srv.PostTranslateModify(context.Background(), req) + require.NoError(t, err) + + ps := srv.programmed.snapshot() + assert.Equal(t, uint64(1), ps.BuildID) + assert.Equal(t, 1, ps.Counts[FamilyWAFRoute], "one WAF-governed route") + assert.Equal(t, 1, ps.Counts[FamilyWAFHCM], "coraza injected into the one RDS HCM, not the readiness listener") + assert.Equal(t, 1, ps.Counts[FamilyConnectorCluster], "one connector cluster replaced") + assert.Equal(t, 1, ps.Counts[FamilyConnectorRoute], "one CONNECT route prepended") + + // The WAF route key must name the governing TPP (wrong-keyed oracle). + require.Len(t, ps.Keys[FamilyWAFRoute], 1) + assert.Contains(t, ps.Keys[FamilyWAFRoute][0], "test-project/test-tpp/Observe") +} + +// TestPostTranslateModify_NoRecordingWhenDisabled proves the production default: +// with the test-only endpoint off, a build records nothing, so the snapshot +// stays empty and no per-build work is done. +func TestPostTranslateModify_NoRecordingWhenDisabled(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(testServerScheme(t)).Build() + // testServerConfig() leaves EnableProgrammedSet at its false default. + srv := New(cl, testServerConfig(), discardLogger()) + + _, err := srv.PostTranslateModify(context.Background(), &pb.PostTranslateModifyRequest{}) + require.NoError(t, err) + + ps := srv.programmed.snapshot() + assert.Equal(t, uint64(0), ps.BuildID, "no build was recorded") + assert.Empty(t, ps.Keys, "nothing recorded while the endpoint is disabled") +} diff --git a/internal/extensionserver/server/server.go b/internal/extensionserver/server/server.go index 5fc21315..57a53cb1 100644 --- a/internal/extensionserver/server/server.go +++ b/internal/extensionserver/server/server.go @@ -7,6 +7,7 @@ package server import ( "context" "log/slog" + "net/http" "time" pb "github.com/envoyproxy/gateway/proto/extension" @@ -36,6 +37,12 @@ type ServerConfig struct { // empty, local-reply injection is a no-op. Sourced from // GatewayConfig.ErrorPage + the embedded/override HTML body. LocalReply mutate.LocalReplyConfig + // EnableProgrammedSet turns on the read-only /debug/programmed-set endpoint + // and the per-build recording that backs it. It exists only to let a test + // confirm the proxy is running exactly the set the build intended, so it is + // off in production and enabled only in the test environment. When off, the + // build does no extra work and the endpoint is not served. + EnableProgrammedSet bool } // Server implements pb.EnvoyGatewayExtensionServer for the NSO production @@ -51,15 +58,29 @@ type Server struct { client client.Client cfg ServerConfig log *slog.Logger + // programmed records what the last build changed, so a test can ask the proxy + // to prove it is running exactly that. Always non-nil; capturing it only reads + // what was already produced. + programmed *programmedRecorder } // New returns a production extension server backed by the given cache client. // In production, cl is the ctrl.Manager.GetClient() from NewManager(). // In tests, cl is a fake client pre-populated with the test objects. func New(cl client.Client, cfg ServerConfig, log *slog.Logger) *Server { - return &Server{client: cl, cfg: cfg, log: log} + return &Server{client: cl, cfg: cfg, log: log, programmed: newProgrammedRecorder()} } +// ProgrammedSetHandler serves what the last build changed, so a test can confirm +// the proxy is running exactly that. Read-only. +func (s *Server) ProgrammedSetHandler() http.HandlerFunc { + return s.programmed.programmedSetHandler() +} + +// ProgrammedSetEndpointPath is exported so the server and the test tooling share +// one definition of where this endpoint lives. +const ProgrammedSetEndpointPath = programmedSetEndpointPath + // PostTranslateModify applies the TPP/WAF and Connector mutation families to // the full xDS snapshot and returns the complete (mutated) resource set. // Secrets are passed through unchanged — the response replaces EG's entire @@ -317,6 +338,16 @@ func (s *Server) PostTranslateModify( extmetrics.ConnectorRoutesTotal.Add(float64(vhCount)) extmetrics.ConnectorOfflineRoutesTotal.Add(float64(offlineRtCount)) + // In the test environment, record what this build changed so a test can later + // confirm the proxy is running exactly that. This only reads the configuration + // just produced; it changes nothing. Off in production, where it does no work. + if s.cfg.EnableProgrammedSet { + s.programmed.record( + listeners, routes, clusters, s.cfg.Coraza.FilterName, + prunedChains, prunedSecrets, listenersLeftIntact, + ) + } + s.log.Info("PostTranslateModify", "clusters", len(clusters), "listeners", len(listeners), diff --git a/test/e2e-edge/README.md b/test/e2e-edge/README.md new file mode 100644 index 00000000..952e28ea --- /dev/null +++ b/test/e2e-edge/README.md @@ -0,0 +1,87 @@ +# End-to-end edge guarantees + +These tests prove that the Datum edge *behaves* the way customers expect, by +sending real traffic through the real proxy and checking what actually comes +back. They are the standing guarantees described in +[`docs/testing/README.md`](../../docs/testing/README.md). + +## How a guarantee is checked + +Every scenario follows the same three-part check, in priority order: + +1. **The traffic verdict (always decisive).** The test makes a real request and + asserts on the real response — a blocked attack, a 503, a branded page, a + 200 from a healthy listener. If the edge behaves wrongly, this fails. A test + is never satisfied by the platform merely *reporting* success. +2. **The configuration is genuinely present.** A + [parity check](../parity/README.md) confirms the running proxy actually + carries the configuration it was told to — closing the blind spot where a + successful-looking response hides a rule that protects nothing. +3. **It's the right configuration serving the request.** A build marker + confirms the response came from the configuration under test, not from stale + config left over from a previous state — so a pass can't be a timing fluke. + +The first is the point; the second and third exist so a surprising result +becomes a diagnosis rather than a mystery. + +## The scenarios + +Each one corresponds to a past production incident, now held in place. + +### Web-app firewall enforcement +A malicious request (matching the firewall's attack rules) must be refused, +while a legitimate request to the same endpoint still succeeds. The test also +flips the policy into observe-only mode and confirms the same attack is then +*allowed* — proving the block is genuinely driven by the customer's firewall +policy and not by some unrelated default. This guards the customer's actual +protection, and the risk that a single bad firewall rule wedges the whole +listener. + +### Offline backend returns a clean 503 +When a backend connector is offline, the customer-facing path must return a +real 503 — not hang, and not serve a blank. This is checked as an observed +response on the user path, because that's what a customer experiences. + +### Branded error page reaches the customer +When the edge serves an error, the customer must receive Datum's styled page, +confirmed by finding the page's content in the actual response body — not by +trusting that the configuration was applied. Production once needed manual +restarts to make this take effect, exactly the kind of inert-configuration gap +this scenario now catches. + +### One bad certificate can't break its neighbors +A single invalid certificate must not take down the other, healthy listeners +sharing the same proxy. The test introduces a genuinely bad certificate and +confirms its listener is isolated while sibling listeners keep serving real +traffic — the "one bad resource freezes everything" failure mode, contained. + +## Running them + +The scenarios run against the production-fidelity environment: + +``` +task test-infra:up # bring the edge online (proxy + extension server + firewall) +task test-infra:smoke # quick confidence check: real traffic serves +task test-infra:e2e # the full guarantee suite above +``` + +See [`Taskfile.test-infra.yml`](../../Taskfile.test-infra.yml) for the +environment these assume. + +## Layout + +- Scenario folders (`waf-enforcement/`, `connector-offline-503/`, + `branded-error-page/`, `atomic-reject-isolation/`) — one guarantee each. +- `_steps/` — shared, reusable checks (send-a-request, confirm-configuration, + capture-the-build-marker) so every scenario asserts behavior the same way. +- `_fixtures/` — the supporting pieces a scenario needs (a sample backend, an + attack corpus, a pre-made bad certificate, the offline-backend stand-in). + +## A note on honesty + +Where the edge genuinely cannot yet do something, the suite says so rather than +papering over it. The "offline backend recovers" path is deliberately held back +because the edge today does not reliably re-apply configuration when a backend +comes *back* online — and the test proves that gap exists instead of pretending +it's closed. A guarantee we can't keep is documented as a gap, not asserted as +a pass. diff --git a/test/e2e-edge/_fixtures/certs/expired-cert-secret.yaml b/test/e2e-edge/_fixtures/certs/expired-cert-secret.yaml new file mode 100644 index 00000000..305c8583 --- /dev/null +++ b/test/e2e-edge/_fixtures/certs/expired-cert-secret.yaml @@ -0,0 +1,14 @@ +# GENERATED by gen-certs.sh — do not edit by hand; re-run the script to refresh. +# +# A pre-minted EXPIRED certificate for bad.e2e.env.datum.net. Its validity window +# (20260621000000Z .. 20260622000000Z) is in the past, so the extension server +# drops the listener that references it while sibling listeners keep serving. +apiVersion: v1 +kind: Secret +metadata: + name: expired-leaf-tls + namespace: default +type: kubernetes.io/tls +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ0ekNDQVY2Z0F3SUJBZ0lVTW5lMFBkKzczaDFPcFh0YWo1eDlIbzRxR2Fjd0NnWUlLb1pJemowRUF3SXcKSURFZU1Cd0dBMVVFQXd3VlltRmtMbVV5WlM1bGJuWXVaR0YwZFcwdWJtVjBNQjRYRFRJMk1EWXlNVEF3TURBdwpNRm9YRFRJMk1EWXlNakF3TURBd01Gb3dJREVlTUJ3R0ExVUVBd3dWWW1Ga0xtVXlaUzVsYm5ZdVpHRjBkVzB1CmJtVjBNRmt3RXdZSEtvWkl6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVKWEYyV0U1UmZzN2lJbkpmVWRWZHNRa0IKWU9aRWk1TmV3cmZ5T3hkRWdiZ2ViNnRGMkpUbHA5L2tMNUYweEkvSWpYeXpFWlF1YU9vMVNwWCtYd0pWS3FOMgpNSFF3SUFZRFZSMFJCQmt3RjRJVlltRmtMbVV5WlM1bGJuWXVaR0YwZFcwdWJtVjBNQXdHQTFVZEV3RUIvd1FDCk1BQXdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CTUdBMVVkSlFRTU1Bb0dDQ3NHQVFVRkJ3TUJNQjBHQTFVZERnUVcKQkJTaTRpOTB2VkEwdmlQU0k2aW9WdFFEdXFhOE5qQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQTJmalMvbGVmNQpTRHFEMEt5UWJRV1hENlltMHg4NDJnU2VPMS9XZ041MHpnSWdYVHd4MWZwSTNVc0U1UXI4bkNXWk0wd044b0cyCkU4a0ttZUo1a3BDL3lFST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUM3Ri9qbkVFSTBYQk12emg4K0lCaVVycG5MMnVGeEpFSUlMQWkyd1Q4eXlvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFSlhGMldFNVJmczdpSW5KZlVkVmRzUWtCWU9aRWk1TmV3cmZ5T3hkRWdiZ2ViNnRGMkpUbApwOS9rTDVGMHhJL0lqWHl6RVpRdWFPbzFTcFgrWHdKVktnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/test/e2e-edge/_fixtures/certs/gen-certs.sh b/test/e2e-edge/_fixtures/certs/gen-certs.sh new file mode 100755 index 00000000..8d5a5d9b --- /dev/null +++ b/test/e2e-edge/_fixtures/certs/gen-certs.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Regenerate the pre-minted EXPIRED TLS certificate the invalid-certificate test +# relies on. +# +# The test needs a certificate that is already expired at test time so the +# extension server drops the listener that references it while sibling listeners +# keep serving. cert-manager cannot help here: it renews before expiry, so it +# cannot hold a certificate in the expired state. We instead mint a self-signed +# certificate whose validity window is in the past. +# +# This is a listener certificate for the customer hostname under test, entirely +# separate from the certificate authority securing the extension server's own +# connection to the gateway; do not wire it to that authority. +# +# Output: expired-cert-secret.yaml — a Secret with the expired certificate and +# key inline. The committed Secret is what the test consumes, so a live run needs +# no openssl; re-run this script only to refresh it. +# +# Usage: ./gen-certs.sh [HOSTNAME] [SECRET_NAME] [SECRET_NAMESPACE] +set -euo pipefail + +HOST="${1:-bad.e2e.env.datum.net}" +SECRET_NAME="${2:-expired-leaf-tls}" +SECRET_NS="${3:-default}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +# ECDSA P-256 key. +openssl ecparam -name prime256v1 -genkey -noout -out "${WORK}/tls.key" + +# A self-signed certificate whose validity window is entirely in the past. We set +# the exact start and end times with OpenSSL 3.x's -not_before/-not_after so the +# certificate is already expired by the time the test runs. +NOT_BEFORE="20260621000000Z" +NOT_AFTER="20260622000000Z" + +cat > "${WORK}/leaf.cnf" < "${SCRIPT_DIR}/expired-cert-secret.yaml" <200 liveness swap, but not real tunnel +// establishment or NAT traversal. +// +// When a connector is online, the extension server points its backend at a path +// that issues an HTTP CONNECT toward the connector's target, which in the test +// is this pod's Service. On a CONNECT, this proxy dials the configured upstream +// (the echo backend) and blindly splices bytes both ways — a minimal forward +// proxy. +// +// Liveness is driven entirely by the connector's annotation on the control plane +// (see _steps/flip-connector-liveness.yaml); this proxy is always up. It exists +// so an online request can only succeed via the tunnel, never via a direct +// fallback route — which is what makes the 503->200 transition meaningful. +package main + +import ( + "io" + "log" + "net" + "net/http" + "os" + "time" +) + +func main() { + listen := envOr("LISTEN_ADDR", ":8080") + // Where CONNECT requests are forwarded. Default to the echo backend Service. + // The proxy ignores the CONNECT target host and always dials this upstream, + // so the test controls the destination via UPSTREAM_ADDR rather than the + // host the proxy sends. + upstream := envOr("UPSTREAM_ADDR", "echo-backend.default.svc.cluster.local:8080") + + srv := &http.Server{ + Addr: listen, + ReadTimeout: 0, // tunnels are long-lived; no read deadline on the hijacked conn + Handler: &proxy{upstream: upstream}, + } + log.Printf("connect-proxy listening on %s, forwarding CONNECT -> %s", listen, upstream) + if err := srv.ListenAndServe(); err != nil { + log.Fatalf("server exited: %v", err) + } +} + +type proxy struct { + upstream string +} + +func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodConnect { + // A plain GET is handy as a liveness/readiness probe. + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "connect-proxy ready\n") + return + } + + dst, err := net.DialTimeout("tcp", p.upstream, 10*time.Second) + if err != nil { + log.Printf("CONNECT %s: dial upstream %s failed: %v", r.Host, p.upstream, err) + http.Error(w, "upstream unavailable", http.StatusBadGateway) + return + } + defer dst.Close() + + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "hijacking unsupported", http.StatusInternalServerError) + return + } + client, _, err := hj.Hijack() + if err != nil { + log.Printf("CONNECT %s: hijack failed: %v", r.Host, err) + return + } + defer client.Close() + + // Tell the client the tunnel is established, then splice bytes both ways. + if _, err := client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil { + log.Printf("CONNECT %s: write 200 failed: %v", r.Host, err) + return + } + + done := make(chan struct{}, 2) + go func() { _, _ = io.Copy(dst, client); done <- struct{}{} }() + go func() { _, _ = io.Copy(client, dst); done <- struct{}{} }() + <-done +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/test/e2e-edge/_fixtures/connector-tunnel/manifest.yaml b/test/e2e-edge/_fixtures/connector-tunnel/manifest.yaml new file mode 100644 index 00000000..d6595904 --- /dev/null +++ b/test/e2e-edge/_fixtures/connector-tunnel/manifest.yaml @@ -0,0 +1,66 @@ +# Connector tunnel stand-in deployment. +# +# The image is built from this directory's Dockerfile and loaded into the +# downstream cluster, e.g.: +# docker build -t connect-proxy:e2e test/e2e/_fixtures/connector-tunnel +# kind load docker-image connect-proxy:e2e --name +# +# When a connector is online, the extension server points it at the connector's +# target. Point that target at this Service so an online request can only succeed +# through the tunnel — there is no direct fallback route to the echo backend. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: connect-proxy + namespace: default + labels: + purpose: connect-proxy +spec: + replicas: 1 + selector: + matchLabels: + purpose: connect-proxy + template: + metadata: + labels: + purpose: connect-proxy + spec: + containers: + - name: connect-proxy + image: connect-proxy:e2e + imagePullPolicy: IfNotPresent + env: + - name: LISTEN_ADDR + value: ":8080" + # Forward CONNECT to the echo backend Service. + - name: UPSTREAM_ADDR + value: "echo-backend.default.svc.cluster.local:8080" + ports: + - containerPort: 8080 + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 1 + periodSeconds: 2 + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + terminationGracePeriodSeconds: 0 +--- +apiVersion: v1 +kind: Service +metadata: + name: connect-proxy + namespace: default +spec: + ports: + - name: connect + port: 8080 + targetPort: 8080 + selector: + purpose: connect-proxy diff --git a/test/e2e-edge/_fixtures/echo-backend.yaml b/test/e2e-edge/_fixtures/echo-backend.yaml new file mode 100644 index 00000000..8806317d --- /dev/null +++ b/test/e2e-edge/_fixtures/echo-backend.yaml @@ -0,0 +1,46 @@ +# Echo / HTTP backend fixture. +# +# go-httpbin gives deterministic endpoints the suite leans on: +# /status/ -> returns that status (drive 200 and 5xx) +# /get -> echoes the request (drive WAF benign/malicious probes) +# Plain HTTP only: the edge proxy terminates TLS per listener and forwards +# plaintext to this backend. +# +# Applied rather than freshly created so scenarios that share the downstream +# cluster coexist regardless of run order. +apiVersion: v1 +kind: Pod +metadata: + name: echo-backend + namespace: default + labels: + purpose: echo-backend +spec: + containers: + - name: backend + image: ghcr.io/mccutchen/go-httpbin:2.18.1 + command: ["/bin/go-httpbin"] + args: ["-host", "0.0.0.0", "-port", "8080"] + ports: + - containerPort: 8080 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + terminationGracePeriodSeconds: 0 +--- +apiVersion: v1 +kind: Service +metadata: + name: echo-backend + namespace: default +spec: + ports: + - name: http + port: 8080 + targetPort: 8080 + selector: + purpose: echo-backend diff --git a/test/e2e-edge/_fixtures/waf-corpus.txt b/test/e2e-edge/_fixtures/waf-corpus.txt new file mode 100644 index 00000000..c9621630 --- /dev/null +++ b/test/e2e-edge/_fixtures/waf-corpus.txt @@ -0,0 +1,30 @@ +# WAF request corpus. +# +# Format (tab- or space-separated; lines starting with # are comments): +#