Skip to content

test(infra): prod-fidelity edge test env + e2e suite #276

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ecv merged 20 commits into from
Jul 14, 2026
Merged

test(infra): prod-fidelity edge test env + e2e suite #276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
d6b6cc4
test(infra): stand up a production-fidelity edge test environment
scotwells Jun 25, 2026
a824579
Enable the programmed-set endpoint in the test environment
scotwells Jun 25, 2026
90b2b12
test(infra): run e2e suite against the two-cluster prod-fidelity env
ecv Jul 14, 2026
86f3c45
test(e2e): target the env GatewayClass in the WAF enforce test
ecv Jul 14, 2026
c4eaa13
test(e2e): ride the shared data plane in the cert tests
ecv Jul 14, 2026
2565543
test(infra): persist a JSON run report from test-e2e
ecv Jul 14, 2026
4466cf5
test(infra): keep env image placeholders in the suite fold
ecv Jul 14, 2026
4aecfe0
test(e2e): adapt the WAF attack/neutralize tests to the env
ecv Jul 14, 2026
e2c1296
ci: run e2e against the prod-fidelity env, retire the bridge
ecv Jul 14, 2026
c2a2126
docs: point e2e docs at the prod-fidelity env
ecv Jul 14, 2026
ff1c2cf
ci: nudge e2e workflow run
ecv Jul 14, 2026
14433d3
docs: guide e2e test authoring against the env
ecv Jul 14, 2026
159ccb7
test(infra): port the billing usage collector into the env
ecv Jul 14, 2026
4b0da7a
fix(test-infra): vendor the EG chart before eg-crds
ecv Jul 14, 2026
662244f
test(e2e): run the billing metering test on the shared data plane
ecv Jul 14, 2026
59aabf4
ci: pin helm to v3.12.3 for kustomize helm builds
ecv Jul 14, 2026
3e0392a
test(infra): drop load-fixtures from env bring-up
ecv Jul 14, 2026
08d6223
test(infra): harden wait-ready against cold-start flakes
ecv Jul 14, 2026
7f208a1
ci: give the manager more cold-start budget + dump it on failure
ecv Jul 14, 2026
cd6f654
test(infra): run the e2e suite serially
ecv Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 43 additions & 54 deletions .github/workflows/test-e2e.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ on:
pull_request:
branches:
- main
workflow_dispatch:

jobs:
test-e2e:
Expand All @@ -23,70 +24,58 @@ jobs:
with:
go-version: '~1.26'

- name: Verify kind installation
run: kind version

- name: Create upstream kind cluster
run: make kind-standard-cluster

- name: Create downstream kind cluster
run: |
make kind-infra-cluster
kubectl config use-context kind-nso-standard

- name: Prepare e2e
run: |
kubectl config use-context kind-nso-standard
make prepare-e2e

kubectl config use-context kind-nso-infra
make prepare-infra-cluster
- name: Install Task
uses: arduino/setup-task@v2
with:
version: 3.x
repo-token: ${{ secrets.GITHUB_TOKEN }}

kind get kubeconfig --name nso-infra --internal > ${TMPDIR}/.kind-nso-infra-internal.yaml
- name: Install Helm
uses: azure/setup-helm@v4
with:
# kustomize v5.5.0 probes helm with `helm version -c`; the -c shorthand
# was removed in helm 3.13, so pin to the last release that accepts it.
version: v3.12.3

kubectl config use-context kind-nso-standard
kubectl create secret -n network-services-operator-system \
generic downstream-cluster-kubeconfig \
--from-file=kubeconfig=${TMPDIR}/.kind-nso-infra-internal.yaml
- name: Install pinned tools
run: make kind kustomize cmctl chainsaw

# Wait for the operator to be ready
kubectl -n network-services-operator-system \
wait deploy network-services-operator-controller-manager \
--for=condition=Available \
--timeout=120s || \
kubectl -n cert-manager describe pods && \
kubectl logs -n cert-manager -l app.kubernetes.io/name=cert-manager --tail=-1 && \
kubectl -n network-services-operator-system describe issuer && \
kubectl -n network-services-operator-system describe certificates && \
kubectl -n network-services-operator-system describe secrets && \
kubectl -n network-services-operator-system describe pod && \
kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=-1
- name: Bring up the prod-fidelity env
run: task test-infra:up

- name: Running Test e2e
run: make test-e2e
- name: Run e2e suite
run: task test-infra:test-e2e

- name: Dump downstream WAF data plane on failure
- name: Dump the downstream data plane on failure
if: failure()
run: |
kubectl config use-context kind-nso-infra
echo "::group::GatewayClasses"
kubectl get gatewayclass -o wide || true
kubectl describe gatewayclass datum-downstream-gateway || true
set +e
UKC=${TMPDIR}/.kind-nso-upstream.yaml
bin/kind-v0.32.0 get kubeconfig --name nso-upstream > "${UKC}"
echo "::group::Upstream NSO manager"
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get pods -o wide
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system describe deploy network-services-operator-controller-manager
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system describe pod -l app.kubernetes.io/name=network-services-operator
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=200 --all-containers
echo "::endgroup::"
echo "::group::EnvoyProxy"
kubectl -n datum-downstream-gateway get envoyproxy -o yaml || true
echo "::group::Upstream cert-manager + webhook cert"
KUBECONFIG="${UKC}" kubectl -n cert-manager get pods -o wide
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get certificate,secret
echo "::endgroup::"
echo "::group::datum-downstream-gateway namespace + pods"
kubectl get ns datum-downstream-gateway --show-labels || true
kubectl -n datum-downstream-gateway get pods -o wide || true
KC=${TMPDIR}/.kind-nso-downstream.yaml
bin/kind-v0.32.0 get kubeconfig --name nso-downstream > "${KC}"
export KUBECONFIG="${KC}"
echo "::group::GatewayClasses"
kubectl get gatewayclass -o wide
echo "::endgroup::"
echo "::group::Downstream Envoy Gateway logs"
kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200 || true
echo "::group::Gateways (all namespaces)"
kubectl get gateway -A -o wide
echo "::endgroup::"
echo "::group::Extension server"
kubectl -n datum-downstream-gateway describe deploy envoy-gateway-extension-server || true
kubectl -n datum-downstream-gateway logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200 || true
echo "::group::datum-downstream-gateway pods"
kubectl -n datum-downstream-gateway get pods -o wide
kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200
echo "::endgroup::"
echo "::group::Certificates"
kubectl -n datum-downstream-gateway get certificate,secret || true
echo "::group::Extension server"
kubectl -n network-services-operator-system get pods -o wide
kubectl -n network-services-operator-system logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200
echo "::endgroup::"
47 changes: 31 additions & 16 deletions CLAUDE.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,27 +63,41 @@ Run `make help` for the full list. Most-used targets:
### Tests
- `make test` — unit/integration tests via **envtest** (excludes `test/e2e`);
runs `manifests generate fmt vet` first, writes `cover.out`
- `make test-e2e` — **Chainsaw** e2e suite in `test/e2e` against the two local
Kind clusters (requires `nso-standard` and `nso-infra` to exist)
- `task test-infra:test-e2e` — **Chainsaw** e2e suite in `test/e2e` against the
prod-fidelity env (below); pass a scenario path after `--` for a single test
- `make test-conformance` — Gateway API conformance suite in
`test/conformance/gatewayapi`

### Local Multi-Cluster Dev Environment (Taskfile)
The dev environment is two Kind clusters: an **upstream** cluster
(`nso-standard`) running the operator, and a **downstream** cluster
(`nso-infra`) running Envoy Gateway, cert-manager, and external-dns.

- `task dev:bootstrap` — full setup: create both clusters, deploy the operator
upstream, prepare downstream, and link them via a kubeconfig secret
- `task dev:redeploy-operator` — rebuild the image, load it into `nso-standard`,
and roll the controller-manager
- `task dev:test [-- <path-or-name>]` — run the e2e suite (optionally a single
test under `test/e2e/`)
- `task dev:destroy` — tear down both clusters
### Prod-Fidelity Test Environment (Taskfile)
The e2e environment is two Kind clusters that mirror how the edge runs in
production: an **upstream** cluster (`nso-upstream`) running the operator, and a
**downstream** cluster (`nso-downstream`) running Envoy Gateway v1.7.4, the
extension server + Coraza WAF data plane, cert-manager, and external-dns. It is
defined in `Taskfile.test-infra.yml` and is what CI runs.

- `task test-infra:up` — build the operator image, create both clusters, install
the downstream data plane, deploy + link the operator
- `task test-infra:test-e2e [-- <path>]` — run the e2e suite against the live env
(JSON report written to `$TMPDIR`); optionally a single scenario
- `task test-infra:down` — tear down both clusters
- `task validate-kustomizations` — `kustomize build` every kustomization
- `task test-prometheus-rules` — `promtool` tests for the alerting rules in
`test/prometheus-rules/`

#### Authoring e2e tests against the env
- Bind to the shared data plane the env stands up: downstream GatewayClass
`datum-downstream-gateway-e2e`, Envoy Gateway namespace
`datum-downstream-gateway`, extension server in
`network-services-operator-system`. Do **not** hardcode the bare
`datum-downstream-gateway` class or self-provision a per-test EnvoyProxy in
`envoy-gateway-system` — that was the retired single-stack bridge's model.
- Every scenario under `test/e2e/` runs against this env in CI. Any setup a
scenario needs (a data-plane component, a namespace) must be installed by
`task test-infra:up` (add it to `Taskfile.test-infra.yml`), not assumed from
outside — a scenario whose dependency isn't provisioned by the env will fail.
- Editing `.github/workflows/*` requires pushing over SSH; the HTTPS token used
by `gh` lacks the `workflow` scope.

## High-Level Architecture

### Multi-Cluster (upstream / downstream)
Expand Down Expand Up @@ -139,7 +153,7 @@ project follows the Kubebuilder v4 layout (`PROJECT`, domain `datumapis.com`).
2. Run `make manifests generate` to regenerate CRDs, RBAC, deepcopy, defaulters.
3. Run `make api-docs` if the CRD reference under `docs/api/` should update.
4. Add/extend reconciler logic in `internal/controller/` and a `_test.go`.
5. `make test` (envtest) and, when relevant, `make test-e2e`.
5. `make test` (envtest) and, when relevant, `task test-infra:test-e2e`.

Never hand-edit generated files (`zz_generated.*`, `config/crd/bases/*`,
`docs/api/*`) — change the source and regenerate.
Expand All @@ -154,7 +168,8 @@ Operator deployment config is composed from Kustomize bases and overlays under

### CI
GitHub Actions enforce the same commands locally available:
`make test` (test.yml), `make test-e2e` (test-e2e.yml), `golangci-lint`
`make test` (test.yml), the `test-infra` e2e env (test-e2e.yml, via
`task test-infra:up` + `task test-infra:test-e2e`), `golangci-lint`
(lint.yml), and kustomize validation (validate-kustomize.yaml). Run them
locally before pushing.

Expand Down
91 changes: 4 additions & 87 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,27 +64,9 @@ vet: ## Run go vet against code.
test: manifests generate fmt vet envtest ## Run tests.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out

# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
# Prometheus and CertManager are installed by default; skip with:
# - PROMETHEUS_INSTALL_SKIP=true
# - CERT_MANAGER_INSTALL_SKIP=true
.PHONY: test-e2e
test-e2e: chainsaw
@command -v kind >/dev/null 2>&1 || { \
echo "Kind is not installed. Please install Kind manually."; \
exit 1; \
}
@kind get clusters | grep -q 'nso-standard' || { \
echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \
exit 1; \
}
$(KIND) get kubeconfig --name nso-standard > $(TMPDIR)/.kind-nso-standard.yaml
$(KIND) get kubeconfig --name nso-infra > $(TMPDIR)/.kind-nso-infra.yaml
$(CHAINSAW) test $(or $(TEST_DIR),./test/e2e) \
--parallel 1 \
--cluster nso-standard=$(TMPDIR)/.kind-nso-standard.yaml \
--cluster nso-infra=$(TMPDIR)/.kind-nso-infra.yaml
# The e2e suite runs against the two-cluster prod-fidelity env; bring it up and
# run it with `task test-infra:up` then `task test-infra:test-e2e`
# (Taskfile.test-infra.yml).

GATEWAY_CONFORMANCE_CLASS ?= gateway-conformance
GATEWAY_CONFORMANCE_FLAGS ?=
Expand Down Expand Up @@ -153,76 +135,14 @@ build-installer: set-image-controller generate ## Generate a consolidated YAML w
set-image-controller: manifests kustomize
cd config/manager && $(KUSTOMIZE) edit set image ghcr.io/datum-cloud/network-services-operator=${IMG}

.PHONY: prepare-infra-cluster
prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds downstream-waf-dataplane billing-usage-collector load-image-infra extension-server configure-eg-extension-manager

.PHONY: downstream-crds
downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the replicator mirrors into it.
$(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_connectors.yaml
$(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml
$(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml

.PHONY: downstream-waf-dataplane
downstream-waf-dataplane: kustomize load-image-infra ## Install the Coraza WAF data plane (downstream EG instance + extension server + datum-downstream-gateway GatewayClass) on the infra cluster.
$(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway-downstream | kubectl apply --server-side=true --force-conflicts -f -
$(KUSTOMIZE) build config/e2e-downstream | kubectl apply --server-side=true --force-conflicts -f -

.PHONY: prepare-e2e
prepare-e2e: chainsaw set-image-controller cert-manager load-image-all deploy-e2e

.PHONY: prepare-dev
prepare-dev: chainsaw set-image-controller cert-manager install

.PHONY: load-image-all
load-image-all: load-image-operator load-image-infra

.PHONY: load-image-operator
load-image-operator: docker-build kind
$(KIND) load docker-image $(IMG) -n nso-standard

.PHONY: load-image-infra
load-image-infra: docker-build kind ## Load operator image into nso-infra kind cluster (needed by the extension server).
$(KIND) load docker-image $(IMG) -n nso-infra

.PHONY: cert-manager
cert-manager: cmctl
$(KUSTOMIZE) build --enable-helm config/tools/cert-manager | kubectl apply --server-side=true --force-conflicts -f -
$(CMCTL) check api --wait=5m

.PHONY: envoy-gateway
envoy-gateway:
$(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway | kubectl apply --server-side=true --force-conflicts -f -

.PHONY: external-dns
external-dns:
$(KUSTOMIZE) build --enable-helm config/tools/external-dns | kubectl apply --server-side=true --force-conflicts -f -

.PHONY: billing-usage-collector
billing-usage-collector:
$(KUSTOMIZE) build --enable-helm config/tools/billing-usage-collector | kubectl apply --server-side=true --force-conflicts -f -

.PHONY: extension-server
extension-server: ## Deploy the NSO extension server to the infra cluster (e2e overlay with cert-manager-issued TLS).
$(KUSTOMIZE) build --enable-helm config/extension-server-e2e | kubectl apply --server-side=true --force-conflicts -f -
kubectl rollout restart deployment/network-services-operator-envoy-gateway-extension-server \
-n network-services-operator-system
kubectl rollout status deployment/network-services-operator-envoy-gateway-extension-server \
-n network-services-operator-system --timeout=5m

.PHONY: configure-eg-extension-manager
configure-eg-extension-manager: ## Patch the EG ConfigMap to enable extensionManager and restart the EG controller.
$(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway/overlays/e2e | kubectl apply --server-side=true --force-conflicts -f -
kubectl rollout restart deployment/envoy-gateway -n envoy-gateway-system
kubectl rollout status deployment/envoy-gateway -n envoy-gateway-system --timeout=3m

.PHONY: kind-standard-cluster
kind-standard-cluster: kind
$(KIND) create cluster --config=config/tools/kind/standard-cluster.yaml

.PHONY: kind-infra-cluster
kind-infra-cluster: kind
$(KIND) create cluster --config=config/tools/kind/infra-cluster.yaml

##@ Deployment

ifndef ignore-not-found
Expand All @@ -241,9 +161,6 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
deploy: set-image-controller ## Deploy controller to the K8s cluster specified in ~/.kube/config.
$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -

.PHONY: deploy-e2e
deploy-e2e: set-image-controller
$(KUSTOMIZE) build config/e2e | $(KUBECTL) apply -f -

.PHONY: undeploy
undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
Expand Down Expand Up @@ -284,7 +201,7 @@ CERTMANAGER_VERSION ?= 1.17.1
CRDOC_VERSION ?= v0.6.4

# renovate: datasource=go depName=sigs.k8s.io/kind
KIND_VERSION ?= v0.27.0
KIND_VERSION ?= v0.32.0

# renovate: datasource=go depName=github.com/kyverno/chainsaw
CHAINSAW_VERSION ?= v0.2.15
Expand Down
Loading
Loading