diff --git a/.github/workflows/test-e2e-federated.yml b/.github/workflows/test-e2e-federated.yml new file mode 100644 index 00000000..24f319a4 --- /dev/null +++ b/.github/workflows/test-e2e-federated.yml @@ -0,0 +1,136 @@ +name: Federated E2E + +# Exercises the production-fidelity edge: two kind clusters (control plane + edge) +# wired with real Karmada federation, the production Envoy Gateway version, the +# Coraza WAF data plane, and the NSO extension server — then runs the edge e2e +# scenarios against real traffic. This is the path the non-federated test/e2e +# suite (test-e2e.yml) never covers. +on: + pull_request: + branches: + - main + paths: + - 'Taskfile.test-infra.yml' + - 'config/e2e-downstream/**' + - 'config/federation/**' + - 'config/tools/**' + - 'config/extension-server/**' + - 'internal/extensionserver/**' + - 'cmd/**' + - 'test/e2e-edge/**' + - 'test/parity/**' + - '.github/workflows/test-e2e-federated.yml' + push: + branches: + - main + workflow_dispatch: + +# One federated run per ref; a newer push cancels an in-flight one. +concurrency: + group: federated-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + federated-e2e: + name: Two-cluster federated edge (Karmada) + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + TMPDIR: /tmp + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '~1.26' + + # Two kind clusters plus a full Karmada control plane and the edge data + # plane need more room than the runner ships with. Reclaim disk from + # toolchains this job never uses. + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL + docker system prune -af || true + df -h / + + # Karmada plus two clusters open a large number of file watches; the + # default runner limits crashloop a component with "too many open files". + - name: Raise inotify limits + run: | + sudo sysctl -w fs.inotify.max_user_instances=8192 + sudo sysctl -w fs.inotify.max_user_watches=1048576 + + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install Helm + uses: azure/setup-helm@v4 + with: + # kustomize v5.5.0 probes helm with `helm version -c`; the -c shorthand + # was removed in helm 3.13, so pin to the last release that accepts it. + version: v3.12.3 + + - name: Install pinned tools (kind, karmadactl, kustomize, cmctl, chainsaw) + run: task test-infra:tools + + - name: Bring up the two-cluster prod-fidelity env + run: task test-infra:up + + - name: Stand up Karmada federation + run: task test-infra:karmada-up + + - name: Run edge e2e scenarios + run: task test-infra:e2e + + # When something fails, capture enough cross-cluster state to diagnose it + # from the logs alone — the runner is torn down immediately after. + - name: Diagnostics on failure + if: failure() + run: | + set +e + echo "::group::upstream pods" + kubectl --context kind-nso-upstream get pods -A -o wide + echo "::endgroup::" + echo "::group::downstream pods" + kubectl --context kind-nso-downstream get pods -A -o wide + echo "::endgroup::" + echo "::group::not-ready pods — describe (both clusters)" + for ctx in kind-nso-upstream kind-nso-downstream; do + echo "### $ctx" + kubectl --context "$ctx" get pods -A \ + --field-selector=status.phase!=Running,status.phase!=Succeeded \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' \ + | while read -r ns name; do + [ -n "$name" ] || continue + kubectl --context "$ctx" -n "$ns" describe pod "$name" 2>/dev/null | tail -30 + done + done + echo "::endgroup::" + echo "::group::cert-manager health (upstream) — restarts + issuance" + kubectl --context kind-nso-upstream -n cert-manager get pods -o wide + kubectl --context kind-nso-upstream -n cert-manager \ + logs -l app.kubernetes.io/name=cert-manager --tail=80 --previous 2>/dev/null || true + kubectl --context kind-nso-upstream get certificate,certificaterequest,issuer,clusterissuer -A + echo "::endgroup::" + echo "::group::karmada member clusters" + KCFG=/tmp/karmada-nso-upstream/karmada-apiserver.config + kubectl --kubeconfig "$KCFG" --context karmada-apiserver get clusters -o wide + kubectl --kubeconfig "$KCFG" --context karmada-apiserver get clusterpropagationpolicy,work -A + echo "::endgroup::" + echo "::group::NSO manager logs (upstream)" + kubectl --context kind-nso-upstream -n network-services-operator-system \ + logs deploy/network-services-operator-controller-manager --tail=200 + echo "::endgroup::" + echo "::group::extension server logs (downstream)" + kubectl --context kind-nso-downstream -n network-services-operator-system \ + logs deploy/network-services-operator-envoy-gateway-extension-server --tail=200 + echo "::endgroup::" + echo "::group::downstream Envoy Gateway logs" + kubectl --context kind-nso-downstream -n datum-downstream-gateway \ + logs deploy/envoy-gateway --tail=200 + echo "::endgroup::" diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 78b070f5..5c4a4c16 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -56,6 +56,35 @@ vars: tasks: + tools: + desc: "Install the pinned tool binaries this env needs into bin/ (kind v0.32.0, karmadactl, kustomize, cmctl, chainsaw). Idempotent — skips any already present. Lets CI and a clean checkout bring up the env without hand-placed binaries." + vars: + KIND_TOOL_VERSION: v0.32.0 + KARMADACTL_VERSION: v1.15.2 + KUSTOMIZE_VERSION: v5.5.0 + CMCTL_VERSION: v2.1.1 + CHAINSAW_VERSION: v0.2.15 + cmds: + - mkdir -p {{.REPO}}/bin + - | + if [ ! -x {{.REPO}}/bin/kind-{{.KIND_TOOL_VERSION}} ]; then + echo "installing kind {{.KIND_TOOL_VERSION}}" + GOBIN={{.REPO}}/bin go install sigs.k8s.io/kind@{{.KIND_TOOL_VERSION}} + mv {{.REPO}}/bin/kind {{.REPO}}/bin/kind-{{.KIND_TOOL_VERSION}} + fi + - '[ -x {{.REPO}}/bin/kustomize ] || GOBIN={{.REPO}}/bin go install sigs.k8s.io/kustomize/kustomize/v5@{{.KUSTOMIZE_VERSION}}' + - '[ -x {{.REPO}}/bin/cmctl ] || GOBIN={{.REPO}}/bin go install github.com/cert-manager/cmctl/v2@{{.CMCTL_VERSION}}' + - '[ -x {{.REPO}}/bin/chainsaw ] || GOBIN={{.REPO}}/bin go install github.com/kyverno/chainsaw@{{.CHAINSAW_VERSION}}' + # karmadactl carries replace directives in its go.mod, so `go install` can't + # build it; pull the published release binary for this OS/arch instead. + - | + if [ ! -x {{.REPO}}/bin/karmadactl ]; then + url="https://github.com/karmada-io/karmada/releases/download/{{.KARMADACTL_VERSION}}/karmadactl-{{OS}}-{{ARCH}}.tgz" + echo "downloading karmadactl {{.KARMADACTL_VERSION}} ({{OS}}/{{ARCH}})" + curl -fsSL "$url" | tar -xz -C {{.REPO}}/bin karmadactl + fi + - echo "tools ready in {{.REPO}}/bin" + up: desc: "Bring the canonical two-cluster prod-fidelity test env ONLINE (clusters + EG v1.7.4 + ext-server + Coraza data plane + NSO manager)." cmds: