From 1fcd122601967872f720e2221f7242af8e435c68 Mon Sep 17 00:00:00 2001 From: Tejas Soham Date: Mon, 12 Oct 2026 21:25:13 +0530 Subject: [PATCH] SECURESIGN-5151: strengthen MVO test coverage and CI gates Signed-off-by: Tejas Soham --- .github/workflows/lint.yml | 82 ++- .github/workflows/test-e2e.yml | 94 +++- .gitignore | 2 + .golangci.yml | 2 +- Dockerfile.coverage | 29 ++ Makefile | 71 ++- ci/kind-ha-config.yaml | 6 + cmd/coverage.go | 5 + codecov.yml | 22 +- config/overlays/production/kustomization.yaml | 1 + .../production/poddisruptionbudget.yaml | 14 + .../production/production-config.yaml | 14 + crdify.yaml | 10 + go.mod | 5 +- go.sum | 2 + internal/constants/images.go | 3 +- internal/webhooks/pod_webhook.go | 15 + internal/webhooks/pod_webhook_cleanup_test.go | 135 +++++ scripts/e2e-coverage.sh | 154 ++++++ scripts/test-e2e-coverage.sh | 89 ++++ scripts/testdata/e2e-coverage/curl | 13 + scripts/testdata/e2e-coverage/kubectl | 23 + scripts/testdata/e2e-coverage/main.go | 18 + test/e2e-olm/config.go | 102 ++++ test/e2e-olm/config_test.go | 122 +++++ test/e2e-olm/helpers_test.go | 218 ++++++++ test/e2e-olm/helpers_unit_test.go | 228 +++++++++ test/e2e-olm/olm_upgrade_suite_test.go | 94 ++-- test/e2e-olm/olm_upgrade_test.go | 216 +++++--- test/e2e-olm/testdata.go | 2 + .../testdata/catalogsource_template.yaml | 2 + test/e2e-olm/testdata/pod_template.yaml | 8 +- .../testdata/subscription_template.yaml | 3 + test/e2e/auth_validation_helpers_test.go | 78 +++ test/e2e/continuous_validation_test.go | 87 ++++ test/e2e/e2e_suite_test.go | 54 +- test/e2e/e2e_test.go | 10 +- test/e2e/ha_test.go | 466 ++++++++++++++++++ test/e2e/integration_test.go | 2 + test/e2e/metrics_template_test.go | 99 ++++ test/e2e/metrics_test.go | 2 + test/e2e/negative_test.go | 340 +++++++++++++ test/e2e/pki_validation_test.go | 56 +++ test/e2e/sigstore_validation_test.go | 60 +++ test/e2e/status_tracking_test.go | 2 + test/e2e/testdata.go | 7 +- .../testdata/curl_metrics_pod_template.yaml | 8 +- test/e2e/testdata/model-data-daemonset.yaml | 23 +- test/e2e/testdata/pod_template.yaml | 3 +- test/e2e/validation_test.go | 2 + test/utils/utils.go | 70 ++- test/utils/utils_test.go | 202 ++++++++ testdata/docker/test-model.Dockerfile | 24 +- 53 files changed, 3204 insertions(+), 195 deletions(-) create mode 100644 Dockerfile.coverage create mode 100644 ci/kind-ha-config.yaml create mode 100644 cmd/coverage.go create mode 100644 config/overlays/production/poddisruptionbudget.yaml create mode 100644 crdify.yaml create mode 100644 internal/webhooks/pod_webhook_cleanup_test.go create mode 100755 scripts/e2e-coverage.sh create mode 100755 scripts/test-e2e-coverage.sh create mode 100755 scripts/testdata/e2e-coverage/curl create mode 100755 scripts/testdata/e2e-coverage/kubectl create mode 100644 scripts/testdata/e2e-coverage/main.go create mode 100644 test/e2e-olm/config.go create mode 100644 test/e2e-olm/config_test.go create mode 100644 test/e2e-olm/helpers_test.go create mode 100644 test/e2e-olm/helpers_unit_test.go create mode 100644 test/e2e/auth_validation_helpers_test.go create mode 100644 test/e2e/continuous_validation_test.go create mode 100644 test/e2e/ha_test.go create mode 100644 test/e2e/metrics_template_test.go create mode 100644 test/e2e/negative_test.go create mode 100644 test/e2e/pki_validation_test.go create mode 100644 test/e2e/sigstore_validation_test.go create mode 100644 test/utils/utils_test.go diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cb4deee1..b67180df 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,12 +11,18 @@ permissions: jobs: lint: - name: Run Linting + name: Lint (${{ matrix.tags || 'default' }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + tags: ["", "integration", "ha", "upgrade", "coverage"] steps: - name: Checkout code uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6 + with: + fetch-depth: 0 - name: Setup Go uses: actions/setup-go@def8c394e3ad351a79bc93815e4a585520fe993b # v5 @@ -28,4 +34,76 @@ jobs: uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0 with: version: v2.4.0 - args: --timeout=5m --issues-exit-code=0 --verbose + # Build with the go.mod toolchain, not a release binary built with an older Go. + install-mode: goinstall + args: --timeout=5m --issues-exit-code=1 --verbose --build-tags=${{ matrix.tags }} + + - name: Test OLM configuration and lifecycle helpers without a cluster + if: matrix.tags == 'upgrade' + run: >- + go test -race -count=1 -tags=upgrade ./test/e2e-olm/ + -run '^(TestUpgradeConfig.*|TestControllerRollout|TestCatalogTemplate|TestExistingCSVs|TestCleanup.*|TestSubscriptionCSVNames)$' + + - name: Test metrics Pod security template without a cluster + if: matrix.tags == 'integration' + run: go test -race -count=1 -tags=integration ./test/e2e/ -run '^TestMetricsPodSecurityTemplate$' + + - name: Test coverage collector failure handling and process restart retention + if: matrix.tags == '' + run: make test-coverage-collector + + crd-compatibility: + name: CRD generation and backward compatibility + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@def8c394e3ad351a79bc93815e4a585520fe993b # v5 + with: + go-version-file: go.mod + cache: true + + - name: Generate and verify committed manifests are current + run: | + make manifests generate + git diff --exit-code -- config/crd/bases config/rbac config/webhook api + untracked=$(git ls-files --others --exclude-standard -- config/crd/bases config/rbac config/webhook api) + if [[ -n "$untracked" ]]; then + echo "Generated artifacts are missing from Git:" + printf '%s\n' "$untracked" + exit 1 + fi + + - name: Install crdify + run: go install sigs.k8s.io/crdify@v0.6.0 + + - name: Check CRD backward compatibility + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + if [[ -z "$BASE_SHA" || "$BASE_SHA" =~ ^0+$ ]]; then + echo "No previous revision is available; skipping CRD comparison" + exit 0 + fi + git show "$BASE_SHA:config/crd/bases/ml.sigstore.dev_modelvalidations.yaml" > /tmp/base-crd.yaml + crdify --config crdify.yaml \ + file:///tmp/base-crd.yaml \ + file://config/crd/bases/ml.sigstore.dev_modelvalidations.yaml + + lint-gate: + # Preserve the existing required-check name while requiring every partition + # and the new generated-schema/compatibility gate to pass. + name: Run Linting + if: ${{ always() }} + needs: [lint, crd-compatibility] + runs-on: ubuntu-latest + steps: + - name: Require all lint and CRD checks to pass + env: + LINT_RESULT: ${{ needs.lint.result }} + CRD_RESULT: ${{ needs.crd-compatibility.result }} + run: test "$LINT_RESULT" = success && test "$CRD_RESULT" = success diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index c77535b9..9dfc1a43 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -11,11 +11,15 @@ env: permissions: contents: read + id-token: write jobs: test-e2e: name: Run E2E Tests runs-on: ubuntu-latest + env: + CONTAINER_FILE: Dockerfile.coverage + E2E_COLLECT_COVERAGE: "true" steps: - name: Checkout code uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6 @@ -44,6 +48,91 @@ jobs: run: | make test-e2e-ci + - name: Collect E2E coverage + if: ${{ !cancelled() }} + run: make collect-e2e-coverage + + - name: Upload E2E coverage to Codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + with: + files: e2e-cover.out + flags: e2e + use_oidc: true + fail_ci_if_error: true + + - name: Collect cluster diagnostics + if: ${{ failure() }} + run: | + mkdir -p e2e-diagnostics + kubectl get pods,events,deployments,endpointslices -n model-validation-operator-system -o yaml > e2e-diagnostics/operator.yaml || true + kubectl get pods,events,modelvalidations -n e2e-webhook-test-ns -o yaml > e2e-diagnostics/tests.yaml || true + kubectl logs -n model-validation-operator-system -l control-plane=controller-manager --all-containers --prefix --tail=1000 > e2e-diagnostics/operator.log || true + + - name: Upload E2E diagnostics and coverage snapshots + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: integration-evidence + path: | + e2e-diagnostics/ + e2e-coverdata/ + e2e-cover.out + if-no-files-found: warn + include-hidden-files: true + retention-days: 7 + + test-e2e-ha: + name: Run HA E2E Tests + runs-on: ubuntu-latest + env: + E2E_ALLOW_NODE_DRAIN: "true" + E2E_DRAIN_NODE_SELECTOR: mvo-e2e-drain=true + steps: + - name: Checkout code + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6 + + - name: Setup Go + uses: actions/setup-go@def8c394e3ad351a79bc93815e4a585520fe993b # v5 + with: + go-version-file: go.mod + cache: true + + - name: Create multi-node kind cluster + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1 + with: + cluster_name: kind + config: ./ci/kind-ha-config.yaml + + - name: Wait for kind cluster to be ready + run: | + kubectl wait --for=condition=Ready nodes --all --timeout=300s + kubectl wait --for=condition=Ready --namespace=kube-system pod --all --timeout=300s + kubectl label nodes -l '!node-role.kubernetes.io/control-plane' mvo-e2e-drain=true + + - name: Swap to dev images + run: make dev-images + + - name: Run HA E2E tests + run: make test-e2e-ha-ci + + - name: Collect HA diagnostics + if: ${{ failure() }} + run: | + mkdir -p e2e-diagnostics + kubectl get nodes -o yaml > e2e-diagnostics/nodes.yaml || true + kubectl get pods,events,deployments,poddisruptionbudgets,leases -n model-validation-operator-system -o yaml > e2e-diagnostics/operator.yaml || true + kubectl get pods,events,modelvalidations -n e2e-webhook-test-ns -o yaml > e2e-diagnostics/tests.yaml || true + kubectl logs -n model-validation-operator-system -l control-plane=controller-manager --all-containers --prefix --tail=1000 > e2e-diagnostics/operator.log || true + + - name: Upload HA diagnostics + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ha-evidence + path: e2e-diagnostics/ + if-no-files-found: warn + retention-days: 7 + test-e2e-olm-upgrade: name: Test OLM upgrade if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository @@ -176,5 +265,8 @@ jobs: env: TEST_BASE_CATALOG: registry.redhat.io/redhat/redhat-operator-index:${{ vars.OLM_INDEX_VERSION }} TEST_TARGET_CATALOG: ${{ env.CATALOG_IMG }} + # CI creates a next-patch upgrade candidate, not an approved GA release. + TEST_EXPECTED_TARGET_CSV: model-validation-operator.v${{ env.OLM_UPGRADE_VERSION }} + TEST_EXPECTED_TARGET_IMAGE: ${{ env.IMG }} run: | - go test ./test/e2e-olm/ -v -ginkgo.v -timeout 20m + go test -tags=upgrade ./test/e2e-olm/ -v -ginkgo.v -timeout 20m diff --git a/.gitignore b/.gitignore index bff59b62..d7e524ee 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ /bin cover.out +e2e-cover.out +e2e-coverdata/ # editor and IDE paraphernalia .idea .vscode diff --git a/.golangci.yml b/.golangci.yml index ca148978..7ec90f67 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -27,7 +27,7 @@ linters: rules: - linters: - lll - source: "^// \\+kubebuilder:validation" + source: "^\\s*// \\+kubebuilder:validation" formatters: enable: diff --git a/Dockerfile.coverage b/Dockerfile.coverage new file mode 100644 index 00000000..da1fcd3d --- /dev/null +++ b/Dockerfile.coverage @@ -0,0 +1,29 @@ +# Copyright 2026 The Sigstore Authors. +# Licensed under the Apache License, Version 2.0 (the "License"); + +FROM golang:1.26 AS builder +ARG TARGETOS +ARG TARGETARCH +ARG AGENT_IMG + +WORKDIR /workspace +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd/ cmd/ +COPY api/ api/ +COPY internal/ internal/ + +# Atomic mode is required by runtime/coverage.WriteCounters. The coverage tag +# embeds Coverport's HTTP collector on port 53700. +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ + -cover -covermode=atomic -tags=coverage -a \ + -ldflags="-X github.com/sigstore/model-validation-operator/internal/constants.ModelValidationAgentImage=${AGENT_IMG}" \ + -o manager ./cmd + +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index a13f19c9..a9a8f0b2 100644 --- a/Makefile +++ b/Makefile @@ -201,6 +201,10 @@ docker-build: test ## Build docker image with the manager. docker-push: ## Push docker image with the manager. $(CONTAINER_TOOL) push ${IMG} +.PHONY: docker-build-coverage +docker-build-coverage: ## Build a coverage-instrumented manager image for E2E tests. + $(CONTAINER_TOOL) build -t ${IMG} --build-arg AGENT_IMG=${AGENT_IMG} -f Dockerfile.coverage . + # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: # - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ @@ -258,7 +262,6 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/overlays/production && $(KUSTOMIZE) edit set image controller=${IMG} - cd config/overlays/production && $(KUSTOMIZE) edit set replicas controller-manager=1 $(KUSTOMIZE) build config/overlays/production | $(KUBECTL) apply -f - .PHONY: undeploy @@ -500,7 +503,7 @@ e2e-sign-test-model: e2e-generate-test-keys .PHONY: e2e-build-test-model e2e-build-test-model: e2e-sign-test-model @echo "Building test model image..." - cd testdata && $(CONTAINER_TOOL) build --no-cache -t $(E2E_TEST_MODEL) -f docker/test-model.Dockerfile . + $(CONTAINER_TOOL) build --no-cache -t $(E2E_TEST_MODEL) -f testdata/docker/test-model.Dockerfile . # install and uninstall cert-manager for tests @@ -597,25 +600,55 @@ e2e-teardown: e2e-cleanup-resources undeploy e2e-uninstall-certmanager .PHONY: test-e2e test-e2e: manifests generate fmt vet ## Run the e2e tests, no setup and teardown. Expects the operator to be deployed. @echo "Running e2e tests (assumes infrastructure is already set up)..." - go test ./test/e2e/ -v -ginkgo.v + go test -tags=integration ./test/e2e/ -v -ginkgo.v -timeout 30m + +.PHONY: test-e2e-integration +test-e2e-integration: test-e2e ## Run the integration-tagged E2E partition. + +.PHONY: test-e2e-ha +test-e2e-ha: manifests generate fmt vet ## Run disruptive HA tests against an existing multi-node cluster. + go test -tags=ha ./test/e2e/ -v -ginkgo.v -timeout 30m .PHONY: test-e2e-full test-e2e-full: manifests generate fmt vet e2e-setup ## Run e2e tests with setup and teardown @echo "Running e2e tests with full infrastructure setup..." - go test ./test/e2e/ -v -ginkgo.v; \ - TEST_RESULT=$$?; \ - $(MAKE) e2e-teardown; \ - exit $$TEST_RESULT + @result=0; \ + go test -tags=integration ./test/e2e/ -v -ginkgo.v -timeout 30m || result=$$?; \ + $(MAKE) e2e-teardown || { cleanup_result=$$?; if [ $$result -eq 0 ]; then result=$$cleanup_result; fi; }; \ + exit $$result .PHONY: test-e2e-ci test-e2e-ci: manifests generate fmt vet e2e-setup ## Run the e2e tests, with setup. No teardown as the CI workflow will throw away kind @echo "Running e2e tests with infrastructure setup for CI..." - go test ./test/e2e/ -v -ginkgo.v + @if [ "$${E2E_COLLECT_COVERAGE:-false}" = "true" ]; then $(MAKE) init-e2e-coverage; fi + go test -tags=integration ./test/e2e/ -v -ginkgo.v -timeout 30m + +.PHONY: test-e2e-ha-ci +test-e2e-ha-ci: manifests generate fmt vet e2e-setup ## Set up and run HA tests in CI. + go test -tags=ha ./test/e2e/ -v -ginkgo.v -timeout 30m + +.PHONY: init-e2e-coverage +init-e2e-coverage: ## Start a fresh coverage session, retaining old sessions for diagnosis. + bash scripts/e2e-coverage.sh init + +.PHONY: snapshot-e2e-coverage +snapshot-e2e-coverage: ## Save cumulative coverage before restarting or terminating managers. + KUBECTL="$(KUBECTL)" E2E_OPERATOR_NAMESPACE="$(E2E_OPERATOR_NAMESPACE)" bash scripts/e2e-coverage.sh snapshot + +.PHONY: collect-e2e-coverage +collect-e2e-coverage: ## Snapshot current managers and merge them with all pre-disruption processes. + KUBECTL="$(KUBECTL)" E2E_OPERATOR_NAMESPACE="$(E2E_OPERATOR_NAMESPACE)" bash scripts/e2e-coverage.sh collect + +.PHONY: test-coverage-collector +test-coverage-collector: ## Test coverage collection with real Go counter data and mocked Kubernetes. + bash scripts/test-e2e-coverage.sh ##@ OLM E2E Test Infrastructure # OLM E2E Variables -OLM_NAMESPACE ?= $(shell $(KUBECTL) get ns openshift-operator-lifecycle-manager --no-headers >/dev/null 2>&1 && echo openshift-operator-lifecycle-manager || echo olm) +# Catalogs outside the marketplace namespace are local to their Subscription. +# The OLM controller namespace is unrelated to CatalogSource visibility. +OLM_CATALOG_NAMESPACE ?= $(E2E_OPERATOR_NAMESPACE) OLM_CHANNEL ?= tech-preview OLM_PACKAGE ?= model-validation-operator @@ -631,15 +664,21 @@ e2e-uninstall-olm: operator-sdk ## Uninstall OLM from the cluster .PHONY: e2e-olm-teardown e2e-olm-teardown: ## Tear down OLM e2e test environment @echo "Tearing down OLM e2e test environment..." - -$(KUBECTL) delete subscription --all -n $(E2E_OPERATOR_NAMESPACE) --timeout=30s - -$(KUBECTL) delete csv --all -n $(E2E_OPERATOR_NAMESPACE) --timeout=60s - -$(KUBECTL) delete operatorgroup --all -n $(E2E_OPERATOR_NAMESPACE) --timeout=30s - -$(KUBECTL) delete catalogsource model-validation-test-catalog -n $(OLM_NAMESPACE) --timeout=30s --ignore-not-found=true - -$(KUBECTL) delete pods --all -n $(E2E_TEST_NAMESPACE) --timeout=30s - -$(KUBECTL) delete modelvalidations --all -n $(E2E_TEST_NAMESPACE) --timeout=30s + $(KUBECTL) delete pods olm-test-pod-initial olm-test-pod-upgrade -n $(E2E_TEST_NAMESPACE) --ignore-not-found --timeout=30s + $(KUBECTL) delete modelvalidation olm-upgrade-test-model -n $(E2E_TEST_NAMESPACE) --ignore-not-found --timeout=30s + @csv_names=$$($(KUBECTL) get subscription model-validation-operator -n $(E2E_OPERATOR_NAMESPACE) --ignore-not-found \ + -o jsonpath='{.status.installedCSV}{" "}{.status.currentCSV}'); \ + $(KUBECTL) delete subscription model-validation-operator -n $(E2E_OPERATOR_NAMESPACE) --ignore-not-found --timeout=30s; \ + for csv in $$csv_names; do \ + $(KUBECTL) delete csv "$$csv" -n $(E2E_OPERATOR_NAMESPACE) --ignore-not-found --timeout=60s; \ + done + $(KUBECTL) delete operatorgroup model-validation-og -n $(E2E_OPERATOR_NAMESPACE) --ignore-not-found --timeout=30s + $(KUBECTL) delete catalogsource model-validation-test-catalog -n $(OLM_CATALOG_NAMESPACE) --ignore-not-found --timeout=30s .PHONY: test-e2e-olm test-e2e-olm: manifests generate fmt vet ## Run OLM upgrade e2e tests @echo "Running OLM upgrade e2e tests..." - go test ./test/e2e-olm/ -v -ginkgo.v -timeout 20m + go test -tags=upgrade ./test/e2e-olm/ -v -ginkgo.v -timeout 20m +.PHONY: test-e2e-upgrade +test-e2e-upgrade: test-e2e-olm ## Run the upgrade-tagged E2E partition. diff --git a/ci/kind-ha-config.yaml b/ci/kind-ha-config.yaml new file mode 100644 index 00000000..752e993c --- /dev/null +++ b/ci/kind-ha-config.yaml @@ -0,0 +1,6 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker +- role: worker diff --git a/cmd/coverage.go b/cmd/coverage.go new file mode 100644 index 00000000..e123097c --- /dev/null +++ b/cmd/coverage.go @@ -0,0 +1,5 @@ +//go:build coverage + +package main + +import _ "github.com/konflux-ci/coverport/instrumentation/go" diff --git a/codecov.yml b/codecov.yml index d85a2ecd..a80cbade 100644 --- a/codecov.yml +++ b/codecov.yml @@ -2,13 +2,27 @@ coverage: status: project: default: - target: auto - informational: true + target: 35% + threshold: 0% + informational: false patch: default: target: 70% - threshold: 5% - informational: true + threshold: 0% + informational: false + +flags: + unit: + carryforward: true + paths: + - api/ + - cmd/ + - internal/ + e2e: + carryforward: true + paths: + - cmd/ + - internal/ flag_management: default_rules: diff --git a/config/overlays/production/kustomization.yaml b/config/overlays/production/kustomization.yaml index 30cd6d63..7f68b773 100644 --- a/config/overlays/production/kustomization.yaml +++ b/config/overlays/production/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - ../../default +- poddisruptionbudget.yaml components: - ../../components/webhook diff --git a/config/overlays/production/poddisruptionbudget.yaml b/config/overlays/production/poddisruptionbudget.yaml new file mode 100644 index 00000000..23c0750f --- /dev/null +++ b/config/overlays/production/poddisruptionbudget.yaml @@ -0,0 +1,14 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: controller-manager + namespace: system + labels: + app.kubernetes.io/name: model-validation-operator + app.kubernetes.io/managed-by: kustomize +spec: + minAvailable: 1 + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: model-validation-operator diff --git a/config/overlays/production/production-config.yaml b/config/overlays/production/production-config.yaml index 39b44960..fbdd4715 100644 --- a/config/overlays/production/production-config.yaml +++ b/config/overlays/production/production-config.yaml @@ -6,6 +6,20 @@ metadata: spec: template: spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + # Compare replicas from the same rollout revision. Counting old + # replicas can place both replacements on the other worker. + matchLabelKeys: + - pod-template-hash + labelSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: model-validation-operator containers: - name: manager resources: diff --git a/crdify.yaml b/crdify.yaml new file mode 100644 index 00000000..3933cb99 --- /dev/null +++ b/crdify.yaml @@ -0,0 +1,10 @@ +# Match OLM v1's fail-closed CRD upgrade safety checks. +conversion: Ignore +unhandledEnforcement: Error +validations: + - name: description + enforcement: Warn + - name: enum + enforcement: Error + configuration: + additionPolicy: Allow diff --git a/go.mod b/go.mod index a1d532c1..18241f18 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ godebug default=go1.25 require ( github.com/go-logr/logr v1.4.3 + github.com/konflux-ci/coverport/instrumentation/go v0.0.0-20260827155954-c914ae07e9f0 github.com/onsi/ginkgo/v2 v2.28.1 github.com/onsi/gomega v1.39.1 github.com/prometheus/client_golang v1.23.2 @@ -15,11 +16,13 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/time v0.15.0 k8s.io/api v0.35.3 + k8s.io/apiextensions-apiserver v0.35.3 k8s.io/apimachinery v0.35.3 k8s.io/client-go v0.35.3 k8s.io/klog/v2 v2.140.0 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.23.3 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -135,7 +138,6 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.35.3 // indirect k8s.io/apiserver v0.35.3 // indirect k8s.io/component-base v0.35.3 // indirect k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 // indirect @@ -143,7 +145,6 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) replace github.com/sigstore/model-signing => github.com/sampras343/model-transparency-go v0.0.0-20260226135740-d987ec6b59e6 diff --git a/go.sum b/go.sum index fbd7affd..6dfdcbca 100644 --- a/go.sum +++ b/go.sum @@ -273,6 +273,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/konflux-ci/coverport/instrumentation/go v0.0.0-20260827155954-c914ae07e9f0 h1:fPsWbCAw78aCmQJ/C7VmibCHnammkgDuWvD853kL63o= +github.com/konflux-ci/coverport/instrumentation/go v0.0.0-20260827155954-c914ae07e9f0/go.mod h1:i6BCROSU0M/Fk4QhzBbnY4R6JVq0hU7jlUSSxjd8dD4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/internal/constants/images.go b/internal/constants/images.go index 551eb943..095be6e5 100644 --- a/internal/constants/images.go +++ b/internal/constants/images.go @@ -25,7 +25,6 @@ var ( // It contains the validation-agent binary which natively validates models // using the model-transparency-go library. Used for both one-shot and // continuous validation modes. - // This can be overridden at build time via ldflags: - // go build -ldflags="-X github.com/sigstore/model-validation-operator/internal/constants.ModelValidationAgentImage=myimage:tag" + // This can be overridden at build time via the LDFLAGS defined in the Makefile. ModelValidationAgentImage = "ghcr.io/sigstore/model-validation-agent:v0.1.0" ) diff --git a/internal/webhooks/pod_webhook.go b/internal/webhooks/pod_webhook.go index e7314dc4..799b9cad 100644 --- a/internal/webhooks/pod_webhook.go +++ b/internal/webhooks/pod_webhook.go @@ -24,6 +24,7 @@ import ( "time" "github.com/sigstore/model-validation-operator/internal/constants" + admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "sigs.k8s.io/controller-runtime/pkg/client" @@ -69,6 +70,20 @@ func (p *podInterceptor) Handle(ctx context.Context, req admission.Request) admi return admission.Errored(http.StatusBadRequest, err) } + // Finalizer removal must not depend on a ModelValidation that may already + // have been deleted. Only bypass injection for an already-terminating Pod: + // the API server's old object prevents a new deletionTimestamp on an active + // Pod update from taking this path. + if req.Operation == admissionv1.Update && !pod.DeletionTimestamp.IsZero() { + oldPod := &corev1.Pod{} + if err := p.decoder.DecodeRaw(req.OldObject, oldPod); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + if !oldPod.DeletionTimestamp.IsZero() && oldPod.UID == pod.UID { + return admission.Allowed("terminating pod cleanup, no injection needed") + } + } + // Check if namespace should be ignored ns := &corev1.Namespace{} if err := p.client.Get(ctx, client.ObjectKey{Name: req.Namespace}, ns); err != nil { diff --git a/internal/webhooks/pod_webhook_cleanup_test.go b/internal/webhooks/pod_webhook_cleanup_test.go new file mode 100644 index 00000000..3e3a7c08 --- /dev/null +++ b/internal/webhooks/pod_webhook_cleanup_test.go @@ -0,0 +1,135 @@ +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webhooks + +import ( + "context" + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/sigstore/model-validation-operator/api/v1alpha1" + "github.com/sigstore/model-validation-operator/internal/constants" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func TestTerminatingPodCleanupAdmission(t *testing.T) { + for _, test := range []struct { + name string + operation admissionv1.Operation + oldTerminating bool + newTerminating bool + uninjected bool + differentUID bool + malformedOld bool + withCR bool + allowed bool + }{ + {name: "remove finalizer after CR deletion", operation: admissionv1.Update, + oldTerminating: true, newTerminating: true, allowed: true}, + {name: "do not inject a terminating uninjected pod", operation: admissionv1.Update, + oldTerminating: true, newTerminating: true, uninjected: true, allowed: true}, + {name: "active create still requires CR", operation: admissionv1.Create}, + {name: "active injected update still requires CR", operation: admissionv1.Update}, + {name: "create cannot bypass CR lookup with deletion timestamp", operation: admissionv1.Create, + newTerminating: true}, + {name: "new deletion timestamp cannot bypass CR lookup", operation: admissionv1.Update, + newTerminating: true}, + {name: "different UID cannot bypass CR lookup", operation: admissionv1.Update, + oldTerminating: true, newTerminating: true, differentUID: true}, + {name: "malformed old pod is rejected", operation: admissionv1.Update, + newTerminating: true, malformedOld: true}, + {name: "normal injected update with existing CR remains allowed", operation: admissionv1.Update, + withCR: true, allowed: true}, + } { + t.Run(test.name, func(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + const namespace = "cleanup-test" + const modelName = "cleanup-model" + objects := []client.Object{&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}} + if test.withCR { + objects = append(objects, &v1alpha1.ModelValidation{ + ObjectMeta: metav1.ObjectMeta{Name: modelName, Namespace: namespace}, + }) + } + handler := NewPodInterceptor(fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build(), + admission.NewDecoder(scheme)) + oldPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cleanup-pod", Namespace: namespace, UID: "original-pod", + Labels: map[string]string{constants.ModelValidationLabel: modelName}, + Finalizers: []string{constants.ModelValidationFinalizer}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "workload", Image: "busybox"}}, + InitContainers: []corev1.Container{{Name: constants.ModelValidationInitContainerName, + Image: "validation-agent"}}, + }, + } + if test.uninjected { + oldPod.Spec.InitContainers = nil + } + deletionTime := metav1.NewTime(time.Now().Add(-time.Minute)) + if test.oldTerminating { + oldPod.DeletionTimestamp = &deletionTime + } + pod := oldPod.DeepCopy() + pod.Finalizers = nil + if test.newTerminating { + pod.DeletionTimestamp = &deletionTime + } + if test.differentUID { + pod.UID = "different-pod" + } + oldBody, err := json.Marshal(oldPod) + if err != nil { + t.Fatal(err) + } + if test.malformedOld { + oldBody = []byte(`{"invalid`) + } + body, err := json.Marshal(pod) + if err != nil { + t.Fatal(err) + } + response := handler.Handle(context.Background(), admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespace, Operation: test.operation, + Object: runtime.RawExtension{Raw: body}, OldObject: runtime.RawExtension{Raw: oldBody}, + }}) + if response.Allowed != test.allowed { + t.Fatalf("allowed = %t, want %t; result: %+v", response.Allowed, test.allowed, response.Result) + } + if len(response.Patches) != 0 { + t.Fatalf("unexpected mutation of cleanup/existing injected pod: %+v", response.Patches) + } + if !test.allowed && (response.Result == nil || response.Result.Code != http.StatusBadRequest) { + t.Fatalf("expected bad request for missing CR or malformed old object, got %+v", response.Result) + } + }) + } +} diff --git a/scripts/e2e-coverage.sh b/scripts/e2e-coverage.sh new file mode 100755 index 00000000..886690f8 --- /dev/null +++ b/scripts/e2e-coverage.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Collect Coverport snapshots without losing counters when a manager restarts. +# Run init once, snapshot before a disruption, and collect after the suite. +set -euo pipefail + +project_root=$(cd "$(dirname "$0")/.." && pwd) +data_dir=${E2E_COVERAGE_DIR:-"$project_root/e2e-coverdata"} +profile=${E2E_COVERAGE_PROFILE:-"$project_root/e2e-cover.out"} +namespace=${E2E_OPERATOR_NAMESPACE:-model-validation-operator-system} +kubectl=${KUBECTL:-kubectl} +go_tool=${GO:-go} +attempts=${E2E_COVERAGE_ATTEMPTS:-30} +retry_delay=${E2E_COVERAGE_RETRY_DELAY:-1} +forward_pid= + +fail() { echo "E2E coverage: $*" >&2; exit 1; } +stop_forward() { + if [[ -n "$forward_pid" ]]; then + kill "$forward_pid" 2>/dev/null || true + wait "$forward_pid" 2>/dev/null || true + forward_pid= + fi +} +trap stop_forward EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +[[ "$data_dir" != *,* ]] || fail "the data directory must not contain commas (covdata input separator)" +[[ "$attempts" =~ ^[1-9][0-9]*$ ]] || fail "E2E_COVERAGE_ATTEMPTS must be positive" +[[ "$retry_delay" =~ ^[0-9]+$ ]] || fail "E2E_COVERAGE_RETRY_DELAY must be a nonnegative integer" + +session_dir() { + [[ -f "$data_dir/.active" ]] || fail "initialize a session first: make init-e2e-coverage" + local active + active=$(<"$data_dir/.active") + [[ "$active" =~ ^run\.[a-zA-Z0-9]+$ ]] || fail "invalid coverage session marker" + [[ -d "$data_dir/$active" ]] || fail "coverage session directory is missing" + printf '%s\n' "$data_dir/$active" +} + +init_session() { + mkdir -p "$data_dir" + local session marker + session=$(mktemp -d "$data_dir/run.XXXXXXXX") + if [[ -f "$profile" ]]; then + mv "$profile" "$session/previous-profile.out" + fi + marker=$(mktemp "$data_dir/.active.XXXXXXXX") + basename "$session" > "$marker" + mv "$marker" "$data_dir/.active" + echo "Initialized coverage session $session (previous sessions retained)" +} + +validate_profile() { + # covdata may succeed for an empty directory; require actual statement records. + awk 'NR == 1 { if ($0 !~ /^mode: (set|count|atomic)$/) exit 1; next } + NF != 3 || $2 !~ /^[0-9]+$/ || $3 !~ /^[0-9]+$/ { exit 1 } + { statements += $2 } + END { if (NR < 2 || statements < 1) exit 1 }' "$1" || fail "empty or malformed coverage profile: $1" +} + +snapshot() { + local session inventory candidates pod uid restarts snapshot_dir port response meta counter marker + for dependency in "$kubectl" "$go_tool" curl jq base64; do + command -v "$dependency" >/dev/null || fail "required command not found: $dependency" + done + session=$(session_dir) + inventory=$(mktemp "$session/pods.XXXXXXXX") + # Check API failures outside command substitution in a for loop. + "$kubectl" --request-timeout=20s get pods -n "$namespace" \ + -l control-plane=controller-manager -o json > "$inventory" + jq -e '.items | type == "array"' "$inventory" >/dev/null || fail "invalid PodList response" + candidates=$(jq -r '.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running") | + . as $pod | .status.containerStatuses[]? | + select(.name == "manager" and .state.running != null) | + [$pod.metadata.name, $pod.metadata.uid, (.restartCount | tostring)] | @tsv' "$inventory") + [[ -n "$candidates" ]] || fail "no running manager processes found in $namespace" + + while IFS=$'\t' read -r pod uid restarts; do + [[ "$pod" =~ ^[a-z0-9.-]+$ && "$uid" =~ ^[a-zA-Z0-9-]+$ && "$restarts" =~ ^[0-9]+$ ]] || + fail "invalid manager process identity" + snapshot_dir=$(mktemp -d "$session/snapshot.XXXXXXXX") + response="$snapshot_dir/response.json" + # Let the OS choose a free local port; never expose Coverport publicly. + "$kubectl" --request-timeout=20s port-forward -n "$namespace" --address=127.0.0.1 \ + --pod-running-timeout=20s "pod/$pod" :53700 > "$snapshot_dir/port-forward.log" 2>&1 & + forward_pid=$! + local collected=false + for ((attempt = 1; attempt <= attempts; attempt++)); do + kill -0 "$forward_pid" 2>/dev/null || break + port=$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> .*/\1/p' "$snapshot_dir/port-forward.log" | head -1) + if [[ -n "$port" ]] && curl --silent --show-error --fail --connect-timeout 2 --max-time 5 \ + "http://127.0.0.1:$port/coverage" > "$response" 2> "$snapshot_dir/curl.log"; then + collected=true + break + fi + sleep "$retry_delay" + done + stop_forward + if [[ "$collected" != true ]]; then + cat "$snapshot_dir/port-forward.log" >&2 + fail "could not collect counters from $pod; diagnostics: $snapshot_dir" + fi + jq -e ' + (.meta_filename | type == "string" and test("^covmeta\\.[a-f0-9]{32}$")) and + (.counters_filename | type == "string" and test("^covcounters\\.[a-f0-9]{32}\\.[0-9]+\\.[0-9]+$")) and + (.meta_data | type == "string" and length > 0 and test("^[A-Za-z0-9+/]*={0,2}$")) and + (.counters_data | type == "string" and length > 0 and test("^[A-Za-z0-9+/]*={0,2}$")) and + ((.meta_filename | split(".")[1]) == (.counters_filename | split(".")[1])) + ' "$response" >/dev/null || fail "malformed Coverport response from $pod" + meta=$(jq -r .meta_filename "$response") + counter=$(jq -r .counters_filename "$response") + local decode=--decode + base64 --decode /dev/null 2>&1 || decode=-D + jq -r .meta_data "$response" | base64 "$decode" > "$snapshot_dir/$meta" + jq -r .counters_data "$response" | base64 "$decode" > "$snapshot_dir/$counter" + [[ -s "$snapshot_dir/$meta" && -s "$snapshot_dir/$counter" ]] || fail "empty binary coverage data from $pod" + "$go_tool" tool covdata textfmt -i="$snapshot_dir" -o="$snapshot_dir/profile.out" + validate_profile "$snapshot_dir/profile.out" + # Publish only a completely validated snapshot. The latest snapshot for a + # process replaces its pointer, avoiding double-counting cumulative hits. + # Restarted containers and replacement Pods get separate process pointers. + marker=$(mktemp "$session/.latest.XXXXXXXX") + basename "$snapshot_dir" > "$marker" + mv "$marker" "$session/latest.$uid.$restarts" + echo "Saved coverage for $pod (UID $uid, restart $restarts)" + done <<< "$candidates" +} + +report() { + local session pointer name inputs= output + session=$(session_dir) + shopt -s nullglob + local pointers=("$session"/latest.*) + ((${#pointers[@]} > 0)) || fail "no validated snapshots in the active session" + for pointer in "${pointers[@]}"; do + name=$(<"$pointer") + [[ "$name" =~ ^snapshot\.[a-zA-Z0-9]+$ && -d "$session/$name" ]] || fail "invalid snapshot pointer" + inputs="${inputs:+$inputs,}$session/$name" + done + output=$(mktemp "$session/merged.XXXXXXXX") + "$go_tool" tool covdata textfmt -i="$inputs" -o="$output" + validate_profile "$output" + mv "$output" "$profile" + echo "Merged ${#pointers[@]} manager processes into $profile" +} + +case "${1:-}" in + init) init_session ;; + snapshot) snapshot ;; + report) report ;; + collect) snapshot; report ;; + *) fail "usage: $0 {init|snapshot|report|collect}" ;; +esac diff --git a/scripts/test-e2e-coverage.sh b/scripts/test-e2e-coverage.sh new file mode 100755 index 00000000..1cd8a365 --- /dev/null +++ b/scripts/test-e2e-coverage.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# No Kubernetes access: use mocked clients and real Go coverage data. +set -euo pipefail +project_root=$(cd "$(dirname "$0")/.." && pwd) +fixtures="$project_root/scripts/testdata/e2e-coverage" +collector="$project_root/scripts/e2e-coverage.sh" +test_dir=$(mktemp -d "${TMPDIR:-/tmp}/mvo-coverage-test.XXXXXXXX") +trap 'rm -rf "$test_dir"' EXIT +export E2E_COVERAGE_DIR="$test_dir/collected" +export E2E_COVERAGE_PROFILE="$test_dir/coverage.out" +export E2E_COVERAGE_ATTEMPTS=4 E2E_COVERAGE_RETRY_DELAY=1 +export MOCK_STATE="$test_dir" MOCK_RESPONSE="$test_dir/before.json" +export KUBECTL="$fixtures/kubectl" PATH="$fixtures:$PATH" + +assert_no_forwards() { + [[ -f "$test_dir/forward-pids" ]] || return 0 + while read -r pid; do + if kill -0 "$pid" 2>/dev/null; then + echo "FAIL: port-forward $pid leaked" >&2 + exit 1 + fi + done < "$test_dir/forward-pids" +} + +expect_failure() { + local mode=$1 + if MOCK_MODE="$mode" bash "$collector" snapshot > "$test_dir/$mode.log" 2>&1; then + echo "FAIL: accepted $mode" >&2 + exit 1 + fi + assert_no_forwards + echo "PASS: rejects $mode and cleans port-forwards" +} + +mkdir "$test_dir/before" "$test_dir/after" +go build -cover -covermode=atomic -o "$test_dir/fixture" "$fixtures/main.go" +GOCOVERDIR="$test_dir/before" "$test_dir/fixture" before >/dev/null +GOCOVERDIR="$test_dir/after" "$test_dir/fixture" after >/dev/null +for phase in before after; do + meta=("$test_dir/$phase"/covmeta.*) + counters=("$test_dir/$phase"/covcounters.*) + jq -n --arg meta "$(basename "${meta[0]}")" --arg counter "$(basename "${counters[0]}")" \ + --arg meta_data "$(base64 < "${meta[0]}" | tr -d '\n')" \ + --arg counter_data "$(base64 < "${counters[0]}" | tr -d '\n')" \ + '{meta_filename:$meta,counters_filename:$counter,meta_data:$meta_data,counters_data:$counter_data}' \ + > "$test_dir/$phase.json" +done + +if bash "$collector" snapshot > "$test_dir/no-session.log" 2>&1; then + echo "FAIL: snapshot should require explicit initialization" >&2 + exit 1 +fi +bash "$collector" init +if bash "$collector" report > "$test_dir/empty-report.log" 2>&1; then + echo "FAIL: empty report should not succeed" >&2 + exit 1 +fi +echo "PASS: requires initialization and nonempty report" +for mode in api-error zero-pods invalid-list port-forward-error http-error malformed missing-counters path-traversal bad-base64 bad-binary; do + expect_failure "$mode" +done + +bash "$collector" snapshot +bash "$collector" report +go tool cover -func="$E2E_COVERAGE_PROFILE" > "$test_dir/before-functions" +awk '$2 == "beforeRestart" && $3 == "100.0%" { found = 1 } END { exit !found }' "$test_dir/before-functions" +awk '$2 == "afterRestart" && $3 == "0.0%" { found = 1 } END { exit !found }' "$test_dir/before-functions" +cp "$E2E_COVERAGE_PROFILE" "$test_dir/first.out" +# Re-reading cumulative counters must replace the process snapshot, not add hits. +bash "$collector" collect +cmp "$test_dir/first.out" "$E2E_COVERAGE_PROFILE" +echo "PASS: multi-replica collection is repeatable without double-counting" + +# Every process is replaced, just as in the scale-to-zero test. Only the second +# response knows afterRestart; beforeRestart must survive through stored data. +MOCK_GENERATION=2 MOCK_RESPONSE="$test_dir/after.json" bash "$collector" collect +go tool cover -func="$E2E_COVERAGE_PROFILE" > "$test_dir/merged-functions" +awk '$2 == "beforeRestart" && $3 == "100.0%" { found = 1 } END { exit !found }' "$test_dir/merged-functions" +awk '$2 == "afterRestart" && $3 == "100.0%" { found = 1 } END { exit !found }' "$test_dir/merged-functions" +assert_no_forwards +echo "PASS: uniquely covered paths survive replacement of every replica" + +bash "$collector" init +[[ ! -e "$E2E_COVERAGE_PROFILE" ]] || { echo "FAIL: stale profile survives initialization" >&2; exit 1; } +if bash "$collector" report > "$test_dir/new-session.log" 2>&1; then + echo "FAIL: a new session reused old snapshots" >&2 + exit 1 +fi +echo "PASS: explicit initialization isolates successive suite runs" diff --git a/scripts/testdata/e2e-coverage/curl b/scripts/testdata/e2e-coverage/curl new file mode 100755 index 00000000..b1a34816 --- /dev/null +++ b/scripts/testdata/e2e-coverage/curl @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +# Assert that each HTTP request is explicitly bounded. +[[ " $* " == *" --connect-timeout 2 "* && " $* " == *" --max-time 5 "* ]] +case "${MOCK_MODE:-success}" in + http-error) exit 22 ;; + malformed) echo 'not json' ;; + missing-counters) jq 'del(.counters_data)' "$MOCK_RESPONSE" ;; + path-traversal) jq '.meta_filename = "../../outside"' "$MOCK_RESPONSE" ;; + bad-base64) jq '.counters_data = "!invalid!"' "$MOCK_RESPONSE" ;; + bad-binary) jq '.counters_data = "aW52YWxpZA=="' "$MOCK_RESPONSE" ;; + *) cat "$MOCK_RESPONSE" ;; +esac diff --git a/scripts/testdata/e2e-coverage/kubectl b/scripts/testdata/e2e-coverage/kubectl new file mode 100755 index 00000000..88dca37c --- /dev/null +++ b/scripts/testdata/e2e-coverage/kubectl @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +case " $* " in + *" get pods "*) + case "${MOCK_MODE:-success}" in + api-error) echo "mock API failure" >&2; exit 1 ;; + zero-pods) echo '{"items":[]}'; exit 0 ;; + invalid-list) echo '{"error":"not a PodList"}'; exit 0 ;; + esac + jq -n --arg generation "${MOCK_GENERATION:-1}" '{items: ["a", "b"] | map({ + metadata: {name: ("manager-" + .), uid: ("uid-" + . + "-" + $generation)}, + status: {phase: "Running", containerStatuses: [{name: "manager", restartCount: 0, state: {running: {}}}]} + })}' + ;; + *" port-forward "*) + echo "$$" >> "$MOCK_STATE/forward-pids" + trap 'exit 0' TERM INT + [[ "${MOCK_MODE:-success}" != port-forward-error ]] || exit 1 + echo "Forwarding from 127.0.0.1:54321 -> 53700" + while :; do sleep 0.1; done + ;; + *) echo "unexpected kubectl command: $*" >&2; exit 1 ;; +esac diff --git a/scripts/testdata/e2e-coverage/main.go b/scripts/testdata/e2e-coverage/main.go new file mode 100644 index 00000000..5f8bd29f --- /dev/null +++ b/scripts/testdata/e2e-coverage/main.go @@ -0,0 +1,18 @@ +// This tiny program supplies real Go binary coverage data to collector tests. +package main + +import ( + "fmt" + "os" +) + +func beforeRestart() { fmt.Println("only executed before restart") } +func afterRestart() { fmt.Println("only executed after restart") } + +func main() { + if len(os.Args) > 1 && os.Args[1] == "before" { + beforeRestart() + } else { + afterRestart() + } +} diff --git a/test/e2e-olm/config.go b/test/e2e-olm/config.go new file mode 100644 index 00000000..fa7ca5ac --- /dev/null +++ b/test/e2e-olm/config.go @@ -0,0 +1,102 @@ +//go:build upgrade + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2eolm + +import ( + "fmt" + "strconv" + "strings" +) + +const ( + operatorNamespace = "model-validation-operator-system" + webhookTestNamespace = "e2e-webhook-test-ns" +) + +// upgradeConfig separates a generic next-build regression from a release +// certification. Only the latter can attest to the named tech-preview/GA inputs. +type upgradeConfig struct { + baseCatalog, targetCatalog string + baseChannel, targetChannel string + packageName string + controllerNamespace, catalogNamespace string + expectedBaseCSV, expectedTargetCSV string + expectedBaseImage, expectedTargetImage string + gaCertification, validateModels bool +} + +func readUpgradeConfig(getenv func(string) string) (upgradeConfig, error) { + value := func(key, fallback string) string { + if v := strings.TrimSpace(getenv(key)); v != "" { + return v + } + return fallback + } + config := upgradeConfig{ + baseCatalog: value("TEST_BASE_CATALOG", ""), + targetCatalog: value("TEST_TARGET_CATALOG", ""), + baseChannel: value("OLM_BASE_CHANNEL", value("OLM_CHANNEL", "tech-preview")), + packageName: value("OLM_PACKAGE", "model-validation-operator"), + controllerNamespace: value("OLM_CONTROLLER_NAMESPACE", ""), + // Namespaced catalogs are visible to subscriptions in that namespace on + // both standalone OLM and OpenShift. The OLM controller namespace is not + // OpenShift's global catalog namespace (openshift-marketplace). + catalogNamespace: value("OLM_CATALOG_NAMESPACE", operatorNamespace), + expectedBaseCSV: value("TEST_EXPECTED_BASE_CSV", ""), + expectedTargetCSV: value("TEST_EXPECTED_TARGET_CSV", ""), + expectedBaseImage: value("TEST_EXPECTED_BASE_IMAGE", ""), + expectedTargetImage: value("TEST_EXPECTED_TARGET_IMAGE", ""), + } + config.targetChannel = value("OLM_TARGET_CHANNEL", config.baseChannel) + for key, target := range map[string]*bool{ + "TEST_GA_CERTIFICATION": &config.gaCertification, + "TEST_VALIDATE_MODELS": &config.validateModels, + } { + var err error + *target, err = strconv.ParseBool(value(key, "false")) + if err != nil { + return config, fmt.Errorf("%s must be a boolean: %w", key, err) + } + } + if config.baseCatalog == "" || config.targetCatalog == "" { + return config, fmt.Errorf("TEST_BASE_CATALOG and TEST_TARGET_CATALOG are required") + } + if config.baseCatalog == config.targetCatalog { + return config, fmt.Errorf("base and target catalog references must differ; use immutable release inputs") + } + if config.gaCertification { + for _, key := range []string{ + "OLM_BASE_CHANNEL", "OLM_TARGET_CHANNEL", + "TEST_EXPECTED_BASE_CSV", "TEST_EXPECTED_TARGET_CSV", + "TEST_EXPECTED_BASE_IMAGE", "TEST_EXPECTED_TARGET_IMAGE", + } { + if value(key, "") == "" { + return config, fmt.Errorf("%s is required for GA certification; obtain approved release inputs", key) + } + } + if !config.validateModels { + return config, fmt.Errorf("GA certification requires TEST_VALIDATE_MODELS=true and prepared signed model fixtures") + } + } + if config.expectedBaseCSV != "" && config.expectedBaseCSV == config.expectedTargetCSV { + return config, fmt.Errorf("expected base and target CSVs must differ") + } + if config.expectedBaseImage != "" && config.expectedBaseImage == config.expectedTargetImage { + return config, fmt.Errorf("expected base and target operator images must differ") + } + return config, nil +} diff --git a/test/e2e-olm/config_test.go b/test/e2e-olm/config_test.go new file mode 100644 index 00000000..97cf3015 --- /dev/null +++ b/test/e2e-olm/config_test.go @@ -0,0 +1,122 @@ +//go:build upgrade + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2eolm + +import ( + "strings" + "testing" +) + +const ( + trueValue = "true" + previewChannel = "tech-preview" +) + +func TestUpgradeConfig(t *testing.T) { + base := func() map[string]string { + return map[string]string{"TEST_BASE_CATALOG": "example.com/base:v1", "TEST_TARGET_CATALOG": "example.com/target:v2"} + } + tests := []struct { + name string + modify func(map[string]string) + wantError string + }{ + {name: "generic upgrade"}, + {name: "missing catalog", modify: func(e map[string]string) { delete(e, "TEST_BASE_CATALOG") }, + wantError: "are required"}, + {name: "same catalog", modify: func(e map[string]string) { e["TEST_TARGET_CATALOG"] = e["TEST_BASE_CATALOG"] }, + wantError: "must differ"}, + {name: "invalid boolean", modify: func(e map[string]string) { e["TEST_GA_CERTIFICATION"] = "yes" }, + wantError: "must be a boolean"}, + {name: "GA cannot default identity", modify: func(e map[string]string) { e["TEST_GA_CERTIFICATION"] = trueValue }, + wantError: "required for GA certification"}, + {name: "GA cannot use admission only", modify: func(e map[string]string) { + addGAInputs(e) + e["TEST_VALIDATE_MODELS"] = "false" + }, wantError: "prepared signed model fixtures"}, + {name: "GA explicit same channel is valid", modify: addGAInputs}, + {name: "same CSV rejected", modify: func(e map[string]string) { + addGAInputs(e) + e["TEST_EXPECTED_TARGET_CSV"] = e["TEST_EXPECTED_BASE_CSV"] + }, wantError: "CSVs must differ"}, + {name: "same operator image rejected", modify: func(e map[string]string) { + addGAInputs(e) + e["TEST_EXPECTED_TARGET_IMAGE"] = e["TEST_EXPECTED_BASE_IMAGE"] + }, wantError: "images must differ"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + env := base() + if tc.modify != nil { + tc.modify(env) + } + config, err := readUpgradeConfig(func(key string) string { return env[key] }) + if tc.wantError != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantError) { + t.Fatalf("wanted %q, got %v", tc.wantError, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if config.catalogNamespace != operatorNamespace { + t.Fatalf("catalog must default to subscription namespace, got %s", config.catalogNamespace) + } + if config.targetChannel != config.baseChannel { + t.Fatalf("target channel should default to base") + } + }) + } +} + +func TestUpgradeConfigNamespaceAndChannelOverrides(t *testing.T) { + env := map[string]string{ + "TEST_BASE_CATALOG": "base:v1", "TEST_TARGET_CATALOG": "target:v2", + "OLM_CONTROLLER_NAMESPACE": "openshift-operator-lifecycle-manager", + "OLM_CATALOG_NAMESPACE": "openshift-marketplace", + "OLM_CHANNEL": "legacy", "OLM_BASE_CHANNEL": previewChannel, "OLM_TARGET_CHANNEL": "stable", + } + config, err := readUpgradeConfig(func(key string) string { return env[key] }) + if err != nil { + t.Fatal(err) + } + if config.controllerNamespace != env["OLM_CONTROLLER_NAMESPACE"] || + config.catalogNamespace != env["OLM_CATALOG_NAMESPACE"] { + t.Fatal("namespace overrides not preserved") + } + if config.baseChannel != previewChannel || config.targetChannel != "stable" { + t.Fatal("channel overrides not preserved") + } + delete(env, "OLM_BASE_CHANNEL") + delete(env, "OLM_TARGET_CHANNEL") + config, err = readUpgradeConfig(func(key string) string { return env[key] }) + if err != nil || config.baseChannel != "legacy" || config.targetChannel != "legacy" { + t.Fatalf("legacy OLM_CHANNEL fallback failed: %#v, %v", config, err) + } +} + +func addGAInputs(env map[string]string) { + env["TEST_GA_CERTIFICATION"] = trueValue + env["TEST_VALIDATE_MODELS"] = trueValue + env["OLM_BASE_CHANNEL"] = previewChannel + env["OLM_TARGET_CHANNEL"] = previewChannel + env["TEST_EXPECTED_BASE_CSV"] = "mvo.v0.1.3" + env["TEST_EXPECTED_TARGET_CSV"] = "mvo.v1.0.0" + env["TEST_EXPECTED_BASE_IMAGE"] = "example.com/mvo:v0.1.3" + env["TEST_EXPECTED_TARGET_IMAGE"] = "example.com/mvo:v1.0.0" +} diff --git a/test/e2e-olm/helpers_test.go b/test/e2e-olm/helpers_test.go new file mode 100644 index 00000000..1d462fff --- /dev/null +++ b/test/e2e-olm/helpers_test.go @@ -0,0 +1,218 @@ +//go:build upgrade + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2eolm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/sigstore/model-validation-operator/internal/constants" + utils "github.com/sigstore/model-validation-operator/test/utils" +) + +// --ignore-not-found suppresses only the API's NotFound result. Authentication, +// transport and authorization errors remain errors, not successful deletion. +func getResourceIfPresent(resource, name, namespace string) (string, error) { + args := []string{"get", resource, name, "--ignore-not-found=true", "--request-timeout=30s", "-o", "json"} + if namespace != "" { + args = append(args, "-n", namespace) + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + output, err := utils.Run(exec.CommandContext(ctx, "kubectl", args...)) + return strings.TrimSpace(output), err +} + +type cleanupTarget struct { + resource, name, namespace string +} + +// Do not fail fast during recovery. Every exact owned target gets one bounded +// attempt, in the specified order; callers assert the aggregate result once. +func cleanupAllResources(targets []cleanupTarget, remove func(string, string, string) error) error { + var failures []error + for _, target := range targets { + if target.name == "" { + failures = append(failures, fmt.Errorf("refusing unnamed %s cleanup", target.resource)) + continue + } + if err := remove(target.resource, target.name, target.namespace); err != nil { + failures = append(failures, fmt.Errorf("cleaning up %s %s/%s: %w", + target.resource, target.namespace, target.name, err)) + } + } + return errors.Join(failures...) +} + +func subscriptionCSVNames(output string) ([]string, error) { + var subscription unstructured.Unstructured + if err := json.Unmarshal([]byte(output), &subscription); err != nil { + return nil, fmt.Errorf("decoding cleanup Subscription: %w", err) + } + var names []string + var failures []error + for _, field := range []string{"installedCSV", "currentCSV"} { + name, _, err := unstructured.NestedString(subscription.Object, "status", field) + if err != nil { + failures = append(failures, fmt.Errorf("reading cleanup Subscription status.%s: %w", field, err)) + } else if name != "" { + names = append(names, name) + } + } + return names, errors.Join(failures...) +} + +func ensureResourceAbsent(resource, name, namespace string) error { + output, err := getResourceIfPresent(resource, name, namespace) + if err != nil { + return err + } + if output != "" { + return fmt.Errorf("refusing to overwrite pre-existing %s %s/%s", resource, namespace, name) + } + return nil +} + +// AllNamespaces operators create informational CSV copies in other namespaces. +// They are not installations owned by this namespace and must not be deleted. +// Still refuse a copied MVO: its source installation would conflict with ours. +func ensureNoConflictingCSVs(csvs []unstructured.Unstructured, namespace, packageName string) error { + for _, csv := range csvs { + source := csv.GetLabels()["olm.copiedFrom"] + reason, _, err := unstructured.NestedString(csv.Object, "status", "reason") + if err != nil { + return fmt.Errorf("cannot classify CSV %s: %w", csv.GetName(), err) + } + if source == "" || source == namespace || csv.GetNamespace() != namespace || reason != "Copied" { + return fmt.Errorf("refusing existing CSV %s/%s: not a confirmed copy from another namespace", + csv.GetNamespace(), csv.GetName()) + } + if strings.HasPrefix(csv.GetName(), packageName+".") { + return fmt.Errorf("MVO CSV %s is copied from %s; remove the conflicting MVO installation first", + csv.GetName(), source) + } + owned, _, err := unstructured.NestedSlice(csv.Object, "spec", "customresourcedefinitions", "owned") + if err != nil { + return fmt.Errorf("cannot inspect CRD ownership of CSV %s: %w", csv.GetName(), err) + } + for _, item := range owned { + crd, ok := item.(map[string]any) + if !ok { + return fmt.Errorf("cannot inspect malformed CRD ownership of CSV %s", csv.GetName()) + } + if crd["name"] == "modelvalidations.ml.sigstore.dev" { + return fmt.Errorf("CSV %s copied from %s already owns the ModelValidation API", csv.GetName(), source) + } + } + } + return nil +} + +func isPodReady(pod corev1.Pod) bool { + if pod.DeletionTimestamp != nil { + return false + } + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + +func controllerRolledOut(expectedImage string) error { + var deployment appsv1.Deployment + err := utils.KubectlGetJSON("deployment", "model-validation-controller-manager", operatorNamespace, &deployment) + if err != nil { + return err + } + return validateControllerRollout(deployment, expectedImage) +} + +func validateControllerRollout(deployment appsv1.Deployment, expectedImage string) error { + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas < 1 { + return fmt.Errorf("controller deployment must have at least one replica") + } + desired := *deployment.Spec.Replicas + status := deployment.Status + if status.ObservedGeneration < deployment.Generation || status.UpdatedReplicas != desired || + status.Replicas != desired || status.ReadyReplicas != desired || + status.AvailableReplicas != desired || status.UnavailableReplicas != 0 { + return fmt.Errorf("controller rollout incomplete: generation %d, observed %d, desired %d, status %+v", + deployment.Generation, status.ObservedGeneration, desired, status) + } + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == "manager" { + if expectedImage != "" && container.Image != expectedImage { + return fmt.Errorf("manager image %q does not match expected %q", container.Image, expectedImage) + } + return nil + } + } + return fmt.Errorf("manager container missing from controller deployment") +} + +func jsonMergePatch(value any) string { + data, err := json.Marshal(value) + if err != nil { + panic(err) + } // Callers supply only maps of string literals. + return string(data) +} + +func olmProbeData(name, modelName string) utils.PodTemplateData { + data := utils.DefaultPodData(name, webhookTestNamespace, modelName, "olm-upgrade") + if !upgradeSettings.validateModels { + // Generic catalog CI has no seeded fixtures. Test admission without + // hostPath/SCC prerequisites, and do not call this validation success. + data.Volumes = nil + data.VolumeMounts = nil + } + return data +} + +func expectModelVerified(podName string) { + By("requiring successful cryptographic verification using the prepared signed PublicKey fixtures") + Eventually(func(g Gomega) { + var pod corev1.Pod + g.Expect(utils.KubectlGetJSON("pod", podName, webhookTestNamespace, &pod)).To(Succeed()) + var verified bool + for _, status := range pod.Status.InitContainerStatuses { + if status.Name == constants.ModelValidationInitContainerName && status.State.Terminated != nil { + g.Expect(status.State.Terminated.ExitCode).To(BeZero(), "validation agent must verify the signed model") + verified = true + } + } + g.Expect(verified).To(BeTrue(), "validation agent must complete") + g.Expect(isPodReady(pod)).To(BeTrue()) + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + logs, err := utils.Run(exec.Command("kubectl", "logs", podName, "-n", webhookTestNamespace, + "-c", constants.ModelValidationInitContainerName)) + Expect(err).NotTo(HaveOccurred()) + Expect(logs).To(ContainSubstring("Verification succeeded")) +} diff --git a/test/e2e-olm/helpers_unit_test.go b/test/e2e-olm/helpers_unit_test.go new file mode 100644 index 00000000..67c00735 --- /dev/null +++ b/test/e2e-olm/helpers_unit_test.go @@ -0,0 +1,228 @@ +//go:build upgrade + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2eolm + +import ( + "bytes" + "errors" + "slices" + "strings" + "testing" + "text/template" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" + + utils "github.com/sigstore/model-validation-operator/test/utils" +) + +func TestCleanupAllResources(t *testing.T) { + firstFailure := errors.New("Pod finalizer did not finish") + secondFailure := errors.New("CSV API rejected deletion") + targets := []cleanupTarget{ + {"pod", "initial", webhookTestNamespace}, + {"pod", "upgraded", webhookTestNamespace}, + {"modelvalidation", "model", webhookTestNamespace}, + {"subscription", "subscription", operatorNamespace}, + {"csv", "candidate", operatorNamespace}, + {"operatorgroup", "group", operatorNamespace}, + {"catalogsource", "catalog", operatorNamespace}, + } + var attempted []cleanupTarget + err := cleanupAllResources(targets, func(resource, name, namespace string) error { + attempted = append(attempted, cleanupTarget{resource, name, namespace}) + switch name { + case "initial": + return firstFailure + case "candidate": + return secondFailure + default: + return nil + } + }) + if !slices.Equal(attempted, targets) { + t.Fatalf("cleanup did not attempt all owned resources in order: %v", attempted) + } + if !errors.Is(err, firstFailure) || !errors.Is(err, secondFailure) { + t.Fatalf("cleanup did not preserve both failures: %v", err) + } + if !strings.Contains(err.Error(), "initial") || !strings.Contains(err.Error(), "candidate") { + t.Fatalf("aggregate error omitted resource identities: %v", err) + } + if err := cleanupAllResources(targets, func(string, string, string) error { return nil }); err != nil { + t.Fatalf("successful cleanup failed: %v", err) + } +} + +func TestCleanupRejectsUnnamedTargetsWithoutSkippingLaterTargets(t *testing.T) { + var attempted []string + err := cleanupAllResources([]cleanupTarget{{"csv", "", operatorNamespace}, {"pod", "owned", webhookTestNamespace}}, + func(_, name, _ string) error { attempted = append(attempted, name); return nil }) + if err == nil || !slices.Equal(attempted, []string{"owned"}) { + t.Fatalf("unnamed target must fail without broad deletion or blocking later cleanup: %v, %v", attempted, err) + } +} + +func TestSubscriptionCSVNames(t *testing.T) { + const prefix = `{"apiVersion":"operators.coreos.com/v1alpha1","kind":"Subscription","status":` + names, err := subscriptionCSVNames(prefix + `{"installedCSV":"base","currentCSV":"candidate"}}`) + if err != nil || !slices.Equal(names, []string{"base", "candidate"}) { + t.Fatalf("failed to discover only Subscription-referenced CSVs: %v, %v", names, err) + } + names, err = subscriptionCSVNames(prefix + `{"installedCSV":"base","currentCSV":true}}`) + if err == nil || !slices.Equal(names, []string{"base"}) { + t.Fatalf("must preserve known identity and report malformed remaining identity: %v, %v", names, err) + } + if _, err := subscriptionCSVNames(`invalid JSON`); err == nil { + t.Fatal("malformed Subscription JSON must not be silently ignored") + } + names, err = subscriptionCSVNames(prefix + `{}}`) + if err != nil || len(names) != 0 { + t.Fatalf("missing CSV references should return no names: %v, %v", names, err) + } +} + +func TestControllerRollout(t *testing.T) { + const expectedImage = "example.com/mvo:release" + tests := []struct { + name string + change func(*appsv1.Deployment) + }{ + {name: "controller has not observed new generation", change: func(d *appsv1.Deployment) { + d.Status.ObservedGeneration = d.Generation - 1 + }}, + {name: "old replica still present", change: func(d *appsv1.Deployment) { d.Status.Replicas++ }}, + {name: "new replica missing", change: func(d *appsv1.Deployment) { d.Status.UpdatedReplicas-- }}, + {name: "replica not ready", change: func(d *appsv1.Deployment) { d.Status.ReadyReplicas-- }}, + {name: "replica not available", change: func(d *appsv1.Deployment) { d.Status.AvailableReplicas-- }}, + {name: "unavailable replicas", change: func(d *appsv1.Deployment) { d.Status.UnavailableReplicas++ }}, + {name: "wrong release image", change: func(d *appsv1.Deployment) { + d.Spec.Template.Spec.Containers[0].Image = "example.com/mvo:other" + }}, + {name: "manager missing", change: func(d *appsv1.Deployment) { d.Spec.Template.Spec.Containers = nil }}, + {name: "replicas unspecified", change: func(d *appsv1.Deployment) { d.Spec.Replicas = nil }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + replicas := int32(2) + deployment := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 2}, + Spec: appsv1.DeploymentSpec{Replicas: &replicas, + Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "manager", Image: expectedImage}}, + }}, + }, + Status: appsv1.DeploymentStatus{ObservedGeneration: 2, Replicas: 2, + UpdatedReplicas: 2, ReadyReplicas: 2, AvailableReplicas: 2}, + } + if err := validateControllerRollout(deployment, expectedImage); err != nil { + t.Fatalf("healthy rollout rejected: %v", err) + } + tc.change(&deployment) + if err := validateControllerRollout(deployment, expectedImage); err == nil { + t.Fatal("incomplete or incorrect rollout accepted") + } + }) + } +} + +func TestCatalogTemplate(t *testing.T) { + tmpl, err := template.New("catalogsource").Parse(string(catalogSourceTemplate)) + if err != nil { + t.Fatal(err) + } + var rendered bytes.Buffer + err = tmpl.Execute(&rendered, utils.CatalogSourceTemplateData{ + Name: "test-catalog", Namespace: operatorNamespace, Image: "example.com/catalog:v1", + }) + if err != nil { + t.Fatal(err) + } + var catalog map[string]any + if err := yaml.Unmarshal(rendered.Bytes(), &catalog); err != nil { + t.Fatal(err) + } + mode, _, err := unstructured.NestedString(catalog, "spec", "grpcPodConfig", "securityContextConfig") + if err != nil || mode != "restricted" { + t.Fatalf("catalog security mode = %q, error = %v", mode, err) + } +} + +func TestExistingCSVs(t *testing.T) { + const packageName = "model-validation-operator" + const copiedFromLabel = "olm.copiedFrom" + tests := []struct { + name string + change func(*unstructured.Unstructured) + wantError bool + }{ + {name: "confirmed unrelated copy is allowed"}, + {name: "missing copy label", change: func(csv *unstructured.Unstructured) { + csv.SetLabels(nil) + }, wantError: true}, + {name: "missing copy reason", change: func(csv *unstructured.Unstructured) { + delete(csv.Object, "status") + }, wantError: true}, + {name: "local installation", change: func(csv *unstructured.Unstructured) { + csv.SetLabels(nil) + csv.Object["status"] = map[string]any{"reason": "InstallSucceeded"} + }, wantError: true}, + {name: "self copy label", change: func(csv *unstructured.Unstructured) { + csv.SetLabels(map[string]string{copiedFromLabel: operatorNamespace}) + }, wantError: true}, + {name: "wrong destination namespace", change: func(csv *unstructured.Unstructured) { + csv.SetNamespace("other") + }, wantError: true}, + {name: "copied MVO by package name", change: func(csv *unstructured.Unstructured) { + csv.SetName(packageName + ".v0.1.2") + }, wantError: true}, + {name: "copied different package owning MVO API", change: func(csv *unstructured.Unstructured) { + csv.Object["spec"] = map[string]any{"customresourcedefinitions": map[string]any{ + "owned": []any{map[string]any{"name": "modelvalidations.ml.sigstore.dev"}}, + }} + }, wantError: true}, + {name: "malformed reason", change: func(csv *unstructured.Unstructured) { + csv.Object["status"] = map[string]any{"reason": true} + }, wantError: true}, + } + if err := ensureNoConflictingCSVs(nil, operatorNamespace, packageName); err != nil { + t.Fatalf("empty namespace rejected: %v", err) + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + csv := unstructured.Unstructured{Object: map[string]any{ + "status": map[string]any{"reason": "Copied"}, + "spec": map[string]any{"customresourcedefinitions": map[string]any{ + "owned": []any{map[string]any{"name": "certificates.cert-manager.io"}}, + }}, + }} + csv.SetName("cert-manager.v1.19.0") + csv.SetNamespace(operatorNamespace) + csv.SetLabels(map[string]string{copiedFromLabel: "cert-manager-operator"}) + if tc.change != nil { + tc.change(&csv) + } + err := ensureNoConflictingCSVs([]unstructured.Unstructured{csv}, operatorNamespace, packageName) + if (err != nil) != tc.wantError { + t.Fatalf("wanted error=%t, got %v", tc.wantError, err) + } + }) + } +} diff --git a/test/e2e-olm/olm_upgrade_suite_test.go b/test/e2e-olm/olm_upgrade_suite_test.go index 6d9a5863..65db6133 100644 --- a/test/e2e-olm/olm_upgrade_suite_test.go +++ b/test/e2e-olm/olm_upgrade_suite_test.go @@ -1,3 +1,5 @@ +//go:build upgrade + // Copyright 2025 The Sigstore Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +18,7 @@ package e2eolm import ( "fmt" + "os" "testing" "time" @@ -23,20 +26,26 @@ import ( . "github.com/onsi/gomega" //nolint:revive utils "github.com/sigstore/model-validation-operator/test/utils" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) -const ( - operatorNamespace = "model-validation-operator-system" - webhookTestNamespace = "e2e-webhook-test-ns" +var ( + olmNamespace string + upgradeSettings upgradeConfig ) -var olmNamespace string - -func detectOLMNamespace() string { - if utils.KubectlResourceExists("ns", "openshift-operator-lifecycle-manager", "") { - return "openshift-operator-lifecycle-manager" +func detectOLMNamespace() (string, error) { + for _, namespace := range []string{"openshift-operator-lifecycle-manager", "olm"} { + output, err := getResourceIfPresent("namespace", namespace, "") + if err != nil { + return "", err + } + if output != "" { + return namespace, nil + } } - return "olm" + return "hosted/external (no local controller namespace)", nil } func TestOLMUpgrade(t *testing.T) { @@ -50,44 +59,49 @@ func TestOLMUpgrade(t *testing.T) { } var _ = BeforeSuite(func() { - By("detecting OLM namespace") - olmNamespace = detectOLMNamespace() - _, _ = fmt.Fprintf(GinkgoWriter, "Using OLM namespace: %s\n", olmNamespace) + var err error + upgradeSettings, err = readUpgradeConfig(os.Getenv) + Expect(err).NotTo(HaveOccurred()) - By("verifying OLM is installed") - Expect(utils.KubectlResourceExists("ns", olmNamespace, "")).To(BeTrue(), - "OLM namespace should exist - run 'make e2e-install-olm' first") - - By("verifying OLM operators are running") - Eventually(func() error { - output, err := utils.KubectlGet("pods", "", olmNamespace, - "jsonpath={range .items[*]}{.status.phase}{\"\\n\"}{end}") - if err != nil { - return err - } - phases := utils.GetNonEmptyLines(output) - if len(phases) == 0 { - return fmt.Errorf("no OLM pods found in %s namespace", olmNamespace) - } - for _, phase := range phases { - if phase != "Running" && phase != "Succeeded" { - return fmt.Errorf("OLM pod not ready, phase: %s", phase) + By("detecting OLM namespace") + olmNamespace = upgradeSettings.controllerNamespace + if olmNamespace == "" { + olmNamespace, err = detectOLMNamespace() + Expect(err).NotTo(HaveOccurred()) + } + _, _ = fmt.Fprintf(GinkgoWriter, "OLM controller location (diagnostic only): %s; catalog namespace: %s\n", + olmNamespace, upgradeSettings.catalogNamespace) + + By("verifying established OLM APIs are available") + // ROSA hosted control planes do not expose OLM controller Pods in the + // workload cluster. Do not confuse missing local Pods with missing OLM. + // Actual controller readiness is proved by catalog resolution and CSV + // installation below, not merely by the presence of these APIs. + for _, name := range []string{ + "catalogsources.operators.coreos.com", "subscriptions.operators.coreos.com", + "clusterserviceversions.operators.coreos.com", "installplans.operators.coreos.com", + } { + var crd apiextensionsv1.CustomResourceDefinition + Expect(utils.KubectlGetJSON("crd", name, "", &crd)).To(Succeed(), "OLM API %s must be installed", name) + established := false + for _, condition := range crd.Status.Conditions { + if condition.Type == apiextensionsv1.Established && condition.Status == apiextensionsv1.ConditionTrue { + established = true } } - return nil - }, 2*time.Minute, 5*time.Second).Should(Succeed(), "OLM operators should be running") - - By("verifying operator namespace exists") - Expect(utils.KubectlResourceExists("ns", operatorNamespace, "")).To(BeTrue(), - "Operator namespace should exist - run 'make e2e-setup-namespaces' first") + Expect(established).To(BeTrue(), "OLM API %s must be established", name) + } - By("verifying test namespace exists") - Expect(utils.KubectlResourceExists("ns", webhookTestNamespace, "")).To(BeTrue(), - "Test namespace should exist - run 'make e2e-setup-namespaces' first") + By("verifying operator, test and catalog namespaces exist") + for _, namespace := range []string{operatorNamespace, webhookTestNamespace, upgradeSettings.catalogNamespace} { + var ns corev1.Namespace + Expect(utils.KubectlGetJSON("namespace", namespace, "", &ns)).To(Succeed(), + "Namespace %s must exist before running the suite", namespace) + } }) var _ = AfterSuite(func() { - _, _ = fmt.Fprintf(GinkgoWriter, "OLM upgrade test suite cleanup complete\n") + _, _ = fmt.Fprintf(GinkgoWriter, "OLM upgrade suite finished; see the spec and cleanup results above\n") }) var _ = JustAfterEach(func() { diff --git a/test/e2e-olm/olm_upgrade_test.go b/test/e2e-olm/olm_upgrade_test.go index 70397a4a..c2e25beb 100644 --- a/test/e2e-olm/olm_upgrade_test.go +++ b/test/e2e-olm/olm_upgrade_test.go @@ -1,3 +1,5 @@ +//go:build upgrade + // Copyright 2025 The Sigstore Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,47 +17,43 @@ package e2eolm import ( + "context" + "errors" "fmt" - "os" "os/exec" + "sort" "time" . "github.com/onsi/ginkgo/v2" //nolint:revive . "github.com/onsi/gomega" //nolint:revive corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/sigstore/model-validation-operator/internal/constants" utils "github.com/sigstore/model-validation-operator/test/utils" ) -func cleanupResource(resource, name, namespace string) { - args := []string{"delete", resource} - if name != "" { - args = append(args, name) - } else { - args = append(args, "--all") +func cleanupResource(resource, name, namespace string) error { + if name == "" { + return fmt.Errorf("refusing cleanup without an explicit %s name", resource) } + args := []string{"delete", resource, name} if namespace != "" { args = append(args, "-n", namespace) } - args = append(args, "--timeout=60s", "--ignore-not-found=true") - _, _ = utils.Run(exec.Command("kubectl", args...)) -} - -func envOrDefault(key, defaultValue string) string { - if v := os.Getenv(key); v != "" { - return v - } - return defaultValue + args = append(args, "--timeout=60s", "--request-timeout=30s", "--ignore-not-found=true") + ctx, cancel := context.WithTimeout(context.Background(), 75*time.Second) + defer cancel() + _, err := utils.Run(exec.CommandContext(ctx, "kubectl", args...)) + return err } var _ = Describe("OLM Upgrade", Ordered, func() { var ( - baseCatalogImage string - targetCatalogImage string - channelName string - packageName string - baseCSV string + baseCSV string + upgradeCSV string + preservedCR unstructured.Unstructured + createdCR bool ) const ( @@ -66,29 +64,69 @@ var _ = Describe("OLM Upgrade", Ordered, func() { ) BeforeAll(func() { - baseCatalogImage = os.Getenv("TEST_BASE_CATALOG") - Expect(baseCatalogImage).NotTo(BeEmpty(), "TEST_BASE_CATALOG env var must be set") - - targetCatalogImage = os.Getenv("TEST_TARGET_CATALOG") - Expect(targetCatalogImage).NotTo(BeEmpty(), "TEST_TARGET_CATALOG env var must be set") - - channelName = envOrDefault("OLM_CHANNEL", "tech-preview") - packageName = envOrDefault("OLM_PACKAGE", "model-validation-operator") - _, _ = fmt.Fprintf(GinkgoWriter, "OLM Upgrade Test Configuration:\n") - _, _ = fmt.Fprintf(GinkgoWriter, " Base catalog: %s\n", baseCatalogImage) - _, _ = fmt.Fprintf(GinkgoWriter, " Target catalog: %s\n", targetCatalogImage) - _, _ = fmt.Fprintf(GinkgoWriter, " Channel: %s\n", channelName) - _, _ = fmt.Fprintf(GinkgoWriter, " Package: %s\n", packageName) + _, _ = fmt.Fprintf(GinkgoWriter, " Base catalog: %s\n", upgradeSettings.baseCatalog) + _, _ = fmt.Fprintf(GinkgoWriter, " Target catalog: %s\n", upgradeSettings.targetCatalog) + _, _ = fmt.Fprintf(GinkgoWriter, " Channels: %s -> %s\n", upgradeSettings.baseChannel, upgradeSettings.targetChannel) + _, _ = fmt.Fprintf(GinkgoWriter, " Package: %s; GA certification: %t; real model verification: %t\n", + upgradeSettings.packageName, upgradeSettings.gaCertification, upgradeSettings.validateModels) + + By("refusing to overwrite a standalone or existing OLM installation") + Expect(ensureResourceAbsent("deployment", "model-validation-controller-manager", operatorNamespace)).To(Succeed()) + for _, resource := range []string{"subscriptions", "operatorgroups"} { + var list unstructured.UnstructuredList + Expect(utils.KubectlGetJSON(resource, "", operatorNamespace, &list)).To(Succeed()) + Expect(list.Items).To(BeEmpty(), "Use a clean operator namespace; refusing to replace existing %s", resource) + } + var csvs unstructured.UnstructuredList + Expect(utils.KubectlGetJSON("clusterserviceversions", "", operatorNamespace, &csvs)).To(Succeed()) + Expect(ensureNoConflictingCSVs(csvs.Items, operatorNamespace, upgradeSettings.packageName)).To(Succeed()) + Expect(ensureResourceAbsent("catalogsource", catalogName, upgradeSettings.catalogNamespace)).To(Succeed()) + for _, name := range []string{"olm-test-pod-initial", "olm-test-pod-upgrade"} { + Expect(ensureResourceAbsent("pod", name, webhookTestNamespace)).To(Succeed()) + } DeferCleanup(func() { By("cleaning up OLM test resources") - cleanupResource("pods", "", webhookTestNamespace) - cleanupResource("modelvalidations", "", webhookTestNamespace) - cleanupResource("subscription", subscriptionName, operatorNamespace) - cleanupResource("csv", "", operatorNamespace) - cleanupResource("operatorgroup", operatorGroupName, operatorNamespace) - cleanupResource("catalogsource", catalogName, olmNamespace) + var cleanupErrors []error + // Delete only resources this suite owns. Capture a partially installed + // CSV before deleting the Subscription, even if the install spec failed. + csvNames := map[string]bool{baseCSV: true, upgradeCSV: true} + output, err := getResourceIfPresent("subscription", subscriptionName, operatorNamespace) + if err != nil { + cleanupErrors = append(cleanupErrors, fmt.Errorf("discovering owned CSVs: %w", err)) + } else if output != "" { + names, err := subscriptionCSVNames(output) + cleanupErrors = append(cleanupErrors, err) + for _, name := range names { + csvNames[name] = true + } + } + var targets []cleanupTarget + for _, name := range []string{"olm-test-pod-initial", "olm-test-pod-upgrade"} { + targets = append(targets, cleanupTarget{"pod", name, webhookTestNamespace}) + } + if createdCR { + targets = append(targets, cleanupTarget{"modelvalidation", testModelName, webhookTestNamespace}) + } + // Attempt all workload cleanup while the operator still exists. An + // individual failure must not prevent attempts on remaining owned + // resources, and is reported after all bounded deletes have run. + targets = append(targets, cleanupTarget{"subscription", subscriptionName, operatorNamespace}) + delete(csvNames, "") + var sortedCSVs []string + for name := range csvNames { + sortedCSVs = append(sortedCSVs, name) + } + sort.Strings(sortedCSVs) + for _, name := range sortedCSVs { + targets = append(targets, cleanupTarget{"csv", name, operatorNamespace}) + } + targets = append(targets, + cleanupTarget{"operatorgroup", operatorGroupName, operatorNamespace}, + cleanupTarget{"catalogsource", catalogName, upgradeSettings.catalogNamespace}) + cleanupErrors = append(cleanupErrors, cleanupAllResources(targets, cleanupResource)) + Expect(errors.Join(cleanupErrors...)).To(Succeed(), "All owned-resource cleanup attempts must succeed") }) }) @@ -96,14 +134,14 @@ var _ = Describe("OLM Upgrade", Ordered, func() { By("creating a CatalogSource with the base catalog image") err := utils.KubectlApply(catalogSourceTemplate, utils.CatalogSourceTemplateData{ Name: catalogName, - Namespace: olmNamespace, - Image: baseCatalogImage, + Namespace: upgradeSettings.catalogNamespace, + Image: upgradeSettings.baseCatalog, }) Expect(err).NotTo(HaveOccurred(), "Failed to create CatalogSource") By("waiting for CatalogSource to be ready") Eventually(func(g Gomega) { - status, err := utils.KubectlGet("catalogsource", catalogName, olmNamespace, + status, err := utils.KubectlGet("catalogsource", catalogName, upgradeSettings.catalogNamespace, "jsonpath={.status.connectionState.lastObservedState}") g.Expect(err).NotTo(HaveOccurred()) g.Expect(status).To(Equal("READY")) @@ -117,14 +155,17 @@ var _ = Describe("OLM Upgrade", Ordered, func() { Expect(err).NotTo(HaveOccurred(), "Failed to create OperatorGroup") By("creating a Subscription to trigger installation") - err = utils.KubectlApply(subscriptionTemplate, utils.SubscriptionTemplateData{ + err = utils.KubectlApply(subscriptionTemplate, struct { + utils.SubscriptionTemplateData + StartingCSV string + }{SubscriptionTemplateData: utils.SubscriptionTemplateData{ Name: subscriptionName, Namespace: operatorNamespace, - Channel: channelName, - PackageName: packageName, + Channel: upgradeSettings.baseChannel, + PackageName: upgradeSettings.packageName, CatalogSourceName: catalogName, - CatalogSourceNamespace: olmNamespace, - }) + CatalogSourceNamespace: upgradeSettings.catalogNamespace, + }, StartingCSV: upgradeSettings.expectedBaseCSV}) Expect(err).NotTo(HaveOccurred(), "Failed to create Subscription") By("waiting for subscription to report an installed CSV") @@ -133,6 +174,9 @@ var _ = Describe("OLM Upgrade", Ordered, func() { "jsonpath={.status.installedCSV}") g.Expect(err).NotTo(HaveOccurred()) g.Expect(csv).NotTo(BeEmpty()) + if upgradeSettings.expectedBaseCSV != "" { + g.Expect(csv).To(Equal(upgradeSettings.expectedBaseCSV)) + } baseCSV = csv }, 5*time.Minute, 5*time.Second).Should(Succeed(), "Subscription should report an installed CSV") @@ -150,8 +194,7 @@ var _ = Describe("OLM Upgrade", Ordered, func() { By("waiting for controller deployment to be available") Eventually(func() error { - return utils.KubectlWait("deployment", "model-validation-controller-manager", - operatorNamespace, "condition=Available", "30s") + return controllerRolledOut(upgradeSettings.expectedBaseImage) }, 3*time.Minute, 10*time.Second).Should(Succeed(), "Controller deployment should be available") @@ -167,15 +210,18 @@ var _ = Describe("OLM Upgrade", Ordered, func() { It("should have a functioning webhook after OLM installation", func() { By("deploying a ModelValidation CR") + Expect(ensureResourceAbsent("modelvalidation", testModelName, webhookTestNamespace)).To(Succeed()) + createdCR = true err := utils.KubectlApply(modelValidationTemplate, utils.CRTemplateData{ ModelName: testModelName, Namespace: webhookTestNamespace, }) Expect(err).NotTo(HaveOccurred(), "Failed to apply ModelValidation CR") + Expect(utils.KubectlGetJSON("modelvalidation", testModelName, webhookTestNamespace, &preservedCR)).To(Succeed()) + Expect(preservedCR.GetUID()).NotTo(BeEmpty()) By("deploying a test pod with model validation label") - err = utils.KubectlApply(podTemplate, utils.DefaultPodData( - "olm-test-pod-initial", webhookTestNamespace, testModelName, "olm-upgrade")) + err = utils.KubectlApply(podTemplate, olmProbeData("olm-test-pod-initial", testModelName)) Expect(err).NotTo(HaveOccurred(), "Failed to deploy test pod") By("verifying the init container is injected by the webhook") @@ -189,6 +235,9 @@ var _ = Describe("OLM Upgrade", Ordered, func() { Equal(constants.ModelValidationInitContainerName)) }, 1*time.Minute, 5*time.Second).Should(Succeed(), "Init container should be injected") + if upgradeSettings.validateModels { + expectModelVerified("olm-test-pod-initial") + } }) It("should upgrade when the catalog is updated", func() { @@ -204,14 +253,18 @@ var _ = Describe("OLM Upgrade", Ordered, func() { }, 1*time.Minute, 5*time.Second).Should(Succeed()) _, _ = fmt.Fprintf(GinkgoWriter, " Pre-upgrade image: %s\n", preUpgradeImage) - By(fmt.Sprintf("patching CatalogSource to use target catalog: %s", targetCatalogImage)) - err := utils.KubectlPatch("catalogsource", catalogName, olmNamespace, "merge", - fmt.Sprintf(`{"spec":{"image":"%s"}}`, targetCatalogImage)) + By(fmt.Sprintf("patching CatalogSource to use target catalog: %s", upgradeSettings.targetCatalog)) + err := utils.KubectlPatch("catalogsource", catalogName, upgradeSettings.catalogNamespace, "merge", + jsonMergePatch(map[string]any{"spec": map[string]string{"image": upgradeSettings.targetCatalog}})) Expect(err).NotTo(HaveOccurred(), "Failed to patch CatalogSource") + if upgradeSettings.targetChannel != upgradeSettings.baseChannel { + Expect(utils.KubectlPatch("subscription", subscriptionName, operatorNamespace, "merge", + jsonMergePatch(map[string]any{"spec": map[string]string{"channel": upgradeSettings.targetChannel}}))).To(Succeed()) + } By("waiting for CatalogSource to reconnect with updated catalog") Eventually(func(g Gomega) { - status, err := utils.KubectlGet("catalogsource", catalogName, olmNamespace, + status, err := utils.KubectlGet("catalogsource", catalogName, upgradeSettings.catalogNamespace, "jsonpath={.status.connectionState.lastObservedState}") g.Expect(err).NotTo(HaveOccurred()) g.Expect(status).To(Equal("READY")) @@ -225,13 +278,13 @@ var _ = Describe("OLM Upgrade", Ordered, func() { g.Expect(err).NotTo(HaveOccurred()) g.Expect(csv).NotTo(BeEmpty()) g.Expect(csv).NotTo(Equal(baseCSV), "installed CSV should change after upgrade") + if upgradeSettings.expectedTargetCSV != "" { + g.Expect(csv).To(Equal(upgradeSettings.expectedTargetCSV)) + } + upgradeCSV = csv }, 5*time.Minute, 10*time.Second).Should(Succeed(), "Subscription should report a new installed CSV after upgrade") - var upgradeCSV string - upgradeCSV, err = utils.KubectlGet("subscription", subscriptionName, operatorNamespace, - "jsonpath={.status.installedCSV}") - Expect(err).NotTo(HaveOccurred()) _, _ = fmt.Fprintf(GinkgoWriter, " Upgraded CSV: %s (was: %s)\n", upgradeCSV, baseCSV) By(fmt.Sprintf("waiting for upgrade CSV %s to reach Succeeded phase", upgradeCSV)) @@ -245,8 +298,7 @@ var _ = Describe("OLM Upgrade", Ordered, func() { By("waiting for controller deployment to be available after upgrade") Eventually(func() error { - return utils.KubectlWait("deployment", "model-validation-controller-manager", - operatorNamespace, "condition=Available", "30s") + return controllerRolledOut(upgradeSettings.expectedTargetImage) }, 3*time.Minute, 10*time.Second).Should(Succeed(), "Controller deployment should be available after upgrade") @@ -265,9 +317,10 @@ var _ = Describe("OLM Upgrade", Ordered, func() { It("should maintain functionality after upgrade", func() { By("verifying existing ModelValidation CR survived the upgrade") - Expect(utils.KubectlResourceExists("modelvalidation", testModelName, - webhookTestNamespace)).To(BeTrue(), - "Existing ModelValidation CR should persist through upgrade") + var upgradedCR unstructured.Unstructured + Expect(utils.KubectlGetJSON("modelvalidation", testModelName, webhookTestNamespace, &upgradedCR)).To(Succeed()) + Expect(upgradedCR.GetUID()).To(Equal(preservedCR.GetUID()), "CR must not be deleted and recreated") + Expect(upgradedCR.Object["spec"]).To(Equal(preservedCR.Object["spec"]), "CR spec must be preserved") By("verifying controller pod is running after upgrade") Eventually(func(g Gomega) { @@ -280,8 +333,7 @@ var _ = Describe("OLM Upgrade", Ordered, func() { By("deploying a new test pod to verify webhook still works after upgrade") Eventually(func() error { - return utils.KubectlApply(podTemplate, utils.DefaultPodData( - "olm-test-pod-upgrade", webhookTestNamespace, testModelName, "olm-upgrade")) + return utils.KubectlApply(podTemplate, olmProbeData("olm-test-pod-upgrade", testModelName)) }, 2*time.Minute, 5*time.Second).Should(Succeed(), "Post-upgrade test pod should be accepted by webhook") @@ -296,17 +348,37 @@ var _ = Describe("OLM Upgrade", Ordered, func() { Equal(constants.ModelValidationInitContainerName)) }, 1*time.Minute, 5*time.Second).Should(Succeed(), "Init container should be injected after upgrade") + if upgradeSettings.validateModels { + expectModelVerified("olm-test-pod-upgrade") + } }) - It("should allow CR deletion after upgrade", func() { + It("should allow CR deletion and subsequent Pod cleanup after upgrade", func() { By("deleting the ModelValidation CR") - cleanupResource("modelvalidation", testModelName, webhookTestNamespace) + Expect(cleanupResource("modelvalidation", testModelName, webhookTestNamespace)).To(Succeed()) By("verifying the ModelValidation CR is deleted") - Eventually(func() bool { - return utils.KubectlResourceExists("modelvalidation", testModelName, webhookTestNamespace) - }, 1*time.Minute, 5*time.Second).Should(BeFalse(), + Eventually(func(g Gomega) { + output, err := getResourceIfPresent("modelvalidation", testModelName, webhookTestNamespace) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(BeEmpty()) + }, 1*time.Minute, 5*time.Second).Should(Succeed(), "ModelValidation CR should be deleted after upgrade") + + By("deleting both old and new Pods after their ModelValidation CR is gone") + podTargets := []cleanupTarget{ + {"pod", "olm-test-pod-initial", webhookTestNamespace}, + {"pod", "olm-test-pod-upgrade", webhookTestNamespace}, + } + Expect(cleanupAllResources(podTargets, cleanupResource)).To(Succeed(), + "Controller finalizer cleanup must work after the referenced CR has been deleted") + Eventually(func(g Gomega) { + for _, target := range podTargets { + output, err := getResourceIfPresent(target.resource, target.name, target.namespace) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(BeEmpty(), "Pod %s must not remain stuck terminating", target.name) + } + }, time.Minute, 5*time.Second).Should(Succeed()) }) }) diff --git a/test/e2e-olm/testdata.go b/test/e2e-olm/testdata.go index c97685ab..095a2f31 100644 --- a/test/e2e-olm/testdata.go +++ b/test/e2e-olm/testdata.go @@ -1,3 +1,5 @@ +//go:build upgrade + /* Copyright 2025. diff --git a/test/e2e-olm/testdata/catalogsource_template.yaml b/test/e2e-olm/testdata/catalogsource_template.yaml index 08b7f76a..14fdc0a8 100644 --- a/test/e2e-olm/testdata/catalogsource_template.yaml +++ b/test/e2e-olm/testdata/catalogsource_template.yaml @@ -5,6 +5,8 @@ metadata: namespace: {{.Namespace}} spec: sourceType: grpc + grpcPodConfig: + securityContextConfig: restricted image: {{.Image}} displayName: Model Validation Operator Test Catalog publisher: E2E Tests diff --git a/test/e2e-olm/testdata/pod_template.yaml b/test/e2e-olm/testdata/pod_template.yaml index 34a888d3..ea03ca0a 100644 --- a/test/e2e-olm/testdata/pod_template.yaml +++ b/test/e2e-olm/testdata/pod_template.yaml @@ -7,6 +7,12 @@ metadata: validation.ml.sigstore.dev/ml: "{{.ModelName}}"{{if .TestBatch}} test-batch: "{{.TestBatch}}"{{end}} spec: + terminationGracePeriodSeconds: 5 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault restartPolicy: Never containers: - name: test-container @@ -29,4 +35,4 @@ spec: - name: {{.Name}}{{if .HostPath}} hostPath: path: {{.HostPath.Path}} - type: {{if .HostPath.Type}}{{.HostPath.Type}}{{else}}Directory{{end}}{{end}}{{end}}{{end}} \ No newline at end of file + type: {{if .HostPath.Type}}{{.HostPath.Type}}{{else}}Directory{{end}}{{end}}{{end}}{{end}} diff --git a/test/e2e-olm/testdata/subscription_template.yaml b/test/e2e-olm/testdata/subscription_template.yaml index dec7eccc..c4e11077 100644 --- a/test/e2e-olm/testdata/subscription_template.yaml +++ b/test/e2e-olm/testdata/subscription_template.yaml @@ -8,4 +8,7 @@ spec: name: {{.PackageName}} source: {{.CatalogSourceName}} sourceNamespace: {{.CatalogSourceNamespace}} +{{- if .StartingCSV}} + startingCSV: {{.StartingCSV}} +{{- end}} installPlanApproval: Automatic diff --git a/test/e2e/auth_validation_helpers_test.go b/test/e2e/auth_validation_helpers_test.go new file mode 100644 index 00000000..5382fb27 --- /dev/null +++ b/test/e2e/auth_validation_helpers_test.go @@ -0,0 +1,78 @@ +//go:build integration + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "os/exec" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive + corev1 "k8s.io/api/core/v1" + + "github.com/sigstore/model-validation-operator/internal/constants" + utils "github.com/sigstore/model-validation-operator/test/utils" +) + +func expectSuccessfulValidation(podName string) { + By("verifying the real validation agent completes successfully") + Eventually(func(g Gomega) { + var pod corev1.Pod + g.Expect(utils.KubectlGetJSON("pod", podName, webhookTestNamespace, &pod)).To(Succeed()) + g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning)) + g.Expect(pod.Status.InitContainerStatuses).ToNot(BeEmpty()) + terminated := pod.Status.InitContainerStatuses[0].State.Terminated + g.Expect(terminated).NotTo(BeNil()) + g.Expect(terminated.ExitCode).To(BeZero()) + }, 90*time.Second, 3*time.Second).Should(Succeed()) + + output, err := utils.Run(exec.Command("kubectl", "logs", podName, "-n", webhookTestNamespace, + "-c", constants.ModelValidationInitContainerName)) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("Verification succeeded")) +} + +func deleteE2EResource(resource, name string) { + _, err := utils.Run(exec.Command("kubectl", "delete", resource, name, "-n", webhookTestNamespace, + "--ignore-not-found=true", "--timeout=45s", "--request-timeout=50s")) + Expect(err).NotTo(HaveOccurred(), "cleanup of %s/%s failed", resource, name) +} + +func authFixturePodData(podName, modelName string) utils.PodTemplateData { + return utils.DefaultPodData(podName, webhookTestNamespace, modelName, "auth-validation") +} + +func authModelValidation(name, config, modelPath, signaturePath string, continuousValidation string) []byte { + ignorePaths := fmt.Sprintf(" ignorePaths:\n - %s/ignore-me\n", modelPath) + if modelPath == "/data" { + ignorePaths = "" + } + return []byte(fmt.Sprintf(`apiVersion: ml.sigstore.dev/v1alpha1 +kind: ModelValidation +metadata: + name: %s + namespace: %s +spec: + config: +%s + model: + path: %s + signaturePath: %s +%s imagePullPolicy: IfNotPresent +%s`, name, webhookTestNamespace, config, modelPath, signaturePath, ignorePaths, continuousValidation)) +} diff --git a/test/e2e/continuous_validation_test.go b/test/e2e/continuous_validation_test.go new file mode 100644 index 00000000..8c11067c --- /dev/null +++ b/test/e2e/continuous_validation_test.go @@ -0,0 +1,87 @@ +//go:build integration + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os/exec" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive + corev1 "k8s.io/api/core/v1" + + "github.com/sigstore/model-validation-operator/internal/constants" + utils "github.com/sigstore/model-validation-operator/test/utils" +) + +var _ = Describe("ContinuousValidation lifecycle", func() { + It("injects a native sidecar, re-validates at the configured interval, and terminates cleanly", func() { + const ( + modelName = "continuous-validation-model" + podName = "continuous-validation-pod" + ) + + DeferCleanup(func() { + deleteE2EResource("pod", podName) + deleteE2EResource("modelvalidation", modelName) + }) + + continuous := ` continuousValidation: + enabled: true + interval: 1m +` + config := ` publicKeyConfig: + keyPath: /keys/test_public_key.pub` + Expect(utils.KubectlApply(authModelValidation(modelName, config, "/data", "/data/model.sig", + continuous), nil)).To(Succeed()) + Expect(utils.KubectlApply(podTemplate, authFixturePodData(podName, modelName))).To(Succeed()) + + By("verifying the native sidecar contract") + Eventually(func(g Gomega) { + var pod corev1.Pod + g.Expect(utils.KubectlGetJSON("pod", podName, webhookTestNamespace, &pod)).To(Succeed()) + g.Expect(pod.Spec.InitContainers).To(HaveLen(1)) + sidecar := pod.Spec.InitContainers[0] + g.Expect(sidecar.RestartPolicy).NotTo(BeNil()) + g.Expect(*sidecar.RestartPolicy).To(Equal(corev1.ContainerRestartPolicyAlways)) + g.Expect(sidecar.Args).To(ContainElement("--interval=1m")) + g.Expect(sidecar.ReadinessProbe).NotTo(BeNil()) + g.Expect(sidecar.ReadinessProbe.HTTPGet).NotTo(BeNil()) + g.Expect(sidecar.LivenessProbe).NotTo(BeNil()) + g.Expect(sidecar.LivenessProbe.HTTPGet).NotTo(BeNil()) + g.Expect(sidecar.ReadinessProbe.HTTPGet.Path).To(Equal("/ready")) + g.Expect(sidecar.LivenessProbe.HTTPGet.Path).To(Equal("/healthz")) + g.Expect(pod.Annotations).To(HaveKeyWithValue(constants.ContinuousValidationAnnotationKey, "true")) + g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning)) + }, 90*time.Second, 3*time.Second).Should(Succeed()) + + By("waiting for a periodic re-validation after the configured one-minute interval") + Eventually(func(g Gomega) { + output, err := utils.Run(exec.Command("kubectl", "logs", podName, "-n", webhookTestNamespace, + "-c", constants.ModelValidationInitContainerName)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Running periodic validation")) + g.Expect(output).To(ContainSubstring("Validation successful")) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + + By("deleting the pod and verifying the sidecar and tracking finalizer do not block cleanup") + deleteE2EResource("pod", podName) + Eventually(func() (bool, error) { + return utils.KubectlResourceAbsent("pod", podName, webhookTestNamespace) + }, 45*time.Second, 2*time.Second).Should(BeTrue()) + }) +}) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 61b8fd68..76e89f33 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,3 +1,5 @@ +//go:build integration || ha + // Copyright 2025 The Sigstore Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +18,8 @@ package e2e import ( + "context" + "errors" "fmt" "os/exec" "strings" @@ -99,31 +103,35 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { By("cleaning up test resources") - - // Clean up any test resources that may have been created during tests - By("cleaning up test pods and CRs") - cmd := exec.Command("kubectl", "delete", "pods", "--all", "-n", webhookTestNamespace, - "--timeout=30s", "--ignore-not-found=true") - _, _ = utils.Run(cmd) - - cmd = exec.Command("kubectl", "delete", "modelvalidations", "--all", "-n", webhookTestNamespace, - "--timeout=30s", "--ignore-not-found=true") - _, _ = utils.Run(cmd) - - By("cleaning up persistent metrics pod") - cleanupTemplateData := utils.CurlPodTemplateData{ - PodName: curlMetricsPodName, - Namespace: operatorNamespace, - ServiceAccount: utils.MetricsServiceAccountName, + // The workload namespace is dedicated to this suite, but fixture DaemonSet + // Pods are reusable infrastructure, not test workloads. Metrics resources are + // individually named because the operator namespace also hosts the manager. + // Attempt every step before failing, so one stuck finalizer does not prevent + // cleanup of unrelated test objects. Bound both API requests and each process. + steps := []struct { + name string + args []string + }{ + {name: "test pods", args: []string{"pods", "--selector=app!=model-data-setup", "-n", webhookTestNamespace}}, + {name: "test CRs", args: []string{"modelvalidations", "--all", "-n", webhookTestNamespace}}, + {name: "metrics pod", args: []string{"pod", curlMetricsPodName, "-n", operatorNamespace}}, + {name: "metrics service account", args: []string{"serviceaccount", utils.MetricsServiceAccountName, + "-n", operatorNamespace}}, + {name: "metrics role binding", args: []string{"clusterrolebinding", utils.MetricsServiceAccountName + "-binding"}}, } - if err := utils.KubectlDelete(curlPodTemplate, &utils.KubectlDeleteOptions{ - IgnoreNotFound: true, - Timeout: "30s", - TemplateData: cleanupTemplateData, - }); err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "Failed to cleanup metrics pod: %s\n", err) + var cleanupErrors []error + for _, step := range steps { + By("cleaning up " + step.name) + args := append([]string{"delete"}, step.args...) + args = append(args, "--timeout=30s", "--request-timeout=15s", "--ignore-not-found=true") + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + _, err := utils.Run(exec.CommandContext(ctx, "kubectl", args...)) + cancel() + if err != nil { + cleanupErrors = append(cleanupErrors, fmt.Errorf("%s: %w", step.name, err)) + } } - + Expect(errors.Join(cleanupErrors...)).To(Succeed(), "suite cleanup failed after attempting every cleanup step") _, _ = fmt.Fprintf(GinkgoWriter, "Test cleanup complete.\n") }) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c9ae26af..3f819ba5 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1,3 +1,5 @@ +//go:build integration + // Copyright 2025 The Sigstore Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,11 +32,11 @@ import ( // metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data const metricsRoleBindingName = "model-validation-operator-metrics-binding" -// curlMetricsPodName is the name of the pod used to access the metrics endpoint -const curlMetricsPodName = "curl-metrics" - const e2eTestPodName = "e2e-test-pod" +//go:embed testdata/clusterrolebinding_template.yaml +var clusterRoleBindingTemplate []byte + var _ = Describe("Manager", Ordered, func() { Context("Manager", func() { It("should run successfully", func() { @@ -53,7 +55,7 @@ var _ = Describe("Manager", Ordered, func() { podOutput, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") podNames := utils.GetNonEmptyLines(podOutput) - g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") + g.Expect(podNames).NotTo(BeEmpty(), "expected at least 1 controller pod running") controllerPodName = podNames[0] g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) diff --git a/test/e2e/ha_test.go b/test/e2e/ha_test.go new file mode 100644 index 00000000..d6a46525 --- /dev/null +++ b/test/e2e/ha_test.go @@ -0,0 +1,466 @@ +//go:build ha + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive + appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + + "github.com/sigstore/model-validation-operator/api/v1alpha1" + "github.com/sigstore/model-validation-operator/internal/constants" + utils "github.com/sigstore/model-validation-operator/test/utils" +) + +const ( + haDeploymentName = "model-validation-controller-manager" + haModelName = "ha-validation-model" + haPDBName = "controller-manager" +) + +func haManagerPods() ([]corev1.Pod, error) { + var deployment appsv1.Deployment + if err := utils.KubectlGetJSON("deployment", haDeploymentName, operatorNamespace, &deployment); err != nil { + return nil, err + } + selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) + if err != nil { + return nil, err + } + output, err := utils.Run(exec.Command("kubectl", "get", "pods", "-n", operatorNamespace, + "-l", selector.String(), "--request-timeout=15s", "-o", "json")) + if err != nil { + return nil, err + } + var pods corev1.PodList + err = json.Unmarshal([]byte(output), &pods) + return pods.Items, err +} + +func haPodReady(pod corev1.Pod) bool { + if pod.DeletionTimestamp != nil { + return false + } + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + +// A completed Deployment rollout can leave the previous Pod's lease identity +// behind until its lease expires. Require a live, Ready manager and re-read the +// lease after looking it up before selecting a failover victim. +func haCurrentReadyLeader(leaseName string) (string, corev1.Pod, error) { + var lease coordinationv1.Lease + if err := utils.KubectlGetJSON("lease", leaseName, operatorNamespace, &lease); err != nil { + return "", corev1.Pod{}, err + } + if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "" { + return "", corev1.Pod{}, fmt.Errorf("leader lease has no holder") + } + holder := *lease.Spec.HolderIdentity + pods, err := haManagerPods() + if err != nil { + return "", corev1.Pod{}, err + } + for _, pod := range pods { + if pod.Name != strings.SplitN(holder, "_", 2)[0] || !haPodReady(pod) { + continue + } + var latest coordinationv1.Lease + if err := utils.KubectlGetJSON("lease", leaseName, operatorNamespace, &latest); err != nil { + return "", corev1.Pod{}, err + } + if latest.UID != lease.UID || latest.Spec.HolderIdentity == nil || *latest.Spec.HolderIdentity != holder { + return "", corev1.Pod{}, fmt.Errorf("leader changed while selecting the failover victim") + } + return holder, pod, nil + } + return "", corev1.Pod{}, fmt.Errorf("lease holder %s is not a live Ready manager", holder) +} + +// Include observedGeneration, exact replica counts and Pod UIDs: old status or +// terminating Pods must not make a rolling restart appear complete. +func haRolloutComplete(generation int64, oldUIDs map[types.UID]bool) error { + var deployment appsv1.Deployment + if err := utils.KubectlGetJSON("deployment", haDeploymentName, operatorNamespace, &deployment); err != nil { + return err + } + want := *deployment.Spec.Replicas + status := deployment.Status + if deployment.Generation < generation || status.ObservedGeneration < deployment.Generation || + status.Replicas != want || status.UpdatedReplicas != want || status.ReadyReplicas != want || + status.AvailableReplicas != want || status.UnavailableReplicas != 0 { + return fmt.Errorf("deployment generation %d observed %d is not fully rolled out: %+v", + deployment.Generation, status.ObservedGeneration, status) + } + pods, err := haManagerPods() + if err != nil { + return err + } + if len(pods) != int(want) { + return fmt.Errorf("expected %d manager Pods including terminating Pods, got %d", want, len(pods)) + } + for _, pod := range pods { + if oldUIDs[pod.UID] || !haPodReady(pod) { + return fmt.Errorf("manager Pod %s is old or not ready", pod.Name) + } + } + return nil +} + +func waitForHAReplicas() { + Eventually(func() error { return haRolloutComplete(0, nil) }, 3*time.Minute, 2*time.Second).Should(Succeed()) +} + +func expectHAManagerSpread(pods []corev1.Pod) { + Expect(pods).To(HaveLen(2)) + Expect(pods[0].Spec.NodeName).NotTo(BeEmpty()) + Expect(pods[1].Spec.NodeName).NotTo(BeEmpty()) + Expect(pods[0].Spec.NodeName).NotTo(Equal(pods[1].Spec.NodeName), + "healthy manager replicas must occupy different nodes after a completed rollout") +} + +// Server-side dry run invokes the real admission webhook without creating +// unlimited workloads during a slow drain/rollout. +func haAdmissionProbe() error { + pod := corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "ha-admission-probe-", Namespace: webhookTestNamespace, + Labels: map[string]string{constants.ModelValidationLabel: haModelName}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "workload", Image: "busybox:latest"}}}, + } + body, err := json.Marshal(pod) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "kubectl", "create", "--dry-run=server", "--request-timeout=15s", + "-f", "-", "-o", "json") + cmd.Stdin = bytes.NewReader(body) + // API admission warnings go to stderr and are not part of the Pod JSON. + var stderr bytes.Buffer + cmd.Stderr = &stderr + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("admission probe failed: %w: %s", err, stderr.String()) + } + if err := json.Unmarshal(output, &pod); err != nil { + return err + } + if !utils.HasValidationContainer(&pod) { + return fmt.Errorf("admission succeeded without injecting the validation container") + } + return nil +} + +func probeDuringHAChange(change func(), complete func() error) { + Expect(haAdmissionProbe()).To(Succeed(), "pre-disruption admission probe") + type probeResult struct { + count int + failures []string + } + stop := make(chan struct{}) + done := make(chan probeResult, 1) + go func() { + result := probeResult{} + for { + select { + case <-stop: + done <- result + return + default: + } + result.count++ + if err := haAdmissionProbe(); err != nil { + result.failures = append(result.failures, fmt.Sprintf("%s: %v", time.Now().UTC().Format(time.RFC3339Nano), err)) + } + timer := time.NewTimer(250 * time.Millisecond) + select { + case <-stop: + timer.Stop() + done <- result + return + case <-timer.C: + } + } + }() + // Also stop traffic if an assertion in change/completion fails. + defer func() { + close(stop) + result := <-done + AddReportEntry("admission probes during disruption", result.count) + Expect(result.count).To(BeNumerically(">", 0)) + Expect(result.failures).To(BeEmpty(), "admission failures must not be retried away") + }() + change() + Eventually(complete, 4*time.Minute, 2*time.Second).Should(Succeed()) + Expect(haAdmissionProbe()).To(Succeed(), "post-recovery admission probe") +} + +func haEvict(pod corev1.Pod) error { + eviction := policyv1.Eviction{ + TypeMeta: metav1.TypeMeta{APIVersion: "policy/v1", Kind: "Eviction"}, + ObjectMeta: metav1.ObjectMeta{Name: pod.Name, Namespace: operatorNamespace}, + DeleteOptions: &metav1.DeleteOptions{ + DryRun: []string{metav1.DryRunAll}, Preconditions: &metav1.Preconditions{UID: &pod.UID}, + }, + } + body, err := json.Marshal(eviction) + if err != nil { + return err + } + path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/eviction", operatorNamespace, pod.Name) + cmd := exec.Command("kubectl", "create", "--raw", path, "--request-timeout=15s", "-f", "-") + cmd.Stdin = bytes.NewReader(body) + _, err = utils.Run(cmd) + return err +} + +func expectHABudget(replicas int32, allowed int32) { + Eventually(func(g Gomega) { + var pdb policyv1.PodDisruptionBudget + g.Expect(utils.KubectlGetJSON("poddisruptionbudget", haPDBName, operatorNamespace, &pdb)).To(Succeed()) + g.Expect(pdb.Spec.MinAvailable).NotTo(BeNil()) + g.Expect(pdb.Spec.MinAvailable.IntValue()).To(Equal(1)) + g.Expect(pdb.Status.ObservedGeneration).To(BeNumerically(">=", pdb.Generation)) + g.Expect(pdb.Status.ExpectedPods).To(Equal(replicas)) + g.Expect(pdb.Status.CurrentHealthy).To(Equal(replicas)) + g.Expect(pdb.Status.DesiredHealthy).To(Equal(int32(1))) + g.Expect(pdb.Status.DisruptionsAllowed).To(Equal(allowed)) + selector, err := metav1.LabelSelectorAsSelector(pdb.Spec.Selector) + g.Expect(err).NotTo(HaveOccurred()) + pods, err := haManagerPods() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(pods).To(HaveLen(int(replicas))) + for _, pod := range pods { + g.Expect(selector.Matches(labels.Set(pod.Labels))).To(BeTrue(), "PDB does not select %s", pod.Name) + } + }, time.Minute, 2*time.Second).Should(Succeed()) +} + +func scaleHAManagers(replicas int32) { + _, err := utils.Run(exec.Command("kubectl", "scale", "deployment", haDeploymentName, + "-n", operatorNamespace, fmt.Sprintf("--replicas=%d", replicas), "--request-timeout=15s")) + Expect(err).NotTo(HaveOccurred()) + waitForHAReplicas() +} + +var _ = Describe("High availability", Ordered, Serial, func() { + BeforeAll(func() { + Expect(utils.KubectlApply(modelValidationTemplate, utils.CRTemplateData{ + ModelName: haModelName, Namespace: webhookTestNamespace, + })).To(Succeed()) + waitForHAReplicas() + pods, err := haManagerPods() + Expect(err).NotTo(HaveOccurred()) + Expect(pods).To(HaveLen(2), "HA tests require exactly two manager replicas") + }) + + AfterAll(func() { + _, err := utils.Run(exec.Command("kubectl", "delete", "pods", "-n", webhookTestNamespace, + "-l", "test-batch=ha", "--ignore-not-found=true", "--timeout=45s")) + Expect(err).NotTo(HaveOccurred()) + _, err = utils.Run(exec.Command("kubectl", "delete", "modelvalidation", haModelName, + "-n", webhookTestNamespace, "--ignore-not-found=true", "--timeout=30s")) + Expect(err).NotTo(HaveOccurred()) + }) + + It("enforces a disruption budget and spreads replicas across nodes", func() { + expectHABudget(2, 1) + var deployment appsv1.Deployment + Expect(utils.KubectlGetJSON("deployment", haDeploymentName, operatorNamespace, &deployment)).To(Succeed()) + Expect(deployment.Spec.Template.Spec.Affinity).NotTo(BeNil()) + Expect(deployment.Spec.Template.Spec.Affinity.PodAntiAffinity).NotTo(BeNil()) + Expect(deployment.Spec.Template.Spec.Affinity.PodAntiAffinity. + PreferredDuringSchedulingIgnoredDuringExecution).To(ContainElement( + HaveField("PodAffinityTerm", And( + HaveField("TopologyKey", "kubernetes.io/hostname"), + HaveField("MatchLabelKeys", ContainElement("pod-template-hash")), + )), + ), "anti-affinity must spread each revision independently during rolling updates") + pods, err := haManagerPods() + Expect(err).NotTo(HaveOccurred()) + expectHAManagerSpread(pods) + Expect(haEvict(pods[0])).To(Succeed(), "a healthy two-replica budget should allow an eviction") + + By("exhausting the budget and proving the Eviction API rejects disruption of the last healthy replica") + DeferCleanup(scaleHAManagers, int32(2)) + scaleHAManagers(1) + expectHABudget(1, 0) + pods, err = haManagerPods() + Expect(err).NotTo(HaveOccurred()) + err = haEvict(pods[0]) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("disruption budget")) + scaleHAManagers(2) + expectHABudget(2, 1) + }) + + It("keeps the webhook available when one replica is killed", func() { + pods, err := haManagerPods() + Expect(err).NotTo(HaveOccurred()) + victim := pods[0] + probeDuringHAChange(func() { + _, deleteErr := utils.Run(exec.Command("kubectl", "delete", "pod", victim.Name, + "-n", operatorNamespace, "--wait=false", "--request-timeout=15s")) + Expect(deleteErr).NotTo(HaveOccurred()) + }, func() error { return haRolloutComplete(0, map[types.UID]bool{victim.UID: true}) }) + }) + + It("allows a voluntary node drain while honoring the PDB", func() { + Expect(os.Getenv("E2E_ALLOW_NODE_DRAIN")).To(Equal("true"), + "node drain is disruptive: explicitly set E2E_ALLOW_NODE_DRAIN=true on an isolated test cluster") + selectorText := os.Getenv("E2E_DRAIN_NODE_SELECTOR") + Expect(selectorText).NotTo(BeEmpty(), "set E2E_DRAIN_NODE_SELECTOR to explicitly approved test workers") + selector, err := labels.Parse(selectorText) + Expect(err).NotTo(HaveOccurred()) + Expect(selector.Empty()).To(BeFalse(), "an empty node selector would authorize all nodes") + pods, err := haManagerPods() + Expect(err).NotTo(HaveOccurred()) + var target *corev1.Node + for _, pod := range pods { + var node corev1.Node + Expect(utils.KubectlGetJSON("node", pod.Spec.NodeName, "", &node)).To(Succeed()) + _, controlPlane := node.Labels["node-role.kubernetes.io/control-plane"] + _, master := node.Labels["node-role.kubernetes.io/master"] + if selector.Matches(labels.Set(node.Labels)) && !controlPlane && !master { + target = &node + break + } + } + Expect(target).NotTo(BeNil(), "no manager is on an explicitly approved non-control-plane node") + DeferCleanup(func() { + action := "uncordon" + if target.Spec.Unschedulable { + action = "cordon" + } + _, restoreErr := utils.Run(exec.Command("kubectl", action, target.Name, "--request-timeout=15s")) + Expect(restoreErr).NotTo(HaveOccurred(), "restore original scheduling state of %s", target.Name) + waitForHAReplicas() + }) + expectHABudget(2, 1) + probeDuringHAChange(func() { + _, drainErr := utils.Run(exec.Command("kubectl", "drain", target.Name, "--ignore-daemonsets", + "--pod-selector=app.kubernetes.io/name=model-validation-operator,control-plane=controller-manager", + "--delete-emptydir-data", "--timeout=180s")) + Expect(drainErr).NotTo(HaveOccurred()) + }, func() error { return haRolloutComplete(0, nil) }) + }) + + It("retains webhook availability throughout a rolling update", func() { + pods, err := haManagerPods() + Expect(err).NotTo(HaveOccurred()) + oldUIDs := make(map[types.UID]bool, len(pods)) + for _, pod := range pods { + oldUIDs[pod.UID] = true + } + var before appsv1.Deployment + Expect(utils.KubectlGetJSON("deployment", haDeploymentName, operatorNamespace, &before)).To(Succeed()) + stamp := time.Now().UTC().Format(time.RFC3339Nano) + probeDuringHAChange(func() { + Expect(utils.KubectlPatch("deployment", haDeploymentName, operatorNamespace, "merge", + fmt.Sprintf(`{"spec":{"template":{"metadata":{"annotations":{"ha-test/restartedAt":%q}}}}}`, stamp))).To(Succeed()) + }, func() error { return haRolloutComplete(before.Generation+1, oldUIDs) }) + var after appsv1.Deployment + Expect(utils.KubectlGetJSON("deployment", haDeploymentName, operatorNamespace, &after)).To(Succeed()) + const revision = "deployment.kubernetes.io/revision" + Expect(after.Annotations[revision]).NotTo(BeEmpty()) + Expect(after.Annotations[revision]).NotTo(Equal(before.Annotations[revision])) + pods, err = haManagerPods() + Expect(err).NotTo(HaveOccurred()) + expectHAManagerSpread(pods) + }) + + It("transfers the leader lease and resumes reconciliation and finalizer cleanup", func() { + const leaseName = "2b2639c9.sigstore.dev" + var oldHolder string + var victim corev1.Pod + Eventually(func() error { + var getErr error + oldHolder, victim, getErr = haCurrentReadyLeader(leaseName) + return getErr + }, 90*time.Second, 2*time.Second).Should(Succeed(), "wait for leadership to settle after prior disruptions") + + // Raw DELETE accepts Kubernetes DeleteOptions, including a UID precondition. + // Never mistake a stale name / NotFound for successfully killing a leader. + deleteOptions := metav1.DeleteOptions{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "DeleteOptions"}, + Preconditions: &metav1.Preconditions{UID: &victim.UID}, + } + body, err := json.Marshal(deleteOptions) + Expect(err).NotTo(HaveOccurred()) + path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", operatorNamespace, victim.Name) + deleteCmd := exec.Command("kubectl", "delete", "--raw", path, "--request-timeout=15s", "-f", "-") + deleteCmd.Stdin = bytes.NewReader(body) + _, err = utils.Run(deleteCmd) + Expect(err).NotTo(HaveOccurred(), "delete the selected live leader, not a stale lease holder") + Eventually(func(g Gomega) { + newHolder, leader, getErr := haCurrentReadyLeader(leaseName) + g.Expect(getErr).NotTo(HaveOccurred()) + g.Expect(newHolder).NotTo(Equal(oldHolder)) + g.Expect(leader.UID).NotTo(Equal(victim.UID)) + }, 90*time.Second, 2*time.Second).Should(Succeed()) + + const podName = "ha-leader-failover" + Expect(utils.KubectlApply(podTemplate, + utils.DefaultPodData(podName, webhookTestNamespace, haModelName, "ha"))).To(Succeed()) + var pod corev1.Pod + Expect(utils.KubectlGetJSON("pod", podName, webhookTestNamespace, &pod)).To(Succeed()) + Expect(pod.Finalizers).To(ContainElement(constants.ModelValidationFinalizer)) + Eventually(func(g Gomega) { + var mv v1alpha1.ModelValidation + g.Expect(utils.KubectlGetJSON("modelvalidation", haModelName, webhookTestNamespace, &mv)).To(Succeed()) + g.Expect(mv.Status.InjectedPodCount).To(Equal(int32(1))) + g.Expect(mv.Status.InjectedPods).To(ContainElement(And( + HaveField("Name", podName), HaveField("UID", string(pod.UID))))) + }, time.Minute, 2*time.Second).Should(Succeed()) + _, err = utils.Run(exec.Command("kubectl", "delete", "pod", podName, "-n", webhookTestNamespace, + "--wait=false", "--request-timeout=15s")) + Expect(err).NotTo(HaveOccurred()) + Eventually(func(g Gomega) { + absent, getErr := utils.KubectlResourceAbsent("pod", podName, webhookTestNamespace) + g.Expect(getErr).NotTo(HaveOccurred()) + g.Expect(absent).To(BeTrue(), "the new leader must remove the tracking finalizer") + var mv v1alpha1.ModelValidation + g.Expect(utils.KubectlGetJSON("modelvalidation", haModelName, webhookTestNamespace, &mv)).To(Succeed()) + g.Expect(mv.Status.InjectedPodCount).To(BeZero()) + g.Expect(mv.Status.InjectedPods).To(BeEmpty()) + }, time.Minute, 2*time.Second).Should(Succeed()) + waitForHAReplicas() + }) +}) diff --git a/test/e2e/integration_test.go b/test/e2e/integration_test.go index 7f71f65e..38eeca66 100644 --- a/test/e2e/integration_test.go +++ b/test/e2e/integration_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright 2025. diff --git a/test/e2e/metrics_template_test.go b/test/e2e/metrics_template_test.go new file mode 100644 index 00000000..2323b261 --- /dev/null +++ b/test/e2e/metrics_template_test.go @@ -0,0 +1,99 @@ +//go:build integration || ha + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "errors" + "io" + "slices" + "testing" + "text/template" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/yaml" + + utils "github.com/sigstore/model-validation-operator/test/utils" +) + +// Keep the metrics helper compatible with restricted PSA and OpenShift's +// namespace-assigned UID range, even when the caller can use an elevated SCC. +func TestMetricsPodSecurityTemplate(t *testing.T) { + tmpl, err := template.New("metrics").Option("missingkey=error").Parse(string(curlPodTemplate)) + if err != nil { + t.Fatal(err) + } + data := utils.CurlPodTemplateData{ + PodName: "metrics-security-test", Namespace: "metrics-test", ServiceAccount: "metrics-reader", + } + var rendered bytes.Buffer + if err := tmpl.Execute(&rendered, data); err != nil { + t.Fatal(err) + } + decoder := yaml.NewYAMLOrJSONDecoder(&rendered, 4096) + podCount := 0 + for { + var pod corev1.Pod + if err := decoder.Decode(&pod); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + if pod.Kind != "Pod" { + continue + } + podCount++ + if pod.Name != data.PodName || pod.Namespace != data.Namespace || + (pod.Spec.ServiceAccountName != data.ServiceAccount && pod.Spec.DeprecatedServiceAccount != data.ServiceAccount) { + t.Fatalf("metrics Pod identity or service account was not rendered correctly: %+v", pod) + } + assertMetricsPodSecurity(t, pod) + } + if podCount != 1 { + t.Fatalf("expected exactly one rendered metrics Pod, got %d", podCount) + } +} + +func assertMetricsPodSecurity(t *testing.T, pod corev1.Pod) { + t.Helper() + if pod.Annotations["openshift.io/required-scc"] != "restricted-v2" { + t.Fatal("metrics Pod must explicitly require restricted-v2 rather than inherit the fixture SCC") + } + if pod.Spec.SecurityContext != nil && (pod.Spec.SecurityContext.RunAsUser != nil || + pod.Spec.SecurityContext.SELinuxOptions != nil) { + t.Fatal("Pod-level security must not override OpenShift's assigned UID or SELinux context") + } + if len(pod.Spec.Containers) != 1 { + t.Fatalf("expected one metrics helper container, got %d", len(pod.Spec.Containers)) + } + security := pod.Spec.Containers[0].SecurityContext + if security == nil || security.RunAsNonRoot == nil || !*security.RunAsNonRoot || + security.AllowPrivilegeEscalation == nil || *security.AllowPrivilegeEscalation { + t.Fatal("metrics container must require non-root execution and disable privilege escalation") + } + if security.RunAsUser != nil || security.SELinuxOptions != nil || + (security.Privileged != nil && *security.Privileged) { + t.Fatal("metrics container must not force a UID, SELinux context, or privileged execution") + } + if security.Capabilities == nil || len(security.Capabilities.Add) != 0 || + !slices.Contains(security.Capabilities.Drop, corev1.Capability("ALL")) { + t.Fatal("metrics container must drop all capabilities without adding any") + } + if security.SeccompProfile == nil || security.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Fatal("metrics container must use RuntimeDefault seccomp") + } +} diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go index 961c5d47..0b69e539 100644 --- a/test/e2e/metrics_test.go +++ b/test/e2e/metrics_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright 2025. diff --git a/test/e2e/negative_test.go b/test/e2e/negative_test.go new file mode 100644 index 00000000..75410419 --- /dev/null +++ b/test/e2e/negative_test.go @@ -0,0 +1,340 @@ +//go:build integration + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + + "github.com/sigstore/model-validation-operator/api/v1alpha1" + "github.com/sigstore/model-validation-operator/internal/constants" + utils "github.com/sigstore/model-validation-operator/test/utils" +) + +var _ = Describe("Negative and failure scenarios", Ordered, func() { + It("rejects malformed ModelValidation resources at the API boundary", func() { + malformed := []byte(fmt.Sprintf(`apiVersion: ml.sigstore.dev/v1alpha1 +kind: ModelValidation +metadata: + name: malformed-validation + namespace: %s +spec: + config: {} + model: + path: relative/path + signaturePath: "" +`, webhookTestNamespace)) + + err := utils.KubectlApply(malformed, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Or(ContainSubstring("exactly one validation method"), + ContainSubstring("spec.model.path"), ContainSubstring("spec.model.signaturePath"))) + }) + + It("fails closed when a labeled pod references a missing CR", func() { + const podName = "missing-cr-pod" + DeferCleanup(deleteE2EResource, "pod", podName) + + err := utils.KubectlApply(podTemplate, authFixturePodData(podName, "does-not-exist")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does-not-exist")) + }) + + It("fails closed while all webhook replicas are unavailable", func() { + const ( + deploymentName = "model-validation-controller-manager" + modelName = "operator-down-model" + podName = "operator-down-pod" + ) + + Expect(utils.KubectlApply(modelValidationTemplate, utils.CRTemplateData{ + ModelName: modelName, + Namespace: webhookTestNamespace, + })).To(Succeed()) + DeferCleanup(deleteE2EResource, "modelvalidation", modelName) + + replicaText, err := utils.KubectlGet("deployment", deploymentName, operatorNamespace, + "jsonpath={.spec.replicas}") + Expect(err).NotTo(HaveOccurred()) + replicas, err := strconv.Atoi(replicaText) + Expect(err).NotTo(HaveOccurred()) + + DeferCleanup(func() { + _, restoreErr := utils.Run(exec.Command("kubectl", "scale", "deployment", deploymentName, + "-n", operatorNamespace, fmt.Sprintf("--replicas=%d", replicas))) + Expect(restoreErr).NotTo(HaveOccurred()) + _, restoreErr = utils.Run(exec.Command("kubectl", "rollout", "status", "deployment/"+deploymentName, + "-n", operatorNamespace, "--timeout=120s")) + Expect(restoreErr).NotTo(HaveOccurred()) + _, restoreErr = utils.Run(exec.Command("kubectl", "wait", "pod", + "-l", "control-plane=controller-manager", "-n", operatorNamespace, + "--for=condition=Ready", "--timeout=120s")) + Expect(restoreErr).NotTo(HaveOccurred()) + Eventually(webhookEndpointAddresses, 60*time.Second, 2*time.Second).ShouldNot(BeEmpty()) + }) + + if os.Getenv("E2E_COLLECT_COVERAGE") == "true" { + By("preserving every live manager's coverage before terminating its process") + _, snapshotErr := utils.Run(exec.Command("make", "snapshot-e2e-coverage")) + Expect(snapshotErr).NotTo(HaveOccurred(), "coverage-enabled runs must retain pre-disruption counters") + } + + _, err = utils.Run(exec.Command("kubectl", "scale", "deployment", deploymentName, + "-n", operatorNamespace, "--replicas=0")) + Expect(err).NotTo(HaveOccurred()) + Eventually(webhookEndpointAddresses, 60*time.Second, 2*time.Second).Should(BeEmpty()) + + err = utils.KubectlApply(podTemplate, authFixturePodData(podName, modelName)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Or(ContainSubstring("failed calling webhook"), + ContainSubstring("no endpoints available"))) + }) + + It("detects a model corrupted after signing", func() { + const ( + modelName = "corrupted-model" + podName = "corrupted-model-pod" + ) + DeferCleanup(func() { + deleteE2EResource("pod", podName) + deleteE2EResource("modelvalidation", modelName) + }) + + Expect(utils.KubectlApply(modelValidationTemplate, utils.CRTemplateData{ + ModelName: modelName, + Namespace: webhookTestNamespace, + })).To(Succeed()) + + pod := []byte(fmt.Sprintf(`apiVersion: v1 +kind: Pod +metadata: + name: %s + namespace: %s + labels: + validation.ml.sigstore.dev/ml: %s +spec: + restartPolicy: Never + terminationGracePeriodSeconds: 1 + initContainers: + - name: corrupt-model + image: busybox:latest + command: ["sh", "-c", "cp -R /source/. /data/ && echo tampered > /data/corrupted"] + volumeMounts: + - {name: source, mountPath: /source, readOnly: true} + - {name: model-copy, mountPath: /data} + containers: + - name: workload + image: busybox:latest + command: ["sh", "-c", "sleep 3600"] + volumeMounts: + - {name: model-copy, mountPath: /data, readOnly: true} + - {name: keys, mountPath: /keys, readOnly: true} + volumes: + - name: source + hostPath: {path: /tmp/e2e-model-data, type: Directory} + - name: keys + hostPath: {path: /tmp/e2e-keys-data, type: Directory} + - name: model-copy + emptyDir: {} +`, podName, webhookTestNamespace, modelName)) + Expect(utils.KubectlApply(pod, nil)).To(Succeed()) + + Eventually(func(g Gomega) { + var current corev1.Pod + g.Expect(utils.KubectlGetJSON("pod", podName, webhookTestNamespace, ¤t)).To(Succeed()) + g.Expect(current.Status.InitContainerStatuses).To(HaveLen(2)) + validationStatus := current.Status.InitContainerStatuses[1] + g.Expect(validationStatus.State.Terminated).NotTo(BeNil()) + g.Expect(validationStatus.State.Terminated.ExitCode).NotTo(BeZero()) + }, 90*time.Second, 3*time.Second).Should(Succeed()) + + logs, err := utils.Run(exec.Command("kubectl", "logs", podName, "-n", webhookTestNamespace, + "-c", constants.ModelValidationInitContainerName)) + Expect(err).NotTo(HaveOccurred()) + Expect(logs).To(ContainSubstring("verification failed")) + }) + + It("reports Sigstore verification failure when trust services are unreachable", func() { + const ( + modelName = "sigstore-network-failure" + podName = "sigstore-network-failure-pod" + modelPath = "/fixtures/v1.1.0-sigstore" + controlModelName = "sigstore-network-control" + controlPodName = "sigstore-network-control-pod" + ) + DeferCleanup(func() { + deleteE2EResource("pod", podName) + deleteE2EResource("pod", controlPodName) + deleteE2EResource("modelvalidation", modelName) + deleteE2EResource("modelvalidation", controlModelName) + }) + + config := ` sigstoreConfig: + certificateIdentity: stefanb@us.ibm.com + certificateOidcIssuer: https://sigstore.verify.ibm.com/oauth2` + // Both Pods have the same deliberately unavailable loopback DNS server. + // Local trust allows the control to prove image pulls, files, identity and + // crypto work under that exact network fault, without public-service uptime. + localTrust := config + ` + clientTrustConfig: + trustConfigPath: /fixtures/keys/sigstore/trusted_root.json` + Expect(utils.KubectlApply(authModelValidation(controlModelName, localTrust, modelPath, + modelPath+"/model.sig", ""), nil)).To(Succeed()) + podTemplateText := `apiVersion: v1 +kind: Pod +metadata: + name: %s + namespace: %s + labels: + validation.ml.sigstore.dev/ml: %s +spec: + restartPolicy: Never + terminationGracePeriodSeconds: 1 + dnsPolicy: None + dnsConfig: + nameservers: ["127.0.0.1"] + options: + - {name: timeout, value: "1"} + - {name: attempts, value: "1"} + containers: + - name: workload + image: busybox:latest + imagePullPolicy: IfNotPresent + command: ["sh", "-c", "sleep 3600"] + volumeMounts: + - {name: fixtures, mountPath: /fixtures, readOnly: true} + volumes: + - name: fixtures + hostPath: {path: /tmp/e2e-fixture-data, type: Directory} +` + controlPod := []byte(fmt.Sprintf(podTemplateText, controlPodName, webhookTestNamespace, controlModelName)) + Expect(utils.KubectlApply(controlPod, nil)).To(Succeed()) + expectSuccessfulValidation(controlPodName) + + By("requiring a trust-root fetch and asserting an actual DNS/connectivity failure") + Expect(utils.KubectlApply(authModelValidation(modelName, config, modelPath, + modelPath+"/model.sig", ""), nil)).To(Succeed()) + pod := []byte(fmt.Sprintf(podTemplateText, podName, webhookTestNamespace, modelName)) + Expect(utils.KubectlApply(pod, nil)).To(Succeed()) + + Eventually(func(g Gomega) { + var current corev1.Pod + g.Expect(utils.KubectlGetJSON("pod", podName, webhookTestNamespace, ¤t)).To(Succeed()) + g.Expect(current.Status.InitContainerStatuses).ToNot(BeEmpty()) + terminated := current.Status.InitContainerStatuses[0].State.Terminated + g.Expect(terminated).NotTo(BeNil()) + g.Expect(terminated.ExitCode).NotTo(BeZero()) + }, 2*time.Minute, 3*time.Second).Should(Succeed()) + logs, err := utils.Run(exec.Command("kubectl", "logs", podName, "-n", webhookTestNamespace, + "-c", constants.ModelValidationInitContainerName)) + Expect(err).NotTo(HaveOccurred()) + Expect(logs).To(ContainSubstring("failed to fetch production trust root")) + Expect(logs).To(ContainSubstring("lookup")) + Expect(logs).To(ContainSubstring("127.0.0.1:53")) + }) + + It("preserves Kubernetes optimistic concurrency on simultaneous CR updates", func() { + const modelName = "concurrent-update-model" + DeferCleanup(deleteE2EResource, "modelvalidation", modelName) + + Expect(utils.KubectlApply(modelValidationTemplate, utils.CRTemplateData{ + ModelName: modelName, + Namespace: webhookTestNamespace, + })).To(Succeed()) + + projectDir, err := utils.GetProjectDir() + Expect(err).NotTo(HaveOccurred()) + + // Status reconciliation bumps resourceVersion, so a single attempt can + // lose to the controller. Retry until one write wins and the other conflicts. + Eventually(func(g Gomega) { + var mv v1alpha1.ModelValidation + g.Expect(utils.KubectlGetJSON("modelvalidation", modelName, webhookTestNamespace, &mv)).To(Succeed()) + mv.Spec.ImagePullPolicy = corev1.PullNever + body, marshalErr := json.Marshal(&mv) + g.Expect(marshalErr).NotTo(HaveOccurred()) + + type result struct { + output string + err error + } + results := make(chan result, 2) + start := make(chan struct{}) + var wg sync.WaitGroup + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + <-start + cmd := exec.Command("kubectl", "replace", "-f", "-") + cmd.Dir = projectDir + cmd.Stdin = bytes.NewReader(body) + output, runErr := cmd.CombinedOutput() + results <- result{output: string(output), err: runErr} + }() + } + close(start) + wg.Wait() + close(results) + + var successes, conflicts int + for update := range results { + if update.err == nil { + successes++ + } else if strings.Contains(update.output, "the object has been modified") { + conflicts++ + } + } + g.Expect(successes).To(Equal(1)) + g.Expect(conflicts).To(Equal(1)) + }, 30*time.Second, time.Second).Should(Succeed()) + }) +}) + +func webhookEndpointAddresses() ([]string, error) { + output, err := utils.Run(exec.Command("kubectl", "get", "endpointslice", + "-l", "kubernetes.io/service-name=model-validation-webhook", + "-n", operatorNamespace, "--request-timeout=15s", "-o", "json")) + if err != nil { + return nil, err + } + var slices discoveryv1.EndpointSliceList + if err := json.Unmarshal([]byte(output), &slices); err != nil { + return nil, err + } + var addresses []string + for _, slice := range slices.Items { + for _, endpoint := range slice.Endpoints { + if endpoint.Conditions.Ready != nil && *endpoint.Conditions.Ready { + addresses = append(addresses, endpoint.Addresses...) + } + } + } + return addresses, nil +} diff --git a/test/e2e/pki_validation_test.go b/test/e2e/pki_validation_test.go new file mode 100644 index 00000000..71eb8c5c --- /dev/null +++ b/test/e2e/pki_validation_test.go @@ -0,0 +1,56 @@ +//go:build integration + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive + corev1 "k8s.io/api/core/v1" + + utils "github.com/sigstore/model-validation-operator/test/utils" +) + +var _ = Describe("PKI validation", func() { + It("verifies a model signature against an X.509 certificate chain", func() { + const ( + modelName = "pki-chain-model" + podName = "pki-chain-pod" + modelPath = "/fixtures/v1.1.0-certificate" + caPath = "/fixtures/keys/certificate/ca-cert.pem" + ) + + DeferCleanup(func() { + deleteE2EResource("pod", podName) + deleteE2EResource("modelvalidation", modelName) + }) + + config := ` pkiConfig: + certificateAuthority: ` + caPath + Expect(utils.KubectlApply(authModelValidation(modelName, config, modelPath, + modelPath+"/model.sig", ""), nil)).To(Succeed()) + Expect(utils.KubectlApply(podTemplate, authFixturePodData(podName, modelName))).To(Succeed()) + + By("verifying PKI-specific arguments were injected") + var pod corev1.Pod + Expect(utils.KubectlGetJSON("pod", podName, webhookTestNamespace, &pod)).To(Succeed()) + Expect(pod.Spec.InitContainers).To(HaveLen(1)) + Expect(pod.Spec.InitContainers[0].Args).To(ContainElements( + "certificate", "--certificate_chain", caPath)) + + expectSuccessfulValidation(podName) + }) +}) diff --git a/test/e2e/sigstore_validation_test.go b/test/e2e/sigstore_validation_test.go new file mode 100644 index 00000000..d493809d --- /dev/null +++ b/test/e2e/sigstore_validation_test.go @@ -0,0 +1,60 @@ +//go:build integration + +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive + corev1 "k8s.io/api/core/v1" + + utils "github.com/sigstore/model-validation-operator/test/utils" +) + +var _ = Describe("Sigstore keyless validation", func() { + It("verifies a keyless bundle with certificate identity and OIDC issuer", func() { + const ( + modelName = "sigstore-keyless-model" + podName = "sigstore-keyless-pod" + modelPath = "/fixtures/v1.1.0-sigstore" + ) + + DeferCleanup(func() { + deleteE2EResource("pod", podName) + deleteE2EResource("modelvalidation", modelName) + }) + + config := ` sigstoreConfig: + certificateIdentity: stefanb@us.ibm.com + certificateOidcIssuer: https://sigstore.verify.ibm.com/oauth2 + clientTrustConfig: + trustConfigPath: /fixtures/keys/sigstore/trusted_root.json` + Expect(utils.KubectlApply(authModelValidation(modelName, config, modelPath, + modelPath+"/model.sig", ""), nil)).To(Succeed()) + Expect(utils.KubectlApply(podTemplate, authFixturePodData(podName, modelName))).To(Succeed()) + + By("verifying Sigstore-specific arguments were injected") + var pod corev1.Pod + Expect(utils.KubectlGetJSON("pod", podName, webhookTestNamespace, &pod)).To(Succeed()) + Expect(pod.Spec.InitContainers).To(HaveLen(1)) + Expect(pod.Spec.InitContainers[0].Args).To(ContainElements( + "sigstore", "--identity", "stefanb@us.ibm.com", "--identity_provider", + "https://sigstore.verify.ibm.com/oauth2", "--trust_config", + "/fixtures/keys/sigstore/trusted_root.json")) + + expectSuccessfulValidation(podName) + }) +}) diff --git a/test/e2e/status_tracking_test.go b/test/e2e/status_tracking_test.go index 9d17afc4..b97e6580 100644 --- a/test/e2e/status_tracking_test.go +++ b/test/e2e/status_tracking_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright 2025. diff --git a/test/e2e/testdata.go b/test/e2e/testdata.go index 58d3d46d..4aa4ea05 100644 --- a/test/e2e/testdata.go +++ b/test/e2e/testdata.go @@ -1,3 +1,5 @@ +//go:build integration || ha + /* Copyright 2025. @@ -18,6 +20,8 @@ package e2e import _ "embed" +const curlMetricsPodName = "curl-metrics" + // ModelValidation CR template used by multiple tests // //go:embed testdata/modelvalidation_template.yaml @@ -30,6 +34,3 @@ var podTemplate []byte //go:embed testdata/curl_metrics_pod_template.yaml var curlPodTemplate []byte - -//go:embed testdata/clusterrolebinding_template.yaml -var clusterRoleBindingTemplate []byte diff --git a/test/e2e/testdata/curl_metrics_pod_template.yaml b/test/e2e/testdata/curl_metrics_pod_template.yaml index a0673404..49c5c7e9 100644 --- a/test/e2e/testdata/curl_metrics_pod_template.yaml +++ b/test/e2e/testdata/curl_metrics_pod_template.yaml @@ -23,7 +23,12 @@ kind: Pod metadata: name: {{.PodName}} namespace: {{.Namespace}} + annotations: + # OpenShift must not select the elevated hostPath fixture SCC for this Pod. + # Kubernetes/Kind treats this annotation as ordinary metadata. + openshift.io/required-scc: restricted-v2 spec: + terminationGracePeriodSeconds: 1 containers: - name: curl image: curlimages/curl:latest @@ -34,8 +39,7 @@ spec: capabilities: drop: ["ALL"] runAsNonRoot: true - runAsUser: 1000 seccompProfile: type: RuntimeDefault serviceAccount: {{.ServiceAccount}} - restartPolicy: Never \ No newline at end of file + restartPolicy: Never diff --git a/test/e2e/testdata/model-data-daemonset.yaml b/test/e2e/testdata/model-data-daemonset.yaml index 524c44e1..a214a217 100644 --- a/test/e2e/testdata/model-data-daemonset.yaml +++ b/test/e2e/testdata/model-data-daemonset.yaml @@ -22,15 +22,26 @@ spec: - sh - -c - | + set -eu echo "Setting up model data on node $NODE_NAME..." - mkdir -p /host-data /host-keys + mkdir -p /host-data /host-keys /host-fixtures + # Go module fixtures can be read-only. A restarted root container + # without DAC_OVERRIDE must restore owner write access before copying. + chmod -R u+rwX /host-data /host-keys /host-fixtures cp -r /data/* /host-data/ cp -r /keys/* /host-keys/ + cp -r /fixtures/* /host-fixtures/ # Make files readable by all users (validation-agent runs as UID 65532) - chmod -R a+rX /host-data /host-keys + chmod -R a+rX /host-data /host-keys /host-fixtures echo "Model data setup complete on node $NODE_NAME" + touch /tmp/model-data-ready # Keep running so DaemonSet stays active sleep infinity + readinessProbe: + exec: + command: ["test", "-f", "/tmp/model-data-ready"] + initialDelaySeconds: 1 + periodSeconds: 2 env: - name: NODE_NAME valueFrom: @@ -41,6 +52,8 @@ spec: mountPath: /host-data - name: host-keys-data mountPath: /host-keys + - name: host-fixture-data + mountPath: /host-fixtures securityContext: runAsNonRoot: false runAsUser: 0 @@ -53,6 +66,10 @@ spec: hostPath: path: /tmp/e2e-keys-data type: DirectoryOrCreate + - name: host-fixture-data + hostPath: + path: /tmp/e2e-fixture-data + type: DirectoryOrCreate tolerations: - operator: Exists - effect: NoSchedule \ No newline at end of file + effect: NoSchedule diff --git a/test/e2e/testdata/pod_template.yaml b/test/e2e/testdata/pod_template.yaml index 34a888d3..52fdc617 100644 --- a/test/e2e/testdata/pod_template.yaml +++ b/test/e2e/testdata/pod_template.yaml @@ -8,6 +8,7 @@ metadata: test-batch: "{{.TestBatch}}"{{end}} spec: restartPolicy: Never + terminationGracePeriodSeconds: 1 containers: - name: test-container image: busybox:latest @@ -29,4 +30,4 @@ spec: - name: {{.Name}}{{if .HostPath}} hostPath: path: {{.HostPath.Path}} - type: {{if .HostPath.Type}}{{.HostPath.Type}}{{else}}Directory{{end}}{{end}}{{end}}{{end}} \ No newline at end of file + type: {{if .HostPath.Type}}{{.HostPath.Type}}{{else}}Directory{{end}}{{end}}{{end}}{{end}} diff --git a/test/e2e/validation_test.go b/test/e2e/validation_test.go index 261f29f4..ab51ec38 100644 --- a/test/e2e/validation_test.go +++ b/test/e2e/validation_test.go @@ -1,3 +1,5 @@ +//go:build integration + /* Copyright 2025. diff --git a/test/utils/utils.go b/test/utils/utils.go index 1691d259..4f5462ab 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "os/exec" + "regexp" "strings" "text/template" @@ -120,25 +121,38 @@ type SubscriptionTemplateData struct { // Run executes the provided command within this context func Run(cmd *exec.Cmd) (string, error) { - dir, _ := GetProjectDir() - cmd.Dir = dir - - if err := os.Chdir(cmd.Dir); err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + if cmd.Dir == "" { + dir, err := GetProjectDir() + if err != nil { + return "", err + } + cmd.Dir = dir } - cmd.Env = append(os.Environ(), "GO111MODULE=on") - command := strings.Join(cmd.Args, " ") + if cmd.Env == nil { + cmd.Env = append(os.Environ(), "GO111MODULE=on") + } + command := redactCommand(strings.Join(cmd.Args, " ")) _, _ = fmt.Fprintf(GinkgoWriter, "running: %s\n", command) output, err := cmd.CombinedOutput() if err != nil { return string(output), fmt.Errorf("%s failed with error: (%v) %s", - command, err, string(output)) + command, err, redactCommand(string(output))) } return string(output), nil } +var ( + bearerPattern = regexp.MustCompile(`(?i)(Bearer\s+)[^\s'";]+`) + secretFlagPattern = regexp.MustCompile(`(?i)(--(?:token|password)(?:=|\s+))[^\s'";]+`) +) + +func redactCommand(command string) string { + command = bearerPattern.ReplaceAllString(command, "${1}[REDACTED]") + return secretFlagPattern.ReplaceAllString(command, "${1}[REDACTED]") +} + // GetNonEmptyLines converts given command output string into individual objects // according to line breakers, and ignores the empty elements in it. func GetNonEmptyLines(output string) []string { @@ -232,7 +246,7 @@ func KubectlDelete(yamlData []byte, opts *KubectlDeleteOptions) error { // KubectlGet retrieves a Kubernetes resource and returns the output // If name is empty, retrieves all resources of the specified type func KubectlGet(resource, name, namespace string, outputFormat string) (string, error) { - args := []string{"get", resource} + args := []string{"get", resource, "--request-timeout=15s"} if name != "" { args = append(args, name) } @@ -300,6 +314,13 @@ func DefaultPodData(podName, namespace, modelName, testBatch string) PodTemplate Type: "Directory", }, } + fixtureDataVolume := Volume{ + Name: "fixture-data", + HostPath: &HostPathVolume{ + Path: "/tmp/e2e-fixture-data", + Type: "Directory", + }, + } modelDataMount := VolumeMount{ Name: "model-data", MountPath: "/data", @@ -310,14 +331,19 @@ func DefaultPodData(podName, namespace, modelName, testBatch string) PodTemplate MountPath: "/keys", ReadOnly: true, } + fixtureDataMount := VolumeMount{ + Name: "fixture-data", + MountPath: "/fixtures", + ReadOnly: true, + } data := PodTemplateData{ PodName: podName, Namespace: namespace, ModelName: modelName, TestBatch: testBatch, - VolumeMounts: []VolumeMount{modelDataMount, keysDataMount}, - Volumes: []Volume{modelDataVolume, keysDataVolume}, + VolumeMounts: []VolumeMount{modelDataMount, keysDataMount, fixtureDataMount}, + Volumes: []Volume{modelDataVolume, keysDataVolume, fixtureDataVolume}, } return data @@ -330,9 +356,20 @@ func GetMetricsOutput(namespace, podName string) string { return "" } + metricsHost := fmt.Sprintf("%s.%s.svc.cluster.local", MetricsServiceName, namespace) + holderIdentity, holderErr := KubectlGet("lease", "2b2639c9.sigstore.dev", namespace, + "jsonpath={.spec.holderIdentity}") + if holderErr == nil && holderIdentity != "" { + leaderPod := strings.SplitN(holderIdentity, "_", 2)[0] + podIP, podErr := KubectlGet("pod", leaderPod, namespace, "jsonpath={.status.podIP}") + if podErr == nil && podIP != "" { + metricsHost = podIP + } + } + curlCommand := fmt.Sprintf( - "curl -s -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics 2>/dev/null", - token, MetricsServiceName, namespace) + "curl -s -k -H 'Authorization: Bearer %s' https://%s:8443/metrics 2>/dev/null", + token, metricsHost) execCmd := exec.Command("kubectl", "exec", podName, "-n", namespace, "--", "/bin/sh", "-c", curlCommand) @@ -373,6 +410,13 @@ func KubectlResourceExists(resource, name, namespace string) bool { return err == nil } +// KubectlResourceAbsent distinguishes API NotFound from connectivity/authentication errors. +func KubectlResourceAbsent(resource, name, namespace string) (bool, error) { + output, err := Run(exec.Command("kubectl", "get", resource, name, "-n", namespace, + "--ignore-not-found=true", "--request-timeout=15s", "-o", "name")) + return strings.TrimSpace(output) == "" && err == nil, err +} + // HasValidationContainer checks if a pod has the model-validation init container func HasValidationContainer(pod *corev1.Pod) bool { for _, container := range pod.Spec.InitContainers { diff --git a/test/utils/utils_test.go b/test/utils/utils_test.go new file mode 100644 index 00000000..a7502cf7 --- /dev/null +++ b/test/utils/utils_test.go @@ -0,0 +1,202 @@ +// Copyright 2026 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils_test //nolint:revive + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestRedactCommand(t *testing.T) { + for _, command := range []string{ + `curl -H 'Authorization: Bearer sensitive-token' https://example.invalid`, + `kubectl --token=sensitive-token get pods`, + `oc login --password sensitive-token`, + } { + result := redactCommand(command) + if strings.Contains(result, "sensitive-token") || !strings.Contains(result, "[REDACTED]") { + t.Errorf("secret was not redacted: %s", result) + } + } +} + +func TestRunPreservesProcessDirectoryAndCommandEnvironment(t *testing.T) { + before, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("sh", "-c", `printf '%s\n%s' "$PWD" "$MVO_TEST_VALUE"`) + cmd.Dir = t.TempDir() + cmd.Env = append(os.Environ(), "MVO_TEST_VALUE=preserved") + output, err := Run(cmd) + if err != nil { + t.Fatal(err) + } + expectedDir, err := filepath.EvalSymlinks(cmd.Dir) + if err != nil { + t.Fatal(err) + } + if output != expectedDir+"\npreserved" { + t.Fatalf("command directory or environment lost: %q", output) + } + after, err := os.Getwd() + if err != nil || before != after { + t.Fatalf("Run changed the process directory: %s -> %s (%v)", before, after, err) + } +} + +func TestRunRedactsSecretInWrappedErrors(t *testing.T) { + _, err := Run(exec.Command("sh", "-c", "echo 'Authorization: Bearer sensitive-token' >&2; exit 1")) + if err == nil || strings.Contains(err.Error(), "sensitive-token") { + t.Fatalf("expected a redacted error, got %v", err) + } +} + +func TestKubectlResourceAbsentDoesNotHideAPIErrors(t *testing.T) { + stub := `#!/bin/sh +case $MVO_FAKE_GET in +missing) exit 0;; +found) echo pod/test;; +*) echo Forbidden >&2; exit 1;; +esac +` + fakeKubectl(t, stub) + for _, test := range []struct { + name string + absent bool + err bool + }{ + {name: "missing", absent: true}, + {name: "found"}, + {name: "forbidden", err: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Setenv("MVO_FAKE_GET", test.name) + absent, err := KubectlResourceAbsent("pod", "test", "test-namespace") + if absent != test.absent || (err != nil) != test.err { + t.Fatalf("absent = %t, error = %v", absent, err) + } + }) + } +} + +func fakeKubectl(t *testing.T, script string) { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "kubectl"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestDefaultPodDataMountsAllVerificationFixturesReadOnly(t *testing.T) { + data := DefaultPodData("test-pod", "test-namespace", "test-model", "test-batch") + if data.PodName != "test-pod" || data.Namespace != "test-namespace" || + data.ModelName != "test-model" || data.TestBatch != "test-batch" { + t.Fatalf("pod identity fields were not preserved: %+v", data) + } + wantMounts := []VolumeMount{ + {Name: "model-data", MountPath: "/data", ReadOnly: true}, + {Name: "keys-data", MountPath: "/keys", ReadOnly: true}, + {Name: "fixture-data", MountPath: "/fixtures", ReadOnly: true}, + } + if !reflect.DeepEqual(data.VolumeMounts, wantMounts) { + t.Errorf("unexpected mounts: %+v", data.VolumeMounts) + } + wantVolumes := []Volume{ + {Name: "model-data", HostPath: &HostPathVolume{Path: "/tmp/e2e-model-data", Type: "Directory"}}, + {Name: "keys-data", HostPath: &HostPathVolume{Path: "/tmp/e2e-keys-data", Type: "Directory"}}, + {Name: "fixture-data", HostPath: &HostPathVolume{Path: "/tmp/e2e-fixture-data", Type: "Directory"}}, + } + if !reflect.DeepEqual(data.Volumes, wantVolumes) { + t.Errorf("unexpected hostPath sources: %+v", data.Volumes) + } +} + +func TestKubectlGetBoundsRequestsAndPreservesResourceArguments(t *testing.T) { + fakeKubectl(t, "#!/bin/sh\nprintf '%s\\n' \"$@\"\n") + output, err := KubectlGet("pods", "test-pod", "test-namespace", "json") + if err != nil { + t.Fatal(err) + } + want := []string{"get", "pods", "--request-timeout=15s", "test-pod", "-n", "test-namespace", "-o", "json"} + if !reflect.DeepEqual(GetNonEmptyLines(output), want) { + t.Errorf("kubectl arguments: %q, want %q", GetNonEmptyLines(output), want) + } + output, err = KubectlGet("nodes", "", "", "") + if err != nil || !reflect.DeepEqual(GetNonEmptyLines(output), []string{"get", "nodes", "--request-timeout=15s"}) { + t.Errorf("cluster-wide list arguments: %q (%v)", output, err) + } +} + +func TestGetMetricsOutputTargetsLeaderOrFallsBackToService(t *testing.T) { + fakeKubectl(t, `#!/bin/sh +case "$1 $2" in +"create token") + [ "$MVO_METRICS_MODE" != "token-error" ] || exit 1 + [ "$3" = "e2e-metrics-reader" ] || exit 2 + printf 'test-metrics-token' + ;; +"get lease") + [ "$MVO_METRICS_MODE" != "lease-error" ] || exit 1 + [ "$MVO_METRICS_MODE" != "lease-empty" ] || exit 0 + printf 'manager-leader_lease-identity' + ;; +"get pod") + [ "$MVO_METRICS_MODE" != "pod-error" ] || exit 1 + [ "$MVO_METRICS_MODE" != "pod-empty" ] || exit 0 + [ "$4" = "manager-leader" ] || exit 2 + printf '10.0.0.17' + ;; +"exec metrics-reader") + [ "$MVO_METRICS_MODE" != "exec-error" ] || exit 1 + for arg in "$@"; do last=$arg; done + printf '%s' "$last" + ;; +*) exit 2;; +esac +`) + const serviceHost = "model-validation-controller-manager-metrics-service.test-namespace.svc.cluster.local" + for _, test := range []struct { + mode string + host string + }{ + {mode: "leader", host: "10.0.0.17"}, + {mode: "lease-error", host: serviceHost}, + {mode: "lease-empty", host: serviceHost}, + {mode: "pod-error", host: serviceHost}, + {mode: "pod-empty", host: serviceHost}, + {mode: "token-error"}, + {mode: "exec-error"}, + } { + t.Run(test.mode, func(t *testing.T) { + t.Setenv("MVO_METRICS_MODE", test.mode) + output := GetMetricsOutput("test-namespace", "metrics-reader") + want := "" + if test.host != "" { + want = fmt.Sprintf( + "curl -s -k -H 'Authorization: Bearer test-metrics-token' https://%s:8443/metrics 2>/dev/null", test.host) + } + if output != want { + t.Errorf("metrics command: %q, want %q", output, want) + } + }) + } +} diff --git a/testdata/docker/test-model.Dockerfile b/testdata/docker/test-model.Dockerfile index c21472c1..663bad03 100644 --- a/testdata/docker/test-model.Dockerfile +++ b/testdata/docker/test-model.Dockerfile @@ -1,15 +1,29 @@ +FROM golang:1.26 AS fixture-source + +WORKDIR /workspace +COPY go.mod go.sum ./ +RUN go mod download +RUN fixture_dir="$(go list -m -f '{{.Dir}}' github.com/sigstore/model-signing)" && \ + sigstore_dir="$(go list -m -f '{{.Dir}}' github.com/sigstore/sigstore)" && \ + mkdir -p /fixtures/keys/certificate /fixtures/keys/sigstore && \ + cp -R "${fixture_dir}/scripts/tests/v1.1.0-sigstore" /fixtures/ && \ + cp -R "${fixture_dir}/scripts/tests/v1.1.0-certificate" /fixtures/ && \ + cp "${fixture_dir}/scripts/tests/keys/certificate/ca-cert.pem" /fixtures/keys/certificate/ && \ + cp "${sigstore_dir}/pkg/tuf/repository/targets/trusted_root.json" /fixtures/keys/sigstore/ + FROM busybox # Copy the real TensorFlow SavedModel files from testdata -COPY tensorflow_saved_model/ /data/ +COPY testdata/tensorflow_saved_model/ /data/ # Copy public keys for verification tests - separate directory RUN mkdir -p /keys -COPY docker/test_public_key.pub /keys/test_public_key.pub -COPY docker/test_invalid_public_key.pub /keys/test_invalid_public_key.pub +COPY testdata/docker/test_public_key.pub /keys/test_public_key.pub +COPY testdata/docker/test_invalid_public_key.pub /keys/test_invalid_public_key.pub +COPY --from=fixture-source /fixtures/ /fixtures/ # Make files readable and ensure no stray public key files in model directory -RUN chmod -R 644 /data /keys && rm -f /data/test_public_key.pub /data/*.pub +RUN chmod -R a+rX /data /keys /fixtures && rm -f /data/test_public_key.pub /data/*.pub # Default command -CMD ["sleep", "3600"] \ No newline at end of file +CMD ["sleep", "3600"]