SECURESIGN-5151: strengthen MVO test coverage and CI gates

.github/workflows/lint.yml

@@ -11,12 +11,18 @@ permissions:
 
 jobs:
   lint:
-    name: Run Linting
+    name: Lint (${{ matrix.tags || 'default' }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: ["", "integration", "ha", "upgrade", "coverage"]
 
     steps:
       - name: Checkout code
         uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6
+        with:
+          fetch-depth: 0
 
       - name: Setup Go
         uses: actions/setup-go@def8c394e3ad351a79bc93815e4a585520fe993b # v5
@@ -28,4 +34,76 @@ jobs:
         uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0
         with:
           version: v2.4.0
-          args: --timeout=5m --issues-exit-code=0 --verbose
+          # Build with the go.mod toolchain, not a release binary built with an older Go.
+          install-mode: goinstall
+          args: --timeout=5m --issues-exit-code=1 --verbose --build-tags=${{ matrix.tags }}
+
+      - name: Test OLM configuration and lifecycle helpers without a cluster
+        if: matrix.tags == 'upgrade'
+        run: >-
+          go test -race -count=1 -tags=upgrade ./test/e2e-olm/
+          -run '^(TestUpgradeConfig.*|TestControllerRollout|TestCatalogTemplate|TestExistingCSVs|TestCleanup.*|TestSubscriptionCSVNames)$'
+
+      - name: Test metrics Pod security template without a cluster
+        if: matrix.tags == 'integration'
+        run: go test -race -count=1 -tags=integration ./test/e2e/ -run '^TestMetricsPodSecurityTemplate$'
+
+      - name: Test coverage collector failure handling and process restart retention
+        if: matrix.tags == ''
+        run: make test-coverage-collector
+
+  crd-compatibility:
+    name: CRD generation and backward compatibility
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@def8c394e3ad351a79bc93815e4a585520fe993b # v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Generate and verify committed manifests are current
+        run: |
+          make manifests generate
+          git diff --exit-code -- config/crd/bases config/rbac config/webhook api
+          untracked=$(git ls-files --others --exclude-standard -- config/crd/bases config/rbac config/webhook api)
+          if [[ -n "$untracked" ]]; then
+            echo "Generated artifacts are missing from Git:"
+            printf '%s\n' "$untracked"
+            exit 1
+          fi
+
+      - name: Install crdify
+        run: go install sigs.k8s.io/crdify@v0.6.0
+
+      - name: Check CRD backward compatibility
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
+        run: |
+          if [[ -z "$BASE_SHA" || "$BASE_SHA" =~ ^0+$ ]]; then
+            echo "No previous revision is available; skipping CRD comparison"
+            exit 0
+          fi
+          git show "$BASE_SHA:config/crd/bases/ml.sigstore.dev_modelvalidations.yaml" > /tmp/base-crd.yaml
+          crdify --config crdify.yaml \
+            file:///tmp/base-crd.yaml \
+            file://config/crd/bases/ml.sigstore.dev_modelvalidations.yaml
+
+  lint-gate:
+    # Preserve the existing required-check name while requiring every partition
+    # and the new generated-schema/compatibility gate to pass.
+    name: Run Linting
+    if: ${{ always() }}
+    needs: [lint, crd-compatibility]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require all lint and CRD checks to pass
+        env:
+          LINT_RESULT: ${{ needs.lint.result }}
+          CRD_RESULT: ${{ needs.crd-compatibility.result }}
+        run: test "$LINT_RESULT" = success && test "$CRD_RESULT" = success

.github/workflows/test-e2e.yml

@@ -11,11 +11,15 @@ env:
 
 permissions:
   contents: read
+  id-token: write
 
 jobs:
   test-e2e:
     name: Run E2E Tests
     runs-on: ubuntu-latest
+    env:
+      CONTAINER_FILE: Dockerfile.coverage
+      E2E_COLLECT_COVERAGE: "true"
     steps:
       - name: Checkout code
         uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6
@@ -44,6 +48,91 @@ jobs:
         run: |
           make test-e2e-ci
 
+      - name: Collect E2E coverage
+        if: ${{ !cancelled() }}
+        run: make collect-e2e-coverage
+
+      - name: Upload E2E coverage to Codecov
+        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
+        with:
+          files: e2e-cover.out
+          flags: e2e
+          use_oidc: true
+          fail_ci_if_error: true
+
+      - name: Collect cluster diagnostics
+        if: ${{ failure() }}
+        run: |
+          mkdir -p e2e-diagnostics
+          kubectl get pods,events,deployments,endpointslices -n model-validation-operator-system -o yaml > e2e-diagnostics/operator.yaml || true
+          kubectl get pods,events,modelvalidations -n e2e-webhook-test-ns -o yaml > e2e-diagnostics/tests.yaml || true
+          kubectl logs -n model-validation-operator-system -l control-plane=controller-manager --all-containers --prefix --tail=1000 > e2e-diagnostics/operator.log || true
+
+      - name: Upload E2E diagnostics and coverage snapshots
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: integration-evidence
+          path: |
+            e2e-diagnostics/
+            e2e-coverdata/
+            e2e-cover.out
+          if-no-files-found: warn
+          include-hidden-files: true
+          retention-days: 7
+
+  test-e2e-ha:
+    name: Run HA E2E Tests
+    runs-on: ubuntu-latest
+    env:
+      E2E_ALLOW_NODE_DRAIN: "true"
+      E2E_DRAIN_NODE_SELECTOR: mvo-e2e-drain=true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6
+
+      - name: Setup Go
+        uses: actions/setup-go@def8c394e3ad351a79bc93815e4a585520fe993b # v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Create multi-node kind cluster
+        uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1
+        with:
+          cluster_name: kind
+          config: ./ci/kind-ha-config.yaml
+
+      - name: Wait for kind cluster to be ready
+        run: |
+          kubectl wait --for=condition=Ready nodes --all --timeout=300s
+          kubectl wait --for=condition=Ready --namespace=kube-system pod --all --timeout=300s
+          kubectl label nodes -l '!node-role.kubernetes.io/control-plane' mvo-e2e-drain=true
+
+      - name: Swap to dev images
+        run: make dev-images
+
+      - name: Run HA E2E tests
+        run: make test-e2e-ha-ci
+
+      - name: Collect HA diagnostics
+        if: ${{ failure() }}
+        run: |
+          mkdir -p e2e-diagnostics
+          kubectl get nodes -o yaml > e2e-diagnostics/nodes.yaml || true
+          kubectl get pods,events,deployments,poddisruptionbudgets,leases -n model-validation-operator-system -o yaml > e2e-diagnostics/operator.yaml || true
+          kubectl get pods,events,modelvalidations -n e2e-webhook-test-ns -o yaml > e2e-diagnostics/tests.yaml || true
+          kubectl logs -n model-validation-operator-system -l control-plane=controller-manager --all-containers --prefix --tail=1000 > e2e-diagnostics/operator.log || true
+
+      - name: Upload HA diagnostics
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: ha-evidence
+          path: e2e-diagnostics/
+          if-no-files-found: warn
+          retention-days: 7
+
   test-e2e-olm-upgrade:
     name: Test OLM upgrade
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
@@ -176,5 +265,8 @@ jobs:
         env:
           TEST_BASE_CATALOG: registry.redhat.io/redhat/redhat-operator-index:${{ vars.OLM_INDEX_VERSION }}
           TEST_TARGET_CATALOG: ${{ env.CATALOG_IMG }}
+          # CI creates a next-patch upgrade candidate, not an approved GA release.
+          TEST_EXPECTED_TARGET_CSV: model-validation-operator.v${{ env.OLM_UPGRADE_VERSION }}
+          TEST_EXPECTED_TARGET_IMAGE: ${{ env.IMG }}
         run: |
-          go test ./test/e2e-olm/ -v -ginkgo.v -timeout 20m
+          go test -tags=upgrade ./test/e2e-olm/ -v -ginkgo.v -timeout 20m

.gitignore

@@ -3,6 +3,8 @@
 /bin
 
 cover.out
+e2e-cover.out
+e2e-coverdata/
 # editor and IDE paraphernalia
 .idea
 .vscode

.golangci.yml

@@ -27,7 +27,7 @@ linters:
     rules:
       - linters:
           - lll
-        source: "^// \\+kubebuilder:validation"
+        source: "^\\s*// \\+kubebuilder:validation"
 
 formatters:
   enable:

Dockerfile.coverage

@@ -0,0 +1,29 @@
+# Copyright 2026 The Sigstore Authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+
+FROM golang:1.26 AS builder
+ARG TARGETOS
+ARG TARGETARCH
+ARG AGENT_IMG
+
+WORKDIR /workspace
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY cmd/ cmd/
+COPY api/ api/
+COPY internal/ internal/
+
+# Atomic mode is required by runtime/coverage.WriteCounters. The coverage tag
+# embeds Coverport's HTTP collector on port 53700.
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \
+    -cover -covermode=atomic -tags=coverage -a \
+    -ldflags="-X github.com/sigstore/model-validation-operator/internal/constants.ModelValidationAgentImage=${AGENT_IMG}" \
+    -o manager ./cmd
+
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/manager .
+USER 65532:65532
+
+ENTRYPOINT ["/manager"]

Makefile

@@ -201,6 +201,10 @@ docker-build: test ## Build docker image with the manager.
 docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${IMG}
 
+.PHONY: docker-build-coverage
+docker-build-coverage: ## Build a coverage-instrumented manager image for E2E tests.
+	$(CONTAINER_TOOL) build -t ${IMG} --build-arg AGENT_IMG=${AGENT_IMG} -f Dockerfile.coverage .
+
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
 # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
 # - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
@@ -258,7 +262,6 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/overlays/production && $(KUSTOMIZE) edit set image controller=${IMG}
-	cd config/overlays/production && $(KUSTOMIZE) edit set replicas controller-manager=1
 	$(KUSTOMIZE) build config/overlays/production | $(KUBECTL) apply -f -
 
 .PHONY: undeploy
@@ -500,7 +503,7 @@ e2e-sign-test-model: e2e-generate-test-keys
 .PHONY: e2e-build-test-model
 e2e-build-test-model: e2e-sign-test-model
 	@echo "Building test model image..."
-	cd testdata && $(CONTAINER_TOOL) build --no-cache -t $(E2E_TEST_MODEL) -f docker/test-model.Dockerfile .
+	$(CONTAINER_TOOL) build --no-cache -t $(E2E_TEST_MODEL) -f testdata/docker/test-model.Dockerfile .
 
 # install and uninstall cert-manager for tests
 
@@ -597,25 +600,55 @@ e2e-teardown: e2e-cleanup-resources undeploy e2e-uninstall-certmanager
 .PHONY: test-e2e
 test-e2e: manifests generate fmt vet ## Run the e2e tests, no setup and teardown.  Expects the operator to be deployed.
 	@echo "Running e2e tests (assumes infrastructure is already set up)..."
-	go test ./test/e2e/ -v -ginkgo.v
+	go test -tags=integration ./test/e2e/ -v -ginkgo.v -timeout 30m
+
+.PHONY: test-e2e-integration
+test-e2e-integration: test-e2e ## Run the integration-tagged E2E partition.
+
+.PHONY: test-e2e-ha
+test-e2e-ha: manifests generate fmt vet ## Run disruptive HA tests against an existing multi-node cluster.
+	go test -tags=ha ./test/e2e/ -v -ginkgo.v -timeout 30m
 
 .PHONY: test-e2e-full
 test-e2e-full: manifests generate fmt vet e2e-setup ## Run e2e tests with setup and teardown
 	@echo "Running e2e tests with full infrastructure setup..."
-	go test ./test/e2e/ -v -ginkgo.v; \
-	TEST_RESULT=$$?; \
-	$(MAKE) e2e-teardown; \
-	exit $$TEST_RESULT
+	@result=0; \
+	go test -tags=integration ./test/e2e/ -v -ginkgo.v -timeout 30m || result=$$?; \
+	$(MAKE) e2e-teardown || { cleanup_result=$$?; if [ $$result -eq 0 ]; then result=$$cleanup_result; fi; }; \
+	exit $$result
 
 .PHONY: test-e2e-ci
 test-e2e-ci: manifests generate fmt vet e2e-setup ## Run the e2e tests, with setup.  No teardown as the CI workflow will throw away kind
 	@echo "Running e2e tests with infrastructure setup for CI..."
-	go test ./test/e2e/ -v -ginkgo.v
+	@if [ "$${E2E_COLLECT_COVERAGE:-false}" = "true" ]; then $(MAKE) init-e2e-coverage; fi
+	go test -tags=integration ./test/e2e/ -v -ginkgo.v -timeout 30m
+
+.PHONY: test-e2e-ha-ci
+test-e2e-ha-ci: manifests generate fmt vet e2e-setup ## Set up and run HA tests in CI.
+	go test -tags=ha ./test/e2e/ -v -ginkgo.v -timeout 30m
+
+.PHONY: init-e2e-coverage
+init-e2e-coverage: ## Start a fresh coverage session, retaining old sessions for diagnosis.
+	bash scripts/e2e-coverage.sh init
+
+.PHONY: snapshot-e2e-coverage
+snapshot-e2e-coverage: ## Save cumulative coverage before restarting or terminating managers.
+	KUBECTL="$(KUBECTL)" E2E_OPERATOR_NAMESPACE="$(E2E_OPERATOR_NAMESPACE)" bash scripts/e2e-coverage.sh snapshot
+
+.PHONY: collect-e2e-coverage
+collect-e2e-coverage: ## Snapshot current managers and merge them with all pre-disruption processes.
+	KUBECTL="$(KUBECTL)" E2E_OPERATOR_NAMESPACE="$(E2E_OPERATOR_NAMESPACE)" bash scripts/e2e-coverage.sh collect
+
+.PHONY: test-coverage-collector
+test-coverage-collector: ## Test coverage collection with real Go counter data and mocked Kubernetes.
+	bash scripts/test-e2e-coverage.sh
 
 ##@ OLM E2E Test Infrastructure
 
 # OLM E2E Variables
-OLM_NAMESPACE ?= $(shell $(KUBECTL) get ns openshift-operator-lifecycle-manager --no-headers >/dev/null 2>&1 && echo openshift-operator-lifecycle-manager || echo olm)
+# Catalogs outside the marketplace namespace are local to their Subscription.
+# The OLM controller namespace is unrelated to CatalogSource visibility.
+OLM_CATALOG_NAMESPACE ?= $(E2E_OPERATOR_NAMESPACE)
 OLM_CHANNEL ?= tech-preview
 OLM_PACKAGE ?= model-validation-operator
 
@@ -631,15 +664,21 @@ e2e-uninstall-olm: operator-sdk ## Uninstall OLM from the cluster
 .PHONY: e2e-olm-teardown
 e2e-olm-teardown: ## Tear down OLM e2e test environment
 	@echo "Tearing down OLM e2e test environment..."
-	-$(KUBECTL) delete subscription --all -n $(E2E_OPERATOR_NAMESPACE) --timeout=30s
-	-$(KUBECTL) delete csv --all -n $(E2E_OPERATOR_NAMESPACE) --timeout=60s
-	-$(KUBECTL) delete operatorgroup --all -n $(E2E_OPERATOR_NAMESPACE) --timeout=30s
-	-$(KUBECTL) delete catalogsource model-validation-test-catalog -n $(OLM_NAMESPACE) --timeout=30s --ignore-not-found=true
-	-$(KUBECTL) delete pods --all -n $(E2E_TEST_NAMESPACE) --timeout=30s
-	-$(KUBECTL) delete modelvalidations --all -n $(E2E_TEST_NAMESPACE) --timeout=30s
+	$(KUBECTL) delete pods olm-test-pod-initial olm-test-pod-upgrade -n $(E2E_TEST_NAMESPACE) --ignore-not-found --timeout=30s
+	$(KUBECTL) delete modelvalidation olm-upgrade-test-model -n $(E2E_TEST_NAMESPACE) --ignore-not-found --timeout=30s
+	@csv_names=$$($(KUBECTL) get subscription model-validation-operator -n $(E2E_OPERATOR_NAMESPACE) --ignore-not-found \
+		-o jsonpath='{.status.installedCSV}{" "}{.status.currentCSV}'); \
+	$(KUBECTL) delete subscription model-validation-operator -n $(E2E_OPERATOR_NAMESPACE) --ignore-not-found --timeout=30s; \
+	for csv in $$csv_names; do \
+		$(KUBECTL) delete csv "$$csv" -n $(E2E_OPERATOR_NAMESPACE) --ignore-not-found --timeout=60s; \
+	done
+	$(KUBECTL) delete operatorgroup model-validation-og -n $(E2E_OPERATOR_NAMESPACE) --ignore-not-found --timeout=30s
+	$(KUBECTL) delete catalogsource model-validation-test-catalog -n $(OLM_CATALOG_NAMESPACE) --ignore-not-found --timeout=30s
 
 .PHONY: test-e2e-olm
 test-e2e-olm: manifests generate fmt vet ## Run OLM upgrade e2e tests
 	@echo "Running OLM upgrade e2e tests..."
-	go test ./test/e2e-olm/ -v -ginkgo.v -timeout 20m
+	go test -tags=upgrade ./test/e2e-olm/ -v -ginkgo.v -timeout 20m
 
+.PHONY: test-e2e-upgrade
+test-e2e-upgrade: test-e2e-olm ## Run the upgrade-tagged E2E partition.

ci/kind-ha-config.yaml

@@ -0,0 +1,6 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker

cmd/coverage.go

@@ -0,0 +1,5 @@
+//go:build coverage
+
+package main
+
+import _ "github.com/konflux-ci/coverport/instrumentation/go"