From 9c36079e3a97cba2c15e538b15d0241576ce7f4a Mon Sep 17 00:00:00 2001 From: JasonPowr Date: Thu, 13 Aug 2026 11:06:13 +0100 Subject: [PATCH] feat: support fips mode, add fips e2e --- .tekton/images-mirror-set.yaml | 16 + ...ntroller-operator-bundle-pull-request.yaml | 2 + ...olicy-controller-operator-bundle-push.yaml | 2 + ...licy-controller-operator-pull-request.yaml | 2 + .tekton/policy-controller-operator-push.yaml | 2 + Dockerfile | 13 +- Makefile | 6 +- bundle.Dockerfile | 2 +- config/default/metrics_service.yaml | 4 +- config/manager/manager.yaml | 8 +- ...roller-operator.clusterserviceversion.yaml | 2 +- .../network-policy/allow-metrics-traffic.yaml | 2 +- .../serving_cert_annotation_patch.yaml | 2 +- config/prometheus/monitor.yaml | 4 +- config/webhook/service.yaml | 4 +- go.mod | 3 + go.sum | 6 + .../policy-controller-operator/values.yaml | 2 +- test/e2e/e2e_suite_test.go | 4 + test/e2e_fips/e2e_fips_suite_test.go | 115 ++++++ test/e2e_fips/fips_test.go | 26 ++ test/e2e_upgrade/e2e_suite_upgrade_test.go | 1 + test/e2e_upgrade/upgrade_test.go | 6 + test/utils/fips.go | 347 ++++++++++++++++++ test/utils/kubernetes.go | 55 +-- 25 files changed, 588 insertions(+), 48 deletions(-) create mode 100644 .tekton/images-mirror-set.yaml create mode 100644 test/e2e_fips/e2e_fips_suite_test.go create mode 100644 test/e2e_fips/fips_test.go create mode 100644 test/utils/fips.go diff --git a/.tekton/images-mirror-set.yaml b/.tekton/images-mirror-set.yaml new file mode 100644 index 00000000..a4f126bc --- /dev/null +++ b/.tekton/images-mirror-set.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageDigestMirrorSet +metadata: + name: registries +spec: + imageDigestMirrors: + - mirrors: + - quay.io/securesign/policy-controller + source: registry.redhat.io/rhtas/policy-controller-rhel9 + - mirrors: + - quay.io/securesign/policy-controller-operator + source: registry.redhat.io/rhtas/policy-controller-rhel9-operator + - mirrors: + - quay.io/securesign/policy-controller-operator-bundle + source: registry.redhat.io/rhtas/policy-controller-operator-bundle diff --git a/.tekton/policy-controller-operator-bundle-pull-request.yaml b/.tekton/policy-controller-operator-bundle-pull-request.yaml index 47ad19ee..d181397a 100644 --- a/.tekton/policy-controller-operator-bundle-pull-request.yaml +++ b/.tekton/policy-controller-operator-bundle-pull-request.yaml @@ -49,6 +49,8 @@ spec: value: "1.1.0" - name: release-version value: "1.5.0" + - name: fips-check + value: "true" pipelineRef: params: - name: url diff --git a/.tekton/policy-controller-operator-bundle-push.yaml b/.tekton/policy-controller-operator-bundle-push.yaml index 7a99f891..97eefb2c 100644 --- a/.tekton/policy-controller-operator-bundle-push.yaml +++ b/.tekton/policy-controller-operator-bundle-push.yaml @@ -46,6 +46,8 @@ spec: value: "1.1.0" - name: release-version value: "1.5.0" + - name: fips-check + value: "true" pipelineRef: params: - name: url diff --git a/.tekton/policy-controller-operator-pull-request.yaml b/.tekton/policy-controller-operator-pull-request.yaml index 7af930e6..643654d5 100644 --- a/.tekton/policy-controller-operator-pull-request.yaml +++ b/.tekton/policy-controller-operator-pull-request.yaml @@ -47,6 +47,8 @@ spec: value: "1.1.0" - name: release-version value: "1.5.0" + - name: fips-check + value: "true" pipelineRef: params: - name: url diff --git a/.tekton/policy-controller-operator-push.yaml b/.tekton/policy-controller-operator-push.yaml index 75f42ac2..052d9455 100644 --- a/.tekton/policy-controller-operator-push.yaml +++ b/.tekton/policy-controller-operator-push.yaml @@ -44,6 +44,8 @@ spec: value: "1.1.0" - name: release-version value: "1.5.0" + - name: fips-check + value: "true" pipelineRef: params: - name: url diff --git a/Dockerfile b/Dockerfile index 9e873abf..6f5d12ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,15 @@ # Build the admission-webhook-controller binary FROM registry.redhat.io/ubi9/go-toolset:latest@sha256:444e81b3e88d8a68b92a081a7b3abc7d3fed2450f8473ea6b3caebf3bb73a0b8 AS admission-webhook-controller WORKDIR /opt/app-root/src/ -ENV GOEXPERIMENT=strictfipsruntime -ENV CGO_ENABLED=1 +ENV CGO_ENABLED=0 +ENV GOFIPS140=v1.0.0 -COPY go.mod go.mod -COPY go.sum go.sum -COPY cmd cmd +COPY --chown=1001:0 go.mod go.mod +COPY --chown=1001:0 go.sum go.sum +COPY --chown=1001:0 cmd cmd -RUN go build -mod=mod -o admission-webhook-controller ./cmd +RUN go mod edit -godebug=fips140=auto && \ + go build -mod=mod -tags 'no_openssl' -o admission-webhook-controller ./cmd # Unpack Helm chart FROM registry.redhat.io/ubi9/ubi-minimal:latest@sha256:7c372902c8d211db2d25c8277ba534a73b92742a334874dced829a63b0f21221 AS unpack-templates diff --git a/Makefile b/Makefile index 883b2ad1..c4dd7c00 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,11 @@ e2e-test: .PHONY: e2e-test-upgrade e2e-test-upgrade: - go test -count=1 -tags=upgrade -v -timeout 30m ./test/... + go test -count=1 -tags=upgrade -v -timeout 30m ./test/... + +.PHONY: e2e-test-fips +e2e-test-fips: + go test -count=1 -tags=fips -v -timeout 30m ./test/... # Generate related image FILE := helm-charts/policy-controller-operator/values.yaml diff --git a/bundle.Dockerfile b/bundle.Dockerfile index a9999df5..0eb70d5b 100644 --- a/bundle.Dockerfile +++ b/bundle.Dockerfile @@ -59,7 +59,7 @@ LABEL name="rhtas/policy-controller-operator-bundle" LABEL features.operators.openshift.io/cni="false" LABEL features.operators.openshift.io/disconnected="false" -LABEL features.operators.openshift.io/fips-compliant="false" +LABEL features.operators.openshift.io/fips-compliant="true" LABEL features.operators.openshift.io/proxy-aware="false" LABEL features.operators.openshift.io/cnf="false" LABEL features.operators.openshift.io/csi="false" diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml index 574cf754..2e0265f6 100644 --- a/config/default/metrics_service.yaml +++ b/config/default/metrics_service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: labels: - control-plane: controller-manager + control-plane: policy-controller-operator app.kubernetes.io/name: policy-controller-operator app.kubernetes.io/managed-by: kustomize name: controller-manager-metrics-service @@ -14,4 +14,4 @@ spec: protocol: TCP targetPort: 8443 selector: - control-plane: controller-manager + control-plane: policy-controller-operator diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index f7efe732..f63f3e94 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Namespace metadata: labels: - control-plane: controller-manager + control-plane: policy-controller-operator app.kubernetes.io/name: policy-controller-operator app.kubernetes.io/managed-by: kustomize name: system @@ -13,20 +13,20 @@ metadata: name: controller-manager namespace: system labels: - control-plane: controller-manager + control-plane: policy-controller-operator app.kubernetes.io/name: policy-controller-operator app.kubernetes.io/managed-by: kustomize spec: selector: matchLabels: - control-plane: controller-manager + control-plane: policy-controller-operator replicas: 1 template: metadata: annotations: kubectl.kubernetes.io/default-container: manager labels: - control-plane: controller-manager + control-plane: policy-controller-operator spec: securityContext: runAsNonRoot: true diff --git a/config/manifests/bases/policy-controller-operator.clusterserviceversion.yaml b/config/manifests/bases/policy-controller-operator.clusterserviceversion.yaml index e6184a73..4cf66720 100644 --- a/config/manifests/bases/policy-controller-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/policy-controller-operator.clusterserviceversion.yaml @@ -9,7 +9,7 @@ metadata: features.operators.openshift.io/cni: "false" features.operators.openshift.io/csi: "false" features.operators.openshift.io/disconnected: "false" - features.operators.openshift.io/fips-compliant: "false" + features.operators.openshift.io/fips-compliant: "true" features.operators.openshift.io/proxy-aware: "false" features.operators.openshift.io/tls-profiles: "false" features.operators.openshift.io/token-auth-aws: "false" diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml index 193a9e54..2d119f1c 100644 --- a/config/network-policy/allow-metrics-traffic.yaml +++ b/config/network-policy/allow-metrics-traffic.yaml @@ -12,7 +12,7 @@ metadata: spec: podSelector: matchLabels: - control-plane: controller-manager + control-plane: policy-controller-operator policyTypes: - Ingress ingress: diff --git a/config/openshift/serving_cert_annotation_patch.yaml b/config/openshift/serving_cert_annotation_patch.yaml index b856e0e4..78d7cd0e 100644 --- a/config/openshift/serving_cert_annotation_patch.yaml +++ b/config/openshift/serving_cert_annotation_patch.yaml @@ -4,6 +4,6 @@ metadata: name: controller-manager-webhook-service namespace: system labels: - control-plane: controller-manager + control-plane: policy-controller-operator annotations: service.beta.openshift.io/serving-cert-secret-name: webhook-cert diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index bef1ae37..54fd71b7 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -3,7 +3,7 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: - control-plane: controller-manager + control-plane: policy-controller-operator app.kubernetes.io/name: policy-controller-operator app.kubernetes.io/managed-by: kustomize name: controller-manager-metrics-monitor @@ -27,4 +27,4 @@ spec: insecureSkipVerify: true selector: matchLabels: - control-plane: controller-manager + control-plane: policy-controller-operator diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml index f9991516..31c0092a 100644 --- a/config/webhook/service.yaml +++ b/config/webhook/service.yaml @@ -4,7 +4,7 @@ metadata: name: controller-manager-webhook-service namespace: system labels: - control-plane: controller-manager + control-plane: policy-controller-operator spec: ports: - name: https-webhook @@ -12,4 +12,4 @@ spec: targetPort: 9443 protocol: TCP selector: - control-plane: controller-manager + control-plane: policy-controller-operator diff --git a/go.mod b/go.mod index 6508cc47..8ff899ff 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,9 @@ require ( github.com/go-openapi/swag/stringutils v0.28.0 // indirect github.com/go-openapi/swag/typeutils v0.28.0 // indirect github.com/go-openapi/swag/yamlutils v0.28.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/klauspost/compress v1.19.2 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/secure-systems-lab/go-securesystemslib v0.11.0 // indirect @@ -48,6 +50,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20260727163830-6c54dddc4772 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gotest.tools/v3 v3.5.2 // indirect + k8s.io/streaming v0.36.3 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect ) diff --git a/go.sum b/go.sum index 0af6c68f..37ab60c7 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFe github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -103,6 +105,8 @@ github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -223,6 +227,8 @@ k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= +k8s.io/streaming v0.36.3 h1:9rAaqBk0C0Pc7+/fqGekj07NV+/Xrew58p647A0JT8w= +k8s.io/streaming v0.36.3/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= diff --git a/helm-charts/policy-controller-operator/values.yaml b/helm-charts/policy-controller-operator/values.yaml index 383a0c8f..ac029e86 100644 --- a/helm-charts/policy-controller-operator/values.yaml +++ b/helm-charts/policy-controller-operator/values.yaml @@ -16,7 +16,7 @@ policy-controller: name: webhook image: repository: registry.redhat.io/rhtas/policy-controller-rhel9 - version: sha256:c784547c9ad24ba39e3826607c945cabcee9cbc7b585795683a79611356589df + version: sha256:8bd5171bc05bdbf77622db9ea77f47a59e881cc876f0fadf2c03f7006eb816d5 pullPolicy: IfNotPresent env: KUBERNETES_MIN_VERSION: "1.29.0-0" diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 9e7e197c..1c4ee79b 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -58,6 +58,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { k8sClient, err = client.New(restCfg, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) + e2e_utils.SetRestConfig(restCfg) ctx = context.Background() @@ -88,6 +89,9 @@ var _ = SynchronizedBeforeSuite(func() []byte { return e2e_utils.WaitForDeploymentReady(ctx, k8sClient, e2e_utils.InstallNamespace, e2e_utils.DeploymentName) }).WithContext(ctx).Should(Succeed(), "timed out waiting for Deployment %q to be ready", e2e_utils.DeploymentName) + By("detecting and configuring FIPS mode if cluster is FIPS-enabled") + e2e_utils.DetectAndConfigureFIPS(ctx, k8sClient) + By("injecting CA") injectCA, err = strconv.ParseBool(strings.TrimSpace(e2e_utils.InjectCA())) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e_fips/e2e_fips_suite_test.go b/test/e2e_fips/e2e_fips_suite_test.go new file mode 100644 index 00000000..7c17c40b --- /dev/null +++ b/test/e2e_fips/e2e_fips_suite_test.go @@ -0,0 +1,115 @@ +//go:build fips + +package e2e_fips + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + e2e_utils "github.com/securesign/policy-controller-operator/test/utils" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/e2e-framework/klient/conf" +) + +var ( + k8sClient client.Client + ctx context.Context + scheme = runtime.NewScheme() +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) +} + +func TestFips(t *testing.T) { + format.MaxLength = 0 + RegisterFailHandler(Fail) + log.SetLogger(GinkgoLogr) + SetDefaultEventuallyTimeout(3 * time.Minute) + SetDefaultEventuallyPollingInterval(1 * time.Second) + EnforceDefaultTimeoutsWhenUsingContexts() + RunSpecs(t, "Policy Controller FIPS E2E Suite") +} + +var _ = SynchronizedBeforeSuite(func() []byte { + kubeconfig := conf.ResolveKubeConfigFile() + data, err := os.ReadFile(kubeconfig) + Expect(err).NotTo(HaveOccurred()) + return data +}, func(data []byte) { + restCfg, err := clientcmd.RESTConfigFromKubeConfig(data) + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err = client.New(restCfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + e2e_utils.SetRestConfig(restCfg) + + ctx = context.Background() + + fmt.Println(">>> Running FIPS E2E tests with the following parameters:") + fmt.Printf(" %-24s %s\n", "Install Namespace:", e2e_utils.InstallNamespace) + fmt.Printf(" %-24s %s\n", "Inject CA:", e2e_utils.InjectCA()) + + By("ensuring the policy-controller-operator namespace exists") + Expect(k8sClient.Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: e2e_utils.InstallNamespace}, + })).To(SatisfyAny(Succeed(), MatchError(ContainSubstring("already exists")))) + + By("applying the PolicyController CR") + renderedPolicyController, err := e2e_utils.RenderTemplate(e2e_utils.PolicyControllerCRPath, map[string]string{ + "NS": e2e_utils.InstallNamespace, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(e2e_utils.ApplyManifest(ctx, k8sClient, renderedPolicyController, "")).To(Succeed()) + + By("ensuring deployment is ready") + Eventually(func(ctx context.Context) error { + return e2e_utils.WaitForDeploymentReady(ctx, k8sClient, e2e_utils.InstallNamespace, e2e_utils.DeploymentName) + }).WithContext(ctx).Should(Succeed(), "timed out waiting for Deployment %q to be ready", e2e_utils.DeploymentName) + + By("detecting and configuring FIPS mode") + e2e_utils.DetectAndConfigureFIPS(ctx, k8sClient) + + By("injecting CA") + injectCA, err := strconv.ParseBool(strings.TrimSpace(e2e_utils.InjectCA())) + Expect(err).NotTo(HaveOccurred()) + if injectCA { + Expect(e2e_utils.InjectCAIntoDeployment(ctx, k8sClient, e2e_utils.DeploymentName, e2e_utils.InstallNamespace)).To(Succeed()) + Eventually(func(ctx context.Context) error { + return e2e_utils.WaitForConfigMapKey(ctx, k8sClient, e2e_utils.InstallNamespace, "trusted-ca-bundle", "ca-bundle.crt") + }).WithContext(ctx).Should(Succeed(), "trusted-ca-bundle never got its ca-bundle.crt") + + Eventually(func(ctx context.Context) error { + return e2e_utils.WaitForDeploymentReady(ctx, k8sClient, e2e_utils.InstallNamespace, e2e_utils.DeploymentName) + }).WithContext(ctx).Should(Succeed(), "timed out waiting for Deployment %q to be ready after CA injection", e2e_utils.DeploymentName) + } + + By("verifying all required resources are created") + e2e_utils.ExpectRequiredResources(ctx, k8sClient) +}) + +var _ = AfterSuite(func() { + By("cleaning up policy controller resources") + gvk := schema.GroupVersionKind{ + Group: "rhtas.charts.redhat.com", + Version: "v1alpha1", + Kind: "PolicyController", + } + Expect(e2e_utils.DeleteResource(ctx, k8sClient, gvk, "policycontroller-sample", e2e_utils.InstallNamespace)).To(Succeed()) +}) diff --git a/test/e2e_fips/fips_test.go b/test/e2e_fips/fips_test.go new file mode 100644 index 00000000..8913e2fe --- /dev/null +++ b/test/e2e_fips/fips_test.go @@ -0,0 +1,26 @@ +//go:build fips + +package e2e_fips + +import ( + . "github.com/onsi/ginkgo/v2" + e2e_utils "github.com/securesign/policy-controller-operator/test/utils" +) + +var _ = Describe("Policy Controller FIPS Strict Mode (fips140=only)", Ordered, func() { + It("verifies the cluster kernel has FIPS enabled", func(ctx SpecContext) { + pod, containerName := e2e_utils.GetDeploymentPod(ctx, k8sClient, e2e_utils.InstallNamespace, e2e_utils.DeploymentName) + e2e_utils.VerifyFIPSKernel(pod.Name, containerName, pod.Namespace) + }) + + It("verifies the policy-controller webhook is running in FIPS mode", func(ctx SpecContext) { + pod, containerName := e2e_utils.GetDeploymentPod(ctx, k8sClient, e2e_utils.InstallNamespace, e2e_utils.DeploymentName) + e2e_utils.VerifyFIPSGoNative(pod.Name, containerName, pod.Namespace) + }) + + It("verifies the admission-webhook-controller is running in FIPS mode", func(ctx SpecContext) { + operatorDep := e2e_utils.FindOperatorDeployment(ctx, k8sClient) + pod, _ := e2e_utils.GetDeploymentPod(ctx, k8sClient, operatorDep.Namespace, operatorDep.Name) + e2e_utils.VerifyFIPSGoNative(pod.Name, "admission-webhook-controller", pod.Namespace) + }) +}) diff --git a/test/e2e_upgrade/e2e_suite_upgrade_test.go b/test/e2e_upgrade/e2e_suite_upgrade_test.go index 2f7b5c75..cbda2940 100644 --- a/test/e2e_upgrade/e2e_suite_upgrade_test.go +++ b/test/e2e_upgrade/e2e_suite_upgrade_test.go @@ -63,6 +63,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { k8sClient, err = client.New(restCfg, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) + e2e_utils.SetRestConfig(restCfg) ctx = context.Background() diff --git a/test/e2e_upgrade/upgrade_test.go b/test/e2e_upgrade/upgrade_test.go index dbbe16a1..64156a02 100644 --- a/test/e2e_upgrade/upgrade_test.go +++ b/test/e2e_upgrade/upgrade_test.go @@ -159,6 +159,9 @@ var _ = Describe("Operator upgrade", Ordered, func() { Eventually(func(ctx context.Context) error { return e2e_utils.WaitForDeploymentReady(ctx, k8sClient, e2e_utils.InstallNamespace, e2e_utils.DeploymentName) }).WithContext(ctx).Should(Succeed(), "timed out waiting for Deployment %q to be ready", e2e_utils.DeploymentName) + + By("detecting and configuring FIPS mode if cluster is FIPS-enabled") + e2e_utils.DetectAndConfigureFIPS(ctx, k8sClient) }) It("injects CA into the policy-controller deployment", func(ctx SpecContext) { @@ -292,6 +295,9 @@ var _ = Describe("Operator upgrade", Ordered, func() { }).WithContext(ctx).Should(Succeed(), "timed out waiting for Deployment %q to be ready after CA injection", e2e_utils.DeploymentName) } + By("re-configuring FIPS mode after upgrade if cluster is FIPS-enabled") + e2e_utils.DetectAndConfigureFIPS(ctx, k8sClient) + By("allowing already signed images through") e2e_utils.Verify(ctx, k8sClient, upgradeTestNS, upgradeTestImage, false) diff --git a/test/utils/fips.go b/test/utils/fips.go new file mode 100644 index 00000000..9a9067e3 --- /dev/null +++ b/test/utils/fips.go @@ -0,0 +1,347 @@ +package e2e_utils + +import ( + "bytes" + "context" + "fmt" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/client-go/util/retry" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + restConfig *rest.Config + clientset *kubernetes.Clientset +) + +// SetRestConfig makes the REST config available to ExecInPod. Suite setup +// must call this alongside constructing the controller-runtime client. +func SetRestConfig(cfg *rest.Config) { + restConfig = cfg + cs, err := kubernetes.NewForConfig(cfg) + Expect(err).ToNot(HaveOccurred(), "building clientset for ExecInPod") + clientset = cs +} + +func ExecInPod(podName, containerName, namespace string, command ...string) (string, error) { + if clientset == nil { + return "", fmt.Errorf("ExecInPod: rest config not set, call SetRestConfig during suite setup") + } + + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec") + req.VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: command, + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(restConfig, "POST", req.URL()) + if err != nil { + return "", fmt.Errorf("creating executor: %w", err) + } + + var stdout, stderr bytes.Buffer + if err := executor.StreamWithContext(context.Background(), remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }); err != nil { + return "", fmt.Errorf("pod exec failed: %w, stderr: %s", err, stderr.String()) + } + return stdout.String(), nil +} + +func getRunningPods(ctx context.Context, k8sClient client.Client, namespace string, selector map[string]string) ([]*corev1.Pod, error) { + podList := &corev1.PodList{} + if err := k8sClient.List(ctx, podList, + client.InNamespace(namespace), + client.MatchingLabels(selector), + ); err != nil { + return nil, err + } + var running []*corev1.Pod + for i := range podList.Items { + pod := &podList.Items[i] + if pod.Status.Phase == corev1.PodRunning && pod.DeletionTimestamp == nil { + running = append(running, pod) + } + } + return running, nil +} + +func GetDeploymentPod(ctx context.Context, k8sClient client.Client, namespace, deploymentName string) (*corev1.Pod, string) { + deploy := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: deploymentName}, deploy)).To(Succeed()) + + pods, err := getRunningPods(ctx, k8sClient, namespace, deploy.Spec.Selector.MatchLabels) + Expect(err).ToNot(HaveOccurred()) + Expect(pods).ToNot(BeEmpty(), "no running pod found for deployment %s", deploymentName) + + pod := pods[0] + Expect(pod.Spec.Containers).ToNot(BeEmpty()) + return pod, pod.Spec.Containers[0].Name +} + +func FindOperatorDeployment(ctx context.Context, k8sClient client.Client) *appsv1.Deployment { + depList := &appsv1.DeploymentList{} + Expect(k8sClient.List(ctx, depList, + client.MatchingLabels{"control-plane": "policy-controller-operator"}, + )).To(Succeed()) + Expect(depList.Items).ToNot(BeEmpty(), + "could not find operator deployment with label control-plane=policy-controller-operator") + return &depList.Items[0] +} + +func PatchDeploymentEnv(ctx context.Context, k8sClient client.Client, namespace, deploymentName string, env corev1.EnvVar) { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + deploy := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: deploymentName}, deploy); err != nil { + return err + } + for i := range deploy.Spec.Template.Spec.Containers { + ensureEnv(&deploy.Spec.Template.Spec.Containers[i], env) + } + return k8sClient.Update(ctx, deploy) + }) + Expect(err).ToNot(HaveOccurred()) +} + +func hasEnvVars(pod *corev1.Pod, envs []corev1.EnvVar) bool { + for _, want := range envs { + found := false + for _, c := range pod.Spec.Containers { + for _, got := range c.Env { + if got.Name == want.Name && got.Value == want.Value { + found = true + break + } + } + if found { + break + } + } + if !found { + return false + } + } + return true +} + +func WaitForDeploymentPodWithEnv(ctx context.Context, k8sClient client.Client, namespace, deploymentName string, envs ...corev1.EnvVar) *corev1.Pod { + deploy := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: deploymentName}, deploy)).To(Succeed()) + selector := deploy.Spec.Selector.MatchLabels + + var result *corev1.Pod + Eventually(func(g Gomega, ctx context.Context) { + pods, err := getRunningPods(ctx, k8sClient, namespace, selector) + g.Expect(err).ToNot(HaveOccurred()) + + for _, pod := range pods { + ready := len(pod.Status.ContainerStatuses) > 0 + for _, cs := range pod.Status.ContainerStatuses { + if !cs.Ready { + ready = false + break + } + } + if !ready { + continue + } + if hasEnvVars(pod, envs) { + result = pod + return + } + } + g.Expect(result).ToNot(BeNil(), "waiting for ready pod with required env vars in deployment %s", deploymentName) + }).WithContext(ctx).Should(Succeed()) + return result +} + +func VerifyFIPSKernel(podName, containerName, namespace string) { + out, err := ExecInPod(podName, containerName, namespace, "cat", "/proc/sys/crypto/fips_enabled") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.TrimSpace(out)).To(Equal("1"), "kernel FIPS mode should be enabled") +} + +func VerifyGodebugFIPSOnly(podName, containerName, namespace string) { + out, err := ExecInPod(podName, containerName, namespace, "printenv", "GODEBUG") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.TrimSpace(out)).To(Equal("fips140=only"), + fmt.Sprintf("container %s should have GODEBUG=fips140=only", containerName)) +} + +func VerifyBinaryGOFIPS(podName, containerName, namespace, binaryPath string) { + out, err := ExecInPod(podName, containerName, namespace, + "grep", "-aom", "1", "GOFIPS140=", binaryPath) + Expect(err).ToNot(HaveOccurred()) + Expect(strings.TrimSpace(out)).To(ContainSubstring("GOFIPS140="), + fmt.Sprintf("binary %s should be built with GOFIPS140", binaryPath)) +} + +func GetBinaryPath(podName, containerName, namespace string) string { + out, err := ExecInPod(podName, containerName, namespace, "readlink", "-f", "/proc/1/exe") + Expect(err).ToNot(HaveOccurred()) + path := strings.TrimSpace(out) + Expect(path).ToNot(BeEmpty(), "could not determine binary path from /proc/1/exe") + return path +} + +func VerifyFIPSGoNative(podName, containerName, namespace string) { + VerifyFIPSKernel(podName, containerName, namespace) + VerifyGodebugFIPSOnly(podName, containerName, namespace) + binaryPath := GetBinaryPath(podName, containerName, namespace) + VerifyBinaryGOFIPS(podName, containerName, namespace, binaryPath) +} + +var ( + fipsEnabled bool + fipsResolved bool +) + +func checkFIPSKernel(podName, containerName, namespace string) (bool, error) { + if fipsResolved { + return fipsEnabled, nil + } + out, err := ExecInPod(podName, containerName, namespace, "cat", "/proc/sys/crypto/fips_enabled") + if err != nil { + return false, err + } + fipsEnabled = strings.TrimSpace(out) == "1" + fipsResolved = true + return fipsEnabled, nil +} + +// IsFIPSCluster checks whether the host kernel has FIPS enabled. +// The result is cached after the first successful check; transient +// ExecInPod failures are retried via Eventually. +func IsFIPSCluster(podName, containerName, namespace string) bool { + var isFIPS bool + Eventually(func() error { + var err error + isFIPS, err = checkFIPSKernel(podName, containerName, namespace) + return err + }).Should(Succeed(), "failed to detect FIPS status by execing into pod %s/%s", namespace, podName) + if isFIPS { + fmt.Println(" FIPS detection: cluster IS FIPS-enabled") + } else { + fmt.Println(" FIPS detection: cluster is NOT FIPS-enabled") + } + return isFIPS +} + +func DetectAndConfigureFIPS(ctx context.Context, k8sClient client.Client) { + pod, containerName := GetDeploymentPod(ctx, k8sClient, InstallNamespace, DeploymentName) + + if !IsFIPSCluster(pod.Name, containerName, pod.Namespace) { + return + } + + godebug := corev1.EnvVar{Name: "GODEBUG", Value: "fips140=only"} + + By("patching operator CSV/deployment with GODEBUG=fips140=only") + operatorDep := FindOperatorDeployment(ctx, k8sClient) + patchOperatorFIPSEnv(ctx, k8sClient, operatorDep, godebug) + WaitForDeploymentPodWithEnv(ctx, k8sClient, operatorDep.Namespace, operatorDep.Name, godebug) + + By("patching webhook deployment with GODEBUG=fips140=only") + PatchDeploymentEnv(ctx, k8sClient, InstallNamespace, DeploymentName, godebug) + WaitForDeploymentPodWithEnv(ctx, k8sClient, InstallNamespace, DeploymentName, godebug) +} + +func patchOperatorFIPSEnv(ctx context.Context, k8sClient client.Client, deploy *appsv1.Deployment, env corev1.EnvVar) { + csvName := findCSVForDeployment(ctx, k8sClient, deploy) + if csvName != "" { + patchCSVDeploymentEnv(ctx, k8sClient, deploy.Namespace, csvName, deploy.Name, env) + return + } + PatchDeploymentEnv(ctx, k8sClient, deploy.Namespace, deploy.Name, env) +} + +func findCSVForDeployment(ctx context.Context, k8sClient client.Client, deploy *appsv1.Deployment) string { + for _, ref := range deploy.OwnerReferences { + if ref.Kind == "ClusterServiceVersion" { + return ref.Name + } + } + csvName, err := GetCSVName(ctx, k8sClient, deploy.Namespace, PackageName) + if err != nil || csvName == "" { + return "" + } + return csvName +} + +func patchCSVDeploymentEnv(ctx context.Context, k8sClient client.Client, namespace, csvName, deployName string, env corev1.EnvVar) { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + csv, err := GetCSV(ctx, k8sClient, namespace, csvName) + if err != nil { + return err + } + + deployments, found, err := unstructured.NestedSlice(csv.Object, "spec", "install", "spec", "deployments") + if err != nil || !found { + return fmt.Errorf("CSV %s has no spec.install.spec.deployments: %w", csvName, err) + } + + for i, raw := range deployments { + dep, ok := raw.(map[string]any) + if !ok { + continue + } + name, _ := dep["name"].(string) + if name != deployName { + continue + } + + containers, _, _ := unstructured.NestedSlice(dep, "spec", "template", "spec", "containers") + for j, cRaw := range containers { + c, ok := cRaw.(map[string]any) + if !ok { + continue + } + envList, _, _ := unstructured.NestedSlice(c, "env") + updated := false + for k, eRaw := range envList { + e, ok := eRaw.(map[string]any) + if !ok { + continue + } + if e["name"] == env.Name { + envList[k] = map[string]any{"name": env.Name, "value": env.Value} + updated = true + break + } + } + if !updated { + envList = append(envList, map[string]any{"name": env.Name, "value": env.Value}) + } + c["env"] = envList + containers[j] = c + } + if err := unstructured.SetNestedSlice(dep, containers, "spec", "template", "spec", "containers"); err != nil { + return err + } + deployments[i] = dep + } + + if err := unstructured.SetNestedSlice(csv.Object, deployments, "spec", "install", "spec", "deployments"); err != nil { + return err + } + return k8sClient.Update(ctx, csv) + }) + Expect(err).ToNot(HaveOccurred(), "failed to patch CSV %s deployment %s with %s=%s", csvName, deployName, env.Name, env.Value) +} diff --git a/test/utils/kubernetes.go b/test/utils/kubernetes.go index cc32f4cc..c6f87a5d 100644 --- a/test/utils/kubernetes.go +++ b/test/utils/kubernetes.go @@ -15,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -149,36 +150,38 @@ func InjectCAIntoDeployment(ctx context.Context, k8sClient client.Client, deploy return fmt.Errorf("ensuring trusted CA ConfigMap: %w", err) } - var deploy appsv1.Deployment - if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: deploymentName}, &deploy); err != nil { - return err - } + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + var deploy appsv1.Deployment + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: deploymentName}, &deploy); err != nil { + return err + } - ensureVolume(&deploy, corev1.Volume{ - Name: "trusted-ca", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: "trusted-ca-bundle"}, - Items: []corev1.KeyToPath{ - {Key: "ca-bundle.crt", Path: "ca-bundle.crt"}, + ensureVolume(&deploy, corev1.Volume{ + Name: "trusted-ca", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "trusted-ca-bundle"}, + Items: []corev1.KeyToPath{ + {Key: "ca-bundle.crt", Path: "ca-bundle.crt"}, + }, }, }, - }, - }) - - for i := range deploy.Spec.Template.Spec.Containers { - ensureMount(&deploy.Spec.Template.Spec.Containers[i], corev1.VolumeMount{ - Name: "trusted-ca", - MountPath: "/etc/ssl/certs/tls-ca-bundle.pem", - SubPath: "ca-bundle.crt", - ReadOnly: true, - }) - ensureEnv(&deploy.Spec.Template.Spec.Containers[i], corev1.EnvVar{ - Name: "SSL_CERT_DIR", - Value: "/var/run/secrets/kubernetes.io/serviceaccount:/etc/ssl/certs", }) - } - return k8sClient.Update(ctx, &deploy) + + for i := range deploy.Spec.Template.Spec.Containers { + ensureMount(&deploy.Spec.Template.Spec.Containers[i], corev1.VolumeMount{ + Name: "trusted-ca", + MountPath: "/etc/ssl/certs/tls-ca-bundle.pem", + SubPath: "ca-bundle.crt", + ReadOnly: true, + }) + ensureEnv(&deploy.Spec.Template.Spec.Containers[i], corev1.EnvVar{ + Name: "SSL_CERT_DIR", + Value: "/var/run/secrets/kubernetes.io/serviceaccount:/etc/ssl/certs", + }) + } + return k8sClient.Update(ctx, &deploy) + }) } func ensureVolume(d *appsv1.Deployment, v corev1.Volume) {