From aa53685a94327f8eea28364eeca25e8e6df61108 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Thu, 30 Apr 2026 18:34:11 +0200 Subject: [PATCH 1/4] Defer router test kubeClient init to BeforeEach Move oc.AdminKubeClient() calls from Describe body into BeforeEach in router config manager tests. Ginkgo Describe bodies run during tree building, so deferring cluster access to BeforeEach avoids unnecessary coupling between test discovery and cluster availability. This improves compatibility with workflows that build the test tree without a live cluster connection. Co-Authored-By: Claude Opus 4.6 --- test/extended/router/config_manager.go | 2 +- test/extended/router/config_manager_ingress.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/extended/router/config_manager.go b/test/extended/router/config_manager.go index 25fa4a44bc38..51a121329045 100644 --- a/test/extended/router/config_manager.go +++ b/test/extended/router/config_manager.go @@ -65,9 +65,9 @@ var _ = g.Describe("[sig-network][Feature:Router][apigroup:route.openshift.io]", // in order to allow binding 80/443 without being root. Without the privilege, // the capability does not take effect and haproxy fails to start. oc = exutil.NewCLIWithPodSecurityLevel("router-config-manager", api.LevelPrivileged) - kubeClient = oc.AdminKubeClient() g.BeforeEach(func() { + kubeClient = oc.AdminKubeClient() ns = oc.Namespace() routerImage, err := exutil.FindRouterImage(oc) diff --git a/test/extended/router/config_manager_ingress.go b/test/extended/router/config_manager_ingress.go index 5325c72943c3..b7d4a1fb5c80 100644 --- a/test/extended/router/config_manager_ingress.go +++ b/test/extended/router/config_manager_ingress.go @@ -52,10 +52,10 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. ctx := context.Background() oc := exutil.NewCLIWithPodSecurityLevel("router-dcm-ingress", api.LevelPrivileged).AsAdmin() - kubeClient := oc.AdminKubeClient() // variables updated on every new test var ( + kubeClient kubernetes.Interface execPod execPodRef controller types.NamespacedName routeSelectorSet labels.Set @@ -69,6 +69,7 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. }) g.BeforeEach(func() { + kubeClient = oc.AdminKubeClient() // ingress controller need to be created in operator's namespace, ... nsOperator := "openshift-ingress-operator" controllerName := names.SimpleNameGenerator.GenerateName("e2e-dcm-") From dc541a22451c31836b462f4401357aaf70c22d74 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Thu, 30 Apr 2026 18:34:20 +0200 Subject: [PATCH 2/4] Add WithPayloadOnly option to ExtractAllTestBinaries Add functional options pattern to ExtractAllTestBinaries so callers can skip non-payload extension discovery, which requires cluster access via oc client. This enables listing tests from payload binaries without a cluster connection. Also adds Name() and HasInfo() on TestBinary, and DetermineRegistryAuthFilePathWithoutCluster() for resolving registry auth without a cluster. Co-Authored-By: Claude Opus 4.6 --- pkg/test/extensions/binary.go | 67 +++++++++++++++++++++++++++-------- pkg/test/extensions/util.go | 22 ++++++++++++ 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/pkg/test/extensions/binary.go b/pkg/test/extensions/binary.go index 1fdf4a13b607..0f24079ad264 100644 --- a/pkg/test/extensions/binary.go +++ b/pkg/test/extensions/binary.go @@ -171,6 +171,16 @@ type TestBinary struct { info *Extension } +// Name returns the base name of the binary. +func (b *TestBinary) Name() string { + return filepath.Base(b.binaryPath) +} + +// HasInfo returns true if Info() has been successfully called on this binary. +func (b *TestBinary) HasInfo() bool { + return b.info != nil +} + // UnpermittedExtension describes a discovered non-payload extension that is not permitted by any TestExtensionAdmission. // Used to generate a synthetic skip test. type UnpermittedExtension struct { @@ -233,10 +243,6 @@ var extensionBinaries = []TestBinary{ imageTag: "cluster-cloud-controller-manager-operator", binaryPath: "/usr/bin/cloud-controller-manager-aws-tests-ext.gz", }, - { - imageTag: "cluster-cloud-controller-manager-operator", - binaryPath: "/usr/bin/cloud-controller-manager-operator-tests-ext.gz", - }, { imageTag: "cluster-config-operator", binaryPath: "/usr/bin/cluster-config-operator-tests-ext.gz", @@ -614,10 +620,31 @@ func (b *TestBinary) ListImages(ctx context.Context) (ImageSet, error) { return result, nil } +// ExtractionOption configures the behavior of ExtractAllTestBinaries. +type ExtractionOption func(*extractionOptions) + +type extractionOptions struct { + payloadOnly bool +} + +// WithPayloadOnly skips non-payload extension discovery, which requires cluster +// access. When set, only payload test binaries are extracted. Registry auth must +// be provided via REGISTRY_AUTH_FILE or CI cluster profile. +func WithPayloadOnly() ExtractionOption { + return func(o *extractionOptions) { + o.payloadOnly = true + } +} + // ExtractAllTestBinaries determines the optimal release payload to use, and extracts all the external // test binaries from it (payload + permitted non-payload), and returns cleanup, binaries, and any // unpermitted non-payload extensions for synthetic skip tests. -func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestBinaries, []UnpermittedExtension, error) { +func ExtractAllTestBinaries(ctx context.Context, parallelism int, opts ...ExtractionOption) (func(), TestBinaries, []UnpermittedExtension, error) { + var options extractionOptions + for _, opt := range opts { + opt(&options) + } + if len(os.Getenv("OPENSHIFT_SKIP_EXTERNAL_TESTS")) > 0 { logrus.Warning("Using built-in tests only due to OPENSHIFT_SKIP_EXTERNAL_TESTS being set") var internalBinaries []*TestBinary @@ -649,8 +676,14 @@ func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestB defer os.RemoveAll(tmpDir) - oc := exutil.NewCLIWithoutNamespace("default") - registryAuthFilePath, err := DetermineRegistryAuthFilePath(tmpDir, oc) + var registryAuthFilePath string + var oc *exutil.CLI + if options.payloadOnly { + registryAuthFilePath, err = DetermineRegistryAuthFilePathWithoutCluster(tmpDir) + } else { + oc = exutil.NewCLIWithoutNamespace("default") + registryAuthFilePath, err = DetermineRegistryAuthFilePath(tmpDir, oc) + } if err != nil { return nil, nil, nil, fmt.Errorf("failed to determine registry auth file path: %w", err) } @@ -660,14 +693,18 @@ func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestB return nil, nil, nil, errors.WithMessage(err, "could not create external binary provider") } - permitPatterns, err := DiscoverNonPayloadBinaryAdmission(ctx, oc.AdminConfig()) - if err != nil { - logrus.Warnf("Skipping non-payload extension discovery (admission check failed): %v", err) - permitPatterns = nil - } - permittedNonPayload, unpermittedNonPayload, err := discoverNonPayloadExtensions(ctx, oc, permitPatterns) - if err != nil { - logrus.Warnf("Non-payload extension discovery failed: %v", err) + var permittedNonPayload []nonPayloadSource + var unpermittedNonPayload []UnpermittedExtension + if !options.payloadOnly { + permitPatterns, err := DiscoverNonPayloadBinaryAdmission(ctx, oc.AdminConfig()) + if err != nil { + logrus.Warnf("Skipping non-payload extension discovery (admission check failed): %v", err) + permitPatterns = nil + } + permittedNonPayload, unpermittedNonPayload, err = discoverNonPayloadExtensions(ctx, oc, permitPatterns) + if err != nil { + logrus.Warnf("Non-payload extension discovery failed: %v", err) + } } var ( diff --git a/pkg/test/extensions/util.go b/pkg/test/extensions/util.go index fb2f2e0f7808..38ea454ed5fd 100644 --- a/pkg/test/extensions/util.go +++ b/pkg/test/extensions/util.go @@ -289,6 +289,28 @@ func ExtractReleaseImageStream(extractPath, releaseImage string, return is, releaseImage, nil } +// DetermineRegistryAuthFilePathWithoutCluster resolves registry auth without +// cluster access. It checks REGISTRY_AUTH_FILE and the CI cluster profile path, +// falling back to unauthenticated access if neither is available. +func DetermineRegistryAuthFilePathWithoutCluster(tmpDir string) (string, error) { + registryAuthFilePath := os.Getenv("REGISTRY_AUTH_FILE") + if len(registryAuthFilePath) != 0 { + logrus.Infof("Using REGISTRY_AUTH_FILE environment variable: %v", registryAuthFilePath) + return registryAuthFilePath, nil + } + + ciProfilePullSecretPath := "/run/secrets/ci.openshift.io/cluster-profile/pull-secret" + if _, err := os.Stat(ciProfilePullSecretPath); err == nil { + logrus.Infof("Detected %v; using cluster profile for image access", ciProfilePullSecretPath) + return ciProfilePullSecretPath, nil + } else if !os.IsNotExist(err) { + return "", fmt.Errorf("failed to check for CI pull secret at %s: %w", ciProfilePullSecretPath, err) + } + + logrus.Warningf("No REGISTRY_AUTH_FILE or cluster profile pull-secret found; falling back to default container credentials") + return "", nil +} + func DetermineRegistryAuthFilePath(tmpDir string, oc *util.CLI) (string, error) { // To extract binaries bearing external tests, we must inspect the release // payload under tests as well as extract content from component images From f408a27845fb3c98778b62309de806079d9e2c64 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Thu, 30 Apr 2026 18:34:28 +0200 Subject: [PATCH 3/4] Add list all-tests command for aggregated test listing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new "openshift-tests list all-tests" subcommand that lists tests from all extension binaries in the release payload. Unlike "list tests" which operates on a single OTE component, this command aggregates tests across all binaries — the same set that "run" operates on. The --suite flag filters tests by a suite's CEL qualifiers, working with both origin-defined suites (like openshift/network/third-party) and suites advertised by extension binaries. This enables validating suite selections without cluster access or --dry-run. Does not require cluster access. Extension binaries that fail info (e.g. due to missing KUBECONFIG) are skipped with a warning. Co-Authored-By: Claude Opus 4.6 --- pkg/cmd/openshift-tests/list/all_tests.go | 157 ++++++++++++++++++++++ pkg/cmd/openshift-tests/list/root.go | 1 + 2 files changed, 158 insertions(+) create mode 100644 pkg/cmd/openshift-tests/list/all_tests.go diff --git a/pkg/cmd/openshift-tests/list/all_tests.go b/pkg/cmd/openshift-tests/list/all_tests.go new file mode 100644 index 000000000000..295723149ff5 --- /dev/null +++ b/pkg/cmd/openshift-tests/list/all_tests.go @@ -0,0 +1,157 @@ +package list + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/openshift/origin/pkg/test/extensions" + "github.com/openshift/origin/pkg/testsuites" +) + +// resolveSuiteQualifiers finds a suite by name, first checking origin's internal +// suites, then checking suites advertised by the already-extracted extension binaries. +func resolveSuiteQualifiers(ctx context.Context, suiteName string, binaries extensions.TestBinaries) ([]string, error) { + for _, s := range testsuites.InternalTestSuites() { + if s.Name == suiteName { + return s.Qualifiers, nil + } + } + + extensionInfos, err := binaries.Info(ctx, 4) + if err != nil { + return nil, fmt.Errorf("failed to get extension info: %w", err) + } + for _, e := range extensionInfos { + for _, s := range e.Suites { + if s.Name == suiteName { + return s.Qualifiers, nil + } + } + } + + return nil, fmt.Errorf("suite %q not found", suiteName) +} + +func NewListAllTestsCommand(streams genericclioptions.IOStreams) *cobra.Command { + var suiteName string + + cmd := &cobra.Command{ + Use: "all-tests", + Short: "List tests from all extension binaries", + Long: templates.LongDesc(` + List all tests discovered from all extension binaries in the release payload. + + Unlike 'list tests', which lists tests from a single extension component, + this command aggregates tests from all extension binaries — the same set + of tests that 'run' operates on. + + Use --suite to filter tests by a suite's qualifiers. This works with both + origin-defined suites (like openshift/network/third-party) and suites + advertised by extension binaries. + + This command does not require cluster access. + `), + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + + const defaultBinaryParallelism = 10 + + extractionContext, extractionContextCancel := context.WithTimeout(ctx, 30*60*1e9) + defer extractionContextCancel() + cleanUpFn, allBinaries, _, err := extensions.ExtractAllTestBinaries(extractionContext, defaultBinaryParallelism, extensions.WithPayloadOnly()) + if err != nil { + return fmt.Errorf("failed to extract test binaries: %w", err) + } + defer cleanUpFn() + + infoContext, infoContextCancel := context.WithTimeout(ctx, 30*60*1e9) + defer infoContextCancel() + logrus.Infof("Fetching info from %d extension binaries", len(allBinaries)) + if _, err := allBinaries.Info(infoContext, defaultBinaryParallelism); err != nil { + logrus.Warnf("Some extension binaries failed info fetch (they may require cluster access): %v", err) + } + + // Filter to binaries that successfully returned info, since ListTests + // requires info to be populated. Binaries that failed info (e.g. due to + // missing cluster access) are skipped. + var availableBinaries extensions.TestBinaries + for _, b := range allBinaries { + if b.HasInfo() { + availableBinaries = append(availableBinaries, b) + } + } + logrus.Infof("%d of %d binaries available for listing", len(availableBinaries), len(allBinaries)) + + listContext, listContextCancel := context.WithTimeout(ctx, 10*60*1e9) + defer listContextCancel() + + specs, err := availableBinaries.ListTests(listContext, defaultBinaryParallelism, nil) + if err != nil { + return fmt.Errorf("failed to list tests: %w", err) + } + + logrus.Infof("Discovered %d total tests", len(specs)) + + if suiteName != "" { + qualifiers, err := resolveSuiteQualifiers(ctx, suiteName, availableBinaries) + if err != nil { + return err + } + + specs, err = extensions.FilterWrappedSpecs(specs, qualifiers) + if err != nil { + return fmt.Errorf("failed to filter tests by suite qualifiers: %w", err) + } + + logrus.Infof("Suite %q selected %d tests", suiteName, len(specs)) + } + + outputFormat, err := cmd.Flags().GetString("output") + if err != nil { + return errors.Wrapf(err, "error accessing flag output for command %s", cmd.Name()) + } + + sort.Slice(specs, func(i, j int) bool { + return specs[i].Name < specs[j].Name + }) + + switch outputFormat { + case "": + for _, spec := range specs { + fmt.Fprintln(streams.Out, spec.Name) + } + case "json": + data, err := json.MarshalIndent(specs, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal tests to JSON: %w", err) + } + fmt.Fprintln(streams.Out, string(data)) + case "yaml": + data, err := yaml.Marshal(specs) + if err != nil { + return fmt.Errorf("failed to marshal tests to YAML: %w", err) + } + fmt.Fprintln(streams.Out, string(data)) + default: + return fmt.Errorf("invalid output format: %s", outputFormat) + } + + return nil + }, + } + + cmd.Flags().StringVar(&suiteName, "suite", "", "Filter tests by the qualifiers of the specified suite") + cmd.Flags().StringP("output", "o", "", "Output format; available options are 'yaml' and 'json'") + return cmd +} diff --git a/pkg/cmd/openshift-tests/list/root.go b/pkg/cmd/openshift-tests/list/root.go index 4bc85953e680..1e102849283d 100644 --- a/pkg/cmd/openshift-tests/list/root.go +++ b/pkg/cmd/openshift-tests/list/root.go @@ -22,6 +22,7 @@ func NewListCommand(streams genericclioptions.IOStreams, extensionRegistry *exte oteListCmd.AddCommand( NewListSuitesCommand(streams), NewListExtensionsCommand(streams), + NewListAllTestsCommand(streams), ) return oteListCmd From ee96fdbb6ef34e1710d0d721ef48c11bd2cad766 Mon Sep 17 00:00:00 2001 From: Petr Muller Date: Thu, 30 Apr 2026 18:34:35 +0200 Subject: [PATCH 4/4] Add e2e test validating payload extension binaries respond to info Add a ginkgo test that extracts all payload extension binaries and verifies each responds to the OTE "info" command. Binaries with known upstream issues are exempted; the test fails if an exempted binary starts succeeding, prompting removal from the exemption list. Runs as part of openshift/conformance/parallel. Co-Authored-By: Claude Opus 4.6 --- test/extended/extension/payload_compliance.go | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 test/extended/extension/payload_compliance.go diff --git a/test/extended/extension/payload_compliance.go b/test/extended/extension/payload_compliance.go new file mode 100644 index 000000000000..21645e7f45c6 --- /dev/null +++ b/test/extended/extension/payload_compliance.go @@ -0,0 +1,53 @@ +package extension + +import ( + "context" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + "github.com/openshift/origin/pkg/test/extensions" + "k8s.io/apimachinery/pkg/util/sets" +) + +// knownInfoFailures lists extension binaries that are known to fail the +// "info" command without cluster access. Each entry should have a tracking +// issue for fixing the upstream binary. Remove entries as fixes land. +var knownInfoFailures = sets.New[string]( + "ovn-kubernetes-tests-ext", // https://github.com/openshift/ovn-kubernetes/pull/3170 + "cloud-controller-manager-aws-tests-ext", // https://github.com/openshift/cluster-cloud-controller-manager-operator/pull/458 +) + +var _ = g.Describe("[sig-ci] [OTE] Payload extension binaries [Suite:openshift/conformance/parallel]", func() { + defer g.GinkgoRecover() + + g.It("should all respond to the info command", func(ctx context.Context) { + extractCtx, extractCancel := context.WithTimeout(ctx, 30*time.Minute) + defer extractCancel() + + cleanUpFn, allBinaries, _, err := extensions.ExtractAllTestBinaries(extractCtx, 10, extensions.WithPayloadOnly()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to extract test binaries from payload") + defer cleanUpFn() + + var failures []string + for _, binary := range allBinaries { + binName := binary.Name() + infoCtx, infoCancel := context.WithTimeout(ctx, 10*time.Minute) + _, err := binary.Info(infoCtx) + infoCancel() + + if err != nil { + if knownInfoFailures.Has(binName) { + g.GinkgoLogr.Info("Skipping known info failure", "binary", binName, "error", err) + continue + } + failures = append(failures, binName+": "+err.Error()) + } else if knownInfoFailures.Has(binName) { + failures = append(failures, binName+": listed in knownInfoFailures but info succeeded — remove from exemption list") + } + } + + o.Expect(failures).To(o.BeEmpty(), "extension binaries failed the OTE info contract") + }) +})