diff --git a/pkg/cmd/openshift-tests/list/all_tests.go b/pkg/cmd/openshift-tests/list/all_tests.go new file mode 100644 index 000000000000..295723149ff5 --- /dev/null +++ b/pkg/cmd/openshift-tests/list/all_tests.go @@ -0,0 +1,157 @@ +package list + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/openshift/origin/pkg/test/extensions" + "github.com/openshift/origin/pkg/testsuites" +) + +// resolveSuiteQualifiers finds a suite by name, first checking origin's internal +// suites, then checking suites advertised by the already-extracted extension binaries. +func resolveSuiteQualifiers(ctx context.Context, suiteName string, binaries extensions.TestBinaries) ([]string, error) { + for _, s := range testsuites.InternalTestSuites() { + if s.Name == suiteName { + return s.Qualifiers, nil + } + } + + extensionInfos, err := binaries.Info(ctx, 4) + if err != nil { + return nil, fmt.Errorf("failed to get extension info: %w", err) + } + for _, e := range extensionInfos { + for _, s := range e.Suites { + if s.Name == suiteName { + return s.Qualifiers, nil + } + } + } + + return nil, fmt.Errorf("suite %q not found", suiteName) +} + +func NewListAllTestsCommand(streams genericclioptions.IOStreams) *cobra.Command { + var suiteName string + + cmd := &cobra.Command{ + Use: "all-tests", + Short: "List tests from all extension binaries", + Long: templates.LongDesc(` + List all tests discovered from all extension binaries in the release payload. + + Unlike 'list tests', which lists tests from a single extension component, + this command aggregates tests from all extension binaries — the same set + of tests that 'run' operates on. + + Use --suite to filter tests by a suite's qualifiers. This works with both + origin-defined suites (like openshift/network/third-party) and suites + advertised by extension binaries. + + This command does not require cluster access. + `), + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + + const defaultBinaryParallelism = 10 + + extractionContext, extractionContextCancel := context.WithTimeout(ctx, 30*60*1e9) + defer extractionContextCancel() + cleanUpFn, allBinaries, _, err := extensions.ExtractAllTestBinaries(extractionContext, defaultBinaryParallelism, extensions.WithPayloadOnly()) + if err != nil { + return fmt.Errorf("failed to extract test binaries: %w", err) + } + defer cleanUpFn() + + infoContext, infoContextCancel := context.WithTimeout(ctx, 30*60*1e9) + defer infoContextCancel() + logrus.Infof("Fetching info from %d extension binaries", len(allBinaries)) + if _, err := allBinaries.Info(infoContext, defaultBinaryParallelism); err != nil { + logrus.Warnf("Some extension binaries failed info fetch (they may require cluster access): %v", err) + } + + // Filter to binaries that successfully returned info, since ListTests + // requires info to be populated. Binaries that failed info (e.g. due to + // missing cluster access) are skipped. + var availableBinaries extensions.TestBinaries + for _, b := range allBinaries { + if b.HasInfo() { + availableBinaries = append(availableBinaries, b) + } + } + logrus.Infof("%d of %d binaries available for listing", len(availableBinaries), len(allBinaries)) + + listContext, listContextCancel := context.WithTimeout(ctx, 10*60*1e9) + defer listContextCancel() + + specs, err := availableBinaries.ListTests(listContext, defaultBinaryParallelism, nil) + if err != nil { + return fmt.Errorf("failed to list tests: %w", err) + } + + logrus.Infof("Discovered %d total tests", len(specs)) + + if suiteName != "" { + qualifiers, err := resolveSuiteQualifiers(ctx, suiteName, availableBinaries) + if err != nil { + return err + } + + specs, err = extensions.FilterWrappedSpecs(specs, qualifiers) + if err != nil { + return fmt.Errorf("failed to filter tests by suite qualifiers: %w", err) + } + + logrus.Infof("Suite %q selected %d tests", suiteName, len(specs)) + } + + outputFormat, err := cmd.Flags().GetString("output") + if err != nil { + return errors.Wrapf(err, "error accessing flag output for command %s", cmd.Name()) + } + + sort.Slice(specs, func(i, j int) bool { + return specs[i].Name < specs[j].Name + }) + + switch outputFormat { + case "": + for _, spec := range specs { + fmt.Fprintln(streams.Out, spec.Name) + } + case "json": + data, err := json.MarshalIndent(specs, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal tests to JSON: %w", err) + } + fmt.Fprintln(streams.Out, string(data)) + case "yaml": + data, err := yaml.Marshal(specs) + if err != nil { + return fmt.Errorf("failed to marshal tests to YAML: %w", err) + } + fmt.Fprintln(streams.Out, string(data)) + default: + return fmt.Errorf("invalid output format: %s", outputFormat) + } + + return nil + }, + } + + cmd.Flags().StringVar(&suiteName, "suite", "", "Filter tests by the qualifiers of the specified suite") + cmd.Flags().StringP("output", "o", "", "Output format; available options are 'yaml' and 'json'") + return cmd +} diff --git a/pkg/cmd/openshift-tests/list/root.go b/pkg/cmd/openshift-tests/list/root.go index 4bc85953e680..1e102849283d 100644 --- a/pkg/cmd/openshift-tests/list/root.go +++ b/pkg/cmd/openshift-tests/list/root.go @@ -22,6 +22,7 @@ func NewListCommand(streams genericclioptions.IOStreams, extensionRegistry *exte oteListCmd.AddCommand( NewListSuitesCommand(streams), NewListExtensionsCommand(streams), + NewListAllTestsCommand(streams), ) return oteListCmd diff --git a/pkg/test/extensions/binary.go b/pkg/test/extensions/binary.go index 1fdf4a13b607..0f24079ad264 100644 --- a/pkg/test/extensions/binary.go +++ b/pkg/test/extensions/binary.go @@ -171,6 +171,16 @@ type TestBinary struct { info *Extension } +// Name returns the base name of the binary. +func (b *TestBinary) Name() string { + return filepath.Base(b.binaryPath) +} + +// HasInfo returns true if Info() has been successfully called on this binary. +func (b *TestBinary) HasInfo() bool { + return b.info != nil +} + // UnpermittedExtension describes a discovered non-payload extension that is not permitted by any TestExtensionAdmission. // Used to generate a synthetic skip test. type UnpermittedExtension struct { @@ -233,10 +243,6 @@ var extensionBinaries = []TestBinary{ imageTag: "cluster-cloud-controller-manager-operator", binaryPath: "/usr/bin/cloud-controller-manager-aws-tests-ext.gz", }, - { - imageTag: "cluster-cloud-controller-manager-operator", - binaryPath: "/usr/bin/cloud-controller-manager-operator-tests-ext.gz", - }, { imageTag: "cluster-config-operator", binaryPath: "/usr/bin/cluster-config-operator-tests-ext.gz", @@ -614,10 +620,31 @@ func (b *TestBinary) ListImages(ctx context.Context) (ImageSet, error) { return result, nil } +// ExtractionOption configures the behavior of ExtractAllTestBinaries. +type ExtractionOption func(*extractionOptions) + +type extractionOptions struct { + payloadOnly bool +} + +// WithPayloadOnly skips non-payload extension discovery, which requires cluster +// access. When set, only payload test binaries are extracted. Registry auth must +// be provided via REGISTRY_AUTH_FILE or CI cluster profile. +func WithPayloadOnly() ExtractionOption { + return func(o *extractionOptions) { + o.payloadOnly = true + } +} + // ExtractAllTestBinaries determines the optimal release payload to use, and extracts all the external // test binaries from it (payload + permitted non-payload), and returns cleanup, binaries, and any // unpermitted non-payload extensions for synthetic skip tests. -func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestBinaries, []UnpermittedExtension, error) { +func ExtractAllTestBinaries(ctx context.Context, parallelism int, opts ...ExtractionOption) (func(), TestBinaries, []UnpermittedExtension, error) { + var options extractionOptions + for _, opt := range opts { + opt(&options) + } + if len(os.Getenv("OPENSHIFT_SKIP_EXTERNAL_TESTS")) > 0 { logrus.Warning("Using built-in tests only due to OPENSHIFT_SKIP_EXTERNAL_TESTS being set") var internalBinaries []*TestBinary @@ -649,8 +676,14 @@ func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestB defer os.RemoveAll(tmpDir) - oc := exutil.NewCLIWithoutNamespace("default") - registryAuthFilePath, err := DetermineRegistryAuthFilePath(tmpDir, oc) + var registryAuthFilePath string + var oc *exutil.CLI + if options.payloadOnly { + registryAuthFilePath, err = DetermineRegistryAuthFilePathWithoutCluster(tmpDir) + } else { + oc = exutil.NewCLIWithoutNamespace("default") + registryAuthFilePath, err = DetermineRegistryAuthFilePath(tmpDir, oc) + } if err != nil { return nil, nil, nil, fmt.Errorf("failed to determine registry auth file path: %w", err) } @@ -660,14 +693,18 @@ func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestB return nil, nil, nil, errors.WithMessage(err, "could not create external binary provider") } - permitPatterns, err := DiscoverNonPayloadBinaryAdmission(ctx, oc.AdminConfig()) - if err != nil { - logrus.Warnf("Skipping non-payload extension discovery (admission check failed): %v", err) - permitPatterns = nil - } - permittedNonPayload, unpermittedNonPayload, err := discoverNonPayloadExtensions(ctx, oc, permitPatterns) - if err != nil { - logrus.Warnf("Non-payload extension discovery failed: %v", err) + var permittedNonPayload []nonPayloadSource + var unpermittedNonPayload []UnpermittedExtension + if !options.payloadOnly { + permitPatterns, err := DiscoverNonPayloadBinaryAdmission(ctx, oc.AdminConfig()) + if err != nil { + logrus.Warnf("Skipping non-payload extension discovery (admission check failed): %v", err) + permitPatterns = nil + } + permittedNonPayload, unpermittedNonPayload, err = discoverNonPayloadExtensions(ctx, oc, permitPatterns) + if err != nil { + logrus.Warnf("Non-payload extension discovery failed: %v", err) + } } var ( diff --git a/pkg/test/extensions/util.go b/pkg/test/extensions/util.go index fb2f2e0f7808..38ea454ed5fd 100644 --- a/pkg/test/extensions/util.go +++ b/pkg/test/extensions/util.go @@ -289,6 +289,28 @@ func ExtractReleaseImageStream(extractPath, releaseImage string, return is, releaseImage, nil } +// DetermineRegistryAuthFilePathWithoutCluster resolves registry auth without +// cluster access. It checks REGISTRY_AUTH_FILE and the CI cluster profile path, +// falling back to unauthenticated access if neither is available. +func DetermineRegistryAuthFilePathWithoutCluster(tmpDir string) (string, error) { + registryAuthFilePath := os.Getenv("REGISTRY_AUTH_FILE") + if len(registryAuthFilePath) != 0 { + logrus.Infof("Using REGISTRY_AUTH_FILE environment variable: %v", registryAuthFilePath) + return registryAuthFilePath, nil + } + + ciProfilePullSecretPath := "/run/secrets/ci.openshift.io/cluster-profile/pull-secret" + if _, err := os.Stat(ciProfilePullSecretPath); err == nil { + logrus.Infof("Detected %v; using cluster profile for image access", ciProfilePullSecretPath) + return ciProfilePullSecretPath, nil + } else if !os.IsNotExist(err) { + return "", fmt.Errorf("failed to check for CI pull secret at %s: %w", ciProfilePullSecretPath, err) + } + + logrus.Warningf("No REGISTRY_AUTH_FILE or cluster profile pull-secret found; falling back to default container credentials") + return "", nil +} + func DetermineRegistryAuthFilePath(tmpDir string, oc *util.CLI) (string, error) { // To extract binaries bearing external tests, we must inspect the release // payload under tests as well as extract content from component images diff --git a/test/extended/extension/payload_compliance.go b/test/extended/extension/payload_compliance.go new file mode 100644 index 000000000000..21645e7f45c6 --- /dev/null +++ b/test/extended/extension/payload_compliance.go @@ -0,0 +1,53 @@ +package extension + +import ( + "context" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + "github.com/openshift/origin/pkg/test/extensions" + "k8s.io/apimachinery/pkg/util/sets" +) + +// knownInfoFailures lists extension binaries that are known to fail the +// "info" command without cluster access. Each entry should have a tracking +// issue for fixing the upstream binary. Remove entries as fixes land. +var knownInfoFailures = sets.New[string]( + "ovn-kubernetes-tests-ext", // https://github.com/openshift/ovn-kubernetes/pull/3170 + "cloud-controller-manager-aws-tests-ext", // https://github.com/openshift/cluster-cloud-controller-manager-operator/pull/458 +) + +var _ = g.Describe("[sig-ci] [OTE] Payload extension binaries [Suite:openshift/conformance/parallel]", func() { + defer g.GinkgoRecover() + + g.It("should all respond to the info command", func(ctx context.Context) { + extractCtx, extractCancel := context.WithTimeout(ctx, 30*time.Minute) + defer extractCancel() + + cleanUpFn, allBinaries, _, err := extensions.ExtractAllTestBinaries(extractCtx, 10, extensions.WithPayloadOnly()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to extract test binaries from payload") + defer cleanUpFn() + + var failures []string + for _, binary := range allBinaries { + binName := binary.Name() + infoCtx, infoCancel := context.WithTimeout(ctx, 10*time.Minute) + _, err := binary.Info(infoCtx) + infoCancel() + + if err != nil { + if knownInfoFailures.Has(binName) { + g.GinkgoLogr.Info("Skipping known info failure", "binary", binName, "error", err) + continue + } + failures = append(failures, binName+": "+err.Error()) + } else if knownInfoFailures.Has(binName) { + failures = append(failures, binName+": listed in knownInfoFailures but info succeeded — remove from exemption list") + } + } + + o.Expect(failures).To(o.BeEmpty(), "extension binaries failed the OTE info contract") + }) +}) diff --git a/test/extended/router/config_manager.go b/test/extended/router/config_manager.go index 25fa4a44bc38..51a121329045 100644 --- a/test/extended/router/config_manager.go +++ b/test/extended/router/config_manager.go @@ -65,9 +65,9 @@ var _ = g.Describe("[sig-network][Feature:Router][apigroup:route.openshift.io]", // in order to allow binding 80/443 without being root. Without the privilege, // the capability does not take effect and haproxy fails to start. oc = exutil.NewCLIWithPodSecurityLevel("router-config-manager", api.LevelPrivileged) - kubeClient = oc.AdminKubeClient() g.BeforeEach(func() { + kubeClient = oc.AdminKubeClient() ns = oc.Namespace() routerImage, err := exutil.FindRouterImage(oc) diff --git a/test/extended/router/config_manager_ingress.go b/test/extended/router/config_manager_ingress.go index 5325c72943c3..b7d4a1fb5c80 100644 --- a/test/extended/router/config_manager_ingress.go +++ b/test/extended/router/config_manager_ingress.go @@ -52,10 +52,10 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. ctx := context.Background() oc := exutil.NewCLIWithPodSecurityLevel("router-dcm-ingress", api.LevelPrivileged).AsAdmin() - kubeClient := oc.AdminKubeClient() // variables updated on every new test var ( + kubeClient kubernetes.Interface execPod execPodRef controller types.NamespacedName routeSelectorSet labels.Set @@ -69,6 +69,7 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. }) g.BeforeEach(func() { + kubeClient = oc.AdminKubeClient() // ingress controller need to be created in operator's namespace, ... nsOperator := "openshift-ingress-operator" controllerName := names.SimpleNameGenerator.GenerateName("e2e-dcm-")