From f082d69512325e4ccca3cabefb0c1439391a5c05 Mon Sep 17 00:00:00 2001 From: Predrag Knezevic Date: Sat, 1 Aug 2026 17:45:38 +0200 Subject: [PATCH] fix: make demo e2e catalog queries resilient to transient failures The generate-demos CI job fails ~45% of the time with jq exit status 5 on the ClusterCatalog Quickstart scenario. Three issues contribute: - jq -s (slurp mode) buffers the entire operatorhubio FBC response in memory before processing, risking system errors on large catalogs - catalog content queries run exactly once with no retry, so any transient port-forward or network hiccup fails the step immediately - bash() does not attach stderr to ExitError, making failures opaque Remove jq slurp mode so each JSON object is processed in constant memory, prefixing filters with 'objects' to skip non-object values in the FBC stream. Wrap CatalogContainsSomePackages, PackageHasSomeChannels, and PackageHasSomeBundles in waitFor for retry on transient errors. Add curl --retry for transport-level resilience, --compressed to handle gzip-encoded responses, and --fail to prevent HTTP error bodies from reaching jq. Re-establish dead port-forwards automatically via liveness checks. Inject stderr into ExitError in bash() to match k8sClient diagnostics. Log catalog query errors at V(0) so CI timeout failures are diagnosable. Co-Authored-By: Claude --- test/e2e/steps/demo_steps.go | 110 ++++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 39 deletions(-) diff --git a/test/e2e/steps/demo_steps.go b/test/e2e/steps/demo_steps.go index d4ac355b1a..733df62714 100644 --- a/test/e2e/steps/demo_steps.go +++ b/test/e2e/steps/demo_steps.go @@ -5,6 +5,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "net" "net/http" @@ -41,6 +42,10 @@ func bash(ctx context.Context, script string) (string, error) { if err != nil { logger.V(1).Info("Failed to run", "command", script, "stderr", stderr, "error", err) + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitErr.Stderr = stderrBuf.Bytes() + } } logger.V(1).Info("Output", "command", script, "output", stdout) @@ -81,7 +86,15 @@ func CatalogReportsConditionWithoutReason(ctx context.Context, catalogUserName, func ensureCatalogPortForward(ctx context.Context) (string, error) { sc := scenarioCtx(ctx) if sc.catalogAddr != "" { - return sc.catalogAddr, nil + if catalogPortForwardAlive(sc.catalogAddr) { + return sc.catalogAddr, nil + } + logger.V(1).Info("Catalog port-forward is dead, re-establishing", "addr", sc.catalogAddr) + if sc.catalogCleanup != nil { + sc.catalogCleanup() + } + sc.catalogAddr = "" + sc.catalogCleanup = nil } addr, cleanup, err := portForward(ctx, componentNamespaces["catalogd"], "service/catalogd-service", 443) @@ -92,68 +105,87 @@ func ensureCatalogPortForward(ctx context.Context) (string, error) { sc.catalogCleanup = cleanup waitFor(ctx, func() bool { - client := &http.Client{ - Timeout: 3 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec - DialContext: (&net.Dialer{Timeout: 2 * time.Second}).DialContext, - }, - } - resp, err := client.Get(fmt.Sprintf("https://%s/", addr)) - if err != nil { - return false - } - resp.Body.Close() - return true + return catalogPortForwardAlive(addr) }) return addr, nil } +func catalogPortForwardAlive(addr string) bool { + client := &http.Client{ + Timeout: 3 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + DialContext: (&net.Dialer{Timeout: 2 * time.Second}).DialContext, + }, + } + resp, err := client.Get(fmt.Sprintf("https://%s/", addr)) + if err != nil { + return false + } + resp.Body.Close() + return true +} + func catalogCurlJq(ctx context.Context, catalogName, jqFilter string) (string, error) { addr, err := ensureCatalogPortForward(ctx) if err != nil { return "", err } script := fmt.Sprintf( - `curl -s -k https://%s/catalogs/%s/api/v1/all | jq -s '%s'`, + `curl -sS -k --compressed --fail --retry 3 --retry-delay 2 --retry-all-errors https://%s/catalogs/%s/api/v1/all | jq '%s'`, addr, catalogName, jqFilter, ) return bash(ctx, script) } func CatalogContainsSomePackages(ctx context.Context, catalogName string) error { - out, err := catalogCurlJq(ctx, catalogName, - `.[] | select(.schema == "olm.package") | .name`) - if err != nil { - return err - } - if strings.TrimSpace(out) == "" { - return fmt.Errorf("catalog %q contains no packages", catalogName) - } + waitFor(ctx, func() bool { + out, err := catalogCurlJq(ctx, catalogName, + `objects | select(.schema == "olm.package") | .name`) + if err != nil { + logger.Info("Catalog query failed, retrying", "catalog", catalogName, "error", err, "stderr", stderrOutput(err)) + return false + } + if strings.TrimSpace(out) == "" { + logger.Info("Catalog returned no packages, retrying", "catalog", catalogName) + return false + } + return true + }) return nil } func PackageHasSomeChannels(ctx context.Context, packageName, catalogName string) error { - out, err := catalogCurlJq(ctx, catalogName, - fmt.Sprintf(`.[] | select(.schema == "olm.channel") | select(.package == "%s") | .name`, packageName)) - if err != nil { - return err - } - if strings.TrimSpace(out) == "" { - return fmt.Errorf("package %q in catalog %q has no channels", packageName, catalogName) - } + waitFor(ctx, func() bool { + out, err := catalogCurlJq(ctx, catalogName, + fmt.Sprintf(`objects | select(.schema == "olm.channel") | select(.package == "%s") | .name`, packageName)) + if err != nil { + logger.Info("Catalog query failed, retrying", "catalog", catalogName, "package", packageName, "error", err, "stderr", stderrOutput(err)) + return false + } + if strings.TrimSpace(out) == "" { + logger.Info("Package has no channels, retrying", "catalog", catalogName, "package", packageName) + return false + } + return true + }) return nil } func PackageHasSomeBundles(ctx context.Context, packageName, catalogName string) error { - out, err := catalogCurlJq(ctx, catalogName, - fmt.Sprintf(`.[] | select(.schema == "olm.bundle") | select(.package == "%s") | .name`, packageName)) - if err != nil { - return err - } - if strings.TrimSpace(out) == "" { - return fmt.Errorf("package %q in catalog %q has no bundles", packageName, catalogName) - } + waitFor(ctx, func() bool { + out, err := catalogCurlJq(ctx, catalogName, + fmt.Sprintf(`objects | select(.schema == "olm.bundle") | select(.package == "%s") | .name`, packageName)) + if err != nil { + logger.Info("Catalog query failed, retrying", "catalog", catalogName, "package", packageName, "error", err, "stderr", stderrOutput(err)) + return false + } + if strings.TrimSpace(out) == "" { + logger.Info("Package has no bundles, retrying", "catalog", catalogName, "package", packageName) + return false + } + return true + }) return nil }