feat(export): strict mode, layout verify, golden tests, revive

.golangci.yml

@@ -10,6 +10,7 @@ linters:
     - gosimple
     - govet
     - ineffassign
+    - revive
     - staticcheck
     - unused
 
@@ -18,6 +19,32 @@ linters-settings:
     errorf: true
     asserts: true
     comparison: true
+  revive:
+    rules:
+      - name: blank-imports
+      - name: context-as-argument
+      - name: context-keys-type
+      - name: dot-imports
+      - name: error-return
+      - name: error-strings
+      - name: error-naming
+      - name: exported
+        disabled: true
+      - name: increment-decrement
+      - name: var-naming
+      - name: package-comments
+        disabled: true
+      - name: range
+      - name: receiver-naming
+      - name: time-naming
+      - name: unexported-return
+        disabled: true
+      - name: indent-error-flow
+      - name: errorf
+      - name: empty-block
+      - name: superfluous-else
+      - name: unreachable-code
+      - name: redefines-builtin-id
 
 issues:
   exclude-dirs:

internal/cli/cli.go

@@ -64,6 +64,7 @@ func RunExport(ctx context.Context, cfg config.ExportConfig) error {
 		OutDir:              cfg.OutDir,
 		IncludeApplications: cfg.IncludeApplications,
 		RedactSecrets:       cfg.RedactSecrets,
+		Strict:              cfg.Strict,
 		MaxConcurrent:       cfg.MaxConcurrent,
 		PerPage:             cfg.PerPage,
 	})

internal/config/config.go

@@ -25,6 +25,7 @@ type ExportConfig struct {
 	OutDir              string
 	IncludeApplications bool
 	RedactSecrets       bool
+	Strict              bool
 	PerPage             int
 	MaxConcurrent       int
 	ToolboxImage        string
@@ -62,6 +63,7 @@ func BindExportFlags(fs *pflag.FlagSet, cfg *ExportConfig) {
 	fs.StringVar(&cfg.OutDir, "output", cfg.OutDir, "export output directory")
 	fs.BoolVar(&cfg.IncludeApplications, "include-applications", cfg.IncludeApplications, "export applications and linked accounts")
 	fs.BoolVar(&cfg.RedactSecrets, "redact-secrets", cfg.RedactSecrets, "mask API keys and OIDC secrets in output")
+	fs.BoolVar(&cfg.Strict, "strict", cfg.Strict, "fail export if any product sidecar JSON cannot be fetched")
 	fs.IntVar(&cfg.PerPage, "per-page", cfg.PerPage, "Admin API page size (max 500)")
 	fs.IntVar(&cfg.MaxConcurrent, "concurrency", cfg.MaxConcurrent, "max concurrent Admin API requests")
 	fs.StringVar(&cfg.ToolboxImage, "toolbox-image", cfg.ToolboxImage, "3scale toolbox container image (Red Hat official)")

internal/export/exporter.go

@@ -18,6 +18,7 @@ type Options struct {
 	OutDir              string
 	IncludeApplications bool
 	RedactSecrets       bool
+	Strict              bool
 	MaxConcurrent       int
 	PerPage             int
 }
@@ -203,6 +204,9 @@ func (s *Service) exportService(ctx context.Context, writer *output.Writer, opts
 	for _, f := range fetches {
 		var payload json.RawMessage
 		if err := s.client.Get(ctx, f.path, &payload); err != nil {
+			if opts.Strict {
+				return fmt.Errorf("%w: product %s: skipped %s (%s): %w", ErrStrictSidecar, svc.SystemName, f.file, f.path, err)
+			}
 			manifest.RecordSkip(svc.SystemName, f.file, f.path, err)
 			continue
 		}

internal/export/integration_test.go

@@ -61,7 +61,7 @@ func TestIntegrationExport(t *testing.T) {
 		t.Fatal(err)
 	}
 	svc := export.NewService(client, integrationToolbox(t))
-	_, err = svc.Export(context.Background(), export.Options{
+	manifest, err := svc.Export(context.Background(), export.Options{
 		AdminURL: adminURL,
 		Token:    token,
 		OutDir:   outDir,
@@ -70,4 +70,13 @@ func TestIntegrationExport(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	if manifest == nil {
+		t.Fatal("expected manifest")
+	}
+	if manifest.SchemaVersion != "1.0" {
+		t.Fatalf("schema_version = %q", manifest.SchemaVersion)
+	}
+	if err := export.VerifyExport(outDir); err != nil {
+		t.Fatal(err)
+	}
 }

internal/export/testdata/golden-export-layout.txt

@@ -0,0 +1,10 @@
+backends/billing-backend.json
+manifest.json
+policies/catalog.json
+products/payments.yaml
+products/payments/application_plans.json
+products/payments/backend_usages.json
+products/payments/metrics.json
+products/payments/oidc_configuration.json
+products/payments/policies.json
+products/payments/proxy.json

internal/export/verify.go

@@ -0,0 +1,107 @@
+package export
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/Everything-is-Code/3scaleextract/internal/output"
+)
+
+var ErrStrictSidecar = errors.New("strict export: missing product sidecar")
+
+// VerifyExport checks manifest fields and on-disk layout after an export completes.
+func VerifyExport(root string) error {
+	root = strings.TrimSpace(root)
+	if root == "" {
+		return errors.New("export directory is required")
+	}
+
+	manifestPath := filepath.Join(root, "manifest.json")
+	data, err := os.ReadFile(manifestPath)
+	if err != nil {
+		return fmt.Errorf("read manifest.json: %w", err)
+	}
+
+	var manifest output.Manifest
+	if err := json.Unmarshal(data, &manifest); err != nil {
+		return fmt.Errorf("parse manifest.json: %w", err)
+	}
+	if manifest.SchemaVersion != output.SchemaVersion {
+		return fmt.Errorf("manifest schema_version = %q, want %q", manifest.SchemaVersion, output.SchemaVersion)
+	}
+	if strings.TrimSpace(manifest.AdminURL) == "" {
+		return errors.New("manifest admin_url is required")
+	}
+	if strings.TrimSpace(manifest.ExportedAt) == "" {
+		return errors.New("manifest exported_at is required")
+	}
+
+	for _, path := range []string{"products", "backends", "policies"} {
+		if _, err := os.Stat(filepath.Join(root, path)); err != nil {
+			return fmt.Errorf("missing directory %s: %w", path, err)
+		}
+	}
+	if _, err := os.Stat(filepath.Join(root, "policies", "catalog.json")); err != nil {
+		return fmt.Errorf("missing policies/catalog.json: %w", err)
+	}
+
+	productYAMLs, err := filepath.Glob(filepath.Join(root, "products", "*.yaml"))
+	if err != nil {
+		return err
+	}
+	if manifest.ProductCount > 0 && len(productYAMLs) == 0 {
+		return errors.New("expected at least one products/*.yaml")
+	}
+	if manifest.ProductCount != len(productYAMLs) {
+		return fmt.Errorf("manifest product_count = %d, found %d products/*.yaml", manifest.ProductCount, len(productYAMLs))
+	}
+
+	backendJSONs, err := filepath.Glob(filepath.Join(root, "backends", "*.json"))
+	if err != nil {
+		return err
+	}
+	if manifest.BackendCount != len(backendJSONs) {
+		return fmt.Errorf("manifest backend_count = %d, found %d backends/*.json", manifest.BackendCount, len(backendJSONs))
+	}
+
+	if manifest.IncludeApplications {
+		appPages, err := filepath.Glob(filepath.Join(root, "applications", "page-*.json"))
+		if err != nil {
+			return err
+		}
+		if manifest.ApplicationCount > 0 && len(appPages) == 0 {
+			return errors.New("expected applications/page-*.json when application_count > 0")
+		}
+	}
+
+	return nil
+}
+
+// ListExportPaths returns sorted relative file paths under an export root (directories excluded).
+func ListExportPaths(root string) ([]string, error) {
+	var paths []string
+	err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		rel, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		paths = append(paths, filepath.ToSlash(rel))
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(paths)
+	return paths, nil
+}

internal/export/verify_test.go

@@ -0,0 +1,125 @@
+package export
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestExportStrictFailsOnMissingSidecar(t *testing.T) {
+	dir := t.TempDir()
+	client := &mockClient{
+		responses: map[string]any{
+			"/services": serviceListResponse{
+				Services: []serviceEntry{{Service: serviceRef{ID: 10, SystemName: "payments"}}},
+			},
+			"/backend_apis": backendListResponse{},
+			"/policies":     map[string]any{},
+			"/services/10/proxy": map[string]any{
+				"proxy": map[string]any{"auth_type": "oidc"},
+			},
+			"/services/10/proxy/policies":    map[string]any{"policies": []any{}},
+			"/services/10/application_plans": map[string]any{"plans": []any{}},
+			"/services/10/backend_usages":    map[string]any{"backend_usages": []any{}},
+			"/services/10/metrics":           map[string]any{"metrics": []any{}},
+		},
+	}
+	toolbox := &mockToolbox{outputs: map[string][]byte{"payments": []byte("apiVersion: v1\n")}}
+
+	svc := NewService(client, toolbox)
+	manifest, err := svc.Export(context.Background(), Options{
+		AdminURL: "https://tenant.example.com",
+		Token:    "tok",
+		OutDir:   dir,
+		Strict:   true,
+	})
+	if err == nil {
+		t.Fatal("expected strict export error")
+	}
+	if !errors.Is(err, ErrStrictSidecar) {
+		t.Fatalf("err = %v", err)
+	}
+	if manifest == nil || !manifest.Incomplete {
+		t.Fatal("expected incomplete manifest on strict failure")
+	}
+}
+
+func TestVerifyExportLayout(t *testing.T) {
+	dir := t.TempDir()
+	client := defaultScopeMockClient()
+	toolbox := &mockToolbox{outputs: map[string][]byte{"payments": []byte("apiVersion: v1\nkind: Product\n")}}
+
+	svc := NewService(client, toolbox)
+	if _, err := svc.Export(context.Background(), Options{
+		AdminURL: "https://tenant.example.com",
+		Token:    "tok",
+		OutDir:   dir,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := VerifyExport(dir); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestExportGoldenLayout(t *testing.T) {
+	dir := t.TempDir()
+	svc := NewService(defaultScopeMockClient(), &mockToolbox{
+		outputs: map[string][]byte{"payments": []byte("apiVersion: v1\nkind: Product\n")},
+	})
+	if _, err := svc.Export(context.Background(), Options{
+		AdminURL: "https://tenant.example.com",
+		Token:    "tok",
+		OutDir:   dir,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := ListExportPaths(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wantRaw, err := os.ReadFile(filepath.Join("testdata", "golden-export-layout.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var want []string
+	for _, line := range strings.Split(strings.TrimSpace(string(wantRaw)), "\n") {
+		if line = strings.TrimSpace(line); line != "" {
+			want = append(want, line)
+		}
+	}
+	if len(got) != len(want) {
+		t.Fatalf("paths = %d, want %d\ngot:  %v\nwant: %v", len(got), len(want), got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("path[%d] = %q, want %q", i, got[i], want[i])
+		}
+	}
+}
+
+func defaultScopeMockClient() *mockClient {
+	return &mockClient{
+		responses: map[string]any{
+			"/services": serviceListResponse{
+				Services: []serviceEntry{{Service: serviceRef{ID: 10, SystemName: "payments"}}},
+			},
+			"/backend_apis": backendListResponse{
+				BackendAPIs: []backendEntry{{BackendAPI: []byte(`{"system_name":"billing-backend"}`)}},
+			},
+			"/policies": map[string]any{"apicast": map[string]any{"version": "builtin"}},
+			"/services/10/proxy": map[string]any{
+				"proxy": map[string]any{"auth_user_key": "key", "auth_type": "api_key"},
+			},
+			"/services/10/proxy/policies":            map[string]any{"policies": []any{}},
+			"/services/10/proxy/oidc_configuration":    map[string]any{},
+			"/services/10/application_plans":           map[string]any{"plans": []any{}},
+			"/services/10/backend_usages":              map[string]any{"backend_usages": []any{}},
+			"/services/10/metrics":                     map[string]any{"metrics": []any{}},
+		},
+	}
+}