From e9ab1c23ac5cfa53cea0a3c37ce7431cc7726f27 Mon Sep 17 00:00:00 2001 From: Francisco Meneses Date: Thu, 11 Jun 2026 16:58:06 -0400 Subject: [PATCH] feat(export): strict mode, layout verify, golden tests, revive lint Add --strict export flag (C2), VerifyExport for integration assertions (T12), golden layout test (T13), and enable revive in golangci-lint. Co-authored-by: Cursor --- .golangci.yml | 27 ++++ internal/cli/cli.go | 1 + internal/config/config.go | 2 + internal/export/exporter.go | 4 + internal/export/integration_test.go | 11 +- .../export/testdata/golden-export-layout.txt | 10 ++ internal/export/verify.go | 107 +++++++++++++++ internal/export/verify_test.go | 125 ++++++++++++++++++ 8 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 internal/export/testdata/golden-export-layout.txt create mode 100644 internal/export/verify.go create mode 100644 internal/export/verify_test.go diff --git a/.golangci.yml b/.golangci.yml index eda6300..3891caa 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -10,6 +10,7 @@ linters: - gosimple - govet - ineffassign + - revive - staticcheck - unused @@ -18,6 +19,32 @@ linters-settings: errorf: true asserts: true comparison: true + revive: + rules: + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + disabled: true + - name: increment-decrement + - name: var-naming + - name: package-comments + disabled: true + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + disabled: true + - name: indent-error-flow + - name: errorf + - name: empty-block + - name: superfluous-else + - name: unreachable-code + - name: redefines-builtin-id issues: exclude-dirs: diff --git a/internal/cli/cli.go b/internal/cli/cli.go index fd740c4..1c4bb6b 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -64,6 +64,7 @@ func RunExport(ctx context.Context, cfg config.ExportConfig) error { OutDir: cfg.OutDir, IncludeApplications: cfg.IncludeApplications, RedactSecrets: cfg.RedactSecrets, + Strict: cfg.Strict, MaxConcurrent: cfg.MaxConcurrent, PerPage: cfg.PerPage, }) diff --git a/internal/config/config.go b/internal/config/config.go index 19d73ff..3c82db9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,6 +25,7 @@ type ExportConfig struct { OutDir string IncludeApplications bool RedactSecrets bool + Strict bool PerPage int MaxConcurrent int ToolboxImage string @@ -62,6 +63,7 @@ func BindExportFlags(fs *pflag.FlagSet, cfg *ExportConfig) { fs.StringVar(&cfg.OutDir, "output", cfg.OutDir, "export output directory") fs.BoolVar(&cfg.IncludeApplications, "include-applications", cfg.IncludeApplications, "export applications and linked accounts") fs.BoolVar(&cfg.RedactSecrets, "redact-secrets", cfg.RedactSecrets, "mask API keys and OIDC secrets in output") + fs.BoolVar(&cfg.Strict, "strict", cfg.Strict, "fail export if any product sidecar JSON cannot be fetched") fs.IntVar(&cfg.PerPage, "per-page", cfg.PerPage, "Admin API page size (max 500)") fs.IntVar(&cfg.MaxConcurrent, "concurrency", cfg.MaxConcurrent, "max concurrent Admin API requests") fs.StringVar(&cfg.ToolboxImage, "toolbox-image", cfg.ToolboxImage, "3scale toolbox container image (Red Hat official)") diff --git a/internal/export/exporter.go b/internal/export/exporter.go index a345cef..813b993 100644 --- a/internal/export/exporter.go +++ b/internal/export/exporter.go @@ -18,6 +18,7 @@ type Options struct { OutDir string IncludeApplications bool RedactSecrets bool + Strict bool MaxConcurrent int PerPage int } @@ -203,6 +204,9 @@ func (s *Service) exportService(ctx context.Context, writer *output.Writer, opts for _, f := range fetches { var payload json.RawMessage if err := s.client.Get(ctx, f.path, &payload); err != nil { + if opts.Strict { + return fmt.Errorf("%w: product %s: skipped %s (%s): %w", ErrStrictSidecar, svc.SystemName, f.file, f.path, err) + } manifest.RecordSkip(svc.SystemName, f.file, f.path, err) continue } diff --git a/internal/export/integration_test.go b/internal/export/integration_test.go index 906cbd8..8292d05 100644 --- a/internal/export/integration_test.go +++ b/internal/export/integration_test.go @@ -61,7 +61,7 @@ func TestIntegrationExport(t *testing.T) { t.Fatal(err) } svc := export.NewService(client, integrationToolbox(t)) - _, err = svc.Export(context.Background(), export.Options{ + manifest, err := svc.Export(context.Background(), export.Options{ AdminURL: adminURL, Token: token, OutDir: outDir, @@ -70,4 +70,13 @@ func TestIntegrationExport(t *testing.T) { if err != nil { t.Fatal(err) } + if manifest == nil { + t.Fatal("expected manifest") + } + if manifest.SchemaVersion != "1.0" { + t.Fatalf("schema_version = %q", manifest.SchemaVersion) + } + if err := export.VerifyExport(outDir); err != nil { + t.Fatal(err) + } } diff --git a/internal/export/testdata/golden-export-layout.txt b/internal/export/testdata/golden-export-layout.txt new file mode 100644 index 0000000..5ebf13f --- /dev/null +++ b/internal/export/testdata/golden-export-layout.txt @@ -0,0 +1,10 @@ +backends/billing-backend.json +manifest.json +policies/catalog.json +products/payments.yaml +products/payments/application_plans.json +products/payments/backend_usages.json +products/payments/metrics.json +products/payments/oidc_configuration.json +products/payments/policies.json +products/payments/proxy.json diff --git a/internal/export/verify.go b/internal/export/verify.go new file mode 100644 index 0000000..72d0bcf --- /dev/null +++ b/internal/export/verify.go @@ -0,0 +1,107 @@ +package export + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Everything-is-Code/3scaleextract/internal/output" +) + +var ErrStrictSidecar = errors.New("strict export: missing product sidecar") + +// VerifyExport checks manifest fields and on-disk layout after an export completes. +func VerifyExport(root string) error { + root = strings.TrimSpace(root) + if root == "" { + return errors.New("export directory is required") + } + + manifestPath := filepath.Join(root, "manifest.json") + data, err := os.ReadFile(manifestPath) + if err != nil { + return fmt.Errorf("read manifest.json: %w", err) + } + + var manifest output.Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return fmt.Errorf("parse manifest.json: %w", err) + } + if manifest.SchemaVersion != output.SchemaVersion { + return fmt.Errorf("manifest schema_version = %q, want %q", manifest.SchemaVersion, output.SchemaVersion) + } + if strings.TrimSpace(manifest.AdminURL) == "" { + return errors.New("manifest admin_url is required") + } + if strings.TrimSpace(manifest.ExportedAt) == "" { + return errors.New("manifest exported_at is required") + } + + for _, path := range []string{"products", "backends", "policies"} { + if _, err := os.Stat(filepath.Join(root, path)); err != nil { + return fmt.Errorf("missing directory %s: %w", path, err) + } + } + if _, err := os.Stat(filepath.Join(root, "policies", "catalog.json")); err != nil { + return fmt.Errorf("missing policies/catalog.json: %w", err) + } + + productYAMLs, err := filepath.Glob(filepath.Join(root, "products", "*.yaml")) + if err != nil { + return err + } + if manifest.ProductCount > 0 && len(productYAMLs) == 0 { + return errors.New("expected at least one products/*.yaml") + } + if manifest.ProductCount != len(productYAMLs) { + return fmt.Errorf("manifest product_count = %d, found %d products/*.yaml", manifest.ProductCount, len(productYAMLs)) + } + + backendJSONs, err := filepath.Glob(filepath.Join(root, "backends", "*.json")) + if err != nil { + return err + } + if manifest.BackendCount != len(backendJSONs) { + return fmt.Errorf("manifest backend_count = %d, found %d backends/*.json", manifest.BackendCount, len(backendJSONs)) + } + + if manifest.IncludeApplications { + appPages, err := filepath.Glob(filepath.Join(root, "applications", "page-*.json")) + if err != nil { + return err + } + if manifest.ApplicationCount > 0 && len(appPages) == 0 { + return errors.New("expected applications/page-*.json when application_count > 0") + } + } + + return nil +} + +// ListExportPaths returns sorted relative file paths under an export root (directories excluded). +func ListExportPaths(root string) ([]string, error) { + var paths []string + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + paths = append(paths, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return nil, err + } + sort.Strings(paths) + return paths, nil +} diff --git a/internal/export/verify_test.go b/internal/export/verify_test.go new file mode 100644 index 0000000..50f3154 --- /dev/null +++ b/internal/export/verify_test.go @@ -0,0 +1,125 @@ +package export + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestExportStrictFailsOnMissingSidecar(t *testing.T) { + dir := t.TempDir() + client := &mockClient{ + responses: map[string]any{ + "/services": serviceListResponse{ + Services: []serviceEntry{{Service: serviceRef{ID: 10, SystemName: "payments"}}}, + }, + "/backend_apis": backendListResponse{}, + "/policies": map[string]any{}, + "/services/10/proxy": map[string]any{ + "proxy": map[string]any{"auth_type": "oidc"}, + }, + "/services/10/proxy/policies": map[string]any{"policies": []any{}}, + "/services/10/application_plans": map[string]any{"plans": []any{}}, + "/services/10/backend_usages": map[string]any{"backend_usages": []any{}}, + "/services/10/metrics": map[string]any{"metrics": []any{}}, + }, + } + toolbox := &mockToolbox{outputs: map[string][]byte{"payments": []byte("apiVersion: v1\n")}} + + svc := NewService(client, toolbox) + manifest, err := svc.Export(context.Background(), Options{ + AdminURL: "https://tenant.example.com", + Token: "tok", + OutDir: dir, + Strict: true, + }) + if err == nil { + t.Fatal("expected strict export error") + } + if !errors.Is(err, ErrStrictSidecar) { + t.Fatalf("err = %v", err) + } + if manifest == nil || !manifest.Incomplete { + t.Fatal("expected incomplete manifest on strict failure") + } +} + +func TestVerifyExportLayout(t *testing.T) { + dir := t.TempDir() + client := defaultScopeMockClient() + toolbox := &mockToolbox{outputs: map[string][]byte{"payments": []byte("apiVersion: v1\nkind: Product\n")}} + + svc := NewService(client, toolbox) + if _, err := svc.Export(context.Background(), Options{ + AdminURL: "https://tenant.example.com", + Token: "tok", + OutDir: dir, + }); err != nil { + t.Fatal(err) + } + if err := VerifyExport(dir); err != nil { + t.Fatal(err) + } +} + +func TestExportGoldenLayout(t *testing.T) { + dir := t.TempDir() + svc := NewService(defaultScopeMockClient(), &mockToolbox{ + outputs: map[string][]byte{"payments": []byte("apiVersion: v1\nkind: Product\n")}, + }) + if _, err := svc.Export(context.Background(), Options{ + AdminURL: "https://tenant.example.com", + Token: "tok", + OutDir: dir, + }); err != nil { + t.Fatal(err) + } + + got, err := ListExportPaths(dir) + if err != nil { + t.Fatal(err) + } + wantRaw, err := os.ReadFile(filepath.Join("testdata", "golden-export-layout.txt")) + if err != nil { + t.Fatal(err) + } + var want []string + for _, line := range strings.Split(strings.TrimSpace(string(wantRaw)), "\n") { + if line = strings.TrimSpace(line); line != "" { + want = append(want, line) + } + } + if len(got) != len(want) { + t.Fatalf("paths = %d, want %d\ngot: %v\nwant: %v", len(got), len(want), got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("path[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +func defaultScopeMockClient() *mockClient { + return &mockClient{ + responses: map[string]any{ + "/services": serviceListResponse{ + Services: []serviceEntry{{Service: serviceRef{ID: 10, SystemName: "payments"}}}, + }, + "/backend_apis": backendListResponse{ + BackendAPIs: []backendEntry{{BackendAPI: []byte(`{"system_name":"billing-backend"}`)}}, + }, + "/policies": map[string]any{"apicast": map[string]any{"version": "builtin"}}, + "/services/10/proxy": map[string]any{ + "proxy": map[string]any{"auth_user_key": "key", "auth_type": "api_key"}, + }, + "/services/10/proxy/policies": map[string]any{"policies": []any{}}, + "/services/10/proxy/oidc_configuration": map[string]any{}, + "/services/10/application_plans": map[string]any{"plans": []any{}}, + "/services/10/backend_usages": map[string]any{"backend_usages": []any{}}, + "/services/10/metrics": map[string]any{"metrics": []any{}}, + }, + } +}