feat(export): extend --redact-secrets coverage for application keys

.gitignore

@@ -5,3 +5,4 @@ dist/
 /export/
 *.test
 coverage.out
+.atl/

README.md

@@ -81,7 +81,7 @@ The hybrid export combines:
 | `--token` | Personal Access Token |
 | `--output` | Output directory (default `./export`) |
 | `--include-applications` | Include applications and accounts (paginated) |
-| `--redact-secrets` | Mask API keys and OIDC secrets |
+| `--redact-secrets` | Opt-in: mask sensitive keys in JSON/YAML artifacts (see [Redaction](#redaction) below) |
 | `--per-page` | Admin API page size (max 500) |
 | `--concurrency` | Concurrent requests (default 4) |
 | `--insecure` | Skip TLS verification on Admin API |
@@ -98,6 +98,20 @@ Self-signed TLS (Admin Portal or toolbox):
   --output ./export
 ```
 
+### Redaction
+
+`--redact-secrets` is **opt-in** (default off). When set, every `.json`, `.yaml`, and `.yml` file under the export root is processed before `manifest.json` is written.
+
+**Fully redacted keys** (value becomes `***REDACTED***`):
+
+`access_token`, `api_key`, `app_id`, `app_key`, `client_id`, `client_secret`, `provider_key`, `provider_verification_key`, `secret`, `user_key`
+
+**Issuer URL stripping** (`issuer_endpoint`, `oidc_issuer_endpoint`): embedded credentials are removed (`https://user:pass@host/path` → `https://host/path`); host, path, and query stay visible.
+
+**Preserved auth-mode flags** (not secrets): `auth_user_key`, `auth_app_id`, `auth_app_key`
+
+After redaction, a cleartext scan runs over the same artifacts. If any sensitive value or issuer userinfo remains, **export fails** with a path-qualified error.
+
 ### Visualize the export
 
 Optional: generate a Markdown report from the export directory (no Admin API or containers):

docs/TEST_CASES.md

@@ -102,22 +102,36 @@ Automation references are verified against the repository at the time of writing
 |-------|-------|
 | Priority | P0 |
 | CLI | `threescale-export` |
-| Automation | `TestExportRedactSecrets` (`internal/export/exporter_test.go`), `TestRedactJSONRemovesCleartextSecrets`, `TestRedactYAMLRemovesCleartextSecrets`, `TestRedactDirectory` (`internal/export/redact_test.go`) |
+| Automation | `TestExportRedactSecrets`, `TestExportWithoutRedactPreservesSecrets`, `TestExportRedactSecretsVerifyGateFailsOnResidual` (`internal/export/exporter_test.go`); `TestVerifyNoCleartextSecretsClean`, `TestVerifyNoCleartextSecretsFailsWithPath` (`internal/export/verify_test.go`); `TestRedactExtendedSensitiveKeys`, `TestRedactIssuerStripsUserinfoJSON`, `TestRedactPreservesAuthProxyFlags`, `TestContainsCleartextSecretYAMLIssuerUserinfo` (`internal/export/redact_test.go`) |
 
 **Preconditions**
 
 - Same as TC-EXP-001
-- Export includes credentials (API keys, OIDC secrets, etc.)
+- Export includes credentials (API keys, OIDC secrets, application identifiers, issuer URLs with embedded credentials, etc.)
 
 **Steps**
 
 1. Run `threescale-export --output ./export --redact-secrets --include-applications`
-2. Search output for cleartext secrets
+2. Search output for cleartext secrets and issuer URL userinfo
 
 **Expected results**
 
-- Sensitive values replaced with `***REDACTED***` in JSON and YAML artifacts
-- Export completes without error
+- Core and extended sensitive keys (`provider_verification_key`, `client_id`, `app_id`, plus existing secret keys) replaced with `***REDACTED***` in JSON and YAML artifacts
+- `issuer_endpoint` and `oidc_issuer_endpoint` have URL userinfo stripped; host and path remain visible
+- `auth_user_key`, `auth_app_id`, and `auth_app_key` are unchanged (auth-mode flags, not secrets)
+- Post-redaction cleartext scan passes; export completes without error
+- If cleartext remains after redaction, export fails with a path-qualified error (e.g. `cleartext secret in products/api/proxy.json`)
+
+**Opt-in default (no flag)**
+
+| Field | Value |
+|-------|-------|
+| Automation | `TestExportWithoutRedactPreservesSecrets` (`internal/export/exporter_test.go`) |
+
+**Expected results (without `--redact-secrets`)**
+
+- Credential values remain cleartext in JSON and YAML artifacts
+- No `***REDACTED***` markers in output
 
 ---
 

internal/config/config.go

@@ -62,7 +62,7 @@ func BindExportFlags(fs *pflag.FlagSet, cfg *ExportConfig) {
 	BindAuthFlags(fs, &cfg.AuthConfig)
 	fs.StringVar(&cfg.OutDir, "output", cfg.OutDir, "export output directory")
 	fs.BoolVar(&cfg.IncludeApplications, "include-applications", cfg.IncludeApplications, "export applications and linked accounts")
-	fs.BoolVar(&cfg.RedactSecrets, "redact-secrets", cfg.RedactSecrets, "mask API keys and OIDC secrets in output")
+	fs.BoolVar(&cfg.RedactSecrets, "redact-secrets", cfg.RedactSecrets, "mask sensitive keys in JSON/YAML output (secrets to ***REDACTED***, issuer URLs strip embedded credentials; export fails if cleartext remains)")
 	fs.BoolVar(&cfg.Strict, "strict", cfg.Strict, "fail export if any product sidecar JSON cannot be fetched")
 	fs.IntVar(&cfg.PerPage, "per-page", cfg.PerPage, "Admin API page size (max 500)")
 	fs.IntVar(&cfg.MaxConcurrent, "concurrency", cfg.MaxConcurrent, "max concurrent Admin API requests")

internal/export/exporter.go

@@ -121,6 +121,9 @@ func (s *Service) Export(ctx context.Context, opts Options) (*output.Manifest, e
 		if err := RedactDirectory(writer.Root()); err != nil {
 			return manifest, err
 		}
+		if err := VerifyNoCleartextSecrets(writer.Root()); err != nil {
+			return manifest, err
+		}
 	}
 
 	if err := writer.WriteManifest(manifest); err != nil {

internal/export/exporter_test.go

@@ -156,7 +156,7 @@ func TestExportIncludeApplications(t *testing.T) {
 	}
 }
 
-func TestExportRedactSecrets(t *testing.T) {
+func TestExportWithoutRedactPreservesSecrets(t *testing.T) {
 	dir := t.TempDir()
 	client := &mockClient{
 		responses: map[string]any{
@@ -168,18 +168,130 @@ func TestExportRedactSecrets(t *testing.T) {
 			"/services/1/proxy": map[string]any{
 				"proxy": map[string]any{"api_key": "secret-value"},
 			},
+			"/services/1/proxy/policies": map[string]any{"policies": []any{}},
+			"/services/1/proxy/oidc_configuration": map[string]any{
+				"oidc_configuration": map[string]any{
+					"oidc_issuer_endpoint": "https://zync-user:zync-pass@idp.example.com/realms/demo",
+					"client_id":            "oidc-client-id",
+				},
+			},
+			"/services/1/application_plans": map[string]any{"plans": []any{}},
+			"/services/1/backend_usages":    map[string]any{"backend_usages": []any{}},
+			"/services/1/metrics":           map[string]any{"metrics": []any{}},
+			"/accounts/7": map[string]any{
+				"account": map[string]any{"id": 7},
+			},
+		},
+		pages: map[string][][]json.RawMessage{
+			"/applications": {
+				{json.RawMessage(`{"application":{"id":1,"account_id":7,"provider_verification_key":"pvk-live","client_id":"app-client","app_id":"app-99"}}`)},
+			},
+		},
+	}
+	toolbox := &mockToolbox{outputs: map[string][]byte{
+		"api": []byte("credentials:\n  client_secret: yaml-secret\n  client_id: yaml-client\n"),
+	}}
+
+	svc := NewService(client, toolbox)
+	_, err := svc.Export(context.Background(), Options{
+		AdminURL:            "https://tenant.example.com",
+		Token:               "tok",
+		OutDir:              dir,
+		IncludeApplications: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	proxyData, err := os.ReadFile(filepath.Join(dir, "products", "api", "proxy.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(proxyData), "secret-value") {
+		t.Fatalf("expected cleartext api_key in proxy: %s", proxyData)
+	}
+
+	oidcData, err := os.ReadFile(filepath.Join(dir, "products", "api", "oidc_configuration.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	oidcStr := string(oidcData)
+	if !strings.Contains(oidcStr, "zync-user") || !strings.Contains(oidcStr, "oidc-client-id") {
+		t.Fatalf("expected cleartext OIDC fields: %s", oidcData)
+	}
+
+	appData, err := os.ReadFile(filepath.Join(dir, "applications", "page-1.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	appStr := string(appData)
+	for _, want := range []string{"pvk-live", "app-client", "app-99"} {
+		if !strings.Contains(appStr, want) {
+			t.Fatalf("expected cleartext %q in applications: %s", want, appData)
+		}
+	}
+
+	yamlData, err := os.ReadFile(filepath.Join(dir, "products", "api.yaml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	yamlStr := string(yamlData)
+	if !strings.Contains(yamlStr, "yaml-secret") || !strings.Contains(yamlStr, "yaml-client") {
+		t.Fatalf("expected cleartext YAML credentials: %s", yamlData)
+	}
+	if strings.Contains(yamlStr, "***REDACTED***") {
+		t.Fatalf("unexpected redaction marker without --redact-secrets: %s", yamlData)
+	}
+}
+
+func TestExportRedactSecrets(t *testing.T) {
+	dir := t.TempDir()
+	client := &mockClient{
+		responses: map[string]any{
+			"/services": serviceListResponse{
+				Services: []serviceEntry{{Service: serviceRef{ID: 1, SystemName: "api"}}},
+			},
+			"/backend_apis": backendListResponse{},
+			"/policies":     map[string]any{},
+			"/services/1/proxy": map[string]any{
+				"proxy": map[string]any{
+					"api_key":       "secret-value",
+					"auth_user_key": "user_key",
+					"auth_app_id":   "app_id",
+					"auth_app_key":  "app_key",
+				},
+			},
+			"/services/1/proxy/policies": map[string]any{"policies": []any{}},
+			"/services/1/proxy/oidc_configuration": map[string]any{
+				"oidc_configuration": map[string]any{
+					"oidc_issuer_endpoint": "https://zync-user:zync-pass@idp.example.com/realms/demo",
+					"client_id":            "oidc-client-id",
+				},
+			},
+			"/services/1/application_plans": map[string]any{"plans": []any{}},
+			"/services/1/backend_usages":    map[string]any{"backend_usages": []any{}},
+			"/services/1/metrics":           map[string]any{"metrics": []any{}},
+			"/accounts/7": map[string]any{
+				"account": map[string]any{"id": 7},
+			},
+		},
+		pages: map[string][][]json.RawMessage{
+			"/applications": {
+				{json.RawMessage(`{"application":{"id":1,"account_id":7,"provider_verification_key":"pvk-live","client_id":"app-client","app_id":"app-99"}}`)},
+			},
 		},
 	}
 	toolbox := &mockToolbox{outputs: map[string][]byte{
-		"api": []byte("credentials:\n  client_secret: yaml-secret\n"),
+		"api": []byte("credentials:\n  client_secret: yaml-secret\n  client_id: yaml-client\n"),
 	}}
 
 	svc := NewService(client, toolbox)
 	_, err := svc.Export(context.Background(), Options{
-		AdminURL:      "https://tenant.example.com",
-		Token:         "tok",
-		OutDir:        dir,
-		RedactSecrets: true,
+		AdminURL:            "https://tenant.example.com",
+		Token:               "tok",
+		OutDir:              dir,
+		RedactSecrets:       true,
+		IncludeApplications: true,
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -192,13 +304,68 @@ func TestExportRedactSecrets(t *testing.T) {
 	if ContainsCleartextSecret(proxyData) {
 		t.Fatalf("proxy still has secret: %s", proxyData)
 	}
+	proxyStr := string(proxyData)
+	for _, want := range []string{`"auth_user_key": "user_key"`, `"auth_app_id": "app_id"`, `"auth_app_key": "app_key"`} {
+		if !strings.Contains(proxyStr, want) {
+			t.Fatalf("auth flag missing %q in %s", want, proxyData)
+		}
+	}
+
+	oidcData, err := os.ReadFile(filepath.Join(dir, "products", "api", "oidc_configuration.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	oidcStr := string(oidcData)
+	if strings.Contains(oidcStr, "zync-user") || strings.Contains(oidcStr, "zync-pass") {
+		t.Fatalf("issuer userinfo still present: %s", oidcData)
+	}
+	if !strings.Contains(oidcStr, "https://idp.example.com/realms/demo") {
+		t.Fatalf("expected stripped issuer host in %s", oidcData)
+	}
+	if strings.Contains(oidcStr, "oidc-client-id") {
+		t.Fatalf("client_id not redacted: %s", oidcData)
+	}
+
+	appData, err := os.ReadFile(filepath.Join(dir, "applications", "page-1.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ContainsCleartextSecret(appData) {
+		t.Fatalf("applications still have secrets: %s", appData)
+	}
+
 	yamlData, err := os.ReadFile(filepath.Join(dir, "products", "api.yaml"))
 	if err != nil {
 		t.Fatal(err)
 	}
-	if strings.Contains(string(yamlData), "yaml-secret") {
+	if strings.Contains(string(yamlData), "yaml-secret") || strings.Contains(string(yamlData), "yaml-client") {
 		t.Fatalf("yaml still has secret: %s", yamlData)
 	}
+
+	if err := VerifyNoCleartextSecrets(dir); err != nil {
+		t.Fatalf("post-export verify gate failed: %v", err)
+	}
+}
+
+func TestExportRedactSecretsVerifyGateFailsOnResidual(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "products", "api"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(
+		filepath.Join(dir, "products", "api", "proxy.json"),
+		[]byte(`{"proxy":{"client_secret":"not-redacted"}}`),
+		0o644,
+	); err != nil {
+		t.Fatal(err)
+	}
+	err := VerifyNoCleartextSecrets(dir)
+	if err == nil {
+		t.Fatal("expected cleartext gate error")
+	}
+	if !strings.Contains(err.Error(), "products/api/proxy.json") {
+		t.Fatalf("error = %v, want path-qualified message", err)
+	}
 }
 
 func TestExportIncompleteOnBackendFailure(t *testing.T) {