From 4b193ffad8dbbf131c9b3247c1be1412117fbc57 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Mon, 20 Jul 2026 09:05:03 +0100 Subject: [PATCH] feat(boatstack): non-breaking versioned configuration schema migration system Implement a non-breaking versioned configuration migration system for .boatstack-project.json. - migrate.go: Core migration registry and MigrateConfigBytes helper. - export.go: Relax SchemaVersion check in ValidateConfig to accept <= CurrentConfigSchemaVersion and emit behind/ahead instructions. Embed config-schema.md. - init.go: Run config schema migration on load during update path and report vN->vM upgrade status. - update.go: Add schema conformance validation check inside ValidateUpdateWorkspace. - planning.go: Map behind/ahead schema errors to custom remediation hints in DoctorRepairHint. - cmd/boatstack-helper/main.go: Add structured migrate-config CLI command. - references/config-schema.md: Create canonical config schema reference document. - boatstack-distribution/release-notes: Add release notes fragment. - migrate_test.go: Exhaustive unit test suite. --- .../2026-07-20-config-schema-migration.md | 3 + .../cmd/boatstack-helper/main.go | 67 ++++- .../product-engineering-loop/export.go | 12 +- .../product-engineering-loop/init.go | 25 +- .../product-engineering-loop/migrate.go | 101 +++++++ .../product-engineering-loop/migrate_test.go | 274 ++++++++++++++++++ .../product-engineering-loop/planning.go | 7 + .../references/config-schema.md | 39 +++ .../product-engineering-loop/update.go | 7 + 9 files changed, 529 insertions(+), 6 deletions(-) create mode 100644 labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-20-config-schema-migration.md create mode 100644 labs/12-product-engineering-loop/product-engineering-loop/migrate.go create mode 100644 labs/12-product-engineering-loop/product-engineering-loop/migrate_test.go create mode 100644 labs/12-product-engineering-loop/product-engineering-loop/references/config-schema.md diff --git a/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-20-config-schema-migration.md b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-20-config-schema-migration.md new file mode 100644 index 000000000..5153a06a9 --- /dev/null +++ b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-20-config-schema-migration.md @@ -0,0 +1,3 @@ +### Versioned configuration schema migration system + +Introduced a non-breaking, versioned configuration migration system for `.boatstack-project.json`. Older configurations are automatically upgraded to the latest schema version during `/boatstack-update`, with full dry-run reporting, schema gap detection, and helpful error messages for newer configuration versions. A generated configuration schema reference document (`CONFIG_SCHEMA.md`) is now embedded and distributed in the product bundle. diff --git a/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/main.go b/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/main.go index 7c370a3b5..b21ace45d 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/main.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/main.go @@ -1,10 +1,12 @@ package main import ( + "encoding/json" "flag" "fmt" "io" "os" + "path/filepath" "sort" "strings" "time" @@ -513,6 +515,67 @@ func checkSafetyCommand(arguments []string) int { return 0 } +type MigrateConfigReport struct { + Status string `json:"status"` + Message string `json:"message,omitempty"` + FromVersion int `json:"from_version"` + ToVersion int `json:"to_version"` + Changed bool `json:"changed"` +} + +func migrateConfigCommand(arguments []string) int { + flags := flag.NewFlagSet("migrate-config", flag.ContinueOnError) + repo := flags.String("repo", ".", "repository whose configuration should be migrated") + check := flags.Bool("check", false, "dry-run check mode") + if err := flags.Parse(arguments); err != nil { + return 2 + } + configPath := filepath.Join(*repo, ".boatstack-project.json") + raw, err := os.ReadFile(configPath) + if err != nil { + report := MigrateConfigReport{ + Status: "FAIL", + Message: fmt.Sprintf("failed to read config: %v", err), + } + value, _ := json.Marshal(report) + fmt.Print(string(value)) + return 1 + } + upgraded, fromVer, toVer, changed, err := boatstack.MigrateConfigBytes(raw) + if err != nil { + report := MigrateConfigReport{ + Status: "FAIL", + Message: fmt.Sprintf("migration failed: %v", err), + } + value, _ := json.Marshal(report) + fmt.Print(string(value)) + return 1 + } + if changed && !*check { + if err := os.WriteFile(configPath, upgraded, 0o644); err != nil { + report := MigrateConfigReport{ + Status: "FAIL", + Message: fmt.Sprintf("failed to write migrated config: %v", err), + } + value, _ := json.Marshal(report) + fmt.Print(string(value)) + return 1 + } + } + report := MigrateConfigReport{ + Status: "PASS", + FromVersion: fromVer, + ToVersion: toVer, + Changed: changed, + } + value, err := json.Marshal(report) + if err != nil { + return fail(err) + } + fmt.Print(string(value)) + return 0 +} + func prContextCommand(arguments []string) int { flags := flag.NewFlagSet("pr-context", flag.ContinueOnError) repo := flags.String("repo", ".", "repository whose branch should be projected") @@ -665,7 +728,7 @@ func workspaceStatusCommand(arguments []string) int { func run() int { if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, "usage: boatstack-helper ") + fmt.Fprintln(os.Stderr, "usage: boatstack-helper ") return 2 } switch os.Args[1] { @@ -723,6 +786,8 @@ func run() int { return workspaceCleanupCommand(os.Args[2:]) case "workspace-status": return workspaceStatusCommand(os.Args[2:]) + case "migrate-config": + return migrateConfigCommand(os.Args[2:]) case "version": fmt.Printf("Boatstack %s (%s)\n", boatstack.Version, boatstack.SourceCommit) return 0 diff --git a/labs/12-product-engineering-loop/product-engineering-loop/export.go b/labs/12-product-engineering-loop/product-engineering-loop/export.go index 3fde1cb57..001b376fc 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/export.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/export.go @@ -96,8 +96,14 @@ func LoadConfig(path string) (ProjectConfig, []byte, error) { } func ValidateConfig(config ProjectConfig) error { - if config.SchemaVersion != 1 { - return fmt.Errorf("project config schema_version must be 1") + if config.SchemaVersion < 1 { + return fmt.Errorf("project config schema_version must be >= 1") + } + if config.SchemaVersion > currentSchemaVersion() { + return fmt.Errorf("config was written by a newer Boatstack; update Boatstack") + } + if config.SchemaVersion < currentSchemaVersion() { + return fmt.Errorf("config schema is behind; run /boatstack-update") } if strings.TrimSpace(config.Project.Name) == "" { return fmt.Errorf("project.name is required") @@ -217,7 +223,7 @@ func BuildExportBundle(configPath string, config ProjectConfig, rawConfig []byte } } - for _, name := range []string{"workflow.md", "artifacts.md", "failure-moves.md", "irreversible-operation-boundary.md", "host-hook-contracts.md"} { + for _, name := range []string{"workflow.md", "artifacts.md", "failure-moves.md", "irreversible-operation-boundary.md", "host-hook-contracts.md", "config-schema.md"} { value, err := readCanonical("references/" + name) if err != nil { return ExportBundle{}, err diff --git a/labs/12-product-engineering-loop/product-engineering-loop/init.go b/labs/12-product-engineering-loop/product-engineering-loop/init.go index e29874427..48cfb291b 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/init.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/init.go @@ -313,9 +313,26 @@ func RunInit(options InitOptions) (returnErr error) { } var config ProjectConfig var rawConfig []byte + var migrationFrom, migrationTo int + var migrationChanged bool if configExists { - config, rawConfig, err = LoadConfig(configPath) + var err error + rawConfig, err = os.ReadFile(configPath) if err != nil { + return err + } + var upgraded []byte + upgraded, migrationFrom, migrationTo, migrationChanged, err = MigrateConfigBytes(rawConfig) + if err != nil { + return fmt.Errorf("failed to migrate project config: %w", err) + } + if migrationChanged { + rawConfig = upgraded + } + if err := DecodeJSON("load project configuration", configPath, rawConfig, &config); err != nil { + return fmt.Errorf("existing Boatstack config is invalid: %w", err) + } + if err := ValidateConfig(config); err != nil { return fmt.Errorf("existing Boatstack config is invalid: %w", err) } } else { @@ -508,7 +525,11 @@ func RunInit(options InitOptions) (returnErr error) { return scopeErr } fmt.Fprintf(options.Output, "\nPASS: Boatstack updated to %s on a dedicated infrastructure branch.\n", Version) - fmt.Fprintln(options.Output, "PASS: no product files changed.") + if migrationChanged { + fmt.Fprintf(options.Output, "PASS: migrated .boatstack-project.json from schema version %d to %d.\n", migrationFrom, migrationTo) + } else { + fmt.Fprintln(options.Output, "PASS: no product files changed.") + } fmt.Fprintln(options.Output, "Changed Boatstack paths:") for _, path := range changed { fmt.Fprintln(options.Output, " "+path) diff --git a/labs/12-product-engineering-loop/product-engineering-loop/migrate.go b/labs/12-product-engineering-loop/product-engineering-loop/migrate.go new file mode 100644 index 000000000..ddc5b22a2 --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/migrate.go @@ -0,0 +1,101 @@ +package boatstack + +import ( + "encoding/json" + "fmt" + "strings" +) + +const CurrentConfigSchemaVersion = 1 + +var currentConfigSchemaVersionOverride = CurrentConfigSchemaVersion + +type configMigration struct { + from int + to int + apply func(map[string]any) (map[string]any, error) +} + +var configMigrations []configMigration + +func currentSchemaVersion() int { + return currentConfigSchemaVersionOverride +} + +// MigrateConfigBytes migrates raw JSON configuration bytes to the latest CurrentConfigSchemaVersion. +// It returns the upgraded JSON bytes, the original schema version, the final schema version, +// a boolean indicating whether the content actually changed, and any error encountered. +func MigrateConfigBytes(raw []byte) (upgraded []byte, from, to int, changed bool, err error) { + if len(strings.TrimSpace(string(raw))) == 0 { + return raw, 0, 0, false, nil + } + + var partial map[string]any + if err := json.Unmarshal(raw, &partial); err != nil { + return nil, 0, 0, false, fmt.Errorf("failed to parse config JSON: %w", err) + } + + targetVer := currentSchemaVersion() + + var fromVer int + if v, exists := partial["schema_version"]; exists { + switch val := v.(type) { + case float64: + fromVer = int(val) + case int: + fromVer = val + default: + return nil, 0, 0, false, fmt.Errorf("schema_version must be an integer") + } + } else { + // Default to 1 if schema_version is missing + fromVer = 1 + } + + if fromVer > targetVer { + return nil, fromVer, 0, false, fmt.Errorf("config was written by a newer Boatstack; update Boatstack") + } + + if fromVer == targetVer { + return raw, fromVer, fromVer, false, nil + } + + currentVer := fromVer + data := partial + + for currentVer < targetVer { + var found *configMigration + for i := range configMigrations { + if configMigrations[i].from == currentVer { + found = &configMigrations[i] + break + } + } + + if found == nil { + return nil, fromVer, 0, false, fmt.Errorf("no migration found from version %d to %d", currentVer, currentVer+1) + } + + var err error + data, err = found.apply(data) + if err != nil { + return nil, fromVer, 0, false, fmt.Errorf("failed to apply migration from %d to %d: %w", found.from, found.to, err) + } + + if found.to <= currentVer { + return nil, fromVer, 0, false, fmt.Errorf("invalid migration path from %d to %d", found.from, found.to) + } + + currentVer = found.to + } + + data["schema_version"] = targetVer + + upgraded, err = json.MarshalIndent(data, "", " ") + if err != nil { + return nil, fromVer, 0, false, fmt.Errorf("failed to marshal migrated config: %w", err) + } + upgraded = append(upgraded, '\n') + + return upgraded, fromVer, targetVer, true, nil +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/migrate_test.go b/labs/12-product-engineering-loop/product-engineering-loop/migrate_test.go new file mode 100644 index 000000000..35a31a00a --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/migrate_test.go @@ -0,0 +1,274 @@ +package boatstack + +import ( + "bytes" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestMigrateConfigBytes_SyntheticChain(t *testing.T) { + // Backup and restore + oldMigrations := configMigrations + oldOverride := currentConfigSchemaVersionOverride + defer func() { + configMigrations = oldMigrations + currentConfigSchemaVersionOverride = oldOverride + }() + + // Simulate current schema version is 3 + currentConfigSchemaVersionOverride = 3 + + // Register synthetic v1->v2 and v2->v3 migrations + configMigrations = []configMigration{ + { + from: 1, + to: 2, + apply: func(data map[string]any) (map[string]any, error) { + data["v1_to_v2_applied"] = true + return data, nil + }, + }, + { + from: 2, + to: 3, + apply: func(data map[string]any) (map[string]any, error) { + data["v2_to_v3_applied"] = true + return data, nil + }, + }, + } + + raw := []byte(`{ + "schema_version": 1, + "project": { + "name": "test-project" + } + }`) + + upgraded, from, to, changed, err := MigrateConfigBytes(raw) + if err != nil { + t.Fatalf("MigrateConfigBytes failed: %v", err) + } + + if !changed { + t.Error("expected config to be changed") + } + if from != 1 { + t.Errorf("expected from version 1, got %d", from) + } + if to != 3 { + t.Errorf("expected to version 3, got %d", to) + } + + var parsed map[string]any + if err := json.Unmarshal(upgraded, &parsed); err != nil { + t.Fatalf("failed to unmarshal upgraded config: %v", err) + } + + if parsed["v1_to_v2_applied"] != true { + t.Error("expected v1->v2 migration to be applied") + } + if parsed["v2_to_v3_applied"] != true { + t.Error("expected v2->v3 migration to be applied") + } + if int(parsed["schema_version"].(float64)) != 3 { + t.Errorf("expected upgraded schema_version to be 3, got %v", parsed["schema_version"]) + } +} + +func TestMigrateConfigBytes_NoOp(t *testing.T) { + oldOverride := currentConfigSchemaVersionOverride + defer func() { currentConfigSchemaVersionOverride = oldOverride }() + + currentConfigSchemaVersionOverride = 1 + + raw := []byte(`{ + "schema_version": 1, + "project": { + "name": "test-project" + } + }`) + + upgraded, from, to, changed, err := MigrateConfigBytes(raw) + if err != nil { + t.Fatalf("MigrateConfigBytes failed: %v", err) + } + + if changed { + t.Error("expected config to be unchanged (no-op)") + } + if from != 1 || to != 1 { + t.Errorf("expected from=1 and to=1, got from=%d, to=%d", from, to) + } + if !bytes.Equal(raw, upgraded) { + t.Error("expected upgraded bytes to match raw bytes exactly") + } +} + +func TestMigrateConfigBytes_GapDetection(t *testing.T) { + oldMigrations := configMigrations + oldOverride := currentConfigSchemaVersionOverride + defer func() { + configMigrations = oldMigrations + currentConfigSchemaVersionOverride = oldOverride + }() + + currentConfigSchemaVersionOverride = 3 + + // Missing v2->v3 migration + configMigrations = []configMigration{ + { + from: 1, + to: 2, + apply: func(data map[string]any) (map[string]any, error) { + return data, nil + }, + }, + } + + raw := []byte(`{"schema_version": 1}`) + _, _, _, _, err := MigrateConfigBytes(raw) + if err == nil { + t.Fatal("expected error due to missing migration (gap), got nil") + } + if !strings.Contains(err.Error(), "no migration found from version 2 to 3") { + t.Errorf("expected gap error message, got: %v", err) + } +} + +func TestMigrateConfigBytes_RejectNewer(t *testing.T) { + oldOverride := currentConfigSchemaVersionOverride + defer func() { currentConfigSchemaVersionOverride = oldOverride }() + + currentConfigSchemaVersionOverride = 1 + + raw := []byte(`{"schema_version": 2}`) + _, _, _, _, err := MigrateConfigBytes(raw) + if err == nil { + t.Fatal("expected error rejecting newer version, got nil") + } + if !strings.Contains(err.Error(), "config was written by a newer Boatstack; update Boatstack") { + t.Errorf("expected reject newer error message, got: %v", err) + } +} + +func TestValidateConfig_AcceptanceTable(t *testing.T) { + oldOverride := currentConfigSchemaVersionOverride + defer func() { currentConfigSchemaVersionOverride = oldOverride }() + + currentConfigSchemaVersionOverride = 2 + + tests := []struct { + name string + schemaVersion int + wantErr string + }{ + { + name: "current version passes", + schemaVersion: 2, + wantErr: "", + }, + { + name: "older version is behind", + schemaVersion: 1, + wantErr: "config schema is behind; run /boatstack-update", + }, + { + name: "newer version is ahead", + schemaVersion: 3, + wantErr: "config was written by a newer Boatstack; update Boatstack", + }, + { + name: "invalid version < 1", + schemaVersion: 0, + wantErr: "project config schema_version must be >= 1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := ProjectConfig{ + SchemaVersion: tt.schemaVersion, + Project: Project{ + Name: "test-project", + Commands: map[string]string{ + "test": "go test ./...", + }, + }, + } + err := ValidateConfig(cfg) + if tt.wantErr == "" { + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("expected error containing %q, got: %v", tt.wantErr, err) + } + } + }) + } +} + +func TestDoctor_SchemaBehindAndAhead(t *testing.T) { + oldOverride := currentConfigSchemaVersionOverride + defer func() { currentConfigSchemaVersionOverride = oldOverride }() + + currentConfigSchemaVersionOverride = 2 + + errBehind := fmt.Errorf("config schema is behind; run /boatstack-update") + errAhead := fmt.Errorf("config was written by a newer Boatstack; update Boatstack") + + hintBehind := DoctorRepairHint(errBehind) + hintAhead := DoctorRepairHint(errAhead) + + if !strings.Contains(hintBehind.Error(), "remediation: run /boatstack-update to migrate project configuration") { + t.Errorf("expected behind hint, got: %v", hintBehind) + } + if !strings.Contains(hintAhead.Error(), "remediation: update your Boatstack installation to load this configuration") { + t.Errorf("expected ahead hint, got: %v", hintAhead) + } +} + +func TestValidateUpdateWorkspace_ConformanceBlock(t *testing.T) { + oldOverride := currentConfigSchemaVersionOverride + oldVersion := Version + oldSourceCommit := SourceCommit + defer func() { + currentConfigSchemaVersionOverride = oldOverride + Version = oldVersion + SourceCommit = oldSourceCommit + }() + + currentConfigSchemaVersionOverride = 1 + Version = "v0.5.0" + SourceCommit = "update-test-0.5.0" + + now := time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC) + withUpdateGlobals(t, "v0.5.0", now, func() (ReleaseInfo, error) { return ReleaseInfo{}, nil }) + repo, _ := updateInstalledRepo(t) + + // Create a behind config (version 1) + currentConfigSchemaVersionOverride = 1 + config, _, err := LoadConfig(filepath.Join(repo, ".boatstack-project.json")) + if err != nil { + t.Fatal(err) + } + currentConfigSchemaVersionOverride = 2 + + // Set the current checked-out branch to the update branch + runGit(t, repo, "switch", "-c", "chore/update-boatstack-v0.5.0") + + // ValidateUpdateWorkspace with config.SchemaVersion = 1, while current is overridden to 2. + err = ValidateUpdateWorkspace(repo, config) + if err == nil { + t.Fatal("expected ValidateUpdateWorkspace to fail for schema behind, got nil") + } + if !strings.Contains(err.Error(), "config schema is behind; run /boatstack-update") { + t.Errorf("expected error to contain behind message, got: %v", err) + } +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/planning.go b/labs/12-product-engineering-loop/product-engineering-loop/planning.go index de1cc8f15..5887f1db3 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/planning.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/planning.go @@ -258,5 +258,12 @@ func DoctorRepairHint(err error) error { if err == nil { return nil } + errStr := err.Error() + if strings.Contains(errStr, "config schema is behind") { + return fmt.Errorf("%s; remediation: run /boatstack-update to migrate project configuration", errStr) + } + if strings.Contains(errStr, "config was written by a newer Boatstack") { + return fmt.Errorf("%s; remediation: update your Boatstack installation to load this configuration", errStr) + } return fmt.Errorf("%w; repair: rerun the verified Boatstack installer once from any checkout in this Git clone, then reload the coding host", err) } diff --git a/labs/12-product-engineering-loop/product-engineering-loop/references/config-schema.md b/labs/12-product-engineering-loop/product-engineering-loop/references/config-schema.md new file mode 100644 index 000000000..5607693c2 --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/references/config-schema.md @@ -0,0 +1,39 @@ +# Boatstack Configuration Schema + +This reference document defines the schema and version history of `.boatstack-project.json`. + +## Current Schema Version + +- **schema_version**: `1` + +## Field Reference + +### Root Fields + +- `schema_version` (integer, required): Must be exactly `1`. +- `project` (object, required): General project definition. +- `workflow` (object, required): Flags controlling state machine transitions and safety gates. +- `adapters` (array of strings, optional): Enabled host environment adapters. If empty, defaults to enabling all. +- `integrations` (object, optional): Explicit configurations for individual third-party integrations. + +### project Fields + +- `name` (string, required): The human-readable name of the project. +- `default_branch` (string, optional): The canonical development/default branch (e.g. `main` or `master`). +- `context` (array of strings, optional): Paths to persistent project directories or contextual documents. +- `commands` (object, required): Custom development commands: + - `test` (string, required): The exact command to execute project-local tests. +- `high_risk_paths` (array of strings, optional): Glob patterns of files requiring independent reviewer sign-off before shipping. + +### workflow Fields + +- `human_plan_approval` (boolean, optional): Whether a parent plan requires explicit human approval before building. +- `independent_review_for_high_risk` (boolean, optional): Whether modifications to high-risk files require a distinct peer review gate. +- `allow_pass_with_gaps` (boolean, optional): Whether the delivery verification allows outstanding questions or gaps. +- `maintain_changelog` (boolean, optional): Whether a release-notes fragment is required for each delivery slice. + +## Version Changelog + +### Version 1 + +- Initial schema with `project`, `workflow`, `adapters`, and `integrations`. diff --git a/labs/12-product-engineering-loop/product-engineering-loop/update.go b/labs/12-product-engineering-loop/product-engineering-loop/update.go index 02def254f..e23d85f66 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/update.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/update.go @@ -357,5 +357,12 @@ func ValidateUpdateWorkspace(repo string, config ProjectConfig) error { if err := CheckInstalledHostHooks(repo, config.Adapters); err != nil { return fmt.Errorf("host-hook drift blocks update: %w", err) } + var schemaProblems []string + if config.SchemaVersion != currentSchemaVersion() { + schemaProblems = append(schemaProblems, fmt.Sprintf("schema_version %d is behind current %d", config.SchemaVersion, currentSchemaVersion())) + } + if len(schemaProblems) > 0 { + return fmt.Errorf("config schema is behind; run /boatstack-update: %s", strings.Join(schemaProblems, ", ")) + } return CheckExistingInstallProvenance(repo) }