diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..c2ebb986 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,38 @@ +name: Release + +# Builds and publishes dbdeployer binaries when a version tag is pushed. +# Uses GoReleaser with the config in .goreleaser.yaml. +# Produces: linux/darwin x amd64/arm64 tarballs + checksums.txt +# +# Security note: no user-controlled inputs are used. Triggers only on +# version tags pushed by maintainers. + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + +jobs: + release: + name: Build and Release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/villagesql_flavor_test.yml b/.github/workflows/villagesql_flavor_test.yml new file mode 100644 index 00000000..587f796d --- /dev/null +++ b/.github/workflows/villagesql_flavor_test.yml @@ -0,0 +1,145 @@ +name: VillageSQL Flavor Test + +# Tests VillageSQL flavor detection, sandbox deployment, and capability +# inheritance using the official VillageSQL 0.0.3 release tarball from +# https://github.com/villagesql/villagesql-server/releases/tag/0.0.3 +# +# The workflow verifies: +# 1. dbdeployer unpack detects the flavor as "villagesql" via the +# share/villagesql_schema.sql marker file +# 2. Single sandbox deployment works with --unpack-version mapping +# 3. Replication deployment works with data verification +# 4. VillageSQL capabilities correctly inherit MySQL capabilities +# +# Note: The 0.0.3 tarball contains two symlinks in +# mysql-test/suite/villagesql/examples/ that point outside the extraction +# directory. These are stripped before unpacking. See +# https://github.com/villagesql/villagesql-server/issues/237 +# +# Security note: this workflow uses no user-controlled inputs (issue +# bodies, PR titles, commit messages, etc.). All values are hardcoded. + +on: + push: + branches: [master] + pull_request: + branches: [master] + +env: + VILLAGESQL_VERSION: "0.0.3" + VILLAGESQL_SHA256: "8b15522a973b17b430ed9e64e8bdbf97bf858cef028bfbc7f9b9608002406393" + VILLAGESQL_TARBALL: "villagesql-dev-server-0.0.3-dev-linux-x86_64.tar.gz" + VILLAGESQL_URL: "https://github.com/villagesql/villagesql-server/releases/download/0.0.3/villagesql-dev-server-0.0.3-dev-linux-x86_64.tar.gz" + # VillageSQL uses its own version scheme (0.0.3). Map to MySQL 8.0.40 for + # capability lookups (mysqld --initialize, CREATE USER, GTID, etc.) + MYSQL_MAPPED_VERSION: "8.0.40" + GO111MODULE: on + SANDBOX_BINARY: ${{ github.workspace }}/opt/mysql + +jobs: + villagesql-deploy: + name: Deploy (VillageSQL ${{ env.VILLAGESQL_VERSION }}) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y libaio1 libnuma1 libncurses5 + + - name: Build dbdeployer + run: go build -o dbdeployer . + + - name: Cache VillageSQL tarball + uses: actions/cache@v4 + with: + path: /tmp/villagesql-tarball + key: villagesql-${{ env.VILLAGESQL_VERSION }}-linux-x86_64-v1 + + - name: Download and verify VillageSQL tarball + run: | + mkdir -p /tmp/villagesql-tarball + if [ ! -f "/tmp/villagesql-tarball/$VILLAGESQL_TARBALL" ]; then + echo "Downloading VillageSQL $VILLAGESQL_VERSION..." + curl -L -f -o "/tmp/villagesql-tarball/$VILLAGESQL_TARBALL" "$VILLAGESQL_URL" + fi + echo "Verifying checksum..." + echo "$VILLAGESQL_SHA256 /tmp/villagesql-tarball/$VILLAGESQL_TARBALL" | sha256sum -c + ls -lh "/tmp/villagesql-tarball/$VILLAGESQL_TARBALL" + + - name: Repack tarball without broken symlinks + run: | + cd /tmp + mkdir -p villagesql-staging villagesql-clean + tar xzf "villagesql-tarball/$VILLAGESQL_TARBALL" -C villagesql-staging + INNER="villagesql-staging/villagesql-dev-server-$VILLAGESQL_VERSION-dev-linux-x86_64" + # Remove symlinks that point outside the extraction directory + # (https://github.com/villagesql/villagesql-server/issues/237) + rm -f "$INNER/mysql-test/suite/villagesql/examples/vsql-complex" + rm -f "$INNER/mysql-test/suite/villagesql/examples/vsql-tvector" + tar czf "villagesql-clean/$VILLAGESQL_TARBALL" -C villagesql-staging \ + "villagesql-dev-server-$VILLAGESQL_VERSION-dev-linux-x86_64" + rm -rf villagesql-staging + + - name: Test unpack with --unpack-version + run: | + mkdir -p "$SANDBOX_BINARY" + ./dbdeployer unpack "/tmp/villagesql-clean/$VILLAGESQL_TARBALL" \ + --sandbox-binary="$SANDBOX_BINARY" \ + --unpack-version="$MYSQL_MAPPED_VERSION" + + # Verify flavor detected as villagesql + FLAVOR_FILE="$SANDBOX_BINARY/$MYSQL_MAPPED_VERSION/FLAVOR" + [ -f "$FLAVOR_FILE" ] || { echo "FAIL: No FLAVOR file"; exit 1; } + FLAVOR=$(cat "$FLAVOR_FILE") + echo "Detected flavor: $FLAVOR" + [ "$FLAVOR" = "villagesql" ] || { echo "FAIL: expected 'villagesql', got '$FLAVOR'"; exit 1; } + + # Verify marker file + MARKER="$SANDBOX_BINARY/$MYSQL_MAPPED_VERSION/share/villagesql_schema.sql" + [ -f "$MARKER" ] || { echo "FAIL: marker file not found"; exit 1; } + echo "OK: flavor=villagesql, marker file present" + + - name: Test deploy single sandbox + run: | + ./dbdeployer deploy single "$MYSQL_MAPPED_VERSION" --sandbox-binary="$SANDBOX_BINARY" + VERSION=$(~/sandboxes/msb_*/use -BN -e "SELECT VERSION();") + echo "Server version: $VERSION" + echo "$VERSION" | grep -qi villagesql || { echo "FAIL: expected villagesql in VERSION()"; exit 1; } + echo "OK: VillageSQL single sandbox running" + ./dbdeployer delete all --skip-confirm + + - name: Test deploy replication sandbox + run: | + ./dbdeployer deploy replication "$MYSQL_MAPPED_VERSION" --sandbox-binary="$SANDBOX_BINARY" + ~/sandboxes/rsandbox_*/check_slaves + ~/sandboxes/rsandbox_*/test_replication + echo "OK: VillageSQL replication sandbox works" + ./dbdeployer delete all --skip-confirm + + - name: Cleanup + if: always() + run: | + ./dbdeployer delete all --skip-confirm 2>/dev/null || true + pkill -9 -u "$USER" mysqld 2>/dev/null || true + + villagesql-capabilities: + name: Capability Inheritance Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Run capability tests + run: go test ./common/... -v -run TestHasCapability -count=1 + + - name: Run copy capabilities tests + run: go test ./common/... -v -run TestCopyCapabilities -count=1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bab7a09..7b2129ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,133 @@ +## 2.2.2 18-Apr-2026 + +## NEW FEATURES + +* Add VillageSQL flavor support. VillageSQL is a MySQL drop-in replacement + with extensions. Its tarballs are detected via the unique marker file + `share/villagesql_schema.sql` and reuse MySQL's sandbox lifecycle (init, + start, stop, grants, replication) unchanged. Because VillageSQL uses its + own version scheme (e.g. 0.0.3), unpacking requires `--unpack-version` + mapped to the MySQL base version (e.g. `--unpack-version=8.0.40`). +* Add VillageSQL to CI: real tarball download, SHA256 checksum verification, + single and replication deployment tests, capability inheritance tests. + +## 2.2.1 09-Apr-2026 + +## BUGS FIXED + +* Fix InnoDB Cluster deployment: let mysqlsh manage Group Replication from + scratch instead of conflicting with dbdeployer's GR setup +* Fix InnoDB Cluster Basedir template pointing to wrong directory +* Fix `--with-proxysql` failing for InnoDB Cluster (wrong sandbox path in + ProxySQL config) +* Fix Router port extraction including config file path in the result +* Fix Router start hanging forever when mysqlrouter process forks +* Fix ProxySQL grep -v exiting under `set -e` in monitoring scripts +* Fix ProxySQL GR monitor seeing all nodes as offline (hostgroup 3) +* Fix fan-in CREATE DATABASE on node2 conflicting with node1's database +* Fix copy of mysqlsh lib/mysqlsh/ directory (only .so files were copied) +* Fix PostgreSQL multiple sandbox directory naming +* Fix symlink opt/mysql for ts tests in CI (HOME path mismatch) +* Remove MariaDB 11.4 from CI (authentication bug #82) + +## CI + +* Add install script test workflow (downloads and verifies dbdeployer install) +* Drop macos-13 from install test (unsupported runner) +* Replace sleep+check with retry loops for replication verification in CI + +## 2.2.0 08-Apr-2026 + +## NEW FEATURES + +* Add MariaDB 10.6/10.11/11.4/11.7/11.8 and Percona Server 5.7/8.0/8.4 + to tarball registry +* Add ts replication test suite (MySQL 5.7, 8.0, 8.4, 9.5) +* Add PostgreSQL ts testscript tests (single + replication) +* Add MySQL 9.5 support for semisync and ts replication tests + +## BUGS FIXED + +* Use Slave|Replica pattern for IO/SQL thread status in multi-source + ts tests (compatibility with MySQL 8.x terminology) +* Resolve CI failures for MariaDB, Percona, ts replication, and fan-in + +## CI + +* Add Percona Server and MariaDB integration tests +* Add PostgreSQL ts testscript tests to CI +* Add ts replication test suite to CI (5.7, 8.0, 8.4, 9.5) + +## 2.1.1 04-Apr-2026 + +## BUGS FIXED + +* Fix macOS --minimal fallback and `--guess` using real URL patterns +* Fix install script to download checksums.txt instead of per-file .sha256 + +## 2.1.0 04-Apr-2026 + +## NEW FEATURES + +* Add InnoDB Cluster topology (`--topology=innodb-cluster`) with MySQL Shell + and MySQL Router support +* Add ProxySQL GR-aware hostgroups for InnoDB Cluster and Group Replication + (`--with-proxysql` configures reader/writer hostgroups automatically) +* Add `--topology=group` single-primary and multi-primary Group Replication + with full CI coverage +* Add fan-in and all-masters replication topologies with data verification +* Add `downloads add-url` command for custom tarball URLs +* Add MySQL 8.4-specific replication and group replication templates +* Add ProxySQL PostgreSQL backend wiring (`pgsql_servers/pgsql_users`) +* Add `--provider` flag and PostgreSQL routing to all deploy commands +* Add `dbdeployer deploy postgresql` standalone command +* Add `dbdeployer init --provider=postgresql` for one-command setup +* Add macOS PostgreSQL support via Postgres.app binary detection +* Add cross-database topology constraint validation +* Add comprehensive topology/provider/proxy reference documentation +* Add group replication, fan-in, all-masters, PostgreSQL multiple tests to CI +* Add InnoDB Cluster integration tests (MySQL 8.4.8 + 9.5.0) +* Add functional verification (write/read) to all integration tests +* Add ProxySQL `--bootstrap` mode test script +* Add admin web UI proof of concept + +## BUGS FIXED + +* Replace `\G` with `--vertical` in all replication templates (MySQL 9.5 compat) +* Fix semisync template variable scoping and version detection +* Fix PostgreSQL deb extraction version detection and binary setup +* Fix PostgreSQL initdb requiring empty data dir (create log dir after initdb) +* Fix PostgreSQL share files for deb-extracted binaries (timezonesets path) +* Remove dead commented-out `semisync_master_enabled` from template +* Fix `gosec` and `staticcheck` lint warnings in PostgreSQL and ProxySQL code + +## 2.0.0 24-Mar-2026 + +Initial release under the ProxySQL organization. Forked from +[datacharmer/dbdeployer](https://github.com/datacharmer/dbdeployer) v1.73.0 +with Giuseppe Maxia's blessing. + +## NEW FEATURES + +* PostgreSQL provider: full provider architecture with `initdb`, config + generation (`postgresql.conf`, `pg_hba.conf`), single sandbox deployment, + streaming replication via `pg_basebackup`, and monitoring scripts +* PostgreSQL deb extraction for binary management (`unpack --provider=postgresql`) +* ProxySQL provider: standalone and topology-integrated deployment + (`--with-proxysql` wires read/write split into any MySQL/PostgreSQL topology) +* Provider interface with `SupportedTopologies` and `CreateReplica` +* Add MySQL 8.4.0–8.4.8, 9.0.1, 9.1.0, 9.2.0, 9.3.0–9.5.0 to tarball registry +* Add `dbdeployer init` with curl-based install script +* Add admin web UI proof of concept +* Add comprehensive website documentation at proxysql.github.io/dbdeployer + +## CI + +* Full GitHub Actions CI pipeline: lint, unit tests, build verification +* Integration tests: MySQL, Percona Server, MariaDB, PostgreSQL, InnoDB Cluster, + Group Replication, fan-in, all-masters, ProxySQL wiring +* Install script test workflow across multiple OS versions + ## 1.73.0 09-Jul-2023 ## NEW FEATURES diff --git a/README.md b/README.md index b8702acb..91a5b757 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,23 @@ dbdeployer deploy replication 16.13 --provider=postgresql > **Note:** The `apt-get download` command downloads `.deb` files to the current directory without installing anything. Your system is untouched. See the [PostgreSQL provider guide](https://proxysql.github.io/dbdeployer/providers/postgresql/) for details and alternative installation methods. +### VillageSQL + +[VillageSQL](https://github.com/villagesql/villagesql-server) is a MySQL drop-in replacement with extensions (custom types, VDFs). Since it uses its own version scheme, unpack with `--unpack-version` mapped to the MySQL base version: + +```bash +# Download from GitHub Releases +curl -L -o villagesql-dev-server-0.0.3-dev-linux-x86_64.tar.gz \ + https://github.com/villagesql/villagesql-server/releases/download/0.0.3/villagesql-dev-server-0.0.3-dev-linux-x86_64.tar.gz + +# Unpack with MySQL 8.0 version mapping (required for capabilities) +dbdeployer unpack villagesql-dev-server-0.0.3-dev-linux-x86_64.tar.gz --unpack-version=8.0.40 + +# Deploy +dbdeployer deploy single 8.0.40 +~/sandboxes/msb_8_0_40/use -e "SELECT VERSION();" +``` + ## Supported Databases | Provider | Single | Replication | Group Replication | ProxySQL Wiring | @@ -71,6 +88,7 @@ dbdeployer deploy replication 16.13 --provider=postgresql | MariaDB | ✓ | ✓ | — | ✓ | | NDB Cluster | ✓ | ✓ | — | — | | Percona XtraDB Cluster | ✓ | ✓ | — | — | +| VillageSQL | ✓ | ✓ | — | — | ## Key Features diff --git a/common/capabilities.go b/common/capabilities.go index aa46f8d2..89d7edd4 100644 --- a/common/capabilities.go +++ b/common/capabilities.go @@ -52,6 +52,7 @@ const ( NdbFlavor = "ndb" PxcFlavor = "pxc" TiDbFlavor = "tidb" + VillageSQLFlavor = "villagesql" // Feature names InstallDb = "installdb" @@ -255,6 +256,13 @@ var FlavorCompositionList = []flavorIndicator{ }, flavor: PerconaServerFlavor, }, + { + AllNeeded: false, + elements: []elementPath{ + {"share", globals.FnVillagesqlSchema}, + }, + flavor: VillageSQLFlavor, + }, { AllNeeded: false, elements: []elementPath{ @@ -287,6 +295,12 @@ var PerconaCapabilities = Capabilities{ Features: MySQLCapabilities.Features, } +var VillageSQLCapabilities = Capabilities{ + Flavor: VillageSQLFlavor, + Description: "VillageSQL server", + Features: MySQLCapabilities.Features, +} + var TiDBCapabilities = Capabilities{ Flavor: TiDbFlavor, Description: "TiDB isolated server", @@ -382,6 +396,7 @@ var MySQLShellCapabilities = Capabilities{ var AllCapabilities = map[string]Capabilities{ MySQLFlavor: MySQLCapabilities, PerconaServerFlavor: PerconaCapabilities, + VillageSQLFlavor: VillageSQLCapabilities, MariaDbFlavor: MariadbCapabilities, TiDbFlavor: TiDBCapabilities, NdbFlavor: NdbCapabilities, diff --git a/common/capabilities_test.go b/common/capabilities_test.go index 8c61d11c..473ea958 100644 --- a/common/capabilities_test.go +++ b/common/capabilities_test.go @@ -30,6 +30,8 @@ type TestCapabilities struct { func TestHasCapability(t *testing.T) { var capabilitiesList = []TestCapabilities{ {[]string{MySQLFlavor, MariaDbFlavor, PerconaServerFlavor}, InstallDb, "5.1.72", true}, + {[]string{VillageSQLFlavor}, InstallDb, "5.1.72", true}, + {[]string{VillageSQLFlavor}, InstallDb, "5.7.0", false}, {[]string{MariaDbFlavor}, InstallDb, "5.5.0", true}, {[]string{MariaDbFlavor}, InstallDb, "10.0.0", true}, {[]string{MariaDbFlavor}, InstallDb, "10.1.0", true}, @@ -52,6 +54,7 @@ func TestHasCapability(t *testing.T) { {[]string{MySQLFlavor, PerconaServerFlavor, MariaDbFlavor}, SemiSynch, "5.5.40", true}, {[]string{MySQLFlavor}, MySQLX, "5.5.40", false}, {[]string{MySQLFlavor}, MySQLX, "5.7.40", true}, + {[]string{VillageSQLFlavor}, MySQLX, "5.7.40", true}, {[]string{MySQLFlavor, PerconaServerFlavor}, MySQLXDefault, "5.7.40", false}, {[]string{MySQLFlavor, PerconaServerFlavor}, MySQLXDefault, "8.0.40", true}, {[]string{MySQLFlavor, PerconaServerFlavor, MariaDbFlavor}, DynVariables, "5.1.72", true}, @@ -63,6 +66,7 @@ func TestHasCapability(t *testing.T) { {[]string{MySQLFlavor, PerconaServerFlavor}, EnhancedGTID, "5.7.40", true}, {[]string{MySQLFlavor, PerconaServerFlavor}, Initialize, "5.6.40", false}, {[]string{MySQLFlavor, PerconaServerFlavor}, Initialize, "5.7.40", true}, + {[]string{VillageSQLFlavor}, Initialize, "5.7.40", true}, {[]string{MySQLFlavor, PerconaServerFlavor}, CreateUser, "5.6.40", false}, {[]string{MySQLFlavor, PerconaServerFlavor}, CreateUser, "5.7.40", true}, {[]string{MySQLFlavor, PerconaServerFlavor}, SuperReadOnly, "5.6.40", false}, diff --git a/common/checks.go b/common/checks.go index ac2ea307..9dc29b42 100644 --- a/common/checks.go +++ b/common/checks.go @@ -154,6 +154,7 @@ func GetCompatibleClientVersion(basedir, serverVersion string) (string, error) { compatibleFlavors := map[string]bool{ MySQLFlavor: true, PerconaServerFlavor: true, + VillageSQLFlavor: true, } serverVersionList, err := VersionToList(serverVersion) if err != nil { @@ -875,6 +876,7 @@ func DetectTarballFlavor(tarballName string) string { TiDbFlavor: `tidb`, PxcFlavor: `Percona-XtraDB-Cluster`, MySQLShellFlavor: `mysql-shell`, + VillageSQLFlavor: `villagesql`, MySQLFlavor: `mysql`, } @@ -887,6 +889,7 @@ func DetectTarballFlavor(tarballName string) string { TiDbFlavor, PxcFlavor, MySQLShellFlavor, + VillageSQLFlavor, MySQLFlavor, } diff --git a/docs/superpowers/plans/2026-03-24-phase2a-provider-abstraction.md b/docs/superpowers/plans/2026-03-24-phase2a-provider-abstraction.md deleted file mode 100644 index a6ad01d1..00000000 --- a/docs/superpowers/plans/2026-03-24-phase2a-provider-abstraction.md +++ /dev/null @@ -1,560 +0,0 @@ -# Phase 2a: Provider Abstraction & MySQL Refactor - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Introduce the Provider abstraction layer and refactor the existing MySQL sandbox code behind it, so that new providers (ProxySQL, Orchestrator, PostgreSQL) can be added cleanly in Phase 2b. - -**Architecture:** Create a `providers/` package with the `Provider` interface and `ProviderRegistry`. Move MySQL-specific sandbox logic into `providers/mysql/`, keeping the existing `sandbox/` package as a thinner orchestration layer that works through the registry. The `cmd/` layer routes through the registry. All existing functionality must continue to work identically — this is a pure refactoring. - -**Tech Stack:** Go 1.22+, existing Cobra CLI framework - -**Spec:** `docs/superpowers/specs/2026-03-23-dbdeployer-revitalization-design.md` - -**Key constraint:** Every task must leave the codebase in a compilable, test-passing state. No big-bang refactor. - ---- - -## File Structure - -### New files to create: -``` -providers/ - provider.go # Provider interface, Instance, PortRange, ProviderRegistry - provider_test.go # Registry tests with mock provider - mysql/ - mysql.go # MySQLProvider implementing Provider interface - mysql_test.go # MySQL provider unit tests -``` - -### Files to modify: -``` -cmd/root.go # Register MySQL provider in existing init() -cmd/single.go # Add provider validation before sandbox creation -cmd/replication.go # Add provider validation before sandbox creation -cmd/multiple.go # Add provider validation before sandbox creation -``` - -Note: `sandbox/sandbox.go` and `sandbox/replication.go` are NOT modified in Phase 2a. Moving sandbox logic behind the provider interface is deferred to Phase 2b when ProxySQL needs it. - -### Files that stay as-is (no changes needed in Phase 2a): -``` -sandbox/templates/ # All .gotxt files unchanged -sandbox/templates.go # Template collections unchanged -sandbox/repl_templates.go # Template collections unchanged -sandbox/group_replication.go # Touched minimally (registry lookup) -sandbox/multiple.go # Touched minimally -sandbox/multi-source-replication.go -sandbox/ndb_replication.go -sandbox/pxc_replication.go -``` - ---- - -### Task 1: Define Provider interface and ProviderRegistry - -**Files:** -- Create: `providers/provider.go` -- Create: `providers/provider_test.go` - -This is the foundation. The interface is intentionally minimal for Phase 2a — just `Name()`, `ValidateVersion()`, and `DefaultPorts()`. The full interface from the spec (with `CreateSandbox`, `Start`, `Stop`, `Destroy`, `HealthCheck`) will be added in Phase 2b when ProxySQL needs it. This establishes the registry pattern first. - -- [ ] **Step 1: Create `providers/provider.go` with interface and registry** - -```go -package providers - -import ( - "fmt" - "sort" -) - -// Provider is the core abstraction for deploying database infrastructure. -type Provider interface { - // Name returns the provider identifier (e.g., "mysql", "proxysql"). - Name() string - - // ValidateVersion checks if the given version string is valid for this provider. - ValidateVersion(version string) error - - // DefaultPorts returns the port allocation strategy for this provider. - DefaultPorts() PortRange -} - -// PortRange defines a provider's default port allocation. -type PortRange struct { - BasePort int // default starting port (e.g., 3306 for MySQL) - PortsPerInstance int // how many ports each instance needs -} - -// Registry manages available providers. -type Registry struct { - providers map[string]Provider -} - -// NewRegistry creates an empty provider registry. -func NewRegistry() *Registry { - return &Registry{providers: make(map[string]Provider)} -} - -// Register adds a provider to the registry. -func (r *Registry) Register(p Provider) error { - name := p.Name() - if _, exists := r.providers[name]; exists { - return fmt.Errorf("provider %q already registered", name) - } - r.providers[name] = p - return nil -} - -// Get retrieves a provider by name. -func (r *Registry) Get(name string) (Provider, error) { - p, exists := r.providers[name] - if !exists { - return nil, fmt.Errorf("provider %q not found", name) - } - return p, nil -} - -// List returns names of all registered providers (sorted). -func (r *Registry) List() []string { - names := make([]string, 0, len(r.providers)) - for name := range r.providers { - names = append(names, name) - } - sort.Strings(names) - return names -} - -// DefaultRegistry is the global provider registry. -var DefaultRegistry = NewRegistry() -``` - -- [ ] **Step 2: Create `providers/provider_test.go`** - -```go -package providers - -import "testing" - -type mockProvider struct { - name string -} - -func (m *mockProvider) Name() string { return m.name } -func (m *mockProvider) ValidateVersion(version string) error { return nil } -func (m *mockProvider) DefaultPorts() PortRange { return PortRange{BasePort: 9999, PortsPerInstance: 1} } - -func TestRegistryRegisterAndGet(t *testing.T) { - reg := NewRegistry() - mock := &mockProvider{name: "test"} - - if err := reg.Register(mock); err != nil { - t.Fatalf("Register failed: %v", err) - } - - p, err := reg.Get("test") - if err != nil { - t.Fatalf("Get failed: %v", err) - } - if p.Name() != "test" { - t.Errorf("expected name 'test', got %q", p.Name()) - } -} - -func TestRegistryDuplicateRegister(t *testing.T) { - reg := NewRegistry() - mock := &mockProvider{name: "test"} - _ = reg.Register(mock) - err := reg.Register(mock) - if err == nil { - t.Fatal("expected error on duplicate register") - } -} - -func TestRegistryGetNotFound(t *testing.T) { - reg := NewRegistry() - _, err := reg.Get("nonexistent") - if err == nil { - t.Fatal("expected error on missing provider") - } -} - -func TestRegistryList(t *testing.T) { - reg := NewRegistry() - _ = reg.Register(&mockProvider{name: "a"}) - _ = reg.Register(&mockProvider{name: "b"}) - names := reg.List() - if len(names) != 2 { - t.Errorf("expected 2 providers, got %d", len(names)) - } -} -``` - -- [ ] **Step 3: Verify tests pass** - -Run: `go test ./providers/... -v` -Expected: All 4 tests pass. - -- [ ] **Step 4: Commit** - -```bash -git add providers/ -git commit -m "feat: add Provider interface and ProviderRegistry" -``` - ---- - -### Task 2: Create MySQLProvider implementing the Provider interface - -**Files:** -- Create: `providers/mysql/mysql.go` -- Create: `providers/mysql/mysql_test.go` - -The MySQL provider starts minimal — just implementing the interface. It doesn't replace any existing functionality yet. That happens in Task 3. - -- [ ] **Step 1: Create `providers/mysql/mysql.go`** - -```go -package mysql - -import ( - "fmt" - "strings" - - "github.com/ProxySQL/dbdeployer/providers" -) - -const ProviderName = "mysql" - -// MySQLProvider implements the Provider interface for MySQL and its flavors -// (Percona, MariaDB, NDB, PXC, TiDB). -type MySQLProvider struct{} - -// NewMySQLProvider creates a new MySQL provider. -func NewMySQLProvider() *MySQLProvider { - return &MySQLProvider{} -} - -func (p *MySQLProvider) Name() string { return ProviderName } - -func (p *MySQLProvider) ValidateVersion(version string) error { - parts := strings.Split(version, ".") - if len(parts) < 2 { - return fmt.Errorf("invalid MySQL version format: %q (expected X.Y or X.Y.Z)", version) - } - return nil -} - -func (p *MySQLProvider) DefaultPorts() providers.PortRange { - return providers.PortRange{ - BasePort: 3306, - PortsPerInstance: 3, // main port + mysqlx port + admin port - } -} - -// Register adds the MySQL provider to the given registry. -func Register(reg *providers.Registry) error { - return reg.Register(NewMySQLProvider()) -} -``` - -- [ ] **Step 2: Create `providers/mysql/mysql_test.go`** - -```go -package mysql - -import ( - "testing" - - "github.com/ProxySQL/dbdeployer/providers" -) - -func TestMySQLProviderName(t *testing.T) { - p := NewMySQLProvider() - if p.Name() != "mysql" { - t.Errorf("expected 'mysql', got %q", p.Name()) - } -} - -func TestMySQLProviderValidateVersion(t *testing.T) { - p := NewMySQLProvider() - tests := []struct { - version string - wantErr bool - }{ - {"8.4.4", false}, - {"9.1.0", false}, - {"5.7", false}, - {"invalid", true}, - } - for _, tt := range tests { - err := p.ValidateVersion(tt.version) - if (err != nil) != tt.wantErr { - t.Errorf("ValidateVersion(%q) error = %v, wantErr %v", tt.version, err, tt.wantErr) - } - } -} - -func TestMySQLProviderRegister(t *testing.T) { - reg := providers.NewRegistry() - if err := Register(reg); err != nil { - t.Fatalf("Register failed: %v", err) - } - p, err := reg.Get("mysql") - if err != nil { - t.Fatalf("Get failed: %v", err) - } - if p.Name() != "mysql" { - t.Errorf("expected 'mysql', got %q", p.Name()) - } -} -``` - -- [ ] **Step 3: Verify tests pass** - -Run: `go test ./providers/... -v` -Expected: All tests pass (both providers/ and providers/mysql/). - -- [ ] **Step 4: Commit** - -```bash -git add providers/mysql/ -git commit -m "feat: add MySQLProvider implementing Provider interface" -``` - ---- - -### Task 3: Register MySQLProvider at startup and wire into cmd/root.go - -**Files:** -- Modify: `cmd/root.go` (add provider registration to existing init function) - -This wires the provider registry into the application lifecycle without changing any existing behavior. No change to `main.go` is needed since it already imports `cmd`. - -- [ ] **Step 1: Add MySQL provider registration to the existing init() in cmd/root.go** - -`cmd/root.go` already has an `init()` function (around line 145). Add the provider registration at the top of that existing function: - -```go -import ( - "github.com/ProxySQL/dbdeployer/providers" - mysqlprovider "github.com/ProxySQL/dbdeployer/providers/mysql" -) - -func init() { - // Register built-in providers - if err := mysqlprovider.Register(providers.DefaultRegistry); err != nil { - // This should never happen at startup - panic(fmt.Sprintf("failed to register MySQL provider: %v", err)) - } -} -``` - -- [ ] **Step 2: Verify the application still builds and runs** - -```bash -go build -o dbdeployer . -./dbdeployer --version -``` -Expected: Outputs version 1.74.1 (or current). No behavior change. - -- [ ] **Step 3: Run all unit tests** - -Run: `go test ./... -timeout 30m 2>&1 | grep -E "^(ok|FAIL)" | grep -v "sandbox\|ts\b"` -Expected: All packages pass. - -- [ ] **Step 4: Commit** - -```bash -git add cmd/root.go -git commit -m "feat: register MySQLProvider at startup via DefaultRegistry" -``` - ---- - -### Task 4: Add provider lookup to cmd/single.go - -**Files:** -- Modify: `cmd/single.go` - -This is the first cmd/ file to use the registry. It looks up the MySQL provider and validates the version before calling the existing sandbox creation. Minimal change — just adds a validation step. - -- [ ] **Step 1: Read cmd/single.go and understand the current flow** - -Find the function that handles `dbdeployer deploy single `. It calls into `sandbox.CreateStandaloneSandbox()`. Add a provider lookup + validation before that call. - -- [ ] **Step 2: Add provider validation** - -After `fillSandboxDefinition()` returns and before `CreateStandaloneSandbox()` is called, add provider validation using `sd.Version` (which is the resolved version, not the raw CLI argument): - -```go -// Validate version with provider -// TODO: Phase 2b — determine provider from sd.Flavor instead of hardcoding "mysql" -p, err := providers.DefaultRegistry.Get("mysql") -if err != nil { - common.Exitf(1, "provider error: %s", err) -} -if err := p.ValidateVersion(sd.Version); err != nil { - common.Exitf(1, "version validation failed: %s", err) -} -``` - -This is additive — existing code continues to work, we just add a validation gate. The `ValidateVersion` call is a seam for future use; the existing code already does extensive version checking. - -- [ ] **Step 3: Verify single sandbox deployment still works** - -```bash -go build -o dbdeployer . -./dbdeployer deploy single 8.4.4 --sandbox-binary=$HOME/opt/mysql -~/sandboxes/msb_8_4_4/use -e "SELECT VERSION()" -./dbdeployer delete all --skip-confirm -``` - -- [ ] **Step 4: Commit** - -```bash -git add cmd/single.go -git commit -m "feat: add provider validation to single sandbox deployment" -``` - ---- - -### Task 5: Add provider lookup to cmd/replication.go and cmd/multiple.go - -**Files:** -- Modify: `cmd/replication.go` -- Modify: `cmd/multiple.go` - -Same pattern as Task 4 — add provider validation before existing sandbox creation calls. - -- [ ] **Step 1: Add provider validation to cmd/replication.go** - -Same pattern: look up "mysql" provider, validate version, then proceed with existing flow. - -- [ ] **Step 2: Add provider validation to cmd/multiple.go** - -Same pattern. - -- [ ] **Step 3: Verify replication deployment still works** - -```bash -go build -o dbdeployer . -./dbdeployer deploy replication 8.4.4 --sandbox-binary=$HOME/opt/mysql -~/sandboxes/rsandbox_8_4_4/check_slaves -./dbdeployer delete all --skip-confirm -``` - -- [ ] **Step 4: Run all unit tests** - -Run: `go test ./cmd/... -v -timeout 30m` -Expected: All cmd tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add cmd/replication.go cmd/multiple.go -git commit -m "feat: add provider validation to replication and multiple deployments" -``` - ---- - -### Task 6: Add `dbdeployer providers list` command - -**Files:** -- Create: `cmd/providers.go` - -A new CLI command that lists registered providers. This makes the provider system visible to users and verifies the registry is wired correctly end-to-end. - -- [ ] **Step 1: Create `cmd/providers.go`** - -```go -package cmd - -import ( - "fmt" - - "github.com/ProxySQL/dbdeployer/providers" - "github.com/spf13/cobra" -) - -var providersCmd = &cobra.Command{ - Use: "providers", - Short: "Shows available deployment providers", - Long: "Lists all registered providers that can be used for sandbox deployment", - Run: func(cmd *cobra.Command, args []string) { - for _, name := range providers.DefaultRegistry.List() { - p, _ := providers.DefaultRegistry.Get(name) - ports := p.DefaultPorts() - fmt.Printf("%-15s (base port: %d, ports per instance: %d)\n", - name, ports.BasePort, ports.PortsPerInstance) - } - }, -} - -func init() { - rootCmd.AddCommand(providersCmd) -} -``` - -- [ ] **Step 2: Build and test** - -```bash -go build -o dbdeployer . -./dbdeployer providers -``` -Expected output: -``` -mysql (base port: 3306, ports per instance: 3) -``` - -- [ ] **Step 3: Commit** - -```bash -git add cmd/providers.go -git commit -m "feat: add 'dbdeployer providers' command to list registered providers" -``` - ---- - -### Task 7: Final validation and cleanup - -- [ ] **Step 1: Run all unit tests** - -```bash -go test ./providers/... ./cmd/... ./common/... ./downloads/... ./ops/... -timeout 30m -``` -Expected: All pass. - -- [ ] **Step 2: Run integration test locally** - -```bash -go build -o dbdeployer . -# Single -./dbdeployer deploy single 8.4.4 --sandbox-binary=$HOME/opt/mysql -~/sandboxes/msb_8_4_4/use -e "SELECT VERSION()" -./dbdeployer delete all --skip-confirm -# Replication -./dbdeployer deploy replication 9.1.0 --sandbox-binary=$HOME/opt/mysql -~/sandboxes/rsandbox_9_1_0/check_slaves -./dbdeployer delete all --skip-confirm -# Providers command -./dbdeployer providers -``` - -- [ ] **Step 3: Verify no regressions in existing behavior** - -The provider layer is purely additive in Phase 2a. No existing command syntax or behavior should change. The only new command is `dbdeployer providers`. - -- [ ] **Step 4: Commit any final fixes** - ---- - -## What Phase 2a Does NOT Do (Deferred to Phase 2b) - -- Does NOT decompose SandboxDef into base + provider-specific structs (that happens when ProxySQL needs a different config shape) -- Does NOT move MySQL sandbox creation logic into providers/mysql/ (the Provider interface is established but MySQL's `CreateSandbox` still lives in `sandbox/`) -- Does NOT add ProxySQL, Orchestrator, or PostgreSQL providers -- Does NOT add topology-aware multi-provider deployment (`--with-proxysql`) -- Does NOT change the sandbox catalog - -These are intentionally deferred to keep Phase 2a small, safe, and mergeable. The Provider interface and Registry are the foundation; Phase 2b builds on them. diff --git a/docs/superpowers/plans/2026-03-24-phase2b-proxysql-provider.md b/docs/superpowers/plans/2026-03-24-phase2b-proxysql-provider.md deleted file mode 100644 index 40a764b8..00000000 --- a/docs/superpowers/plans/2026-03-24-phase2b-proxysql-provider.md +++ /dev/null @@ -1,842 +0,0 @@ -# Phase 2b: ProxySQL Provider Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add ProxySQL as the first non-MySQL provider in dbdeployer, supporting standalone ProxySQL sandboxes and topology-aware deployment with MySQL replication. - -**Architecture:** ProxySQL provider uses system-installed binaries (no tarball management). Deploys local ProxySQL instances with generated config files, data directories, and lifecycle scripts. Topology-aware deployment (`--with-proxysql`) automatically configures ProxySQL backends based on the MySQL topology type. - -**Tech Stack:** Go 1.22+, ProxySQL admin interface (MySQL protocol), existing Cobra CLI - -**Spec:** `docs/superpowers/specs/2026-03-23-dbdeployer-revitalization-design.md` - ---- - -## Key Design Decisions - -### Binary management -- First iteration: ProxySQL must be installed on the system (deb/rpm/compiled) -- Provider locates `proxysql` binary in PATH or user-configured location -- `Unpack()` is a no-op — tarball support deferred to when ProxySQL distributes tarballs - -### ProxySQL sandbox structure -``` -~/sandboxes/proxysql_2_7_0/ - proxysql.cnf # generated config - data/ # ProxySQL SQLite datadir - start # lifecycle script - stop # - status # - use # connects to admin interface via mysql client - use_proxy # connects through ProxySQL's MySQL port - my.proxy.cnf # client defaults for admin connection -``` - -### Topology-aware config generation -ProxySQL config varies by MySQL topology: - -| MySQL Topology | Hostgroups | Monitoring | -|---------------|-----------|------------| -| Single | HG 0 only (one backend) | Basic health check | -| Replication | HG 0 = writer (master), HG 1 = readers (slaves) | read_only + replication lag | -| Group Replication | HG 0 = writer, HG 1 = readers | group_replication monitoring | - -No query rules are generated — users configure those themselves. - -### Monitor user -Uses the existing `msandbox` user for backend monitoring (already has SELECT privileges on all nodes). - ---- - -## File Structure - -### New files: -``` -providers/proxysql/ - proxysql.go # ProxySQLProvider implementing Provider - proxysql_test.go # unit tests - config.go # config file generation for different topologies - config_test.go # config generation tests - templates/ - proxysql.cnf.gotxt # ProxySQL config template - start.gotxt # start script template - stop.gotxt # stop script template - status.gotxt # status script template - use.gotxt # admin connection script - use_proxy.gotxt # proxy connection script -``` - -### Files to modify: -``` -providers/provider.go # Extend Provider interface with CreateSandbox, Start, Stop -cmd/root.go # Register ProxySQL provider -cmd/single.go # Add --with-proxysql flag -cmd/replication.go # Add --with-proxysql flag -sandbox/replication.go # Hook for post-deploy ProxySQL wiring -``` - ---- - -### Task 1: Extend Provider interface with lifecycle methods - -**Files:** -- Modify: `providers/provider.go` -- Modify: `providers/provider_test.go` -- Modify: `providers/mysql/mysql.go` - -The Phase 2a interface only has `Name`, `ValidateVersion`, `DefaultPorts`. Now add the methods needed for ProxySQL to actually deploy sandboxes. - -- [ ] **Step 1: Add SandboxConfig and lifecycle methods to Provider interface** - -In `providers/provider.go`, add: - -```go -// SandboxConfig holds provider-agnostic sandbox configuration. -type SandboxConfig struct { - Version string - Dir string // sandbox directory path - Port int // primary port - AdminPort int // admin/management port (0 if not applicable) - Host string // bind address - DbUser string // admin username - DbPassword string // admin password - Options map[string]string // provider-specific key-value options -} - -// SandboxInfo describes a running sandbox instance. -type SandboxInfo struct { - Dir string - Port int - Socket string - Status string // "running", "stopped" -} -``` - -Extend the Provider interface: - -```go -type Provider interface { - Name() string - ValidateVersion(version string) error - DefaultPorts() PortRange - // FindBinary returns the path to the provider's main binary, or error if not found. - FindBinary(version string) (string, error) - // CreateSandbox deploys a new sandbox instance. - CreateSandbox(config SandboxConfig) (*SandboxInfo, error) - // StartSandbox starts a stopped sandbox. - StartSandbox(dir string) error - // StopSandbox stops a running sandbox. - StopSandbox(dir string) error -} -``` - -- [ ] **Step 2: Add stub implementations to MySQLProvider** - -In `providers/mysql/mysql.go`, add no-op stubs so it still compiles: - -```go -func (p *MySQLProvider) FindBinary(version string) (string, error) { - return "", fmt.Errorf("MySQLProvider.FindBinary: use sandbox package directly (not yet migrated)") -} - -func (p *MySQLProvider) CreateSandbox(config providers.SandboxConfig) (*providers.SandboxInfo, error) { - return nil, fmt.Errorf("MySQLProvider.CreateSandbox: use sandbox package directly (not yet migrated)") -} - -func (p *MySQLProvider) StartSandbox(dir string) error { - return fmt.Errorf("MySQLProvider.StartSandbox: use sandbox package directly (not yet migrated)") -} - -func (p *MySQLProvider) StopSandbox(dir string) error { - return fmt.Errorf("MySQLProvider.StopSandbox: use sandbox package directly (not yet migrated)") -} -``` - -- [ ] **Step 3: Update mock in provider_test.go** - -Add stub methods to the mock provider so tests compile. - -- [ ] **Step 4: Verify all tests pass** - -Run: `go test ./providers/... -v` - -- [ ] **Step 5: Commit** - -```bash -git add providers/ -git commit -m "feat: extend Provider interface with FindBinary, CreateSandbox, Start, Stop" -``` - ---- - -### Task 2: Create ProxySQL provider — binary detection and registration - -**Files:** -- Create: `providers/proxysql/proxysql.go` -- Create: `providers/proxysql/proxysql_test.go` -- Modify: `cmd/root.go` (register proxysql provider) - -- [ ] **Step 1: Create `providers/proxysql/proxysql.go`** - -```go -package proxysql - -import ( - "fmt" - "os/exec" - "strings" - - "github.com/ProxySQL/dbdeployer/providers" -) - -const ProviderName = "proxysql" - -type ProxySQLProvider struct{} - -func NewProxySQLProvider() *ProxySQLProvider { - return &ProxySQLProvider{} -} - -func (p *ProxySQLProvider) Name() string { return ProviderName } - -func (p *ProxySQLProvider) ValidateVersion(version string) error { - parts := strings.Split(version, ".") - if len(parts) < 2 { - return fmt.Errorf("invalid ProxySQL version format: %q", version) - } - return nil -} - -func (p *ProxySQLProvider) DefaultPorts() providers.PortRange { - return providers.PortRange{ - BasePort: 6032, // admin port - PortsPerInstance: 2, // admin port + mysql port - } -} - -// FindBinary locates the proxysql binary on the system. -func (p *ProxySQLProvider) FindBinary(version string) (string, error) { - path, err := exec.LookPath("proxysql") - if err != nil { - return "", fmt.Errorf("proxysql binary not found in PATH: %w", err) - } - return path, nil -} - -func (p *ProxySQLProvider) CreateSandbox(config providers.SandboxConfig) (*providers.SandboxInfo, error) { - // Implemented in Task 3 - return nil, fmt.Errorf("not yet implemented") -} - -func (p *ProxySQLProvider) StartSandbox(dir string) error { - return fmt.Errorf("not yet implemented") -} - -func (p *ProxySQLProvider) StopSandbox(dir string) error { - return fmt.Errorf("not yet implemented") -} - -func Register(reg *providers.Registry) error { - return reg.Register(NewProxySQLProvider()) -} -``` - -- [ ] **Step 2: Create `providers/proxysql/proxysql_test.go`** - -```go -package proxysql - -import ( - "testing" - - "github.com/ProxySQL/dbdeployer/providers" -) - -func TestProxySQLProviderName(t *testing.T) { - p := NewProxySQLProvider() - if p.Name() != "proxysql" { - t.Errorf("expected 'proxysql', got %q", p.Name()) - } -} - -func TestProxySQLProviderValidateVersion(t *testing.T) { - p := NewProxySQLProvider() - tests := []struct { - version string - wantErr bool - }{ - {"2.7.0", false}, - {"3.0.0", false}, - {"invalid", true}, - } - for _, tt := range tests { - err := p.ValidateVersion(tt.version) - if (err != nil) != tt.wantErr { - t.Errorf("ValidateVersion(%q) error = %v, wantErr %v", tt.version, err, tt.wantErr) - } - } -} - -func TestProxySQLProviderRegister(t *testing.T) { - reg := providers.NewRegistry() - if err := Register(reg); err != nil { - t.Fatalf("Register failed: %v", err) - } - p, err := reg.Get("proxysql") - if err != nil { - t.Fatalf("Get failed: %v", err) - } - if p.Name() != "proxysql" { - t.Errorf("expected 'proxysql', got %q", p.Name()) - } -} - -func TestProxySQLFindBinary(t *testing.T) { - p := NewProxySQLProvider() - path, err := p.FindBinary("2.7.0") - if err != nil { - t.Skipf("proxysql not installed, skipping: %v", err) - } - if path == "" { - t.Error("expected non-empty path") - } -} -``` - -- [ ] **Step 3: Register ProxySQL provider in cmd/root.go** - -Add alongside the MySQL registration: - -```go -import proxysqlprovider "github.com/ProxySQL/dbdeployer/providers/proxysql" - -// In init(): -// ProxySQL registration is non-fatal — it's OK if proxysql isn't installed -_ = proxysqlprovider.Register(providers.DefaultRegistry) -``` - -- [ ] **Step 4: Verify** - -```bash -go build -o dbdeployer . -./dbdeployer providers -``` -Expected: -``` -mysql (base port: 3306, ports per instance: 3) -proxysql (base port: 6032, ports per instance: 2) -``` - -- [ ] **Step 5: Commit** - -```bash -git add providers/proxysql/ cmd/root.go -git commit -m "feat: add ProxySQL provider with binary detection" -``` - ---- - -### Task 3: ProxySQL sandbox creation — config generation and lifecycle scripts - -**Files:** -- Create: `providers/proxysql/config.go` -- Create: `providers/proxysql/config_test.go` -- Modify: `providers/proxysql/proxysql.go` (implement CreateSandbox, StartSandbox, StopSandbox) - -This is the core of the ProxySQL provider. It generates a proxysql.cnf, creates the sandbox directory structure, and writes lifecycle scripts. - -- [ ] **Step 1: Create `providers/proxysql/config.go`** - -Config generation function that builds a proxysql.cnf string: - -```go -package proxysql - -import ( - "fmt" - "strings" -) - -// BackendServer represents a MySQL backend for ProxySQL configuration. -type BackendServer struct { - Host string - Port int - Hostgroup int - MaxConns int - Weight int -} - -// ProxySQLConfig holds all settings needed to generate proxysql.cnf. -type ProxySQLConfig struct { - AdminHost string - AdminPort int - AdminUser string - AdminPassword string - MySQLPort int - DataDir string - Backends []BackendServer - MonitorUser string - MonitorPass string -} - -// GenerateConfig produces a proxysql.cnf file content. -func GenerateConfig(cfg ProxySQLConfig) string { - var b strings.Builder - - b.WriteString("datadir=\"" + cfg.DataDir + "\"\n\n") - - b.WriteString("admin_variables=\n{\n") - b.WriteString(fmt.Sprintf(" admin_credentials=\"%s:%s\"\n", cfg.AdminUser, cfg.AdminPassword)) - b.WriteString(fmt.Sprintf(" mysql_ifaces=\"%s:%d\"\n", cfg.AdminHost, cfg.AdminPort)) - b.WriteString("}\n\n") - - b.WriteString("mysql_variables=\n{\n") - b.WriteString(fmt.Sprintf(" interfaces=\"%s:%d\"\n", cfg.AdminHost, cfg.MySQLPort)) - b.WriteString(fmt.Sprintf(" monitor_username=\"%s\"\n", cfg.MonitorUser)) - b.WriteString(fmt.Sprintf(" monitor_password=\"%s\"\n", cfg.MonitorPass)) - b.WriteString(" monitor_connect_interval=2000\n") - b.WriteString(" monitor_ping_interval=2000\n") - b.WriteString("}\n\n") - - if len(cfg.Backends) > 0 { - b.WriteString("mysql_servers=\n(\n") - for i, srv := range cfg.Backends { - b.WriteString(" {\n") - b.WriteString(fmt.Sprintf(" address=\"%s\"\n", srv.Host)) - b.WriteString(fmt.Sprintf(" port=%d\n", srv.Port)) - b.WriteString(fmt.Sprintf(" hostgroup=%d\n", srv.Hostgroup)) - maxConns := srv.MaxConns - if maxConns == 0 { - maxConns = 200 - } - b.WriteString(fmt.Sprintf(" max_connections=%d\n", maxConns)) - b.WriteString(" }") - if i < len(cfg.Backends)-1 { - b.WriteString(",") - } - b.WriteString("\n") - } - b.WriteString(")\n\n") - } - - b.WriteString("mysql_users=\n(\n") - b.WriteString(" {\n") - b.WriteString(fmt.Sprintf(" username=\"%s\"\n", cfg.MonitorUser)) - b.WriteString(fmt.Sprintf(" password=\"%s\"\n", cfg.MonitorPass)) - b.WriteString(" default_hostgroup=0\n") - b.WriteString(" }\n") - b.WriteString(")\n") - - return b.String() -} -``` - -- [ ] **Step 2: Create `providers/proxysql/config_test.go`** - -```go -package proxysql - -import ( - "strings" - "testing" -) - -func TestGenerateConfigBasic(t *testing.T) { - cfg := ProxySQLConfig{ - AdminHost: "127.0.0.1", - AdminPort: 6032, - AdminUser: "admin", - AdminPassword: "admin", - MySQLPort: 6033, - DataDir: "/tmp/proxysql-test", - MonitorUser: "msandbox", - MonitorPass: "msandbox", - } - result := GenerateConfig(cfg) - if !strings.Contains(result, `admin_credentials="admin:admin"`) { - t.Error("missing admin credentials") - } - if !strings.Contains(result, `interfaces="127.0.0.1:6033"`) { - t.Error("missing mysql interfaces") - } - if !strings.Contains(result, `monitor_username="msandbox"`) { - t.Error("missing monitor username") - } -} - -func TestGenerateConfigWithBackends(t *testing.T) { - cfg := ProxySQLConfig{ - AdminHost: "127.0.0.1", - AdminPort: 6032, - AdminUser: "admin", - AdminPassword: "admin", - MySQLPort: 6033, - DataDir: "/tmp/proxysql-test", - MonitorUser: "msandbox", - MonitorPass: "msandbox", - Backends: []BackendServer{ - {Host: "127.0.0.1", Port: 3306, Hostgroup: 0, MaxConns: 100}, - {Host: "127.0.0.1", Port: 3307, Hostgroup: 1, MaxConns: 100}, - }, - } - result := GenerateConfig(cfg) - if !strings.Contains(result, "mysql_servers=") { - t.Error("missing mysql_servers section") - } - if !strings.Contains(result, "port=3306") { - t.Error("missing first backend port") - } - if !strings.Contains(result, "hostgroup=1") { - t.Error("missing reader hostgroup") - } -} -``` - -- [ ] **Step 3: Implement CreateSandbox, StartSandbox, StopSandbox in proxysql.go** - -Update the provider to actually create sandbox directories with config and scripts: - -```go -func (p *ProxySQLProvider) CreateSandbox(config providers.SandboxConfig) (*providers.SandboxInfo, error) { - binaryPath, err := p.FindBinary(config.Version) - if err != nil { - return nil, err - } - - // Create directory structure - dataDir := filepath.Join(config.Dir, "data") - if err := os.MkdirAll(dataDir, 0755); err != nil { - return nil, fmt.Errorf("creating data directory: %w", err) - } - - adminPort := config.AdminPort - if adminPort == 0 { - adminPort = config.Port - } - mysqlPort := adminPort + 1 - - // Generate config - proxyCfg := ProxySQLConfig{ - AdminHost: config.Host, - AdminPort: adminPort, - AdminUser: config.DbUser, - AdminPassword: config.DbPassword, - MySQLPort: mysqlPort, - DataDir: dataDir, - MonitorUser: config.Options["monitor_user"], - MonitorPass: config.Options["monitor_password"], - } - - // Parse backends from options if provided - // (populated by topology-aware deployment) - proxyCfg.Backends = parseBackends(config.Options) - - cfgContent := GenerateConfig(proxyCfg) - cfgPath := filepath.Join(config.Dir, "proxysql.cnf") - if err := os.WriteFile(cfgPath, []byte(cfgContent), 0644); err != nil { - return nil, fmt.Errorf("writing config: %w", err) - } - - // Write lifecycle scripts - writeScript(config.Dir, "start", fmt.Sprintf( - "#!/bin/bash\n%s --config %s -D %s &\necho $! > %s/proxysql.pid\necho 'ProxySQL started'\n", - binaryPath, cfgPath, dataDir, config.Dir)) - - writeScript(config.Dir, "stop", fmt.Sprintf( - "#!/bin/bash\nif [ -f %s/proxysql.pid ]; then\n kill $(cat %s/proxysql.pid) 2>/dev/null\n rm -f %s/proxysql.pid\n echo 'ProxySQL stopped'\nfi\n", - config.Dir, config.Dir, config.Dir)) - - writeScript(config.Dir, "status", fmt.Sprintf( - "#!/bin/bash\nif [ -f %s/proxysql.pid ] && kill -0 $(cat %s/proxysql.pid) 2>/dev/null; then\n echo 'ProxySQL running (pid '$(cat %s/proxysql.pid)')'\nelse\n echo 'ProxySQL not running'\n exit 1\nfi\n", - config.Dir, config.Dir, config.Dir)) - - writeScript(config.Dir, "use", fmt.Sprintf( - "#!/bin/bash\nmysql -h %s -P %d -u %s -p%s --prompt 'ProxySQL Admin> ' \"$@\"\n", - config.Host, adminPort, config.DbUser, config.DbPassword)) - - writeScript(config.Dir, "use_proxy", fmt.Sprintf( - "#!/bin/bash\nmysql -h %s -P %d -u %s -p%s --prompt 'ProxySQL> ' \"$@\"\n", - config.Host, mysqlPort, config.Options["monitor_user"], config.Options["monitor_password"])) - - return &providers.SandboxInfo{ - Dir: config.Dir, - Port: adminPort, - Status: "stopped", - }, nil -} - -func (p *ProxySQLProvider) StartSandbox(dir string) error { - startScript := filepath.Join(dir, "start") - cmd := exec.Command("bash", startScript) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("start failed: %s: %w", string(output), err) - } - return nil -} - -func (p *ProxySQLProvider) StopSandbox(dir string) error { - stopScript := filepath.Join(dir, "stop") - cmd := exec.Command("bash", stopScript) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("stop failed: %s: %w", string(output), err) - } - return nil -} - -func writeScript(dir, name, content string) error { - path := filepath.Join(dir, name) - return os.WriteFile(path, []byte(content), 0755) -} - -func parseBackends(options map[string]string) []BackendServer { - // Format: "host1:port1:hg1,host2:port2:hg2" - raw, ok := options["backends"] - if !ok || raw == "" { - return nil - } - var backends []BackendServer - for _, entry := range strings.Split(raw, ",") { - parts := strings.Split(entry, ":") - if len(parts) >= 3 { - port, _ := strconv.Atoi(parts[1]) - hg, _ := strconv.Atoi(parts[2]) - backends = append(backends, BackendServer{ - Host: parts[0], - Port: port, - Hostgroup: hg, - MaxConns: 200, - }) - } - } - return backends -} -``` - -- [ ] **Step 4: Run tests** - -Run: `go test ./providers/... -v` -Expected: All tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add providers/proxysql/ -git commit -m "feat: implement ProxySQL sandbox creation with config generation and lifecycle scripts" -``` - ---- - -### Task 4: Add `dbdeployer deploy proxysql` command - -**Files:** -- Create: `cmd/deploy_proxysql.go` - -A new subcommand that deploys a standalone ProxySQL sandbox using the system-installed binary. - -- [ ] **Step 1: Create `cmd/deploy_proxysql.go`** - -```go -package cmd - -// Adds a "dbdeployer deploy proxysql" command that: -// 1. Looks up the ProxySQL provider from the registry -// 2. Finds the proxysql binary on the system -// 3. Creates a sandbox directory in ~/sandboxes/proxysql_/ -// 4. Generates proxysql.cnf with admin/mysql ports -// 5. Writes lifecycle scripts (start, stop, status, use) -// 6. Optionally starts the sandbox -// -// Usage: dbdeployer deploy proxysql [--port=6032] [--admin-user=admin] [--admin-password=admin] -``` - -The command should use `providers.DefaultRegistry.Get("proxysql")` and call `CreateSandbox()`. - -- [ ] **Step 2: Verify** - -```bash -go build -o dbdeployer . -./dbdeployer deploy proxysql --port 6032 -ls ~/sandboxes/proxysql_6032/ -cat ~/sandboxes/proxysql_6032/proxysql.cnf -~/sandboxes/proxysql_6032/start -~/sandboxes/proxysql_6032/use -e "SELECT 1" -~/sandboxes/proxysql_6032/stop -``` - -- [ ] **Step 3: Commit** - -```bash -git add cmd/deploy_proxysql.go -git commit -m "feat: add 'dbdeployer deploy proxysql' command" -``` - ---- - -### Task 5: Add `--with-proxysql` flag to replication deployment - -**Files:** -- Modify: `cmd/replication.go` (add flag) -- Create: `sandbox/proxysql_topology.go` (topology wiring logic) - -This is the topology-aware deployment. When `--with-proxysql` is passed to `dbdeployer deploy replication`, after the MySQL replication sandbox is created, a ProxySQL sandbox is deployed and configured with the MySQL backends. - -- [ ] **Step 1: Create `sandbox/proxysql_topology.go`** - -Logic to wire ProxySQL to a MySQL replication sandbox: - -```go -package sandbox - -// DeployProxySQLForReplication creates a ProxySQL sandbox configured -// for a MySQL replication topology. -// -// Parameters: -// - replicationDir: path to the MySQL replication sandbox (e.g. ~/sandboxes/rsandbox_8_4_4) -// - masterPort: MySQL master port -// - slavePorts: MySQL slave ports -// - proxysqlPort: port for ProxySQL admin interface -// -// ProxySQL configuration: -// - Hostgroup 0: writer (master) -// - Hostgroup 1: readers (slaves) -// - Monitor user: msandbox/msandbox -// - No query rules (user configures) -``` - -- [ ] **Step 2: Add `--with-proxysql` flag to cmd/replication.go** - -Add a `--with-proxysql` boolean flag. When set, after the replication sandbox deploys successfully, call the topology wiring function to deploy ProxySQL alongside it. - -- [ ] **Step 3: Test end-to-end** - -```bash -go build -o dbdeployer . -./dbdeployer deploy replication 8.4.4 --sandbox-binary=$HOME/opt/mysql --with-proxysql -# Verify MySQL replication works -~/sandboxes/rsandbox_8_4_4/check_slaves -# Verify ProxySQL sandbox exists -ls ~/sandboxes/rsandbox_8_4_4/proxysql/ -# Verify ProxySQL is running and has backends -~/sandboxes/rsandbox_8_4_4/proxysql/use -e "SELECT * FROM mysql_servers" -# Connect through ProxySQL to MySQL -~/sandboxes/rsandbox_8_4_4/proxysql/use_proxy -e "SELECT @@hostname, @@port" -# Cleanup -./dbdeployer delete all --skip-confirm -``` - -- [ ] **Step 4: Commit** - -```bash -git add sandbox/proxysql_topology.go cmd/replication.go -git commit -m "feat: add --with-proxysql flag for topology-aware ProxySQL deployment" -``` - ---- - -### Task 6: Add `--with-proxysql` to single deployment - -**Files:** -- Modify: `cmd/single.go` (add flag) - -Simpler than replication — just one backend in hostgroup 0. - -- [ ] **Step 1: Add `--with-proxysql` flag to cmd/single.go** - -When set, deploy a ProxySQL sandbox alongside the single MySQL sandbox with one backend. - -- [ ] **Step 2: Test** - -```bash -./dbdeployer deploy single 8.4.4 --sandbox-binary=$HOME/opt/mysql --with-proxysql -~/sandboxes/msb_8_4_4/proxysql/use -e "SELECT * FROM mysql_servers" -./dbdeployer delete all --skip-confirm -``` - -- [ ] **Step 3: Commit** - -```bash -git add cmd/single.go -git commit -m "feat: add --with-proxysql flag for single sandbox deployment" -``` - ---- - -### Task 7: Update sandbox deletion to handle ProxySQL - -**Files:** -- Modify: `cmd/delete.go` or sandbox deletion logic - -Ensure `dbdeployer delete` properly stops and removes ProxySQL sandboxes alongside MySQL ones. - -- [ ] **Step 1: Update deletion to check for ProxySQL sub-sandbox** - -When deleting a sandbox that has a `proxysql/` subdirectory, run `proxysql/stop` first. - -- [ ] **Step 2: Test** - -```bash -./dbdeployer deploy replication 8.4.4 --sandbox-binary=$HOME/opt/mysql --with-proxysql -./dbdeployer delete all --skip-confirm -# Verify no stale proxysql processes -ps aux | grep proxysql | grep -v grep -``` - -- [ ] **Step 3: Commit** - -```bash -git add cmd/delete.go -git commit -m "feat: handle ProxySQL cleanup during sandbox deletion" -``` - ---- - -### Task 8: Final validation and documentation - -- [ ] **Step 1: Run all unit tests** - -```bash -go test ./providers/... ./cmd/... ./common/... -timeout 30m -``` - -- [ ] **Step 2: Full integration test** - -```bash -# Standalone ProxySQL -./dbdeployer deploy proxysql --port 16032 -./dbdeployer delete all --skip-confirm - -# Single MySQL + ProxySQL -./dbdeployer deploy single 8.4.4 --sandbox-binary=$HOME/opt/mysql --with-proxysql -./dbdeployer delete all --skip-confirm - -# Replication + ProxySQL -./dbdeployer deploy replication 9.1.0 --sandbox-binary=$HOME/opt/mysql --with-proxysql -~/sandboxes/rsandbox_9_1_0/proxysql/use -e "SELECT * FROM mysql_servers" -~/sandboxes/rsandbox_9_1_0/check_slaves -./dbdeployer delete all --skip-confirm -``` - -- [ ] **Step 3: Verify `dbdeployer providers` shows both** - -```bash -./dbdeployer providers -``` -Expected: -``` -mysql (base port: 3306, ports per instance: 3) -proxysql (base port: 6032, ports per instance: 2) -``` - -- [ ] **Step 4: Update README with ProxySQL usage examples** - ---- - -## What Phase 2b Does NOT Do (Deferred) - -- No Orchestrator provider (separate Phase 2c) -- No tarball management for ProxySQL (no tarballs distributed yet) -- No query rules in generated config (users configure manually) -- No `--with-proxysql` for group replication (can be added incrementally) -- No ProxySQL version detection from system binary (uses user-specified or "system") diff --git a/docs/superpowers/plans/2026-03-24-phase3-postgresql-provider.md b/docs/superpowers/plans/2026-03-24-phase3-postgresql-provider.md deleted file mode 100644 index bc4996ad..00000000 --- a/docs/superpowers/plans/2026-03-24-phase3-postgresql-provider.md +++ /dev/null @@ -1,2389 +0,0 @@ -# Phase 3 — PostgreSQL Provider Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a PostgreSQL provider to dbdeployer supporting single sandbox, streaming replication, cross-database topology constraints, and ProxySQL+PostgreSQL backend wiring. - -**Architecture:** Extend the Provider interface with `SupportedTopologies()` and `CreateReplica()`. Implement a PostgreSQL provider that uses `initdb`/`pg_ctl`/`pg_basebackup` for sandbox lifecycle. Add deb extraction for binary management. Wire into existing cmd layer via `--provider` flag. Extend ProxySQL config generator for PostgreSQL backends. - -**Tech Stack:** Go, PostgreSQL CLI tools (initdb, pg_ctl, pg_basebackup, psql), dpkg-deb - -**Spec:** `docs/superpowers/specs/2026-03-24-phase3-postgresql-provider-design.md` - ---- - -## File Structure - -### New Files -- `providers/postgresql/postgresql.go` — Provider struct, registration, Name/ValidateVersion/DefaultPorts/FindBinary/StartSandbox/StopSandbox/SupportedTopologies/CreateReplica -- `providers/postgresql/sandbox.go` — CreateSandbox implementation (initdb, config gen, script gen) -- `providers/postgresql/config.go` — postgresql.conf and pg_hba.conf generation functions -- `providers/postgresql/scripts.go` — lifecycle script generation (start, stop, status, restart, use, clear) -- `providers/postgresql/unpack.go` — deb extraction logic -- `providers/postgresql/postgresql_test.go` — unit tests for provider methods -- `providers/postgresql/config_test.go` — unit tests for config generation -- `providers/postgresql/unpack_test.go` — unit tests for deb extraction -- `providers/postgresql/integration_test.go` — integration tests (build-tagged) -- `cmd/deploy_postgresql.go` — `dbdeployer deploy postgresql ` standalone command - -### Modified Files -- `providers/provider.go` — add `SupportedTopologies()`, `CreateReplica()`, `ErrNotSupported` -- `providers/provider_test.go` — update mock, add topology/validation tests -- `providers/mysql/mysql.go` — implement new interface methods -- `providers/proxysql/proxysql.go` — implement new interface methods -- `providers/proxysql/config.go` — add PostgreSQL backend config generation -- `providers/proxysql/proxysql_test.go` — update for new interface methods -- `providers/proxysql/config_test.go` — test PostgreSQL backend config -- `sandbox/proxysql_topology.go` — accept `backendProvider` parameter -- `cmd/root.go` — register PostgreSQL provider -- `cmd/single.go` — add `--provider` flag, route to provider -- `cmd/multiple.go` — add `--provider` flag, route to provider -- `cmd/replication.go` — add `--provider` flag, PostgreSQL replication flow -- `cmd/unpack.go` — add `--provider` flag for deb extraction -- `globals/globals.go` — PostgreSQL constants - ---- - -## Task 1: Extend Provider Interface - -**Files:** -- Modify: `providers/provider.go` -- Modify: `providers/provider_test.go` -- Modify: `providers/mysql/mysql.go` -- Modify: `providers/proxysql/proxysql.go` -- Modify: `providers/proxysql/proxysql_test.go` - -- [ ] **Step 1: Write failing test for SupportedTopologies on mock provider** - -In `providers/provider_test.go`, add `SupportedTopologies` and `CreateReplica` to `mockProvider`, then write a test: - -```go -func (m *mockProvider) SupportedTopologies() []string { - return []string{"single", "multiple"} -} -func (m *mockProvider) CreateReplica(primary SandboxInfo, config SandboxConfig) (*SandboxInfo, error) { - return nil, ErrNotSupported -} - -func TestErrNotSupported(t *testing.T) { - mock := &mockProvider{name: "test"} - _, err := mock.CreateReplica(SandboxInfo{}, SandboxConfig{}) - if err != ErrNotSupported { - t.Errorf("expected ErrNotSupported, got %v", err) - } -} - -func TestSupportedTopologies(t *testing.T) { - mock := &mockProvider{name: "test"} - topos := mock.SupportedTopologies() - if len(topos) != 2 || topos[0] != "single" { - t.Errorf("unexpected topologies: %v", topos) - } -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cd /data/rene/dbdeployer && go test ./providers/ -run TestErrNotSupported -v` -Expected: Compilation error — `ErrNotSupported` and `SupportedTopologies` not defined on interface. - -- [ ] **Step 3: Add interface methods and ErrNotSupported to provider.go** - -In `providers/provider.go`, add: - -```go -import ( - "errors" - "fmt" - "sort" -) - -var ErrNotSupported = errors.New("operation not supported by this provider") - -type Provider interface { - Name() string - ValidateVersion(version string) error - DefaultPorts() PortRange - FindBinary(version string) (string, error) - CreateSandbox(config SandboxConfig) (*SandboxInfo, error) - StartSandbox(dir string) error - StopSandbox(dir string) error - SupportedTopologies() []string - CreateReplica(primary SandboxInfo, config SandboxConfig) (*SandboxInfo, error) -} -``` - -- [ ] **Step 4: Update MySQLProvider to implement new methods** - -In `providers/mysql/mysql.go`, add: - -```go -func (p *MySQLProvider) SupportedTopologies() []string { - return []string{"single", "multiple", "replication", "group", "fan-in", "all-masters", "ndb", "pxc"} -} - -func (p *MySQLProvider) CreateReplica(primary providers.SandboxInfo, config providers.SandboxConfig) (*providers.SandboxInfo, error) { - return nil, providers.ErrNotSupported -} -``` - -- [ ] **Step 5: Update ProxySQLProvider to implement new methods** - -In `providers/proxysql/proxysql.go`, add: - -```go -func (p *ProxySQLProvider) SupportedTopologies() []string { - return []string{"single"} -} - -func (p *ProxySQLProvider) CreateReplica(primary providers.SandboxInfo, config providers.SandboxConfig) (*providers.SandboxInfo, error) { - return nil, providers.ErrNotSupported -} -``` - -- [ ] **Step 6: Run all provider tests to verify they pass** - -Run: `cd /data/rene/dbdeployer && go test ./providers/... -v` -Expected: All tests pass, including the new ones and existing ProxySQL tests. - -- [ ] **Step 7: Commit** - -```bash -git add providers/provider.go providers/provider_test.go providers/mysql/mysql.go providers/proxysql/proxysql.go -git commit -m "feat: extend Provider interface with SupportedTopologies and CreateReplica" -``` - ---- - -## Task 2: PostgreSQL Provider — Core Structure and Version Validation - -**Files:** -- Create: `providers/postgresql/postgresql.go` -- Create: `providers/postgresql/postgresql_test.go` - -- [ ] **Step 1: Write failing tests for PostgreSQL provider basics** - -Create `providers/postgresql/postgresql_test.go`: - -```go -package postgresql - -import ( - "testing" - - "github.com/ProxySQL/dbdeployer/providers" -) - -func TestPostgreSQLProviderName(t *testing.T) { - p := NewPostgreSQLProvider() - if p.Name() != "postgresql" { - t.Errorf("expected 'postgresql', got %q", p.Name()) - } -} - -func TestPostgreSQLProviderValidateVersion(t *testing.T) { - p := NewPostgreSQLProvider() - tests := []struct { - version string - wantErr bool - }{ - {"16.13", false}, - {"17.1", false}, - {"12.0", false}, - {"11.5", true}, // major < 12 - {"16", true}, // missing minor - {"16.13.1", true}, // three parts - {"abc", true}, - {"", true}, - } - for _, tt := range tests { - err := p.ValidateVersion(tt.version) - if (err != nil) != tt.wantErr { - t.Errorf("ValidateVersion(%q) error = %v, wantErr %v", tt.version, err, tt.wantErr) - } - } -} - -func TestPostgreSQLProviderDefaultPorts(t *testing.T) { - p := NewPostgreSQLProvider() - ports := p.DefaultPorts() - if ports.BasePort != 15000 { - t.Errorf("expected BasePort 15000, got %d", ports.BasePort) - } - if ports.PortsPerInstance != 1 { - t.Errorf("expected PortsPerInstance 1, got %d", ports.PortsPerInstance) - } -} - -func TestPostgreSQLProviderSupportedTopologies(t *testing.T) { - p := NewPostgreSQLProvider() - topos := p.SupportedTopologies() - expected := map[string]bool{"single": true, "multiple": true, "replication": true} - if len(topos) != len(expected) { - t.Fatalf("expected %d topologies, got %d: %v", len(expected), len(topos), topos) - } - for _, topo := range topos { - if !expected[topo] { - t.Errorf("unexpected topology %q", topo) - } - } -} - -func TestPostgreSQLVersionToPort(t *testing.T) { - tests := []struct { - version string - expected int - }{ - {"16.13", 16613}, - {"16.3", 16603}, - {"17.1", 16701}, - {"17.10", 16710}, - {"12.0", 16200}, - } - for _, tt := range tests { - port, err := VersionToPort(tt.version) - if err != nil { - t.Errorf("VersionToPort(%q) unexpected error: %v", tt.version, err) - continue - } - if port != tt.expected { - t.Errorf("VersionToPort(%q) = %d, want %d", tt.version, port, tt.expected) - } - } -} - -func TestPostgreSQLProviderRegister(t *testing.T) { - reg := providers.NewRegistry() - if err := Register(reg); err != nil { - t.Fatalf("Register failed: %v", err) - } - p, err := reg.Get("postgresql") - if err != nil { - t.Fatalf("Get failed: %v", err) - } - if p.Name() != "postgresql" { - t.Errorf("expected 'postgresql', got %q", p.Name()) - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -v` -Expected: Compilation error — package doesn't exist. - -- [ ] **Step 3: Implement PostgreSQL provider core** - -Create `providers/postgresql/postgresql.go`: - -```go -package postgresql - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - - "github.com/ProxySQL/dbdeployer/providers" -) - -const ProviderName = "postgresql" - -type PostgreSQLProvider struct{} - -func NewPostgreSQLProvider() *PostgreSQLProvider { return &PostgreSQLProvider{} } - -func (p *PostgreSQLProvider) Name() string { return ProviderName } - -func (p *PostgreSQLProvider) ValidateVersion(version string) error { - parts := strings.Split(version, ".") - if len(parts) != 2 { - return fmt.Errorf("invalid PostgreSQL version format: %q (expected major.minor, e.g. 16.13)", version) - } - major, err := strconv.Atoi(parts[0]) - if err != nil { - return fmt.Errorf("invalid PostgreSQL major version %q: %w", parts[0], err) - } - if major < 12 { - return fmt.Errorf("PostgreSQL major version must be >= 12, got %d", major) - } - if _, err := strconv.Atoi(parts[1]); err != nil { - return fmt.Errorf("invalid PostgreSQL minor version %q: %w", parts[1], err) - } - return nil -} - -func (p *PostgreSQLProvider) DefaultPorts() providers.PortRange { - return providers.PortRange{BasePort: 15000, PortsPerInstance: 1} -} - -func (p *PostgreSQLProvider) SupportedTopologies() []string { - return []string{"single", "multiple", "replication"} -} - -// VersionToPort converts a PostgreSQL version to a port number. -// Formula: BasePort + major*100 + minor -// Example: 16.13 -> 15000 + 1600 + 13 = 16613 -func VersionToPort(version string) (int, error) { - parts := strings.Split(version, ".") - if len(parts) != 2 { - return 0, fmt.Errorf("invalid version format: %q", version) - } - major, err := strconv.Atoi(parts[0]) - if err != nil { - return 0, err - } - minor, err := strconv.Atoi(parts[1]) - if err != nil { - return 0, err - } - return 15000 + major*100 + minor, nil -} - -// FindBinary returns the path to the postgres binary for the given version. -// Looks in ~/opt/postgresql//bin/postgres by default. -func (p *PostgreSQLProvider) FindBinary(version string) (string, error) { - home, err := os.UserHomeDir() - if err != nil { - return "", fmt.Errorf("cannot determine home directory: %w", err) - } - binPath := filepath.Join(home, "opt", "postgresql", version, "bin", "postgres") - if _, err := os.Stat(binPath); err != nil { - return "", fmt.Errorf("PostgreSQL binary not found at %s: %w", binPath, err) - } - return binPath, nil -} - -// basedirFromVersion returns the base directory for a PostgreSQL version. -func basedirFromVersion(version string) (string, error) { - home, err := os.UserHomeDir() - if err != nil { - return "", fmt.Errorf("cannot determine home directory: %w", err) - } - return filepath.Join(home, "opt", "postgresql", version), nil -} - -func (p *PostgreSQLProvider) StartSandbox(dir string) error { - cmd := exec.Command("bash", filepath.Join(dir, "start")) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("start failed: %s: %w", string(output), err) - } - return nil -} - -func (p *PostgreSQLProvider) StopSandbox(dir string) error { - cmd := exec.Command("bash", filepath.Join(dir, "stop")) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("stop failed: %s: %w", string(output), err) - } - return nil -} - -func Register(reg *providers.Registry) error { - return reg.Register(NewPostgreSQLProvider()) -} -``` - -Note: `CreateSandbox` and `CreateReplica` are implemented in Task 4 and Task 6 respectively, in separate files. Add stubs for now: - -```go -func (p *PostgreSQLProvider) CreateSandbox(config providers.SandboxConfig) (*providers.SandboxInfo, error) { - return nil, fmt.Errorf("PostgreSQLProvider.CreateSandbox: not yet implemented") -} - -func (p *PostgreSQLProvider) CreateReplica(primary providers.SandboxInfo, config providers.SandboxConfig) (*providers.SandboxInfo, error) { - return nil, fmt.Errorf("PostgreSQLProvider.CreateReplica: not yet implemented") -} -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -v` -Expected: All tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add providers/postgresql/postgresql.go providers/postgresql/postgresql_test.go -git commit -m "feat: add PostgreSQL provider core structure and version validation" -``` - ---- - -## Task 3: PostgreSQL Config Generation - -**Files:** -- Create: `providers/postgresql/config.go` -- Create: `providers/postgresql/config_test.go` - -- [ ] **Step 1: Write failing tests for config generation** - -Create `providers/postgresql/config_test.go`: - -```go -package postgresql - -import ( - "strings" - "testing" -) - -func TestGeneratePostgresqlConf(t *testing.T) { - conf := GeneratePostgresqlConf(PostgresqlConfOptions{ - Port: 5433, - ListenAddresses: "127.0.0.1", - UnixSocketDir: "/tmp/sandbox/data", - LogDir: "/tmp/sandbox/data/log", - Replication: false, - }) - if !strings.Contains(conf, "port = 5433") { - t.Error("missing port setting") - } - if !strings.Contains(conf, "listen_addresses = '127.0.0.1'") { - t.Error("missing listen_addresses") - } - if !strings.Contains(conf, "unix_socket_directories = '/tmp/sandbox/data'") { - t.Error("missing unix_socket_directories") - } - if !strings.Contains(conf, "logging_collector = on") { - t.Error("missing logging_collector") - } - if strings.Contains(conf, "wal_level") { - t.Error("should not contain wal_level when replication is false") - } -} - -func TestGeneratePostgresqlConfWithReplication(t *testing.T) { - conf := GeneratePostgresqlConf(PostgresqlConfOptions{ - Port: 5433, - ListenAddresses: "127.0.0.1", - UnixSocketDir: "/tmp/sandbox/data", - LogDir: "/tmp/sandbox/data/log", - Replication: true, - }) - if !strings.Contains(conf, "wal_level = replica") { - t.Error("missing wal_level = replica") - } - if !strings.Contains(conf, "max_wal_senders = 10") { - t.Error("missing max_wal_senders") - } - if !strings.Contains(conf, "hot_standby = on") { - t.Error("missing hot_standby") - } -} - -func TestGeneratePgHbaConf(t *testing.T) { - conf := GeneratePgHbaConf(false) - if !strings.Contains(conf, "local all") { - t.Error("missing local all entry") - } - if !strings.Contains(conf, "host all") { - t.Error("missing host all entry") - } - if strings.Contains(conf, "replication") { - t.Error("should not contain replication when replication is false") - } -} - -func TestGeneratePgHbaConfWithReplication(t *testing.T) { - conf := GeneratePgHbaConf(true) - if !strings.Contains(conf, "host replication") { - t.Error("missing replication entry") - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -run TestGenerate -v` -Expected: Compilation error — functions not defined. - -- [ ] **Step 3: Implement config generation** - -Create `providers/postgresql/config.go`: - -```go -package postgresql - -import ( - "fmt" - "strings" -) - -type PostgresqlConfOptions struct { - Port int - ListenAddresses string - UnixSocketDir string - LogDir string - Replication bool -} - -func GeneratePostgresqlConf(opts PostgresqlConfOptions) string { - var b strings.Builder - b.WriteString(fmt.Sprintf("port = %d\n", opts.Port)) - b.WriteString(fmt.Sprintf("listen_addresses = '%s'\n", opts.ListenAddresses)) - b.WriteString(fmt.Sprintf("unix_socket_directories = '%s'\n", opts.UnixSocketDir)) - b.WriteString("logging_collector = on\n") - b.WriteString(fmt.Sprintf("log_directory = '%s'\n", opts.LogDir)) - - if opts.Replication { - b.WriteString("\n# Replication settings\n") - b.WriteString("wal_level = replica\n") - b.WriteString("max_wal_senders = 10\n") - b.WriteString("hot_standby = on\n") - } - - return b.String() -} - -func GeneratePgHbaConf(replication bool) string { - var b strings.Builder - b.WriteString("# TYPE DATABASE USER ADDRESS METHOD\n") - b.WriteString("local all all trust\n") - b.WriteString("host all all 127.0.0.1/32 trust\n") - b.WriteString("host all all ::1/128 trust\n") - - if replication { - b.WriteString("host replication all 127.0.0.1/32 trust\n") - } - - return b.String() -} -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -run TestGenerate -v` -Expected: All pass. - -- [ ] **Step 5: Commit** - -```bash -git add providers/postgresql/config.go providers/postgresql/config_test.go -git commit -m "feat: add PostgreSQL config generation (postgresql.conf, pg_hba.conf)" -``` - ---- - -## Task 4: PostgreSQL Script Generation and CreateSandbox - -**Files:** -- Create: `providers/postgresql/scripts.go` -- Create: `providers/postgresql/sandbox.go` -- Modify: `providers/postgresql/postgresql.go` (replace CreateSandbox stub) -- Modify: `providers/postgresql/postgresql_test.go` (add script tests) - -- [ ] **Step 1: Write failing tests for script generation** - -Add to `providers/postgresql/postgresql_test.go`: - -```go -func TestGenerateScripts(t *testing.T) { - opts := ScriptOptions{ - SandboxDir: "/tmp/pg_sandbox", - DataDir: "/tmp/pg_sandbox/data", - BinDir: "/opt/postgresql/16.13/bin", - LibDir: "/opt/postgresql/16.13/lib", - Port: 16613, - LogFile: "/tmp/pg_sandbox/postgresql.log", - } - scripts := GenerateScripts(opts) - - // Verify all expected scripts exist - expectedScripts := []string{"start", "stop", "status", "restart", "use", "clear"} - for _, name := range expectedScripts { - if _, ok := scripts[name]; !ok { - t.Errorf("missing script %q", name) - } - } - - // Verify start script contents - start := scripts["start"] - if !strings.Contains(start, "pg_ctl") { - t.Error("start script missing pg_ctl") - } - if !strings.Contains(start, "LD_LIBRARY_PATH") { - t.Error("start script missing LD_LIBRARY_PATH") - } - if !strings.Contains(start, "unset PGDATA") { - t.Error("start script missing PGDATA unset") - } - - // Verify use script - use := scripts["use"] - if !strings.Contains(use, "psql") { - t.Error("use script missing psql") - } - if !strings.Contains(use, "16613") { - t.Error("use script missing port") - } -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -run TestGenerateScripts -v` -Expected: Compilation error — `ScriptOptions` and `GenerateScripts` not defined. - -- [ ] **Step 3: Implement script generation** - -Create `providers/postgresql/scripts.go`: - -```go -package postgresql - -import "fmt" - -type ScriptOptions struct { - SandboxDir string - DataDir string - BinDir string - LibDir string - Port int - LogFile string -} - -const envPreamble = `#!/bin/bash -export LD_LIBRARY_PATH="%s" -unset PGDATA PGPORT PGHOST PGUSER PGDATABASE -` - -func GenerateScripts(opts ScriptOptions) map[string]string { - preamble := fmt.Sprintf(envPreamble, opts.LibDir) - - return map[string]string{ - "start": fmt.Sprintf("%s%s/pg_ctl -D %s -l %s start\n", - preamble, opts.BinDir, opts.DataDir, opts.LogFile), - - "stop": fmt.Sprintf("%s%s/pg_ctl -D %s stop -m fast\n", - preamble, opts.BinDir, opts.DataDir), - - "status": fmt.Sprintf("%s%s/pg_ctl -D %s status\n", - preamble, opts.BinDir, opts.DataDir), - - "restart": fmt.Sprintf("%s%s/pg_ctl -D %s -l %s restart\n", - preamble, opts.BinDir, opts.DataDir, opts.LogFile), - - "use": fmt.Sprintf("%s%s/psql -h 127.0.0.1 -p %d -U postgres \"$@\"\n", - preamble, opts.BinDir, opts.Port), - - "clear": fmt.Sprintf("%s%s/pg_ctl -D %s stop -m fast 2>/dev/null\nrm -rf %s\n%s/initdb -D %s --auth=trust --username=postgres\necho \"Sandbox cleared.\"\n", - preamble, opts.BinDir, opts.DataDir, opts.DataDir, opts.BinDir, opts.DataDir), - } -} -``` - -- [ ] **Step 4: Run script generation tests to verify they pass** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -run TestGenerateScripts -v` -Expected: PASS. - -- [ ] **Step 5: Implement CreateSandbox** - -Create `providers/postgresql/sandbox.go`: - -```go -package postgresql - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - - "github.com/ProxySQL/dbdeployer/providers" -) - -func (p *PostgreSQLProvider) CreateSandbox(config providers.SandboxConfig) (*providers.SandboxInfo, error) { - basedir, err := p.resolveBasedir(config) - if err != nil { - return nil, err - } - binDir := filepath.Join(basedir, "bin") - libDir := filepath.Join(basedir, "lib") - dataDir := filepath.Join(config.Dir, "data") - logDir := filepath.Join(dataDir, "log") - logFile := filepath.Join(config.Dir, "postgresql.log") - - replication := config.Options["replication"] == "true" - - // Create log directory - if err := os.MkdirAll(logDir, 0755); err != nil { - return nil, fmt.Errorf("creating log directory: %w", err) - } - - // Run initdb - initdbPath := filepath.Join(binDir, "initdb") - initCmd := exec.Command(initdbPath, "-D", dataDir, "--auth=trust", "--username=postgres") - initCmd.Env = append(os.Environ(), fmt.Sprintf("LD_LIBRARY_PATH=%s", libDir)) - if output, err := initCmd.CombinedOutput(); err != nil { - os.RemoveAll(config.Dir) // cleanup on failure - return nil, fmt.Errorf("initdb failed: %s: %w", string(output), err) - } - - // Generate and write postgresql.conf - pgConf := GeneratePostgresqlConf(PostgresqlConfOptions{ - Port: config.Port, - ListenAddresses: "127.0.0.1", - UnixSocketDir: dataDir, - LogDir: logDir, - Replication: replication, - }) - confPath := filepath.Join(dataDir, "postgresql.conf") - if err := os.WriteFile(confPath, []byte(pgConf), 0644); err != nil { - os.RemoveAll(config.Dir) - return nil, fmt.Errorf("writing postgresql.conf: %w", err) - } - - // Generate and write pg_hba.conf - hbaConf := GeneratePgHbaConf(replication) - hbaPath := filepath.Join(dataDir, "pg_hba.conf") - if err := os.WriteFile(hbaPath, []byte(hbaConf), 0644); err != nil { - os.RemoveAll(config.Dir) - return nil, fmt.Errorf("writing pg_hba.conf: %w", err) - } - - // Generate and write lifecycle scripts - scripts := GenerateScripts(ScriptOptions{ - SandboxDir: config.Dir, - DataDir: dataDir, - BinDir: binDir, - LibDir: libDir, - Port: config.Port, - LogFile: logFile, - }) - for name, content := range scripts { - scriptPath := filepath.Join(config.Dir, name) - if err := os.WriteFile(scriptPath, []byte(content), 0755); err != nil { - os.RemoveAll(config.Dir) - return nil, fmt.Errorf("writing script %s: %w", name, err) - } - } - - return &providers.SandboxInfo{ - Dir: config.Dir, - Port: config.Port, - Status: "stopped", - }, nil -} - -// resolveBasedir determines the PostgreSQL base directory. -// Uses config.Options["basedir"] if set, otherwise ~/opt/postgresql/. -func (p *PostgreSQLProvider) resolveBasedir(config providers.SandboxConfig) (string, error) { - if bd, ok := config.Options["basedir"]; ok && bd != "" { - return bd, nil - } - return basedirFromVersion(config.Version) -} -``` - -- [ ] **Step 6: Remove the CreateSandbox stub from postgresql.go** - -In `providers/postgresql/postgresql.go`, remove the stub `CreateSandbox` method (now implemented in sandbox.go). - -- [ ] **Step 7: Run all PostgreSQL provider tests** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -v` -Expected: All pass. - -- [ ] **Step 8: Commit** - -```bash -git add providers/postgresql/scripts.go providers/postgresql/sandbox.go providers/postgresql/postgresql.go providers/postgresql/postgresql_test.go -git commit -m "feat: implement PostgreSQL CreateSandbox with initdb, config gen, and lifecycle scripts" -``` - ---- - -## Task 5: Deb Extraction for PostgreSQL Binaries - -**Files:** -- Create: `providers/postgresql/unpack.go` -- Create: `providers/postgresql/unpack_test.go` - -- [ ] **Step 1: Write failing tests for deb filename parsing and validation** - -Create `providers/postgresql/unpack_test.go`: - -```go -package postgresql - -import "testing" - -func TestParseDebVersion(t *testing.T) { - tests := []struct { - filename string - wantVer string - wantErr bool - }{ - {"postgresql-16_16.13-0ubuntu0.24.04.1_amd64.deb", "16.13", false}, - {"postgresql-17_17.2-1_amd64.deb", "17.2", false}, - {"postgresql-client-16_16.13-0ubuntu0.24.04.1_amd64.deb", "16.13", false}, - {"random-file.tar.gz", "", true}, - {"postgresql-16_bad-version.deb", "", true}, - } - for _, tt := range tests { - ver, err := ParseDebVersion(tt.filename) - if (err != nil) != tt.wantErr { - t.Errorf("ParseDebVersion(%q) error = %v, wantErr %v", tt.filename, err, tt.wantErr) - continue - } - if ver != tt.wantVer { - t.Errorf("ParseDebVersion(%q) = %q, want %q", tt.filename, ver, tt.wantVer) - } - } -} - -func TestClassifyDebs(t *testing.T) { - files := []string{ - "postgresql-16_16.13-0ubuntu0.24.04.1_amd64.deb", - "postgresql-client-16_16.13-0ubuntu0.24.04.1_amd64.deb", - } - server, client, err := ClassifyDebs(files) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if server != files[0] { - t.Errorf("server = %q, want %q", server, files[0]) - } - if client != files[1] { - t.Errorf("client = %q, want %q", client, files[1]) - } -} - -func TestClassifyDebsMissingClient(t *testing.T) { - files := []string{"postgresql-16_16.13-0ubuntu0.24.04.1_amd64.deb"} - _, _, err := ClassifyDebs(files) - if err == nil { - t.Error("expected error for missing client deb") - } -} - -func TestRequiredBinaries(t *testing.T) { - expected := []string{"postgres", "initdb", "pg_ctl", "psql", "pg_basebackup"} - got := RequiredBinaries() - if len(got) != len(expected) { - t.Fatalf("expected %d binaries, got %d", len(expected), len(got)) - } - for i, name := range expected { - if got[i] != name { - t.Errorf("binary[%d] = %q, want %q", i, got[i], name) - } - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -run "TestParseDeb|TestClassify|TestRequired" -v` -Expected: Compilation error — functions not defined. - -- [ ] **Step 3: Implement deb extraction logic** - -Create `providers/postgresql/unpack.go`: - -```go -package postgresql - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" -) - -var debVersionRegex = regexp.MustCompile(`^postgresql(?:-client)?-(\d+)_(\d+\.\d+)`) - -// ParseDebVersion extracts the PostgreSQL version from a deb filename. -func ParseDebVersion(filename string) (string, error) { - base := filepath.Base(filename) - matches := debVersionRegex.FindStringSubmatch(base) - if matches == nil { - return "", fmt.Errorf("cannot parse PostgreSQL version from %q (expected postgresql[-client]-NN_X.Y-*)", base) - } - return matches[2], nil -} - -// ClassifyDebs identifies server and client debs from a list of filenames. -func ClassifyDebs(files []string) (server, client string, err error) { - for _, f := range files { - base := filepath.Base(f) - if strings.HasPrefix(base, "postgresql-client-") { - client = f - } else if strings.HasPrefix(base, "postgresql-") && strings.HasSuffix(base, ".deb") { - server = f - } - } - if server == "" { - return "", "", fmt.Errorf("no server deb found (expected postgresql-NN_*.deb)") - } - if client == "" { - return "", "", fmt.Errorf("no client deb found (expected postgresql-client-NN_*.deb)") - } - return server, client, nil -} - -// RequiredBinaries returns the binaries that must exist after extraction. -func RequiredBinaries() []string { - return []string{"postgres", "initdb", "pg_ctl", "psql", "pg_basebackup"} -} - -// UnpackDebs extracts PostgreSQL server and client debs into the target directory. -// targetDir is the final layout dir, e.g. ~/opt/postgresql/16.13/ -func UnpackDebs(serverDeb, clientDeb, targetDir string) error { - tmpDir, err := os.MkdirTemp("", "dbdeployer-pg-unpack-*") - if err != nil { - return fmt.Errorf("creating temp directory: %w", err) - } - defer os.RemoveAll(tmpDir) - - // Extract both debs - for _, deb := range []string{serverDeb, clientDeb} { - cmd := exec.Command("dpkg-deb", "-x", deb, tmpDir) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("extracting %s: %s: %w", filepath.Base(deb), string(output), err) - } - } - - // Determine the major version directory inside the extracted tree - version, err := ParseDebVersion(serverDeb) - if err != nil { - return err - } - major := strings.Split(version, ".")[0] - - // Source paths within extracted debs - srcBin := filepath.Join(tmpDir, "usr", "lib", "postgresql", major, "bin") - srcLib := filepath.Join(tmpDir, "usr", "lib", "postgresql", major, "lib") - srcShare := filepath.Join(tmpDir, "usr", "share", "postgresql", major) - - // Create target directories - dstBin := filepath.Join(targetDir, "bin") - dstLib := filepath.Join(targetDir, "lib") - dstShare := filepath.Join(targetDir, "share") - - for _, dir := range []string{dstBin, dstLib, dstShare} { - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("creating directory %s: %w", dir, err) - } - } - - // Copy files using cp -a to preserve permissions and symlinks - copies := []struct{ src, dst string }{ - {srcBin, dstBin}, - {srcLib, dstLib}, - {srcShare, dstShare}, - } - for _, c := range copies { - if _, err := os.Stat(c.src); os.IsNotExist(err) { - continue // some dirs may not exist in the client deb - } - cmd := exec.Command("cp", "-a", c.src+"/.", c.dst+"/") - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("copying %s to %s: %s: %w", c.src, c.dst, string(output), err) - } - } - - // Validate required binaries - for _, bin := range RequiredBinaries() { - binPath := filepath.Join(dstBin, bin) - if _, err := os.Stat(binPath); err != nil { - return fmt.Errorf("required binary %q not found at %s after extraction", bin, binPath) - } - } - - return nil -} -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -run "TestParseDeb|TestClassify|TestRequired" -v` -Expected: All pass. - -- [ ] **Step 5: Commit** - -```bash -git add providers/postgresql/unpack.go providers/postgresql/unpack_test.go -git commit -m "feat: add PostgreSQL deb extraction for binary management" -``` - ---- - -## Task 6: PostgreSQL Replication (CreateReplica) - -**Files:** -- Modify: `providers/postgresql/postgresql.go` (replace CreateReplica stub) -- Create: `providers/postgresql/replication.go` -- Modify: `providers/postgresql/postgresql_test.go` (add replication config tests) - -- [ ] **Step 1: Write failing tests for replication monitoring script generation** - -Add to `providers/postgresql/postgresql_test.go`: - -```go -func TestGenerateCheckReplicationScript(t *testing.T) { - script := GenerateCheckReplicationScript(ScriptOptions{ - BinDir: "/opt/postgresql/16.13/bin", - LibDir: "/opt/postgresql/16.13/lib", - Port: 16613, - }) - if !strings.Contains(script, "pg_stat_replication") { - t.Error("missing pg_stat_replication query") - } - if !strings.Contains(script, "16613") { - t.Error("missing primary port") - } -} - -func TestGenerateCheckRecoveryScript(t *testing.T) { - ports := []int{16614, 16615} - script := GenerateCheckRecoveryScript(ScriptOptions{ - BinDir: "/opt/postgresql/16.13/bin", - LibDir: "/opt/postgresql/16.13/lib", - }, ports) - if !strings.Contains(script, "pg_is_in_recovery") { - t.Error("missing pg_is_in_recovery query") - } - if !strings.Contains(script, "16614") || !strings.Contains(script, "16615") { - t.Error("missing replica ports") - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -run "TestGenerateCheck" -v` -Expected: Compilation error — functions not defined. - -- [ ] **Step 3: Add monitoring script generators to scripts.go** - -Add to `providers/postgresql/scripts.go`: - -```go -func GenerateCheckReplicationScript(opts ScriptOptions) string { - preamble := fmt.Sprintf(envPreamble, opts.LibDir) - return fmt.Sprintf(`%s%s/psql -h 127.0.0.1 -p %d -U postgres -c \ - "SELECT client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn FROM pg_stat_replication;" -`, preamble, opts.BinDir, opts.Port) -} - -func GenerateCheckRecoveryScript(opts ScriptOptions, replicaPorts []int) string { - preamble := fmt.Sprintf(envPreamble, opts.LibDir) - var b strings.Builder - b.WriteString(preamble) - for _, port := range replicaPorts { - b.WriteString(fmt.Sprintf("echo \"=== Replica port %d ===\"\n", port)) - b.WriteString(fmt.Sprintf("%s/psql -h 127.0.0.1 -p %d -U postgres -c \"SELECT pg_is_in_recovery();\"\n", opts.BinDir, port)) - } - return b.String() -} -``` - -Add `"strings"` to imports in `scripts.go`. - -- [ ] **Step 4: Implement CreateReplica** - -Create `providers/postgresql/replication.go`: - -```go -package postgresql - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/ProxySQL/dbdeployer/providers" -) - -func (p *PostgreSQLProvider) CreateReplica(primary providers.SandboxInfo, config providers.SandboxConfig) (*providers.SandboxInfo, error) { - basedir, err := p.resolveBasedir(config) - if err != nil { - return nil, err - } - binDir := filepath.Join(basedir, "bin") - libDir := filepath.Join(basedir, "lib") - dataDir := filepath.Join(config.Dir, "data") - logFile := filepath.Join(config.Dir, "postgresql.log") - - // pg_basebackup from the running primary - pgBasebackup := filepath.Join(binDir, "pg_basebackup") - bbCmd := exec.Command(pgBasebackup, - "-h", "127.0.0.1", - "-p", fmt.Sprintf("%d", primary.Port), - "-U", "postgres", - "-D", dataDir, - "-Fp", "-Xs", "-R", - ) - bbCmd.Env = append(os.Environ(), fmt.Sprintf("LD_LIBRARY_PATH=%s", libDir)) - if output, err := bbCmd.CombinedOutput(); err != nil { - os.RemoveAll(config.Dir) // cleanup on failure - return nil, fmt.Errorf("pg_basebackup failed: %s: %w", string(output), err) - } - - // Modify replica's postgresql.conf: update port and unix_socket_directories - confPath := filepath.Join(dataDir, "postgresql.conf") - confBytes, err := os.ReadFile(confPath) - if err != nil { - os.RemoveAll(config.Dir) - return nil, fmt.Errorf("reading postgresql.conf: %w", err) - } - - conf := string(confBytes) - // Replace port line - lines := strings.Split(conf, "\n") - var newLines []string - for _, line := range lines { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "port =") || strings.HasPrefix(trimmed, "port=") { - newLines = append(newLines, fmt.Sprintf("port = %d", config.Port)) - } else if strings.HasPrefix(trimmed, "unix_socket_directories =") || strings.HasPrefix(trimmed, "unix_socket_directories=") { - newLines = append(newLines, fmt.Sprintf("unix_socket_directories = '%s'", dataDir)) - } else { - newLines = append(newLines, line) - } - } - - if err := os.WriteFile(confPath, []byte(strings.Join(newLines, "\n")), 0644); err != nil { - os.RemoveAll(config.Dir) - return nil, fmt.Errorf("writing modified postgresql.conf: %w", err) - } - - // Write lifecycle scripts - scripts := GenerateScripts(ScriptOptions{ - SandboxDir: config.Dir, - DataDir: dataDir, - BinDir: binDir, - LibDir: libDir, - Port: config.Port, - LogFile: logFile, - }) - for name, content := range scripts { - scriptPath := filepath.Join(config.Dir, name) - if err := os.WriteFile(scriptPath, []byte(content), 0755); err != nil { - os.RemoveAll(config.Dir) - return nil, fmt.Errorf("writing script %s: %w", name, err) - } - } - - // Start the replica - if err := p.StartSandbox(config.Dir); err != nil { - os.RemoveAll(config.Dir) - return nil, fmt.Errorf("starting replica: %w", err) - } - - return &providers.SandboxInfo{ - Dir: config.Dir, - Port: config.Port, - Status: "running", - }, nil -} -``` - -- [ ] **Step 5: Remove CreateReplica stub from postgresql.go** - -In `providers/postgresql/postgresql.go`, remove the stub `CreateReplica` method. - -- [ ] **Step 6: Run tests to verify they pass** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -v` -Expected: All tests pass (unit tests; replication flow is integration-tested). - -- [ ] **Step 7: Commit** - -```bash -git add providers/postgresql/replication.go providers/postgresql/scripts.go providers/postgresql/postgresql.go providers/postgresql/postgresql_test.go -git commit -m "feat: implement PostgreSQL CreateReplica with pg_basebackup and monitoring scripts" -``` - ---- - -## Task 7: Register Provider and Add --provider Flag to Commands - -**Files:** -- Modify: `cmd/root.go` -- Modify: `cmd/single.go` -- Modify: `cmd/multiple.go` -- Modify: `cmd/replication.go` -- Modify: `globals/globals.go` -- Modify: `providers/provider.go` (add `ContainsString` helper) -- Modify: `sandbox/proxysql_topology.go` (add `backendProvider` parameter) - -**Note:** This task introduces `cmd/deploy_postgresql.go` (Task 11) and splits files not in the original spec (`sandbox.go`, `scripts.go`). These are intentional improvements for code organization and UX. - -- [ ] **Step 1: Add PostgreSQL constants and ContainsString helper to providers** - -In `globals/globals.go`, add near the existing constant blocks: - -```go -const ( - ProviderLabel = "provider" - ProviderValue = "mysql" // default provider -) -``` - -In `providers/provider.go`, add an exported helper: - -```go -// ContainsString checks if a string slice contains a given value. -func ContainsString(slice []string, s string) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} -``` - -- [ ] **Step 2: Register PostgreSQL provider in cmd/root.go** - -In `cmd/root.go`, add import for PostgreSQL provider and register it in `init()`: - -```go -import ( - // existing imports... - postgresqlprovider "github.com/ProxySQL/dbdeployer/providers/postgresql" -) - -// In init(), after proxysql registration: -_ = postgresqlprovider.Register(providers.DefaultRegistry) -``` - -- [ ] **Step 3: Update DeployProxySQLForTopology signature** - -In `sandbox/proxysql_topology.go`, add a `backendProvider` parameter. All callers must be updated: - -```go -func DeployProxySQLForTopology(sandboxDir string, masterPort int, slavePorts []int, proxysqlPort int, host string, backendProvider string) error { - // ... existing code unchanged until config building ... - config := providers.SandboxConfig{ - // ... existing fields ... - Options: map[string]string{ - "monitor_user": "msandbox", - "monitor_password": "msandbox", - "backends": strings.Join(backendParts, ","), - "backend_provider": backendProvider, // NEW: "" for mysql, "postgresql" for pg - }, - } - // ... rest unchanged ... -} -``` - -**Callers to update** (pass `""` to preserve existing MySQL behavior): -- `cmd/single.go:485` — `sandbox.DeployProxySQLForTopology(sandboxDir, masterPort, nil, 0, "127.0.0.1", "")` -- `cmd/replication.go:135` — `sandbox.DeployProxySQLForTopology(sandboxDir, masterPort, slavePorts, 0, "127.0.0.1", "")` - -- [ ] **Step 4: Update cmd/single.go — add --provider flag and routing** - -The key design decision: for non-MySQL providers, we **skip `fillSandboxDefinition` entirely** because it is deeply MySQL-specific (checks for MySQL directories, runs `common.CheckLibraries`, calls `getFlavor`, etc.). Instead, non-MySQL providers build a `providers.SandboxConfig` directly from CLI flags. - -Replace `singleSandbox()` with this structure: - -```go -func singleSandbox(cmd *cobra.Command, args []string) { - flags := cmd.Flags() - providerName, _ := flags.GetString(globals.ProviderLabel) - - // Non-MySQL providers: bypass fillSandboxDefinition entirely - if providerName != "mysql" { - deploySingleNonMySQL(cmd, args, providerName) - return - } - - // Existing MySQL path — completely unchanged - var sd sandbox.SandboxDef - var err error - common.CheckOrigin(args) - sd, err = fillSandboxDefinition(cmd, args, false) - // ... rest of existing code unchanged, BUT update DeployProxySQLForTopology call: - // sandbox.DeployProxySQLForTopology(sandboxDir, masterPort, nil, 0, "127.0.0.1", "") -} - -func deploySingleNonMySQL(cmd *cobra.Command, args []string, providerName string) { - flags := cmd.Flags() - version := args[0] - - p, err := providers.DefaultRegistry.Get(providerName) - if err != nil { - common.Exitf(1, "provider error: %s", err) - } - - // Flavor validation: --flavor is MySQL-only - flavor, _ := flags.GetString(globals.FlavorLabel) - if flavor != "" { - common.Exitf(1, "--flavor is only valid with --provider=mysql") - } - - // Topology validation - if !providers.ContainsString(p.SupportedTopologies(), "single") { - common.Exitf(1, "provider %q does not support topology \"single\"\nSupported topologies: %s", - providerName, strings.Join(p.SupportedTopologies(), ", ")) - } - - if err := p.ValidateVersion(version); err != nil { - common.Exitf(1, "version validation failed: %s", err) - } - - if _, err := p.FindBinary(version); err != nil { - common.Exitf(1, "binaries not found: %s", err) - } - - // Compute port from provider's default port range - portRange := p.DefaultPorts() - port := portRange.BasePort - // For PostgreSQL, use VersionToPort - if providerName == "postgresql" { - port, _ = postgresql.VersionToPort(version) - } - freePort, portErr := common.FindFreePort(port, []int{}, portRange.PortsPerInstance) - if portErr == nil { - port = freePort - } - - sandboxHome := defaults.Defaults().SandboxHome - sandboxDir := path.Join(sandboxHome, fmt.Sprintf("%s_sandbox_%d", providerName, port)) - if common.DirExists(sandboxDir) { - common.Exitf(1, "sandbox directory %s already exists", sandboxDir) - } - - skipStart, _ := flags.GetBool(globals.SkipStartLabel) - config := providers.SandboxConfig{ - Version: version, - Dir: sandboxDir, - Port: port, - Host: "127.0.0.1", - DbUser: "postgres", - Options: map[string]string{}, - } - - if _, err := p.CreateSandbox(config); err != nil { - common.Exitf(1, "error creating sandbox: %s", err) - } - - if !skipStart { - if err := p.StartSandbox(sandboxDir); err != nil { - common.Exitf(1, "error starting sandbox: %s", err) - } - } - - // Handle --with-proxysql - withProxySQL, _ := flags.GetBool("with-proxysql") - if withProxySQL { - if !providers.ContainsString(providers.CompatibleAddons["proxysql"], providerName) { - common.Exitf(1, "--with-proxysql is not compatible with provider %q", providerName) - } - err := sandbox.DeployProxySQLForTopology(sandboxDir, port, nil, 0, "127.0.0.1", providerName) - if err != nil { - common.Exitf(1, "ProxySQL deployment failed: %s", err) - } - } - - fmt.Printf("%s %s sandbox deployed in %s (port: %d)\n", providerName, version, sandboxDir, port) -} -``` - -Add flag in `init()`: - -```go -singleCmd.PersistentFlags().String(globals.ProviderLabel, globals.ProviderValue, "Database provider (mysql, postgresql)") -``` - -Add imports for `postgresql` and `providers` packages. - -- [ ] **Step 5: Update cmd/multiple.go — add --provider flag and routing** - -Same bypass pattern. For non-MySQL providers, create N instances with sequential ports: - -```go -func multipleSandbox(cmd *cobra.Command, args []string) { - flags := cmd.Flags() - providerName, _ := flags.GetString(globals.ProviderLabel) - - if providerName != "mysql" { - deployMultipleNonMySQL(cmd, args, providerName) - return - } - - // Existing MySQL path unchanged, no modification needed - // ... -} - -func deployMultipleNonMySQL(cmd *cobra.Command, args []string, providerName string) { - flags := cmd.Flags() - version := args[0] - nodes, _ := flags.GetInt(globals.NodesLabel) - - p, err := providers.DefaultRegistry.Get(providerName) - if err != nil { - common.Exitf(1, "provider error: %s", err) - } - - flavor, _ := flags.GetString(globals.FlavorLabel) - if flavor != "" { - common.Exitf(1, "--flavor is only valid with --provider=mysql") - } - - if !providers.ContainsString(p.SupportedTopologies(), "multiple") { - common.Exitf(1, "provider %q does not support topology \"multiple\"\nSupported topologies: %s", - providerName, strings.Join(p.SupportedTopologies(), ", ")) - } - - if err := p.ValidateVersion(version); err != nil { - common.Exitf(1, "version validation failed: %s", err) - } - - if _, err := p.FindBinary(version); err != nil { - common.Exitf(1, "binaries not found: %s", err) - } - - // Compute base port - basePort := p.DefaultPorts().BasePort - if providerName == "postgresql" { - basePort, _ = postgresql.VersionToPort(version) - } - - sandboxHome := defaults.Defaults().SandboxHome - topologyDir := path.Join(sandboxHome, fmt.Sprintf("%s_multi_%d", providerName, basePort)) - if common.DirExists(topologyDir) { - common.Exitf(1, "sandbox directory %s already exists", topologyDir) - } - os.MkdirAll(topologyDir, 0755) - - skipStart, _ := flags.GetBool(globals.SkipStartLabel) - - for i := 1; i <= nodes; i++ { - port := basePort + i - freePort, err := common.FindFreePort(port, []int{}, 1) - if err == nil { - port = freePort - } - - nodeDir := path.Join(topologyDir, fmt.Sprintf("node%d", i)) - config := providers.SandboxConfig{ - Version: version, - Dir: nodeDir, - Port: port, - Host: "127.0.0.1", - DbUser: "postgres", - Options: map[string]string{}, - } - - if _, err := p.CreateSandbox(config); err != nil { - common.Exitf(1, "error creating node %d: %s", i, err) - } - - if !skipStart { - if err := p.StartSandbox(nodeDir); err != nil { - common.Exitf(1, "error starting node %d: %s", i, err) - } - } - - fmt.Printf(" Node %d deployed in %s (port: %d)\n", i, nodeDir, port) - } - - fmt.Printf("%s multiple sandbox (%d nodes) deployed in %s\n", providerName, nodes, topologyDir) -} -``` - -Add flag in `init()`: - -```go -multipleCmd.PersistentFlags().String(globals.ProviderLabel, globals.ProviderValue, "Database provider (mysql, postgresql)") -``` - -- [ ] **Step 6: Update cmd/replication.go — add --provider flag and PostgreSQL replication flow** - -Same bypass pattern. For PostgreSQL: create primary with replication options, start it, then CreateReplica for each replica sequentially: - -```go -func replicationSandbox(cmd *cobra.Command, args []string) { - flags := cmd.Flags() - providerName, _ := flags.GetString(globals.ProviderLabel) - - if providerName != "mysql" { - deployReplicationNonMySQL(cmd, args, providerName) - return - } - - // Existing MySQL path unchanged, BUT update DeployProxySQLForTopology call: - // sandbox.DeployProxySQLForTopology(sandboxDir, masterPort, slavePorts, 0, "127.0.0.1", "") - // ... -} - -func deployReplicationNonMySQL(cmd *cobra.Command, args []string, providerName string) { - flags := cmd.Flags() - version := args[0] - nodes, _ := flags.GetInt(globals.NodesLabel) - - p, err := providers.DefaultRegistry.Get(providerName) - if err != nil { - common.Exitf(1, "provider error: %s", err) - } - - flavor, _ := flags.GetString(globals.FlavorLabel) - if flavor != "" { - common.Exitf(1, "--flavor is only valid with --provider=mysql") - } - - if !providers.ContainsString(p.SupportedTopologies(), "replication") { - common.Exitf(1, "provider %q does not support topology \"replication\"\nSupported topologies: %s", - providerName, strings.Join(p.SupportedTopologies(), ", ")) - } - - if err := p.ValidateVersion(version); err != nil { - common.Exitf(1, "version validation failed: %s", err) - } - - if _, err := p.FindBinary(version); err != nil { - common.Exitf(1, "binaries not found: %s", err) - } - - // Compute base port - basePort := p.DefaultPorts().BasePort - if providerName == "postgresql" { - basePort, _ = postgresql.VersionToPort(version) - } - - sandboxHome := defaults.Defaults().SandboxHome - topologyDir := path.Join(sandboxHome, fmt.Sprintf("%s_repl_%d", providerName, basePort)) - if common.DirExists(topologyDir) { - common.Exitf(1, "sandbox directory %s already exists", topologyDir) - } - os.MkdirAll(topologyDir, 0755) - - skipStart, _ := flags.GetBool(globals.SkipStartLabel) - primaryPort := basePort - - // 1. Create and start primary with replication options - primaryDir := path.Join(topologyDir, "primary") - primaryConfig := providers.SandboxConfig{ - Version: version, - Dir: primaryDir, - Port: primaryPort, - Host: "127.0.0.1", - DbUser: "postgres", - Options: map[string]string{"replication": "true"}, - } - - if _, err := p.CreateSandbox(primaryConfig); err != nil { - common.Exitf(1, "error creating primary: %s", err) - } - - if !skipStart { - if err := p.StartSandbox(primaryDir); err != nil { - common.Exitf(1, "error starting primary: %s", err) - } - } - - fmt.Printf(" Primary deployed in %s (port: %d)\n", primaryDir, primaryPort) - - primaryInfo := providers.SandboxInfo{Dir: primaryDir, Port: primaryPort, Status: "running"} - - // 2. Create replicas sequentially (pg_basebackup requires running primary) - var replicaPorts []int - for i := 1; i <= nodes-1; i++ { - replicaPort := primaryPort + i - freePort, err := common.FindFreePort(replicaPort, []int{}, 1) - if err == nil { - replicaPort = freePort - } - - replicaDir := path.Join(topologyDir, fmt.Sprintf("replica%d", i)) - replicaConfig := providers.SandboxConfig{ - Version: version, - Dir: replicaDir, - Port: replicaPort, - Host: "127.0.0.1", - DbUser: "postgres", - Options: map[string]string{}, - } - - if _, err := p.CreateReplica(primaryInfo, replicaConfig); err != nil { - // Cleanup: stop primary and any already-running replicas - p.StopSandbox(primaryDir) - for j := 1; j < i; j++ { - p.StopSandbox(path.Join(topologyDir, fmt.Sprintf("replica%d", j))) - } - common.Exitf(1, "error creating replica %d: %s", i, err) - } - - replicaPorts = append(replicaPorts, replicaPort) - fmt.Printf(" Replica %d deployed in %s (port: %d)\n", i, replicaDir, replicaPort) - } - - // 3. Generate topology-level monitoring scripts - home, _ := os.UserHomeDir() - basedir := path.Join(home, "opt", "postgresql", version) - binDir := path.Join(basedir, "bin") - libDir := path.Join(basedir, "lib") - - scriptOpts := postgresql.ScriptOptions{ - BinDir: binDir, - LibDir: libDir, - Port: primaryPort, - } - - checkReplScript := postgresql.GenerateCheckReplicationScript(scriptOpts) - os.WriteFile(path.Join(topologyDir, "check_replication"), []byte(checkReplScript), 0755) - - checkRecovScript := postgresql.GenerateCheckRecoveryScript(scriptOpts, replicaPorts) - os.WriteFile(path.Join(topologyDir, "check_recovery"), []byte(checkRecovScript), 0755) - - // 4. Handle --with-proxysql - withProxySQL, _ := flags.GetBool("with-proxysql") - if withProxySQL { - if !providers.ContainsString(providers.CompatibleAddons["proxysql"], providerName) { - common.Exitf(1, "--with-proxysql is not compatible with provider %q", providerName) - } - err := sandbox.DeployProxySQLForTopology(topologyDir, primaryPort, replicaPorts, 0, "127.0.0.1", providerName) - if err != nil { - common.Exitf(1, "ProxySQL deployment failed: %s", err) - } - } - - fmt.Printf("%s replication sandbox (1 primary + %d replicas) deployed in %s\n", - providerName, nodes-1, topologyDir) -} -``` - -Add flag in `init()`: - -```go -replicationCmd.PersistentFlags().String(globals.ProviderLabel, globals.ProviderValue, "Database provider (mysql, postgresql)") -``` - -- [ ] **Step 7: Run full test suite to verify nothing is broken** - -Run: `cd /data/rene/dbdeployer && go test ./... -v -timeout 5m` -Expected: All existing tests pass. No regressions. - -- [ ] **Step 8: Commit** - -```bash -git add globals/globals.go providers/provider.go cmd/root.go cmd/single.go cmd/multiple.go cmd/replication.go sandbox/proxysql_topology.go -git commit -m "feat: add --provider flag and PostgreSQL routing to deploy commands" -``` - ---- - -## Task 8: Unpack Command for PostgreSQL Debs - -**Files:** -- Modify: `cmd/unpack.go` - -- [ ] **Step 1: Add --provider flag to unpack command** - -In `cmd/unpack.go`, modify `unpackTarball()` to check `--provider` flag. When `--provider=postgresql`, route to PostgreSQL deb extraction instead of MySQL tarball extraction: - -```go -providerName, _ := flags.GetString(globals.ProviderLabel) -if providerName == "postgresql" { - // PostgreSQL deb extraction - if len(args) < 2 { - common.Exitf(1, "PostgreSQL unpack requires both server and client .deb files\n"+ - "Usage: dbdeployer unpack --provider=postgresql postgresql-16_*.deb postgresql-client-16_*.deb") - } - server, client, err := postgresql.ClassifyDebs(args) - if err != nil { - common.Exitf(1, "error classifying deb files: %s", err) - } - version := Version // from --unpack-version flag - if version == "" { - version, err = postgresql.ParseDebVersion(server) - if err != nil { - common.Exitf(1, "cannot detect version from filename: %s\nUse --unpack-version to specify", err) - } - } - targetDir := filepath.Join(home, "opt", "postgresql", version) - if err := postgresql.UnpackDebs(server, client, targetDir); err != nil { - common.Exitf(1, "error unpacking PostgreSQL debs: %s", err) - } - fmt.Printf("PostgreSQL %s unpacked to %s\n", version, targetDir) - return -} -// ... existing MySQL tarball path unchanged -``` - -Add the `--provider` flag in `init()`: - -```go -unpackCmd.PersistentFlags().String(globals.ProviderLabel, globals.ProviderValue, "Database provider (mysql, postgresql)") -``` - -Update `unpackCmd` to accept variadic args for PostgreSQL (currently `Args: cobra.ExactArgs(1)`): - -```go -Args: cobra.MinimumNArgs(1), -``` - -- [ ] **Step 2: Run tests to verify no regressions** - -Run: `cd /data/rene/dbdeployer && go test ./... -timeout 5m` -Expected: All pass. - -- [ ] **Step 3: Commit** - -```bash -git add cmd/unpack.go -git commit -m "feat: add --provider=postgresql support to dbdeployer unpack for deb extraction" -``` - ---- - -## Task 9: ProxySQL + PostgreSQL Backend Wiring - -**Files:** -- Modify: `providers/proxysql/config.go` -- Modify: `providers/proxysql/config_test.go` (or create if absent) -- Modify: `providers/proxysql/proxysql.go` -- Modify: `sandbox/proxysql_topology.go` - -- [ ] **Step 1: Write failing test for PostgreSQL backend config generation** - -Add to `providers/proxysql/config_test.go` (create if needed): - -```go -package proxysql - -import ( - "strings" - "testing" -) - -func TestGenerateConfigMySQL(t *testing.T) { - cfg := ProxySQLConfig{ - AdminHost: "127.0.0.1", - AdminPort: 6032, - AdminUser: "admin", - AdminPassword: "admin", - MySQLPort: 6033, - DataDir: "/tmp/proxysql/data", - MonitorUser: "msandbox", - MonitorPass: "msandbox", - Backends: []BackendServer{ - {Host: "127.0.0.1", Port: 3306, Hostgroup: 0, MaxConns: 200}, - }, - } - config := GenerateConfig(cfg) - if !strings.Contains(config, "mysql_servers") { - t.Error("expected mysql_servers block") - } - if !strings.Contains(config, "mysql_variables") { - t.Error("expected mysql_variables block") - } -} - -func TestGenerateConfigPostgreSQL(t *testing.T) { - cfg := ProxySQLConfig{ - AdminHost: "127.0.0.1", - AdminPort: 6032, - AdminUser: "admin", - AdminPassword: "admin", - MySQLPort: 6033, - DataDir: "/tmp/proxysql/data", - MonitorUser: "postgres", - MonitorPass: "postgres", - BackendProvider: "postgresql", - Backends: []BackendServer{ - {Host: "127.0.0.1", Port: 16613, Hostgroup: 0, MaxConns: 200}, - {Host: "127.0.0.1", Port: 16614, Hostgroup: 1, MaxConns: 200}, - }, - } - config := GenerateConfig(cfg) - if !strings.Contains(config, "pgsql_servers") { - t.Error("expected pgsql_servers block") - } - if !strings.Contains(config, "pgsql_users") { - t.Error("expected pgsql_users block") - } - if !strings.Contains(config, "pgsql_variables") { - t.Error("expected pgsql_variables block") - } - if strings.Contains(config, "mysql_servers") { - t.Error("should not contain mysql_servers for postgresql backend") - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cd /data/rene/dbdeployer && go test ./providers/proxysql/ -run TestGenerateConfig -v` -Expected: Fail — `BackendProvider` field doesn't exist yet. - -- [ ] **Step 3: Add BackendProvider field to ProxySQLConfig and update GenerateConfig** - -In `providers/proxysql/config.go`: - -Add `BackendProvider string` field to `ProxySQLConfig`. - -Update `GenerateConfig` to branch on `BackendProvider`: - -```go -func GenerateConfig(cfg ProxySQLConfig) string { - var b strings.Builder - b.WriteString(fmt.Sprintf("datadir=\"%s\"\n\n", cfg.DataDir)) - - b.WriteString("admin_variables=\n{\n") - b.WriteString(fmt.Sprintf(" admin_credentials=\"%s:%s\"\n", cfg.AdminUser, cfg.AdminPassword)) - b.WriteString(fmt.Sprintf(" mysql_ifaces=\"%s:%d\"\n", cfg.AdminHost, cfg.AdminPort)) - b.WriteString("}\n\n") - - isPgsql := cfg.BackendProvider == "postgresql" - - if isPgsql { - b.WriteString("pgsql_variables=\n{\n") - b.WriteString(fmt.Sprintf(" interfaces=\"%s:%d\"\n", cfg.AdminHost, cfg.MySQLPort)) - b.WriteString(fmt.Sprintf(" monitor_username=\"%s\"\n", cfg.MonitorUser)) - b.WriteString(fmt.Sprintf(" monitor_password=\"%s\"\n", cfg.MonitorPass)) - b.WriteString("}\n\n") - } else { - b.WriteString("mysql_variables=\n{\n") - b.WriteString(fmt.Sprintf(" interfaces=\"%s:%d\"\n", cfg.AdminHost, cfg.MySQLPort)) - b.WriteString(fmt.Sprintf(" monitor_username=\"%s\"\n", cfg.MonitorUser)) - b.WriteString(fmt.Sprintf(" monitor_password=\"%s\"\n", cfg.MonitorPass)) - b.WriteString(" monitor_connect_interval=2000\n") - b.WriteString(" monitor_ping_interval=2000\n") - b.WriteString("}\n\n") - } - - serversKey := "mysql_servers" - usersKey := "mysql_users" - if isPgsql { - serversKey = "pgsql_servers" - usersKey = "pgsql_users" - } - - if len(cfg.Backends) > 0 { - b.WriteString(fmt.Sprintf("%s=\n(\n", serversKey)) - for i, srv := range cfg.Backends { - b.WriteString(" {\n") - b.WriteString(fmt.Sprintf(" address=\"%s\"\n", srv.Host)) - b.WriteString(fmt.Sprintf(" port=%d\n", srv.Port)) - b.WriteString(fmt.Sprintf(" hostgroup=%d\n", srv.Hostgroup)) - maxConns := srv.MaxConns - if maxConns == 0 { - maxConns = 200 - } - b.WriteString(fmt.Sprintf(" max_connections=%d\n", maxConns)) - b.WriteString(" }") - if i < len(cfg.Backends)-1 { - b.WriteString(",") - } - b.WriteString("\n") - } - b.WriteString(")\n\n") - } - - b.WriteString(fmt.Sprintf("%s=\n(\n", usersKey)) - b.WriteString(" {\n") - b.WriteString(fmt.Sprintf(" username=\"%s\"\n", cfg.MonitorUser)) - b.WriteString(fmt.Sprintf(" password=\"%s\"\n", cfg.MonitorPass)) - b.WriteString(" default_hostgroup=0\n") - b.WriteString(" }\n") - b.WriteString(")\n") - - return b.String() -} -``` - -- [ ] **Step 4: Update ProxySQLProvider.CreateSandbox to pass BackendProvider** - -In `providers/proxysql/proxysql.go`, set `BackendProvider` from `config.Options["backend_provider"]`: - -```go -proxyCfg := ProxySQLConfig{ - // ... existing fields ... - BackendProvider: config.Options["backend_provider"], -} -``` - -Also update the `use_proxy` script generation to use `psql` when backend is PostgreSQL: - -```go -if config.Options["backend_provider"] == "postgresql" { - scripts["use_proxy"] = fmt.Sprintf("#!/bin/bash\npsql -h %s -p %d -U %s \"$@\"\n", - host, mysqlPort, monitorUser) -} else { - scripts["use_proxy"] = fmt.Sprintf("#!/bin/bash\nmysql -h %s -P %d -u %s -p%s --prompt 'ProxySQL> ' \"$@\"\n", - host, mysqlPort, monitorUser, monitorPass) -} -``` - -- [ ] **Step 5: Update DeployProxySQLForTopology to accept backend provider** - -In `sandbox/proxysql_topology.go`, add a `backendProvider` parameter: - -```go -func DeployProxySQLForTopology(sandboxDir string, masterPort int, slavePorts []int, proxysqlPort int, host string, backendProvider string) error { - // ... existing code ... - config.Options["backend_provider"] = backendProvider - // ... -} -``` - -Update all callers in `cmd/single.go` and `cmd/replication.go` to pass `""` (empty string = mysql default) or `"postgresql"` when appropriate. - -- [ ] **Step 6: Run all tests** - -Run: `cd /data/rene/dbdeployer && go test ./... -timeout 5m` -Expected: All pass. - -- [ ] **Step 7: Commit** - -```bash -git add providers/proxysql/config.go providers/proxysql/config_test.go providers/proxysql/proxysql.go sandbox/proxysql_topology.go cmd/single.go cmd/replication.go -git commit -m "feat: add ProxySQL PostgreSQL backend wiring (pgsql_servers/pgsql_users config)" -``` - ---- - -## Task 10: Cross-Database Topology Constraints - -**Files:** -- Modify: `cmd/single.go` -- Modify: `cmd/multiple.go` -- Modify: `cmd/replication.go` -- Modify: `providers/provider_test.go` - -This task ensures the validation logic added in Task 7 is properly tested. - -- [ ] **Step 1: Write tests for topology and cross-provider validation** - -Add to `providers/provider_test.go`: - -```go -func TestTopologyValidation(t *testing.T) { - mock := &mockProvider{name: "test"} - topos := mock.SupportedTopologies() - if !containsString(topos, "single") { - t.Error("expected single in supported topologies") - } - if containsString(topos, "group") { - t.Error("did not expect group in supported topologies") - } -} - -func containsString(slice []string, s string) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} -``` - -Also add a test for the addon compatibility map (define it in `providers/provider.go` or `cmd/` depending on where it lives): - -```go -var CompatibleAddons = map[string][]string{ - "proxysql": {"mysql", "postgresql"}, -} - -func TestAddonCompatibility(t *testing.T) { - if !containsString(CompatibleAddons["proxysql"], "postgresql") { - t.Error("proxysql should be compatible with postgresql") - } - if containsString(CompatibleAddons["proxysql"], "fake") { - t.Error("proxysql should not be compatible with fake") - } -} -``` - -- [ ] **Step 2: Add CompatibleAddons map to providers/provider.go** - -```go -// CompatibleAddons maps addon names to the list of providers they work with. -var CompatibleAddons = map[string][]string{ - "proxysql": {"mysql", "postgresql"}, -} -``` - -- [ ] **Step 3: Run tests** - -Run: `cd /data/rene/dbdeployer && go test ./providers/ -v` -Expected: All pass. - -- [ ] **Step 4: Commit** - -```bash -git add providers/provider.go providers/provider_test.go -git commit -m "feat: add cross-database topology constraint validation" -``` - ---- - -## Task 11: Standalone PostgreSQL Deploy Command - -**Files:** -- Create: `cmd/deploy_postgresql.go` - -- [ ] **Step 1: Create deploy postgresql subcommand** - -Create `cmd/deploy_postgresql.go` following the pattern from `cmd/deploy_proxysql.go`: - -```go -package cmd - -import ( - "fmt" - "path" - - "github.com/ProxySQL/dbdeployer/common" - "github.com/ProxySQL/dbdeployer/defaults" - "github.com/ProxySQL/dbdeployer/providers" - "github.com/ProxySQL/dbdeployer/providers/postgresql" - "github.com/spf13/cobra" -) - -func deploySandboxPostgreSQL(cmd *cobra.Command, args []string) { - version := args[0] - flags := cmd.Flags() - skipStart, _ := flags.GetBool("skip-start") - - p, err := providers.DefaultRegistry.Get("postgresql") - if err != nil { - common.Exitf(1, "PostgreSQL provider not available: %s", err) - } - - if err := p.ValidateVersion(version); err != nil { - common.Exitf(1, "invalid version: %s", err) - } - - if _, err := p.FindBinary(version); err != nil { - common.Exitf(1, "PostgreSQL binaries not found: %s\nRun: dbdeployer unpack --provider=postgresql ", err) - } - - port, err := postgresql.VersionToPort(version) - if err != nil { - common.Exitf(1, "error computing port: %s", err) - } - freePort, portErr := common.FindFreePort(port, []int{}, 1) - if portErr == nil { - port = freePort - } - - sandboxHome := defaults.Defaults().SandboxHome - sandboxDir := path.Join(sandboxHome, fmt.Sprintf("pg_sandbox_%d", port)) - - if common.DirExists(sandboxDir) { - common.Exitf(1, "sandbox directory %s already exists", sandboxDir) - } - - config := providers.SandboxConfig{ - Version: version, - Dir: sandboxDir, - Port: port, - Host: "127.0.0.1", - DbUser: "postgres", - DbPassword: "", - Options: map[string]string{}, - } - - if _, err := p.CreateSandbox(config); err != nil { - common.Exitf(1, "error creating PostgreSQL sandbox: %s", err) - } - - if !skipStart { - if err := p.StartSandbox(sandboxDir); err != nil { - common.Exitf(1, "error starting PostgreSQL: %s", err) - } - } - - fmt.Printf("PostgreSQL %s sandbox deployed in %s (port: %d)\n", version, sandboxDir, port) -} - -var deployPostgreSQLCmd = &cobra.Command{ - Use: "postgresql version", - Short: "deploys a PostgreSQL sandbox", - Long: `postgresql deploys a standalone PostgreSQL instance as a sandbox. -It creates a sandbox directory with data, configuration, start/stop scripts, and a -psql client script. - -Requires PostgreSQL binaries to be extracted first: - dbdeployer unpack --provider=postgresql postgresql-16_*.deb postgresql-client-16_*.deb - -Example: - dbdeployer deploy postgresql 16.13 - dbdeployer deploy postgresql 17.1 --skip-start -`, - Args: cobra.ExactArgs(1), - Run: deploySandboxPostgreSQL, -} - -func init() { - deployCmd.AddCommand(deployPostgreSQLCmd) - deployPostgreSQLCmd.Flags().Bool("skip-start", false, "Do not start PostgreSQL after deployment") -} -``` - -- [ ] **Step 2: Run build to verify compilation** - -Run: `cd /data/rene/dbdeployer && go build -o /dev/null .` -Expected: Build succeeds. - -- [ ] **Step 3: Commit** - -```bash -git add cmd/deploy_postgresql.go -git commit -m "feat: add 'dbdeployer deploy postgresql' standalone command" -``` - ---- - -## Task 12: Integration Tests - -**Files:** -- Create: `providers/postgresql/integration_test.go` - -- [ ] **Step 1: Write integration tests (build-tagged)** - -Create `providers/postgresql/integration_test.go`: - -```go -//go:build integration - -package postgresql - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "testing" - "time" - - "github.com/ProxySQL/dbdeployer/common" - "github.com/ProxySQL/dbdeployer/providers" -) - -func findPostgresVersion(t *testing.T) string { - t.Helper() - home, _ := os.UserHomeDir() - entries, err := os.ReadDir(filepath.Join(home, "opt", "postgresql")) - if err != nil { - t.Skipf("no PostgreSQL installations found: %v", err) - } - for _, e := range entries { - if e.IsDir() { - return e.Name() - } - } - t.Skip("no PostgreSQL version directories found") - return "" -} - -func TestIntegrationSingleSandbox(t *testing.T) { - version := findPostgresVersion(t) - p := NewPostgreSQLProvider() - - tmpDir := t.TempDir() - sandboxDir := filepath.Join(tmpDir, "pg_test") - - config := providers.SandboxConfig{ - Version: version, - Dir: sandboxDir, - Port: 15432, - Host: "127.0.0.1", - DbUser: "postgres", - Options: map[string]string{}, - } - - // Create - info, err := p.CreateSandbox(config) - if err != nil { - t.Fatalf("CreateSandbox failed: %v", err) - } - if info.Port != 15432 { - t.Errorf("expected port 15432, got %d", info.Port) - } - - // Start - if err := p.StartSandbox(sandboxDir); err != nil { - t.Fatalf("StartSandbox failed: %v", err) - } - stopped := false - defer func() { - if !stopped { - p.StopSandbox(sandboxDir) - } - }() - time.Sleep(2 * time.Second) - - // Connect via psql - home, _ := os.UserHomeDir() - psql := filepath.Join(home, "opt", "postgresql", version, "bin", "psql") - cmd := exec.Command(psql, "-h", "127.0.0.1", "-p", "15432", "-U", "postgres", "-c", "SELECT 1;") - cmd.Env = append(os.Environ(), fmt.Sprintf("LD_LIBRARY_PATH=%s", - filepath.Join(home, "opt", "postgresql", version, "lib"))) - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("psql connection failed: %s: %v", string(output), err) - } - - // Stop - if err := p.StopSandbox(sandboxDir); err != nil { - t.Fatalf("StopSandbox failed: %v", err) - } - stopped = true -} - -func TestIntegrationReplication(t *testing.T) { - version := findPostgresVersion(t) - p := NewPostgreSQLProvider() - - tmpDir := t.TempDir() - primaryDir := filepath.Join(tmpDir, "primary") - replica1Dir := filepath.Join(tmpDir, "replica1") - replica2Dir := filepath.Join(tmpDir, "replica2") - - // Create and start primary with replication - primaryConfig := providers.SandboxConfig{ - Version: version, - Dir: primaryDir, - Port: 15500, - Host: "127.0.0.1", - DbUser: "postgres", - Options: map[string]string{"replication": "true"}, - } - - _, err := p.CreateSandbox(primaryConfig) - if err != nil { - t.Fatalf("CreateSandbox (primary) failed: %v", err) - } - if err := p.StartSandbox(primaryDir); err != nil { - t.Fatalf("StartSandbox (primary) failed: %v", err) - } - defer p.StopSandbox(primaryDir) - time.Sleep(2 * time.Second) - - primaryInfo := providers.SandboxInfo{Dir: primaryDir, Port: 15500} - - // Create replicas - for i, rDir := range []string{replica1Dir, replica2Dir} { - rConfig := providers.SandboxConfig{ - Version: version, - Dir: rDir, - Port: 15501 + i, - Host: "127.0.0.1", - DbUser: "postgres", - Options: map[string]string{}, - } - _, err := p.CreateReplica(primaryInfo, rConfig) - if err != nil { - t.Fatalf("CreateReplica %d failed: %v", i+1, err) - } - defer p.StopSandbox(rDir) - } - - time.Sleep(2 * time.Second) - - // Verify pg_stat_replication on primary shows 2 replicas - home, _ := os.UserHomeDir() - psql := filepath.Join(home, "opt", "postgresql", version, "bin", "psql") - libDir := filepath.Join(home, "opt", "postgresql", version, "lib") - - cmd := exec.Command(psql, "-h", "127.0.0.1", "-p", "15500", "-U", "postgres", "-t", "-c", - "SELECT count(*) FROM pg_stat_replication;") - cmd.Env = append(os.Environ(), fmt.Sprintf("LD_LIBRARY_PATH=%s", libDir)) - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("replication check failed: %s: %v", string(output), err) - } - - // Verify replicas are in recovery - for _, port := range []int{15501, 15502} { - cmd := exec.Command(psql, "-h", "127.0.0.1", "-p", fmt.Sprintf("%d", port), "-U", "postgres", "-t", "-c", - "SELECT pg_is_in_recovery();") - cmd.Env = append(os.Environ(), fmt.Sprintf("LD_LIBRARY_PATH=%s", libDir)) - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("recovery check on port %d failed: %s: %v", port, string(output), err) - } - } -} -``` - -- [ ] **Step 2: Verify unit tests still pass (integration tests skipped by default)** - -Run: `cd /data/rene/dbdeployer && go test ./providers/postgresql/ -v` -Expected: All unit tests pass. Integration tests are not compiled without the build tag. - -- [ ] **Step 3: Commit** - -```bash -git add providers/postgresql/integration_test.go -git commit -m "test: add PostgreSQL integration tests (build-tagged)" -``` - ---- - -## Task 13: Create GitHub Issues for CI Follow-Up - -**Files:** None (GitHub issues only) - -- [ ] **Step 1: Create GitHub issue for PostgreSQL deb caching in CI** - -```bash -gh issue create --title "CI: Add PostgreSQL deb caching to CI pipeline" \ - --body "Add caching of PostgreSQL server and client .deb packages to CI, similar to MySQL tarball caching. This enables running PostgreSQL integration tests in CI." \ - --label "enhancement,ci" -``` - -- [ ] **Step 2: Create GitHub issue for PostgreSQL integration tests in CI matrix** - -```bash -gh issue create --title "CI: Add PostgreSQL integration tests to CI matrix" \ - --body "Add PostgreSQL integration tests (providers/postgresql/integration_test.go) to the CI test matrix. Requires PostgreSQL deb caching (#) to be in place." \ - --label "enhancement,ci" -``` - -- [ ] **Step 3: Create GitHub issue for nightly PostgreSQL topology tests** - -```bash -gh issue create --title "CI: Add nightly PostgreSQL replication topology tests" \ - --body "Add nightly CI job that runs full PostgreSQL replication topology tests (primary + replicas, ProxySQL wiring)." \ - --label "enhancement,ci" -``` - -- [ ] **Step 4: Commit (no code changes, just documenting)** - -No commit needed — issues are tracked in GitHub. - ---- - -## Execution Notes - -### Dependencies between tasks -- Task 1 (interface changes) must complete before all other tasks -- Tasks 2-5 can run in parallel after Task 1 -- Task 6 (replication) depends on Tasks 2-4 -- Task 7 (cmd layer) depends on Tasks 2-6 -- Task 8 (unpack cmd) depends on Task 5 -- Task 9 (ProxySQL wiring) depends on Tasks 6-7 -- Task 10 (constraints) depends on Task 7 -- Task 11 (deploy command) depends on Tasks 2-4, 7 -- Task 12 (integration tests) depends on all implementation tasks -- Task 13 (GitHub issues) is independent - -### Running integration tests locally - -```bash -# Extract PostgreSQL binaries first -apt-get download postgresql-16 postgresql-client-16 -./dbdeployer unpack --provider=postgresql postgresql-16_*.deb postgresql-client-16_*.deb - -# Run integration tests -cd /data/rene/dbdeployer && go test ./providers/postgresql/ -tags integration -v -timeout 10m -``` diff --git a/docs/superpowers/plans/2026-03-24-website.md b/docs/superpowers/plans/2026-03-24-website.md deleted file mode 100644 index 165209ca..00000000 --- a/docs/superpowers/plans/2026-03-24-website.md +++ /dev/null @@ -1,927 +0,0 @@ -# dbdeployer Website Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build a documentation website for dbdeployer with Astro + Starlight, deployed to GitHub Pages, featuring a marketing landing page, migrated wiki docs, quickstart guides, providers page, and blog. - -**Architecture:** Astro project in `website/` at the repo root. Starlight handles the docs section (sidebar, search, dark mode). Custom Astro pages for landing, providers, and blog. A shell script copies `docs/wiki/*.md` into Starlight's content collection with frontmatter injection and link rewriting. GitHub Actions builds and deploys to `gh-pages` on push. - -**Tech Stack:** Astro 4.x, Starlight, Node.js 20 LTS, GitHub Actions, GitHub Pages - -**Spec:** `docs/superpowers/specs/2026-03-24-website-design.md` - ---- - -## File Structure - -### New Files (in `website/`) -- `package.json` — Astro project dependencies -- `astro.config.mjs` — Astro + Starlight config with sidebar, base path, sitemap -- `tsconfig.json` — TypeScript config (Astro default) -- `src/content/config.ts` — Content collection schemas (docs via Starlight, blog custom) -- `src/pages/index.astro` — Landing page -- `src/pages/providers.astro` — Providers comparison page -- `src/pages/404.astro` — Custom 404 -- `src/pages/blog/index.astro` — Blog index -- `src/pages/blog/[...slug].astro` — Blog post pages -- `src/components/Hero.astro` — Hero section component -- `src/components/FeatureGrid.astro` — Feature cards grid -- `src/components/ProviderCard.astro` — Provider card component -- `src/components/Terminal.astro` — Terminal demo component -- `src/components/BlogPostCard.astro` — Blog post preview card -- `src/layouts/Landing.astro` — Layout for marketing pages -- `src/layouts/BlogPost.astro` — Layout for blog posts -- `src/styles/global.css` — Global styles -- `src/content/blog/2026-03-24-new-maintainership.md` — Launch blog post 1 -- `src/content/blog/2026-03-24-postgresql-support.md` — Launch blog post 2 -- `src/content/docs/getting-started/quickstart-mysql-single.md` — New quickstart guide -- `src/content/docs/getting-started/quickstart-mysql-replication.md` — New quickstart guide -- `src/content/docs/getting-started/quickstart-postgresql.md` — New quickstart guide -- `src/content/docs/getting-started/quickstart-proxysql.md` — New quickstart guide -- `src/content/docs/providers/postgresql.md` — New provider docs -- `public/favicon.svg` — Favicon -- `public/og-image.png` — OG social image (placeholder) -- `scripts/copy-wiki.sh` — Wiki content pipeline script - -### New Files (repo root) -- `.github/workflows/deploy-website.yml` — GitHub Actions deployment workflow - ---- - -## Task 1: Scaffold Astro + Starlight Project - -**Files:** -- Create: `website/package.json` -- Create: `website/astro.config.mjs` -- Create: `website/tsconfig.json` -- Create: `website/src/content/config.ts` -- Create: `website/src/content/docs/index.mdx` (Starlight requires at least one doc) -- Create: `website/public/favicon.svg` - -- [ ] **Step 1: Initialize Astro project** - -```bash -cd /data/rene/dbdeployer -mkdir -p website -cd website -npm create astro@latest -- --template starlight --no-git --no-install -y . -``` - -If the template prompt is interactive, manually create the files instead: - -```bash -npm init -y -npm install astro @astrojs/starlight @astrojs/sitemap -``` - -- [ ] **Step 2: Configure astro.config.mjs** - -```js -import { defineConfig } from 'astro/config'; -import starlight from '@astrojs/starlight'; -import sitemap from '@astrojs/sitemap'; - -export default defineConfig({ - site: 'https://proxysql.github.io', - base: '/dbdeployer', - integrations: [ - starlight({ - title: 'dbdeployer', - description: 'Deploy MySQL & PostgreSQL sandboxes in seconds', - social: { - github: 'https://github.com/ProxySQL/dbdeployer', - }, - sidebar: [ - { - label: 'Getting Started', - items: [ - { label: 'Installation', slug: 'getting-started/installation' }, - { label: 'Quick Start: MySQL Single', slug: 'getting-started/quickstart-mysql-single' }, - { label: 'Quick Start: MySQL Replication', slug: 'getting-started/quickstart-mysql-replication' }, - { label: 'Quick Start: PostgreSQL', slug: 'getting-started/quickstart-postgresql' }, - { label: 'Quick Start: ProxySQL Integration', slug: 'getting-started/quickstart-proxysql' }, - ], - }, - { - label: 'Core Concepts', - items: [ - { label: 'Sandboxes', slug: 'concepts/sandboxes' }, - { label: 'Versions & Flavors', slug: 'concepts/flavors' }, - { label: 'Ports & Networking', slug: 'concepts/ports' }, - { label: 'Environment Variables', slug: 'concepts/environment-variables' }, - ], - }, - { - label: 'Deploying', - items: [ - { label: 'Single Sandbox', slug: 'deploying/single' }, - { label: 'Multiple Sandboxes', slug: 'deploying/multiple' }, - { label: 'Replication', slug: 'deploying/replication' }, - { label: 'Group Replication', slug: 'deploying/group-replication' }, - { label: 'Fan-In & All-Masters', slug: 'deploying/fan-in-all-masters' }, - { label: 'NDB Cluster', slug: 'deploying/ndb-cluster' }, - ], - }, - { - label: 'Providers', - items: [ - { label: 'MySQL', slug: 'providers/mysql' }, - { label: 'PostgreSQL', slug: 'providers/postgresql' }, - { label: 'ProxySQL', slug: 'providers/proxysql' }, - { label: 'Percona XtraDB Cluster', slug: 'providers/pxc' }, - ], - }, - { - label: 'Managing Sandboxes', - items: [ - { label: 'Starting & Stopping', slug: 'managing/starting-stopping' }, - { label: 'Using Sandboxes', slug: 'managing/using' }, - { label: 'Customization', slug: 'managing/customization' }, - { label: 'Database Users', slug: 'managing/users' }, - { label: 'Logs', slug: 'managing/logs' }, - { label: 'Deletion & Cleanup', slug: 'managing/deletion' }, - ], - }, - { - label: 'Advanced', - items: [ - { label: 'Concurrent Deployment', slug: 'advanced/concurrent' }, - { label: 'Importing Databases', slug: 'advanced/importing' }, - { label: 'Inter-Sandbox Replication', slug: 'advanced/inter-sandbox-replication' }, - { label: 'Cloning', slug: 'advanced/cloning' }, - { label: 'Using as a Go Library', slug: 'advanced/go-library' }, - { label: 'Compiling from Source', slug: 'advanced/compiling' }, - ], - }, - { - label: 'Reference', - items: [ - { label: 'CLI Commands', slug: 'reference/cli-commands' }, - { label: 'Configuration', slug: 'reference/configuration' }, - { label: 'API Changelog', slug: 'reference/api-changelog' }, - ], - }, - ], - }), - sitemap(), - ], -}); -``` - -- [ ] **Step 3: Create minimal tsconfig.json** - -```json -{ - "extends": "astro/tsconfigs/strict" -} -``` - -- [ ] **Step 4: Create content collection config** - -Create `src/content/config.ts`: - -```ts -import { defineCollection, z } from 'astro:content'; -import { docsSchema } from '@astrojs/starlight/schema'; - -const blog = defineCollection({ - type: 'content', - schema: z.object({ - title: z.string(), - date: z.date(), - author: z.string(), - description: z.string(), - tags: z.array(z.string()).optional(), - }), -}); - -export const collections = { - docs: defineCollection({ schema: docsSchema() }), - blog, -}; -``` - -- [ ] **Step 5: Create placeholder doc page** - -Create `src/content/docs/index.mdx`: - -```mdx ---- -title: dbdeployer Documentation -description: Deploy MySQL & PostgreSQL sandboxes in seconds ---- - -Welcome to dbdeployer documentation. Use the sidebar to navigate. -``` - -- [ ] **Step 6: Create favicon** - -Create `public/favicon.svg` — a simple database icon: - -```svg - - - - - -``` - -- [ ] **Step 7: Install dependencies and verify build** - -```bash -cd website -npm install -npm run build -``` - -Expected: Build succeeds, `dist/` directory created. - -- [ ] **Step 8: Verify dev server** - -```bash -npm run dev -``` - -Expected: Starlight docs site loads at `http://localhost:4321/dbdeployer/` with the placeholder doc page. - -- [ ] **Step 9: Commit** - -```bash -cd /data/rene/dbdeployer -git add website/ -git commit -m "feat: scaffold Astro + Starlight website project" -``` - ---- - -## Task 2: Wiki Content Pipeline (copy-wiki.sh) - -**Files:** -- Create: `website/scripts/copy-wiki.sh` - -- [ ] **Step 1: Create the copy script** - -Create `website/scripts/copy-wiki.sh`: - -```bash -#!/bin/bash -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -WIKI_DIR="$REPO_ROOT/docs/wiki" -DOCS_DIR="$(cd "$(dirname "$0")/.." && pwd)/src/content/docs" - -# Clean previously copied docs (preserve new content written directly) -# Only clean directories that map from wiki -for dir in concepts deploying providers managing advanced reference; do - rm -rf "$DOCS_DIR/$dir" -done - -# Create target directories -for dir in getting-started concepts deploying providers managing advanced reference; do - mkdir -p "$DOCS_DIR/$dir" -done - -# Function to copy a wiki file with frontmatter and link rewriting -copy_wiki() { - local src="$1" - local dst="$2" - local title="$3" - - if [ ! -f "$src" ]; then - echo "WARNING: Source file not found: $src" - return - fi - - # Create frontmatter + content, strip wiki nav links, rewrite .md links - { - echo "---" - echo "title: \"$title\"" - echo "---" - echo "" - cat "$src" - } | sed '/\[\[HOME\]\]/d' \ - | sed 's/\[([^]]*)\](\([^)]*\)\.md)/[\1](\/dbdeployer\/docs\/\2\/)/g' \ - > "$dst" - - echo " Copied: $(basename "$src") -> $dst" -} - -echo "=== Copying wiki pages ===" - -# Getting Started -copy_wiki "$WIKI_DIR/installation.md" "$DOCS_DIR/getting-started/installation.md" "Installation" - -# Core Concepts -copy_wiki "$WIKI_DIR/default-sandbox.md" "$DOCS_DIR/concepts/sandboxes.md" "Sandboxes" -copy_wiki "$WIKI_DIR/database-server-flavors.md" "$DOCS_DIR/concepts/flavors.md" "Versions & Flavors" -copy_wiki "$WIKI_DIR/ports-management.md" "$DOCS_DIR/concepts/ports.md" "Ports & Networking" -copy_wiki "$REPO_ROOT/docs/env_variables.md" "$DOCS_DIR/concepts/environment-variables.md" "Environment Variables" - -# Deploying -copy_wiki "$WIKI_DIR/main-operations.md" "$DOCS_DIR/deploying/single.md" "Single Sandbox" -copy_wiki "$WIKI_DIR/multiple-sandboxes,-same-version-and-type.md" "$DOCS_DIR/deploying/multiple.md" "Multiple Sandboxes" -copy_wiki "$WIKI_DIR/replication-topologies.md" "$DOCS_DIR/deploying/replication.md" "Replication" - -# Providers -copy_wiki "$WIKI_DIR/standard-and-non-standard-basedir-names.md" "$DOCS_DIR/providers/mysql.md" "MySQL" -copy_wiki "$REPO_ROOT/docs/proxysql-guide.md" "$DOCS_DIR/providers/proxysql.md" "ProxySQL" - -# Managing Sandboxes -copy_wiki "$WIKI_DIR/sandbox-management.md" "$DOCS_DIR/managing/starting-stopping.md" "Starting & Stopping" -copy_wiki "$WIKI_DIR/using-the-latest-sandbox.md" "$DOCS_DIR/managing/using.md" "Using Sandboxes" -copy_wiki "$WIKI_DIR/sandbox-customization.md" "$DOCS_DIR/managing/customization.md" "Customization" -copy_wiki "$WIKI_DIR/database-users.md" "$DOCS_DIR/managing/users.md" "Database Users" -copy_wiki "$WIKI_DIR/database-logs-management..md" "$DOCS_DIR/managing/logs.md" "Logs" -copy_wiki "$WIKI_DIR/sandbox-deletion.md" "$DOCS_DIR/managing/deletion.md" "Deletion & Cleanup" - -# Advanced -copy_wiki "$WIKI_DIR/concurrent-deployment-and-deletion.md" "$DOCS_DIR/advanced/concurrent.md" "Concurrent Deployment" -copy_wiki "$WIKI_DIR/importing-databases-into-sandboxes.md" "$DOCS_DIR/advanced/importing.md" "Importing Databases" -copy_wiki "$WIKI_DIR/replication-between-sandboxes.md" "$DOCS_DIR/advanced/inter-sandbox-replication.md" "Inter-Sandbox Replication" -copy_wiki "$WIKI_DIR/cloning-databases.md" "$DOCS_DIR/advanced/cloning.md" "Cloning" -copy_wiki "$WIKI_DIR/using-dbdeployer-source-for-other-projects.md" "$DOCS_DIR/advanced/go-library.md" "Using as a Go Library" -copy_wiki "$WIKI_DIR/compiling-dbdeployer.md" "$DOCS_DIR/advanced/compiling.md" "Compiling from Source" - -# Reference -copy_wiki "$WIKI_DIR/command-line-completion.md" "$DOCS_DIR/reference/cli-commands.md" "CLI Commands" -copy_wiki "$WIKI_DIR/initializing-the-environment.md" "$DOCS_DIR/reference/configuration.md" "Configuration" - -echo "=== Done ===" -``` - -- [ ] **Step 2: Make executable and test** - -```bash -chmod +x website/scripts/copy-wiki.sh -bash website/scripts/copy-wiki.sh -``` - -Expected: Files copied with frontmatter into `website/src/content/docs/` subdirectories. - -- [ ] **Step 3: Verify build with copied docs** - -```bash -cd website && npm run build -``` - -Expected: Build succeeds. Some wiki pages may have broken internal links (acceptable for now — link rewriting is best-effort via sed). - -- [ ] **Step 4: Commit** - -```bash -git add website/scripts/copy-wiki.sh -git commit -m "feat: add wiki content pipeline script (copy-wiki.sh)" -``` - ---- - -## Task 3: Stub Docs for Sidebar Completeness - -Several sidebar entries need placeholder pages that don't come from the wiki (new content, extracted sections, or consolidated pages). Create stubs so the build doesn't break on missing slugs. - -**Files:** -- Create: Multiple stub .md files in `website/src/content/docs/` - -- [ ] **Step 1: Create stub pages** - -For each of these, create a minimal markdown file with frontmatter: - -```bash -# Deploying section (extracted from replication-topologies.md — write stubs for now) -cat > website/src/content/docs/deploying/group-replication.md << 'EOF' ---- -title: "Group Replication" ---- - -Content to be extracted from the replication topologies page. See [Replication](/dbdeployer/docs/deploying/replication/) for the full reference. -EOF - -cat > website/src/content/docs/deploying/fan-in-all-masters.md << 'EOF' ---- -title: "Fan-In & All-Masters" ---- - -Content to be extracted from the replication topologies page. See [Replication](/dbdeployer/docs/deploying/replication/) for the full reference. -EOF - -cat > website/src/content/docs/deploying/ndb-cluster.md << 'EOF' ---- -title: "NDB Cluster" ---- - -Content to be extracted from the replication topologies page. See [Replication](/dbdeployer/docs/deploying/replication/) for the full reference. -EOF - -# Providers section -cat > website/src/content/docs/providers/postgresql.md << 'EOF' ---- -title: "PostgreSQL" ---- - -PostgreSQL support is available starting with dbdeployer v1.75.0. - -## Binary Management - -PostgreSQL does not distribute pre-compiled tarballs. Use `.deb` packages: - -```bash -apt-get download postgresql-16 postgresql-client-16 -dbdeployer unpack --provider=postgresql postgresql-16_*.deb postgresql-client-16_*.deb -``` - -## Deploy a Single Sandbox - -```bash -dbdeployer deploy postgresql 16.13 -``` - -## Streaming Replication - -```bash -dbdeployer deploy replication 16.13 --provider=postgresql -``` - -## With ProxySQL - -```bash -dbdeployer deploy replication 16.13 --provider=postgresql --with-proxysql -``` -EOF - -cat > website/src/content/docs/providers/pxc.md << 'EOF' ---- -title: "Percona XtraDB Cluster" ---- - -Percona XtraDB Cluster (PXC) is deployed using the `pxc` topology within the MySQL provider. - -```bash -dbdeployer deploy replication 8.0.35 --topology=pxc -``` - -See [Replication](/dbdeployer/docs/deploying/replication/) for topology details. -EOF - -# Reference -cat > website/src/content/docs/reference/api-changelog.md << 'EOF' ---- -title: "API Changelog" ---- - -See the [full API history on GitHub](https://github.com/ProxySQL/dbdeployer/tree/master/docs/API) for all versions. -EOF -``` - -- [ ] **Step 2: Run copy script + build** - -```bash -bash website/scripts/copy-wiki.sh -cd website && npm run build -``` - -Expected: Build succeeds with all sidebar entries resolving. - -- [ ] **Step 3: Commit** - -```bash -git add website/src/content/docs/ -git commit -m "feat: add stub docs for sidebar completeness" -``` - ---- - -## Task 4: Getting Started Quickstart Guides - -**Files:** -- Create: `website/src/content/docs/getting-started/quickstart-mysql-single.md` -- Create: `website/src/content/docs/getting-started/quickstart-mysql-replication.md` -- Create: `website/src/content/docs/getting-started/quickstart-postgresql.md` -- Create: `website/src/content/docs/getting-started/quickstart-proxysql.md` - -These are **new content** — polished, tutorial-style, written fresh. Each should be short (under 50 lines), copy-pasteable, and satisfying in under 2 minutes. - -- [ ] **Step 1: Write MySQL Single quickstart** - -Create `website/src/content/docs/getting-started/quickstart-mysql-single.md`: - -```markdown ---- -title: "Quick Start: MySQL Single" -description: "Deploy a MySQL sandbox in 30 seconds" ---- - -## 1. Install dbdeployer - -```bash -# Download the latest release -curl -L https://github.com/ProxySQL/dbdeployer/releases/latest/download/dbdeployer-linux-amd64 -o dbdeployer -chmod +x dbdeployer -sudo mv dbdeployer /usr/local/bin/ -``` - -## 2. Get MySQL binaries - -```bash -dbdeployer downloads get-by-version 8.4 -``` - -## 3. Deploy - -```bash -dbdeployer deploy single 8.4.4 -``` - -## 4. Connect - -```bash -~/sandboxes/msb_8_4_4/use -``` - -You're now in a MySQL shell. Try: - -```sql -SELECT @@version; -SHOW DATABASES; -``` - -## 5. Clean up - -```bash -dbdeployer delete msb_8_4_4 -``` -``` - -- [ ] **Step 2: Write MySQL Replication quickstart** - -Create `website/src/content/docs/getting-started/quickstart-mysql-replication.md` — similar structure: deploy replication, show `./check_slaves`, connect to master/slave. - -- [ ] **Step 3: Write PostgreSQL quickstart** - -Create `website/src/content/docs/getting-started/quickstart-postgresql.md` — unpack debs, deploy, connect via psql, clean up. - -- [ ] **Step 4: Write ProxySQL Integration quickstart** - -Create `website/src/content/docs/getting-started/quickstart-proxysql.md` — deploy replication with `--with-proxysql`, connect through proxy, show hostgroup routing. - -- [ ] **Step 5: Verify build** - -```bash -bash website/scripts/copy-wiki.sh -cd website && npm run build -``` - -- [ ] **Step 6: Commit** - -```bash -git add website/src/content/docs/getting-started/ -git commit -m "feat: add getting started quickstart guides" -``` - ---- - -## Task 5: Landing Page - -**Files:** -- Create: `website/src/layouts/Landing.astro` -- Create: `website/src/components/Hero.astro` -- Create: `website/src/components/FeatureGrid.astro` -- Create: `website/src/components/Terminal.astro` -- Create: `website/src/styles/global.css` -- Create: `website/src/pages/index.astro` - -- [ ] **Step 1: Create Landing layout** - -`src/layouts/Landing.astro` — full HTML layout with nav, main slot, footer. Includes global CSS. Sets OG meta tags. - -- [ ] **Step 2: Create Hero component** - -`src/components/Hero.astro` — tagline, subtitle, two CTA buttons (Get Started, View on GitHub). - -- [ ] **Step 3: Create FeatureGrid component** - -`src/components/FeatureGrid.astro` — 4 feature cards: Any Topology, Multiple Databases, ProxySQL Integration, No Root/No Docker. - -- [ ] **Step 4: Create Terminal component** - -`src/components/Terminal.astro` — styled code block showing a deploy + connect flow. Static (not animated — keep it simple for v1). - -- [ ] **Step 5: Create global CSS** - -`src/styles/global.css` — CSS custom properties for colors, basic typography, responsive utilities. Keep minimal — Starlight handles docs styling. - -- [ ] **Step 6: Assemble landing page** - -`src/pages/index.astro` — imports Landing layout and all components. Includes: -1. Nav bar -2. Hero -3. Quick install snippet (`` block with copy button) -4. Feature grid -5. Terminal demo -6. Provider cards linking to `/providers` -7. Footer - -- [ ] **Step 7: Verify locally** - -```bash -cd website && npm run dev -``` - -Open `http://localhost:4321/dbdeployer/` — landing page should render with all sections. - -- [ ] **Step 8: Commit** - -```bash -git add website/src/layouts/ website/src/components/ website/src/styles/ website/src/pages/index.astro -git commit -m "feat: add marketing landing page with hero, features, and terminal demo" -``` - ---- - -## Task 6: Providers Page - -**Files:** -- Create: `website/src/components/ProviderCard.astro` -- Create: `website/src/pages/providers.astro` - -- [ ] **Step 1: Create ProviderCard component** - -`src/components/ProviderCard.astro` — accepts name, description, example command, docs link. Renders a card with code snippet. - -- [ ] **Step 2: Create providers page** - -`src/pages/providers.astro` — uses Landing layout. Contains: -1. Intro paragraph about provider architecture -2. HTML comparison matrix table (from spec) -3. Three ProviderCards (MySQL, PostgreSQL, ProxySQL) -4. "Coming Soon" teaser for Orchestrator - -- [ ] **Step 3: Verify locally** - -Visit `http://localhost:4321/dbdeployer/providers` — comparison matrix and cards render correctly. - -- [ ] **Step 4: Commit** - -```bash -git add website/src/components/ProviderCard.astro website/src/pages/providers.astro -git commit -m "feat: add providers comparison page" -``` - ---- - -## Task 7: Blog - -**Files:** -- Create: `website/src/layouts/BlogPost.astro` -- Create: `website/src/components/BlogPostCard.astro` -- Create: `website/src/pages/blog/index.astro` -- Create: `website/src/pages/blog/[...slug].astro` -- Create: `website/src/content/blog/2026-03-24-new-maintainership.md` -- Create: `website/src/content/blog/2026-03-24-postgresql-support.md` - -- [ ] **Step 1: Create BlogPost layout** - -`src/layouts/BlogPost.astro` — extends Landing layout. Adds article header (title, date, author), content slot, back-to-blog link. - -- [ ] **Step 2: Create BlogPostCard component** - -`src/components/BlogPostCard.astro` — title, date, description excerpt. Links to full post. - -- [ ] **Step 3: Create blog index page** - -`src/pages/blog/index.astro` — queries blog collection, renders BlogPostCards in reverse chronological order. - -- [ ] **Step 4: Create blog post dynamic route** - -`src/pages/blog/[...slug].astro` — renders individual blog posts using BlogPost layout. - -- [ ] **Step 5: Write launch blog posts** - -Create `src/content/blog/2026-03-24-new-maintainership.md`: - -```markdown ---- -title: "dbdeployer Under New Maintainership" -date: 2026-03-24 -author: "Rene Cannao" -description: "The ProxySQL team takes over dbdeployer with modern MySQL support, a provider architecture, and PostgreSQL on the horizon." -tags: ["announcement", "roadmap"] ---- - -dbdeployer is now maintained by the ProxySQL team... -``` - -Create `src/content/blog/2026-03-24-postgresql-support.md`: - -```markdown ---- -title: "PostgreSQL Support is Here" -date: 2026-03-24 -author: "Rene Cannao" -description: "dbdeployer now supports PostgreSQL sandboxes with streaming replication and ProxySQL integration." -tags: ["release", "postgresql"] ---- - -We're excited to announce PostgreSQL support in dbdeployer... -``` - -- [ ] **Step 6: Add "What's New" strip to landing page** - -Update `src/pages/index.astro` — query latest 2 blog posts, render BlogPostCards above the footer. - -- [ ] **Step 7: Verify locally** - -- `http://localhost:4321/dbdeployer/blog/` — index with 2 posts -- `http://localhost:4321/dbdeployer/blog/2026-03-24-new-maintainership/` — full post -- Landing page shows blog strip - -- [ ] **Step 8: Commit** - -```bash -git add website/src/layouts/BlogPost.astro website/src/components/BlogPostCard.astro website/src/pages/blog/ website/src/content/blog/ website/src/pages/index.astro -git commit -m "feat: add blog with launch posts and landing page integration" -``` - ---- - -## Task 8: 404 Page and OG Image - -**Files:** -- Create: `website/src/pages/404.astro` -- Create: `website/public/og-image.png` - -- [ ] **Step 1: Create 404 page** - -`src/pages/404.astro` — uses Landing layout. Simple message: "Page not found" with links to Home and Docs. - -- [ ] **Step 2: Create placeholder OG image** - -Create a simple 1200x630 PNG or use a placeholder. Can be generated with any tool or be a solid color with text overlay. This can be improved later. - -- [ ] **Step 3: Commit** - -```bash -git add website/src/pages/404.astro website/public/og-image.png -git commit -m "feat: add 404 page and OG social image" -``` - ---- - -## Task 9: GitHub Actions Deployment Workflow - -**Files:** -- Create: `.github/workflows/deploy-website.yml` - -- [ ] **Step 1: Create the workflow** - -```yaml -name: Deploy Website - -on: - push: - branches: [master] - paths: - - 'website/**' - - 'docs/wiki/**' - - 'docs/proxysql-guide.md' - - 'docs/env_variables.md' - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: website/package-lock.json - - - name: Install dependencies - working-directory: website - run: npm ci - - - name: Copy wiki content - run: bash website/scripts/copy-wiki.sh - - - name: Build website - working-directory: website - run: npm run build - - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: website/dist - - deploy: - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 -``` - -- [ ] **Step 2: Verify workflow syntax** - -```bash -# Check YAML is valid -python3 -c "import yaml; yaml.safe_load(open('.github/workflows/deploy-website.yml'))" -``` - -- [ ] **Step 3: Commit** - -```bash -git add .github/workflows/deploy-website.yml -git commit -m "ci: add GitHub Actions workflow for website deployment" -``` - ---- - -## Task 10: Final Build Verification and Cleanup - -- [ ] **Step 1: Full build from scratch** - -```bash -cd /data/rene/dbdeployer -bash website/scripts/copy-wiki.sh -cd website -rm -rf node_modules dist -npm install -npm run build -``` - -Expected: Clean build succeeds. - -- [ ] **Step 2: Verify all pages render** - -```bash -npm run preview -``` - -Spot-check: -- `/dbdeployer/` — landing page -- `/dbdeployer/docs/` — docs home -- `/dbdeployer/docs/getting-started/installation/` — wiki-migrated page -- `/dbdeployer/docs/getting-started/quickstart-mysql-single/` — new quickstart -- `/dbdeployer/providers` — providers comparison -- `/dbdeployer/blog/` — blog index - -- [ ] **Step 3: Add website/ to .gitignore entries** - -Add to `.gitignore`: -``` -website/node_modules/ -website/dist/ -website/.astro/ -``` - -- [ ] **Step 4: Final commit** - -```bash -git add .gitignore -git commit -m "chore: add website build artifacts to gitignore" -``` - ---- - -## Execution Notes - -### Task dependencies -- Task 1 (scaffold) must complete before all others -- Task 2 (copy script) must complete before Task 3 (stubs) -- Tasks 4-8 can run in parallel after Tasks 1-3 -- Task 9 (deployment) is independent of content tasks -- Task 10 (verification) runs last - -### Local development during implementation - -```bash -# Terminal 1: watch for changes -cd website && npm run dev - -# Terminal 2: re-run copy script after wiki edits -bash website/scripts/copy-wiki.sh -``` - -### After merging - -1. Enable GitHub Pages in repo settings: Settings → Pages → Source: GitHub Actions -2. First push to master with `website/**` changes triggers the deployment -3. Site available at `https://proxysql.github.io/dbdeployer/` diff --git a/docs/superpowers/specs/2026-03-24-admin-webui-poc-design.md b/docs/superpowers/specs/2026-03-24-admin-webui-poc-design.md deleted file mode 100644 index 3fa195f2..00000000 --- a/docs/superpowers/specs/2026-03-24-admin-webui-poc-design.md +++ /dev/null @@ -1,132 +0,0 @@ -# Admin Web UI POC Design - -**Date:** 2026-03-24 -**Author:** Rene (ProxySQL) -**Status:** POC - -## Goal - -Prove that dbdeployer can be a platform, not just a CLI. A `dbdeployer admin` command launches a localhost web dashboard showing all deployed sandboxes with start/stop/destroy controls. - -## Scope (POC only) - -- Dashboard showing all sandboxes as cards grouped by topology -- Start/stop/destroy actions via the UI -- OTP authentication (CLI generates token, browser validates) -- Localhost only (127.0.0.1) -- Go templates + HTMX, embedded in binary - -## NOT in scope (future) - -- Deploy new sandboxes via UI -- Real-time log streaming -- Topology graph visualization -- Multi-user / remote access -- Persistent sessions - -## Architecture - -``` -dbdeployer admin - └─ starts HTTP server on 127.0.0.1: - └─ generates OTP, prints to terminal - └─ opens browser to http://127.0.0.1:/login?token= - └─ serves embedded HTML templates via Go's html/template - └─ HTMX handles dynamic actions (no page reload for start/stop/destroy) - └─ API endpoints read sandbox catalog + execute lifecycle commands -``` - -### Authentication Flow - -1. `dbdeployer admin` generates a random OTP (32-char hex) -2. Prints: `Admin UI: http://127.0.0.1:9090/login?token=` -3. Browser hits `/login?token=` → server validates → sets session cookie -4. Session cookie used for all subsequent requests -5. OTP is single-use (invalidated after first login) -6. Session expires when server stops (in-memory) - -### API Endpoints - -| Method | Path | Description | -|--------|------|-------------| -| GET | `/login` | Validate OTP, set session cookie, redirect to dashboard | -| GET | `/` | Dashboard (HTML) | -| GET | `/api/sandboxes` | JSON list of all sandboxes | -| POST | `/api/sandboxes/:name/start` | Start a sandbox | -| POST | `/api/sandboxes/:name/stop` | Stop a sandbox | -| POST | `/api/sandboxes/:name/destroy` | Destroy a sandbox (requires confirmation) | - -### Dashboard Layout - -**Header:** "dbdeployer admin" + sandbox count + server uptime - -**Sandbox cards grouped by topology:** - -``` -┌─ Replication: rsandbox_8_4_4 ────────────────────────┐ -│ │ -│ ┌─ master ─────────┐ ┌─ node1 ──────────┐ │ -│ │ Port: 8404 │ │ Port: 8405 │ │ -│ │ ● Running │ │ ● Running │ │ -│ │ [Stop] │ │ [Stop] │ │ -│ └──────────────────┘ └──────────────────┘ │ -│ │ -│ ┌─ node2 ──────────┐ ┌─ proxysql ───────┐ │ -│ │ Port: 8406 │ │ Port: 6032/6033 │ │ -│ │ ● Running │ │ ● Running │ │ -│ │ [Stop] │ │ [Stop] │ │ -│ └──────────────────┘ └──────────────────┘ │ -│ │ -│ [Stop All] [Destroy] ──────────────────────────────│ -└────────────────────────────────────────────────────────┘ - -┌─ Single: msb_8_4_4 ──────────────────────────────────┐ -│ Port: 8404 │ ● Running │ [Stop] [Destroy] │ -└────────────────────────────────────────────────────────┘ -``` - -### Sandbox Data Source - -Read from `~/.dbdeployer/sandboxes.json` (the existing sandbox catalog). Each entry has: -- Sandbox name and directory -- Type (single, multiple, replication, group, etc.) -- Ports -- Nodes (for multi-node topologies) - -Status is determined by checking if the sandbox's PID file exists / process is running. - -### Technology - -- **Server:** Go `net/http` (stdlib, no framework) -- **Templates:** Go `html/template` with `//go:embed` -- **Interactivity:** HTMX (loaded from CDN or embedded) -- **Styling:** Inline CSS in the template (single file, dark theme matching the website) -- **Session:** In-memory map, cookie-based - -## File Structure - -``` -cmd/admin.go # Cobra command: dbdeployer admin -admin/ - server.go # HTTP server, routes, middleware - auth.go # OTP generation, session management - handlers.go # API handlers (list, start, stop, destroy) - sandbox_status.go # Read catalog, check process status - templates/ - layout.html # Base layout (head, nav, footer) - dashboard.html # Dashboard with sandbox cards - login.html # Login page (auto-submits with OTP) - components/ - sandbox-card.html # Single sandbox card partial - topology-group.html # Topology group wrapper partial - static/ - htmx.min.js # HTMX library (embedded) - style.css # Dashboard styles -``` - -All templates and static files embedded via `//go:embed admin/templates/* admin/static/*`. - -## Port Selection - -Default: 9090. If busy, find next free port. Print the URL to terminal. -Flag: `--port` to override. diff --git a/docs/superpowers/specs/2026-03-24-phase3-postgresql-provider-design.md b/docs/superpowers/specs/2026-03-24-phase3-postgresql-provider-design.md deleted file mode 100644 index cb562b88..00000000 --- a/docs/superpowers/specs/2026-03-24-phase3-postgresql-provider-design.md +++ /dev/null @@ -1,346 +0,0 @@ -# Phase 3 — PostgreSQL Provider Design - -**Date:** 2026-03-24 -**Author:** Rene (ProxySQL) -**Status:** Draft -**Prerequisite:** Phase 2b complete (provider interface, MySQL/ProxySQL providers) - -## Context - -dbdeployer's provider architecture (Phase 2) introduced a `Provider` interface with MySQL and ProxySQL implementations. Phase 3 validates that this architecture scales to a fundamentally different database system — PostgreSQL — where initialization, configuration, replication, and binary management all differ significantly from MySQL. - -**Primary motivation:** Enable ProxySQL protocol compatibility testing against PostgreSQL backends, and prove the provider model generalizes beyond MySQL-family databases. - -## Scope - -- PostgreSQL provider: binary management (deb extraction), single sandbox, lifecycle -- Streaming replication topology -- Cross-database topology constraints and validation -- ProxySQL + PostgreSQL backend wiring -- Unit tests from day one; integration tests written but CI-gated as manual - -## Provider Interface Changes - -Two methods added to the `Provider` interface: - -```go -type Provider interface { - // ... existing methods ... - - // SupportedTopologies returns which topology types this provider can deploy. - // The cmd layer validates against this before attempting deployment. - SupportedTopologies() []string - - // CreateReplica creates a replica from a running primary instance. - // Returns ErrNotSupported if the provider doesn't support replication. - // Called by the topology layer after the primary is started. - CreateReplica(primary SandboxInfo, config SandboxConfig) (*SandboxInfo, error) -} -``` - -**Per-provider topology support:** - -| Provider | Supported Topologies | -|------------|---------------------------------------------------------------------------| -| mysql | single, multiple, replication, group, fan-in, all-masters, ndb, pxc | -| proxysql | single | -| postgresql | single, multiple, replication | - -**MySQL provider** returns the full topology list from `SupportedTopologies()` — these topologies are served by the legacy `sandbox` package, not through the provider interface's `CreateSandbox`/`CreateReplica` methods. The topology list is accurate to what dbdeployer can deploy; it just flows through the old code path. `CreateReplica` returns `ErrNotSupported`. - -**ProxySQL provider** returns `["single"]` and `ErrNotSupported` from `CreateReplica`. - -**Binary resolution in `CreateReplica`:** The replica's `config.Version` is used to resolve binaries internally via `FindBinary(config.Version)`. This avoids needing to pass basedir through `SandboxInfo`. - -### Cleanup on Failure - -If `CreateSandbox` or `CreateReplica` fails partway through (e.g., initdb succeeds but config generation fails), the method cleans up its own sandbox directory before returning the error. The caller is not responsible for partial cleanup within a single sandbox. - -For multi-node replication topologies, if replica N fails, the topology layer is responsible for stopping and destroying the primary and any previously created replicas. This matches the existing MySQL behavior where partial topology failures trigger full cleanup. - -## Binary Management — Deb Extraction - -PostgreSQL does not distribute pre-compiled tarballs. Binaries are extracted from `.deb` packages. - -### Usage - -```bash -# User downloads debs (familiar apt workflow) -apt-get download postgresql-16 postgresql-client-16 - -# dbdeployer extracts and lays out binaries -dbdeployer unpack --provider=postgresql postgresql-16_16.13.deb postgresql-client-16_16.13.deb -``` - -### Extraction Flow - -1. Validate both debs are provided (server + client) -2. Extract each via `dpkg-deb -x` to a temp directory -3. Copy `usr/lib/postgresql/16/bin/` → `~/opt/postgresql/16.13/bin/` -4. Copy `usr/lib/postgresql/16/lib/` → `~/opt/postgresql/16.13/lib/` -5. Copy `usr/share/postgresql/16/` → `~/opt/postgresql/16.13/share/` -6. Validate required binaries exist: `postgres`, `initdb`, `pg_ctl`, `psql`, `pg_basebackup` -7. Clean up temp directory - -### Version Detection - -Extracted from deb filename pattern `postgresql-NN_X.Y-*`. Overridable via `--version=16.13`. - -### Target Layout - -``` -~/opt/postgresql/16.13/ - bin/ (postgres, initdb, pg_ctl, psql, pg_basebackup, pg_dump, ...) - lib/ (shared libraries) - share/ (timezone data, extension SQL — required by initdb) -``` - -**Implementation:** `providers/postgresql/unpack.go`, called from `cmd/unpack.go` when `--provider=postgresql`. - -## PostgreSQL Provider — Single Sandbox - -### Registration - -Same pattern as ProxySQL: `Register()` called from `cmd/root.go` init. - -### Port Allocation - -`DefaultPorts()` returns `{BasePort: 15000, PortsPerInstance: 1}`. - -Version-to-port formula: `BasePort + major * 100 + minor`. Examples: -- `16.13` → `15000 + 1600 + 13` = `16613` -- `16.3` → `15000 + 1600 + 3` = `16603` -- `17.1` → `15000 + 1700 + 1` = `16701` -- `17.10` → `15000 + 1700 + 10` = `16710` - -Single port per instance (PostgreSQL uses one port for all connections). - -### Version Validation - -`ValidateVersion()` accepts exactly `major.minor` format where both parts are integers. Major must be >= 12 (oldest supported PostgreSQL with streaming replication via `pg_basebackup -R`). Three-part versions like `16.13.1` are rejected (PostgreSQL does not use them). - -### FindBinary - -Looks in `~/opt/postgresql//bin/postgres`. Provider determines base path (`~/opt/postgresql/`); `--basedir` overrides for custom locations. - -### CreateSandbox Flow - -1. **Create log directory:** `mkdir -p /data/log` - -2. **Init database:** - ```bash - initdb -D /data --auth=trust --username=postgres - ``` - Note: `initdb` locates `share/` data relative to its own binary path (`../share/`). Since the extraction layout places `share/` as a sibling of `bin/`, no `-L` flag is needed. If the layout ever changes, `-L /share` can be added as a fallback. - -3. **Generate `postgresql.conf`:** - ``` - port = - listen_addresses = '127.0.0.1' - unix_socket_directories = '/data' - logging_collector = on - log_directory = '/data/log' - ``` - -4. **Generate `pg_hba.conf`** (overwrite initdb default): - ``` - local all all trust - host all all 127.0.0.1/32 trust - host all all ::1/128 trust - ``` - -5. **Write lifecycle scripts** (inline generation, like ProxySQL): - - `start` — `pg_ctl -D -l /postgresql.log start` - - `stop` — `pg_ctl -D stop -m fast` - - `status` — `pg_ctl -D status` - - `restart` — `pg_ctl -D -l /postgresql.log restart` - - `use` — `psql -h 127.0.0.1 -p -U postgres` - - `clear` — stop + remove data directory + re-init - -6. **Set environment in all scripts:** - - `LD_LIBRARY_PATH=/lib/` (extracted debs need this for shared libraries) - - Unset `PGDATA`, `PGPORT`, `PGHOST`, `PGUSER`, `PGDATABASE` to prevent environment contamination from the user's shell - -7. **Return `SandboxInfo`** with dir, port. `Socket` field is left empty (lifecycle scripts use TCP via `127.0.0.1`, matching the ProxySQL provider pattern). The unix socket exists at `/data/.s.PGSQL.` but is not the primary connection method. - -### Multiple Topology - -`dbdeployer deploy multiple 16.13 --provider=postgresql` creates N independent PostgreSQL instances using `CreateSandbox` with sequential port allocation. No additional configuration beyond what single provides — each instance is standalone with no replication relationship. - -## PostgreSQL Replication - -### CreateReplica Flow - -1. **No `initdb`** — replica data comes from the running primary via `pg_basebackup`: - ```bash - pg_basebackup -h 127.0.0.1 -p -U postgres -D /data -Fp -Xs -R - ``` - - `-Fp` = plain format - - `-Xs` = stream WAL during backup - - `-R` = auto-create `standby.signal` + write `primary_conninfo` to `postgresql.auto.conf` - -2. **Modify replica's `postgresql.conf`:** - - Change `port` to replica's assigned port - - Change `unix_socket_directories` to replica's sandbox dir - -3. **Write lifecycle scripts** — same as single sandbox with replica's port - -4. **Start replica** — `pg_ctl -D -l /postgresql.log start` - -### Primary-Side Configuration - -When replication is intended (`config.Options["replication"] = "true"`), `CreateSandbox` adds: - -**postgresql.conf:** -``` -wal_level = replica -max_wal_senders = 10 -hot_standby = on -``` - -**pg_hba.conf:** -``` -host replication all 127.0.0.1/32 trust -``` - -### Topology Layer Flow - -For `dbdeployer deploy replication 16.13 --provider=postgresql`: - -1. `CreateSandbox()` for primary with replication options -2. `StartSandbox()` for primary — **must be running before replicas** -3. For each replica: `CreateReplica(primaryInfo, replicaConfig)` — **sequential, not concurrent** -4. Each replica starts automatically as part of `CreateReplica` - -### Monitoring Scripts - -Generated in the topology directory: - -**`check_replication`** — connects to primary, shows connected replicas: -```bash -psql -h 127.0.0.1 -p -U postgres -c \ - "SELECT client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn FROM pg_stat_replication;" -``` - -**`check_recovery`** — connects to each replica, verifies standby status: -```bash -# For each replica: -psql -h 127.0.0.1 -p -U postgres -c "SELECT pg_is_in_recovery();" -``` - -## ProxySQL + PostgreSQL Wiring - -### Triggering - -```bash -dbdeployer deploy replication 16.13 --provider=postgresql --with-proxysql -``` - -### Config Generation - -ProxySQL supports PostgreSQL backends natively. The backend provider type is passed via: -```go -config.Options["backend_provider"] = "postgresql" -``` - -The ProxySQL config generator (`providers/proxysql/config.go`) branches: - -| Backend Provider | Config Blocks | -|------------------|-------------------------------------------------------| -| mysql (default) | `mysql_servers`, `mysql_users`, `mysql_variables` | -| postgresql | `pgsql_servers`, `pgsql_users`, `pgsql_variables` | - -### End-to-End Flow - -1. Deploy PostgreSQL primary + replicas (streaming replication) -2. Deploy ProxySQL with `pgsql_servers` pointing to primary (HG 0) + replicas (HG 1) -3. Generate `use_proxy` script: `psql -h 127.0.0.1 -p -U postgres` - -### Port Allocation - -ProxySQL admin port stays on its usual range (6032+). The frontend port uses the next consecutive port, same as today. The ProxySQL `ProxySQLConfig` struct's `MySQLPort` field is reused for the frontend listener port regardless of backend type — the field name is a misnomer but changing it would break the MySQL path. The `use_proxy` script uses `psql` instead of `mysql` when `backend_provider` is `postgresql`. - -## Cross-Database Topology Constraints - -### Topology Validation - -Cmd layer validates provider supports the requested topology before any sandbox creation: - -``` -$ dbdeployer deploy group 16.13 --provider=postgresql -Error: provider "postgresql" does not support topology "group" -Supported topologies: single, multiple, replication -``` - -### Flavor Validation - -`--flavor` is MySQL-specific. Rejected when `--provider` is not `mysql`: - -``` -$ dbdeployer deploy single 16.13 --provider=postgresql --flavor=ndb -Error: --flavor is only valid with --provider=mysql -``` - -### Cross-Provider Wiring Validation - -Compatibility map determines which addons work with which providers: - -```go -var compatibleAddons = map[string][]string{ - "proxysql": {"mysql", "postgresql"}, - // future: "orchestrator": {"mysql"}, -} -``` - -``` -$ dbdeployer deploy single 16.13 --provider=postgresql --with-orchestrator -Error: --with-orchestrator is not compatible with provider "postgresql" -``` - -## Testing Strategy - -### Unit Tests (no binaries needed) - -- `providers/postgresql/postgresql_test.go` — `ValidateVersion()`, `DefaultPorts()`, `SupportedTopologies()`, port calculation, config generation (postgresql.conf, pg_hba.conf), script generation -- `providers/postgresql/unpack_test.go` — deb filename parsing, version extraction, required binary validation -- `providers/proxysql/config_test.go` — extend for PostgreSQL backend config (`pgsql_servers`/`pgsql_users`) -- `providers/provider_test.go` — extend for topology validation, flavor rejection, cross-provider compatibility -- Cmd-level tests — `--provider=postgresql --flavor=ndb` errors, unsupported topologies error - -### Integration Tests (`//go:build integration`) - -`providers/postgresql/integration_test.go`: -- Single sandbox: initdb → start → connect via psql → stop → destroy -- Replication: primary + 2 replicas → verify `pg_stat_replication` shows 2 senders → verify `pg_is_in_recovery() = true` -- With ProxySQL: replication + proxysql → connect through ProxySQL → verify routing -- Deb extraction: unpack real .deb files → verify binary layout - -### CI Follow-Up (tracked as GitHub issues) - -1. Add PostgreSQL deb caching to CI pipeline -2. Add PostgreSQL integration tests to CI matrix -3. Nightly topology tests for PostgreSQL replication - -Integration tests run locally until CI is set up. - -## File Structure - -``` -providers/postgresql/ - postgresql.go # Provider implementation (CreateSandbox, CreateReplica, lifecycle) - unpack.go # Deb extraction logic - config.go # postgresql.conf and pg_hba.conf generation - postgresql_test.go # Unit tests - unpack_test.go # Deb extraction unit tests - integration_test.go # Integration tests (build-tagged) -``` - -Modifications to existing files: -- `providers/provider.go` — add `SupportedTopologies()`, `CreateReplica()` to interface -- `providers/mysql/mysql.go` — implement new interface methods (return full topology list, ErrNotSupported for CreateReplica) -- `providers/proxysql/proxysql.go` — implement new interface methods -- `providers/proxysql/config.go` — PostgreSQL backend config generation -- `cmd/root.go` — register PostgreSQL provider -- `cmd/single.go`, `cmd/multiple.go`, `cmd/replication.go` — `--provider` flag, topology validation -- `cmd/unpack.go` — `--provider` flag for deb extraction -- `globals/globals.go` — PostgreSQL constants, flag labels diff --git a/docs/superpowers/specs/2026-03-24-website-design.md b/docs/superpowers/specs/2026-03-24-website-design.md deleted file mode 100644 index 645d1030..00000000 --- a/docs/superpowers/specs/2026-03-24-website-design.md +++ /dev/null @@ -1,316 +0,0 @@ -# dbdeployer Website Design - -**Date:** 2026-03-24 -**Author:** Rene (ProxySQL) -**Status:** Draft - -## Context - -dbdeployer has rich documentation (44 wiki pages, 54 API versions, ProxySQL guide, PostgreSQL provider docs) but no proper website. The current setup is a default Jekyll theme on GitHub Pages rendering the README. The project is evolving from a MySQL-only sandbox tool into a multi-database infrastructure tool under ProxySQL's maintainership, and needs a web presence that reflects this. - -## Goals - -- **Primary audience:** MySQL/PostgreSQL developers searching for local sandbox/testing tools (SEO-first) -- **Secondary goal:** Introduce ProxySQL integration as a natural next step -- **Tone:** Documentation-focused with a commercial/marketing polish — not a corporate site, but professional enough to build confidence - -## Tech Stack - -- **Framework:** Astro with Starlight integration (Astro's official docs theme) -- **Why Starlight:** sidebar navigation, Pagefind search, dark/light mode, content collections, i18n-ready — all out of the box. Custom pages (landing, providers, blog) use standard Astro layouts outside Starlight. -- **Node.js:** 20 LTS (Astro 4.x requires Node 18.17+) -- **Hosting:** GitHub Pages -- **Deployment:** GitHub Actions → builds Astro → pushes to `gh-pages` branch -- **Base path:** `astro.config.mjs` must set `base: '/dbdeployer'` and `site: 'https://proxysql.github.io'` since this is a project repo (not org root) - -## Project Structure - -``` -website/ - astro.config.mjs - package.json - src/ - content/ - config.ts # Content collection schemas (docs + blog) - docs/ # Starlight docs (migrated wiki pages) - blog/ # Blog posts as .md files - pages/ - index.astro # Landing page (custom, not Starlight) - providers.astro # Providers comparison page - 404.astro # Custom 404 page (links back to home/docs) - blog/ - index.astro # Blog index (reverse-chronological list) - [...slug].astro # Individual blog post pages - components/ # Reusable Astro components (Hero, FeatureGrid, etc.) - layouts/ # Custom layouts for landing/blog - styles/ # Global CSS - public/ - favicon.svg # Site favicon - og-image.png # Default Open Graph image for social sharing - images/ # Screenshots, diagrams - scripts/ - copy-wiki.sh # Build step: copies docs/wiki/ into src/content/docs/ -``` - -Source lives in `website/` at the repo root. The `gh-pages` branch contains only the built output. - -## Site Sections - -### Home (Landing Page) - -Custom `index.astro` — not a Starlight page. Marketing-oriented. - -**Structure (top to bottom):** - -1. **Nav bar** — logo/name, links: Getting Started, Docs, Providers, Blog, GitHub -2. **Hero section:** - - Tagline: *"Deploy MySQL & PostgreSQL sandboxes in seconds"* - - Subtitle: *"Create single instances, replication topologies, and full testing stacks — locally, without root, without Docker"* - - CTAs: "Get Started" → quickstart guide, "View on GitHub" → repo -3. **Quick install snippet** — one-liner in a code block with copy button -4. **Feature grid** — 3-4 cards: - - "Any Topology" — single, replication, group replication, fan-in, all-masters - - "Multiple Databases" — MySQL, PostgreSQL, Percona, MariaDB - - "ProxySQL Integration" — deploy read/write split stacks in one command - - "No Root, No Docker" — runs entirely in userspace -5. **Terminal demo** — animated or static code block showing a deploy + connect flow -6. **Providers section** — brief cards for MySQL, PostgreSQL, ProxySQL linking to Providers page -7. **"What's New" strip** — latest 1-2 blog posts -8. **Footer** — links, GitHub, license - -### Getting Started - -Four polished, tutorial-style guides — **new content**, written fresh: - -1. **Quick Start: MySQL Single** — install, deploy, connect, destroy -2. **Quick Start: MySQL Replication** — deploy replication, check status, test failover -3. **Quick Start: PostgreSQL** — unpack debs, deploy, connect via psql -4. **Quick Start: ProxySQL Integration** — deploy replication with `--with-proxysql`, connect through proxy - -These are the hook — short, copy-pasteable, satisfying in under 2 minutes. - -### Docs - -The 44 existing wiki pages reorganized into a Starlight sidebar: - -``` -Getting Started - ├── Installation - ├── Quick Start: MySQL Single - ├── Quick Start: MySQL Replication - ├── Quick Start: PostgreSQL - └── Quick Start: ProxySQL Integration - -Core Concepts - ├── Sandboxes - ├── Versions & Flavors - ├── Ports & Networking - └── Environment Variables - -Deploying - ├── Single Sandbox - ├── Multiple Sandboxes - ├── Replication - ├── Group Replication - ├── Fan-In & All-Masters - └── NDB Cluster - -Providers - ├── MySQL - ├── PostgreSQL - ├── ProxySQL - └── Percona XtraDB Cluster - -Managing Sandboxes - ├── Starting & Stopping - ├── Using Sandboxes - ├── Customization - ├── Database Users - ├── Logs - └── Deletion & Cleanup - -Advanced - ├── Concurrent Deployment - ├── Importing Databases - ├── Inter-Sandbox Replication - ├── Cloning - ├── Using as a Go Library - └── Compiling from Source - -Reference - ├── CLI Commands - ├── Configuration - └── API Changelog -``` - -**Content strategy:** existing wiki markdown is kept mostly as-is. Navigation is restructured. Pages that don't fit are merged or dropped. Frontmatter is added/adjusted during the build copy step. - -### Providers Page - -Custom layout at `/providers` — the marketing angle for the provider architecture. - -**Structure:** - -1. **Intro** — dbdeployer's provider architecture, one CLI for multiple databases -2. **Comparison matrix:** - -| | MySQL | PostgreSQL | ProxySQL | -|---|---|---|---| -| Single sandbox | ✓ | ✓ | ✓ | -| Multiple sandboxes | ✓ | ✓ | — | -| Replication | ✓ | ✓ (streaming) | — | -| Group replication | ✓ | — | — | -| ProxySQL wiring | ✓ | ✓ | — | -| Binary source | Tarballs | .deb extraction | System binary | - -Note: MariaDB and Percona Server are MySQL-compatible flavors (same binary format, same provider) and are not listed as separate columns. The docs explain this under Providers > MySQL. - -3. **Per-provider cards** — description, example command, link to docs -4. **"Coming Soon" teaser** — Orchestrator integration (from roadmap) - -This is where ProxySQL gets introduced naturally — users browsing providers see the integration story. - -### Blog - -Content collection in `src/content/blog/`. Each post is a `.md` with frontmatter (title, date, author, tags, description). - -**Blog index** at `/blog` — reverse-chronological, custom layout. - -**Launch posts:** -1. "dbdeployer Under New Maintainership" — ProxySQL team story, what changed, roadmap -2. "PostgreSQL Support is Here" — Phase 3 announcement, examples - -**Home integration:** latest 1-2 posts shown in "What's New" strip above footer. - -## Docs Content Pipeline - -Wiki pages are authored in `docs/wiki/` (close to the Go code). A build script (`website/scripts/copy-wiki.sh`) copies them into Starlight's content collection with transformations. - -### Copy Script Responsibilities - -The script (`copy-wiki.sh`) runs before `npm run build` and does: - -1. **Copy files** from `docs/wiki/*.md` into `website/src/content/docs/
/` per the mapping table below -2. **Normalize filenames** — remove commas, double dots, convert to lowercase kebab-case -3. **Add Starlight frontmatter** — inject `title:` and `sidebar:` fields based on the mapping -4. **Rewrite links** — convert wiki-style links (`[text](other-page.md)`) to Starlight paths (`[text](/docs/
/other-page/)`) -5. **Strip wiki navigation** — remove `[[HOME]]`-style nav links (Starlight sidebar replaces these) -6. **Copy ProxySQL guide** — `docs/proxysql-guide.md` → `website/src/content/docs/providers/proxysql.md` - -### Wiki Page Mapping - -| Wiki File | Target Path | Sidebar Label | -|---|---|---| -| `installation.md` | `getting-started/installation` | Installation | -| *(new content)* | `getting-started/quickstart-mysql-single` | Quick Start: MySQL Single | -| *(new content)* | `getting-started/quickstart-mysql-replication` | Quick Start: MySQL Replication | -| *(new content)* | `getting-started/quickstart-postgresql` | Quick Start: PostgreSQL | -| *(new content)* | `getting-started/quickstart-proxysql` | Quick Start: ProxySQL Integration | -| `default-sandbox.md` | `concepts/sandboxes` | Sandboxes | -| `database-server-flavors.md` | `concepts/flavors` | Versions & Flavors | -| `ports-management.md` | `concepts/ports` | Ports & Networking | -| `../env_variables.md` | `concepts/environment-variables` | Environment Variables | -| `main-operations.md` | `deploying/single` | Single Sandbox | -| `multiple-sandboxes,-same-version-and-type.md` | `deploying/multiple` | Multiple Sandboxes | -| `replication-topologies.md` | `deploying/replication` | Replication | -| *(extract from replication-topologies.md)* | `deploying/group-replication` | Group Replication | -| *(extract from replication-topologies.md)* | `deploying/fan-in-all-masters` | Fan-In & All-Masters | -| *(extract from replication-topologies.md)* | `deploying/ndb-cluster` | NDB Cluster | -| `standard-and-non-standard-basedir-names.md` | `providers/mysql` | MySQL | -| *(new content)* | `providers/postgresql` | PostgreSQL | -| `../proxysql-guide.md` | `providers/proxysql` | ProxySQL | -| *(extract from replication-topologies.md)* | `providers/pxc` | Percona XtraDB Cluster | -| `skip-server-start.md` + `sandbox-management.md` | `managing/starting-stopping` | Starting & Stopping | -| `using-the-latest-sandbox.md` | `managing/using` | Using Sandboxes | -| `sandbox-customization.md` | `managing/customization` | Customization | -| `database-users.md` | `managing/users` | Database Users | -| `database-logs-management..md` | `managing/logs` | Logs | -| `sandbox-deletion.md` | `managing/deletion` | Deletion & Cleanup | -| `concurrent-deployment-and-deletion.md` | `advanced/concurrent` | Concurrent Deployment | -| `importing-databases-into-sandboxes.md` | `advanced/importing` | Importing Databases | -| `replication-between-sandboxes.md` | `advanced/inter-sandbox-replication` | Inter-Sandbox Replication | -| `cloning-databases.md` | `advanced/cloning` | Cloning | -| `using-dbdeployer-source-for-other-projects.md` | `advanced/go-library` | Using as a Go Library | -| `compiling-dbdeployer.md` | `advanced/compiling` | Compiling from Source | -| `command-line-completion.md` | `reference/cli-commands` | CLI Commands | -| `initializing-the-environment.md` | `reference/configuration` | Configuration | -| *(consolidated)* | `reference/api-changelog` | API Changelog | - -### Dropped/Merged Pages - -These wiki pages are NOT mapped to the sidebar (content merged into other pages or no longer relevant): - -| Wiki File | Disposition | -|---|---| -| `Home.md` | Replaced by landing page | -| `do-not-edit.md` | Internal tooling note, drop | -| `generating-additional-documentation.md` | Internal tooling, drop | -| `semantic-versioning.md` | Merge into Reference > Configuration | -| `practical-examples.md` | Content absorbed into quickstart guides | -| `sandbox-macro-operations.md` | Merge into Managing > Using Sandboxes | -| `sandbox-upgrade.md` | Merge into Managing > Using Sandboxes | -| `dedicated-admin-address.md` | Merge into Deploying > Single Sandbox | -| `running-sysbench.md` | Merge into Advanced > Importing (or drop) | -| `mysql-document-store,-mysqlsh,-and-defaults..md` | Merge into Providers > MySQL | -| `installing-mysql-shell.md` | Merge into Providers > MySQL | -| `loading-sample-data-into-sandboxes.md` | Merge into Advanced > Importing | -| `using-dbdeployer-in-scripts.md` | Merge into Advanced > Go Library | -| `using-short-version-numbers.md` | Merge into Concepts > Versions & Flavors | -| `using-the-direct-path-to-the-expanded-tarball.md` | Merge into Concepts > Versions & Flavors | -| `getting-remote-tarballs.md` | Merge into Getting Started > Installation | -| `updating-dbdeployer.md` | Merge into Getting Started > Installation | -| `obtaining-sandbox-metadata.md` | Merge into Managing > Using Sandboxes | -| `exporting-dbdeployer-structure.md` | Merge into Reference > CLI Commands | -| `dbdeployer-operations-logging.md` | Merge into Managing > Logs | - -### API Changelog Strategy - -The 54 API version files (`docs/API/API-1.0.md` through `docs/API/1.68.md`) are **not** published individually. Instead: - -- A single `reference/api-changelog.md` page is generated that consolidates the last 5 versions with full content -- Older versions link to the GitHub directory: "See [full API history on GitHub](https://github.com/ProxySQL/dbdeployer/tree/master/docs/API)" - -This keeps the sidebar clean and avoids 54 pages of version diffs. - -### Pipeline Summary - -- Docs live near the code (developers edit `docs/wiki/`) -- The website automatically picks up changes -- No manual sync between repo and site - -## Assets & Metadata - -### SEO & Social - -- **Favicon:** `public/favicon.svg` — simple dbdeployer logo/icon -- **OG image:** `public/og-image.png` — branded card (1200x630) with tagline, used as default `og:image` -- **Meta tags:** Starlight handles `` and `<meta description>` from frontmatter for docs pages. Custom pages (landing, providers, blog) set their own `<meta>` tags in `<head>` -- **Sitemap:** Astro's `@astrojs/sitemap` integration generates `sitemap.xml` automatically - -### Wiki Deprecation - -After the website launches, add a notice to the top of the GitHub wiki `Home.md` (if the wiki is still accessible): - -> "This wiki has moved to [proxysql.github.io/dbdeployer](https://proxysql.github.io/dbdeployer/docs/). These pages are no longer maintained." - -The wiki pages in `docs/wiki/` remain in the repo as the source of truth — they're just served through the website now. - -## Deployment - -**Workflow:** `.github/workflows/deploy-website.yml` - -Triggers: -- Push to `master` when `website/**` or `docs/wiki/**` change -- Manual `workflow_dispatch` - -Steps: -1. Checkout repo -2. Setup Node.js 20 LTS (`actions/setup-node` with `node-version: '20'`) -3. `npm ci` in `website/` -4. Run copy script: `bash website/scripts/copy-wiki.sh` — transforms and copies `docs/wiki/*.md` into `website/src/content/docs/` -5. `npm run build` -6. Deploy `dist/` to `gh-pages` branch via `actions/deploy-pages` - -**Site URL:** `proxysql.github.io/dbdeployer` (GitHub Pages default for org repos). Custom domain can be configured later. - -**GitHub Pages config:** Settings → Pages → Source: GitHub Actions. diff --git a/docs/wiki/database-server-flavors.md b/docs/wiki/database-server-flavors.md index 5c798e3e..bae89260 100644 --- a/docs/wiki/database-server-flavors.md +++ b/docs/wiki/database-server-flavors.md @@ -9,6 +9,7 @@ Before version 1.19.0, dbdeployer assumed that it was dealing to some version of * `pxc`: Percona Xtradb Cluster * `ndb`: MySQL Cluster (NDB) * `tidb`: A stand-alone TiDB server. +* `villagesql`: VillageSQL server, a MySQL drop-in replacement with extensions. It uses the same capabilities as MySQL and is detected by the presence of `share/villagesql_schema.sql` in the tarball. To see what every flavor can do, you can use the command `dbdeployer admin capabilities`. @@ -27,3 +28,73 @@ $ dbdeployer admin capabilities mysql 5.7.11 $ dbdeployer admin capabilities mysql 5.7.13 ``` +## Using dbdeployer with VillageSQL + +VillageSQL is a MySQL drop-in replacement with extensions (custom types, VDFs). dbdeployer supports it as a first-class flavor starting from version 2.2.2. + +### Download + +Download the VillageSQL tarball from [GitHub Releases](https://github.com/villagesql/villagesql-server/releases): + +```shell +curl -L -o villagesql-dev-server-0.0.3-dev-linux-x86_64.tar.gz \ + https://github.com/villagesql/villagesql-server/releases/download/0.0.3/villagesql-dev-server-0.0.3-dev-linux-x86_64.tar.gz +``` + +### Important: unpack with --unpack-version + +VillageSQL uses its own version scheme (`0.0.3`) which does not correspond to MySQL's version numbers. Since VillageSQL is built on MySQL 8.x, you must tell dbdeployer which MySQL version to use for capability lookups: + +```shell +dbdeployer unpack villagesql-dev-server-0.0.3-dev-linux-x86_64.tar.gz --unpack-version=8.0.40 +``` + +This maps VillageSQL to MySQL 8.0.40 capabilities (mysqld --initialize, CREATE USER, GTID, etc.), which is required for sandbox deployment to work. Without `--unpack-version`, dbdeployer would extract version `0.0.3`, which is below every MySQL capability threshold, resulting in a broken init script. + +You can verify the flavor was detected correctly: + +```shell +$ cat ~/opt/mysql/8.0.40/FLAVOR +villagesql +``` + +### Deploy a single sandbox + +```shell +dbdeployer deploy single 8.0.40 +~/sandboxes/msb_8_0_40/use -e "SELECT VERSION();" +# +-----------------------------------------+ +# | VERSION() | +# +-----------------------------------------+ +# | 8.4.8-villagesql-0.0.3-dev-78e24815 | +# +-----------------------------------------+ +``` + +### Deploy replication + +```shell +dbdeployer deploy replication 8.0.40 +~/sandboxes/rsandbox_8_0_40/test_replication +``` + +### Tarball symlink issue (0.0.3 only) + +The VillageSQL 0.0.3 tarball contains two symlinks that point outside the extraction directory: + +``` +mysql-test/suite/villagesql/examples/vsql-complex -> ../../../../villagesql/examples/vsql-complex/test +mysql-test/suite/villagesql/examples/vsql-tvector -> ../../../../villagesql/examples/vsql-tvector/test +``` + +dbdeployer's security check rejects these. If you encounter this error, remove the broken symlinks before unpacking: + +```shell +tar xzf villagesql-dev-server-0.0.3-dev-linux-x86_64.tar.gz +rm -f villagesql-dev-server-0.0.3-dev-linux-x86_64/mysql-test/suite/villagesql/examples/vsql-complex +rm -f villagesql-dev-server-0.0.3-dev-linux-x86_64/mysql-test/suite/villagesql/examples/vsql-tvector +tar czf villagesql-clean.tar.gz villagesql-dev-server-0.0.3-dev-linux-x86_64 +dbdeployer unpack villagesql-clean.tar.gz --unpack-version=8.0.40 +``` + +This issue is tracked at [villagesql/villagesql-server#237](https://github.com/villagesql/villagesql-server/issues/237). + diff --git a/downloads/remote_registry_test.go b/downloads/remote_registry_test.go index 44b730da..8f243b1f 100644 --- a/downloads/remote_registry_test.go +++ b/downloads/remote_registry_test.go @@ -19,6 +19,7 @@ import ( "reflect" "strings" "testing" + "time" "github.com/ProxySQL/dbdeployer/common" "github.com/ProxySQL/dbdeployer/compare" @@ -153,20 +154,33 @@ func TestTarballRegistry(t *testing.T) { // Allow a small number of transient failures without failing the test. maxAllowedFailures := 3 failures := 0 + transient403s := 0 for _, tarball := range DefaultTarballRegistry.Tarballs { - size, err := checkRemoteUrl(tarball.Url) + size, err := CheckRemoteUrl(tarball.Url) if err != nil { - failures++ - t.Logf("WARN - tarball %s check failed (%d/%d allowed): %s", tarball.Name, failures, maxAllowedFailures, err) + // HTTP 403 from MySQL CDN is rate-limiting, not a broken URL. + // Count separately and don't let it fail the test. + if strings.Contains(err.Error(), "received code 403") { + transient403s++ + t.Logf("WARN - tarball %s rate-limited by CDN (403): %s", tarball.Name, err) + } else { + failures++ + t.Logf("WARN - tarball %s check failed (%d/%d allowed): %s", tarball.Name, failures, maxAllowedFailures, err) + } } else { t.Logf("ok - tarball %s found", tarball.Name) if size == 0 { t.Logf("note - size 0 for tarball %s (size not recorded in registry)", tarball.Name) } } + // Small delay to avoid triggering CDN rate limits + time.Sleep(50 * time.Millisecond) } + if transient403s > 0 { + t.Logf("INFO: %d tarballs returned HTTP 403 (CDN rate limit) — not counted as failures", transient403s) + } if failures > maxAllowedFailures { t.Errorf("too many tarball URL failures: %d (max allowed: %d)", failures, maxAllowedFailures) } diff --git a/globals/globals.go b/globals/globals.go index e12d2668..e8accca9 100644 --- a/globals/globals.go +++ b/globals/globals.go @@ -475,6 +475,7 @@ const ( FnNdbdMtd = "ndbmtd" FnTableH = "table.h" FnTiDbServer = "tidb-server" + FnVillagesqlSchema = "villagesql_schema.sql" ) var AllowedTopologies = []string{