diff --git a/apps/cli-go/docs/supabase/db/diff.md b/apps/cli-go/docs/supabase/db/diff.md index 822c752485..0c0cf05a4d 100644 --- a/apps/cli-go/docs/supabase/db/diff.md +++ b/apps/cli-go/docs/supabase/db/diff.md @@ -10,6 +10,8 @@ By default, all schemas in the target database are diffed. Use the `--schema pub Projects created by a recent `supabase init` default to the pg-delta diff engine (`[experimental.pgdelta] enabled = true` in `config.toml`). Existing projects are unaffected and keep using migra unless they opt in. To fall back to the legacy migra engine, set `enabled = false` under `[experimental.pgdelta]`, or pass `--use-migra` for a single run. +With the pg-delta engine the diff SQL is formatted by default with the same settings the declarative export uses (uppercase keywords, wrapped at a max width of 180, indented and column-aligned); execution-aware transaction boundaries are preserved as per-unit header comments in the output. Configure overrides with `[experimental.pgdelta] format_options`, or set `format_options = "null"` to emit raw, unformatted statements. + While the diff command is able to capture most schema changes, there are cases where it is known to fail. Currently, this could happen if you schema contains: - Changes to publication diff --git a/apps/cli-go/docs/supabase/db/pull.md b/apps/cli-go/docs/supabase/db/pull.md index ff35019715..e10c0679f7 100644 --- a/apps/cli-go/docs/supabase/db/pull.md +++ b/apps/cli-go/docs/supabase/db/pull.md @@ -12,6 +12,10 @@ If no entries exist in the migration history table, the default diff engine uses Pass `--diff-engine pg-delta` to keep the migration-file `db pull` workflow while using pg-delta for the shadow diff step. On initial pull, pg-delta replaces `pg_dump` and produces the full migration from the shadow diff alone. Pass `--declarative` to switch to the declarative pg-delta export workflow instead. +pg-delta plans are execution-aware: when a plan crosses a transaction boundary — for example `ALTER TYPE ... ADD VALUE` followed by a statement that uses the new enum value, which cannot run in the same transaction — `db pull` writes one ordered migration file per plan unit instead of a single file (for example `_remote_schema_schema_changes.sql` and `_remote_schema_after_enum_values.sql`), each recorded in the migration history. The common case (a single unit) still produces exactly one `_remote_schema.sql` file. + +By default the emitted SQL is formatted with the same settings the declarative export uses (uppercase keywords, wrapped at a max width of 180, indented and column-aligned). Configure overrides with `[experimental.pgdelta] format_options` in `config.toml`, or set `format_options = "null"` to opt out and emit raw, unformatted statements. + When `[experimental.pgdelta] enabled = true` (the default for projects created by a recent `supabase init`), the migration-file `db pull` workflow uses pg-delta for the shadow diff step by default; it does not switch to declarative output. Existing projects without the section are unaffected and keep using migra. To fall back to the legacy migra engine, set `enabled = false` under `[experimental.pgdelta]`, or pass `--diff-engine migra` for a single run. When pulling from a remote database with `--db-url`, prefer a direct connection (`db..supabase.co:5432`) over the connection pooler so pg-delta can introspect the full catalog reliably. diff --git a/apps/cli-go/internal/db/declarative/declarative.go b/apps/cli-go/internal/db/declarative/declarative.go index c79df6289f..b84087bf9f 100644 --- a/apps/cli-go/internal/db/declarative/declarative.go +++ b/apps/cli-go/internal/db/declarative/declarative.go @@ -204,7 +204,7 @@ func SyncToMigrations(ctx context.Context, schema []string, file string, noCache if len(strings.TrimSpace(file)) == 0 { file = "declarative_sync" } - if err := diff.SaveDiff(result.DiffSQL, file, fsys); err != nil { + if err := diff.SaveDiff(diff.DatabaseDiff{SQL: result.DiffSQL}, file, fsys); err != nil { return err } if len(result.DropWarnings) > 0 { diff --git a/apps/cli-go/internal/db/diff/diff.go b/apps/cli-go/internal/db/diff/diff.go index 85c9c09c9c..32fabb2c21 100644 --- a/apps/cli-go/internal/db/diff/diff.go +++ b/apps/cli-go/internal/db/diff/diff.go @@ -38,7 +38,7 @@ func Run(ctx context.Context, schema []string, file string, config pgconn.Config out := result.SQL branch := utils.GetGitBranch(fsys) fmt.Fprintln(os.Stderr, "Finished "+utils.Aqua("supabase db diff")+" on branch "+utils.Aqua(branch)+".\n") - if err := SaveDiff(out, file, fsys); err != nil { + if err := SaveDiff(result, file, fsys); err != nil { return err } drops := findDropStatements(out) @@ -225,22 +225,31 @@ func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w } else { fmt.Fprintln(w, "Diffing schemas...") } - if IsPgDeltaDebugEnabled() && usePgDelta { - // Capture the shadow baseline catalog and edge-runtime stderr so an - // empty diff can be inspected later. DiffPgDeltaRefDetailed mirrors the - // pg-delta differ but additionally surfaces stderr, which differ() drops. - debugCapture := &PgDeltaDebugCapture{} - if snapshot, exportErr := exportCatalogPgDelta(ctx, utils.ToPostgresURL(shadowConfig), "postgres", options...); exportErr == nil { - debugCapture.SourceCatalog = snapshot - } else { - fmt.Fprintf(w, "Warning: failed to export shadow pg-delta catalog: %v\n", exportErr) + if usePgDelta { + // pg-delta always goes through the diffPgDeltaRefDetailed seam so callers get + // the execution-aware per-unit files (db pull writes one migration file each); + // db diff/declarative flatten them back via SQL. This mirrors the config-based + // differ (DiffPgDelta) exactly, so it is safe to bypass the injected differ() + // here — differ() remains the migra engine path below. + var debugCapture *PgDeltaDebugCapture + if IsPgDeltaDebugEnabled() { + // Capture the shadow baseline catalog and edge-runtime stderr so an + // empty diff can be inspected later. + debugCapture = &PgDeltaDebugCapture{} + if snapshot, exportErr := exportCatalogPgDelta(ctx, utils.ToPostgresURL(shadowConfig), "postgres", options...); exportErr == nil { + debugCapture.SourceCatalog = snapshot + } else { + fmt.Fprintf(w, "Warning: failed to export shadow pg-delta catalog: %v\n", exportErr) + } } - result, err := DiffPgDeltaRefDetailed(ctx, utils.ToPostgresURL(shadowConfig), utils.ToPostgresURL(config), schema, pgDeltaFormatOptions(), options...) + result, err := diffPgDeltaRefDetailed(ctx, utils.ToPostgresURL(shadowConfig), utils.ToPostgresURL(config), schema, pgDeltaFormatOptions(), options...) if err != nil { return DatabaseDiff{}, err } - debugCapture.Stderr = result.Stderr - return DatabaseDiff{SQL: result.SQL, Debug: debugCapture}, nil + if debugCapture != nil { + debugCapture.Stderr = result.Stderr + } + return DatabaseDiff{SQL: joinPgDeltaFiles(result.Files), Files: result.Files, Debug: debugCapture}, nil } output, err := differ(ctx, shadowConfig, config, schema, options...) if err != nil { diff --git a/apps/cli-go/internal/db/diff/diff_test.go b/apps/cli-go/internal/db/diff/diff_test.go index 92fd01164b..aff3242699 100644 --- a/apps/cli-go/internal/db/diff/diff_test.go +++ b/apps/cli-go/internal/db/diff/diff_test.go @@ -224,12 +224,23 @@ func TestRun(t *testing.T) { Reply("CREATE FUNCTION"). Query(tableSQL). Reply("CREATE TABLE") + // pg-delta bypasses the injected DiffFunc and runs the real edge-runtime + // pipeline, so stub the seam DiffDatabase uses (mirrors exportCatalogPgDelta). + // The migra differ must never be reached on this path. + originalDiffPgDelta := diffPgDeltaRefDetailed + t.Cleanup(func() { diffPgDeltaRefDetailed = originalDiffPgDelta }) diffCalled := false - differ := func(_ context.Context, _, target pgconn.Config, schema []string, _ ...func(*pgx.ConnConfig)) (string, error) { + diffPgDeltaRefDetailed = func(_ context.Context, _, targetRef string, schema []string, _ string, _ ...func(*pgx.ConnConfig)) (PgDeltaDiffResult, error) { diffCalled = true - assert.Equal(t, "contrib_regression", target.Database) + assert.Contains(t, targetRef, "contrib_regression") assert.Equal(t, []string{"public"}, schema) - return generated, nil + return PgDeltaDiffResult{ + Files: []PgDeltaPlanFile{{Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: generated}}, + }, nil + } + differ := func(context.Context, pgconn.Config, pgconn.Config, []string, ...func(*pgx.ConnConfig)) (string, error) { + t.Fatal("migra differ must not be called on the pg-delta path") + return "", nil } localConfig := pgconn.Config{ Host: utils.Config.Hostname, diff --git a/apps/cli-go/internal/db/diff/pgadmin.go b/apps/cli-go/internal/db/diff/pgadmin.go index d53cd4660e..9f1a692971 100644 --- a/apps/cli-go/internal/db/diff/pgadmin.go +++ b/apps/cli-go/internal/db/diff/pgadmin.go @@ -5,6 +5,7 @@ import ( _ "embed" "fmt" "os" + "time" "github.com/jackc/pgconn" "github.com/spf13/afero" @@ -17,13 +18,26 @@ import ( var warnDiff = `WARNING: The diff tool is not foolproof, so you may need to manually rearrange and modify the generated migration. Run ` + utils.Aqua("supabase db reset") + ` to verify that the new migration does not generate errors.` -func SaveDiff(out, file string, fsys afero.Fs) error { +func SaveDiff(result DatabaseDiff, file string, fsys afero.Fs) error { + out := result.SQL if len(out) < 2 { fmt.Fprintln(os.Stderr, "No schema changes found") } else if len(file) > 0 { - path := new.GetMigrationPath(utils.GetCurrentTimestamp(), file) - if err := utils.WriteFile(path, []byte(out), fsys); err != nil { - return err + // A pg-delta plan that crosses a transaction boundary yields more than one + // ordered unit; writing them into a single migration file would later fail + // when `db push`/`reset` applies it as one transaction. Write one migration + // file per unit in that case (Go's `WritePgDeltaMigrations`). The migra / + // pgadmin engines and single-unit pg-delta plans keep the exact single-file + // path, byte-identical to before. + if len(result.Files) > 1 { + if _, err := WritePgDeltaMigrations(result.Files, time.Now(), file, fsys); err != nil { + return err + } + } else { + path := new.GetMigrationPath(utils.GetCurrentTimestamp(), file) + if err := utils.WriteFile(path, []byte(out), fsys); err != nil { + return err + } } fmt.Fprintln(os.Stderr, warnDiff) } else { @@ -44,7 +58,7 @@ func RunPgAdmin(ctx context.Context, schema []string, file string, config pgconn return err } - return SaveDiff(output, file, fsys) + return SaveDiff(DatabaseDiff{SQL: output}, file, fsys) } var output string diff --git a/apps/cli-go/internal/db/diff/pgadmin_test.go b/apps/cli-go/internal/db/diff/pgadmin_test.go new file mode 100644 index 0000000000..7bf65e7d78 --- /dev/null +++ b/apps/cli-go/internal/db/diff/pgadmin_test.go @@ -0,0 +1,88 @@ +package diff + +import ( + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/utils" +) + +func TestSaveDiff(t *testing.T) { + t.Run("reports no changes on empty diff", func(t *testing.T) { + fsys := afero.NewMemMapFs() + require.NoError(t, SaveDiff(DatabaseDiff{SQL: ""}, "my_diff", fsys)) + // Nothing written when there are no schema changes. + entries, err := afero.ReadDir(fsys, utils.MigrationsDir) + assert.Error(t, err) + assert.Empty(t, entries) + }) + + t.Run("writes a single migration file for a single-unit plan", func(t *testing.T) { + fsys := afero.NewMemMapFs() + files := []PgDeltaPlanFile{ + {Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: "create table a ();"}, + } + result := DatabaseDiff{SQL: joinPgDeltaFiles(files), Files: files} + require.NoError(t, SaveDiff(result, "my_diff", fsys)) + entries, err := afero.ReadDir(fsys, utils.MigrationsDir) + require.NoError(t, err) + require.Len(t, entries, 1) + // A single-unit plan keeps the plain `_.sql` name and the exact + // diff SQL, byte-identical to the pre-multi-file behavior (no trailing newline + // added, no unit-name suffix). + assert.Regexp(t, `^\d{14}_my_diff\.sql$`, entries[0].Name()) + contents, err := afero.ReadFile(fsys, utils.MigrationsDir+"/"+entries[0].Name()) + require.NoError(t, err) + assert.Equal(t, "create table a ();", string(contents)) + }) + + t.Run("writes one migration file per unit for a multi-unit plan", func(t *testing.T) { + fsys := afero.NewMemMapFs() + files := []PgDeltaPlanFile{ + {Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: "alter type mood add value 'ok';"}, + {Order: 2, Name: "after_enum_values", TransactionMode: "transactional", SQL: "insert into t values ('ok');"}, + } + result := DatabaseDiff{SQL: joinPgDeltaFiles(files), Files: files} + require.NoError(t, SaveDiff(result, "my_diff", fsys)) + entries, err := afero.ReadDir(fsys, utils.MigrationsDir) + require.NoError(t, err) + require.Len(t, entries, 2) + // Multi-unit plans split into one ordered file per unit, each suffixed with the + // unit name, so `db push`/`reset` applies each unit as its own transaction. + assert.Regexp(t, `^\d{14}_my_diff_schema_changes\.sql$`, entries[0].Name()) + assert.Regexp(t, `^\d{14}_my_diff_after_enum_values\.sql$`, entries[1].Name()) + }) + + t.Run("prints diff to stdout when no file is given", func(t *testing.T) { + fsys := afero.NewMemMapFs() + require.NoError(t, SaveDiff(DatabaseDiff{SQL: "create table a ();"}, "", fsys)) + entries, _ := afero.ReadDir(fsys, utils.MigrationsDir) + assert.Empty(t, entries) + }) + + t.Run("creates nested parent directories for a nested single-unit name", func(t *testing.T) { + fsys := afero.NewMemMapFs() + require.NoError(t, SaveDiff(DatabaseDiff{SQL: "create table a ();"}, "snapshots/remote", fsys)) + matches, err := afero.Glob(fsys, utils.MigrationsDir+"/*_snapshots/remote.sql") + require.NoError(t, err) + require.Len(t, matches, 1) + contents, err := afero.ReadFile(fsys, matches[0]) + require.NoError(t, err) + assert.Equal(t, "create table a ();", string(contents)) + }) + + t.Run("creates nested parent directories for a nested multi-unit name", func(t *testing.T) { + fsys := afero.NewMemMapFs() + files := []PgDeltaPlanFile{ + {Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: "alter type mood add value 'ok';"}, + {Order: 2, Name: "after_enum_values", TransactionMode: "transactional", SQL: "insert into t values ('ok');"}, + } + result := DatabaseDiff{SQL: joinPgDeltaFiles(files), Files: files} + require.NoError(t, SaveDiff(result, "snapshots/remote", fsys)) + matches, err := afero.Glob(fsys, utils.MigrationsDir+"/*_snapshots/remote_*.sql") + require.NoError(t, err) + require.Len(t, matches, 2) + }) +} diff --git a/apps/cli-go/internal/db/diff/pgdelta.go b/apps/cli-go/internal/db/diff/pgdelta.go index d9d5edfc46..5267f0c6dd 100644 --- a/apps/cli-go/internal/db/diff/pgdelta.go +++ b/apps/cli-go/internal/db/diff/pgdelta.go @@ -43,6 +43,34 @@ type DeclarativeOutput struct { Files []DeclarativeFile `json:"files"` } +// PgDeltaPlanFile is one execution-aware migration unit rendered by pg-delta's +// renderPlanFiles: a numbered SQL file whose header comments record the unit +// number, transaction mode and boundary reason. +type PgDeltaPlanFile struct { + Order int `json:"order"` + Name string `json:"name"` + TransactionMode string `json:"transactionMode"` + SQL string `json:"sql"` +} + +// PgDeltaDiffOutput is the top-level diff envelope emitted by templates/pgdelta.ts. +type PgDeltaDiffOutput struct { + Version int `json:"version"` + Files []PgDeltaPlanFile `json:"files"` +} + +// joinPgDeltaFiles flattens the per-unit files back into a single SQL string for +// callers (db diff, declarative sync) that consume one blob. The per-unit header +// comments keep the transaction boundaries visible in the reviewed output; empty +// files produce an empty string, preserving "no changes" detection. +func joinPgDeltaFiles(files []PgDeltaPlanFile) string { + blocks := make([]string, len(files)) + for i, file := range files { + blocks[i] = file.SQL + } + return strings.Join(blocks, "\n\n") +} + func isPostgresURL(ref string) bool { return strings.HasPrefix(ref, "postgres://") || strings.HasPrefix(ref, "postgresql://") } @@ -101,7 +129,7 @@ func DiffPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []s if err != nil { return "", err } - return result.SQL, nil + return joinPgDeltaFiles(result.Files), nil } // DiffPgDeltaRefDetailed is like DiffPgDeltaRef but also returns edge-runtime stderr. @@ -136,15 +164,37 @@ func DiffPgDeltaRefDetailed(ctx context.Context, sourceRef, targetRef string, sc if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error diffing schema", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return PgDeltaDiffResult{}, err } - return PgDeltaDiffResult{ - SQL: stdout.String(), - Stderr: stderr.String(), - }, nil + return parsePgDeltaDiffOutput(stdout.String(), stderr.String()) +} + +// parsePgDeltaDiffOutput turns the pg-delta diff script's stdout envelope into a +// result. The template always prints the envelope on the success path, even for +// an empty plan (`{"version":1,"files":[]}`); a truly empty stdout means no +// envelope was produced, which we surface as "no changes" (empty Files) rather +// than an error. Non-empty stdout that is not valid envelope JSON is a parse +// error carrying the edge-runtime stderr for diagnosis. +func parsePgDeltaDiffOutput(stdout, stderr string) (PgDeltaDiffResult, error) { + result := PgDeltaDiffResult{Stderr: stderr} + if len(strings.TrimSpace(stdout)) == 0 { + return result, nil + } + var envelope PgDeltaDiffOutput + if err := json.Unmarshal([]byte(stdout), &envelope); err != nil { + return PgDeltaDiffResult{}, errors.Errorf("failed to parse pg-delta diff output: %w:\n%s", err, stderr) + } + result.Files = envelope.Files + return result, nil } // exportCatalogPgDelta is overridden in tests to mock catalog export. var exportCatalogPgDelta = ExportCatalogPgDelta +// diffPgDeltaRefDetailed is the seam DiffDatabase uses for the pg-delta engine. +// Tests override it to stub the real edge-runtime pipeline (which the injected +// DiffFunc differ cannot, since pg-delta bypasses differ), the same pattern as +// exportCatalogPgDelta above. +var diffPgDeltaRefDetailed = DiffPgDeltaRefDetailed + // DeclarativeExportPgDelta exports target schema as declarative file payloads // while keeping a config-based API for existing call sites. func DeclarativeExportPgDelta(ctx context.Context, source, target pgconn.Config, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (DeclarativeOutput, error) { diff --git a/apps/cli-go/internal/db/diff/pgdelta_debug.go b/apps/cli-go/internal/db/diff/pgdelta_debug.go index b901edd5f7..1439018e80 100644 --- a/apps/cli-go/internal/db/diff/pgdelta_debug.go +++ b/apps/cli-go/internal/db/diff/pgdelta_debug.go @@ -17,9 +17,10 @@ func IsPgDeltaDebugEnabled() bool { } } -// PgDeltaDiffResult holds pg-delta diff output and edge-runtime stderr. +// PgDeltaDiffResult holds the parsed pg-delta diff envelope (one file per +// execution-aware plan unit) and the edge-runtime stderr. type PgDeltaDiffResult struct { - SQL string + Files []PgDeltaPlanFile Stderr string } @@ -31,7 +32,10 @@ type PgDeltaDebugCapture struct { // DatabaseDiff is the result of diffing a target database against a shadow baseline. type DatabaseDiff struct { - SQL string + SQL string + // Files carries the per-unit pg-delta plan files (empty for the migra engine). + // SQL is the flattened join of these, kept for callers that consume one blob. + Files []PgDeltaPlanFile Debug *PgDeltaDebugCapture } diff --git a/apps/cli-go/internal/db/diff/pgdelta_migrations.go b/apps/cli-go/internal/db/diff/pgdelta_migrations.go new file mode 100644 index 0000000000..c8f4a8b244 --- /dev/null +++ b/apps/cli-go/internal/db/diff/pgdelta_migrations.go @@ -0,0 +1,108 @@ +package diff + +import ( + "os" + "path/filepath" + "time" + + "github.com/go-errors/errors" + "github.com/spf13/afero" + "github.com/supabase/cli/internal/migration/new" + "github.com/supabase/cli/internal/utils" +) + +// maxVersionCollisionAttempts bounds the base-timestamp bump retry so a directory +// already full of same-second migrations can't spin forever. +const maxVersionCollisionAttempts = 60 + +// WrittenMigration is a migration file produced by a diff/pull, paired with the +// version to record in the remote migration history. +type WrittenMigration struct { + Path string + Version string +} + +// WritePgDeltaMigrations writes one ordered migration file per plan unit. A +// single-unit plan (the common case) keeps the exact `_.sql` filename; +// multi-unit plans append the unit name and give each file a strictly increasing +// timestamp (real time arithmetic on the base, never string increment) so their +// execution order and migration-history order stay stable. +// +// Before writing anything, the FULL set of generated filenames is collision-checked +// against the filesystem: if any target path already exists the base is advanced by +// one second and every version recomputed, so the set stays strictly ascending AND +// unique against pre-existing migrations. The base only ever moves forward — never +// backdated below the caller's wall clock, since backdating could sort a new file +// before pre-existing migrations. The resulting ≤N−1s future-dating is inherent to +// second-granularity versions and acceptable once uniqueness is enforced. +func WritePgDeltaMigrations(files []PgDeltaPlanFile, base time.Time, name string, fsys afero.Fs) (_ []WrittenMigration, err error) { + single := len(files) == 1 + buildSet := func(b time.Time) []WrittenMigration { + set := make([]WrittenMigration, len(files)) + for i, file := range files { + version := utils.GetVersionTimestamp(b.Add(time.Duration(i) * time.Second)) + fileName := name + if !single { + fileName = name + "_" + file.Name + } + set[i] = WrittenMigration{Path: new.GetMigrationPath(version, fileName), Version: version} + } + return set + } + + set := buildSet(base) + for attempt := 0; ; attempt++ { + collision := false + for _, w := range set { + exists, err := afero.Exists(fsys, w.Path) + if err != nil { + return nil, errors.Errorf("failed to check migration file: %w", err) + } + if exists { + collision = true + break + } + } + if !collision { + break + } + if attempt+1 >= maxVersionCollisionAttempts { + return nil, errors.Errorf("failed to find a unique migration version after %d attempts", maxVersionCollisionAttempts) + } + base = base.Add(time.Second) + set = buildSet(base) + } + + written := make([]WrittenMigration, 0, len(files)) + // Best-effort cleanup: if any open/write fails mid-loop, remove every file this + // invocation already wrote so a partial multi-file migration isn't left behind. + // A removal failure never masks the original error. + defer func() { + if err != nil { + for _, w := range written { + _ = fsys.Remove(w.Path) + } + } + }() + for i, file := range files { + w := set[i] + if err = utils.MkdirIfNotExistFS(fsys, filepath.Dir(w.Path)); err != nil { + return nil, err + } + // O_EXCL (not O_TRUNC): a race that created the file between the collision + // check and here must never silently overwrite an existing migration. + f, openErr := fsys.OpenFile(w.Path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + if openErr != nil { + err = errors.Errorf("failed to open migration file: %w", openErr) + return nil, err + } + if _, writeErr := f.WriteString(file.SQL + "\n"); writeErr != nil { + f.Close() + err = errors.Errorf("failed to write migration file: %w", writeErr) + return nil, err + } + f.Close() + written = append(written, w) + } + return written, nil +} diff --git a/apps/cli-go/internal/db/diff/pgdelta_migrations_test.go b/apps/cli-go/internal/db/diff/pgdelta_migrations_test.go new file mode 100644 index 0000000000..16ace1b7ff --- /dev/null +++ b/apps/cli-go/internal/db/diff/pgdelta_migrations_test.go @@ -0,0 +1,133 @@ +package diff + +import ( + "os" + "testing" + "time" + + "github.com/go-errors/errors" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/migration/new" +) + +// failOnNthOpenFs fails the Nth create-for-write OpenFile so a mid-loop write +// failure can be exercised deterministically. Stat/mkdir/read calls pass through. +type failOnNthOpenFs struct { + afero.Fs + failOn int + count int +} + +func (f *failOnNthOpenFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + if flag&os.O_CREATE != 0 { + f.count++ + if f.count == f.failOn { + return nil, errors.New("simulated open failure") + } + } + return f.Fs.OpenFile(name, flag, perm) +} + +func TestWritePgDeltaMigrations(t *testing.T) { + base := time.Date(2026, 7, 17, 15, 18, 48, 0, time.UTC) + + t.Run("writes a single unit with the unchanged name", func(t *testing.T) { + fsys := afero.NewMemMapFs() + files := []PgDeltaPlanFile{ + {Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: "-- unit 1\n\ncreate table a ();"}, + } + written, err := WritePgDeltaMigrations(files, base, "remote_schema", fsys) + require.NoError(t, err) + require.Len(t, written, 1) + assert.Equal(t, "20260717151848", written[0].Version) + expectedPath := new.GetMigrationPath("20260717151848", "remote_schema") + assert.Equal(t, expectedPath, written[0].Path) + contents, err := afero.ReadFile(fsys, expectedPath) + require.NoError(t, err) + assert.Equal(t, "-- unit 1\n\ncreate table a ();\n", string(contents)) + }) + + t.Run("writes one ordered file per unit with strictly increasing versions", func(t *testing.T) { + fsys := afero.NewMemMapFs() + files := []PgDeltaPlanFile{ + {Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: "-- unit 1\n\nalter type mood add value 'ok';"}, + {Order: 2, Name: "after_enum_values", TransactionMode: "transactional", SQL: "-- unit 2\n\ninsert into t values ('ok');"}, + {Order: 3, Name: "non_transactional", TransactionMode: "none", SQL: "-- unit 3\n\ncreate index concurrently i on t (c);"}, + } + written, err := WritePgDeltaMigrations(files, base, "remote_schema", fsys) + require.NoError(t, err) + require.Len(t, written, 3) + + wantVersions := []string{"20260717151848", "20260717151849", "20260717151850"} + wantNames := []string{"remote_schema_schema_changes", "remote_schema_after_enum_values", "remote_schema_non_transactional"} + for i, w := range written { + assert.Equal(t, wantVersions[i], w.Version) + assert.Equal(t, new.GetMigrationPath(wantVersions[i], wantNames[i]), w.Path) + contents, err := afero.ReadFile(fsys, w.Path) + require.NoError(t, err) + assert.Equal(t, files[i].SQL+"\n", string(contents)) + } + // Versions are strictly increasing so history + execution order stay stable. + assert.True(t, written[0].Version < written[1].Version) + assert.True(t, written[1].Version < written[2].Version) + }) + + t.Run("creates nested parent directories for a nested migration name", func(t *testing.T) { + fsys := afero.NewMemMapFs() + files := []PgDeltaPlanFile{ + {Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: "create table a ();"}, + {Order: 2, Name: "after_enum_values", TransactionMode: "transactional", SQL: "insert into t values ('ok');"}, + } + written, err := WritePgDeltaMigrations(files, base, "snapshots/remote", fsys) + require.NoError(t, err) + require.Len(t, written, 2) + for i, w := range written { + contents, err := afero.ReadFile(fsys, w.Path) + require.NoError(t, err) + assert.Equal(t, files[i].SQL+"\n", string(contents)) + } + }) + + t.Run("bumps the base version when a target file already exists", func(t *testing.T) { + fsys := afero.NewMemMapFs() + files := []PgDeltaPlanFile{ + {Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: "create table a ();"}, + {Order: 2, Name: "after_enum_values", TransactionMode: "transactional", SQL: "insert into t values ('ok');"}, + } + // Pre-existing migration at the first version the base would otherwise use. + existing := new.GetMigrationPath("20260717151848", "remote_schema_schema_changes") + require.NoError(t, afero.WriteFile(fsys, existing, []byte("-- pre-existing\n"), 0644)) + + written, err := WritePgDeltaMigrations(files, base, "remote_schema", fsys) + require.NoError(t, err) + require.Len(t, written, 2) + // The whole set advances one second so it skips the colliding version and + // stays strictly ascending against the pre-existing file. + assert.Equal(t, "20260717151849", written[0].Version) + assert.Equal(t, "20260717151850", written[1].Version) + assert.True(t, written[0].Version < written[1].Version) + // The pre-existing file is untouched (never overwritten). + contents, err := afero.ReadFile(fsys, existing) + require.NoError(t, err) + assert.Equal(t, "-- pre-existing\n", string(contents)) + }) + + t.Run("removes already-written files when a later write fails", func(t *testing.T) { + fsys := &failOnNthOpenFs{Fs: afero.NewMemMapFs(), failOn: 2} + files := []PgDeltaPlanFile{ + {Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: "create table a ();"}, + {Order: 2, Name: "after_enum_values", TransactionMode: "transactional", SQL: "insert into t values ('ok');"}, + } + written, err := WritePgDeltaMigrations(files, base, "remote_schema", fsys) + require.Error(t, err) + assert.Nil(t, written) + // The first unit's file was written then removed on the failure, so nothing + // from this invocation is left behind. + first := new.GetMigrationPath("20260717151848", "remote_schema_schema_changes") + exists, statErr := afero.Exists(fsys, first) + require.NoError(t, statErr) + assert.False(t, exists) + }) +} diff --git a/apps/cli-go/internal/db/diff/pgdelta_test.go b/apps/cli-go/internal/db/diff/pgdelta_test.go index 671414a069..ad312273c5 100644 --- a/apps/cli-go/internal/db/diff/pgdelta_test.go +++ b/apps/cli-go/internal/db/diff/pgdelta_test.go @@ -32,3 +32,40 @@ func TestContainerRef(t *testing.T) { assert.Equal(t, "/workspace/supabase/.temp/pgdelta/catalog-baseline-17.6.1.106.json", containerRef(ref)) }) } + +func TestParsePgDeltaDiffOutput(t *testing.T) { + t.Run("parses a multi-file envelope", func(t *testing.T) { + stdout := `{"version":1,"files":[` + + `{"order":1,"name":"schema_changes","transactionMode":"transactional","sql":"-- unit 1\n\nCREATE TABLE a ();"},` + + `{"order":2,"name":"after_enum_values","transactionMode":"transactional","sql":"-- unit 2\n\nINSERT INTO a VALUES (1);"}` + + `]}` + result, err := parsePgDeltaDiffOutput(stdout, "debug stderr") + assert.NoError(t, err) + assert.Equal(t, "debug stderr", result.Stderr) + assert.Len(t, result.Files, 2) + assert.Equal(t, PgDeltaPlanFile{Order: 1, Name: "schema_changes", TransactionMode: "transactional", SQL: "-- unit 1\n\nCREATE TABLE a ();"}, result.Files[0]) + assert.Equal(t, "after_enum_values", result.Files[1].Name) + // The flattened join keeps unit boundaries visible via header comments. + assert.Equal(t, "-- unit 1\n\nCREATE TABLE a ();\n\n-- unit 2\n\nINSERT INTO a VALUES (1);", joinPgDeltaFiles(result.Files)) + }) + + t.Run("treats an empty envelope as no changes", func(t *testing.T) { + result, err := parsePgDeltaDiffOutput(`{"version":1,"files":[]}`, "") + assert.NoError(t, err) + assert.Empty(t, result.Files) + assert.Equal(t, "", joinPgDeltaFiles(result.Files)) + }) + + t.Run("treats empty stdout as no changes", func(t *testing.T) { + result, err := parsePgDeltaDiffOutput(" \n", "") + assert.NoError(t, err) + assert.Empty(t, result.Files) + }) + + t.Run("fails on malformed json and embeds stderr", func(t *testing.T) { + _, err := parsePgDeltaDiffOutput("not json", "boom on the edge runtime") + assert.Error(t, err) + assert.ErrorContains(t, err, "failed to parse pg-delta diff output") + assert.ErrorContains(t, err, "boom on the edge runtime") + }) +} diff --git a/apps/cli-go/internal/db/diff/templates/pgdelta.ts b/apps/cli-go/internal/db/diff/templates/pgdelta.ts index 0fd9d00e30..2a9d2d3f2a 100644 --- a/apps/cli-go/internal/db/diff/templates/pgdelta.ts +++ b/apps/cli-go/internal/db/diff/templates/pgdelta.ts @@ -1,7 +1,7 @@ import { createPlan, deserializeCatalog, - formatSqlStatements, + renderPlanFiles, } from "npm:@supabase/pg-delta@1.0.0-alpha.20"; import { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.20/integrations/supabase"; @@ -32,10 +32,19 @@ if (includedSchemas) { } const formatOptionsRaw = Deno.env.get("FORMAT_OPTIONS"); -let formatOptions = undefined; -if (formatOptionsRaw) { - formatOptions = JSON.parse(formatOptionsRaw); -} +const parsedFormatOptions = formatOptionsRaw ? JSON.parse(formatOptionsRaw) : undefined; +// Format the emitted SQL by default with the same sensible settings the +// declarative export uses (`exportDeclarativeSchema` in @supabase/pg-delta: +// `{ ...DEFAULT_OPTIONS, maxWidth: 180, keywordCase: "upper", ...userOptions }`), +// so `db pull` / `db diff` produce readable migrations even when config sets no +// `[experimental.pgdelta] format_options`. The formatter fills DEFAULT_OPTIONS +// for missing keys itself, so only the two overrides are passed here. Setting +// `format_options = "null"` (parsed to `null`) is the explicit opt-out: raw, +// unformatted statements, mirroring declarative export's `formatOptions === null`. +const sqlFormatOptions = + parsedFormatOptions === null + ? undefined + : { maxWidth: 180, keywordCase: "upper", ...parsedFormatOptions }; try { const result = await createPlan( @@ -46,14 +55,32 @@ try { skipDefaultPrivilegeSubtraction: true, }, ); - let statements = result?.plan.statements ?? []; - if (formatOptions != null) { - statements = formatSqlStatements(statements, formatOptions); - } + // pg-delta >= 1.0.0-alpha.32 groups plan statements into execution-aware + // `units` with transaction boundaries. `renderPlanFiles` turns those into one + // numbered SQL file per unit (header comments included). `includeTransactions: + // false` because the CLI appliers already wrap each migration file in a single + // transaction (Go's implicit ExecBatch / the TS BEGIN/COMMIT wrap), so embedded + // BEGIN/COMMIT would nest; format options are applied per unit here instead of a + // manual `formatSqlStatements` pass. + const files = result + ? renderPlanFiles(result.plan, { + includeTransactions: false, + sqlFormatOptions, + }) + : []; + const envelope = files.map((file, index) => ({ + order: index + 1, + // The unit name is the rendered path minus its numeric prefix and `.sql` + // extension (e.g. `001_after_enum_values.sql` -> `after_enum_values`). + name: file.path.replace(/^\d+_/, "").replace(/\.sql$/, ""), + transactionMode: file.unit.transactionMode, + sql: file.sql, + })); if (Deno.env.get("PGDELTA_DEBUG")) { console.error( JSON.stringify({ - statementCount: statements.length, + statementCount: files.reduce((total, file) => total + file.unit.statements.length, 0), + fileCount: files.length, source: source ? "connected" : "null", target: target ? "connected" : "null", includedSchemas: includedSchemas ?? null, @@ -61,11 +88,13 @@ try { }), ); } - for (const sql of statements) { - console.log(`${sql};`); - } + console.log(JSON.stringify({ version: 1, files: envelope })); } catch (e) { console.error(e); + // Emit a sentinel so the CLI runner can distinguish a real script crash from a + // successful empty diff, even though the forced-exit non-zero code below is + // suppressed by the "main worker has been destroyed" handling. + console.error("PGDELTA_SCRIPT_ERROR"); // Force close event loop throw new Error(""); } diff --git a/apps/cli-go/internal/db/diff/templates/pgdelta_catalog_export.ts b/apps/cli-go/internal/db/diff/templates/pgdelta_catalog_export.ts index 6b7d426ce1..dfecc58da1 100644 --- a/apps/cli-go/internal/db/diff/templates/pgdelta_catalog_export.ts +++ b/apps/cli-go/internal/db/diff/templates/pgdelta_catalog_export.ts @@ -12,6 +12,10 @@ const role = Deno.env.get("ROLE") ?? undefined; if (!target) { console.error("TARGET is required"); + // Emit a sentinel so the CLI runner treats this as a real script crash rather + // than a successful empty catalog, even though the forced-exit non-zero code is + // suppressed by the "main worker has been destroyed" handling. + console.error("PGDELTA_SCRIPT_ERROR"); throw new Error(""); } const { pool, close } = await createManagedPool(target, { role }); @@ -21,6 +25,10 @@ try { console.log(stringifyCatalogSnapshot(serializeCatalog(catalog))); } catch (e) { console.error(e); + // Emit a sentinel so the CLI runner can distinguish a real script crash from a + // successful empty catalog, even though the forced-exit non-zero code below is + // suppressed by the "main worker has been destroyed" handling. + console.error("PGDELTA_SCRIPT_ERROR"); // Force close event loop throw new Error(""); } finally { diff --git a/apps/cli-go/internal/db/diff/templates/pgdelta_declarative_export.ts b/apps/cli-go/internal/db/diff/templates/pgdelta_declarative_export.ts index 4656e4690e..18820ff7b9 100644 --- a/apps/cli-go/internal/db/diff/templates/pgdelta_declarative_export.ts +++ b/apps/cli-go/internal/db/diff/templates/pgdelta_declarative_export.ts @@ -68,6 +68,10 @@ try { } } catch (e) { console.error(e); + // Emit a sentinel so the CLI runner can distinguish a real script crash from a + // successful empty export, even though the forced-exit non-zero code below is + // suppressed by the "main worker has been destroyed" handling. + console.error("PGDELTA_SCRIPT_ERROR"); // Force close event loop throw new Error(""); } diff --git a/apps/cli-go/internal/db/pull/pull.go b/apps/cli-go/internal/db/pull/pull.go index 07503687a7..21e0db87f0 100644 --- a/apps/cli-go/internal/db/pull/pull.go +++ b/apps/cli-go/internal/db/pull/pull.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/go-errors/errors" "github.com/jackc/pgconn" @@ -59,21 +60,26 @@ func Run(ctx context.Context, schema []string, config pgconn.Config, name string // TODO: handle managed schemas return format.WriteStructuredSchemas(ctx, &buf, fsys) } - // 2. Pull schema - timestamp := utils.GetCurrentTimestamp() - path := new.GetMigrationPath(timestamp, name) - if err := run(ctx, schema, path, conn, usePgDeltaDiff, differ, fsys, options...); err != nil { + // 2. Pull schema. pg-delta plans with transaction boundaries produce more than + // one ordered migration file; migra always produces exactly one. + base := time.Now().UTC() + written, err := run(ctx, schema, base, name, conn, usePgDeltaDiff, differ, fsys, options...) + if err != nil { return err } - if err := ensureMigrationWritten(fsys, path); err != nil { - return err + if len(written) == 0 { + return errors.New(errInSync) + } + // 3. Insert a row to `schema_migrations` for every file written. + versions := make([]string, len(written)) + for i, w := range written { + fmt.Fprintln(os.Stderr, "Schema written to "+utils.Bold(w.Path)) + versions[i] = w.Version } - // 3. Insert a row to `schema_migrations` - fmt.Fprintln(os.Stderr, "Schema written to "+utils.Bold(path)) if shouldUpdate, err := utils.NewConsole().PromptYesNo(ctx, "Update remote migration history table?", true); err != nil { return err } else if shouldUpdate { - return repair.UpdateMigrationTable(ctx, conn, []string{timestamp}, repair.Applied, false, fsys) + return repair.UpdateMigrationTable(ctx, conn, versions, repair.Applied, false, fsys) } return nil } @@ -114,8 +120,10 @@ func pullDeclarativePgDelta(ctx context.Context, schema []string, config pgconn. return nil } -func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { +func run(ctx context.Context, schema []string, base time.Time, name string, conn *pgx.Conn, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) ([]diff.WrittenMigration, error) { config := conn.Config().Config + timestamp := utils.GetVersionTimestamp(base) + path := new.GetMigrationPath(timestamp, name) // 1. Assert `supabase/migrations` and `schema_migrations` are in sync. if err := assertRemoteInSync(ctx, conn, fsys); errors.Is(err, errMissing) { // pg_dump strips ownership when restored as a non-superuser, so platform @@ -126,19 +134,27 @@ func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, useP if !usePgDeltaDiff { // Ignore schemas flag when working on the initial pull if err = dumpRemoteSchema(ctx, path, config, fsys); err != nil { - return err + return nil, err } } // For the legacy path this is a second pass that captures changes // pg_dump cannot emit (default privileges, managed schemas). For the // pg-delta path this is the only pass and produces the full schema. - err = swallowInitialInSync(diffRemoteSchema(ctx, nil, path, config, usePgDeltaDiff, differ, fsys, options...), fsys, path) - return err + written, err := diffRemoteSchema(ctx, nil, base, name, config, usePgDeltaDiff, differ, fsys, options...) + if err = swallowInitialInSync(err, fsys, path); err != nil { + return nil, err + } + // The migra initial pull seeds `path` with a pg_dump even when the follow-up + // diff is empty and swallowed above, so record that single migration. + if !usePgDeltaDiff && len(written) == 0 { + written = []diff.WrittenMigration{{Path: path, Version: timestamp}} + } + return written, nil } else if err != nil { - return err + return nil, err } // 2. Fetch remote schema changes - return diffRemoteSchema(ctx, schema, path, config, usePgDeltaDiff, differ, fsys, options...) + return diffRemoteSchema(ctx, schema, base, name, config, usePgDeltaDiff, differ, fsys, options...) } func dumpRemoteSchema(ctx context.Context, path string, config pgconn.Config, fsys afero.Fs) error { @@ -157,7 +173,7 @@ func dumpRemoteSchema(ctx context.Context, path string, config pgconn.Config, fs }) } -func diffRemoteSchema(ctx context.Context, schema []string, path string, config pgconn.Config, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { +func diffRemoteSchema(ctx context.Context, schema []string, base time.Time, name string, config pgconn.Config, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) ([]diff.WrittenMigration, error) { // Diff remote db (source) & shadow db (target) and write it as a new migration. result, err := diff.DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDeltaDiff, options...) if err != nil { @@ -166,37 +182,47 @@ func diffRemoteSchema(ctx context.Context, schema []string, path string, config // so the whole db pull workflow is self-healing, not just the dump pass. poolerConfig, ok := dump.PoolerFallbackConfig(ctx, config, err) if !ok { - return err + return nil, err } if result, err = diff.DiffDatabase(ctx, schema, poolerConfig, os.Stderr, fsys, differ, usePgDeltaDiff, options...); err != nil { - return err + return nil, err } } - output := result.SQL - if trimmed := strings.TrimSpace(output); len(trimmed) == 0 { - if usePgDeltaDiff && diff.IsPgDeltaDebugEnabled() { - if debugDir, debugErr := saveEmptyPgDeltaPullDebug(ctx, config, result.Debug, fsys, options...); debugErr != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to save pg-delta debug bundle: %v\n", debugErr) - } else if len(debugDir) > 0 { - return errors.Errorf("%w (debug bundle: %s)", errInSync, debugDir) + // pg-delta path: one migration file per execution-aware plan unit. + if usePgDeltaDiff { + if len(result.Files) == 0 { + if diff.IsPgDeltaDebugEnabled() { + if debugDir, debugErr := saveEmptyPgDeltaPullDebug(ctx, config, result.Debug, fsys, options...); debugErr != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to save pg-delta debug bundle: %v\n", debugErr) + } else if len(debugDir) > 0 { + return nil, errors.Errorf("%w (debug bundle: %s)", errInSync, debugDir) + } } + return nil, errors.New(errInSync) } - return errors.New(errInSync) + return diff.WritePgDeltaMigrations(result.Files, base, name, fsys) } + // migra path: a single migration file, appended when seeded by dumpRemoteSchema. + output := result.SQL + if trimmed := strings.TrimSpace(output); len(trimmed) == 0 { + return nil, errors.New(errInSync) + } + timestamp := utils.GetVersionTimestamp(base) + path := new.GetMigrationPath(timestamp, name) if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(path)); err != nil { - return err + return nil, err } // Append to existing migration file when we run this after dumpRemoteSchema; - // for the pg-delta path this is the only writer and creates the file fresh. + // for a non-initial pull this creates the file fresh. f, err := fsys.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) if err != nil { - return errors.Errorf("failed to open migration file: %w", err) + return nil, errors.Errorf("failed to open migration file: %w", err) } defer f.Close() if _, err := f.WriteString(output); err != nil { - return errors.Errorf("failed to write migration file: %w", err) + return nil, errors.Errorf("failed to write migration file: %w", err) } - return nil + return []diff.WrittenMigration{{Path: path, Version: timestamp}}, nil } func assertRemoteInSync(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error { diff --git a/apps/cli-go/internal/db/pull/pull_test.go b/apps/cli-go/internal/db/pull/pull_test.go index e9fbae9198..ec3e784f83 100644 --- a/apps/cli-go/internal/db/pull/pull_test.go +++ b/apps/cli-go/internal/db/pull/pull_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/h2non/gock" "github.com/jackc/pgconn" @@ -14,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/supabase/cli/internal/db/diff" + "github.com/supabase/cli/internal/migration/new" "github.com/supabase/cli/internal/testing/apitest" "github.com/supabase/cli/internal/testing/fstest" "github.com/supabase/cli/internal/utils" @@ -75,11 +77,13 @@ func TestPullSchema(t *testing.T) { conn.Query(migration.LIST_MIGRATION_VERSION). Reply("SELECT 0") // Run test - err := run(context.Background(), nil, "0_test.sql", conn.MockClient(t), false, diff.DiffSchemaMigra, fsys) + base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + path := new.GetMigrationPath(utils.GetVersionTimestamp(base), "test") + _, err := run(context.Background(), nil, base, "test", conn.MockClient(t), false, diff.DiffSchemaMigra, fsys) // Check error assert.ErrorIs(t, err, errNetwork) assert.Empty(t, apitest.ListUnmatchedRequests()) - contents, err := afero.ReadFile(fsys, "0_test.sql") + contents, err := afero.ReadFile(fsys, path) assert.NoError(t, err) assert.Equal(t, []byte("test"), contents) }) @@ -102,12 +106,14 @@ func TestPullSchema(t *testing.T) { conn.Query(migration.LIST_MIGRATION_VERSION). Reply("SELECT 0") // Run test with usePgDeltaDiff=true - err := run(context.Background(), nil, "0_test.sql", conn.MockClient(t), true, diff.DiffPgDelta, fsys) + base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + path := new.GetMigrationPath(utils.GetVersionTimestamp(base), "test") + _, err := run(context.Background(), nil, base, "test", conn.MockClient(t), true, diff.DiffPgDelta, fsys) // Failure must come from shadow-creation image inspect (proving we // reached the diff step), not from pg_dump. assert.ErrorIs(t, err, errNetwork) assert.Empty(t, apitest.ListUnmatchedRequests()) - exists, err := afero.Exists(fsys, "0_test.sql") + exists, err := afero.Exists(fsys, path) assert.NoError(t, err) assert.False(t, exists, "pg_dump should be skipped for pg-delta diff engine") }) @@ -129,7 +135,8 @@ func TestPullSchema(t *testing.T) { conn.Query(migration.LIST_MIGRATION_VERSION). Reply("SELECT 1", []any{"0"}) // Run test - err := run(context.Background(), []string{"public"}, "", conn.MockClient(t), false, diff.DiffSchemaMigra, fsys) + base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + _, err := run(context.Background(), []string{"public"}, base, "test", conn.MockClient(t), false, diff.DiffSchemaMigra, fsys) // Check error assert.ErrorContains(t, err, "network error") assert.Empty(t, apitest.ListUnmatchedRequests()) diff --git a/apps/cli-go/internal/utils/edgeruntime.go b/apps/cli-go/internal/utils/edgeruntime.go index c81169ea6f..8e54afa628 100644 --- a/apps/cli-go/internal/utils/edgeruntime.go +++ b/apps/cli-go/internal/utils/edgeruntime.go @@ -13,6 +13,18 @@ import ( "github.com/spf13/viper" ) +// EdgeRuntimeScriptErrorSentinel is printed to stderr by the pg-delta Deno +// templates when their body throws (see the `catch` blocks in +// internal/db/diff/templates/*.ts). The templates force the edge-runtime worker +// to exit by throwing on both the success and failure paths, and that non-zero +// exit is otherwise suppressed here when stderr contains "main worker has been +// destroyed". Without a distinct marker a crashed script would be +// indistinguishable from a successful empty diff, so `db pull` would report "No +// schema changes found" while the real error (e.g. a permission-denied catalog +// query) was silently swallowed. Templates and tests must reference this exact +// string. See supabase/cli#5826. +const EdgeRuntimeScriptErrorSentinel = "PGDELTA_SCRIPT_ERROR" + // edgeRuntimeFile is a single file dropped into the edge-runtime container's // working directory before the configured command is run. type edgeRuntimeFile struct { @@ -119,6 +131,14 @@ func RunEdgeRuntimeScript(ctx context.Context, env []string, script string, bind ); err != nil && !strings.Contains(stderr.String(), "main worker has been destroyed") { return errors.Errorf("%s: %w:\n%s", errPrefix, err, stderr.String()) } + // The templates suppress their own non-zero exit (they throw to force the + // worker to exit, which surfaces as "main worker has been destroyed"), so a + // script crash can slip past the check above. Treat the sentinel — printed + // only by the templates' catch blocks — as a hard failure so the real error, + // collected in stderr, reaches the user instead of looking like an empty diff. + if strings.Contains(stderr.String(), EdgeRuntimeScriptErrorSentinel) { + return errors.Errorf("%s: error running script:\n%s", errPrefix, stderr.String()) + } return nil } diff --git a/apps/cli-go/internal/utils/edgeruntime_test.go b/apps/cli-go/internal/utils/edgeruntime_test.go index 471a518ee6..2497630255 100644 --- a/apps/cli-go/internal/utils/edgeruntime_test.go +++ b/apps/cli-go/internal/utils/edgeruntime_test.go @@ -1,14 +1,97 @@ package utils import ( + "bytes" + "context" + "io" + "net/http" "strconv" "strings" "testing" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/stdcopy" + "github.com/h2non/gock" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/testing/apitest" ) +// mockEdgeRuntimeLogs registers the docker responses RunEdgeRuntimeScript needs: +// a one-shot log read multiplexing stdout+stderr, an inspect reporting the exit +// code, and the container delete. Mirrors apitest.MockDockerErrorLogs but also +// carries stdout so we can assert the success path preserves the script output. +func mockEdgeRuntimeLogs(t *testing.T, containerID, stdout, stderr string, exitCode int) { + t.Helper() + var body bytes.Buffer + if len(stdout) > 0 { + _, err := io.Copy(stdcopy.NewStdWriter(&body, stdcopy.Stdout), strings.NewReader(stdout)) + require.NoError(t, err) + } + if len(stderr) > 0 { + _, err := io.Copy(stdcopy.NewStdWriter(&body, stdcopy.Stderr), strings.NewReader(stderr)) + require.NoError(t, err) + } + gock.New(Docker.DaemonHost()). + Get("/v"+Docker.ClientVersion()+"/containers/"+containerID+"/logs"). + Reply(http.StatusOK). + SetHeader("Content-Type", "application/vnd.docker.raw-stream"). + Body(&body) + gock.New(Docker.DaemonHost()). + Get("/v" + Docker.ClientVersion() + "/containers/" + containerID + "/json"). + Reply(http.StatusOK). + JSON(container.InspectResponse{ContainerJSONBase: &container.ContainerJSONBase{ + State: &container.State{ExitCode: exitCode}, + }}) + gock.New(Docker.DaemonHost()). + Delete("/v" + Docker.ClientVersion() + "/containers/" + containerID). + Reply(http.StatusOK) +} + +func TestRunEdgeRuntimeScript(t *testing.T) { + const containerID = "test-edge-runtime" + imageUrl := GetRegistryImageUrl(Config.EdgeRuntime.Image) + + t.Run("surfaces the real error when the script crashes behind the worker-destroyed message", func(t *testing.T) { + viper.Set("INTERNAL_IMAGE_REGISTRY", "docker.io") + t.Cleanup(func() { viper.Set("INTERNAL_IMAGE_REGISTRY", "") }) + require.NoError(t, apitest.MockDocker(Docker)) + defer gock.OffAll() + apitest.MockDockerStart(Docker, imageUrl, containerID) + // The pg-delta template throws to force the worker to exit (surfacing as a + // non-zero exit + "main worker has been destroyed"), and its catch block + // prints the real error and the sentinel. This must NOT look like an empty diff. + stderr := "error: permission denied for table pg_user_mapping\n" + + EdgeRuntimeScriptErrorSentinel + "\n" + + "worker boot error\nmain worker has been destroyed\n" + mockEdgeRuntimeLogs(t, containerID, "", stderr, 1) + + var stdout, stderrBuf bytes.Buffer + err := RunEdgeRuntimeScript(context.Background(), nil, "console.log('x')", nil, "error diffing schema", &stdout, &stderrBuf) + require.Error(t, err) + assert.Contains(t, err.Error(), "error diffing schema: error running script:") + // The real, actionable error must reach the user, not "No schema changes found". + assert.Contains(t, err.Error(), "permission denied for table pg_user_mapping") + }) + + t.Run("still ignores a worker-destroyed exit when no sentinel is present", func(t *testing.T) { + viper.Set("INTERNAL_IMAGE_REGISTRY", "docker.io") + t.Cleanup(func() { viper.Set("INTERNAL_IMAGE_REGISTRY", "") }) + require.NoError(t, apitest.MockDocker(Docker)) + defer gock.OffAll() + apitest.MockDockerStart(Docker, imageUrl, containerID) + // Success path: the template forces the worker to exit after writing output, + // so the exit is non-zero with "main worker has been destroyed" but no sentinel. + mockEdgeRuntimeLogs(t, containerID, "ALTER TABLE x;\n", "main worker has been destroyed\n", 1) + + var stdout, stderrBuf bytes.Buffer + err := RunEdgeRuntimeScript(context.Background(), nil, "console.log('x')", nil, "error diffing schema", &stdout, &stderrBuf) + require.NoError(t, err) + assert.Equal(t, "ALTER TABLE x;\n", stdout.String()) + }) +} + func TestBuildEdgeRuntimeEntrypoint(t *testing.T) { t.Run("emits a single heredoc when only the script is provided", func(t *testing.T) { got := buildEdgeRuntimeEntrypoint( diff --git a/apps/cli-go/internal/utils/misc.go b/apps/cli-go/internal/utils/misc.go index 8e9870531b..3faf385966 100644 --- a/apps/cli-go/internal/utils/misc.go +++ b/apps/cli-go/internal/utils/misc.go @@ -129,7 +129,14 @@ func IsPgDeltaEnabled() bool { func GetCurrentTimestamp() string { // Magic number: https://stackoverflow.com/q/45160822. - return time.Now().UTC().Format(layoutVersion) + return GetVersionTimestamp(time.Now()) +} + +// GetVersionTimestamp formats t as a migration version (UTC `YYYYMMDDHHMMSS`). +// Callers that write several ordered migration files in one pass add real time +// offsets to a shared base rather than incrementing the formatted string. +func GetVersionTimestamp(t time.Time) string { + return t.UTC().Format(layoutVersion) } func GetCurrentBranchFS(fsys afero.Fs) (string, error) { diff --git a/apps/cli-go/pkg/config/pgdelta_version.go b/apps/cli-go/pkg/config/pgdelta_version.go index 26be07f8a0..a38641ae7c 100644 --- a/apps/cli-go/pkg/config/pgdelta_version.go +++ b/apps/cli-go/pkg/config/pgdelta_version.go @@ -4,7 +4,7 @@ import "strings" // DefaultPgDeltaNpmVersion is the npm dist-tag/version used for @supabase/pg-delta // when supabase/.temp/pgdelta-version is absent or empty. -const DefaultPgDeltaNpmVersion = "1.0.0-alpha.27" +const DefaultPgDeltaNpmVersion = "1.0.0-alpha.32" const pgDeltaNpmVersionPlaceholder = "1.0.0-alpha.20" diff --git a/apps/cli/src/legacy/commands/db/diff/diff.handler.ts b/apps/cli/src/legacy/commands/db/diff/diff.handler.ts index c7dcdb5fad..72bc7c2e58 100644 --- a/apps/cli/src/legacy/commands/db/diff/diff.handler.ts +++ b/apps/cli/src/legacy/commands/db/diff/diff.handler.ts @@ -27,6 +27,7 @@ import { legacyGetMigrationPath, } from "../../../shared/legacy-migration-file.ts"; import { legacyDiffMigra } from "../shared/legacy-migra.ts"; +import { legacyWritePgDeltaMigrations } from "../shared/legacy-pgdelta-migrations.write.ts"; import { type LegacyPgDeltaContext, legacyDiffPgDelta } from "../shared/legacy-pgdelta.ts"; import { LegacyDeclarativeSeam } from "../shared/legacy-pgdelta.seam.service.ts"; import type { LegacyDbDiffFlags } from "./diff.command.ts"; @@ -406,7 +407,7 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy projectRef: connType === "linked" ? linkedRef : undefined, }); - const out = yield* Effect.gen(function* () { + const diffResult = yield* Effect.gen(function* () { const target = shadow.targetUrlOverride ?? targetUrl; yield* output.raw( flags.schema.length > 0 @@ -421,15 +422,22 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy schema: flags.schema, formatOptions, }); - return result.sql; + // Keep the per-unit plan files so a multi-unit plan can be written as one + // migration file each (Go's `DatabaseDiff.Files`); `sql` stays the flattened + // join for stdout review + machine payloads. + return { sql: result.sql, files: result.files }; } - return yield* legacyDiffMigra(ctx, { + const sql = yield* legacyDiffMigra(ctx, { source: shadow.sourceUrl, target, schema: flags.schema, connectOptions: { isLocal: resolved.isLocal, dnsResolver }, }); + // The migra engine has no execution-aware plan units, so it always writes a + // single migration file (Go's `SaveDiff` single-file path). + return { sql, files: undefined }; }).pipe(Effect.ensuring(seam.removeShadowContainer(shadow.container))); + const out = diffResult.sql; // Detect the branch from the resolved workdir, not the caller's CWD: Go // chdirs into --workdir in PersistentPreRunE before GetGitBranch @@ -444,27 +452,46 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy // Go's `SaveDiff` (`pgadmin.go:20`) + the drop-statement warning (`diff.go:44`). const engine = useDelta ? "pg-delta" : "migra"; const drops = legacyFindDropStatements(out); - let writtenFile: string | null = null; + const writtenFiles: Array = []; if (out.length < 2) { yield* output.raw("No schema changes found\n", "stderr"); // Go's `SaveDiff` gates the file write on `len(file) > 0` (`pgadmin.go`), so // an empty `--file=""` (e.g. an unset shell var) falls through to stdout // rather than writing a `_.sql` migration with no name. } else if (Option.isSome(flags.file) && flags.file.value.length > 0) { - const timestamp = legacyFormatMigrationTimestamp(yield* Clock.currentTimeMillis); - const migrationPath = legacyGetMigrationPath( - path, - cliConfig.workdir, - timestamp, - flags.file.value, - ); - yield* legacyMakeDir(fs, path.dirname(migrationPath)).pipe( - Effect.mapError((cause) => new LegacyDbDiffWriteError({ message: cause.message })), - ); - yield* fs - .writeFileString(migrationPath, out) - .pipe(Effect.mapError((cause) => new LegacyDbDiffWriteError({ message: cause.message }))); - writtenFile = migrationPath; + const fileName = flags.file.value; + // A pg-delta plan that crosses a transaction boundary yields more than one + // ordered unit; writing them into a single migration file would later fail + // when `db push`/`reset` applies it as one transaction. Write one migration + // file per unit in that case via the shared writer (Go's + // `WritePgDeltaMigrations`, `internal/db/diff/pgdelta_migrations.go`): each + // file appends the unit name and gets a strictly increasing timestamp, the + // full set is collision-checked against existing migrations, and every file is + // written exclusively so a pre-existing migration is never overwritten. A + // single-unit plan (and the migra engine) keeps the exact `_.sql` + // file (Go's `utils.WriteFile`), byte-identical to before. + const planFiles = diffResult.files ?? []; + if (planFiles.length > 1) { + const writtenUnits = yield* legacyWritePgDeltaMigrations(fs, path, { + workdir: cliConfig.workdir, + baseMillis: yield* Clock.currentTimeMillis, + name: fileName, + files: planFiles.map((file) => ({ name: file.name, sql: file.sql })), + }).pipe(Effect.mapError((cause) => new LegacyDbDiffWriteError({ message: cause.message }))); + for (const unit of writtenUnits) writtenFiles.push(unit.path); + } else { + const timestamp = legacyFormatMigrationTimestamp(yield* Clock.currentTimeMillis); + const migrationPath = legacyGetMigrationPath(path, cliConfig.workdir, timestamp, fileName); + // Create parent dirs per written path (mirroring Go's `utils.WriteFile`), so a + // nested `--file snapshots/remote` name creates `_snapshots/` first. + yield* legacyMakeDir(fs, path.dirname(migrationPath)).pipe( + Effect.mapError((cause) => new LegacyDbDiffWriteError({ message: cause.message })), + ); + yield* fs + .writeFileString(migrationPath, out) + .pipe(Effect.mapError((cause) => new LegacyDbDiffWriteError({ message: cause.message }))); + writtenFiles.push(migrationPath); + } yield* output.raw(`${warnDiff}\n`, "stderr"); } else if (output.format === "text") { yield* output.raw(`${out}\n`); @@ -479,7 +506,12 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy if (output.format !== "text") { yield* output.success("Diff complete.", { diff: out, - file: writtenFile, + // `file` keeps the first written path for released consumers that read the + // string field (null when nothing was written); `files` lists EVERY written + // migration path in write order (a pg-delta plan writes one file per unit), + // mirroring pull's `schemaFiles` so machine callers see all of them. + file: writtenFiles[0] ?? null, + files: writtenFiles, schemas: flags.schema, engine, dropStatements: drops, diff --git a/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts b/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts index 0278e7e41e..f4e6fb4ea6 100644 --- a/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts @@ -1,10 +1,11 @@ -import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, Layer, Option } from "effect"; import { + legacyFailWriteStringOnNthCallFsLayer, mockLegacyCliConfig, mockLegacyLinkedProjectCacheTracked, mockLegacyTelemetryStateTracked, @@ -35,10 +36,15 @@ interface SetupOpts { readonly isLocal?: boolean; readonly linkedRef?: string; readonly diffSql?: string; + // When set, the pg-delta edge mock emits a multi-unit plan envelope (one file + // per entry) instead of the single-unit wrap of `diffSql`. + readonly diffFiles?: ReadonlyArray<{ readonly name: string; readonly sql: string }>; readonly targetOverride?: string; readonly oom?: boolean; // edge-runtime OOMs; the bash fallback returns `diffSql` readonly delegateStdout?: string; // stdout returned by a captured Go-delegate run readonly networkId?: string; // --network-id value forwarded to docker runs + // When set, the Nth `writeFileString` fails, exercising cleanup-on-failure. + readonly failWriteOnCall?: number; } function setup(workdir: string, opts: SetupOpts = {}) { @@ -87,7 +93,28 @@ function setup(workdir: string, opts: SetupOpts = {}) { new LegacyEdgeRuntimeScriptError({ message: "Fatal JavaScript out of memory" }), ); } - return Effect.succeed({ stdout: opts.diffSql ?? "", stderr: "" }); + const diffSql = opts.diffSql ?? ""; + // The pg-delta diff script (uniquely identified by `renderPlanFiles`) prints a + // JSON envelope with one file per plan unit; wrap the test's raw SQL into a + // single-unit envelope so `legacyDiffPgDelta` parses it. The migra script + // returns raw SQL unchanged. + const isPgDelta = runOpts.script.includes("renderPlanFiles"); + const planFiles = + opts.diffFiles !== undefined + ? opts.diffFiles.map((file, i) => ({ + order: i + 1, + name: file.name, + transactionMode: "transactional", + sql: file.sql, + })) + : diffSql.length > 0 + ? [{ order: 1, name: "schema_changes", transactionMode: "transactional", sql: diffSql }] + : []; + const stdout = + isPgDelta && planFiles.length > 0 + ? JSON.stringify({ version: 1, files: planFiles }) + : diffSql; + return Effect.succeed({ stdout, stderr: "" }); }, }); @@ -141,7 +168,7 @@ function setup(workdir: string, opts: SetupOpts = {}) { }), }); - const layer = Layer.mergeAll( + const baseLayer = Layer.mergeAll( out.layer, telemetry.layer, cache.layer, @@ -164,6 +191,12 @@ function setup(workdir: string, opts: SetupOpts = {}) { mockRuntimeInfo(), BunServices.layer, ); + // Merged last so its `FileSystem` overrides `BunServices` (last-wins); `Path` + // still resolves from `BunServices`. + const layer = + opts.failWriteOnCall === undefined + ? baseLayer + : Layer.merge(baseLayer, legacyFailWriteStringOnNthCallFsLayer(opts.failWriteOnCall)); return { layer, @@ -438,6 +471,122 @@ describe("legacy db diff", () => { }).pipe(Effect.provide(s.layer)); }); + it.effect("writes one migration file per unit for a multi-unit pg-delta plan", () => { + // A pg-delta plan that crosses a transaction boundary yields more than one + // ordered unit; writing them into one migration would fail when db push/reset + // applies it as a single transaction. Each unit becomes its own file (Go's + // WritePgDeltaMigrations), named `_` with strictly increasing + // timestamps, and the machine payload's `files` lists them all. + const s = setup(tmp.current, { + format: "json", + diffFiles: [ + { name: "schema_changes", sql: "alter type mood add value 'ok';" }, + { name: "after_enum_values", sql: "insert into t values ('ok');" }, + ], + }); + return Effect.gen(function* () { + yield* legacyDbDiff(flags({ usePgDelta: Option.some(true), file: Option.some("my_diff") })); + const dir = join(tmp.current, "supabase", "migrations"); + const files = readdirSync(dir).sort(); + expect(files).toHaveLength(2); + expect(files[0]).toMatch(/^\d{14}_my_diff_schema_changes\.sql$/); + expect(files[1]).toMatch(/^\d{14}_my_diff_after_enum_values\.sql$/); + // Each unit's file carries only that unit's SQL, terminated with a newline. + expect(readFileSync(join(dir, files[0]!), "utf8")).toBe("alter type mood add value 'ok';\n"); + const success = s.out.messages.find((m) => m.type === "success"); + const data = success?.data as { file: string; files: ReadonlyArray }; + expect(data.files).toHaveLength(2); + // `file` stays the first written path for released string-field consumers. + expect(data.file).toBe(data.files[0]); + }).pipe(Effect.provide(s.layer)); + }); + + it.effect("creates nested parent directories for a nested single-unit --file name", () => { + // `db diff -f snapshots/remote` must create the `_snapshots/` parent dir + // before writing, mirroring Go's `utils.WriteFile`. + const s = setup(tmp.current, { diffSql: "create table g ();\n" }); + return Effect.gen(function* () { + yield* legacyDbDiff(flags({ file: Option.some("snapshots/remote") })); + const migrationsRoot = join(tmp.current, "supabase", "migrations"); + const dirs = readdirSync(migrationsRoot); + expect(dirs).toHaveLength(1); + expect(dirs[0]).toMatch(/^\d{14}_snapshots$/); + expect(readdirSync(join(migrationsRoot, dirs[0]!))).toEqual(["remote.sql"]); + }).pipe(Effect.provide(s.layer)); + }); + + it.effect("creates nested parent directories for a nested multi-unit --file name", () => { + const s = setup(tmp.current, { + format: "json", + diffFiles: [ + { name: "schema_changes", sql: "alter type mood add value 'ok';" }, + { name: "after_enum_values", sql: "insert into t values ('ok');" }, + ], + }); + return Effect.gen(function* () { + yield* legacyDbDiff( + flags({ usePgDelta: Option.some(true), file: Option.some("snapshots/remote") }), + ); + const success = s.out.messages.find((m) => m.type === "success"); + const data = success?.data as { files: ReadonlyArray }; + expect(data.files).toHaveLength(2); + for (const written of data.files) expect(existsSync(written)).toBe(true); + expect(data.files[0]).toMatch(/\d{14}_snapshots\/remote_schema_changes\.sql$/u); + expect(data.files[1]).toMatch(/\d{14}_snapshots\/remote_after_enum_values\.sql$/u); + }).pipe(Effect.provide(s.layer)); + }); + + it.effect("bumps the version set when a target migration file already exists", () => { + // The full generated set is collision-checked before writing; if any target + // exists the base advances one second so the new files stay strictly ascending + // AND never overwrite the pre-existing migration. + const s = setup(tmp.current, { + format: "json", + diffFiles: [ + { name: "schema_changes", sql: "a" }, + { name: "after_enum_values", sql: "b" }, + ], + }); + return Effect.gen(function* () { + const dir = join(tmp.current, "supabase", "migrations"); + mkdirSync(dir, { recursive: true }); + // TestClock starts at epoch 0, so the first version the writer tries is + // 19700101000000; pre-seed a colliding file at that version. + const clashing = join(dir, "19700101000000_my_diff_schema_changes.sql"); + writeFileSync(clashing, "-- pre-existing\n"); + yield* legacyDbDiff(flags({ usePgDelta: Option.some(true), file: Option.some("my_diff") })); + expect(readdirSync(dir).sort()).toEqual([ + "19700101000000_my_diff_schema_changes.sql", + "19700101000001_my_diff_schema_changes.sql", + "19700101000002_my_diff_after_enum_values.sql", + ]); + // The pre-existing file was never overwritten. + expect(readFileSync(clashing, "utf8")).toBe("-- pre-existing\n"); + }).pipe(Effect.provide(s.layer)); + }); + + it.effect("removes already-written unit files when a later unit write fails", () => { + // A mid-loop write failure best-effort removes every file this invocation + // already wrote, so no partial multi-file migration is left behind. + const s = setup(tmp.current, { + format: "json", + failWriteOnCall: 2, + diffFiles: [ + { name: "schema_changes", sql: "a" }, + { name: "after_enum_values", sql: "b" }, + ], + }); + return Effect.gen(function* () { + const exit = yield* legacyDbDiff( + flags({ usePgDelta: Option.some(true), file: Option.some("my_diff") }), + ).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + const dir = join(tmp.current, "supabase", "migrations"); + const remaining = existsSync(dir) ? readdirSync(dir) : []; + expect(remaining).toEqual([]); + }).pipe(Effect.provide(s.layer)); + }); + it.effect("explicit --from local --to linked prints the diff to stdout", () => { const s = setup(tmp.current, { isLocal: false, diffSql: "create table e ();\n" }); return Effect.gen(function* () { diff --git a/apps/cli/src/legacy/commands/db/pull/pull.handler.ts b/apps/cli/src/legacy/commands/db/pull/pull.handler.ts index e95fc934f2..858bb50c1a 100644 --- a/apps/cli/src/legacy/commands/db/pull/pull.handler.ts +++ b/apps/cli/src/legacy/commands/db/pull/pull.handler.ts @@ -44,6 +44,7 @@ import { legacyShouldUsePgDelta, } from "../shared/legacy-diff-engine.ts"; import { legacyDiffMigra } from "../shared/legacy-migra.ts"; +import { legacyWritePgDeltaMigrations } from "../shared/legacy-pgdelta-migrations.write.ts"; import { type LegacyDumpOptions, legacyBuildSchemaDumpEnv } from "../shared/legacy-pg-dump.env.ts"; import { legacyStreamPgDump } from "../shared/legacy-pg-dump.run.ts"; import { @@ -428,7 +429,8 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy } // Migration-file path (Go's `pull.run`). - const timestamp = legacyFormatMigrationTimestamp(yield* Clock.currentTimeMillis); + const nowMillis = yield* Clock.currentTimeMillis; + const timestamp = legacyFormatMigrationTimestamp(nowMillis); const migrationPath = legacyGetMigrationPath(path, cliConfig.workdir, timestamp, name); const remote = yield* legacyListRemoteMigrations(session); @@ -620,6 +622,7 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy }); return { sql: result.sql, + files: result.files, capture: debug ? { sourceCatalog, stderr: result.stderr } : undefined, }; } @@ -629,7 +632,7 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy schema: diffSchema, connectOptions: { isLocal: resolved.isLocal, dnsResolver }, }); - return { sql, capture: undefined }; + return { sql, files: undefined, capture: undefined }; }), ); }).pipe(Effect.ensuring(seam.removeShadowContainer(shadow.container))); @@ -678,55 +681,87 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy ); } - if (!diffEmpty) { - if (seededFromDump) { - // Append the migra diff to the dump-seeded file (Go's `diffRemoteSchema` - // opens the migration file `O_APPEND`, `pull.go:191`). - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* fs.open(migrationPath, { flag: "a" }).pipe( - Effect.mapError( - (cause) => - new LegacyDbPullWriteError({ - message: `failed to open migration file: ${cause.message}`, - }), - ), - ); - yield* file.writeAll(new TextEncoder().encode(out)).pipe( - Effect.mapError( - (cause) => - new LegacyDbPullWriteError({ - message: `failed to write migration file: ${cause.message}`, - }), - ), - ); - }), - ); - } else { - yield* legacyMakeDir(fs, path.dirname(migrationPath)).pipe( - Effect.mapError((cause) => new LegacyDbPullWriteError({ message: cause.message })), - ); - yield* fs.writeFileString(migrationPath, out).pipe( - Effect.mapError( - (cause) => - new LegacyDbPullWriteError({ - message: `failed to write migration file: ${cause.message}`, - }), - ), + // Build the list of migration files to record in the remote history. The + // migra engine writes exactly one file (the dump-seeded or freshly written + // migrationPath); the pg-delta engine writes one ordered file per + // execution-aware plan unit. + const writtenMigrations: Array<{ path: string; version: string }> = []; + if (usePgDeltaDiff) { + // pg-delta: one migration file per plan unit via the shared writer. A + // single-unit plan (the common case) keeps the exact `_.sql` + // filename; multi-unit plans append the unit name and give each file a + // strictly increasing timestamp so execution + migration-history order stay + // stable. The full set is collision-checked against existing migrations and + // each file is written exclusively so a pre-existing migration is never + // overwritten. Mirrors Go's `WritePgDeltaMigrations` + // (`internal/db/diff/pgdelta_migrations.go`). Empty plans are handled by the + // `diffEmpty` in-sync branch above, so `planFiles` is non-empty here. + const planFiles = diffOutcome.files ?? []; + const writtenUnits = yield* legacyWritePgDeltaMigrations(fs, path, { + workdir: cliConfig.workdir, + baseMillis: nowMillis, + name, + files: planFiles.map((file) => ({ name: file.name, sql: file.sql })), + }).pipe( + Effect.mapError((cause) => new LegacyDbPullWriteError({ message: cause.message })), + ); + for (const unit of writtenUnits) { + writtenMigrations.push({ path: unit.path, version: unit.version }); + } + } else { + if (!diffEmpty) { + if (seededFromDump) { + // Append the migra diff to the dump-seeded file (Go's `diffRemoteSchema` + // opens the migration file `O_APPEND`, `pull.go:191`). + yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* fs.open(migrationPath, { flag: "a" }).pipe( + Effect.mapError( + (cause) => + new LegacyDbPullWriteError({ + message: `failed to open migration file: ${cause.message}`, + }), + ), + ); + yield* file.writeAll(new TextEncoder().encode(out)).pipe( + Effect.mapError( + (cause) => + new LegacyDbPullWriteError({ + message: `failed to write migration file: ${cause.message}`, + }), + ), + ); + }), + ); + } else { + yield* legacyMakeDir(fs, path.dirname(migrationPath)).pipe( + Effect.mapError((cause) => new LegacyDbPullWriteError({ message: cause.message })), + ); + yield* fs.writeFileString(migrationPath, out).pipe( + Effect.mapError( + (cause) => + new LegacyDbPullWriteError({ + message: `failed to write migration file: ${cause.message}`, + }), + ), + ); + } + } + + // Go's `ensureMigrationWritten` (`pull.go:68,263-268`): a dump that produced + // nothing followed by an empty diff leaves the file empty → in sync. + if (seededFromDump && !seedWroteBytes && diffEmpty) { + return yield* Effect.fail( + new LegacyDbPullInSyncError({ message: "No schema changes found" }), ); } + writtenMigrations.push({ path: migrationPath, version: timestamp }); } - // Go's `ensureMigrationWritten` (`pull.go:68,263-268`): a dump that produced - // nothing followed by an empty diff leaves the file empty → in sync. - if (seededFromDump && !seedWroteBytes && diffEmpty) { - return yield* Effect.fail( - new LegacyDbPullInSyncError({ message: "No schema changes found" }), - ); + for (const written of writtenMigrations) { + yield* output.raw(`Schema written to ${legacyBold(written.path)}\n`, "stderr"); } - yield* output.raw(`Schema written to ${legacyBold(migrationPath)}\n`, "stderr"); - // Prompt to update the remote migration history table. Go calls // `PromptYesNo(ctx, "Update remote migration history table?", true)` // (`internal/db/pull/pull.go:73`), which returns the default (`true`) on @@ -739,14 +774,19 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy // (`console.go:64-82`), and otherwise prompts on a real TTY. const shouldUpdate = yield* legacyPromptYesNo(output, yes, updateHistoryTitle, true); if (shouldUpdate) { - yield* legacyUpdateMigrationHistory(session, fs, path, migrationPath, timestamp); + yield* legacyUpdateMigrationHistory(session, fs, path, writtenMigrations); remoteHistoryUpdated = true; } if (output.format !== "text") { yield* output.success("Schema pulled.", { declarative: false, - schemaWritten: migrationPath, + // `schemaWritten` keeps the first written path for released consumers that + // read the string field; `schemaFiles` lists EVERY written migration path + // in write order (a pg-delta plan writes one file per unit), so machine + // callers see all of them, not just the first. + schemaWritten: writtenMigrations[0]?.path ?? migrationPath, + schemaFiles: writtenMigrations.map((written) => written.path), remoteHistoryUpdated, engine: usePgDeltaDiff ? "pg-delta" : "migra", }); diff --git a/apps/cli/src/legacy/commands/db/pull/pull.integration.test.ts b/apps/cli/src/legacy/commands/db/pull/pull.integration.test.ts index 1e15768888..248c0be283 100644 --- a/apps/cli/src/legacy/commands/db/pull/pull.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/pull/pull.integration.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, Layer, Option } from "effect"; import { + legacyFailWriteStringOnNthCallFsLayer, mockLegacyCliConfig, mockLegacyLinkedProjectCacheTracked, mockLegacyTelemetryStateTracked, @@ -44,6 +45,21 @@ const EXPORT_JSON = JSON.stringify({ files: [{ path: "schemas/public/t.sql", order: 0, statements: 1, sql: "create table t ();" }], }); +// Builds the pg-delta diff envelope printed by `templates/pgdelta.ts`: one file +// per execution-aware plan unit (`{version:1,files:[{order,name,transactionMode,sql}]}`). +const pgDeltaDiffEnvelope = ( + units: ReadonlyArray<{ name: string; sql: string; transactionMode?: string }>, +): string => + JSON.stringify({ + version: 1, + files: units.map((unit, index) => ({ + order: index + 1, + name: unit.name, + transactionMode: unit.transactionMode ?? "transactional", + sql: unit.sql, + })), + }); + interface SetupOpts { readonly format?: OutputFormat; readonly remoteVersions?: ReadonlyArray; @@ -78,6 +94,8 @@ interface SetupOpts { // `--declarative` and `--use-pg-delta` are present, to replay pflag's // last-occurrence-wins ordering; defaults to empty. readonly args?: ReadonlyArray; + // When set, the Nth `writeFileString` fails, exercising cleanup-on-failure. + readonly failWriteOnCall?: number; } function setup(workdir: string, opts: SetupOpts = {}) { @@ -222,7 +240,7 @@ function setup(workdir: string, opts: SetupOpts = {}) { }), }); - const layer = Layer.mergeAll( + const baseLayer = Layer.mergeAll( out.layer, telemetry.layer, cache.layer, @@ -250,6 +268,12 @@ function setup(workdir: string, opts: SetupOpts = {}) { mockRuntimeInfo(), BunServices.layer, ); + // Merged last so its `FileSystem` overrides `BunServices` (last-wins); `Path` + // still resolves from `BunServices`. + const layer = + opts.failWriteOnCall === undefined + ? baseLayer + : Layer.merge(baseLayer, legacyFailWriteStringOnNthCallFsLayer(opts.failWriteOnCall)); return { layer, @@ -303,20 +327,158 @@ describe("legacy db pull", () => { seedMigration(tmp.current, "20240101000000"); const s = setup(tmp.current, { remoteVersions: ["20240101000000"], - edgeStdout: "create table remote ();\n", + edgeStdout: pgDeltaDiffEnvelope([ + { + name: "schema_changes", + sql: "-- Migration unit 1: schema_changes\n\ncreate table remote ();", + }, + ]), yes: true, }); return Effect.gen(function* () { yield* legacyDbPull(flags({ diffEngine: Option.some("pg-delta") })); const dir = join(tmp.current, "supabase", "migrations"); expect(existsSync(join(dir, `${"20240101000000"}_local.sql`))).toBe(true); - // A new timestamped remote_schema migration was written. + // A single-unit plan keeps the unchanged `_remote_schema.sql` filename. + const written = readdirSync(dir).filter((f) => f.endsWith("_remote_schema.sql")); + expect(written).toHaveLength(1); + expect(readFileSync(join(dir, written[0] ?? ""), "utf8")).toContain( + "create table remote ();", + ); expect(streamText(s.out, "stderr")).toContain("Schema written to"); expect(s.historyUpserts.length).toBe(1); expect(streamText(s.out, "stdout")).toContain("Finished supabase db pull."); }).pipe(Effect.provide(s.layer)); }); + it.effect( + "a pg-delta plan with transaction boundaries writes one ordered migration file per unit", + () => { + // pg-delta plans that cross a transaction boundary (e.g. ALTER TYPE ... ADD + // VALUE then a statement using the new value) come back as several units; each + // is written to its own migration file with a strictly increasing timestamp and + // recorded in the remote history. Mirrors Go's `writePgDeltaMigrations`. + seedMigration(tmp.current, "20240101000000"); + const s = setup(tmp.current, { + remoteVersions: ["20240101000000"], + edgeStdout: pgDeltaDiffEnvelope([ + { name: "schema_changes", sql: "-- unit 1\n\nalter type mood add value 'ok';" }, + { name: "after_enum_values", sql: "-- unit 2\n\ninsert into t values ('ok');" }, + { + name: "non_transactional", + transactionMode: "none", + sql: "-- unit 3\n\ncreate index concurrently i on t (c);", + }, + ]), + yes: true, + }); + return Effect.gen(function* () { + yield* legacyDbPull(flags({ diffEngine: Option.some("pg-delta") })); + const dir = join(tmp.current, "supabase", "migrations"); + const written = readdirSync(dir) + .filter((f) => f !== "20240101000000_local.sql") + .sort(); + expect(written).toHaveLength(3); + // Multi-unit plans append the unit name and carry strictly increasing versions. + expect(written[0]).toMatch(/^\d{14}_remote_schema_schema_changes\.sql$/u); + expect(written[1]).toMatch(/^\d{14}_remote_schema_after_enum_values\.sql$/u); + expect(written[2]).toMatch(/^\d{14}_remote_schema_non_transactional\.sql$/u); + const versions = written.map((f) => f.slice(0, 14)); + expect((versions[0] ?? "") < (versions[1] ?? "")).toBe(true); + expect((versions[1] ?? "") < (versions[2] ?? "")).toBe(true); + expect(readFileSync(join(dir, written[2] ?? ""), "utf8")).toContain( + "create index concurrently i on t (c);", + ); + // One "Schema written to" line and one history upsert per unit. + expect(streamText(s.out, "stderr").match(/Schema written to/gu)).toHaveLength(3); + expect(s.historyUpserts.length).toBe(3); + // Go's UpdateMigrationTable prints all versions space-separated. + expect(streamText(s.out, "stderr")).toContain( + `Repaired migration history: [${versions.join(" ")}] => applied`, + ); + }).pipe(Effect.provide(s.layer)); + }, + ); + + it.effect( + "a multi-unit pg-delta pull reports every written migration path in the json payload", + () => { + // The structured payload must list ALL written migration files in write order, + // not just the first (`schemaWritten`). A pg-delta plan writes one file per unit. + seedMigration(tmp.current, "20240101000000"); + const s = setup(tmp.current, { + format: "json", + remoteVersions: ["20240101000000"], + edgeStdout: pgDeltaDiffEnvelope([ + { name: "schema_changes", sql: "-- unit 1\n\nalter type mood add value 'ok';" }, + { name: "after_enum_values", sql: "-- unit 2\n\ninsert into t values ('ok');" }, + { + name: "non_transactional", + transactionMode: "none", + sql: "-- unit 3\n\ncreate index concurrently i on t (c);", + }, + ]), + }); + return Effect.gen(function* () { + yield* legacyDbPull(flags({ diffEngine: Option.some("pg-delta") })); + const success = s.out.messages.find((m) => m.type === "success"); + const data = success?.data as + | { schemaWritten?: string; schemaFiles?: Array } + | undefined; + expect(data?.schemaFiles).toHaveLength(3); + // Paths appear in write order, each carrying its unit name. + expect(data?.schemaFiles?.[0]).toMatch(/_remote_schema_schema_changes\.sql$/u); + expect(data?.schemaFiles?.[1]).toMatch(/_remote_schema_after_enum_values\.sql$/u); + expect(data?.schemaFiles?.[2]).toMatch(/_remote_schema_non_transactional\.sql$/u); + // `schemaWritten` stays the first written path (unchanged string contract). + expect(data?.schemaWritten).toBe(data?.schemaFiles?.[0]); + }).pipe(Effect.provide(s.layer)); + }, + ); + + it.effect("removes already-written unit files when a later pg-delta unit write fails", () => { + // A mid-loop write failure best-effort removes every migration file this + // invocation already wrote, so no partial multi-file pull is left behind. + seedMigration(tmp.current, "20240101000000"); + const s = setup(tmp.current, { + failWriteOnCall: 2, + remoteVersions: ["20240101000000"], + edgeStdout: pgDeltaDiffEnvelope([ + { name: "schema_changes", sql: "alter type mood add value 'ok';" }, + { name: "after_enum_values", sql: "insert into t values ('ok');" }, + ]), + yes: true, + }); + return Effect.gen(function* () { + const exit = yield* legacyDbPull(flags({ diffEngine: Option.some("pg-delta") })).pipe( + Effect.exit, + ); + expect(Exit.isFailure(exit)).toBe(true); + const dir = join(tmp.current, "supabase", "migrations"); + // Only the pre-seeded local migration remains; the first written unit was + // rolled back and the failing second unit never landed. + expect(readdirSync(dir)).toEqual(["20240101000000_local.sql"]); + // No history rows were upserted because the write failed before the prompt. + expect(s.historyUpserts.length).toBe(0); + }).pipe(Effect.provide(s.layer)); + }); + + it.effect("a malformed pg-delta diff envelope surfaces a parse error, not 'in sync'", () => { + seedMigration(tmp.current, "20240101000000"); + const s = setup(tmp.current, { + remoteVersions: ["20240101000000"], + edgeStdout: "not a valid envelope{", + yes: true, + }); + return Effect.gen(function* () { + const error = yield* legacyDbPull(flags({ diffEngine: Option.some("pg-delta") })).pipe( + Effect.flip, + ); + expect(error.message).toContain("failed to parse pg-delta diff output"); + expect(error.message).not.toContain("No schema changes found"); + }).pipe(Effect.provide(s.layer)); + }); + it.effect("pulls with the default migra engine", () => { seedMigration(tmp.current, "20240101000000"); const s = setup(tmp.current, { @@ -527,8 +689,14 @@ describe("legacy db pull", () => { remoteHistoryUpdated: true, engine: "migra", }); - const data = success?.data as { schemaWritten?: string } | undefined; + const data = success?.data as + | { schemaWritten?: string; schemaFiles?: Array } + | undefined; expect(data?.schemaWritten).toMatch(/_remote_schema\.sql$/u); + // The single-unit case lists exactly one written migration path, and it is the + // same path as the singular `schemaWritten` field. + expect(data?.schemaFiles).toHaveLength(1); + expect(data?.schemaFiles?.[0]).toBe(data?.schemaWritten); }).pipe(Effect.provide(s.layer)); }); @@ -1045,7 +1213,7 @@ describe("legacy db pull", () => { writeFileSync(join(tmp.current, "supabase", ".env"), "SUPABASE_EXPERIMENTAL_PG_DELTA=true\n"); const s = setup(tmp.current, { remoteVersions: ["20240101000000"], - edgeStdout: "create table remote ();\n", + edgeStdout: pgDeltaDiffEnvelope([{ name: "schema_changes", sql: "create table remote ();" }]), yes: true, }); return Effect.gen(function* () { @@ -1158,7 +1326,7 @@ describe("legacy db pull", () => { ); const s = setup(tmp.current, { remoteVersions: ["20240101000000"], - edgeStdout: "create table remote ();\n", + edgeStdout: pgDeltaDiffEnvelope([{ name: "schema_changes", sql: "create table remote ();" }]), yes: true, resolvedRef: "abcdefghijklmnopqrst", }); @@ -1180,7 +1348,7 @@ describe("legacy db pull", () => { const s = setup(tmp.current, { remoteVersions: ["20240101000000"], edgeFailFirstWith: "error diffing schema:\nfailed to connect: network is unreachable", - edgeStdout: "create table remote ();\n", + edgeStdout: pgDeltaDiffEnvelope([{ name: "schema_changes", sql: "create table remote ();" }]), yes: true, poolerAvailable: true, }); diff --git a/apps/cli/src/legacy/commands/db/pull/pull.sync.integration.test.ts b/apps/cli/src/legacy/commands/db/pull/pull.sync.integration.test.ts new file mode 100644 index 0000000000..05b3423db6 --- /dev/null +++ b/apps/cli/src/legacy/commands/db/pull/pull.sync.integration.test.ts @@ -0,0 +1,100 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Path } from "effect"; + +import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; +import { LegacyDbExecError } from "../../../shared/legacy-db-connection.errors.ts"; +import type { LegacyDbSession } from "../../../shared/legacy-db-connection.service.ts"; +import { LegacyDbPullWriteError } from "./pull.errors.ts"; +import { legacyUpdateMigrationHistory, type LegacyPulledMigration } from "./pull.sync.ts"; + +// Records exec statements and successful upserts in one ordered log so the tests +// can assert the transaction envelope (BEGIN / UPSERT / COMMIT / ROLLBACK) around +// the version writes. `failUpsertAt` fails the Nth upsert to simulate a dropped +// connection mid-loop. +function mockSession(opts: { readonly failUpsertAt?: number } = {}) { + const calls: Array = []; + let upsertCount = 0; + const session: LegacyDbSession = { + exec: (sql: string) => Effect.sync(() => void calls.push(sql)), + query: (sql: string) => { + if (/INSERT INTO supabase_migrations/u.test(sql)) { + upsertCount += 1; + if (opts.failUpsertAt === upsertCount) { + return Effect.fail(new LegacyDbExecError({ message: "connection reset by peer" })); + } + calls.push("UPSERT"); + } + return Effect.succeed([] as ReadonlyArray>); + }, + extensionExists: () => Effect.die("extensionExists unused"), + copyToCsv: () => Effect.die("copyToCsv unused"), + queryRaw: () => Effect.die("queryRaw unused"), + }; + return { session, calls }; +} + +function writeMigrations(dir: string): ReadonlyArray { + const migrations: ReadonlyArray = [ + { path: join(dir, "20240101000000_a.sql"), version: "20240101000000" }, + { path: join(dir, "20240101000001_b.sql"), version: "20240101000001" }, + ]; + writeFileSync(migrations[0]!.path, "create table a ();"); + writeFileSync(migrations[1]!.path, "create table b ();"); + return migrations; +} + +describe("legacyUpdateMigrationHistory", () => { + it.effect("wraps the upserts in one BEGIN + N upserts + COMMIT transaction", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = mkdtempSync(join(tmpdir(), "pull-sync-")); + const migrations = writeMigrations(dir); + const out = mockOutput(); + const { session, calls } = mockSession(); + + yield* legacyUpdateMigrationHistory(session, fs, path, migrations).pipe( + Effect.provide(out.layer), + ); + + // The create-table setup runs its own BEGIN/COMMIT first; the upsert + // transaction is the trailing envelope around every version write. + expect(calls).not.toContain("ROLLBACK"); + expect(calls.slice(-4)).toEqual(["BEGIN", "UPSERT", "UPSERT", "COMMIT"]); + // The success line is byte-identical to Go's repair output. + expect(out.stderrText).toContain( + "Repaired migration history: [20240101000000 20240101000001] => applied", + ); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("rolls back and surfaces the error when an upsert fails mid-loop", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = mkdtempSync(join(tmpdir(), "pull-sync-")); + const migrations = writeMigrations(dir); + const out = mockOutput(); + const { session, calls } = mockSession({ failUpsertAt: 2 }); + + const error = yield* legacyUpdateMigrationHistory(session, fs, path, migrations).pipe( + Effect.provide(out.layer), + Effect.flip, + ); + + // First upsert applied, second failed → the upsert transaction ends in + // ROLLBACK, never COMMIT (the create-table setup's own COMMIT ran earlier). + expect(calls.slice(-3)).toEqual(["BEGIN", "UPSERT", "ROLLBACK"]); + expect(calls[calls.length - 1]).toBe("ROLLBACK"); + // Error message shape stays byte-identical to the pre-transaction version. + expect(error).toBeInstanceOf(LegacyDbPullWriteError); + expect(error.message).toBe("failed to update migration table: connection reset by peer"); + // No success line when the repair failed. + expect(out.stderrText).not.toContain("Repaired migration history"); + }).pipe(Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/src/legacy/commands/db/pull/pull.sync.ts b/apps/cli/src/legacy/commands/db/pull/pull.sync.ts index 3e251e4c1b..7d00895074 100644 --- a/apps/cli/src/legacy/commands/db/pull/pull.sync.ts +++ b/apps/cli/src/legacy/commands/db/pull/pull.sync.ts @@ -10,50 +10,80 @@ import { import { legacySplitAndTrim } from "../../../shared/legacy-sql-split.ts"; import { LegacyDbPullWriteError } from "./pull.errors.ts"; +/** A pulled migration file paired with the version to record in the history. */ +export interface LegacyPulledMigration { + readonly path: string; + readonly version: string; +} + /** - * Records the pulled migration as applied in `supabase_migrations.schema_migrations` - * WITHOUT re-executing it (the schema already exists on the remote). Mirrors Go's - * `repair.UpdateMigrationTable(conn, [version], Applied, false, fsys)` + * Records the pulled migration(s) as applied in + * `supabase_migrations.schema_migrations` WITHOUT re-executing them (the schema + * already exists on the remote). Mirrors Go's + * `repair.UpdateMigrationTable(conn, versions, Applied, false, fsys)` * (`internal/migration/repair/repair.go:58`): create the history table, then UPSERT - * the version row with the migration's name + statements. + * each version row with the migration's name + statements. A pg-delta pull whose + * plan crosses a transaction boundary writes several ordered files, so several + * versions are recorded in one pass. */ export const legacyUpdateMigrationHistory = ( session: LegacyDbSession, fs: FileSystem.FileSystem, path: Path.Path, - migrationPath: string, - timestamp: string, + migrations: ReadonlyArray, ) => Effect.gen(function* () { const output = yield* Output; - const match = MIGRATE_FILE_PATTERN.exec(path.basename(migrationPath)); - if (match === null || match[1] !== timestamp) { - // Go resolves the repair file by globbing `_*.sql` against the - // migrations dir and fails with `os.ErrNotExist` when nothing matches - // (`repair.GetMigrationFile`, `internal/migration/repair/repair.go:90-99`). - // The glob is anchored on the GENERATED `timestamp` and `*` never crosses a - // path separator, so a migration name with a separator (`supabase db pull - // dir/...`) writes a nested file the glob can't reach — even when the nested - // basename is itself a valid migration filename (`dir/20250101000000_backfill` - // → basename `20250101000000_backfill.sql`, which DOES match the regex but - // carries the user's nested timestamp, not the generated one). Require the - // basename to both match the pattern AND carry the generated timestamp, - // mirroring Go's anchored glob, rather than trusting `path.basename`. - return yield* Effect.fail( - new LegacyDbPullWriteError({ - message: `glob supabase/migrations/${timestamp}_*.sql: file does not exist`, - }), - ); + // Resolve each file the way Go's `repair.GetMigrationFile` globs + // `_*.sql` against the migrations dir, failing with `os.ErrNotExist` + // when nothing matches (`internal/migration/repair/repair.go:90-99`). The glob + // is anchored on the GENERATED version and `*` never crosses a path separator, + // so a migration name with a separator writes a nested file the glob can't + // reach — require the basename to both match the pattern AND carry the + // generated version rather than trusting `path.basename`. + const resolved: Array<{ version: string; name: string; migrationPath: string }> = []; + for (const migration of migrations) { + const match = MIGRATE_FILE_PATTERN.exec(path.basename(migration.path)); + if (match === null || match[1] !== migration.version) { + return yield* Effect.fail( + new LegacyDbPullWriteError({ + message: `glob supabase/migrations/${migration.version}_*.sql: file does not exist`, + }), + ); + } + resolved.push({ + version: migration.version, + name: match[2] ?? "", + migrationPath: migration.path, + }); } - // Guarded above: match[1] === timestamp, so use the generated timestamp - // directly (avoids re-deriving a `string | undefined` from the regex group). - const version = timestamp; - const name = match[2] ?? ""; yield* Effect.gen(function* () { - const content = yield* fs.readFileString(migrationPath); - const statements = legacySplitAndTrim(content); + // Create the history schema/table first, in its OWN transaction — Go runs + // `CreateMigrationTable` before the upsert batch (`repair.go:59`). Keeping it + // outside the upsert transaction below avoids nesting BEGINs + // (`legacyCreateMigrationTable` issues its own BEGIN/COMMIT). yield* legacyCreateMigrationTable(session); - yield* session.query(UPSERT_MIGRATION_VERSION, [version, name, statements]); + // Record every version in ONE explicit transaction: a mid-loop failure + // (dropped connection, unreadable migration file) must record NONE of them. + // Go queues all upserts in a single `pgx.Batch` (`repair.go:63-83`), which + // Postgres executes as one implicit transaction; without a transaction here + // each UPSERT autocommits, so a failure partway through would leave partial + // remote history that fails the next pull's sync check. + yield* Effect.gen(function* () { + yield* session.exec("BEGIN"); + for (const entry of resolved) { + const content = yield* fs.readFileString(entry.migrationPath); + const statements = legacySplitAndTrim(content); + yield* session.query(UPSERT_MIGRATION_VERSION, [entry.version, entry.name, statements]); + } + yield* session.exec("COMMIT"); + }).pipe( + // Roll back on ANY failure inside the transaction — including a migration + // file read that fails after BEGIN. `Effect.ignore` keeps a ROLLBACK + // failure from masking the original error (`tapError` re-raises the + // original). Mirrors `legacyCreateMigrationTable`'s rollback handling. + Effect.tapError(() => session.exec("ROLLBACK").pipe(Effect.ignore)), + ); }).pipe( Effect.mapError( (cause) => @@ -63,8 +93,9 @@ export const legacyUpdateMigrationHistory = ( ), ); // Match Go's `repair.UpdateMigrationTable(..., repairAll=false, ...)`, which - // prints `Repaired migration history: [] => applied` to stderr - // (`internal/migration/repair/repair.go`). Plain text on stderr, so it does - // not interfere with machine-output payloads on stdout. - yield* output.raw(`Repaired migration history: [${version}] => applied\n`, "stderr"); + // prints `Repaired migration history: [ ...] => applied` to stderr + // (Go's `%v` over the `[]string` of versions, space-separated). Plain text on + // stderr, so it does not interfere with machine-output payloads on stdout. + const versions = resolved.map((entry) => entry.version).join(" "); + yield* output.raw(`Repaired migration history: [${versions}] => applied\n`, "stderr"); }); diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts b/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts index ad474a9a77..2b581df3b0 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts @@ -41,7 +41,20 @@ function mockEdge(stdout: string) { const layer = Layer.succeed(LegacyEdgeRuntimeScript, { run: (opts: LegacyEdgeRuntimeRunOpts) => { calls.push(opts); - return Effect.succeed({ stdout, stderr: "" }); + // The pg-delta diff script (uniquely identified by `renderPlanFiles`) prints a + // JSON envelope with one file per plan unit; wrap the test's raw SQL into a + // single-unit envelope so `legacyDiffPgDelta` parses it. Other scripts + // (declarative export) return their stdout unchanged. + const wrapped = + opts.script.includes("renderPlanFiles") && stdout.length > 0 + ? JSON.stringify({ + version: 1, + files: [ + { order: 1, name: "schema_changes", transactionMode: "transactional", sql: stdout }, + ], + }) + : stdout; + return Effect.succeed({ stdout: wrapped, stderr: "" }); }, }); return { layer, calls }; diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts index 44dd7548b2..fa04d716dd 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -96,15 +96,33 @@ function setup(workdir: string, opts: SetupOpts = {}) { removeShadowContainer: () => Effect.void, }); const edge = Layer.succeed(LegacyEdgeRuntimeScript, { - run: (runOpts: LegacyEdgeRuntimeRunOpts) => - Effect.succeed({ - stdout: - opts.exportJson !== undefined && - runOpts.errPrefix === "error exporting declarative schema" - ? opts.exportJson - : (opts.diffSql ?? ""), - stderr: "", - }), + run: (runOpts: LegacyEdgeRuntimeRunOpts) => { + if ( + opts.exportJson !== undefined && + runOpts.errPrefix === "error exporting declarative schema" + ) { + return Effect.succeed({ stdout: opts.exportJson, stderr: "" }); + } + const diffSql = opts.diffSql ?? ""; + // The pg-delta diff script (uniquely identified by `renderPlanFiles`) prints a + // JSON envelope with one file per plan unit; wrap the test's raw SQL into a + // single-unit envelope so `legacyDiffPgDelta` parses it. + const stdout = + runOpts.script.includes("renderPlanFiles") && diffSql.length > 0 + ? JSON.stringify({ + version: 1, + files: [ + { + order: 1, + name: "schema_changes", + transactionMode: "transactional", + sql: diffSql, + }, + ], + }) + : diffSql; + return Effect.succeed({ stdout, stderr: "" }); + }, }); const dbExec: string[] = []; const dbConn = Layer.succeed(LegacyDbConnection, { diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta-migrations.write.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta-migrations.write.ts new file mode 100644 index 0000000000..e02b5c720f --- /dev/null +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta-migrations.write.ts @@ -0,0 +1,135 @@ +import { Data, Effect, type FileSystem, type Path } from "effect"; + +import { legacyMakeDir } from "../../../shared/legacy-make-dir.ts"; +import { + legacyFormatMigrationTimestamp, + legacyGetMigrationPath, +} from "../../../shared/legacy-migration-file.ts"; + +/** A migration file written by a diff/pull, paired with its history version. */ +export interface LegacyWrittenMigration { + readonly path: string; + readonly version: string; +} + +/** + * A write failure from `legacyWritePgDeltaMigrations`. Callers map this to their + * own command-domain write error (`LegacyDbDiffWriteError` / `LegacyDbPullWriteError`). + */ +export class LegacyPgDeltaMigrationWriteError extends Data.TaggedError( + "LegacyPgDeltaMigrationWriteError", +)<{ + readonly message: string; +}> {} + +/** + * Bounds the base-timestamp bump retry so a directory already full of same-second + * migrations can't spin forever. Mirrors Go's `maxVersionCollisionAttempts`. + */ +const MAX_VERSION_COLLISION_ATTEMPTS = 60; + +/** + * Port of Go's `WritePgDeltaMigrations` (`apps/cli-go/internal/db/diff/pgdelta_migrations.go`). + * + * Writes one ordered migration file per plan unit. A single-unit plan (the common + * case) keeps the exact `_.sql` filename; multi-unit plans append the + * unit name and give each file a strictly increasing timestamp (real time + * arithmetic on the base millis, never string increment) so their execution order + * and migration-history order stay stable. + * + * Before writing anything the FULL set of generated filenames is collision-checked + * against the filesystem: if any target path already exists the base is advanced by + * one second and every version recomputed, so the set stays strictly ascending AND + * unique against pre-existing migrations. The base only ever moves forward — never + * backdated below the caller's wall clock, since backdating could sort a new file + * before pre-existing migrations. The resulting ≤N−1s future-dating is inherent to + * second-granularity versions and acceptable once uniqueness is enforced. + * + * Each file is written with the exclusive `"wx"` flag so a race between the + * collision check and the write can still never silently overwrite an existing + * migration. If any open/write fails mid-loop, every file already written by THIS + * invocation is best-effort removed before the error surfaces (a removal failure + * never masks the original error). + */ +export const legacyWritePgDeltaMigrations = ( + fs: FileSystem.FileSystem, + pathSvc: Path.Path, + opts: { + readonly workdir: string; + readonly baseMillis: number; + readonly name: string; + readonly files: ReadonlyArray<{ readonly name: string; readonly sql: string }>; + }, +): Effect.Effect, LegacyPgDeltaMigrationWriteError> => + Effect.gen(function* () { + const { workdir, name, files } = opts; + const single = files.length === 1; + const buildSet = (baseMillis: number): Array => + files.map((file, i) => { + const version = legacyFormatMigrationTimestamp(baseMillis + i * 1000); + const unitName = single ? name : `${name}_${file.name}`; + return { path: legacyGetMigrationPath(pathSvc, workdir, version, unitName), version }; + }); + + let baseMillis = opts.baseMillis; + let set = buildSet(baseMillis); + for (let attempt = 0; ; attempt++) { + let collision = false; + for (const w of set) { + const exists = yield* fs.exists(w.path).pipe( + Effect.mapError( + (cause) => + new LegacyPgDeltaMigrationWriteError({ + message: `failed to check migration file: ${cause.message}`, + }), + ), + ); + if (exists) { + collision = true; + break; + } + } + if (!collision) break; + if (attempt + 1 >= MAX_VERSION_COLLISION_ATTEMPTS) { + return yield* Effect.fail( + new LegacyPgDeltaMigrationWriteError({ + message: `failed to find a unique migration version after ${MAX_VERSION_COLLISION_ATTEMPTS} attempts`, + }), + ); + } + baseMillis += 1000; + set = buildSet(baseMillis); + } + + const written: Array = []; + const writeAll = Effect.gen(function* () { + for (let i = 0; i < files.length; i++) { + const w = set[i]!; + const file = files[i]!; + yield* legacyMakeDir(fs, pathSvc.dirname(w.path)).pipe( + Effect.mapError( + (cause) => new LegacyPgDeltaMigrationWriteError({ message: cause.message }), + ), + ); + yield* fs.writeFileString(w.path, `${file.sql}\n`, { flag: "wx" }).pipe( + Effect.mapError( + (cause) => + new LegacyPgDeltaMigrationWriteError({ + message: + cause.reason._tag === "AlreadyExists" + ? `failed to open migration file: ${cause.message}` + : `failed to write migration file: ${cause.message}`, + }), + ), + ); + written.push(w); + } + return written; + }); + + return yield* writeAll.pipe( + Effect.tapError(() => + Effect.forEach(written, (w) => fs.remove(w.path).pipe(Effect.ignore), { discard: true }), + ), + ); + }); diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.deno-templates.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.deno-templates.ts index e43e9828a8..5a6fb7060b 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.deno-templates.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.deno-templates.ts @@ -15,15 +15,15 @@ /** `templates/pgdelta.ts` — diffs SOURCE→TARGET and prints SQL statements. */ export const legacyPgDeltaDiffScript = - 'import {\n createPlan,\n deserializeCatalog,\n formatSqlStatements,\n} from "npm:@supabase/pg-delta@1.0.0-alpha.20";\nimport { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.20/integrations/supabase";\n\nasync function resolveInput(ref: string | undefined) {\n if (!ref) {\n return null;\n }\n if (ref.startsWith("postgres://") || ref.startsWith("postgresql://")) {\n return ref;\n }\n const json = await Deno.readTextFile(ref);\n return deserializeCatalog(JSON.parse(json));\n}\n\nconst source = Deno.env.get("SOURCE");\nconst target = Deno.env.get("TARGET");\n\nconst includedSchemas = Deno.env.get("INCLUDED_SCHEMAS");\nif (includedSchemas) {\n const schemas = includedSchemas.split(",");\n const schemaFilter = {\n or: [{ "*/schema": schemas }, { "schema/name": schemas }],\n };\n // CompositionPattern `and` is valid FilterDSL; Deno\'s structural typing is strict on `or` branches.\n supabase.filter = {\n and: [supabase.filter!, schemaFilter],\n } as typeof supabase.filter;\n}\n\nconst formatOptionsRaw = Deno.env.get("FORMAT_OPTIONS");\nlet formatOptions = undefined;\nif (formatOptionsRaw) {\n formatOptions = JSON.parse(formatOptionsRaw);\n}\n\ntry {\n const result = await createPlan(\n await resolveInput(source),\n await resolveInput(target),\n {\n ...supabase,\n skipDefaultPrivilegeSubtraction: true,\n },\n );\n let statements = result?.plan.statements ?? [];\n if (formatOptions != null) {\n statements = formatSqlStatements(statements, formatOptions);\n }\n if (Deno.env.get("PGDELTA_DEBUG")) {\n console.error(\n JSON.stringify({\n statementCount: statements.length,\n source: source ? "connected" : "null",\n target: target ? "connected" : "null",\n includedSchemas: includedSchemas ?? null,\n skipDefaultPrivilegeSubtraction: true,\n }),\n );\n }\n for (const sql of statements) {\n console.log(`${sql};`);\n }\n} catch (e) {\n console.error(e);\n // Force close event loop\n throw new Error("");\n}\n// Force close the event loop on the success path too. When SOURCE/TARGET are\n// live database URLs the plan opens connections whose keepalive handles can keep\n// the Edge Runtime worker alive after the diff has been written, so the container\n// never exits and the CLI — which follows this container\'s logs — hangs\n// indefinitely at 0% CPU (supabase/pg-toolbelt#312).\nthrow new Error("");\n'; + 'import {\n createPlan,\n deserializeCatalog,\n renderPlanFiles,\n} from "npm:@supabase/pg-delta@1.0.0-alpha.20";\nimport { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.20/integrations/supabase";\n\nasync function resolveInput(ref: string | undefined) {\n if (!ref) {\n return null;\n }\n if (ref.startsWith("postgres://") || ref.startsWith("postgresql://")) {\n return ref;\n }\n const json = await Deno.readTextFile(ref);\n return deserializeCatalog(JSON.parse(json));\n}\n\nconst source = Deno.env.get("SOURCE");\nconst target = Deno.env.get("TARGET");\n\nconst includedSchemas = Deno.env.get("INCLUDED_SCHEMAS");\nif (includedSchemas) {\n const schemas = includedSchemas.split(",");\n const schemaFilter = {\n or: [{ "*/schema": schemas }, { "schema/name": schemas }],\n };\n // CompositionPattern `and` is valid FilterDSL; Deno\'s structural typing is strict on `or` branches.\n supabase.filter = {\n and: [supabase.filter!, schemaFilter],\n } as typeof supabase.filter;\n}\n\nconst formatOptionsRaw = Deno.env.get("FORMAT_OPTIONS");\nconst parsedFormatOptions = formatOptionsRaw ? JSON.parse(formatOptionsRaw) : undefined;\n// Format the emitted SQL by default with the same sensible settings the\n// declarative export uses (`exportDeclarativeSchema` in @supabase/pg-delta:\n// `{ ...DEFAULT_OPTIONS, maxWidth: 180, keywordCase: "upper", ...userOptions }`),\n// so `db pull` / `db diff` produce readable migrations even when config sets no\n// `[experimental.pgdelta] format_options`. The formatter fills DEFAULT_OPTIONS\n// for missing keys itself, so only the two overrides are passed here. Setting\n// `format_options = "null"` (parsed to `null`) is the explicit opt-out: raw,\n// unformatted statements, mirroring declarative export\'s `formatOptions === null`.\nconst sqlFormatOptions =\n parsedFormatOptions === null\n ? undefined\n : { maxWidth: 180, keywordCase: "upper", ...parsedFormatOptions };\n\ntry {\n const result = await createPlan(\n await resolveInput(source),\n await resolveInput(target),\n {\n ...supabase,\n skipDefaultPrivilegeSubtraction: true,\n },\n );\n // pg-delta >= 1.0.0-alpha.32 groups plan statements into execution-aware\n // `units` with transaction boundaries. `renderPlanFiles` turns those into one\n // numbered SQL file per unit (header comments included). `includeTransactions:\n // false` because the CLI appliers already wrap each migration file in a single\n // transaction (Go\'s implicit ExecBatch / the TS BEGIN/COMMIT wrap), so embedded\n // BEGIN/COMMIT would nest; format options are applied per unit here instead of a\n // manual `formatSqlStatements` pass.\n const files = result\n ? renderPlanFiles(result.plan, {\n includeTransactions: false,\n sqlFormatOptions,\n })\n : [];\n const envelope = files.map((file, index) => ({\n order: index + 1,\n // The unit name is the rendered path minus its numeric prefix and `.sql`\n // extension (e.g. `001_after_enum_values.sql` -> `after_enum_values`).\n name: file.path.replace(/^\\d+_/, "").replace(/\\.sql$/, ""),\n transactionMode: file.unit.transactionMode,\n sql: file.sql,\n }));\n if (Deno.env.get("PGDELTA_DEBUG")) {\n console.error(\n JSON.stringify({\n statementCount: files.reduce((total, file) => total + file.unit.statements.length, 0),\n fileCount: files.length,\n source: source ? "connected" : "null",\n target: target ? "connected" : "null",\n includedSchemas: includedSchemas ?? null,\n skipDefaultPrivilegeSubtraction: true,\n }),\n );\n }\n console.log(JSON.stringify({ version: 1, files: envelope }));\n} catch (e) {\n console.error(e);\n // Emit a sentinel so the CLI runner can distinguish a real script crash from a\n // successful empty diff, even though the forced-exit non-zero code below is\n // suppressed by the "main worker has been destroyed" handling.\n console.error("PGDELTA_SCRIPT_ERROR");\n // Force close event loop\n throw new Error("");\n}\n// Force close the event loop on the success path too. When SOURCE/TARGET are\n// live database URLs the plan opens connections whose keepalive handles can keep\n// the Edge Runtime worker alive after the diff has been written, so the container\n// never exits and the CLI — which follows this container\'s logs — hangs\n// indefinitely at 0% CPU (supabase/pg-toolbelt#312).\nthrow new Error("");\n'; /** `templates/pgdelta_declarative_export.ts` — exports declarative file payloads. */ export const legacyPgDeltaDeclarativeExportScript = - '// This script is executed inside Edge Runtime by the CLI to export a target\n// schema as declarative file payloads. It accepts either live DB URLs or\n// catalog-file references for SOURCE/TARGET, which enables cached sync flows.\nimport {\n createPlan,\n deserializeCatalog,\n exportDeclarativeSchema,\n} from "npm:@supabase/pg-delta@1.0.0-alpha.20";\nimport { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.20/integrations/supabase";\n\nasync function resolveInput(ref: string | undefined) {\n if (!ref) {\n return null;\n }\n if (ref.startsWith("postgres://") || ref.startsWith("postgresql://")) {\n return ref;\n }\n const json = await Deno.readTextFile(ref);\n return deserializeCatalog(JSON.parse(json));\n}\n\nconst source = Deno.env.get("SOURCE");\nconst target = Deno.env.get("TARGET");\n\nconst includedSchemas = Deno.env.get("INCLUDED_SCHEMAS");\nif (includedSchemas) {\n const schemas = includedSchemas.split(",");\n const schemaFilter = {\n or: [{ "*/schema": schemas }, { "schema/name": schemas }],\n };\n supabase.filter = {\n and: [supabase.filter!, schemaFilter],\n } as unknown as typeof supabase.filter;\n}\n\nconst formatOptionsRaw = Deno.env.get("FORMAT_OPTIONS");\nlet formatOptions = undefined;\nif (formatOptionsRaw) {\n formatOptions = JSON.parse(formatOptionsRaw);\n}\ntry {\n const result = await createPlan(\n await resolveInput(source),\n await resolveInput(target),\n {\n ...supabase,\n skipDefaultPrivilegeSubtraction: true,\n },\n );\n if (!result) {\n console.log(\n JSON.stringify({\n version: 1,\n mode: "declarative",\n files: [],\n }),\n );\n } else {\n const output = exportDeclarativeSchema(result, {\n integration: supabase,\n formatOptions,\n });\n console.log(\n JSON.stringify(output, (_key, value) =>\n typeof value === "bigint" ? Number(value) : value,\n ),\n );\n }\n} catch (e) {\n console.error(e);\n // Force close event loop\n throw new Error("");\n}\n// Force close the event loop on the success path too. When SOURCE/TARGET are\n// live database URLs the plan opens connections whose keepalive handles can keep\n// the Edge Runtime worker alive after the export has been written, so the\n// container never exits and the CLI — which follows this container\'s logs —\n// hangs indefinitely at 0% CPU (supabase/pg-toolbelt#312).\nthrow new Error("");\n'; + '// This script is executed inside Edge Runtime by the CLI to export a target\n// schema as declarative file payloads. It accepts either live DB URLs or\n// catalog-file references for SOURCE/TARGET, which enables cached sync flows.\nimport {\n createPlan,\n deserializeCatalog,\n exportDeclarativeSchema,\n} from "npm:@supabase/pg-delta@1.0.0-alpha.20";\nimport { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.20/integrations/supabase";\n\nasync function resolveInput(ref: string | undefined) {\n if (!ref) {\n return null;\n }\n if (ref.startsWith("postgres://") || ref.startsWith("postgresql://")) {\n return ref;\n }\n const json = await Deno.readTextFile(ref);\n return deserializeCatalog(JSON.parse(json));\n}\n\nconst source = Deno.env.get("SOURCE");\nconst target = Deno.env.get("TARGET");\n\nconst includedSchemas = Deno.env.get("INCLUDED_SCHEMAS");\nif (includedSchemas) {\n const schemas = includedSchemas.split(",");\n const schemaFilter = {\n or: [{ "*/schema": schemas }, { "schema/name": schemas }],\n };\n supabase.filter = {\n and: [supabase.filter!, schemaFilter],\n } as unknown as typeof supabase.filter;\n}\n\nconst formatOptionsRaw = Deno.env.get("FORMAT_OPTIONS");\nlet formatOptions = undefined;\nif (formatOptionsRaw) {\n formatOptions = JSON.parse(formatOptionsRaw);\n}\ntry {\n const result = await createPlan(\n await resolveInput(source),\n await resolveInput(target),\n {\n ...supabase,\n skipDefaultPrivilegeSubtraction: true,\n },\n );\n if (!result) {\n console.log(\n JSON.stringify({\n version: 1,\n mode: "declarative",\n files: [],\n }),\n );\n } else {\n const output = exportDeclarativeSchema(result, {\n integration: supabase,\n formatOptions,\n });\n console.log(\n JSON.stringify(output, (_key, value) =>\n typeof value === "bigint" ? Number(value) : value,\n ),\n );\n }\n} catch (e) {\n console.error(e);\n // Emit a sentinel so the CLI runner can distinguish a real script crash from a\n // successful empty export, even though the forced-exit non-zero code below is\n // suppressed by the "main worker has been destroyed" handling.\n console.error("PGDELTA_SCRIPT_ERROR");\n // Force close event loop\n throw new Error("");\n}\n// Force close the event loop on the success path too. When SOURCE/TARGET are\n// live database URLs the plan opens connections whose keepalive handles can keep\n// the Edge Runtime worker alive after the export has been written, so the\n// container never exits and the CLI — which follows this container\'s logs —\n// hangs indefinitely at 0% CPU (supabase/pg-toolbelt#312).\nthrow new Error("");\n'; /** `templates/pgdelta_catalog_export.ts` — serializes a catalog snapshot for caching. */ export const legacyPgDeltaCatalogExportScript = - '// This script serializes a database catalog for caching/reuse in declarative\n// sync workflows, so later diff/export operations can run from file references.\nimport {\n createManagedPool,\n extractCatalog,\n serializeCatalog,\n stringifyCatalogSnapshot,\n} from "npm:@supabase/pg-delta@1.0.0-alpha.20";\n\nconst target = Deno.env.get("TARGET");\nconst role = Deno.env.get("ROLE") ?? undefined;\n\nif (!target) {\n console.error("TARGET is required");\n throw new Error("");\n}\nconst { pool, close } = await createManagedPool(target, { role });\n\ntry {\n const catalog = await extractCatalog(pool);\n console.log(stringifyCatalogSnapshot(serializeCatalog(catalog)));\n} catch (e) {\n console.error(e);\n // Force close event loop\n throw new Error("");\n} finally {\n await close();\n}\n// Force close the event loop on the success path too. The connection pool can\n// leave keepalive handles registered even after close() resolves, which keeps\n// the Edge Runtime worker (and therefore the container) alive after the catalog\n// has already been written to stdout. The CLI streams this container\'s logs with\n// Follow:true, so a worker that never exits hangs the parent `__catalog`\n// subprocess — and the declarative-sync command that spawned it — indefinitely\n// at 0% CPU (supabase/pg-toolbelt#312).\nthrow new Error("");\n'; + '// This script serializes a database catalog for caching/reuse in declarative\n// sync workflows, so later diff/export operations can run from file references.\nimport {\n createManagedPool,\n extractCatalog,\n serializeCatalog,\n stringifyCatalogSnapshot,\n} from "npm:@supabase/pg-delta@1.0.0-alpha.20";\n\nconst target = Deno.env.get("TARGET");\nconst role = Deno.env.get("ROLE") ?? undefined;\n\nif (!target) {\n console.error("TARGET is required");\n // Emit a sentinel so the CLI runner treats this as a real script crash rather\n // than a successful empty catalog, even though the forced-exit non-zero code is\n // suppressed by the "main worker has been destroyed" handling.\n console.error("PGDELTA_SCRIPT_ERROR");\n throw new Error("");\n}\nconst { pool, close } = await createManagedPool(target, { role });\n\ntry {\n const catalog = await extractCatalog(pool);\n console.log(stringifyCatalogSnapshot(serializeCatalog(catalog)));\n} catch (e) {\n console.error(e);\n // Emit a sentinel so the CLI runner can distinguish a real script crash from a\n // successful empty catalog, even though the forced-exit non-zero code below is\n // suppressed by the "main worker has been destroyed" handling.\n console.error("PGDELTA_SCRIPT_ERROR");\n // Force close event loop\n throw new Error("");\n} finally {\n await close();\n}\n// Force close the event loop on the success path too. The connection pool can\n// leave keepalive handles registered even after close() resolves, which keeps\n// the Edge Runtime worker (and therefore the container) alive after the catalog\n// has already been written to stdout. The CLI streams this container\'s logs with\n// Follow:true, so a worker that never exits hangs the parent `__catalog`\n// subprocess — and the declarative-sync command that spawned it — indefinitely\n// at 0% CPU (supabase/pg-toolbelt#312).\nthrow new Error("");\n'; /** `internal/pgdelta/templates/pgdelta_declarative_apply.ts` — applies declarative files to TARGET. */ export const legacyPgDeltaDeclarativeApplyScript = @@ -35,7 +35,7 @@ export const legacyPgDeltaDeclarativeApplyScript = * config field) is absent or empty. Mirrors Go's `DefaultPgDeltaNpmVersion` * (`apps/cli-go/pkg/config/pgdelta_version.go:7`). */ -export const LEGACY_DEFAULT_PG_DELTA_NPM_VERSION = "1.0.0-alpha.27"; +export const LEGACY_DEFAULT_PG_DELTA_NPM_VERSION = "1.0.0-alpha.32"; /** * The literal version baked into the embedded templates above, replaced by diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.errors.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.errors.ts index b52d173fbe..6e12afd324 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.errors.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.errors.ts @@ -48,6 +48,15 @@ export class LegacyDeclarativeParseOutputError extends Data.TaggedError( readonly message: string; }> {} +/** + * Parsing the pg-delta diff envelope failed. Byte-matches Go's + * `"failed to parse pg-delta diff output: " + err + ":\n" + stderr` + * (`apps/cli-go/internal/db/diff/pgdelta.go`, `parsePgDeltaDiffOutput`). + */ +export class LegacyPgDeltaDiffParseError extends Data.TaggedError("LegacyPgDeltaDiffParseError")<{ + readonly message: string; +}> {} + /** * Materializing the declarative export on disk failed. Byte-matches Go's * `WriteDeclarativeSchemas` errors (`declarative.go:239`): diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.integration.test.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.integration.test.ts index f6c816c461..c64f9f624a 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.integration.test.ts @@ -58,7 +58,20 @@ describe("legacyDiffPgDelta", () => { it.effect( "returns the SQL + stderr and passes the interpolated diff script + env + binds", () => { - const edge = fakeEdgeRuntime({ stdout: "ALTER TABLE x;\n", stderr: "warn" }); + const edge = fakeEdgeRuntime({ + stdout: JSON.stringify({ + version: 1, + files: [ + { + order: 1, + name: "schema_changes", + transactionMode: "transactional", + sql: "-- unit 1\n\nALTER TABLE x;", + }, + ], + }), + stderr: "warn", + }); return legacyDiffPgDelta(CTX, { targetRef: "postgresql://u:p@127.0.0.1:54320/postgres?connect_timeout=10", sourceRef: "supabase/.temp/catalog.json", @@ -67,7 +80,10 @@ describe("legacyDiffPgDelta", () => { }).pipe( Effect.tap((result) => Effect.sync(() => { - expect(result.sql).toBe("ALTER TABLE x;\n"); + // The envelope is parsed into per-unit files and a flattened SQL join. + expect(result.sql).toBe("-- unit 1\n\nALTER TABLE x;"); + expect(result.files).toHaveLength(1); + expect(result.files[0]?.name).toBe("schema_changes"); expect(result.stderr).toBe("warn"); const opts = edge.calls[0]!; expect(opts.errPrefix).toBe("error diffing schema"); @@ -139,6 +155,27 @@ describe("legacyDiffPgDelta", () => { Effect.provide(Layer.mergeAll(edge.layer, probe, BunServices.layer)), ); }); + + it.effect("fails with LegacyPgDeltaDiffParseError on a malformed envelope", () => { + const edge = fakeEdgeRuntime({ stdout: "not json{", stderr: "boom" }); + return legacyDiffPgDelta(CTX, { + targetRef: "postgresql://t", + sourceRef: "", + schema: [], + formatOptions: "", + }).pipe( + Effect.exit, + Effect.tap((exit) => + Effect.sync(() => { + expect(failError(exit)?.constructor.name).toBe("LegacyPgDeltaDiffParseError"); + const message = (failError(exit) as { message: string }).message; + expect(message).toContain("failed to parse pg-delta diff output"); + expect(message).toContain("boom"); + }), + ), + Effect.provide(Layer.mergeAll(edge.layer, probe, BunServices.layer)), + ); + }); }); describe("legacyDeclarativeExportPgDelta", () => { diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.ts index a7b49c230d..4dc3de042f 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.ts @@ -19,6 +19,7 @@ import { LegacyDeclarativeEdgeRuntimeError, LegacyDeclarativeEmptyOutputError, LegacyDeclarativeParseOutputError, + LegacyPgDeltaDiffParseError, } from "./legacy-pgdelta.errors.ts"; const PG_DELTA_NPM_REGISTRY_ENV = "PGDELTA_NPM_REGISTRY"; @@ -38,9 +39,32 @@ export interface LegacyDeclarativeOutput { readonly files: ReadonlyArray; } -/** Result of a pg-delta diff: the SQL statements plus edge-runtime stderr. */ +/** + * One execution-aware migration unit from a pg-delta diff plan. Mirrors Go's + * `PgDeltaPlanFile` (`internal/db/diff/pgdelta.go`): a numbered SQL file whose + * header comments record the unit number, transaction mode and boundary reason. + */ +interface LegacyPgDeltaPlanFile { + readonly order: number; + readonly name: string; + readonly transactionMode: string; + readonly sql: string; +} + +/** The pg-delta diff envelope. Mirrors Go's `PgDeltaDiffOutput`. */ +interface LegacyPgDeltaDiffOutput { + readonly version: number; + readonly files: ReadonlyArray; +} + +/** + * Result of a pg-delta diff: the per-unit plan `files`, a `sql` flattening of + * them (kept for `db diff` / declarative callers that consume one blob), and the + * edge-runtime `stderr`. + */ interface LegacyPgDeltaDiffResult { readonly sql: string; + readonly files: ReadonlyArray; readonly stderr: string; } @@ -189,7 +213,27 @@ export const legacyDiffPgDelta = Effect.fnUntraced(function* ( denoVersion: ctx.denoVersion, }) .pipe(Effect.mapError(toDeclarativeEdgeRuntimeError)); - return { sql: result.stdout, stderr: result.stderr } satisfies LegacyPgDeltaDiffResult; + // The template always prints the diff envelope on the success path, even for an + // empty plan (`{"version":1,"files":[]}`); a truly empty stdout means no envelope + // was produced, which we surface as "no changes" rather than a parse error. + // Mirrors Go's `parsePgDeltaDiffOutput` (`internal/db/diff/pgdelta.go`). + if (result.stdout.trim().length === 0) { + return { sql: "", files: [], stderr: result.stderr } satisfies LegacyPgDeltaDiffResult; + } + const envelope = yield* Effect.try({ + try: () => JSON.parse(result.stdout) as LegacyPgDeltaDiffOutput, + catch: (cause) => + new LegacyPgDeltaDiffParseError({ + message: `failed to parse pg-delta diff output: ${ + cause instanceof Error ? cause.message : String(cause) + }:\n${result.stderr}`, + }), + }); + const files = envelope.files ?? []; + // Flatten to one blob for callers that need it; unit header comments keep the + // transaction boundaries visible (mirrors Go's `joinPgDeltaFiles`). + const sql = files.map((file) => file.sql).join("\n\n"); + return { sql, files, stderr: result.stderr } satisfies LegacyPgDeltaDiffResult; }); /** diff --git a/apps/cli/src/legacy/shared/legacy-db-connection.sql-pg.layer.ts b/apps/cli/src/legacy/shared/legacy-db-connection.sql-pg.layer.ts index f291140c54..7553b1a3e0 100644 --- a/apps/cli/src/legacy/shared/legacy-db-connection.sql-pg.layer.ts +++ b/apps/cli/src/legacy/shared/legacy-db-connection.sql-pg.layer.ts @@ -2,8 +2,9 @@ import { readFileSync } from "node:fs"; import * as net from "node:net"; import type { ConnectionOptions } from "node:tls"; import { PgClient } from "@effect/sql-pg"; -import { Duration, Effect, Layer, Redacted, type Scope } from "effect"; +import { Duration, Effect, Exit, Layer, Scope } from "effect"; import * as Reactivity from "effect/unstable/reactivity/Reactivity"; +import { ConnectionError, SqlError } from "effect/unstable/sql/SqlError"; // `pg` is also `@effect/sql-pg`'s transitive driver; we depend on it directly only // for the COPY protocol (which `@effect/sql-pg` does not expose). Keep the direct // `pg` version constraint in package.json aligned with the one `@effect/sql-pg` @@ -27,10 +28,16 @@ import { legacyResolveHostsOverHttps } from "./legacy-db-dns.ts"; // node-postgres honors `queryMode: "extended"` to force the Parse/Bind/Execute // protocol (`pg/lib/query.js` `requiresPreparation`), but `@types/pg` doesn't declare // it. Augment `QueryConfig` so `queryRaw` can request it without an `as` cast. +// pg-pool likewise honors a `verify(client, callback)` option — run for every +// brand-new physical connection before it is handed to a waiting checkout +// (`pg-pool/index.js` `_acquireClient`) — that `@types/pg` doesn't declare either. declare module "pg" { interface QueryConfig { queryMode?: "extended" | "simple"; } + interface PoolConfig { + verify?: (client: import("pg").PoolClient, callback: (err?: Error) => void) => void; + } } // Go's role step-down (`apps/cli-go/internal/utils/connect.go:200-220`, @@ -327,6 +334,178 @@ export function legacySslConfigsFor( return [legacySslOptionFor(sslmode, false, servername, caCert, clientCert)]; } +/** + * The raw `pg.ClientConfig` for a dial target, choosing the connection-string form + * whenever a libpq `options`/`runtimeParams` payload must reach the server (see + * `legacyBuildConnectionUrl`) and discrete fields otherwise (to avoid round-tripping + * the password through a URL). `copyToCsv` / `queryRaw` reuse it to open a dedicated + * node-postgres client for the COPY protocol and full result metadata (neither is + * surfaced by `@effect/sql-pg`), against whichever target the primary connection won. + */ +export function legacyBuildRawPgConfig( + cfg: LegacyPgConnInput, + host: string, + port: number, + sslOption: boolean | ConnectionOptions | undefined, + connectTimeoutSeconds: number, +): Pg.ClientConfig { + const hasOptions = legacyMergedConnectionOptions(cfg) !== undefined; + return { + ...(hasOptions + ? { connectionString: legacyBuildConnectionUrl(cfg, host, port) } + : { host, port, user: cfg.user, password: cfg.password, database: cfg.database }), + ...(sslOption === undefined ? {} : { ssl: sslOption }), + connectionTimeoutMillis: connectTimeoutSeconds * 1000, + }; +} + +/** + * The `pg.PoolConfig` for the primary pooled connection. Extends the raw client + * config with the two pool controls this layer depends on: + * + * - `max: 1` — a single physical connection, so a session-scoped `SET SESSION ROLE` + * (the remote step-down) and any session GUCs persist across `exec`/`query` calls. + * - `idleTimeoutMillis: 0` — node-postgres documents `0` as disabling auto-reaping of + * idle connections. `PgClient.make` leaves this unset (→ node-postgres default + * 10_000ms), which during a long-idle `db pull` (shadow-DB provisioning + diff + + * interactive confirm prompt) reaps the stepped-down connection after 10s idle and + * transparently redials a fresh one for the final migration-table write; that fresh + * connection never ran the step-down, so the DDL executes as the bare `cli_login_*` + * role and fails with `permission denied` (42501). Disabling the reaper keeps the + * single stepped-down connection alive for the whole session. + * + * `application_name` mirrors `PgClient.make`'s default so the server-side session name + * is unchanged from the previous `PgClient.make` path. + * + * When `stepDownRequired`, the pool also gets the `verify` step-down hook so every + * new physical connection runs `SET SESSION ROLE postgres` before it is handed out + * (see `legacyPoolStepDownVerify`). + */ +export function legacyBuildPoolConfig( + cfg: LegacyPgConnInput, + host: string, + port: number, + sslOption: boolean | ConnectionOptions | undefined, + connectTimeoutSeconds: number, + stepDownRequired: boolean, +): Pg.PoolConfig { + return { + ...legacyBuildRawPgConfig(cfg, host, port, sslOption, connectTimeoutSeconds), + idleTimeoutMillis: 0, + max: 1, + application_name: "@effect/sql-pg", + ...(stepDownRequired ? { verify: legacyPoolStepDownVerify } : {}), + }; +} + +// Minimal structural views of `pg.Pool` / `pg.PoolClient` used by the pool hooks, +// so the wiring can be unit-tested with a tiny fake at the driver boundary instead +// of a live pool. A real `pg.Pool`/`pg.PoolClient` satisfies these (their `on` +// overloads and `query` signatures are wider). +interface LegacyStepDownClient { + readonly query: (sql: string) => Promise; +} +interface LegacyPoolErrorSource { + readonly on: (event: "error", listener: (error: Error) => void) => unknown; +} + +/** + * pg-pool `verify` hook that re-runs the remote role step-down on EVERY new + * physical connection, matching Go's `AfterConnect` (`connect.go:355-361`) — + * including the silent redial after a dropped connection, which the post-connect + * one-shot in `connect` can't reach. pg-pool invokes `verify` for a brand-new + * client BEFORE resolving the pending checkout (`pg-pool/index.js` + * `_acquireClient`), so the `SET` completes before any caller query runs on that + * connection. (A `"connect"`-listener `client.query()` would instead overlap + * pg-pool's own synchronous dispatch of the checked-out query — node-postgres' + * "Calling client.query() when the client is already executing a query" + * deprecation, whose internal queueing pg@9 removes.) A failure propagates to + * the checkout and fails the caller's query, like Go's `AfterConnect` failing + * the connect. + */ +export function legacyPoolStepDownVerify( + client: LegacyStepDownClient, + callback: (err?: Error) => void, +): void { + client.query(SET_SESSION_ROLE).then( + () => callback(), + (error) => callback(error instanceof Error ? error : new Error(String(error))), + ); +} + +/** + * Swallow the pool's async background errors, like `PgClient.make`: an idle + * client's connection-level failure emits `"error"` on the pool and would crash + * the process without a listener; the next checkout simply redials (and the + * redial re-runs the `verify` step-down). + */ +export function legacyInstallPoolErrorSwallow(pool: LegacyPoolErrorSource): void { + pool.on("error", () => {}); +} + +// Minimal structural view of the `pg.Pool` surface `legacyAcquireProbedPool` +// drives — a real `pg.Pool` satisfies it (its `on`/`query`/`end` signatures are +// wider), and a tiny fake can stand in at the driver boundary for unit tests. +interface LegacyProbePool extends LegacyPoolErrorSource { + readonly query: (sql: string) => Promise; + readonly end: () => Promise; +} + +/** + * Acquire a `pg` pool and probe it with `SELECT 1`, guaranteeing the pool is + * `.end()`ed on EVERY non-success path — a probe rejection, a + * `connectTimeoutSeconds` timeout, or an interruption. + * + * The pool.end() finalizer is registered the MOMENT the pool is constructed, via + * `Effect.acquireRelease` with a synchronous (never-failing) resource step, BEFORE + * the probe runs. So even if the probe rejects or the outer timeout fires (a + * black-holed host), the finalizer is already installed and closes the pool along + * with its in-flight dial (sockets/timers) when the scope unwinds. Upstream + * `@effect/sql-pg`'s `PgClient.make` instead probes INSIDE the acquireRelease + * acquire, so a failed/timed-out probe never installs the finalizer and leaks the + * pool — we deliberately diverge to fix that leak while keeping the SqlError / + * ConnectionError shapes identical. + * + * The probe pool is generic so tests can inject a lightweight fake at the driver + * boundary; a real `pg.Pool` is returned unchanged for `PgClient.fromPool`. + */ +export const legacyAcquireProbedPool =

( + makePool: () => P, + connectTimeoutSeconds: number, +): Effect.Effect => + Effect.gen(function* () { + const pool = yield* Effect.acquireRelease(Effect.sync(makePool), (pool) => + Effect.promise(() => pool.end()).pipe(Effect.timeoutOption(1000)), + ); + legacyInstallPoolErrorSwallow(pool); + yield* Effect.tryPromise({ + try: () => pool.query("SELECT 1"), + catch: (cause) => + new SqlError({ + reason: new ConnectionError({ + cause, + message: "PgClient: Failed to connect", + operation: "connect", + }), + }), + }).pipe( + Effect.timeoutOrElse({ + duration: Duration.seconds(connectTimeoutSeconds), + orElse: () => + Effect.fail( + new SqlError({ + reason: new ConnectionError({ + cause: new Error("Connection timed out"), + message: "PgClient: Connection timed out", + operation: "connect", + }), + }), + ), + }), + ); + return pool; + }); + /** * Default `LegacyDbConnection` layer, backed by `@effect/sql-pg` (pure-JS `pg` * driver, no native addon — bundles under `bun build --compile`). Each @@ -360,53 +539,46 @@ const connect = ( dialTargets.push({ dialHost, port, servername: dialHost === host ? undefined : host }); } } - // Route through the connection string whenever a libpq `options` param OR - // parsed `runtimeParams` are present, so both reach the live connection. - const hasOptions = legacyMergedConnectionOptions(cfg) !== undefined; // Connect timeout parity: Go's `ToPostgresURL` always sets `connect_timeout`, // defaulting to 10s (`connect.go:24-28`); `ConnectLocalPostgres` uses 2s for // local (`connect.go:143-145`). A DSN/`PGCONNECT_TIMEOUT` value (>0) overrides // both. Without this a black-holed host would hang to the OS/driver default. const connectTimeoutSeconds = cfg.connectTimeoutSeconds ?? (isLocal ? 2 : 10); + // Whether the remote step-down runs on this connection. Go installs the + // `AfterConnect` hook only on the remote path (`ConnectByConfigStream`, + // `connect.go:342-362`), not `ConnectLocalPostgres`, so gate on `!isLocal`. + const stepDownRequired = !isLocal && needsRoleStepDown(cfg.user); + // Build the primary connection over a self-managed `pg.Pool` (via + // `PgClient.fromPool`) rather than `PgClient.make`, so we control two pool + // behaviors `PgClient.make` does not expose: `idleTimeoutMillis: 0` (never reap + // the single pooled connection — see `legacyBuildPoolConfig`; the fix for the + // `db pull` step-down loss) and the per-connection role step-down `verify` hook + // (see `legacyPoolStepDownVerify`). The `acquire` uses `legacyAcquireProbedPool`: + // probe with `SELECT 1`, bound the connect by `connectTimeoutSeconds`, and close + // the pool on scope exit AND on every failure/timeout (the leak `PgClient.make` + // has). `probe` (below) runs each attempt in a forked scope so a failed fallback + // attempt's pool closes immediately, before the next host is dialed. const makeClient = ( dialHost: string, port: number, sslOption: boolean | ConnectionOptions | undefined, - ) => - PgClient.make({ - // When a libpq `options` param is present, route everything through the - // connection string so it reaches the server (see `buildConnectionUrl`); - // otherwise pass discrete fields to avoid round-tripping the password. - ...(hasOptions - ? { url: Redacted.make(legacyBuildConnectionUrl(cfg, dialHost, port)) } - : { - host: dialHost, + ) => { + const acquire = legacyAcquireProbedPool( + () => + new Pg.Pool( + legacyBuildPoolConfig( + cfg, + dialHost, port, - username: cfg.user, - password: Redacted.make(cfg.password), - database: cfg.database, - }), - // TLS parity with Go (`internal/utils/connect.go`): see `legacySslOptionFor`. - ...(sslOption === undefined ? {} : { ssl: sslOption }), - connectTimeout: Duration.seconds(connectTimeoutSeconds), - maxConnections: 1, - }).pipe(Effect.provide(Reactivity.layer)); - - // The raw `pg.ClientConfig` for the same dial target, mirroring `makeClient`'s - // discrete-vs-url choice. `copyToCsv` uses it to open a dedicated node-postgres - // connection for the COPY protocol (which `@effect/sql-pg` does not expose), - // against whichever target the primary connection won. - const buildRawPgConfig = ( - dialHost: string, - port: number, - sslOption: boolean | ConnectionOptions | undefined, - ): Pg.ClientConfig => ({ - ...(hasOptions - ? { connectionString: legacyBuildConnectionUrl(cfg, dialHost, port) } - : { host: dialHost, port, user: cfg.user, password: cfg.password, database: cfg.database }), - ...(sslOption === undefined ? {} : { ssl: sslOption }), - connectionTimeoutMillis: connectTimeoutSeconds * 1000, - }); + sslOption, + connectTimeoutSeconds, + stepDownRequired, + ), + ), + connectTimeoutSeconds, + ); + return PgClient.fromPool({ acquire }).pipe(Effect.provide(Reactivity.layer)); + }; // Go's `ConnectByUrl` calls `SetConnectSuggestion(err)` on every connect failure // (`connect.go:187`), mapping the driver error to an actionable hint that replaces @@ -483,7 +655,7 @@ const connect = ( // failed attempt used TLS (`pgconn.go:182`, gated on `fc.TLSConfig != nil`); // a TLS config is any non-plaintext `ssl` value. usedTls: ssl !== undefined && ssl !== false, - rawConfig: buildRawPgConfig(dialHost, port, ssl), + rawConfig: legacyBuildRawPgConfig(cfg, dialHost, port, ssl, connectTimeoutSeconds), }), ), ); @@ -500,10 +672,26 @@ const connect = ( // follow-up query. The winning attempt's `rawConfig` is carried out so `copyToCsv` // can reuse the exact dial target the primary connection succeeded against. const probe = (attempt: (typeof attempts)[number]) => - attempt.client.pipe( - Effect.tap((candidate) => candidate`select 1`), - Effect.map((candidate) => ({ candidate, rawConfig: attempt.rawConfig })), - ); + Effect.gen(function* () { + // Run each attempt's pool in its OWN scope, forked from the session scope. + // Forking (not a detached `Scope.make`) means an abandoned winner — e.g. a + // later step-down failure — is still closed when the session scope unwinds. + // On a probe FAILURE we close the child immediately, so a failed fallback + // attempt's pool (and its in-flight dial) is released before the next host is + // dialed, not left open until session end. On SUCCESS we leave the child open + // (a child of the session scope), so the winning pool lives for the whole + // session and closes with it. + const sessionScope = yield* Scope.Scope; + const attemptScope = yield* Scope.fork(sessionScope); + return yield* attempt.client.pipe( + Effect.tap((candidate) => candidate`select 1`), + Effect.map((candidate) => ({ candidate, rawConfig: attempt.rawConfig })), + Scope.provide(attemptScope), + Effect.onExit((exit) => + Exit.isSuccess(exit) ? Effect.void : Scope.close(attemptScope, exit), + ), + ); + }); const lastIndex = attempts.length - 1; const { candidate: client, rawConfig: winningRawConfig } = yield* attempts .slice(0, lastIndex) @@ -521,9 +709,13 @@ const connect = ( // Step down from the temp/privileged login role before any further SQL — but // only for remote connections: Go installs this hook in `ConnectByConfigStream`, // not `ConnectLocalPostgres`, so a local `--db-url` using `supabase_admin`/ - // `cli_login_*` must not run it. `maxConnections: 1` guarantees the single - // physical connection is reused, so the session-scoped role persists for `exec`. - if (!isLocal && needsRoleStepDown(cfg.user)) { + // `cli_login_*` must not run it. The pool's `"connect"` hook already ran this on + // the physical connection (and on any silent redial); this explicit one-shot is + // the fail-fast path — the hook swallows errors, so a real role-privilege problem + // only surfaces here, as `LegacyDbConnectError: failed to set session role: ...` + // (Go parity). `max: 1` + `idleTimeoutMillis: 0` keep the stepped-down connection + // alive so the session-scoped role persists for every later `exec`/`query`. + if (stepDownRequired) { yield* client.unsafe(SET_SESSION_ROLE).pipe( Effect.asVoid, Effect.mapError( @@ -563,7 +755,7 @@ const connect = ( try: () => fresh.connect(), catch: toConnectError, }); - if (!isLocal && needsRoleStepDown(cfg.user)) { + if (stepDownRequired) { yield* Effect.tryPromise({ try: () => fresh.query(SET_SESSION_ROLE), catch: (error) => diff --git a/apps/cli/src/legacy/shared/legacy-db-connection.sql-pg.unit.test.ts b/apps/cli/src/legacy/shared/legacy-db-connection.sql-pg.unit.test.ts index c3407c8018..334ecda513 100644 --- a/apps/cli/src/legacy/shared/legacy-db-connection.sql-pg.unit.test.ts +++ b/apps/cli/src/legacy/shared/legacy-db-connection.sql-pg.unit.test.ts @@ -1,7 +1,14 @@ +import { EventEmitter } from "node:events"; +import { Effect, Exit } from "effect"; import { describe, expect, it } from "vitest"; import { + legacyAcquireProbedPool, legacyBuildConnectionUrl, + legacyBuildPoolConfig, + legacyBuildRawPgConfig, + legacyInstallPoolErrorSwallow, + legacyPoolStepDownVerify, legacyIsTerminalConnectError, legacyIsUnixSocketHost, legacyMergedConnectionOptions, @@ -271,6 +278,185 @@ describe("legacySslConfigsFor (pgconn fallback list)", () => { }); }); +describe("legacyBuildRawPgConfig", () => { + const base = { user: "postgres", password: "pw", port: 5432, database: "postgres", host: "h" }; + + it("uses discrete fields (no connection string) when no options/runtimeParams", () => { + const c = legacyBuildRawPgConfig( + { ...base }, + "db.example.com", + 5432, + { rejectUnauthorized: false }, + 10, + ); + expect(c).toMatchObject({ + host: "db.example.com", + port: 5432, + user: "postgres", + password: "pw", + database: "postgres", + connectionTimeoutMillis: 10_000, + ssl: { rejectUnauthorized: false }, + }); + expect(c.connectionString).toBeUndefined(); + }); + + it("routes through a connection string when a libpq options payload is present", () => { + const c = legacyBuildRawPgConfig( + { ...base, options: "reference=abc" }, + "db.example.com", + 6543, + false, + 2, + ); + expect(c.connectionString).toContain("@db.example.com:6543/"); + expect(c.connectionString).toContain("options=reference%3Dabc"); + expect(c.host).toBeUndefined(); + expect(c.connectionTimeoutMillis).toBe(2000); + expect(c.ssl).toBe(false); + }); + + it("omits the ssl field entirely when the ssl option is undefined", () => { + const c = legacyBuildRawPgConfig({ ...base }, "h", 5432, undefined, 5); + expect("ssl" in c).toBe(false); + }); +}); + +describe("legacyBuildPoolConfig", () => { + const base = { user: "postgres", password: "pw", port: 5432, database: "postgres", host: "h" }; + + it("disables idle reaping (idleTimeoutMillis 0) and pins one connection (max 1) for a remote config", () => { + // The `db pull` bug: `PgClient.make` left idleTimeoutMillis unset (node-postgres + // default 10s), reaping the stepped-down connection during a long idle. + const c = legacyBuildPoolConfig( + { ...base }, + "db.example.com", + 5432, + { rejectUnauthorized: false }, + 10, + true, + ); + expect(c.idleTimeoutMillis).toBe(0); + expect(c.max).toBe(1); + expect(c.application_name).toBe("@effect/sql-pg"); + // carries the raw client config fields through + expect(c).toMatchObject({ host: "db.example.com", connectionTimeoutMillis: 10_000 }); + }); + + it("disables idle reaping and pins one connection for a local (plaintext) config too", () => { + const c = legacyBuildPoolConfig({ ...base }, "127.0.0.1", 54322, false, 2, false); + expect(c.idleTimeoutMillis).toBe(0); + expect(c.max).toBe(1); + expect(c.ssl).toBe(false); + }); + + it("installs the step-down verify hook only when required", () => { + // Every NEW physical connection (initial + silent redials) runs the step-down + // before pg-pool hands it to a checkout, mirroring Go's per-connection + // AfterConnect. + const remote = legacyBuildPoolConfig({ ...base }, "db.example.com", 5432, false, 10, true); + expect(remote.verify).toBe(legacyPoolStepDownVerify); + const local = legacyBuildPoolConfig({ ...base }, "127.0.0.1", 54322, false, 2, false); + expect("verify" in local).toBe(false); + }); +}); + +describe("legacyPoolStepDownVerify", () => { + it("runs SET SESSION ROLE postgres and reports success to the pool", async () => { + const queries: Array = []; + const client = { query: (sql: string) => (queries.push(sql), Promise.resolve()) }; + const done = await new Promise((resolve) => { + legacyPoolStepDownVerify(client, resolve); + }); + expect(queries).toEqual(["SET SESSION ROLE postgres"]); + expect(done).toBeUndefined(); + }); + + it("propagates a failing step-down to the pool callback so the checkout fails (Go AfterConnect parity)", async () => { + const failure = new Error("permission denied to set role"); + const client = { query: () => Promise.reject(failure) }; + const done = await new Promise((resolve) => { + legacyPoolStepDownVerify(client, resolve); + }); + expect(done).toBe(failure); + }); + + it("wraps a non-Error rejection into an Error for the pool callback", async () => { + const client = { query: () => Promise.reject("boom") }; + const done = await new Promise((resolve) => { + legacyPoolStepDownVerify(client, resolve); + }); + expect(done).toBeInstanceOf(Error); + expect(String(done)).toContain("boom"); + }); +}); + +describe("legacyInstallPoolErrorSwallow", () => { + it("swallows pool background errors so a dropped idle client never crashes the process", () => { + const pool = new EventEmitter(); + legacyInstallPoolErrorSwallow(pool); + expect(pool.listenerCount("error")).toBe(1); + expect(() => pool.emit("error", new Error("idle client boom"))).not.toThrow(); + }); +}); + +describe("legacyAcquireProbedPool", () => { + // Tiny fake at the driver boundary: records query/end calls. Standing in for a + // real `pg.Pool` proves the pool is always `.end()`ed — the leak this fixes. + function makeFakePool(query: () => Promise) { + const calls = { query: 0, end: 0 }; + const pool = { + query: (_sql: string) => { + calls.query += 1; + return query(); + }, + end: () => { + calls.end += 1; + return Promise.resolve(); + }, + on: (_event: "error", _listener: (error: Error) => void) => undefined, + }; + return { pool, calls }; + } + + it("ends the pool when the connect probe rejects", async () => { + const fake = makeFakePool(() => Promise.reject(new Error("ECONNREFUSED"))); + const exit = await Effect.runPromiseExit( + legacyAcquireProbedPool(() => fake.pool, 2).pipe(Effect.scoped), + ); + expect(Exit.isFailure(exit)).toBe(true); + expect(fake.calls.query).toBe(1); + // The finalizer, installed the moment the pool exists, closes it on failure. + expect(fake.calls.end).toBe(1); + }); + + it("ends the pool when the connect probe times out (black-holed host)", async () => { + // A never-resolving probe models a black-holed host: `timeoutOrElse` fires and + // the already-installed finalizer still closes the pool + its in-flight dial. + const fake = makeFakePool(() => new Promise(() => {})); + const exit = await Effect.runPromiseExit( + legacyAcquireProbedPool(() => fake.pool, 0.05).pipe(Effect.scoped), + ); + expect(Exit.isFailure(exit)).toBe(true); + expect(fake.calls.end).toBe(1); + }); + + it("keeps the pool open until the scope closes on a successful probe", async () => { + const fake = makeFakePool(() => Promise.resolve({ rows: [{ "?column?": 1 }] })); + const observed = await Effect.runPromise( + Effect.gen(function* () { + const pool = yield* legacyAcquireProbedPool(() => fake.pool, 2); + // While the scope is open the pool is live and not yet ended. + return { isSamePool: pool === fake.pool, endWhileOpen: fake.calls.end }; + }).pipe(Effect.scoped), + ); + expect(observed.isSamePool).toBe(true); + expect(observed.endWhileOpen).toBe(0); + // Closing the scope ends the pool exactly once. + expect(fake.calls.end).toBe(1); + }); +}); + describe("legacyIsUnixSocketHost", () => { it("treats an absolute path as a unix socket and a hostname/IP as not", () => { expect(legacyIsUnixSocketHost("/var/run/postgresql")).toBe(true); diff --git a/apps/cli/src/legacy/shared/legacy-edge-runtime-script.layer.integration.test.ts b/apps/cli/src/legacy/shared/legacy-edge-runtime-script.layer.integration.test.ts new file mode 100644 index 0000000000..dc7a3fb052 --- /dev/null +++ b/apps/cli/src/legacy/shared/legacy-edge-runtime-script.layer.integration.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "@effect/vitest"; +import { BunServices } from "@effect/platform-bun"; +import { Effect, Exit, Layer, Option } from "effect"; + +import { LegacyDebugFlag, LegacyNetworkIdFlag } from "../../shared/legacy/global-flags.ts"; +import { RuntimeInfo } from "../../shared/runtime/runtime-info.service.ts"; +import { LegacyCliConfig } from "../config/legacy-cli-config.service.ts"; +import { LegacyDockerRun } from "./legacy-docker-run.service.ts"; +import { legacyEdgeRuntimeScriptLayer } from "./legacy-edge-runtime-script.layer.ts"; +import { LegacyEdgeRuntimeScript } from "./legacy-edge-runtime-script.service.ts"; + +// Fakes a `docker run --rm` capture: the pg-delta scripts throw to force the +// worker to exit, so a real diff always comes back with a non-zero exit and +// "main worker has been destroyed" in stderr — the crash path only differs by +// the sentinel line the templates' catch blocks add. +function fakeDocker(result: { exitCode: number; stdout?: string; stderr?: string }) { + return Layer.succeed(LegacyDockerRun, { + runCapture: () => + Effect.succeed({ + exitCode: result.exitCode, + stdout: new TextEncoder().encode(result.stdout ?? ""), + stderr: result.stderr ?? "", + }), + run: () => Effect.die("unused"), + runStream: () => Effect.die("unused"), + }); +} + +// `workdir` points at a directory without `supabase/.temp/edge-runtime-version`, +// so the image resolver falls back to the default tag (the read is orElseSucceed). +const cliConfig = Layer.succeed(LegacyCliConfig, { + profile: "supabase", + apiUrl: "https://api.supabase.com", + projectHost: "supabase.co", + poolerHost: "supabase.co", + dashboardUrl: "https://supabase.com/dashboard", + accessToken: Option.none(), + projectId: Option.none(), + workdir: "/nonexistent-workdir", + userAgent: "test", +}); + +function setup(result: { exitCode: number; stdout?: string; stderr?: string }) { + return legacyEdgeRuntimeScriptLayer.pipe( + Layer.provideMerge( + Layer.mergeAll( + fakeDocker(result), + cliConfig, + Layer.succeed(RuntimeInfo, { + cwd: "/nonexistent-workdir", + platform: "darwin", + arch: "arm64", + homeDir: "/home/test", + execPath: "/usr/bin/bun", + pid: 1, + }), + Layer.succeed(LegacyDebugFlag, false), + Layer.succeed(LegacyNetworkIdFlag, Option.none()), + BunServices.layer, + ), + ), + ); +} + +const runScript = Effect.fnUntraced(function* () { + const edge = yield* LegacyEdgeRuntimeScript; + return yield* edge.run({ + script: "console.log('x')", + env: {}, + binds: [], + errPrefix: "error diffing schema", + denoVersion: 2, + }); +}); + +describe("legacyEdgeRuntimeScriptLayer sentinel handling", () => { + it.effect( + "fails with the real error when the script crashes behind the worker-destroyed message", + () => { + // The catch block emits the real error, the sentinel, and the worker is torn + // down — all with a non-zero exit. This must surface, not read as an empty diff. + const stderr = + "error: permission denied for table pg_user_mapping\n" + + "PGDELTA_SCRIPT_ERROR\n" + + "worker boot error\nmain worker has been destroyed\n"; + return runScript().pipe( + Effect.exit, + Effect.tap((exit) => + Effect.sync(() => { + expect(Exit.isFailure(exit)).toBe(true); + const error = Exit.isFailure(exit) + ? exit.cause.reasons.find((r) => r._tag === "Fail")?.error + : undefined; + const message = (error as { message: string } | undefined)?.message ?? ""; + expect(message).toContain("error diffing schema: error running script:"); + // The actionable permission error must reach the user. + expect(message).toContain("permission denied for table pg_user_mapping"); + }), + ), + Effect.provide(setup({ exitCode: 1, stdout: "", stderr })), + ); + }, + ); + + it.effect("still succeeds on a worker-destroyed exit when no sentinel is present", () => { + // Success path: the template forces the worker to exit after writing output, so + // the exit is non-zero with "main worker has been destroyed" but no sentinel. + return runScript().pipe( + Effect.tap((res) => + Effect.sync(() => { + expect(res.stdout).toBe("ALTER TABLE x;\n"); + }), + ), + Effect.provide( + setup({ + exitCode: 1, + stdout: "ALTER TABLE x;\n", + stderr: "main worker has been destroyed\n", + }), + ), + ); + }); +}); diff --git a/apps/cli/src/legacy/shared/legacy-edge-runtime-script.layer.ts b/apps/cli/src/legacy/shared/legacy-edge-runtime-script.layer.ts index 5b7d273c78..3fa93dad83 100644 --- a/apps/cli/src/legacy/shared/legacy-edge-runtime-script.layer.ts +++ b/apps/cli/src/legacy/shared/legacy-edge-runtime-script.layer.ts @@ -10,6 +10,7 @@ import { LegacyDockerRun } from "./legacy-docker-run.service.ts"; import { legacyResolveEdgeRuntimeImage } from "./legacy-edge-runtime-image.ts"; import { LegacyEdgeRuntimeScriptError } from "./legacy-edge-runtime-script.errors.ts"; import { + LEGACY_EDGE_RUNTIME_SCRIPT_ERROR_SENTINEL, LegacyEdgeRuntimeScript, legacyBuildEdgeRuntimeEntrypoint, legacyBuildEdgeRuntimeStartCmd, @@ -132,6 +133,21 @@ export const legacyEdgeRuntimeScriptLayer = Layer.effect( ); } + // The pg-delta templates force the worker to exit by throwing, so a + // script crash is masked by the "main worker has been destroyed" + // suppression above. The sentinel — printed only by the templates' + // catch blocks — marks a real failure so the collected stderr (which + // holds the real error) reaches the user instead of looking like an + // empty diff. Byte-for-byte port of Go's check in + // apps/cli-go/internal/utils/edgeruntime.go. + if (result.stderr.includes(LEGACY_EDGE_RUNTIME_SCRIPT_ERROR_SENTINEL)) { + return yield* Effect.fail( + new LegacyEdgeRuntimeScriptError({ + message: `${opts.errPrefix}: error running script:\n${result.stderr}`, + }), + ); + } + return { stdout: new TextDecoder().decode(result.stdout), stderr: result.stderr, diff --git a/apps/cli/src/legacy/shared/legacy-edge-runtime-script.service.ts b/apps/cli/src/legacy/shared/legacy-edge-runtime-script.service.ts index 8f6e970817..8d679b8be6 100644 --- a/apps/cli/src/legacy/shared/legacy-edge-runtime-script.service.ts +++ b/apps/cli/src/legacy/shared/legacy-edge-runtime-script.service.ts @@ -2,6 +2,19 @@ import { Context, type Effect, Option } from "effect"; import type { LegacyEdgeRuntimeScriptError } from "./legacy-edge-runtime-script.errors.ts"; +/** + * Printed to stderr by the pg-delta Deno templates when their body throws (see + * the `catch` blocks in `apps/cli-go/internal/db/diff/templates/*.ts`). The + * templates force the edge-runtime worker to exit by throwing on both the + * success and failure paths, and that non-zero exit is otherwise suppressed when + * stderr contains `"main worker has been destroyed"`. Without a distinct marker + * a crashed script is indistinguishable from a successful empty diff, so + * `db pull` reports "No schema changes found" while the real error is swallowed. + * Byte-for-byte mirror of Go's `EdgeRuntimeScriptErrorSentinel` + * (`apps/cli-go/internal/utils/edgeruntime.go`). See supabase/cli#5826. + */ +export const LEGACY_EDGE_RUNTIME_SCRIPT_ERROR_SENTINEL = "PGDELTA_SCRIPT_ERROR"; + /** A file dropped alongside `index.ts` in the container's working directory. */ export interface LegacyEdgeRuntimeFile { readonly name: string; diff --git a/apps/cli/tests/helpers/legacy-mocks.ts b/apps/cli/tests/helpers/legacy-mocks.ts index d8ab445cc2..e02e241fbb 100644 --- a/apps/cli/tests/helpers/legacy-mocks.ts +++ b/apps/cli/tests/helpers/legacy-mocks.ts @@ -4,7 +4,8 @@ import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { type ApiClient, makeApiClient, type SupabaseApiConfigError } from "@supabase/api/effect"; -import { Effect, Layer, Option, Redacted } from "effect"; +import { Effect, FileSystem, Layer, Option, Redacted } from "effect"; +import { PlatformError, SystemError } from "effect/PlatformError"; import * as HttpClient from "effect/unstable/http/HttpClient"; import * as HttpClientError from "effect/unstable/http/HttpClientError"; import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; @@ -642,6 +643,47 @@ export function useLegacyTempWorkdir(prefix = "supabase-legacy-test-"): { }; } +// --------------------------------------------------------------------------- +// Failing filesystem — wraps the real Bun `FileSystem` and fails the Nth +// `writeFileString` call with a `PlatformError`, so cleanup-on-failure paths +// (e.g. the pg-delta multi-file migration writer) can be exercised +// deterministically. Every other call delegates to the real filesystem, so +// config reads / earlier writes behave normally. Merge this AFTER +// `BunServices.layer` (last-wins) so it overrides only `FileSystem`; `Path` +// still comes from `BunServices`. +// --------------------------------------------------------------------------- + +export function legacyFailWriteStringOnNthCallFsLayer( + failOnCall: number, +): Layer.Layer { + return Layer.effect( + FileSystem.FileSystem, + Effect.map(FileSystem.FileSystem, (real) => { + let calls = 0; + return FileSystem.FileSystem.of({ + ...real, + writeFileString: (path, data, options) => { + calls += 1; + if (calls === failOnCall) { + return Effect.fail( + new PlatformError( + new SystemError({ + _tag: "Unknown", + module: "FileSystem", + method: "writeFileString", + pathOrDescriptor: path, + description: "simulated write failure", + }), + ), + ); + } + return real.writeFileString(path, data, options); + }, + }); + }), + ).pipe(Layer.provide(BunServices.layer)); +} + // --------------------------------------------------------------------------- // Runtime composition — bundles the entire Layer.mergeAll(...) graph that // every native-port integration test re-builds, including the easy-to-mis-wire