diff --git a/README.md b/README.md index 384147f..dc58fc4 100644 --- a/README.md +++ b/README.md @@ -806,6 +806,39 @@ drop policy if exists access_by_numbers on mytable; create policy access_by_numbers on mytable for update using (myfunction(4, 2) < 42); ``` +#### Automatically including changed fixtures (experimental) + +Set `autoFixtures: true` in `.gmrc` to automatically include new and changed SQL +files under `migrations/fixtures`, including subdirectories. This works with +both `current.sql` and `current/`. Comment-only changes are ignored. + +Automatic includes are kept in marked blocks: + +```sql +-- graphile-migrate: generated autoFixtures; +--!include functions/myfunction.sql +-- graphile-migrate: end of autoFixtures; +``` + +Place blocks wherever the fixtures should run. Existing entries keep their +block; new entries are appended to the last block in migration order. Manual +includes outside the blocks take precedence. Keep custom SQL outside the +markers. If an automatic fixture includes a manually placed fixture, refresh +reports the conflict so you can place the outer fixture explicitly. + +If there is no block, one is appended to `current.sql`. In directory mode, a new +file is placed last using your number width: `1-current.sql` gives +`9-auto-fixtures.sql`, and `000001-init.sql` gives `999999-auto-fixtures.sql`. +If that numeric prefix is already used, another digit is added. Empty blocks are +kept so their placement is preserved. + +`watch` updates the blocks as fixtures change. You can also run +`graphile-migrate status --skipDatabase` to review includes before committing. +New entries use filename order; use nested includes or place entries in blocks +when execution order matters. Deleting a fixture removes its pending include; +write any necessary database cleanup SQL yourself. Fixtures should be +idempotent. + ### Committed migration(s) The files for migrations that you've committed with `graphile-migrate commit` diff --git a/__tests__/commit.test.ts b/__tests__/commit.test.ts index 5bf2321..bcd9d78 100644 --- a/__tests__/commit.test.ts +++ b/__tests__/commit.test.ts @@ -169,3 +169,33 @@ describe.each([[undefined], ["My Commit Message"]])( }); }, ); + +it.each([false, true])( + "preserves includes after a failed commit (automatic: %s)", + async (automatic) => { + const original = automatic ? "-- current\n" : "--!include fail.sql\n"; + mockFs({ + "migrations/current/1-current.sql": original, + "migrations/fixtures/fail.sql": "select * from missing_fixture_table;", + }); + await expect( + commit({ ...settings, autoFixtures: automatic }), + ).rejects.toThrow("does not exist"); + const filename = automatic ? "9-auto-fixtures.sql" : "1-current.sql"; + const restored = await fsp.readFile( + `migrations/current/${filename}`, + "utf8", + ); + expect(restored).toContain("--!include fail.sql"); + expect(restored).not.toContain("--! Included"); + expect(await fsp.readFile("migrations/current/1-current.sql", "utf8")).toBe( + original, + ); + expect(await fsp.readdir("migrations/committed")).toEqual([]); + await fsp.writeFile("migrations/fixtures/fail.sql", "select 1;"); + await commit({ ...settings, autoFixtures: automatic }); + expect( + await fsp.readFile("migrations/committed/000001.sql", "utf8"), + ).toContain("select 1;"); + }, +); diff --git a/__tests__/fixtures.test.ts b/__tests__/fixtures.test.ts new file mode 100644 index 0000000..9b4a6d1 --- /dev/null +++ b/__tests__/fixtures.test.ts @@ -0,0 +1,548 @@ +import "./helpers"; // Has side-effects; must come first + +import { promises as fsp } from "fs"; +import mockFs from "mock-fs"; + +import { commit, current, status, uncommit } from "../src"; +import { + getCurrentMigrationLocation, + readCurrentMigration, +} from "../src/current"; +import { + generatedFixtureFooter, + generatedFixtureHeader, +} from "../src/fixtures"; +import { calculateHash } from "../src/hash"; +import * as migration from "../src/migration"; +import { compileIncludes, serializeMigration } from "../src/migration"; +import { withClient } from "../src/pg"; +import { parseSettings } from "../src/settings"; +import { resetDb, settings as baseSettings } from "./helpers"; + +const settings = { ...baseSettings, autoFixtures: true }; + +beforeEach(() => { + mockFs({ + "migrations/current/001.sql": "-- current\n", + "migrations/fixtures/example.sql": "select '--literal'; -- comment\n", + }); +}); +afterEach(() => mockFs.restore()); + +async function recordCommit( + paths = ["example.sql"], + number = 1, + previous: string | null = null, +) { + const parsed = await parseSettings(settings); + const body = await compileIncludes( + parsed, + paths.map((p) => `--!include ${p}`).join("\n"), + new Set(), + ); + const hash = calculateHash(body, previous); + await fsp.mkdir("migrations/committed", { recursive: true }); + await fsp.writeFile( + `migrations/committed/${String(number).padStart(6, "0")}.sql`, + serializeMigration(body, { Previous: previous || "-", Hash: hash }), + ); + return hash; +} + +async function pending(autoFixtures = true) { + const parsed = await parseSettings({ ...settings, autoFixtures }); + const location = await getCurrentMigrationLocation(parsed); + const body = await readCurrentMigration(parsed, location); + const automaticFixtures: string[] = []; + const files = location.isFile + ? [location.path] + : (await fsp.readdir(location.path)).map( + (file) => `${location.path}/${file}`, + ); + for (const file of files) { + const text = await fsp.readFile(file, "utf8"); + for (const block of text.matchAll( + /^-- graphile-migrate: generated autoFixtures;\n([\s\S]*?)^-- graphile-migrate: end of autoFixtures;/gm, + )) { + automaticFixtures.push( + ...[...block[1].matchAll(/^--!include (.+)$/gm)].map( + (match) => match[1], + ), + ); + } + } + return { body, automaticFixtures }; +} + +it("generates a visible include file without changing user SQL", async () => { + const result = await pending(); + expect(result.automaticFixtures).toEqual(["example.sql"]); + expect(result.body).toContain( + "--! Included example.sql\nselect '--literal';", + ); + expect(await fsp.readFile("migrations/current/001.sql", "utf8")).toBe( + "-- current\n", + ); + const generated = await fsp.readFile( + "migrations/current/999-auto-fixtures.sql", + "utf8", + ); + expect(generated).toBe( + generatedFixtureHeader + + "\n--!include example.sql\n" + + generatedFixtureFooter + + "\n", + ); + const before = await fsp.stat("migrations/current/999-auto-fixtures.sql"); + expect(await pending()).toEqual(result); + expect( + (await fsp.stat("migrations/current/999-auto-fixtures.sql")).mtimeMs, + ).toBe(before.mtimeMs); +}); + +it.each([false, undefined])( + "requires opt-in (autoFixtures: %s)", + async (autoFixtures) => { + const parsed = await parseSettings({ ...baseSettings, autoFixtures }); + await readCurrentMigration( + parsed, + await getCurrentMigrationLocation(parsed), + ); + expect(await fsp.readdir("migrations/current")).toEqual(["001.sql"]); + }, +); + +it.each(["current.sql", "current/001.sql"])( + "preserves existing block placement in %s", + async (file) => { + await fsp.writeFile("migrations/fixtures/second.sql", "select 2;"); + const first = `${generatedFixtureHeader}\n--!include example.sql\n${generatedFixtureFooter}\n`; + const last = `${generatedFixtureHeader}\n${generatedFixtureFooter}\n`; + if (file === "current.sql") + await fsp.rm("migrations/current", { recursive: true }); + await fsp.writeFile( + `migrations/${file}`, + `select 0;\n${first}select 1;\n${last}select 3;\n`, + ); + const result = await pending(); + const content = await fsp.readFile(`migrations/${file}`, "utf8"); + expect(content).toBe( + `select 0;\n${first}select 1;\n${generatedFixtureHeader}\n--!include second.sql\n${generatedFixtureFooter}\nselect 3;\n`, + ); + expect(result.body.indexOf("select '--literal'")).toBeLessThan( + result.body.indexOf("select 1;"), + ); + expect(result.body.indexOf("select 1;")).toBeLessThan( + result.body.indexOf("select 2;"), + ); + }, +); + +it("appends new fixtures to the last block across current files", async () => { + await fsp.writeFile( + "migrations/current/001.sql", + `${generatedFixtureHeader}\n--!include example.sql\n${generatedFixtureFooter}\n`, + ); + await fsp.writeFile( + "migrations/current/005-last.sql", + `${generatedFixtureHeader}\n${generatedFixtureFooter}\n`, + ); + await fsp.writeFile("migrations/fixtures/second.sql", "select 2;"); + await pending(); + expect( + await fsp.readFile("migrations/current/005-last.sql", "utf8"), + ).toContain("--!include second.sql"); + expect(await fsp.readdir("migrations/current")).toEqual([ + "001.sql", + "005-last.sql", + ]); +}); + +it("creates a block at the end of current.sql and respects manual includes", async () => { + await fsp.rm("migrations/current", { recursive: true }); + await fsp.writeFile( + "migrations/current.sql", + "--!include example.sql\nselect 0;\n", + ); + await fsp.writeFile("migrations/fixtures/second.sql", "select 2;"); + await pending(); + expect(await fsp.readFile("migrations/current.sql", "utf8")).toBe( + `--!include example.sql\nselect 0;\n\n${generatedFixtureHeader}\n--!include second.sql\n${generatedFixtureFooter}\n`, + ); +}); + +it.each([ + `${generatedFixtureHeader}\n--!include example.sql\n`, + `${generatedFixtureHeader}\nselect 1;\n${generatedFixtureFooter}\n`, +])( + "rejects malformed blocks without changing user content", + async (content) => { + await fsp.writeFile("migrations/current/001.sql", content); + await expect(pending()).rejects.toThrow(); + expect(await fsp.readFile("migrations/current/001.sql", "utf8")).toBe( + content, + ); + }, +); + +it("removes generated includes when included from a manual current file", async () => { + await pending(); + await fsp.writeFile( + "migrations/fixtures/wrapper.sql", + "--!include example.sql", + ); + await fsp.writeFile("migrations/current/001.sql", "--!include wrapper.sql"); + const result = await pending(); + expect(result.automaticFixtures).toEqual([]); + expect(result.body.match(/select '--literal'/g)).toHaveLength(1); + expect(await fsp.readdir("migrations/current")).toEqual([ + "001.sql", + "999-auto-fixtures.sql", + ]); +}); + +it.each([ + ["current.sql", "--!include example.sql"], + ["current/001.sql", "--!include bridge.sql"], +])( + "rejects automatic wrappers that replay manual includes in %s", + async (file, include) => { + if (file === "current.sql") + await fsp.rm("migrations/current", { recursive: true }); + const content = "--!include example.sql\nselect 42;\n"; + await fsp.writeFile(`migrations/${file}`, content); + if (include.includes("bridge.sql")) { + await fsp.writeFile( + "migrations/fixtures/bridge.sql", + "--!include example.sql", + ); + } + await fsp.writeFile( + "migrations/fixtures/wrapper.sql", + `${include}\nselect 2;`, + ); + await expect(pending()).rejects.toThrow( + "Automatic fixture 'wrapper.sql' includes manually placed fixture 'example.sql'", + ); + expect(await fsp.readFile(`migrations/${file}`, "utf8")).toBe(content); + }, +); + +it.each([ + ["current.sql", true], + ["current/001.sql", true], + ["current.sql", false], +] as const)( + "retries edits made during refresh (%s, exists: %s)", + async (file, exists) => { + if (file === "current.sql") + await fsp.rm("migrations/current", { recursive: true }); + const block = `${generatedFixtureHeader}\n${generatedFixtureFooter}\n`; + const path = `migrations/${file}`; + if (exists) await fsp.writeFile(path, block); + const edited = `select 42;\n--!include example.sql\n${block}select 43;\n`; + const getAllMigrations = migration.getAllMigrations; + const history = jest + .spyOn(migration, "getAllMigrations") + .mockImplementationOnce(async (...args) => { + await fsp.writeFile(path, edited); + return getAllMigrations(...args); + }); + try { + const result = await pending(); + expect(await fsp.readFile(path, "utf8")).toBe(edited); + expect(result.automaticFixtures).toEqual([]); + expect(result.body.match(/select '--literal'/g)).toHaveLength(1); + } finally { + history.mockRestore(); + } + }, +); + +it("retries when a new current file introduces a manual include during refresh", async () => { + const getAllMigrations = migration.getAllMigrations; + const history = jest + .spyOn(migration, "getAllMigrations") + .mockImplementationOnce(async (...args) => { + await fsp.writeFile( + "migrations/current/002-late.sql", + "--!include example.sql\n", + ); + return getAllMigrations(...args); + }); + try { + const result = await pending(); + expect(result.automaticFixtures).toEqual([]); + expect(result.body.match(/select '--literal'/g)).toHaveLength(1); + } finally { + history.mockRestore(); + } +}); + +it("stops retrying without overwriting files that keep changing", async () => { + let edits = 0; + const getAllMigrations = migration.getAllMigrations; + const history = jest + .spyOn(migration, "getAllMigrations") + .mockImplementation(async (...args) => { + await fsp.writeFile("migrations/current/001.sql", `select ${++edits};\n`); + return getAllMigrations(...args); + }); + try { + await expect(pending()).rejects.toThrow( + "Current migration files kept changing", + ); + expect(await fsp.readFile("migrations/current/001.sql", "utf8")).toBe( + `select ${edits};\n`, + ); + expect(await fsp.readdir("migrations/current")).toEqual(["001.sql"]); + } finally { + history.mockRestore(); + } +}); + +it("keeps the generated block in place when another user file is added", async () => { + await pending(); + await fsp.writeFile("migrations/current/002-next.sql", "select 2;"); + await pending(); + expect(await fsp.readdir("migrations/current")).toEqual([ + "001.sql", + "002-next.sql", + "999-auto-fixtures.sql", + ]); +}); + +it.each([ + ["1-current.sql", "9-auto-fixtures.sql"], + ["0001-current.sql", "9999-auto-fixtures.sql"], + ["000001-init.sql", "999999-auto-fixtures.sql"], + ["009-current.sql", "999-auto-fixtures.sql"], + ["9-current.sql", "99-auto-fixtures.sql"], +])("matches the numeric prefix convention of %s", async (file, generated) => { + await fsp.rename("migrations/current/001.sql", `migrations/current/${file}`); + await pending(); + expect(await fsp.readdir("migrations/current")).toEqual( + [file, generated].sort(), + ); +}); + +it("still rejects duplicate numeric prefixes among user files", async () => { + await fsp.writeFile("migrations/current/001-duplicate.sql", "select 2;"); + const reads = jest.spyOn(fsp, "readFile"); + try { + await expect(pending()).rejects.toThrow("Current migration filename clash"); + } finally { + // The reader may have started other reads before detecting the duplicate. + await Promise.allSettled(reads.mock.results.map(({ value }) => value)); + reads.mockRestore(); + } +}); + +it("removes stale generated includes when a fixture is deleted", async () => { + await pending(); + await fsp.unlink("migrations/fixtures/example.sql"); + await pending(); + expect(await fsp.readdir("migrations/current")).toEqual([ + "001.sql", + "999-auto-fixtures.sql", + ]); +}); + +it("does not overwrite a user file with the generator's filename suffix", async () => { + await fsp.writeFile("migrations/current/2-auto-fixtures.sql", "select 2;"); + await pending(); + expect( + await fsp.readFile("migrations/current/2-auto-fixtures.sql", "utf8"), + ).toBe("select 2;"); + expect(await fsp.readdir("migrations/current")).toContain( + "999-auto-fixtures.sql", + ); +}); + +it("preserves custom SQL outside a generated block", async () => { + await pending(); + await fsp.appendFile( + "migrations/current/999-auto-fixtures.sql", + "select 2;\n", + ); + expect((await pending()).body).toContain("select 2;"); + await fsp.unlink("migrations/fixtures/example.sql"); + expect((await pending()).body).toContain("select 2;"); +}); + +it("ignores comment-only changes inside function bodies", async () => { + const sql = ( + comment: string, + value: number, + ) => `create or replace function fixture_example() returns integer as $$ + -- ${comment} + select ${value}; + $$ language sql;`; + await fsp.writeFile( + "migrations/fixtures/example.sql", + sql("first comment", 1), + ); + await recordCommit(); + await fsp.writeFile( + "migrations/fixtures/example.sql", + sql("changed comment", 1), + ); + expect((await pending()).automaticFixtures).toEqual([]); + await fsp.writeFile( + "migrations/fixtures/example.sql", + sql("changed comment", 2), + ); + expect((await pending()).automaticFixtures).toEqual(["example.sql"]); +}); + +it("uses the latest committed version and follows history when a commit is removed", async () => { + const first = await recordCommit(); + await fsp.writeFile("migrations/fixtures/example.sql", "select 2;"); + await recordCommit(["example.sql"], 2, first); + expect((await pending()).automaticFixtures).toEqual([]); + await fsp.unlink("migrations/committed/000002.sql"); + expect((await pending()).automaticFixtures).toEqual(["example.sql"]); + await fsp.writeFile("migrations/fixtures/example.sql", "select '--literal';"); + expect((await pending()).automaticFixtures).toEqual([]); +}); + +it("detects changes in nested includes", async () => { + await fsp.writeFile( + "migrations/fixtures/example.sql", + "select 1;\n--!include helper.sql", + ); + await fsp.writeFile("migrations/fixtures/helper.sql", "select 2;"); + await recordCommit(); + expect((await pending()).automaticFixtures).toEqual([]); + await fsp.writeFile("migrations/fixtures/helper.sql", "select 3;"); + const result = await pending(); + expect(result.automaticFixtures).toEqual(["example.sql"]); + expect(result.body).toContain("select 3;"); +}); + +it("appends discovered fixtures in filename order after a current directory", async () => { + await fsp.writeFile("migrations/current/001.sql", "select 0;"); + await fsp.writeFile("migrations/fixtures/second.sql", "select 2;"); + const result = await pending(); + expect(result.automaticFixtures).toEqual(["example.sql", "second.sql"]); + expect(result.body.indexOf("select 0;")).toBeLessThan( + result.body.indexOf("select '--literal'"), + ); + expect(result.body.indexOf("select '--literal'")).toBeLessThan( + result.body.indexOf("select 2;"), + ); +}); + +it("discovers new files in nested fixture directories without a fixture list", async () => { + await recordCommit(); + expect((await pending()).automaticFixtures).toEqual([]); + await fsp.mkdir("migrations/fixtures/functions/nested", { recursive: true }); + await fsp.writeFile( + "migrations/fixtures/functions/nested/new.sql", + "select 42;", + ); + const result = await pending(); + expect(result.automaticFixtures).toEqual(["functions/nested/new.sql"]); + expect(result.body).toContain("select 42;"); +}); + +it("ignores new comment-only fixtures", async () => { + await fsp.writeFile("migrations/fixtures/example.sql", "-- comments only"); + expect((await pending()).automaticFixtures).toEqual([]); +}); + +it("does not allow automatic fixtures to bypass current directory transaction rules", async () => { + await fsp.writeFile( + "migrations/fixtures/example.sql", + "--! no-transaction\nselect 1;", + ); + await expect(pending()).rejects.toThrow("no-transaction"); +}); + +describe("database lifecycle", () => { + beforeEach(resetDb); + + it.each(["directory", "empty directory", "current.sql"])( + "runs, commits, uncommits and recommits fixtures (%s)", + async (mode) => { + if (mode === "current.sql") { + await fsp.rm("migrations/current", { recursive: true }); + await fsp.writeFile("migrations/current.sql", "-- current\n"); + } else if (mode === "empty directory") { + await fsp.unlink("migrations/current/001.sql"); + } + await fsp.writeFile( + "migrations/fixtures/example.sql", + "create table if not exists fixture_test (id integer);\ntruncate fixture_test;\ninsert into fixture_test values (42);", + ); + expect(await status(settings, { skipDatabase: true })).toMatchObject({ + hasCurrentMigration: true, + }); + await current(settings); + const parsed = await parseSettings(settings); + const rows = await withClient( + parsed.connectionString, + parsed, + async (client) => + (await client.query("select * from fixture_test")).rows, + ); + expect(rows).toEqual([{ id: 42 }]); + // Development runs never advance the committed fixture baseline. + expect((await pending()).automaticFixtures).toEqual(["example.sql"]); + await commit(settings); + const firstCommit = await fsp.readFile( + "migrations/committed/000001.sql", + "utf8", + ); + expect(firstCommit).toContain("--! Included example.sql"); + expect(await status(settings, { skipDatabase: true })).toMatchObject({ + hasCurrentMigration: false, + }); + await uncommit(settings); + expect( + (await pending()).body.match(/insert into fixture_test/g), + ).toHaveLength(1); + await commit(settings); + expect( + await fsp.readFile("migrations/committed/000001.sql", "utf8"), + ).toBe(firstCommit); + }, + ); + + it.each([false, true])( + "can uncommit with pending fixture blocks (current.sql: %s)", + async (singleFile) => { + if (singleFile) { + await fsp.rm("migrations/current", { recursive: true }); + await fsp.writeFile("migrations/current.sql", "-- current\n"); + } + await commit(settings); + await fsp.writeFile("migrations/fixtures/example.sql", "select 2;"); + await status(settings, { skipDatabase: true }); + await uncommit(settings); + expect((await pending()).body.match(/select 2;/g)).toHaveLength(1); + await commit(settings); + expect( + await fsp.readFile("migrations/committed/000001.sql", "utf8"), + ).toContain("select 2;"); + expect((await pending()).automaticFixtures).toEqual([]); + }, + ); + + it("can commit a changed fixture with no current SQL and ignores later comment edits", async () => { + await commit(settings); + await fsp.writeFile("migrations/fixtures/example.sql", "select 2;"); + await commit(settings); + expect( + await fsp.readFile("migrations/committed/000002.sql", "utf8"), + ).toContain("select 2;"); + await fsp.writeFile( + "migrations/fixtures/example.sql", + "select 2; -- just a comment", + ); + await expect(commit(settings)).rejects.toThrow( + "Current migration is blank", + ); + expect((await pending()).automaticFixtures).toEqual([]); + }); +}); diff --git a/__tests__/settings.test.ts b/__tests__/settings.test.ts index 94b55a6..1349fc9 100644 --- a/__tests__/settings.test.ts +++ b/__tests__/settings.test.ts @@ -392,3 +392,15 @@ module.exports = { mockFs.restore(); }); }); + +it.each(["true", ["example.sql"], 1])( + "rejects non-boolean autoFixtures: %p", + async (autoFixtures) => { + await expect( + parseSettings({ + connectionString: exampleConnectionString, + autoFixtures: autoFixtures as any, + }), + ).rejects.toThrow("Setting 'autoFixtures': Expected a boolean"); + }, +); diff --git a/__tests__/watch.test.ts b/__tests__/watch.test.ts index 7f7b44c..8029566 100644 --- a/__tests__/watch.test.ts +++ b/__tests__/watch.test.ts @@ -1,10 +1,14 @@ jest.mock("child_process"); +jest.mock("chokidar"); import "./helpers"; // Has side-effects; must come first +import * as chokidar from "chokidar"; +import { EventEmitter } from "events"; import mockFs from "mock-fs"; import { _watch } from "../src/commands/watch"; +import * as currentRunner from "../src/currentRunner"; import { makeCurrentMigrationRunner } from "../src/currentRunner"; import { parseSettings } from "../src/settings"; import { makeMigrations } from "./helpers"; @@ -113,3 +117,37 @@ SELECT ':DATABASE_NAME', 2 * 2; "afterCurrent", ]); }); + +it("coalesces fixture changes while a migration is running", async () => { + mockFs({ "migrations/current.sql": "-- current\n" }); + const watcher = new EventEmitter(); + jest.mocked(chokidar.watch).mockReturnValue(watcher as chokidar.FSWatcher); + let finish!: () => void; + const run = jest.fn( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const runner = jest + .spyOn(currentRunner, "makeCurrentMigrationRunner") + .mockReturnValue(run); + try { + const parsedSettings = await parseSettings({ + connectionString: TEST_DATABASE_URL, + }); + await _watch(parsedSettings); + watcher.emit("ready"); + watcher.emit("change", "migrations/fixtures/one.sql"); + watcher.emit("change", "migrations/fixtures/two.sql"); + expect(run).toHaveBeenCalledTimes(1); + finish(); + await new Promise((resolve) => setImmediate(resolve)); + expect(run).toHaveBeenCalledTimes(2); + finish(); + await new Promise((resolve) => setImmediate(resolve)); + expect(run).toHaveBeenCalledTimes(2); + } finally { + runner.mockRestore(); + } +}); diff --git a/src/commands/commit.ts b/src/commands/commit.ts index 53900ad..e4a2db1 100644 --- a/src/commands/commit.ts +++ b/src/commands/commit.ts @@ -117,7 +117,12 @@ export async function _commit( logDbError(parsedSettings, e); parsedSettings.logger.error("ABORTING..."); - await writeCurrentMigration(parsedSettings, currentLocation, body); + // Restore include directives, including the generated fixture file. + const restored = body.replace( + /^--! Included (?\S+)$[\s\S]*?^--! EndIncluded \k$/gm, + (_, filename: string) => `--!include ${filename}`, + ); + await writeCurrentMigration(parsedSettings, currentLocation, restored); await fsp.unlink(newMigrationFilepath); parsedSettings.logger.error("ABORTED AND ROLLED BACK"); throw e; diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index 058b0fd..5e49bec 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -31,6 +31,7 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { const currentBody = await readCurrentMigration( parsedSettings, currentLocation, + { fixtures: false }, ); const minifiedCurrentBody = pgMinify(currentBody); if (minifiedCurrentBody !== "") { diff --git a/src/commands/watch.ts b/src/commands/watch.ts index 7034860..494ce8e 100644 --- a/src/commands/watch.ts +++ b/src/commands/watch.ts @@ -38,6 +38,7 @@ export async function _watch( const queue = (): void => { if (running) { runAgain = true; + return; } running = true; diff --git a/src/current.ts b/src/current.ts index e6de650..a841950 100644 --- a/src/current.ts +++ b/src/current.ts @@ -1,6 +1,7 @@ import * as assert from "assert"; import { promises as fsp, Stats } from "fs"; +import { removeAutomaticFixtures, updateAutomaticFixtures } from "./fixtures"; import { isNoTransactionDefined } from "./header"; import { errorCode } from "./lib"; import { @@ -89,14 +90,18 @@ export async function getCurrentMigrationLocation( export async function readCurrentMigration( parsedSettings: ParsedSettings, location: CurrentMigrationLocation, + { fixtures = true }: { fixtures?: boolean } = {}, ): Promise { + if (fixtures && parsedSettings.autoFixtures) { + await updateAutomaticFixtures(parsedSettings, location); + } if (location.isFile) { const content = await readFileOrNull(location.path); // If file doesn't exist, treat it as if it were empty. return compileIncludes( parsedSettings, - content || "", + fixtures ? content || "" : removeAutomaticFixtures(content || ""), new Set([location.path]), ); } else { @@ -150,7 +155,7 @@ export async function readCurrentMigration( const rawContents = await bodyPromise; const contents = await compileIncludes( parsedSettings, - rawContents, + fixtures ? rawContents : removeAutomaticFixtures(rawContents), new Set([filePath]), ); const { body, headers } = parseMigrationText(filePath, contents, false); diff --git a/src/fixtures.ts b/src/fixtures.ts new file mode 100644 index 0000000..965849f --- /dev/null +++ b/src/fixtures.ts @@ -0,0 +1,304 @@ +import * as fsp from "fs/promises"; +import { join, posix, relative, sep } from "path"; +import pgMinify from "pg-minify"; + +import type { CurrentMigrationLocation } from "./current"; +import { calculateHash } from "./hash"; +import { errorCode } from "./lib"; +import { compileIncludes, getAllMigrations } from "./migration"; +import { idFromFilename, migrationFilenameMatches } from "./migrationFilename"; +import type { ParsedSettings } from "./settings"; + +/** Recover fixture state from the immutable include blocks already in migrations. */ +function includedFixtures(body: string): Map { + const result = new Map(); + const stack: Array<{ path: string; start: number }> = []; + const markers = /^--! (Included|EndIncluded) (.+)$/gm; + for (const match of body.matchAll(markers)) { + const [, kind, path] = match; + if (kind === "Included") { + stack.push({ path, start: match.index + match[0].length + 1 }); + } else { + const opening = stack.pop(); + if (!opening || opening.path !== path) { + throw new Error(`Invalid fixture include markers for '${path}'`); + } + result.set(posix.normalize(path), body.slice(opening.start, match.index)); + } + } + if (stack.length) { + throw new Error("Unclosed fixture include marker"); + } + return result; +} + +function fixtureHash(sql: string): string { + return calculateHash(pgMinify(sql), null, "sha256"); +} + +async function fixturePaths(directory: string): Promise { + try { + const entries = await fsp.readdir(directory, { + recursive: true, + withFileTypes: true, + }); + return entries + .filter((entry) => entry.isFile() || entry.isSymbolicLink()) + .map((entry) => + relative(directory, join(entry.parentPath, entry.name)) + .split(sep) + .join("/"), + ) + .filter( + (path) => + path.endsWith(".sql") && + !path.split("/").some((part) => part.startsWith(".")), + ) + .sort(); + } catch (error) { + if (errorCode(error) === "ENOENT") { + return []; + } + throw error; + } +} + +export const generatedFixtureHeader = + "-- graphile-migrate: generated autoFixtures;"; +export const generatedFixtureFooter = + "-- graphile-migrate: end of autoFixtures;"; + +function fixtureBlocks(content: string) { + const pattern = + /^-- graphile-migrate: generated autoFixtures;[ \t]*\r?\n([\s\S]*?)^-- graphile-migrate: end of autoFixtures;[ \t]*(?:\r?\n|$)/gm; + const blocks = [...content.matchAll(pattern)].map((match) => { + const paths = match[1] + .split(/\r?\n/) + .filter((line) => line.trim() !== "") + .map((line) => { + const include = /^--!include (.+\.sql)$/.exec(line); + if (!include) { + throw new Error( + "Automatic fixture blocks may only contain include directives; move custom SQL outside the block.", + ); + } + return posix.normalize(include[1]); + }); + return { start: match.index, end: match.index + match[0].length, paths }; + }); + const outside = content.replace(pattern, ""); + if ( + outside.includes(generatedFixtureHeader) || + outside.includes(generatedFixtureFooter) + ) { + throw new Error("Unmatched automatic fixture block marker"); + } + return blocks; +} + +export function removeAutomaticFixtures(content: string): string { + let result = content; + for (const block of fixtureBlocks(content).reverse()) { + result = result.slice(0, block.start) + result.slice(block.end); + } + return result; +} + +function renderBlock(paths: string[]): string { + return [ + generatedFixtureHeader, + ...paths.map((path) => `--!include ${path}`), + generatedFixtureFooter, + "", + ].join("\n"); +} + +/** Refresh automatic include blocks before the ordinary migration reader runs. */ +export async function updateAutomaticFixtures( + settings: ParsedSettings, + location: CurrentMigrationLocation, +): Promise { + for (let attempt = 0; attempt < 3; attempt++) { + if (await refreshAutomaticFixtures(settings, location)) return; + } + throw new Error( + "Current migration files kept changing during automatic fixture refresh; retry once editing has stopped.", + ); +} + +async function readCurrentFile(path: string): Promise { + try { + return await fsp.readFile(path, "utf8"); + } catch (error) { + if (errorCode(error) === "ENOENT") return null; + throw error; + } +} + +async function currentFileNames( + location: CurrentMigrationLocation, +): Promise { + return location.isFile + ? ["current.sql"] + : (await fsp.readdir(location.path)) + .filter((file) => !file.startsWith(".") && file.endsWith(".sql")) + .sort((a, b) => idFromFilename(a) - idFromFilename(b)); +} + +async function refreshAutomaticFixtures( + settings: ParsedSettings, + location: CurrentMigrationLocation, +): Promise { + const names = await currentFileNames(location); + const files = await Promise.all( + names.map(async (name) => { + const path = location.isFile ? location.path : `${location.path}/${name}`; + const original = await readCurrentFile(path); + const content = original ?? ""; + return { path, original, content, blocks: fixtureBlocks(content) }; + }), + ); + const paths = await fixturePaths(`${settings.migrationsFolder}/fixtures`); + const committed = new Map(); + for (const migration of paths.length + ? await getAllMigrations(settings) + : []) { + for (const [path, sql] of includedFixtures(migration.body)) { + committed.set(path, sql); + } + } + const included = includedFixtures( + await compileIncludes( + settings, + files.map(({ content }) => removeAutomaticFixtures(content)).join("\n"), + new Set(), + ), + ); + const candidates = await Promise.all( + paths + .filter((path) => !included.has(path)) + .map(async (path) => { + const compiled = await compileIncludes( + settings, + `--!include ${path}`, + new Set(), + ); + const fixtures = includedFixtures(compiled); + if (!location.isFile && /^--!\s*no-transaction\b/m.test(compiled)) { + throw new Error( + `Automatic fixture '${path}' contains '--! no-transaction'; fixtures require transactional SQL in current/ mode.`, + ); + } + return { path, fixtures, sql: fixtures.get(path)! }; + }), + ); + const nested = new Set( + candidates.flatMap(({ path, fixtures }) => + [...fixtures.keys()].filter((included) => included !== path), + ), + ); + const changed = candidates.filter(({ path, sql }) => { + if (nested.has(path)) { + return false; + } + const previous = committed.get(path); + return previous === undefined + ? pgMinify(sql) !== "" + : fixtureHash(sql) !== fixtureHash(previous); + }); + for (const { path, fixtures } of changed) { + for (const fixture of fixtures.keys()) { + if (included.has(fixture)) { + throw new Error( + `Automatic fixture '${path}' includes manually placed fixture '${fixture}'; place '${path}' manually instead to control its execution order.`, + ); + } + } + } + const updates: Array<{ + path: string; + original: string | null; + content: string; + }> = []; + const remaining = new Set(changed.map(({ path }) => path)); + const blocks = files.flatMap((file) => file.blocks); + for (const block of blocks) { + block.paths = block.paths.filter((path) => remaining.delete(path)); + } + if (blocks.length) { + blocks[blocks.length - 1].paths.push(...remaining); + } else if (remaining.size) { + if (location.isFile) { + const file = files[0]; + file.blocks.push({ + start: file.content.length, + end: file.content.length, + paths: [...remaining], + }); + } else { + const highestId = Math.max(0, ...names.map(idFromFilename)); + let width = Math.max( + 1, + ...names.map((name) => migrationFilenameMatches(name)![1].length), + ); + while (Number("9".repeat(width)) <= highestId) width++; + const prefix = "9".repeat(width); + if (!Number.isSafeInteger(Number(prefix))) { + throw new Error( + "Cannot allocate a numeric prefix for the automatic fixture file", + ); + } + updates.push({ + path: `${location.path}/${prefix}-auto-fixtures.sql`, + original: null, + content: renderBlock([...remaining]), + }); + } + } + for (const file of files) { + let content = file.content; + for (const block of [...file.blocks].reverse()) { + const separator = + block.start === file.content.length && content + ? content.endsWith("\n") + ? "\n" + : "\n\n" + : ""; + content = + content.slice(0, block.start) + + separator + + renderBlock(block.paths) + + content.slice(block.end); + } + if (content !== file.content) { + updates.push({ path: file.path, original: file.original, content }); + } + } + // Recompute placement and manual precedence if any input changed during the async work. + if ( + JSON.stringify(await currentFileNames(location)) !== JSON.stringify(names) + ) + return false; + for (const file of files) { + if ((await readCurrentFile(file.path)) !== file.original) return false; + } + for (const { path, original, content } of updates) { + if (original !== null && !(await fsp.lstat(path)).isFile()) { + throw new Error( + `Cannot overwrite automatic fixture block in symlink '${path}'`, + ); + } + // Check this file again immediately before writing, including after earlier writes. + if ((await readCurrentFile(path)) !== original) return false; + try { + await fsp.writeFile(path, content, { + flag: original === null ? "wx" : "w", + }); + } catch (error) { + if (errorCode(error) === "EEXIST" || errorCode(error) === "ENOENT") + return false; + throw error; + } + } + return true; +} diff --git a/src/settings.ts b/src/settings.ts index 79e665e..aac762f 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -69,6 +69,7 @@ export interface Settings { rootConnectionString?: string; databaseOwner?: string; migrationsFolder?: string; + autoFixtures?: boolean; manageGraphileMigrateSchema?: boolean; pgSettings?: { [key: string]: string; @@ -189,6 +190,12 @@ export async function parseSettings( }, ); + await check("autoFixtures", (value) => { + if (value !== undefined && typeof value !== "boolean") { + throw new Error("Expected a boolean"); + } + }); + const { user, database: databaseName } = parse(connectionString || ""); const databaseOwner = await check( "databaseOwner",