From 09c7005f15870a42eab12a13b6de16fb86d05956 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Jun 2026 09:14:38 -0700 Subject: [PATCH 1/7] New migrations system, ported from sqlite-migrate Refs #752 --- docs/changelog.rst | 9 +- docs/cli-reference.rst | 37 +++++- docs/cli.rst | 23 ++++ docs/index.rst | 1 + docs/migrations.rst | 164 +++++++++++++++++++++++++++ sqlite_utils/__init__.py | 3 +- sqlite_utils/cli.py | 121 ++++++++++++++++++++ sqlite_utils/migrations.py | 119 +++++++++++++++++++ tests/test_cli_migrate.py | 227 +++++++++++++++++++++++++++++++++++++ tests/test_migrations.py | 110 ++++++++++++++++++ 10 files changed, 811 insertions(+), 3 deletions(-) create mode 100644 docs/migrations.rst create mode 100644 sqlite_utils/migrations.py create mode 100644 tests/test_cli_migrate.py create mode 100644 tests/test_migrations.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 140611610..098f00009 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v_unreleased: + +Unreleased +---------- + +- New ``sqlite-utils migrate`` command for applying Python database migrations, incorporating functionality that was previously provided by the separate `sqlite-migrate `__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using ``sqlite-utils migrate database.db migrations.py`` or the :ref:`migrations Python API `. See :ref:`migrations` for details. (:issue:`752`) + .. _v3_39: 3.39 (2025-11-24) @@ -182,7 +189,7 @@ This release introduces a new :ref:`plugin system `. Read more about th - Conversion functions passed to :ref:`table.convert(...) ` can now return lists or dictionaries, which will be inserted into the database as JSON strings. (:issue:`495`) - ``sqlite-utils install`` and ``sqlite-utils uninstall`` commands for installing packages into the same virtual environment as ``sqlite-utils``, :ref:`described here `. (:issue:`483`) - New :ref:`sqlite_utils.utils.flatten() ` utility function. (:issue:`500`) -- Documentation on :ref:`using Just ` to run tests, linters and build documentation. +- Documentation on :ref:`using Just ` to run tests, linters and build documentation. - Documentation now covers the :ref:`release_process` for this package. .. _v3_29: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c3bf7a084..0cc65ce51 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -20,7 +20,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "query", "memory", "insert", "upsert", "bulk", "search", "transform", "extract", "schema", "insert-files", "analyze-tables", "convert", "tables", "views", "rows", "triggers", "indexes", "create-database", "create-table", "create-index", - "enable-fts", "populate-fts", "rebuild-fts", "disable-fts" + "migrate", "enable-fts", "populate-fts", "rebuild-fts", "disable-fts" ] refs = { "query": "cli_query", @@ -49,6 +49,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "enable-wal": "cli_wal", "enable-counts": "cli_enable_counts", "bulk": "cli_bulk", + "migrate": "cli_migrate", "create-database": "cli_create_database", "create-table": "cli_create_table", "drop-table": "cli_drop_table", @@ -965,6 +966,40 @@ See :ref:`cli_create_index`. -h, --help Show this message and exit. +.. _cli_ref_migrate: + +migrate +======= + +See :ref:`cli_migrate`. + +:: + + Usage: sqlite-utils migrate [OPTIONS] DB_PATH [MIGRATIONS]... + + Apply pending database migrations. + + Usage: + + sqlite-utils migrate database.db + + This will find the migrations.py file in the current directory or + subdirectories and apply any pending migrations. + + Or pass paths to one or more migrations.py files directly: + + sqlite-utils migrate database.db path/to/migrations.py + + Pass --list to see a list of applied and pending migrations without applying + them. + + Options: + --stop-before TEXT Stop before applying this migration + --list List migrations without running them + -v, --verbose Show verbose output + -h, --help Show this message and exit. + + .. _cli_ref_enable_fts: enable-fts diff --git a/docs/cli.rst b/docs/cli.rst index 415e4c281..acf302d03 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1058,6 +1058,29 @@ That will look for SpatiaLite in a set of predictable locations. To load it from sqlite-utils create-database empty.db --init-spatialite --load-extension /path/to/spatialite.so +.. _cli_migrate: + +Running migrations +================== + +The ``migrate`` command applies pending Python migrations to a database. For the full migration file format and Python API, see :ref:`migrations`. + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py + +If you omit the migration path it will search the current directory and subdirectories for files called ``migrations.py``: + +.. code-block:: bash + + sqlite-utils migrate creatures.db + +Use ``--list`` to list applied and pending migrations without running them: + +.. code-block:: bash + + sqlite-utils migrate creatures.db --list + .. _cli_inserting_data: Inserting JSON data diff --git a/docs/index.rst b/docs/index.rst index 27a22c330..052b790fc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,6 +36,7 @@ Contents installation cli python-api + migrations plugins reference cli-reference diff --git a/docs/migrations.rst b/docs/migrations.rst new file mode 100644 index 000000000..6b2715406 --- /dev/null +++ b/docs/migrations.rst @@ -0,0 +1,164 @@ +.. _migrations: + +==================== + Database migrations +==================== + +``sqlite-utils`` includes a small migration system for applying repeatable changes to SQLite database files. + +A migration is a Python function that receives a :class:`sqlite_utils.Database` instance. Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database. + +Applying migrations in Python +============================= + +Create a :class:`sqlite_utils.Migrations` object, decorate migration functions with it and call ``.apply(db)`` against a :class:`sqlite_utils.Database` instance: + +.. code-block:: python + + from sqlite_utils import Database, Migrations + + migrations = Migrations("creatures") + + @migrations() + def create_table(db): + db["creatures"].create( + {"id": int, "name": str, "species": str}, + pk="id", + ) + + @migrations() + def add_weight(db): + db["creatures"].add_column("weight", float) + + db = Database("creatures.db") + migrations.apply(db) + +Running ``migrations.apply(db)`` repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped. + +The name passed to ``Migrations("creatures")`` identifies that set of migrations. Use a name that is unique for your project, since multiple migration sets can be applied to the same database. + +Migration functions are applied in the order their decorators run. The function name is used as the migration name unless you pass one explicitly: + +.. code-block:: python + + @migrations(name="001_create_table") + def create_table(db): + db["creatures"].create({"id": int, "name": str}, pk="id") + +You can also stop before a named migration: + +.. code-block:: python + + migrations.apply(db, stop_before="add_weight") + +Migration files +=============== + +The ``sqlite-utils migrate`` command looks for migration sets in Python files, usually named ``migrations.py``. A migration file should define one or more :class:`sqlite_utils.Migrations` objects: + +.. code-block:: python + + from sqlite_utils import Migrations + + migrations = Migrations("creatures") + + @migrations() + def create_table(db): + db["creatures"].create( + {"id": int, "name": str, "species": str}, + pk="id", + ) + + @migrations() + def add_weight(db): + db["creatures"].add_column("weight", float) + +Applying migrations using the CLI +================================= + +Run migrations using the ``sqlite-utils migrate`` command: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py + +The first argument is the database file. The remaining arguments can be paths to migration files or directories containing migration files. + +If you omit migration paths, ``sqlite-utils`` searches the current directory and subdirectories for files called ``migrations.py``: + +.. code-block:: bash + + sqlite-utils migrate creatures.db + +You can also pass a directory. Every ``migrations.py`` file in that directory tree will be considered: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/project/ + +Running the command repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped. + +Listing migrations +================== + +Use ``--list`` to show applied and pending migrations without running them: + +.. code-block:: bash + + sqlite-utils migrate creatures.db --list + +Example output: + +.. code-block:: output + + Migrations for: creatures + + Applied: + create_table - 2026-06-09 17:23:12.048092+00:00 + add_weight - 2026-06-09 17:23:12.051249+00:00 + + Pending: + add_age + +Stopping before a migration +=========================== + +When applying a single migration file, you can stop before a named migration: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py --stop-before add_weight + +This applies any pending migrations before ``add_weight`` and leaves ``add_weight`` and later migrations pending. + +Verbose output +============== + +Use ``--verbose`` or ``-v`` to show the schema before and after migrations are applied, plus a unified diff when the schema changes: + +.. code-block:: bash + + sqlite-utils migrate creatures.db --verbose + +Migrating from sqlite-migrate +============================= + +This system uses the same migration table format as the separate ``sqlite-migrate`` package. To use existing migration files directly with ``sqlite-utils``, update their import from ``sqlite_migrate`` to ``sqlite_utils``: + +.. code-block:: python + + from sqlite_utils import Migrations + + migration = Migrations("creatures") + + @migration() + def create_table(db): + db["creatures"].create({"id": int, "name": str}, pk="id") + +Python API +========== + +.. autoclass:: sqlite_utils.migrations.Migrations + :members: + :undoc-members: + :exclude-members: _Migration, _AppliedMigration diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index b8046f6ae..58ee7ab12 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -2,5 +2,6 @@ from .hookspecs import hookimpl from .hookspecs import hookspec from .db import Database +from .migrations import Migrations -__all__ = ["Database", "suggest_column_types", "hookimpl", "hookspec"] +__all__ = ["Database", "Migrations", "suggest_column_types", "hookimpl", "hookspec"] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5844dfc0c..6bd0367ba 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,4 +1,5 @@ import base64 +import difflib from typing import Any import click from click_default_group import DefaultGroup @@ -3252,6 +3253,125 @@ def create_spatial_index(db_path, table, column_name, load_extension): db.table(table).create_spatial_index(column_name) +def _find_migration_files(migrations): + if not migrations: + migrations = [pathlib.Path(".").resolve()] + files = set() + for path_str in migrations: + path = pathlib.Path(path_str) + if path.is_dir(): + files.update(path.rglob("migrations.py")) + else: + files.add(path) + return sorted(files) + + +def _compatible_migration_set(obj): + return isinstance(obj, sqlite_utils.Migrations) or all( + hasattr(obj, attr) for attr in ("name", "applied", "pending", "apply") + ) + + +def _load_migration_sets(files): + migration_sets = [] + for filepath in files: + code = filepath.read_text() + namespace = { + "__file__": str(filepath), + "__name__": "__sqlite_utils_migration__", + } + exec(code, namespace) + migration_sets.extend( + obj for obj in namespace.values() if _compatible_migration_set(obj) + ) + return migration_sets + + +def _display_migration_list(db, migration_sets): + for migration_set in migration_sets: + click.echo("Migrations for: {}".format(migration_set.name)) + click.echo() + click.echo(" Applied:") + for migration in migration_set.applied(db): + click.echo(" {} - {}".format(migration.name, migration.applied_at)) + click.echo() + click.echo(" Pending:") + output = False + for migration in migration_set.pending(db): + output = True + click.echo(" {}".format(migration.name)) + if not output: + click.echo(" (none)") + click.echo() + + +@click.command() +@click.argument( + "db_path", type=click.Path(dir_okay=False, readable=True, writable=True) +) +@click.argument("migrations", type=click.Path(dir_okay=True, exists=True), nargs=-1) +@click.option("--stop-before", help="Stop before applying this migration") +@click.option( + "list_", "--list", is_flag=True, help="List migrations without running them" +) +@click.option("-v", "--verbose", is_flag=True, help="Show verbose output") +def migrate(db_path, migrations, stop_before, list_, verbose): + """ + Apply pending database migrations. + + Usage: + + sqlite-utils migrate database.db + + This will find the migrations.py file in the current directory + or subdirectories and apply any pending migrations. + + Or pass paths to one or more migrations.py files directly: + + sqlite-utils migrate database.db path/to/migrations.py + + Pass --list to see a list of applied and pending migrations + without applying them. + """ + files = _find_migration_files(migrations) + migration_sets = _load_migration_sets(files) + if not migration_sets: + raise click.ClickException("No migrations.py files found") + + if stop_before and len(migration_sets) > 1: + raise click.ClickException( + "--stop-before can only be used with a single migrations.py file" + ) + + db = sqlite_utils.Database(db_path) + _register_db_for_cleanup(db) + + if list_: + _display_migration_list(db, migration_sets) + return + + prev_schema = db.schema + if verbose: + click.echo("Migrating {}".format(db_path)) + click.echo("\nSchema before:\n") + click.echo(textwrap.indent(prev_schema, " ") or " (empty)") + click.echo() + for migration_set in migration_sets: + migration_set.apply(db, stop_before=stop_before) + if verbose: + click.echo("Schema after:\n") + post_schema = db.schema + if post_schema == prev_schema: + click.echo(" (unchanged)") + else: + click.echo(textwrap.indent(post_schema, " ")) + click.echo("\nSchema diff:\n") + diff = list( + difflib.unified_diff(prev_schema.splitlines(), post_schema.splitlines()) + ) + click.echo("\n".join(diff[3:])) + + @cli.command(name="plugins") def plugins_list(): "List installed plugins" @@ -3259,6 +3379,7 @@ def plugins_list(): pm.hook.register_commands(cli=cli) +cli.add_command(migrate) def _render_common(title, values): diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py new file mode 100644 index 000000000..4e2d85695 --- /dev/null +++ b/sqlite_utils/migrations.py @@ -0,0 +1,119 @@ +from dataclasses import dataclass +import datetime +from typing import Callable, cast, TYPE_CHECKING + +if TYPE_CHECKING: + from sqlite_utils.db import Database, Table + + +class Migrations: + migrations_table = "_sqlite_migrations" + + @dataclass + class _Migration: + name: str + fn: Callable + + @dataclass + class _AppliedMigration: + name: str + applied_at: datetime.datetime + + def __init__(self, name: str): + """ + :param name: The name of the migration set. This should be unique. + """ + self.name = name + self._migrations: list[Migrations._Migration] = [] + + def __call__(self, *, name: str | None = None) -> Callable: + """ + :param name: The name to use for this migration - if not provided, + the name of the function will be used. + """ + + def inner(func: Callable) -> Callable: + self._migrations.append( + self._Migration(name or getattr(func, "__name__"), func) + ) + return func + + return inner + + def pending(self, db: "Database") -> list["Migrations._Migration"]: + """ + Return a list of pending migrations. + """ + self.ensure_migrations_table(db) + already_applied = { + r["name"] + for r in db[self.migrations_table].rows_where( + "migration_set = ?", [self.name] + ) + } + return [ + migration + for migration in self._migrations + if migration.name not in already_applied + ] + + def applied(self, db: "Database") -> list["Migrations._AppliedMigration"]: + """ + Return a list of applied migrations. + """ + self.ensure_migrations_table(db) + return [ + self._AppliedMigration(name=row["name"], applied_at=row["applied_at"]) + for row in db[self.migrations_table].rows_where( + "migration_set = ?", [self.name] + ) + ] + + def apply(self, db: "Database", *, stop_before: str | None = None): + """ + Apply any pending migrations to the database. + """ + self.ensure_migrations_table(db) + for migration in self.pending(db): + name = migration.name + if name == stop_before: + return + migration.fn(db) + _table(db, self.migrations_table).insert( + { + "migration_set": self.name, + "name": name, + "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), + } + ) + + def ensure_migrations_table(self, db: "Database"): + """ + Ensure the _sqlite_migrations table exists and has the correct schema. + """ + table = _table(db, self.migrations_table) + if not table.exists(): + table.create( + { + "id": int, + "migration_set": str, + "name": str, + "applied_at": str, + }, + pk="id", + ) + table.create_index(["migration_set", "name"], unique=True) + elif table.pks != ["id"]: + table.transform(pk="id") + unique_indexes = {tuple(index.columns) for index in table.indexes} + if ("migration_set", "name") not in unique_indexes: + table.create_index(["migration_set", "name"], unique=True) + + def __repr__(self): + return "".format( + self.name, ", ".join(m.name for m in self._migrations) + ) + + +def _table(db: "Database", name: str) -> "Table": + return cast("Table", db[name]) diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py new file mode 100644 index 000000000..4b7d3f315 --- /dev/null +++ b/tests/test_cli_migrate.py @@ -0,0 +1,227 @@ +import pathlib + +from click.testing import CliRunner +import pytest +import sqlite_utils +import sqlite_utils.cli + +TWO_MIGRATIONS = """ +from sqlite_utils import Migrations + +m = Migrations("hello") + +@m() +def foo(db): + db["foo"].insert({"hello": "world"}) + +@m() +def bar(db): + db["bar"].insert({"hello": "world"}) +""" + + +@pytest.fixture +def two_migrations(tmpdir): + path = pathlib.Path(tmpdir) + (path / "foo").mkdir() + migrations_py = path / "foo" / "migrations.py" + migrations_py.write_text(TWO_MIGRATIONS, "utf-8") + return path, migrations_py + + +@pytest.mark.parametrize("arg", ("TMPDIR", "TMPDIR/foo/migrations.py", "TMPDIR/foo/")) +def test_basic(two_migrations, arg): + path, _ = two_migrations + db_path = str(path / "test.db") + + runner = CliRunner() + + def _list(): + list_result = runner.invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, "--list", arg.replace("TMPDIR", str(path))], + ) + assert list_result.exit_code == 0 + return list_result.output + + assert _list() == ( + "Migrations for: hello\n\n" + " Applied:\n\n" + " Pending:\n" + " foo\n" + " bar\n\n" + ) + + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, arg.replace("TMPDIR", str(path))] + ) + assert result.exit_code == 0, result.output + + list_output = _list() + assert "Migrations for: hello\n\n Applied:\n " in list_output + prior_to_pending = list_output.split(" Pending")[0] + assert " foo" in prior_to_pending + assert " bar" in prior_to_pending + assert " Pending:\n (none)" in list_output + + db = sqlite_utils.Database(db_path) + assert db["foo"].exists() + assert db["bar"].exists() + assert db["_sqlite_migrations"].exists() + rows = list(db["_sqlite_migrations"].rows) + assert len(rows) == 2 + assert rows[0]["name"] == "foo" + assert rows[1]["name"] == "bar" + + +def test_list_same_migration_names_in_different_sets(capsys): + applied = sqlite_utils.Migrations("applied") + + @applied(name="foo") + def applied_foo(db): + db["applied"].insert({"hello": "world"}) + + pending = sqlite_utils.Migrations("pending") + + @pending(name="foo") + def pending_foo(db): + db["pending"].insert({"hello": "world"}) + + db = sqlite_utils.Database(memory=True) + applied.apply(db) + + sqlite_utils.cli._display_migration_list(db, [applied, pending]) + + output = capsys.readouterr().out + assert ( + "Migrations for: pending\n\n" " Applied:\n\n" " Pending:\n" " foo\n\n" + ) in output + + +def test_verbose(tmpdir): + path = pathlib.Path(tmpdir) + (path / "foo").mkdir() + migrations_py = path / "foo" / "migrations.py" + migrations_py.write_text( + """ +from sqlite_utils import Migrations + +m = Migrations("hello") + +@m() +def foo(db): + db["dogs"].insert({"id": 1, "name": "Cleo"}) + """, + "utf-8", + ) + db_path = str(path / "test.db") + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py)] + ) + assert result.exit_code == 0 + + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"] + ) + assert result.exit_code == 0 + expected = """ +Schema before: + + CREATE TABLE "_sqlite_migrations" ( + "id" INTEGER PRIMARY KEY, + "migration_set" TEXT, + "name" TEXT, + "applied_at" TEXT + ); + CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name" + ON "_sqlite_migrations" ("migration_set", "name"); + CREATE TABLE "dogs" ( + "id" INTEGER, + "name" TEXT + ); + +Schema after: + + (unchanged) +""".strip() + assert expected in result.output + + new_migration = """ +@m() +def bar(db): + db["dogs"].add_column("age", int) + db["dogs"].add_column("weight", float) + db["dogs"].transform() +""" + migrations_py.write_text(migrations_py.read_text("utf-8") + new_migration) + + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"] + ) + assert result.exit_code == 0 + expected_diff = """ +Schema diff: + + ON "_sqlite_migrations" ("migration_set", "name"); + CREATE TABLE "dogs" ( + "id" INTEGER, +- "name" TEXT ++ "name" TEXT, ++ "age" INTEGER, ++ "weight" REAL + ); +""".strip() + assert expected_diff in result.output + + +def test_stop_before(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(path / "foo" / "migrations.py"), + "--stop-before", + "bar", + ], + ) + assert result.exit_code == 0 + db = sqlite_utils.Database(db_path) + assert db["foo"].exists() + assert not db["bar"].exists() + + +def test_stop_before_error(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + (path / "foo" / "migrations2.py").write_text( + """ +from sqlite_utils import Migrations + +m = Migrations("hello2") + +@m() +def foo(db): + db["foo"].insert({"hello": "world"}) + """, + "utf-8", + ) + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(path / "foo" / "migrations.py"), + str(path / "foo" / "migrations2.py"), + "--stop-before", + "foo", + ], + ) + assert result.exit_code == 1 + assert ( + "--stop-before can only be used with a single migrations.py file" + in result.output + ) diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 000000000..6a0dd3392 --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,110 @@ +import pytest +import sqlite_utils +from sqlite_utils import Migrations + + +@pytest.fixture +def migrations(): + migrations = Migrations("test") + + @migrations() + def m001(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations() + def m002(db): + db["cats"].create({"name": str}) + db.query("insert into dogs (name) values ('Pancakes')") + + return migrations + + +@pytest.fixture +def migrations_not_ordered_alphabetically(): + # Names order alphabetically in the wrong direction but this + # should still be applied correctly. + migrations = Migrations("test") + + @migrations() + def m002(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations() + def m001(db): + db["cats"].create({"name": str}) + db.query("insert into dogs (name) values ('Pancakes')") + + return migrations + + +@pytest.fixture +def migrations2(): + migrations = Migrations("test2") + + @migrations() + def m001(db): + db["dogs2"].insert({"name": "Cleo"}) + + return migrations + + +def test_basic(migrations): + db = sqlite_utils.Database(memory=True) + assert db.table_names() == [] + migrations.apply(db) + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} + + +def test_stop_before(migrations): + db = sqlite_utils.Database(memory=True) + assert db.table_names() == [] + migrations.apply(db, stop_before="m002") + assert set(db.table_names()) == {"_sqlite_migrations", "dogs"} + migrations.apply(db) + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} + + +def test_two_migration_sets(migrations, migrations2): + db = sqlite_utils.Database(memory=True) + assert db.table_names() == [] + migrations.apply(db) + migrations2.apply(db) + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats", "dogs2"} + + +def test_order_does_not_matter(migrations, migrations_not_ordered_alphabetically): + db1 = sqlite_utils.Database(memory=True) + db2 = sqlite_utils.Database(memory=True) + migrations.apply(db1) + migrations_not_ordered_alphabetically.apply(db2) + assert db1.schema == db2.schema + + +@pytest.mark.parametrize( + "create_table,pk", + ( + ( + { + "migration_set": str, + "name": str, + "applied_at": str, + }, + "name", + ), + ( + { + "migration_set": str, + "name": str, + "applied_at": str, + }, + ("migration_set", "name"), + ), + ), +) +def test_upgrades_sqlite_migrations(migrations, create_table, pk): + db = sqlite_utils.Database(memory=True) + db["_sqlite_migrations"].create(create_table, pk=pk) + assert db.table_names() == ["_sqlite_migrations"] + assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk)) + migrations.apply(db) + assert db["_sqlite_migrations"].pks == ["id"] From 711718c6940e9964a01e73fcb0f603bcdf5f2ad1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 08:44:03 -0700 Subject: [PATCH 2/7] Better changelog entry --- docs/changelog.rst | 2 +- docs/migrations.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 098f00009..d5c41aa72 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Unreleased ---------- -- New ``sqlite-utils migrate`` command for applying Python database migrations, incorporating functionality that was previously provided by the separate `sqlite-migrate `__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using ``sqlite-utils migrate database.db migrations.py`` or the :ref:`migrations Python API `. See :ref:`migrations` for details. (:issue:`752`) +- New :ref:`database migrations system `, incorporating functionality that was previously provided by the separate `sqlite-migrate `__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using the ``sqlite-utils migrate`` command or the :ref:`migrations Python API `. (:issue:`752`) .. _v3_39: diff --git a/docs/migrations.rst b/docs/migrations.rst index 6b2715406..ef1d6db94 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -8,6 +8,8 @@ A migration is a Python function that receives a :class:`sqlite_utils.Database` instance. Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database. +.. _migrations_python: + Applying migrations in Python ============================= From 34cab63963438ccd0836cdffa9eb322862cec147 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 09:03:33 -0700 Subject: [PATCH 3/7] Final edits to migrations documentation, refs #752 --- docs/migrations.rst | 65 +++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/docs/migrations.rst b/docs/migrations.rst index ef1d6db94..fdffc006a 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -1,19 +1,29 @@ .. _migrations: -==================== +===================== Database migrations -==================== +===================== -``sqlite-utils`` includes a small migration system for applying repeatable changes to SQLite database files. +``sqlite-utils`` includes a migration system for applying repeatable changes to SQLite database files. -A migration is a Python function that receives a :class:`sqlite_utils.Database` instance. Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database. +A migration is a Python function that receives a :class:`sqlite_utils.Database` instance and then executes Python code to modify that database - creating or transforming tables, adding indexes, inserting rows, or any other operation suppored by SQLite. -.. _migrations_python: +Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database. -Applying migrations in Python -============================= +This means you can run the migrate operation multiple times and it will only apply migrations that have not previously been recorded. + +.. _migrations_define: + +Defining migrations +=================== -Create a :class:`sqlite_utils.Migrations` object, decorate migration functions with it and call ``.apply(db)`` against a :class:`sqlite_utils.Database` instance: +Ordered migration sets are defined by first creating a :class:`sqlite_utils.Migrations` object. + +Individual migrations are Python functions that are then registered with that migration set. Each migration function is passed a single argument that is a :ref:`sqlite_utils.Database ` instance. + +The name passed to ``Migrations("creatures")`` identifies that set of migrations. Use a name that is unique for your project, since multiple migration sets can be applied to the same database. + +Here is a simple example of a ``migrations.py`` file which creates a table, then adds an extra column to that table in a second migration: .. code-block:: python @@ -32,14 +42,21 @@ Create a :class:`sqlite_utils.Migrations` object, decorate migration functions w def add_weight(db): db["creatures"].add_column("weight", float) +.. _migrations_python: + +Applying migrations in Python +============================= + +Once you have a ``Migrations(name)`` collection with one or more migrations registered to it, you can eexcute them in Python code like this: + +.. code-block:: python + db = Database("creatures.db") migrations.apply(db) Running ``migrations.apply(db)`` repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped. -The name passed to ``Migrations("creatures")`` identifies that set of migrations. Use a name that is unique for your project, since multiple migration sets can be applied to the same database. - -Migration functions are applied in the order their decorators run. The function name is used as the migration name unless you pass one explicitly: +Migration functions are applied in the order that they were registered. The function name is used as the migration name unless you pass one explicitly: .. code-block:: python @@ -47,34 +64,12 @@ Migration functions are applied in the order their decorators run. The function def create_table(db): db["creatures"].create({"id": int, "name": str}, pk="id") -You can also stop before a named migration: +When you apply a sit of migrations you can stop part way through by specifying a ``stop_before=`` migration name: .. code-block:: python migrations.apply(db, stop_before="add_weight") -Migration files -=============== - -The ``sqlite-utils migrate`` command looks for migration sets in Python files, usually named ``migrations.py``. A migration file should define one or more :class:`sqlite_utils.Migrations` objects: - -.. code-block:: python - - from sqlite_utils import Migrations - - migrations = Migrations("creatures") - - @migrations() - def create_table(db): - db["creatures"].create( - {"id": int, "name": str, "species": str}, - pk="id", - ) - - @migrations() - def add_weight(db): - db["creatures"].add_column("weight", float) - Applying migrations using the CLI ================================= @@ -145,7 +140,7 @@ Use ``--verbose`` or ``-v`` to show the schema before and after migrations are a Migrating from sqlite-migrate ============================= -This system uses the same migration table format as the separate ``sqlite-migrate`` package. To use existing migration files directly with ``sqlite-utils``, update their import from ``sqlite_migrate`` to ``sqlite_utils``: +This system uses the same migration table format as the older `sqlite-migrate `__ package. To use existing migration files directly with ``sqlite-utils``, update their import from ``sqlite_migrate`` to ``sqlite_utils``: .. code-block:: python From 4c265933b9bf98bf5114d9779b0c61f44194a743 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 09:08:13 -0700 Subject: [PATCH 4/7] Typo fix --- docs/migrations.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migrations.rst b/docs/migrations.rst index fdffc006a..55f9547b2 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -6,7 +6,7 @@ ``sqlite-utils`` includes a migration system for applying repeatable changes to SQLite database files. -A migration is a Python function that receives a :class:`sqlite_utils.Database` instance and then executes Python code to modify that database - creating or transforming tables, adding indexes, inserting rows, or any other operation suppored by SQLite. +A migration is a Python function that receives a :class:`sqlite_utils.Database` instance and then executes Python code to modify that database - creating or transforming tables, adding indexes, inserting rows, or any other operation supported by SQLite. Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database. From 49fbb1b26238978f9e785059e46778c2a0529515 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 09:15:47 -0700 Subject: [PATCH 5/7] Spelling and grammar sweep --- docs/migrations.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/migrations.rst b/docs/migrations.rst index 55f9547b2..11e8ea8cd 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -47,7 +47,7 @@ Here is a simple example of a ``migrations.py`` file which creates a table, then Applying migrations in Python ============================= -Once you have a ``Migrations(name)`` collection with one or more migrations registered to it, you can eexcute them in Python code like this: +Once you have a ``Migrations(name)`` collection with one or more migrations registered to it, you can execute them in Python code like this: .. code-block:: python @@ -64,7 +64,7 @@ Migration functions are applied in the order that they were registered. The func def create_table(db): db["creatures"].create({"id": int, "name": str}, pk="id") -When you apply a sit of migrations you can stop part way through by specifying a ``stop_before=`` migration name: +When you apply a set of migrations you can stop part way through by specifying a ``stop_before=`` migration name: .. code-block:: python @@ -120,7 +120,7 @@ Example output: Stopping before a migration =========================== -When applying a single migration file, you can stop before a named migration: +When applying a single migration set, you can stop before a named migration: .. code-block:: bash From b4a38be626b08585e8d74461b0416103477e505b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 09:16:09 -0700 Subject: [PATCH 6/7] Fixes for ty type checks --- sqlite_utils/cli.py | 49 ++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 6bd0367ba..f326996c2 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -11,6 +11,7 @@ from sqlite_utils.db import ( AlterError, BadMultiValues, + DEFAULT, DescIndex, NoTable, quote_identifier, @@ -2579,7 +2580,6 @@ def transform( _register_db_for_cleanup(db) _load_extensions(db, load_extension) types = {} - kwargs = {} for column, ctype in type: if ctype.upper() not in VALID_COLUMN_TYPES: raise click.ClickException( @@ -2599,29 +2599,46 @@ def transform( for column in default_none: default_dict[column] = None - kwargs["types"] = types - kwargs["drop"] = set(drop) - kwargs["rename"] = dict(rename) - kwargs["column_order"] = column_order or None - kwargs["not_null"] = not_null_dict + drop_set = set(drop) + rename_dict = dict(rename) + column_order_list = list(column_order) or None + drop_foreign_keys_value = drop_foreign_keys or None + add_foreign_keys_value = add_foreign_keys or None + pk_value = DEFAULT if pk: if len(pk) == 1: - kwargs["pk"] = pk[0] + pk_value = pk[0] else: - kwargs["pk"] = pk + pk_value = pk elif pk_none: - kwargs["pk"] = None - kwargs["defaults"] = default_dict - if drop_foreign_keys: - kwargs["drop_foreign_keys"] = drop_foreign_keys - if add_foreign_keys: - kwargs["add_foreign_keys"] = add_foreign_keys + pk_value = None + table_obj = db.table(table) if sql: - for line in db.table(table).transform_sql(**kwargs): + for line in table_obj.transform_sql( + types=types, + drop=drop_set, + rename=rename_dict, + column_order=column_order_list, + not_null=not_null_dict, + pk=pk_value, + defaults=default_dict, + drop_foreign_keys=drop_foreign_keys_value, + add_foreign_keys=add_foreign_keys_value, + ): click.echo(line) else: - db.table(table).transform(**kwargs) + table_obj.transform( + types=types, + drop=drop_set, + rename=rename_dict, + column_order=column_order_list, + not_null=not_null_dict, + pk=pk_value, + defaults=default_dict, + drop_foreign_keys=drop_foreign_keys_value, + add_foreign_keys=add_foreign_keys_value, + ) @cli.command() From 3423195b3ace817b815804fa14ffca84f032ac68 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 09:28:34 -0700 Subject: [PATCH 7/7] Better --stop-before Refs https://github.com/simonw/sqlite-utils/issues/752#issuecomment-4762557380 --- docs/cli-reference.rst | 6 ++- docs/cli.rst | 8 ++++ docs/migrations.rst | 14 ++++++- sqlite_utils/cli.py | 31 ++++++++++---- sqlite_utils/migrations.py | 11 ++++- tests/test_cli_migrate.py | 86 +++++++++++++++++++++++++++++++++++--- 6 files changed, 139 insertions(+), 17 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 0cc65ce51..64a27c261 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -993,8 +993,12 @@ See :ref:`cli_migrate`. Pass --list to see a list of applied and pending migrations without applying them. + Use --stop-before migration_set:name to stop before a migration. This option + can be used multiple times. + Options: - --stop-before TEXT Stop before applying this migration + --stop-before TEXT Stop before applying this migration. Use set:name to + target a migration set. --list List migrations without running them -v, --verbose Show verbose output -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index acf302d03..c9389d88c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1081,6 +1081,14 @@ Use ``--list`` to list applied and pending migrations without running them: sqlite-utils migrate creatures.db --list +Use ``--stop-before`` to stop before a named migration. The option can be passed more than once, and can target a specific migration set using ``migration_set:migration_name``: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py \ + --stop-before creatures:add_weight \ + --stop-before sales:drop_index + .. _cli_inserting_data: Inserting JSON data diff --git a/docs/migrations.rst b/docs/migrations.rst index 11e8ea8cd..948b46887 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -120,13 +120,23 @@ Example output: Stopping before a migration =========================== -When applying a single migration set, you can stop before a named migration: +When applying migrations using the CLI, you can stop before a named migration: .. code-block:: bash sqlite-utils migrate creatures.db path/to/migrations.py --stop-before add_weight -This applies any pending migrations before ``add_weight`` and leaves ``add_weight`` and later migrations pending. +This applies any pending migrations before ``add_weight`` and leaves ``add_weight`` and later migrations pending. An unqualified migration name matches in any migration set. + +You can also target a specific migration set using ``migration_set:migration_name``. This is useful if a migrations file contains more than one migration set, or if multiple sets use the same migration name: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py \ + --stop-before creatures:add_weight \ + --stop-before sales:drop_index + +The ``--stop-before`` option can be passed more than once. Verbose output ============== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index f326996c2..1c3ce8d98 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3322,12 +3322,28 @@ def _display_migration_list(db, migration_sets): click.echo() +def _stop_before_for_migration_set(stop_before, migration_set_name): + matches = [] + for value in stop_before: + set_name, separator, migration_name = value.partition(":") + if separator: + if set_name == migration_set_name: + matches.append(migration_name) + else: + matches.append(value) + return matches + + @click.command() @click.argument( "db_path", type=click.Path(dir_okay=False, readable=True, writable=True) ) @click.argument("migrations", type=click.Path(dir_okay=True, exists=True), nargs=-1) -@click.option("--stop-before", help="Stop before applying this migration") +@click.option( + "--stop-before", + multiple=True, + help="Stop before applying this migration. Use set:name to target a migration set.", +) @click.option( "list_", "--list", is_flag=True, help="List migrations without running them" ) @@ -3349,17 +3365,15 @@ def migrate(db_path, migrations, stop_before, list_, verbose): Pass --list to see a list of applied and pending migrations without applying them. + + Use --stop-before migration_set:name to stop before a + migration. This option can be used multiple times. """ files = _find_migration_files(migrations) migration_sets = _load_migration_sets(files) if not migration_sets: raise click.ClickException("No migrations.py files found") - if stop_before and len(migration_sets) > 1: - raise click.ClickException( - "--stop-before can only be used with a single migrations.py file" - ) - db = sqlite_utils.Database(db_path) _register_db_for_cleanup(db) @@ -3374,7 +3388,10 @@ def migrate(db_path, migrations, stop_before, list_, verbose): click.echo(textwrap.indent(prev_schema, " ") or " (empty)") click.echo() for migration_set in migration_sets: - migration_set.apply(db, stop_before=stop_before) + migration_set.apply( + db, + stop_before=_stop_before_for_migration_set(stop_before, migration_set.name), + ) if verbose: click.echo("Schema after:\n") post_schema = db.schema diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 4e2d85695..f349b17ad 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -1,3 +1,4 @@ +from collections.abc import Iterable from dataclasses import dataclass import datetime from typing import Callable, cast, TYPE_CHECKING @@ -69,14 +70,20 @@ def applied(self, db: "Database") -> list["Migrations._AppliedMigration"]: ) ] - def apply(self, db: "Database", *, stop_before: str | None = None): + def apply(self, db: "Database", *, stop_before: str | Iterable[str] | None = None): """ Apply any pending migrations to the database. """ self.ensure_migrations_table(db) + if stop_before is None: + stop_before_names = set() + elif isinstance(stop_before, str): + stop_before_names = {stop_before} + else: + stop_before_names = set(stop_before) for migration in self.pending(db): name = migration.name - if name == stop_before: + if name in stop_before_names: return migration.fn(db) _table(db, self.migrations_table).insert( diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 4b7d3f315..229bc9cdf 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -29,6 +29,39 @@ def two_migrations(tmpdir): return path, migrations_py +@pytest.fixture +def two_sets_same_migration_name(tmpdir): + path = pathlib.Path(tmpdir) + migrations_py = path / "migrations.py" + migrations_py.write_text( + """ +from sqlite_utils import Migrations + +creatures = Migrations("creatures") + +@creatures() +def create_table(db): + db["creatures"].insert({"name": "Cleo"}) + +@creatures() +def add_weight(db): + db["creature_weights"].insert({"weight": 4.2}) + +sales = Migrations("sales") + +@sales() +def create_table(db): + db["sales"].insert({"id": 1}) + +@sales() +def add_weight(db): + db["sales_weights"].insert({"weight": 10}) +""", + "utf-8", + ) + return path, migrations_py + + @pytest.mark.parametrize("arg", ("TMPDIR", "TMPDIR/foo/migrations.py", "TMPDIR/foo/")) def test_basic(two_migrations, arg): path, _ = two_migrations @@ -194,7 +227,7 @@ def test_stop_before(two_migrations): assert not db["bar"].exists() -def test_stop_before_error(two_migrations): +def test_stop_before_multiple_sets_unqualified(two_migrations): path, _ = two_migrations db_path = str(path / "test.db") (path / "foo" / "migrations2.py").write_text( @@ -220,8 +253,51 @@ def foo(db): "foo", ], ) - assert result.exit_code == 1 - assert ( - "--stop-before can only be used with a single migrations.py file" - in result.output + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert db.table_names() == ["_sqlite_migrations"] + assert list(db["_sqlite_migrations"].rows) == [] + + +def test_stop_before_qualified_only_affects_named_set(two_sets_same_migration_name): + path, migrations_py = two_sets_same_migration_name + db_path = str(path / "test.db") + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(migrations_py), + "--stop-before", + "creatures:add_weight", + ], + ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert db["creatures"].exists() + assert not db["creature_weights"].exists() + assert db["sales"].exists() + assert db["sales_weights"].exists() + + +def test_stop_before_multiple_qualified(two_sets_same_migration_name): + path, migrations_py = two_sets_same_migration_name + db_path = str(path / "test.db") + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(migrations_py), + "--stop-before", + "creatures:add_weight", + "--stop-before", + "sales:add_weight", + ], ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert db["creatures"].exists() + assert not db["creature_weights"].exists() + assert db["sales"].exists() + assert not db["sales_weights"].exists()