diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index fbe6feaba..3be254b8a 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -111,7 +111,8 @@ UUID ~~~~ .. autoclass:: UUID4 - :members: + +.. autoclass:: UUID7 ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index 7a53e1dd0..de947e5fd 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -22,8 +22,8 @@ Make sure the test database exists: .. code-block:: console cockroach sql --insecure - >>> create database piccolo - >>> use piccolo + >>> create database piccolo; + >>> use piccolo; ------------------------------------------------------------------------------- diff --git a/docs/src/piccolo/tutorials/index.rst b/docs/src/piccolo/tutorials/index.rst index f4db575c5..28fe32efb 100644 --- a/docs/src/piccolo/tutorials/index.rst +++ b/docs/src/piccolo/tutorials/index.rst @@ -14,3 +14,4 @@ help you solve common problems: ./avoiding_circular_imports ./moving_table_between_apps ./uuid_columns_in_piccolo_1.31.0 + ./uuid_v7_support.rst diff --git a/docs/src/piccolo/tutorials/uuid_v7_support.rst b/docs/src/piccolo/tutorials/uuid_v7_support.rst new file mode 100644 index 000000000..b246a19fb --- /dev/null +++ b/docs/src/piccolo/tutorials/uuid_v7_support.rst @@ -0,0 +1,26 @@ +UUID v7 support +=============== + +.. note:: CockroachDB does not currently support UUID v7. + +With Postgres 18 and above, UUID v7 is natively supported. + +For this to work in older versions of Postgres, a function has to be registered +in the database. + + +The easiest option is to add the following to :class:`PostgresEngine `: + +.. code-block:: python + + DB = PostgresEngine( + polyfills=['uuid7'], + ... + ) + +Which will try and register a ``uuid7`` function on initialisation. + +Alternatives +------------ + +If you'd rather, you can register an extension such as `pg_uuidv7 `_. diff --git a/piccolo/columns/defaults/uuid.py b/piccolo/columns/defaults/uuid.py index ad0e34679..4a4c159c7 100644 --- a/piccolo/columns/defaults/uuid.py +++ b/piccolo/columns/defaults/uuid.py @@ -3,6 +3,8 @@ from enum import Enum from typing import Union +from piccolo.utils.uuid import uuid7 + from .base import Default @@ -17,6 +19,18 @@ class UUID4(Default): @property def postgres(self): + """ + Historically we had to use `uuid_generate_v4()` from the `uuid-ossp` + extension. + + Since Postgres 13 there is a built-in `gen_random_uuid` function which + generates UUID v4 values. + + In Postgres 18, `uuidv4` was added, which is the same as + `gen_random_uuid`, but more precisely named. We will move to this at + some point in the future. + + """ return "gen_random_uuid()" @property @@ -31,7 +45,42 @@ def python(self): return uuid.uuid4() -UUIDArg = Union[UUID4, uuid.UUID, str, Enum, None, Callable[[], uuid.UUID]] +class UUID7(Default): + @property + def postgres(self): + """ + Supported in Postgres 18 and above. + + For older versions. a custom function has to be loaded into the + database (see ``PostgresEngine.polyfills``). + + """ + return "uuidv7()" + + @property + def cockroach(self): + """ + Unfortunately CockroachDB doesn't current support this. + """ + raise NotImplementedError() + + @property + def sqlite(self): + return "''" + + def python(self): + return uuid7() + + +UUIDArg = Union[ + UUID4, + UUID7, + uuid.UUID, + str, + Enum, + None, + Callable[[], uuid.UUID], +] -__all__ = ["UUIDArg", "UUID4"] +__all__ = ["UUIDArg", "UUID4", "UUID7"] diff --git a/piccolo/engine/postgres.py b/piccolo/engine/postgres.py index ec8c11b91..5a997823e 100644 --- a/piccolo/engine/postgres.py +++ b/piccolo/engine/postgres.py @@ -3,7 +3,7 @@ import contextvars from collections.abc import Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Mapping, Optional, Union +from typing import TYPE_CHECKING, Any, Literal, Mapping, Optional, Union from typing_extensions import Self @@ -352,6 +352,7 @@ class PostgresEngine(Engine[PostgresTransaction]): "config", "extensions", "extra_nodes", + "polyfills", "pool", ) @@ -362,12 +363,14 @@ def __init__( log_queries: bool = False, log_responses: bool = False, extra_nodes: Optional[Mapping[str, PostgresEngine]] = None, + polyfills: list[Literal["uuidv7"]] = [], ) -> None: if extra_nodes is None: extra_nodes = {} self.config = config self.extensions = extensions + self.polyfills = polyfills self.log_queries = log_queries self.log_responses = log_responses self.extra_nodes = extra_nodes @@ -380,7 +383,7 @@ def __init__( engine_type="postgres", log_queries=log_queries, log_responses=log_responses, - min_version_number=10, + min_version_number=13, ) @staticmethod @@ -434,6 +437,11 @@ async def prep_database(self): level=Level.medium, ) + if "uuidv7" in self.polyfills: + from piccolo.utils.uuid import UUID7_DB_POLYFILL + + await self._run_in_new_connection(UUID7_DB_POLYFILL) + ########################################################################### # These typos existed in the codebase for a while, so leaving these proxy # methods for now to ensure backwards compatibility. diff --git a/piccolo/utils/uuid/__init__.py b/piccolo/utils/uuid/__init__.py new file mode 100644 index 000000000..80205a038 --- /dev/null +++ b/piccolo/utils/uuid/__init__.py @@ -0,0 +1,29 @@ +try: + from uuid import uuid7 # type: ignore +except ImportError: + # For version < Python 3.14 + from ._uuid_backport import uuid7 + + +# https://github.com/dverite/postgres-uuidv7-sql/blob/396a44433e6e0eb63b1d9d1517e9098256d97351/sql/uuidv7-sql--1.0.sql#L6-L19 +UUID7_DB_POLYFILL = """ + CREATE OR REPLACE FUNCTION uuidv7( + timestamptz DEFAULT clock_timestamp() + ) RETURNS uuid + AS $$ + -- Replace the first 48 bits of a uuidv4 with the current + -- number of milliseconds since 1970-01-01 UTC + -- and set the "ver" field to 7 by setting additional bits + select encode( + set_bit( + set_bit( + overlay(uuid_send(gen_random_uuid()) placing + substring(int8send((extract(epoch from $1)*1000)::bigint) from 3) + from 1 for 6), + 52, 1), + 53, 1), 'hex')::uuid; + $$ LANGUAGE sql volatile parallel safe; + """ + + +__all__ = ("uuid7", "UUID7_DB_POLYFILL") diff --git a/piccolo/utils/uuid/_uuid_backport.py b/piccolo/utils/uuid/_uuid_backport.py new file mode 100644 index 000000000..6973f1971 --- /dev/null +++ b/piccolo/utils/uuid/_uuid_backport.py @@ -0,0 +1,95 @@ +# flake8: noqa + +r""" +Backport of uuid7 from Python 3.14 - we highly recommend using Python 3.14 if +using uuid7. This is here as a convenience for testing. + +Original author: Ka-Ping Yee +""" + +import os +import time +from uuid import UUID, SafeUUID + +_UINT_128_MAX = (1 << 128) - 1 +# RFC 4122 variant bits and version bits to activate on a UUID integral value. +_RFC_4122_VERSION_7_FLAGS = (7 << 76) | (0x8000 << 48) + + +_last_timestamp_v7 = None +_last_counter_v7 = 0 # 42-bit counter + + +def _uuid7_get_counter_and_tail(): + rand = int.from_bytes(os.urandom(10), byteorder="big") + # 42-bit counter with MSB set to 0 + counter = (rand >> 32) & 0x1FF_FFFF_FFFF + # 32-bit random data + tail = rand & 0xFFFF_FFFF + return counter, tail + + +def uuid7(): + """Generate a UUID from a Unix timestamp in milliseconds and random bits. + + UUIDv7 objects feature monotonicity within a millisecond. + """ + # --- 48 --- -- 4 -- --- 12 --- -- 2 -- --- 30 --- - 32 - + # unix_ts_ms | version | counter_hi | variant | counter_lo | random + # + # 'counter = counter_hi | counter_lo' is a 42-bit counter constructed + # with Method 1 of RFC 9562, ยง6.2, and its MSB is set to 0. + # + # 'random' is a 32-bit random value regenerated for every new UUID. + # + # If multiple UUIDs are generated within the same millisecond, the LSB + # of 'counter' is incremented by 1. When overflowing, the timestamp is + # advanced and the counter is reset to a random 42-bit integer with MSB + # set to 0. + + global _last_timestamp_v7 + global _last_counter_v7 + + nanoseconds = time.time_ns() + timestamp_ms = nanoseconds // 1_000_000 + + if _last_timestamp_v7 is None or timestamp_ms > _last_timestamp_v7: + counter, tail = _uuid7_get_counter_and_tail() + else: + if timestamp_ms < _last_timestamp_v7: + timestamp_ms = _last_timestamp_v7 + 1 + # advance the 42-bit counter + counter = _last_counter_v7 + 1 + if counter > 0x3FF_FFFF_FFFF: + # advance the 48-bit timestamp + timestamp_ms += 1 + counter, tail = _uuid7_get_counter_and_tail() + else: + # 32-bit random data + tail = int.from_bytes(os.urandom(4)) + + unix_ts_ms = timestamp_ms & 0xFFFF_FFFF_FFFF + counter_msbs = counter >> 30 + # keep 12 counter's MSBs and clear variant bits + counter_hi = counter_msbs & 0x0FFF + # keep 30 counter's LSBs and clear version bits + counter_lo = counter & 0x3FFF_FFFF + # ensure that the tail is always a 32-bit integer (by construction, + # it is already the case, but future interfaces may allow the user + # to specify the random tail) + tail &= 0xFFFF_FFFF + + int_uuid_7 = unix_ts_ms << 80 + int_uuid_7 |= counter_hi << 64 + int_uuid_7 |= counter_lo << 32 + int_uuid_7 |= tail + # by construction, the variant and version bits are already cleared + int_uuid_7 |= _RFC_4122_VERSION_7_FLAGS + + assert 0 <= int_uuid_7 <= _UINT_128_MAX, repr(int_uuid_7) + res = UUID(int=int_uuid_7, is_safe=SafeUUID.unknown) + + # defer global update until all computations are done + _last_timestamp_v7 = timestamp_ms + _last_counter_v7 = counter + return res diff --git a/tests/cockroach_conf.py b/tests/cockroach_conf.py index 11b9bf651..0d9c79120 100644 --- a/tests/cockroach_conf.py +++ b/tests/cockroach_conf.py @@ -10,7 +10,7 @@ "user": os.environ.get("PG_USER", "root"), "password": os.environ.get("PG_PASSWORD", ""), "database": os.environ.get("PG_DATABASE", "piccolo"), - } + }, ) diff --git a/tests/columns/test_uuid.py b/tests/columns/test_uuid.py index 3dcce88a1..4a60becb5 100644 --- a/tests/columns/test_uuid.py +++ b/tests/columns/test_uuid.py @@ -1,19 +1,36 @@ import uuid from piccolo.columns.column_types import UUID +from piccolo.columns.defaults.uuid import UUID7 from piccolo.table import Table -from piccolo.testing.test_case import TableTest +from piccolo.testing.test_case import AsyncTableTest +from tests.base import engines_only -class MyTable(Table): +class UUIDTable(Table): uuid = UUID() -class TestUUID(TableTest): - tables = [MyTable] +class TestUUID(AsyncTableTest): + tables = [UUIDTable] - def test_return_type(self): - row = MyTable() - row.save().run_sync() + async def test_return_type(self): + row = UUIDTable() + await row.save() self.assertIsInstance(row.uuid, uuid.UUID) + + +class UUID7Table(Table): + uuid_7 = UUID(default=UUID7()) + + +@engines_only("postgres", "sqlite") +class TestUUID7(AsyncTableTest): + tables = [UUID7Table] + + async def test_return_type(self): + row = UUID7Table() + await row.save() + + self.assertIsInstance(row.uuid_7, uuid.UUID) diff --git a/tests/postgres_conf.py b/tests/postgres_conf.py index af21dcbc5..eeaf96d0b 100644 --- a/tests/postgres_conf.py +++ b/tests/postgres_conf.py @@ -10,7 +10,8 @@ "user": os.environ.get("PG_USER", "postgres"), "password": os.environ.get("PG_PASSWORD", ""), "database": os.environ.get("PG_DATABASE", "piccolo"), - } + }, + polyfills=["uuidv7"], )