diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index bd92a0a48f1f70..d5288a0826d80a 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -864,6 +864,10 @@ uuid (Contributed by Yury Selivanov in :gh:`139123`.) +* :func:`uuid.uuid6` now has a C implementation, with a pure Python fallback. + (Contributed by Yury Selivanov in :gh:`160153`.) + + xml --- diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index cf5bcbb2941d3d..70e9f9e3f9b743 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -366,6 +366,7 @@ _PyStaticObjects_CheckAll(PyInterpreterState *interp) { _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(check_same_thread), "check_same_thread", 17); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(clamp), "clamp", 5); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(clear), "clear", 5); + _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(clock_seq), "clock_seq", 9); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(close), "close", 5); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(closed), "closed", 6); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(closefd), "closefd", 7); @@ -675,7 +676,9 @@ _PyStaticObjects_CheckAll(PyInterpreterState *interp) { _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(newlines), "newlines", 8); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(next), "next", 4); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(nlocals), "nlocals", 7); + _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(node), "node", 4); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(node_depth), "node_depth", 10); + _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(node_func), "node_func", 9); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(node_offset), "node_offset", 11); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(nothing), "nothing", 7); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(ns), "ns", 2); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index bc0a149bc93973..0fc7cb77b5b624 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -378,6 +378,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(check_same_thread) STRUCT_FOR_ID(clamp) STRUCT_FOR_ID(clear) + STRUCT_FOR_ID(clock_seq) STRUCT_FOR_ID(close) STRUCT_FOR_ID(closed) STRUCT_FOR_ID(closefd) @@ -687,7 +688,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(newlines) STRUCT_FOR_ID(next) STRUCT_FOR_ID(nlocals) + STRUCT_FOR_ID(node) STRUCT_FOR_ID(node_depth) + STRUCT_FOR_ID(node_func) STRUCT_FOR_ID(node_offset) STRUCT_FOR_ID(nothing) STRUCT_FOR_ID(ns) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 630b5d78671335..52948c190f500c 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1653,6 +1653,7 @@ extern "C" { INIT_ID(check_same_thread), \ INIT_ID(clamp), \ INIT_ID(clear), \ + INIT_ID(clock_seq), \ INIT_ID(close), \ INIT_ID(closed), \ INIT_ID(closefd), \ @@ -1962,7 +1963,9 @@ extern "C" { INIT_ID(newlines), \ INIT_ID(next), \ INIT_ID(nlocals), \ + INIT_ID(node), \ INIT_ID(node_depth), \ + INIT_ID(node_func), \ INIT_ID(node_offset), \ INIT_ID(nothing), \ INIT_ID(ns), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 621a03016301ab..a9bbc5af564d12 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -330,6 +330,7 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { offsetof(struct _Py_global_strings, identifiers._py_check_same_thread._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_clamp._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_clear._ascii.ob_base), + offsetof(struct _Py_global_strings, identifiers._py_clock_seq._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_close._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_closed._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_closefd._ascii.ob_base), @@ -639,7 +640,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { offsetof(struct _Py_global_strings, identifiers._py_newlines._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_next._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_nlocals._ascii.ob_base), + offsetof(struct _Py_global_strings, identifiers._py_node._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_node_depth._ascii.ob_base), + offsetof(struct _Py_global_strings, identifiers._py_node_func._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_node_offset._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_nothing._ascii.ob_base), offsetof(struct _Py_global_strings, identifiers._py_ns._ascii.ob_base), diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index ee6e57b93966a2..5b760acc8fb29a 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -897,6 +897,21 @@ def test_uuid5(self): equal(u, self.uuid.UUID(v)) equal(str(u), v) + @contextlib.contextmanager + def _uuid6_hooks(self, nanoseconds, *, random_func=None, node_func=None): + time_func = nanoseconds if callable(nanoseconds) else lambda: nanoseconds + self.uuid._install_py_hooks(random_func=random_func, + time_func=time_func, node_func=node_func) + if self.is_c_uuid: + self.uuid._install_c_hooks(random_func=random_func, + time_func=time_func, node_func=node_func) + try: + yield + finally: + self.uuid._install_py_hooks(random_func=None, time_func=None) + if self.is_c_uuid: + self.uuid._install_c_hooks(random_func=None, time_func=None) + def test_uuid6(self): equal = self.assertEqual u = self.uuid.uuid6() @@ -906,12 +921,14 @@ def test_uuid6(self): fake_nanoseconds = 0x1571_20a1_de1a_c533 fake_node_value = 0x54e1_acf6_da7f fake_clock_seq = 0x14c5 - with ( - mock.patch.object(self.uuid, '_last_timestamp_v6', None), - mock.patch.object(self.uuid, 'getnode', return_value=fake_node_value), - mock.patch('time.time_ns', return_value=fake_nanoseconds), - mock.patch('random.getrandbits', return_value=fake_clock_seq) - ): + + def random_func(size): + # The top two entropy bits must be discarded from the clock sequence. + data = (fake_clock_seq | 0xc000).to_bytes(2, 'big') + return (data * ((size + 1) // 2))[:size] + + with self._uuid6_hooks(fake_nanoseconds, random_func=random_func, + node_func=lambda: fake_node_value): u = self.uuid.uuid6() equal(u.variant, self.uuid.RFC_4122) equal(u.version, 6) @@ -945,20 +962,19 @@ def test_uuid6_uniqueness(self): timestamp = 0x1ec9414c_232a_b00 fake_nanoseconds = (timestamp - 0x1b21dd21_3814_000) * 100 - with mock.patch('time.time_ns', return_value=fake_nanoseconds): - def gen(): - with mock.patch.object(self.uuid, '_last_timestamp_v6', None): - return self.uuid.uuid6(node=0, clock_seq=None) - - # By the birthday paradox, sampling N = 1024 UUIDs with identical - # node IDs and timestamps results in duplicates with probability - # close to 1 (not having a duplicate happens with probability of - # order 1E-15) since only the 14-bit clock sequence is randomized. - N = 1024 - uuids = {gen() for _ in range(N)} - self.assertSetEqual({u.node for u in uuids}, {0}) - self.assertSetEqual({u.time for u in uuids}, {timestamp}) - self.assertLess(len(uuids), N, 'collision property does not hold') + def gen(): + with self._uuid6_hooks(fake_nanoseconds): + return self.uuid.uuid6(node=0, clock_seq=None) + + # By the birthday paradox, sampling N = 1024 UUIDs with identical + # node IDs and timestamps results in duplicates with probability + # close to 1 (not having a duplicate happens with probability of + # order 1E-15) since only the 14-bit clock sequence is randomized. + N = 1024 + uuids = {gen() for _ in range(N)} + self.assertSetEqual({u.node for u in uuids}, {0}) + self.assertSetEqual({u.time for u in uuids}, {timestamp}) + self.assertLess(len(uuids), N, 'collision property does not hold') def test_uuid6_node(self): # Make sure the given node ID appears in the UUID. @@ -1016,6 +1032,56 @@ def test_uuid6_clock_seq(self): u = self.uuid.uuid6(node=node, clock_seq=clock_seq) self.assertEqual(get_clock_seq(u), clock_seq & 0x3fff) + def test_uuid6_argument_masking(self): + # Both fields retain their low bits, including for negative and large + # integers. Custom bitwise operations must still be called. + class Maskable: + def __and__(self, mask): + return 0x123456789abcdef & mask + + for value in (-1, -(1 << 100), 1 << 100, True, Maskable()): + with self.subTest(value=value): + u = self.uuid.uuid6(node=value, clock_seq=value) + self.assertEqual(u.node, value & 0xffffffffffff) + self.assertEqual(u.clock_seq, value & 0x3fff) + + class Indexable: + def __index__(self): + return 1 + + for value in (1.5, '1', Indexable()): + for field in ('node', 'clock_seq'): + with self.subTest(value=value, field=field): + with self.assertRaises(TypeError): + self.uuid.uuid6(**{field: value}) + + def test_uuid6_callback_errors(self): + # Entropy and node lookup errors must propagate unchanged. + error = RuntimeError('UUIDv6 callback failed') + + def random_func(size): + raise error + + with self._uuid6_hooks(0, random_func=random_func): + with self.assertRaises(RuntimeError) as cm: + self.uuid.uuid6(node=0) + self.assertIs(cm.exception, error) + def node_func(): + raise error + + with self._uuid6_hooks(0, node_func=node_func): + with self.assertRaises(RuntimeError) as cm: + self.uuid.uuid6(clock_seq=0) + self.assertIs(cm.exception, error) + + def test_uuid6_monotonicity(self): + # Repeated and backward clock readings must still advance the timestamp. + readings = iter((1000, 1000, -1)) + with self._uuid6_hooks(lambda: next(readings)): + values = [self.uuid.uuid6(node=0, clock_seq=0) for _ in range(3)] + self.assertEqual(values[1].time, values[0].time + 1) + self.assertEqual(values[2].time, values[1].time + 1) + def test_uuid6_test_vectors(self): equal = self.assertEqual # https://www.rfc-editor.org/rfc/rfc9562#name-test-vectors @@ -1026,10 +1092,7 @@ def test_uuid6_test_vectors(self): node = 0x9f6bdeced846 clock_seq = (3 << 12) | 0x3c8 - with ( - mock.patch.object(self.uuid, '_last_timestamp_v6', None), - mock.patch('time.time_ns', return_value=fake_nanoseconds) - ): + with self._uuid6_hooks(fake_nanoseconds): u = self.uuid.uuid6(node=node, clock_seq=clock_seq) equal(str(u).upper(), '1EC9414C-232A-6B00-B3C8-9F6BDECED846') # 32 16 4 12 2 14 48 @@ -1854,23 +1917,215 @@ def full_test(p, u): self.assertEqual(len(all_ps), len(all_us)) self.assertEqual(len(all_ps), len(uuids)) - def _install_hooks(self, uuid_mod, *, start_at=0, inc_by=0, seed=0): + def _install_hooks(self, uuid_mod, *, start_at=0, inc_by=0, seed=0, + node_func=None): py_hooks = UuidHooks(start_at=start_at, inc_by=inc_by, seed=seed) uuid_mod._install_py_hooks( random_func=py_hooks.random_func, - time_func=py_hooks.time_func + time_func=py_hooks.time_func, + node_func=node_func, ) c_hooks = UuidHooks(start_at=start_at, inc_by=inc_by, seed=seed) uuid_mod._install_c_hooks( random_func=c_hooks.random_func, - time_func=c_hooks.time_func + time_func=c_hooks.time_func, + node_func=node_func, ) def _reset_hooks(self, uuid_mod): uuid_mod._install_c_hooks(random_func=None, time_func=None) uuid_mod._install_py_hooks(random_func=None, time_func=None) + def test_exact_same_algo_uuid6(self): + # Compare every bit with the Python algorithm across repeated, forward, + # backward, and negative clocks, and masked/default node/sequence values. + # Generate enough values to cross the C entropy cache's refill boundary. + import uuid + + values = (None, 0, -1, (1 << 100) | 0x123456789abc) + try: + for start_at in (-1001, 0, 1672533296123456789): + for inc_by in (-101, 0, 101, 1000000): + for node, clock_seq in product(values, repeat=2): + self._install_hooks(uuid, start_at=start_at, + inc_by=inc_by, + node_func=lambda: 0x123456789abc) + with self.subTest(start=start_at, inc=inc_by, + node=node, clock_seq=clock_seq): + for _ in range(200): + self.assertEqual( + uuid._py_uuid6(node, clock_seq).bytes, + uuid._c_uuid6(node, clock_seq).bytes, + ) + finally: + self._reset_hooks(uuid) + + def test_exact_same_algo_mixed_generators(self): + # UUIDv4/v6/v7 share the C entropy cache. Interleaving their differently + # sized reads must consume the same byte stream as the Python versions, + # including reads that straddle a cache refill. + import uuid + + generators = ( + (uuid._py_uuid4, uuid._c_uuid4, {}), + (uuid._py_uuid6, uuid._c_uuid6, {'node': 0}), + (uuid._py_uuid7, uuid._c_uuid7, {}), + ) + self._install_hooks(uuid, start_at=1672533296123456789, inc_by=101) + try: + for sequence in range(200): + for python_func, c_func, kwargs in generators: + with self.subTest(sequence=sequence, generator=c_func.__name__): + self.assertEqual(python_func(**kwargs).bytes, + c_func(**kwargs).bytes) + finally: + self._reset_hooks(uuid) + + def test_uuid6_node_cache(self): + # Cache the masked default node, including zero. Explicit node arguments + # bypass the cache, and resetting the testing hooks invalidates it. + import uuid + + self.addCleanup(uuid._install_c_hooks, random_func=None, time_func=None) + for node in (0, -1): + with self.subTest(node=node): + getnode = mock.Mock(return_value=node) + uuid._install_c_hooks(random_func=None, time_func=None, + node_func=getnode) + self.assertEqual(uuid.uuid6(clock_seq=0).node, + node & 0xffffffffffff) + self.assertEqual(uuid.uuid6(node=42, clock_seq=0).node, 42) + self.assertEqual(uuid.uuid6(clock_seq=0).node, + node & 0xffffffffffff) + getnode.assert_called_once_with() + + def test_uuid6_node_cache_error(self): + # A failed lookup must leave the cache empty so the next call can retry. + import uuid + + self.addCleanup(uuid._install_c_hooks, random_func=None, time_func=None) + error = RuntimeError('node lookup failed') + getnode = mock.Mock(side_effect=[error, 42]) + uuid._install_c_hooks(random_func=None, time_func=None, + node_func=getnode) + with self.assertRaises(RuntimeError) as cm: + uuid.uuid6(clock_seq=0) + self.assertIs(cm.exception, error) + self.assertEqual(uuid.uuid6(clock_seq=0).node, 42) + self.assertEqual(uuid.uuid6(clock_seq=0).node, 42) + self.assertEqual(getnode.call_count, 2) + + def test_uuid6_node_cache_reentrant(self): + # getnode() can reenter uuid6(). The recursive call initializes the cache; + # the outer call must reuse that value without deadlocking or replacing it. + import uuid + + self.addCleanup(uuid._install_c_hooks, random_func=None, time_func=None) + recursive = False + + def getnode(): + nonlocal recursive + if not recursive: + recursive = True + self.assertEqual(uuid.uuid6(clock_seq=0).node, 11) + return 22 + return 11 + + uuid._install_c_hooks(random_func=None, time_func=None, + node_func=getnode) + self.assertEqual(uuid.uuid6(clock_seq=0).node, 11) + + @threading_helper.requires_working_threading() + @threading_helper.reap_threads + def test_concurrent_uuid6_node_cache(self): + # Pause one lookup while a second thread fills the cache. Both calls + # must use the value published first, even if their lookups disagree. + from concurrent.futures import ThreadPoolExecutor + import threading + import uuid + + entered = threading.Event() + resume = threading.Event() + local = threading.local() + local.id = 2 + + def getnode(): + if local.id == 1: + entered.set() + if not resume.wait(support.SHORT_TIMEOUT): + raise TimeoutError('node lookup was not resumed') + return local.id + + def generate(): + local.id = 1 + return uuid.uuid6(clock_seq=0) + + uuid._install_c_hooks(random_func=None, time_func=None, + node_func=getnode) + self.addCleanup(uuid._install_c_hooks, random_func=None, time_func=None) + with ThreadPoolExecutor(1) as pool: + pending = pool.submit(generate) + try: + self.assertTrue(entered.wait(support.SHORT_TIMEOUT)) + other = uuid.uuid6(clock_seq=0) + finally: + resume.set() + first = pending.result(timeout=support.SHORT_TIMEOUT) + self.assertEqual(first.node, 2) + self.assertEqual(other.node, 2) + + @threading_helper.requires_working_threading() + @threading_helper.reap_threads + def test_concurrent_uuid6_state(self): + # Pause a call in argument conversion or entropy fetching while another + # call completes. + # With fixed time, node, and sequence, only the shared timestamp can + # distinguish them; the resumed call must advance from the newer state. + from concurrent.futures import ThreadPoolExecutor + import threading + import uuid + + for field in ('node', 'clock_seq', 'entropy'): + with self.subTest(field=field): + entered = threading.Event() + resume = threading.Event() + + class Paused: + def __and__(self, mask): + pause() + return 0 + + def pause(): + entered.set() + if not resume.wait(support.SHORT_TIMEOUT): + raise TimeoutError('UUIDv6 callback was not resumed') + + def random_func(size): + pause() + return bytes(size) + + uuid._install_c_hooks( + random_func=random_func if field == 'entropy' else None, + time_func=lambda: 0) + try: + kwargs = dict(node=0, clock_seq=0) + if field == 'entropy': + kwargs['clock_seq'] = None + else: + kwargs[field] = Paused() + with ThreadPoolExecutor(1) as pool: + pending = pool.submit(uuid.uuid6, **kwargs) + try: + self.assertTrue(entered.wait(support.SHORT_TIMEOUT)) + other = uuid.uuid6(node=0, clock_seq=0) + finally: + resume.set() + first = pending.result(timeout=support.SHORT_TIMEOUT) + self.assertEqual(first.time, other.time + 1) + finally: + uuid._install_c_hooks(random_func=None, time_func=None) + def test_invalid_time_hook_result(self): import uuid diff --git a/Lib/uuid.py b/Lib/uuid.py index 2070f63fa54adf..ed844ae5035271 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -113,6 +113,7 @@ class SafeUUID: _random_hook = None _time_hook = None +_node_hook = None def _gen_random(size): @@ -129,14 +130,24 @@ def _gen_time(): return _time_hook() -def _install_py_hooks(*, random_func, time_func): +def _gen_node(): + if _node_hook is None: + return getnode() + return _node_hook() + + +def _install_py_hooks(*, random_func, time_func, node_func=None): global _random_hook global _time_hook + global _node_hook global _last_timestamp_v7 + global _last_timestamp_v6 _random_hook = random_func _time_hook = time_func - # Reset _last_timestamp_v7 for repeatability of tests + _node_hook = node_func + # Reset timestamps for repeatability of tests. _last_timestamp_v7 = None + _last_timestamp_v6 = None # Import optional C extension at toplevel, to help disabling it when testing @@ -833,8 +844,7 @@ def uuid6(node=None, clock_seq=None): of the original 60-bit timestamp. """ global _last_timestamp_v6 - import time - nanoseconds = time.time_ns() + nanoseconds = _gen_time() # 0x01b21dd213814000 is the number of 100-ns intervals between the # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. timestamp = nanoseconds // 100 + 0x01b21dd213814000 @@ -842,13 +852,12 @@ def uuid6(node=None, clock_seq=None): timestamp = _last_timestamp_v6 + 1 _last_timestamp_v6 = timestamp if clock_seq is None: - import random - clock_seq = random.getrandbits(14) # instead of stable storage + clock_seq = int.from_bytes(_gen_random(2)) time_hi_and_mid = (timestamp >> 12) & 0xffff_ffff_ffff time_lo = timestamp & 0x0fff # keep 12 bits and clear version bits clock_s = clock_seq & 0x3fff # keep 14 bits and clear variant bits if node is None: - node = getnode() + node = _gen_node() # --- 32 + 16 --- -- 4 -- -- 12 -- -- 2 -- -- 14 --- 48 # time_hi_and_mid | version | time_lo | variant | clock_seq | node int_uuid_6 = time_hi_and_mid << 80 @@ -964,18 +973,21 @@ def uuid8(a=None, b=None, c=None): _py_uuid4 = uuid4 +_py_uuid6 = uuid6 _py_uuid7 = uuid7 _py_UUID = UUID try: - from _uuid import UUID, uuid4, uuid7 + from _uuid import UUID, uuid4, uuid6, uuid7 from _uuid import _install_c_hooks as _install_c_hooks except ImportError: _c_UUID = None _c_uuid4 = None + _c_uuid6 = None _c_uuid7 = None else: _c_UUID = UUID _c_uuid4 = uuid4 + _c_uuid6 = uuid6 _c_uuid7 = uuid7 def main(): diff --git a/Misc/NEWS.d/next/Library/2026-10-21-12-30-00.gh-issue-139122.Ju6c01.rst b/Misc/NEWS.d/next/Library/2026-10-21-12-30-00.gh-issue-139122.Ju6c01.rst new file mode 100644 index 00000000000000..534edab3137db2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-21-12-30-00.gh-issue-139122.Ju6c01.rst @@ -0,0 +1,3 @@ +Add a C implementation of :func:`uuid.uuid6`, preserving the pure Python +implementation as a fallback. Generate default clock sequences from operating +system entropy instead of the :mod:`random` module. diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index e957071a4ecef9..27971ac3ccbbb0 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -2,7 +2,7 @@ * Python UUID module: * - wraps libuuid or Windows rpcrt4.dll. * - implements fast version of the uuid.py:UUID class. - * - re-implements uuid4() and uuid7() functions with improved performance + * - re-implements uuid4(), uuid6(), and uuid7() functions with improved performance * by virtue of them being implemented in C and better entropy fetching * strategy. * @@ -167,6 +167,7 @@ typedef struct uuidobject { #define RANDOM_BUF_SIZE 256 #define MAX_FREE_LIST_SIZE 32 +#define NODE_NOT_INITIALIZED UINT64_MAX /* State of the _uuid module */ typedef struct { @@ -179,6 +180,7 @@ typedef struct { PyObject *random_func; PyObject *random_size_int; PyObject *time_func; + PyObject *node_func; // A freelist for uuid objects -- 15-20% performance boost. uuidobject *freelist; @@ -190,6 +192,12 @@ typedef struct { uint64_t random_idx; uint64_t random_last_pid; + // UUID v6 state + PyMutex node_mutex; + uint64_t node_id; + uint64_t last_timestamp_v6; + uint8_t last_timestamp_v6_init; + // UUID v7 state uint64_t last_timestamp_v7; uint64_t last_counter_v7; @@ -209,6 +217,7 @@ static int from_bytes_le(uuidobject *self, PyObject *bytes_le); static int from_int(uuidobject *self, PyObject *int_value, int validate); static int from_fields(uuidobject *self, PyObject *fields); static PyObject *get_SafeUUID(uuid_state *state); +static PyObject *get_uuidmod_attr(const char *name); static uint64_t uuid_getpid(void) @@ -381,6 +390,155 @@ _uuid_uuid4_impl(PyObject *module) return uuid_from_bytes_array(state->uuid_type, bytes); } +static int +uuid6_mask(PyObject *value, uint64_t mask, uint64_t *result) +{ + if (PyLong_CheckExact(value)) { + *result = PyLong_AsUnsignedLongLongMask(value) & mask; + return 0; + } + + // Preserve overridden bitwise operations and reject objects that only + // implement __index__, which the Python implementation does not accept. + PyObject *mask_obj = PyLong_FromUnsignedLongLong(mask); + if (mask_obj == NULL) { + return -1; + } + PyObject *masked = PyNumber_And(value, mask_obj); + Py_DECREF(mask_obj); + if (masked == NULL) { + return -1; + } + int ret = PyLong_AsUInt64(masked, result); + Py_DECREF(masked); + return ret; +} + +static int +uuid6_get_node(uuid_state *state, uint64_t *result) +{ + // Cached reads need no lock. The sentinel is outside the 48-bit node range. + uint64_t node_id = _Py_atomic_load_uint64_relaxed(&state->node_id); + if (node_id == NODE_NOT_INITIALIZED) { + int ret = 0; + Py_BEGIN_CRITICAL_SECTION_MUTEX(&state->node_mutex); + node_id = _Py_atomic_load_uint64_relaxed(&state->node_id); + if (node_id == NODE_NOT_INITIALIZED) { + PyObject *getnode = state->node_func != NULL + ? Py_NewRef(state->node_func) : get_uuidmod_attr("getnode"); + PyObject *value = NULL; + if (getnode != NULL) { + value = PyObject_CallNoArgs(getnode); + Py_DECREF(getnode); + } + if (value == NULL) { + ret = -1; + } + else { + uint64_t resolved; + ret = uuid6_mask(value, 0xffffffffffff, &resolved); + Py_DECREF(value); + if (ret == 0) { + // getnode() may suspend this critical section or reenter + // uuid6(). Keep a value cached by an intervening call. + node_id = _Py_atomic_load_uint64_relaxed(&state->node_id); + if (node_id == NODE_NOT_INITIALIZED) { + node_id = resolved; + _Py_atomic_store_uint64_relaxed(&state->node_id, node_id); + } + } + } + } + Py_END_CRITICAL_SECTION(); + if (ret < 0) { + return -1; + } + } + *result = node_id; + return 0; +} + +/*[clinic input] +_uuid.uuid6 + + node: object = None + clock_seq: object = None + +Generate a UUID with reordered UUIDv1 fields for improved DB locality. + +More precisely, given a 60-bit timestamp value as specified for UUIDv1, +for UUIDv6 the first 48 most significant bits are stored first, followed +by the 4-bit version (same position), followed by the remaining 12 bits +of the original 60-bit timestamp. +[clinic start generated code]*/ + +static PyObject * +_uuid_uuid6_impl(PyObject *module, PyObject *node, PyObject *clock_seq) +/*[clinic end generated code: output=ee0446987b1efc6c input=64fd6ac090b44c37]*/ +{ + uuid_state *state = get_uuid_state(module); + PyTime_t nanoseconds; + uint8_t random[2]; + int ret; + Py_BEGIN_CRITICAL_SECTION(module); + ret = gen_time(state, &nanoseconds); + if (ret == 0 && clock_seq == Py_None) { + ret = gen_random_lock_held(module, state, random, sizeof(random)); + } + Py_END_CRITICAL_SECTION(); + if (ret < 0) { + return NULL; + } + + uint64_t clock_s; + if (clock_seq == Py_None) { + clock_s = ((uint64_t)(random[0] & 0x3f) << 8) | random[1]; + } + else if (uuid6_mask(clock_seq, 0x3fff, &clock_s) < 0) { + return NULL; + } + + uint64_t node_id; + if (node == Py_None) { + if (uuid6_get_node(state, &node_id) < 0) { + return NULL; + } + } + else if (uuid6_mask(node, 0xffffffffffff, &node_id) < 0) { + return NULL; + } + + // Match Python's floor division, including times before the Unix epoch. + int64_t ticks = nanoseconds / 100; + if (nanoseconds < 0 && nanoseconds % 100 != 0) { + ticks--; + } + // Number of 100-ns intervals from 1582-10-15 to 1970-01-01. + uint64_t timestamp = (uint64_t)(ticks + 0x01b21dd213814000); + // Calls above may release the module lock or reenter uuid6(). Read and + // advance the shared timestamp only after all Python callbacks have run. + Py_BEGIN_CRITICAL_SECTION(module); + if (state->last_timestamp_v6_init && timestamp <= state->last_timestamp_v6) { + timestamp = state->last_timestamp_v6 + 1; + } + state->last_timestamp_v6 = timestamp; + state->last_timestamp_v6_init = 1; + Py_END_CRITICAL_SECTION(); + + uint8_t bytes[UUID_SIZE]; + // time_high(32) | time_mid(16) | version(4) | time_low(12) | + // variant(2) | clock_seq(14) | node(48) + for (int i = 0; i < 6; i++) { + bytes[i] = (uint8_t)(timestamp >> (52 - i * 8)); + bytes[10 + i] = (uint8_t)(node_id >> (40 - i * 8)); + } + bytes[6] = (uint8_t)(((timestamp >> 8) & 0x0f) | 0x60); + bytes[7] = (uint8_t)timestamp; + bytes[8] = (uint8_t)((clock_s >> 8) | 0x80); + bytes[9] = (uint8_t)clock_s; + return uuid_from_bytes_array(state->uuid_type, bytes); +} + static inline int uuid7_get_counter_and_tail(PyObject *module, uuid_state *state, uint64_t *counter, uint8_t *tail) @@ -499,21 +657,27 @@ _uuid._install_c_hooks * random_func: object time_func: object + node_func: object = None [clinic start generated code]*/ static PyObject * _uuid__install_c_hooks_impl(PyObject *module, PyObject *random_func, - PyObject *time_func) -/*[clinic end generated code: output=884aa6e91b2ea832 input=6c5017297067e2ea]*/ + PyObject *time_func, PyObject *node_func) +/*[clinic end generated code: output=e742978b99d5a2ea input=beb6f73390b31d93]*/ { uuid_state *state = get_uuid_state(module); // Reset bufferred entropy -- tests need to always start fresh - // when the random function is changed. Reset last_timestamp_v7 -- - // important for repeatable tests + // when the random function is changed. Reset timestamps for repeatable tests. state->random_idx = RANDOM_BUF_SIZE; + state->last_timestamp_v6_init = 0; state->last_timestamp_v7_init = 0; + Py_BEGIN_CRITICAL_SECTION_MUTEX(&state->node_mutex); + Py_XSETREF(state->node_func, + node_func == Py_None ? NULL : Py_NewRef(node_func)); + _Py_atomic_store_uint64_relaxed(&state->node_id, NODE_NOT_INITIALIZED); + Py_END_CRITICAL_SECTION(); if (random_func == Py_None) { Py_CLEAR(state->random_func); @@ -1764,6 +1928,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->uint128_max); Py_VISIT(state->random_func); Py_VISIT(state->time_func); + Py_VISIT(state->node_func); Py_VISIT(state->random_size_int); return 0; } @@ -1778,6 +1943,7 @@ module_clear(PyObject *mod) Py_CLEAR(state->uint128_max); Py_CLEAR(state->random_func); Py_CLEAR(state->time_func); + Py_CLEAR(state->node_func); Py_CLEAR(state->random_size_int); if (state->freelist != NULL) { @@ -1850,12 +2016,17 @@ uuid_exec(PyObject *module) return -1; } + state->node_mutex = (PyMutex){0}; + _Py_atomic_store_uint64_relaxed(&state->node_id, NODE_NOT_INITIALIZED); + state->last_timestamp_v6 = 0; + state->last_timestamp_v6_init = 0; state->last_timestamp_v7 = 0; state->last_timestamp_v7_init = 0; state->last_counter_v7 = 0; state->random_last_pid = uuid_getpid(); state->time_func = NULL; + state->node_func = NULL; state->random_func = NULL; state->random_size_int = PyLong_FromSize_t((Py_ssize_t)RANDOM_BUF_SIZE); if (state->random_size_int == NULL) { @@ -1871,6 +2042,7 @@ uuid_exec(PyObject *module) static PyMethodDef uuid_methods[] = { _UUID_UUID4_METHODDEF + _UUID_UUID6_METHODDEF _UUID_UUID7_METHODDEF _UUID__INSTALL_C_HOOKS_METHODDEF #if defined(HAVE_UUID_UUID_H) || defined(HAVE_UUID_H) diff --git a/Modules/clinic/_uuidmodule.c.h b/Modules/clinic/_uuidmodule.c.h index cd7643846ca91a..8ade392a7c14b0 100644 --- a/Modules/clinic/_uuidmodule.c.h +++ b/Modules/clinic/_uuidmodule.c.h @@ -33,6 +33,81 @@ _uuid_uuid4(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(_uuid_uuid6__doc__, +"uuid6($module, /, node=None, clock_seq=None)\n" +"--\n" +"\n" +"Generate a UUID with reordered UUIDv1 fields for improved DB locality.\n" +"\n" +"More precisely, given a 60-bit timestamp value as specified for UUIDv1,\n" +"for UUIDv6 the first 48 most significant bits are stored first, followed\n" +"by the 4-bit version (same position), followed by the remaining 12 bits\n" +"of the original 60-bit timestamp."); + +#define _UUID_UUID6_METHODDEF \ + {"uuid6", _PyCFunction_CAST(_uuid_uuid6), METH_FASTCALL|METH_KEYWORDS, _uuid_uuid6__doc__}, + +static PyObject * +_uuid_uuid6_impl(PyObject *module, PyObject *node, PyObject *clock_seq); + +static PyObject * +_uuid_uuid6(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(node), &_Py_ID(clock_seq), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"node", "clock_seq", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "uuid6", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *node = Py_None; + PyObject *clock_seq = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + node = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + clock_seq = args[1]; +skip_optional_pos: + return_value = _uuid_uuid6_impl(module, node, clock_seq); + +exit: + return return_value; +} + PyDoc_STRVAR(_uuid_uuid7__doc__, "uuid7($module, /)\n" "--\n" @@ -60,7 +135,7 @@ _uuid_uuid7(PyObject *module, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(_uuid__install_c_hooks__doc__, -"_install_c_hooks($module, /, *, random_func, time_func)\n" +"_install_c_hooks($module, /, *, random_func, time_func, node_func=None)\n" "--\n" "\n"); @@ -69,7 +144,7 @@ PyDoc_STRVAR(_uuid__install_c_hooks__doc__, static PyObject * _uuid__install_c_hooks_impl(PyObject *module, PyObject *random_func, - PyObject *time_func); + PyObject *time_func, PyObject *node_func); static PyObject * _uuid__install_c_hooks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -77,7 +152,7 @@ _uuid__install_c_hooks(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -86,7 +161,7 @@ _uuid__install_c_hooks(PyObject *module, PyObject *const *args, Py_ssize_t nargs } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(random_func), &_Py_ID(time_func), }, + .ob_item = { &_Py_ID(random_func), &_Py_ID(time_func), &_Py_ID(node_func), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -95,16 +170,18 @@ _uuid__install_c_hooks(PyObject *module, PyObject *const *args, Py_ssize_t nargs # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"random_func", "time_func", NULL}; + static const char * const _keywords[] = {"random_func", "time_func", "node_func", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "_install_c_hooks", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; PyObject *random_func; PyObject *time_func; + PyObject *node_func = Py_None; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 2, /*varpos*/ 0, argsbuf); @@ -113,8 +190,13 @@ _uuid__install_c_hooks(PyObject *module, PyObject *const *args, Py_ssize_t nargs } random_func = args[0]; time_func = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + node_func = args[2]; +skip_optional_kwonly: Py_BEGIN_CRITICAL_SECTION(module); - return_value = _uuid__install_c_hooks_impl(module, random_func, time_func); + return_value = _uuid__install_c_hooks_impl(module, random_func, time_func, node_func); Py_END_CRITICAL_SECTION(); exit: @@ -296,4 +378,4 @@ _uuid_UUID___setstate__(PyObject *self, PyObject *state) return return_value; } -/*[clinic end generated code: output=929e54bf30e649aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e836f3b61a0fec26 input=a9049054013a1b77]*/