Skip to content

Commit bed01e8

Browse files
committed
gh-153199: Optimize dict.fromkeys() fast-clone path for combined dicts
When the source iterable is a combined, compact dict (no deleted entries), skip the per-key insertdict() loop entirely. Instead, clone the source PyDictKeysObject in one memcpy and replace every active entry's value with a new reference to the fill value. This avoids O(n) hash lookups and collision probing on the target dict. Benchmark (1000-key string dict, 10k iterations): Before: ~365-423 ms After: ~90 ms (+~79% faster) All test_dict and test_dictviews tests pass.
1 parent 564c58c commit bed01e8

1 file changed

Lines changed: 95 additions & 1 deletion

File tree

Objects/dictobject.c

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1054,6 +1054,67 @@ clone_combined_dict_keys(PyDictObject *orig)
10541054
return keys;
10551055
}
10561056

1057+
/* Like clone_combined_dict_keys but replaces every active entry's value
1058+
with a new reference to `fill_value`. Used by dict.fromkeys() to skip
1059+
per-key insertdict() calls when the source dict is clean and combined. */
1060+
static PyDictKeysObject *
1061+
clone_combined_dict_keys_with_value(PyDictObject *orig, PyObject *fill_value)
1062+
{
1063+
assert(PyAnyDict_Check(orig));
1064+
assert(Py_TYPE(orig)->tp_iter == dict_iter);
1065+
assert(orig->ma_values == NULL);
1066+
assert(orig->ma_keys != Py_EMPTY_KEYS);
1067+
assert(orig->ma_keys->dk_refcnt == 1);
1068+
1069+
if (!PyFrozenDict_Check(orig)) {
1070+
ASSERT_DICT_LOCKED(orig);
1071+
}
1072+
1073+
size_t keys_size = _PyDict_KeysSize(orig->ma_keys);
1074+
PyDictKeysObject *keys = PyMem_Malloc(keys_size);
1075+
if (keys == NULL) {
1076+
PyErr_NoMemory();
1077+
return NULL;
1078+
}
1079+
1080+
memcpy(keys, orig->ma_keys, keys_size);
1081+
1082+
/* Walk every entry: incref the key and replace the value with
1083+
fill_value (incref'd per active slot). */
1084+
PyObject **pkey, **pvalue;
1085+
size_t offs;
1086+
if (DK_IS_UNICODE(orig->ma_keys)) {
1087+
PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(keys);
1088+
pkey = &ep0->me_key;
1089+
pvalue = &ep0->me_value;
1090+
offs = sizeof(PyDictUnicodeEntry) / sizeof(PyObject*);
1091+
}
1092+
else {
1093+
PyDictKeyEntry *ep0 = DK_ENTRIES(keys);
1094+
pkey = &ep0->me_key;
1095+
pvalue = &ep0->me_value;
1096+
offs = sizeof(PyDictKeyEntry) / sizeof(PyObject*);
1097+
}
1098+
1099+
Py_ssize_t n = keys->dk_nentries;
1100+
for (Py_ssize_t i = 0; i < n; i++) {
1101+
if (*pvalue != NULL) {
1102+
/* Active entry: incref key and replace value with fill_value. */
1103+
Py_INCREF(*pkey);
1104+
Py_INCREF(fill_value);
1105+
*pvalue = fill_value;
1106+
}
1107+
pvalue += offs;
1108+
pkey += offs;
1109+
}
1110+
1111+
#ifdef Py_REF_DEBUG
1112+
_Py_IncRefTotal(_PyThreadState_GET());
1113+
#endif
1114+
return keys;
1115+
}
1116+
1117+
10571118
PyObject *
10581119
PyDict_New(void)
10591120
{
@@ -3365,11 +3426,44 @@ dict_dict_fromkeys(PyDictObject *mp, PyObject *iterable, PyObject *value)
33653426
{
33663427
assert(can_modify_dict(mp));
33673428

3429+
PyDictObject *other = (PyDictObject *)iterable;
3430+
PyDictKeysObject *okeys = other->ma_keys;
3431+
3432+
/* Fast path: if the source dict is combined, compact (no deleted entries),
3433+
* and target is a freshly-allocated empty combined dict, skip the per-key
3434+
* insertdict() loop entirely. Instead, clone the source keys structure
3435+
* (which copies the hash table and entry array in one memcpy) and fill
3436+
* every active slot's value with a new reference to `value`.
3437+
* This avoids repeated hash lookups + collision probing on the target.
3438+
*/
3439+
if (mp->ma_used == 0 &&
3440+
mp->ma_values == NULL &&
3441+
other->ma_values == NULL &&
3442+
other->ma_used > 0 &&
3443+
other->ma_used == okeys->dk_nentries &&
3444+
Py_TYPE(other)->tp_iter == dict_iter &&
3445+
(DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
3446+
USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used))
3447+
{
3448+
PyDictKeysObject *keys = clone_combined_dict_keys_with_value(other, value);
3449+
if (keys == NULL) {
3450+
Py_DECREF(mp);
3451+
return NULL;
3452+
}
3453+
ensure_shared_on_resize(mp);
3454+
dictkeys_decref(mp->ma_keys, IS_DICT_SHARED(mp));
3455+
set_keys(mp, keys);
3456+
STORE_USED(mp, other->ma_used);
3457+
ASSERT_CONSISTENT(mp);
3458+
return mp;
3459+
}
3460+
3461+
/* Slow path: source is sparse, split, or has deleted entries. */
33683462
PyObject *oldvalue;
33693463
Py_ssize_t pos = 0;
33703464
PyObject *key;
33713465
Py_hash_t hash;
3372-
int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys);
3466+
int unicode = DK_IS_UNICODE(okeys);
33733467
uint8_t new_size = Py_MAX(
33743468
estimate_log2_keysize(PyDict_GET_SIZE(iterable)),
33753469
DK_LOG_SIZE(mp->ma_keys));

0 commit comments

Comments
 (0)