[3.14] gh-146452: Fix pickle segfault on concurrent mutation of dict in pickle (GH-146470) (#149940)

gh-146452: Fix pickle segfault on concurrent mutation of dict in pickle (GH-146470)
(cherry picked from commit e62a611)

Co-authored-by: Farhan Saif <fsaif@uic.edu>
Co-authored-by: Kumar Aditya <kumaraditya@python.org>

Author: 3 people

Lib/test/test_free_threading/test_pickle.py

@@ -0,0 +1,44 @@
+import pickle
+import threading
+import unittest
+
+from test.support import threading_helper
+
+
+@threading_helper.requires_working_threading()
+class TestPickleFreeThreading(unittest.TestCase):
+
+    def test_pickle_dumps_with_concurrent_dict_mutation(self):
+        # gh-146452: Pickling a dict while another thread mutates it
+        # used to segfault. batch_dict_exact() iterated dict items via
+        # PyDict_Next() which returns borrowed references, and a
+        # concurrent pop/replace could free the value before Py_INCREF
+        # got to it.
+        shared = {str(i): list(range(20)) for i in range(50)}
+
+        def dumper():
+            for _ in range(1000):
+                try:
+                    pickle.dumps(shared)
+                except RuntimeError:
+                    # "dictionary changed size during iteration" is expected
+                    pass
+
+        def mutator():
+            for j in range(1000):
+                key = str(j % 50)
+                shared[key] = list(range(j % 20))
+                if j % 10 == 0:
+                    shared.pop(key, None)
+                    shared[key] = [j]
+
+        threads = []
+        for _ in range(10):
+            threads.append(threading.Thread(target=dumper))
+        threads.append(threading.Thread(target=mutator))
+
+        with threading_helper.start_threads(threads):
+            pass
+
+if __name__ == "__main__":
+    unittest.main()

Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst

@@ -0,0 +1,2 @@
+Fix segfault in :mod:`pickle` when pickling a dictionary concurrently
+mutated by another thread in the free-threaded build.

Modules/_pickle.c

@@ -3353,7 +3353,7 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *or
  * Note that this currently doesn't work for protocol 0.
  */
 static int
-batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj)
+batch_dict_exact_impl(PickleState *state, PicklerObject *self, PyObject *obj)
 {
     PyObject *key = NULL, *value = NULL;
     int i;
@@ -3424,6 +3424,18 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj)
     return -1;
 }
 
+/* gh-146452: Wrap the dict iteration in a critical section to prevent
+   concurrent mutation from invalidating PyDict_Next() iteration state. */
+static int
+batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj)
+{
+    int ret;
+    Py_BEGIN_CRITICAL_SECTION(obj);
+    ret = batch_dict_exact_impl(state, self, obj);
+    Py_END_CRITICAL_SECTION();
+    return ret;
+}
+
 static int
 save_dict(PickleState *state, PicklerObject *self, PyObject *obj)
 {