From 8f55561513fd39b3e129abadfe2c095b921fd155 Mon Sep 17 00:00:00 2001 From: Farhan Saif Date: Thu, 26 Mar 2026 10:01:20 -0500 Subject: [PATCH 1/4] gh-146452: Fix pickle segfault when pickling dict with concurrent mutation --- Lib/test/test_free_threading/test_pickle.py | 49 +++++++++++++++++++ ...3-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst | 4 ++ Modules/_pickle.c | 14 +++++- 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_free_threading/test_pickle.py create mode 100644 Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst diff --git a/Lib/test/test_free_threading/test_pickle.py b/Lib/test/test_free_threading/test_pickle.py new file mode 100644 index 00000000000000..9d7024983377e9 --- /dev/null +++ b/Lib/test/test_free_threading/test_pickle.py @@ -0,0 +1,49 @@ +import pickle +import threading +import unittest + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestPickleFreeThreading(unittest.TestCase): + + def test_pickle_dumps_with_concurrent_dict_mutation(self): + # gh-146452: Pickling a dict while another thread mutates it + # used to segfault. batch_dict_exact() iterated dict items via + # PyDict_Next() which returns borrowed references, and a + # concurrent pop/replace could free the value before Py_INCREF + # got to it. + shared = {str(i): list(range(20)) for i in range(50)} + + def dumper(): + for _ in range(1000): + try: + pickle.dumps(shared) + except RuntimeError: + # "dictionary changed size during iteration" is expected + pass + + def mutator(): + for j in range(1000): + key = str(j % 50) + shared[key] = list(range(j % 20)) + if j % 10 == 0: + shared.pop(key, None) + shared[key] = [j] + + threads = [] + for _ in range(10): + threads.append(threading.Thread(target=dumper)) + threads.append(threading.Thread(target=mutator)) + + for t in threads: + t.start() + for t in threads: + t.join() + + # If we get here without a segfault, the test passed. + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst b/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst new file mode 100644 index 00000000000000..1bd98f4f42cca9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst @@ -0,0 +1,4 @@ +Fix segfault in :mod:`pickle` when pickling a dictionary concurrently +mutated by another thread in the free-threaded build. The dict iteration in +``batch_dict_exact`` now holds a critical section to prevent borrowed +references from being invalidated mid-iteration. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index a55e04290b8fdd..7a23513570b17e 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -3451,7 +3451,7 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *or * Note that this currently doesn't work for protocol 0. */ static int -batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) +batch_dict_exact_impl(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *key = NULL, *value = NULL; int i; @@ -3524,6 +3524,18 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) return -1; } +/* gh-146452: Wrap the dict iteration in a critical section to prevent + concurrent mutation from invalidating PyDict_Next() iteration state. */ +static int +batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(obj); + ret = batch_dict_exact_impl(state, self, obj); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int save_dict(PickleState *state, PicklerObject *self, PyObject *obj) { From 3e57547eda6ee4e6a3a5711d9060a68d3658ce40 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sun, 17 May 2026 13:01:56 +0530 Subject: [PATCH 2/4] Apply suggestion from @kumaraditya303 --- .../Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst b/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst index 1bd98f4f42cca9..f36f414436abb4 100644 --- a/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst +++ b/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst @@ -1,4 +1,2 @@ Fix segfault in :mod:`pickle` when pickling a dictionary concurrently -mutated by another thread in the free-threaded build. The dict iteration in -``batch_dict_exact`` now holds a critical section to prevent borrowed -references from being invalidated mid-iteration. +mutated by another thread in the free-threaded build. From 6276525b90ca12ce1314469b95522210348db31b Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sun, 17 May 2026 13:03:26 +0530 Subject: [PATCH 3/4] change test to use threading_helper.start_threads --- Lib/test/test_free_threading/test_pickle.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_free_threading/test_pickle.py b/Lib/test/test_free_threading/test_pickle.py index 9d7024983377e9..85a644dc72ecb4 100644 --- a/Lib/test/test_free_threading/test_pickle.py +++ b/Lib/test/test_free_threading/test_pickle.py @@ -37,13 +37,8 @@ def mutator(): threads.append(threading.Thread(target=dumper)) threads.append(threading.Thread(target=mutator)) - for t in threads: - t.start() - for t in threads: - t.join() - - # If we get here without a segfault, the test passed. - + with threading_helper.start_threads(threads): + pass if __name__ == "__main__": unittest.main() From 62b5811203dae17a8e9197811184d110c1760689 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sun, 17 May 2026 13:15:57 +0530 Subject: [PATCH 4/4] fix news --- .../Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst b/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst index f36f414436abb4..99f3cce33497a1 100644 --- a/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst +++ b/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst @@ -1,2 +1,2 @@ Fix segfault in :mod:`pickle` when pickling a dictionary concurrently -mutated by another thread in the free-threaded build. +mutated by another thread in the free-threaded build.