From 22df94bbdb19b0e76dae67438c8477058abf2d81 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jul 2026 13:52:33 +0300 Subject: [PATCH 1/9] gh-153410: Add `PyDict_AsFrozenDictAndClear` C-API --- Doc/c-api/dict.rst | 14 +++++ Doc/whatsnew/3.16.rst | 4 +- Include/cpython/dictobject.h | 4 ++ Lib/test/test_capi/test_dict.py | 20 ++++++ ...-07-09-13-52-17.gh-issue-153410.neaDgY.rst | 3 + Modules/_testcapi/dict.c | 8 +++ Objects/dictobject.c | 62 +++++++++++++++++-- Python/marshal.c | 2 +- 8 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-07-09-13-52-17.gh-issue-153410.neaDgY.rst diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 87d09ad2412e405..de096017397bcd5 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -507,6 +507,20 @@ Dictionary objects modified by another thread. +.. c:function:: PyObject* PyDict_AsFrozenDictAndClear(PyObject *dict) + + Return a new :class:`frozendict` created + from the existing :c:type:`PyDictObject` instance. + Expects *dict* to be an exact :class:`dict` instance. + + Transfers all keys and values from *dict* + to the newly allocated :c:type:`PyFrozenDict` with O(1) complexity. + Clears the input *dict* on success. + Returns ``NULL`` with the exception set on error. + + .. versionadded:: next + + .. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback) Register *callback* as a dictionary watcher. Return a non-negative integer diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index e871ad822bbcedc..11a7ca2666ca1bb 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -668,7 +668,9 @@ C API changes New features ------------ -* TODO +* Add a new way to efficitly create :class:`frozendict` instance + from existing :class:`dict` instances: :c:func:`PyDict_AsFrozenDictAndClear`. + Porting to Python 3.16 ---------------------- diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 5e7811416aba63f..e6cdd04adb06a96 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -106,3 +106,7 @@ PyAPI_FUNC(int) PyDict_Unwatch(int watcher_id, PyObject* dict); // Create a frozendict. Create an empty dictionary if iterable is NULL. PyAPI_FUNC(PyObject*) PyFrozenDict_New(PyObject *iterable); + +// Create a frozendict from existing `PyDictObject`. +// Transfers existing keys and values. +PyAPI_FUNC(PyObject*) PyDict_AsFrozenDictAndClear(PyObject *dict); diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 4cf404e9f563272..7fbaf5c65d7f6c5 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -609,6 +609,26 @@ def test_dict_popstring(self): # CRASHES dict_popstring({}, NULL) # CRASHES dict_popstring({"a": 1}, NULL) + def test_dict_as_frozendict_and_clear(self): + # Test PyDict_AsFrozenDictAndClear() + check = _testcapi.dict_as_frozendict_and_clear + d = {1: 2, 'a': 'b'} + f = check(d) + self.assertIs(type(f), frozendict) + self.assertEqual(f, {1: 2, 'a': 'b'}) + self.assertIs(type(d), dict) + self.assertEqual(d, {}) + + d = {} + f = check(d) + self.assertIs(type(f), frozendict) + self.assertEqual(f, {}) + self.assertIs(type(d), dict) + self.assertEqual(d, {}) + # CRASHES check([]) + # CRASHES check(frozendict()) + # CRASHES check(NULL) + def test_frozendict_check(self): # Test PyFrozenDict_Check() check = _testcapi.frozendict_check diff --git a/Misc/NEWS.d/next/C_API/2026-07-09-13-52-17.gh-issue-153410.neaDgY.rst b/Misc/NEWS.d/next/C_API/2026-07-09-13-52-17.gh-issue-153410.neaDgY.rst new file mode 100644 index 000000000000000..259981100b5bf15 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-07-09-13-52-17.gh-issue-153410.neaDgY.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyDict_AsFrozenDictAndClear` C-API function to create +:class:`frozendict` from existing :class:`dict` object and clear the input +dict instance. diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 172591b03182abf..5e169d99c179d57 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -257,6 +257,13 @@ test_dict_iteration(PyObject* self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } +static PyObject * +dict_as_frozendict_and_clear(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_AsFrozenDictAndClear(obj); +} + static PyObject * frozendict_check(PyObject *self, PyObject *obj) @@ -306,6 +313,7 @@ static PyMethodDef test_methods[] = { {"dict_popstring", dict_popstring, METH_VARARGS}, {"dict_popstring_null", dict_popstring_null, METH_VARARGS}, {"test_dict_iteration", test_dict_iteration, METH_NOARGS}, + {"dict_as_frozendict_and_clear", dict_as_frozendict_and_clear, METH_O}, {"frozendict_check", frozendict_check, METH_O}, {"frozendict_checkexact", frozendict_checkexact, METH_O}, {"anydict_check", anydict_check, METH_O}, diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 246ffe6c18e9b51..7adc2416fbe9b26 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3120,6 +3120,18 @@ clear_embedded_values(PyDictValues *values, Py_ssize_t nentries) } } +// This function is used in both `PyDict_Clear` +// and `PyDict_AsFrozenDictAndClear`, place all the common parts here. +static void +clear_common(PyDictObject *mp) +{ + /* Empty the dict... */ + _PyDict_NotifyEvent(PyDict_EVENT_CLEARED, mp, NULL, NULL); + // We don't inc ref empty keys because they're immortal + ensure_shared_on_resize(mp); + STORE_USED(mp, 0); +} + static void clear_lock_held(PyObject *op) { @@ -3138,11 +3150,7 @@ clear_lock_held(PyObject *op) if (oldkeys == Py_EMPTY_KEYS) { return; } - /* Empty the dict... */ - _PyDict_NotifyEvent(PyDict_EVENT_CLEARED, mp, NULL, NULL); - // We don't inc ref empty keys because they're immortal - ensure_shared_on_resize(mp); - STORE_USED(mp, 0); + clear_common(mp); if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); assert(oldkeys->dk_refcnt == 1); @@ -8521,6 +8529,50 @@ frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return d; } +static void +transfer_keys_and_values_lock_held(PyObject *res, PyObject *dict) +{ + + PyDictObject *new = (PyDictObject *)res; + PyDictObject *old = (PyDictObject *)dict; + // Fast path: do nothing on an empty dict: + if (old->ma_keys == Py_EMPTY_KEYS) { + return; + } + + // Transfer keys and values from dict to res: + STORE_USED(new, old->ma_used); + set_keys(new, old->ma_keys); + set_values(new, old->ma_values); + ASSERT_CONSISTENT(new); + + // Now, clear the old dict keys and values, but do not decref them: + clear_common((PyDictObject *)dict); + set_keys(old, Py_EMPTY_KEYS); + set_values(old, NULL); + ASSERT_CONSISTENT(dict); +} + +PyObject * +PyDict_AsFrozenDictAndClear(PyObject *dict) +{ + assert(dict != NULL); + assert(PyDict_CheckExact(dict)); + assert(can_modify_dict((PyDictObject *)dict)); + PyObject *res = frozendict_new_untracked(&PyFrozenDict_Type); + if (res == NULL) { + return NULL; + } + + Py_BEGIN_CRITICAL_SECTION(dict); + transfer_keys_and_values_lock_held(res, dict); + Py_END_CRITICAL_SECTION(); + + _PyObject_GC_TRACK(res); + assert(_PyFrozenDictObject_CAST(res)->ma_hash == -1); + return res; +} + PyObject* PyFrozenDict_New(PyObject *iterable) diff --git a/Python/marshal.c b/Python/marshal.c index 9688d426419c2fa..bb69858e9ac8475 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1501,7 +1501,7 @@ r_object(RFILE *p) Py_CLEAR(v); } if (type == TYPE_FROZENDICT && v != NULL) { - Py_SETREF(v, PyFrozenDict_New(v)); + Py_SETREF(v, PyDict_AsFrozenDictAndClear(v)); } retval = v; break; From b629c3fb2c637496d4cfc756aabbed34f33d546d Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jul 2026 14:05:41 +0300 Subject: [PATCH 2/9] Fix docs --- Doc/c-api/dict.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index de096017397bcd5..8ad6208acbddfbb 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -514,7 +514,7 @@ Dictionary objects Expects *dict* to be an exact :class:`dict` instance. Transfers all keys and values from *dict* - to the newly allocated :c:type:`PyFrozenDict` with O(1) complexity. + to the newly allocated :class:`!frozendict` with O(1) complexity. Clears the input *dict* on success. Returns ``NULL`` with the exception set on error. From 9eafd4af0f9f1ff8da7b42babf0ec86e8355873f Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jul 2026 14:06:45 +0300 Subject: [PATCH 3/9] Fix the comment --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 7adc2416fbe9b26..29164880b5f3237 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3125,7 +3125,6 @@ clear_embedded_values(PyDictValues *values, Py_ssize_t nentries) static void clear_common(PyDictObject *mp) { - /* Empty the dict... */ _PyDict_NotifyEvent(PyDict_EVENT_CLEARED, mp, NULL, NULL); // We don't inc ref empty keys because they're immortal ensure_shared_on_resize(mp); @@ -3150,6 +3149,7 @@ clear_lock_held(PyObject *op) if (oldkeys == Py_EMPTY_KEYS) { return; } + /* Empty the dict... */ clear_common(mp); if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); From acd868af49e9fdb40fe1478ae793309f8aa4d79d Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jul 2026 14:33:51 +0300 Subject: [PATCH 4/9] Update Lib/test/test_capi/test_dict.py Co-authored-by: Victor Stinner --- Lib/test/test_capi/test_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 7fbaf5c65d7f6c5..8a8cbc772ade475 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -615,7 +615,7 @@ def test_dict_as_frozendict_and_clear(self): d = {1: 2, 'a': 'b'} f = check(d) self.assertIs(type(f), frozendict) - self.assertEqual(f, {1: 2, 'a': 'b'}) + self.assertEqual(f, frozendict({1: 2, 'a': 'b'})) self.assertIs(type(d), dict) self.assertEqual(d, {}) From 4bbaea2f644ffd6f6c1472845da2ad3d6453115a Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jul 2026 14:35:59 +0300 Subject: [PATCH 5/9] Address review --- Doc/c-api/dict.rst | 8 ++++++-- Lib/test/test_capi/test_dict.py | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 8ad6208acbddfbb..38bc660576e936e 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -510,14 +510,18 @@ Dictionary objects .. c:function:: PyObject* PyDict_AsFrozenDictAndClear(PyObject *dict) Return a new :class:`frozendict` created - from the existing :c:type:`PyDictObject` instance. + from the existing :class:`dict` instance. Expects *dict* to be an exact :class:`dict` instance. Transfers all keys and values from *dict* - to the newly allocated :class:`!frozendict` with O(1) complexity. + to the newly allocated :class:`!frozendict`. Clears the input *dict* on success. Returns ``NULL`` with the exception set on error. + .. impl-detail:: + + Works with O(1) complexity. + .. versionadded:: next diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 8a8cbc772ade475..d936fa0d3dfb377 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -611,18 +611,18 @@ def test_dict_popstring(self): def test_dict_as_frozendict_and_clear(self): # Test PyDict_AsFrozenDictAndClear() - check = _testcapi.dict_as_frozendict_and_clear + as_frozendict = _testcapi.dict_as_frozendict_and_clear d = {1: 2, 'a': 'b'} - f = check(d) + f = as_frozendict(d) self.assertIs(type(f), frozendict) self.assertEqual(f, frozendict({1: 2, 'a': 'b'})) self.assertIs(type(d), dict) self.assertEqual(d, {}) d = {} - f = check(d) + f = as_frozendict(d) self.assertIs(type(f), frozendict) - self.assertEqual(f, {}) + self.assertEqual(f, frozendict()) self.assertIs(type(d), dict) self.assertEqual(d, {}) # CRASHES check([]) From e80b8b3d42a242072741c3f7e48249169f88ba3f Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jul 2026 15:27:14 +0300 Subject: [PATCH 6/9] Address Victors review --- Lib/test/test_capi/test_dict.py | 10 +++++++--- Objects/dictobject.c | 29 ++++++++++++++++++----------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index d936fa0d3dfb377..a6b48840a0c33df 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -625,9 +625,13 @@ def test_dict_as_frozendict_and_clear(self): self.assertEqual(f, frozendict()) self.assertIs(type(d), dict) self.assertEqual(d, {}) - # CRASHES check([]) - # CRASHES check(frozendict()) - # CRASHES check(NULL) + + class DictSubtype(dict): ... + + for wrong_input in (frozendict(), DictSubtype(), [], None): + with self.subTest(wrong_input=wrong_input): + with self.assertRaises(SystemError): + as_frozendict(wrong_input) def test_frozendict_check(self): # Test PyFrozenDict_Check() diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 29164880b5f3237..d1c81442f90bf3f 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8532,33 +8532,40 @@ frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static void transfer_keys_and_values_lock_held(PyObject *res, PyObject *dict) { - PyDictObject *new = (PyDictObject *)res; PyDictObject *old = (PyDictObject *)dict; + assert(can_modify_dict(old)); + // Fast path: do nothing on an empty dict: if (old->ma_keys == Py_EMPTY_KEYS) { return; } - // Transfer keys and values from dict to res: - STORE_USED(new, old->ma_used); - set_keys(new, old->ma_keys); - set_values(new, old->ma_values); - ASSERT_CONSISTENT(new); + Py_ssize_t used = old->ma_used; + PyDictKeysObject *keys = old->ma_keys; + PyDictValues *values = old->ma_values; - // Now, clear the old dict keys and values, but do not decref them: + // Clear the old dict keys and values, but do not decref them: clear_common((PyDictObject *)dict); set_keys(old, Py_EMPTY_KEYS); set_values(old, NULL); - ASSERT_CONSISTENT(dict); + ASSERT_CONSISTENT(old); + + // Transfer keys and values from dict to frozendict: + new->ma_used = used; + new->ma_keys = keys; + new->ma_values = values; + ASSERT_CONSISTENT(new); } PyObject * PyDict_AsFrozenDictAndClear(PyObject *dict) { - assert(dict != NULL); - assert(PyDict_CheckExact(dict)); - assert(can_modify_dict((PyDictObject *)dict)); + if (dict == NULL || !PyDict_CheckExact(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyObject *res = frozendict_new_untracked(&PyFrozenDict_Type); if (res == NULL) { return NULL; From d0d772b69e6e218dab02afe7d223414d77cc65de Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jul 2026 15:31:36 +0300 Subject: [PATCH 7/9] Fix the cast --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index d1c81442f90bf3f..27c9cc058e04101 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8546,7 +8546,7 @@ transfer_keys_and_values_lock_held(PyObject *res, PyObject *dict) PyDictValues *values = old->ma_values; // Clear the old dict keys and values, but do not decref them: - clear_common((PyDictObject *)dict); + clear_common(old); set_keys(old, Py_EMPTY_KEYS); set_values(old, NULL); ASSERT_CONSISTENT(old); From f9df030b358e007f105b15bd63c43da23e329b84 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jul 2026 15:44:56 +0300 Subject: [PATCH 8/9] Support `dict` subtypes --- Lib/test/test_capi/test_dict.py | 9 ++++++++- Objects/dictobject.c | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index a6b48840a0c33df..a51f0c077995595 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -628,7 +628,14 @@ def test_dict_as_frozendict_and_clear(self): class DictSubtype(dict): ... - for wrong_input in (frozendict(), DictSubtype(), [], None): + d = DictSubtype({'x': None}) + f = as_frozendict(d) + self.assertIs(type(f), frozendict) + self.assertEqual(f, frozendict({'x': None})) + self.assertIs(type(d), DictSubtype) + self.assertEqual(d, DictSubtype({})) + + for wrong_input in (frozendict(), [], None): with self.subTest(wrong_input=wrong_input): with self.assertRaises(SystemError): as_frozendict(wrong_input) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 27c9cc058e04101..767d6c7ffb14c85 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8561,7 +8561,7 @@ transfer_keys_and_values_lock_held(PyObject *res, PyObject *dict) PyObject * PyDict_AsFrozenDictAndClear(PyObject *dict) { - if (dict == NULL || !PyDict_CheckExact(dict)) { + if (dict == NULL || !PyDict_Check(dict)) { PyErr_BadInternalCall(); return NULL; } From 3133dd11b9d4ed6813c1cf6b473f764ea74e3ecb Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Jul 2026 16:29:27 +0300 Subject: [PATCH 9/9] Adjust docs --- Doc/c-api/dict.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 38bc660576e936e..6389cce8ff211f0 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -511,7 +511,6 @@ Dictionary objects Return a new :class:`frozendict` created from the existing :class:`dict` instance. - Expects *dict* to be an exact :class:`dict` instance. Transfers all keys and values from *dict* to the newly allocated :class:`!frozendict`.