From 48b2e5437c5153725f2df4cfd25ec4fa1b46b36c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Aug 2026 15:04:54 +0200 Subject: [PATCH 1/5] gh-155742: Check singletons consistencty in the garbage collector If Python is built in debug mode or with assertions, the garbage collector now checks singletons consistencty to detect data corruption in C extension. Add _Py_CheckSingletons() function. Call this function on a GC collection. Add tests checking that corrupting a singleton is properly detected. --- Include/internal/pycore_object.h | 4 + Lib/test/test_capi/test_misc.py | 53 ++++++++ ...-08-14-16-21-47.gh-issue-155742.Xjqupt.rst | 3 + Modules/_testcapi/bytes.c | 13 ++ Modules/_testcapi/long.c | 25 ++++ Modules/_testcapi/unicode.c | 14 ++ Objects/object.c | 121 ++++++++++++++++++ Python/gc.c | 6 + Python/gc_free_threading.c | 6 + 9 files changed, 245 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-14-16-21-47.gh-issue-155742.Xjqupt.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 41786cb267c2e96..ce06d4aa898b6d6 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -1037,6 +1037,10 @@ static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) * references. */ PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg); +#ifndef NDEBUG +extern void _Py_CheckSingletons(void); +#endif + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 7d668843d07debc..a37ebadc8efe740 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -3094,5 +3094,58 @@ def test_ceval_decref(self): self.assertEqual(lines.count("DESTROY list"), 2) +class TestCheckSingleton(unittest.TestCase): + # Test _Py_CheckSingletons() which is called by gc.collect() + # + # Corrupt some singleton objects and make sure that the data corruption + # is detected. + + def check(self, func): + code = f"""if 1: + import _testcapi + from test import support + support.SuppressCrashReport().__enter__() + _testcapi.{func}() + """ + proc = assert_python_failure("-c", code) + return proc.err + + def test_corrupt_bytes_singleton(self): + stderr = self.check("corrupt_bytes_singleton") + + # In fact, it's the character b'a' which is corrupted + self.assertIn(b"object repr : b'A'", stderr) + self.assertIn((b'check_singleton_bytes: ' + b'Assertion "str[0] == ch" failed'), + stderr) + + def test_corrupt_unicode_singleton(self): + stderr = self.check("corrupt_unicode_singleton") + + # In fact, it's the character b'a' which is corrupted + self.assertIn(b"object repr : 'A'", stderr) + self.assertIn((b'check_singleton_unicode: Assertion ' + b'"PyUnicode_READ_CHAR(((PyObject*)((obj))), (0))' + b' == ch" failed'), + stderr) + + def test_corrupt_bool_singleton(self): + stderr = self.check("corrupt_bool_singleton") + + self.assertIn(b"object repr : True", stderr) + self.assertIn((b'check_singleton_long: ' + b'Assertion "compact == value" failed'), + stderr) + + def test_corrupt_long_singleton(self): + stderr = self.check("corrupt_long_singleton") + + # In fact, it's the number 5 which is corrupted + self.assertIn(b"object repr : 42", stderr) + self.assertIn((b'check_singleton_long: ' + b'Assertion "compact == value" failed'), + stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-14-16-21-47.gh-issue-155742.Xjqupt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-14-16-21-47.gh-issue-155742.Xjqupt.rst new file mode 100644 index 000000000000000..ff8543bd2e9d6d9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-14-16-21-47.gh-issue-155742.Xjqupt.rst @@ -0,0 +1,3 @@ +If Python is built in debug mode or with assertions, the garbage collector +now checks singletons consistencty to detect data corruption in C extension. +Patch by Victor Stinner. diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index f12fc7f5f3a2a86..9af9f3e66a9b98e 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -351,12 +351,25 @@ byteswriter_highlevel(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } +static PyObject * +corrupt_bytes_singleton(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + PyObject *obj = PyBytes_FromStringAndSize("a", 1); + assert(obj != NULL); + PyBytes_AS_STRING(obj)[0] = 'A'; + + PyGC_Collect(); + Py_RETURN_NONE; +} + + static PyMethodDef test_methods[] = { {"bytes_resize", bytes_resize, METH_VARARGS}, {"bytes_join", bytes_join, METH_VARARGS}, {"byteswriter_abc", byteswriter_abc, METH_NOARGS}, {"byteswriter_resize", byteswriter_resize, METH_NOARGS}, {"byteswriter_highlevel", byteswriter_highlevel, METH_NOARGS}, + {"corrupt_bytes_singleton", corrupt_bytes_singleton, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 008a7d37726869b..b2f8fad8e820394 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -281,6 +281,29 @@ get_pylong_layout(PyObject *module, PyObject *Py_UNUSED(args)) } +static PyObject * +corrupt_bool_singleton(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + PyObject *obj = Py_True; + ((PyLongObject*)obj)->long_value.ob_digit[0] = 0; + + PyGC_Collect(); + Py_RETURN_NONE; +} + + +static PyObject * +corrupt_long_singleton(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + PyObject *obj = PyLong_FromLong(5); + assert(obj != NULL); + ((PyLongObject*)obj)->long_value.ob_digit[0] = 42; + + PyGC_Collect(); + Py_RETURN_NONE; +} + + static PyMethodDef test_methods[] = { _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF {"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS}, @@ -295,6 +318,8 @@ static PyMethodDef test_methods[] = { {"pylong_ispositive", pylong_ispositive, METH_O}, {"pylong_isnegative", pylong_isnegative, METH_O}, {"pylong_iszero", pylong_iszero, METH_O}, + {"corrupt_bool_singleton", corrupt_bool_singleton, METH_NOARGS}, + {"corrupt_long_singleton", corrupt_long_singleton, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 915c9230f66b52e..6d3f3b1962fe088 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -563,6 +563,19 @@ static PyType_Spec Writer_spec = { }; +static PyObject * +corrupt_unicode_singleton(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + PyObject *obj = PyUnicode_FromOrdinal('a'); + assert(obj != NULL); + assert(PyUnicode_KIND(obj) == PyUnicode_1BYTE_KIND); + PyUnicode_1BYTE_DATA(obj)[0] = 'A'; + + PyGC_Collect(); + Py_RETURN_NONE; +} + + static PyMethodDef TestMethods[] = { {"unicode_new", unicode_new, METH_VARARGS}, {"unicode_fill", unicode_fill, METH_VARARGS}, @@ -572,6 +585,7 @@ static PyMethodDef TestMethods[] = { {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {"unicode_GET_CACHED_HASH", unicode_GET_CACHED_HASH, METH_O}, + {"corrupt_unicode_singleton", corrupt_unicode_singleton, METH_NOARGS}, {NULL}, }; diff --git a/Objects/object.c b/Objects/object.c index fadd9273a36607c..8acf80ed7245a16 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3526,3 +3526,124 @@ Py_ssize_t Py_REFCNT(PyObject *ob) { return _Py_REFCNT(ob); } Py_ssize_t Py_SIZE(PyObject *o) { return _Py_SIZE_impl(o); } int Py_IS_TYPE(PyObject *o, PyTypeObject *t) { return _Py_IS_TYPE_impl(o, t); } void Py_SET_SIZE(PyVarObject *o, Py_ssize_t s) { _Py_SET_SIZE_impl(o, s); } + + +#ifndef NDEBUG +static void +check_singleton(PyObject *obj, PyTypeObject *type) +{ + // Check PyObject.ob_refcnt + _PyObject_ASSERT(obj, _Py_IsImmortal(obj)); + + // Check PyObject.ob_type + _PyObject_ASSERT(obj, Py_TYPE(obj) == type); +} + + +static void +check_singleton_long(PyObject *obj, long value, int is_bool) +{ + PyTypeObject *type = is_bool ? &PyBool_Type : &PyLong_Type; + check_singleton(obj, type); + + // Check _PyLong_CompactValue() + Py_ssize_t compact = _PyLong_CompactValue((const PyLongObject *)obj); + _PyObject_ASSERT(obj, compact == value); + + // Check tv_tag and ob_digit[0] + _PyLongValue *long_value = &((PyLongObject*)obj)->long_value; + int sign = (value == 0) ? 0 : ((value < 0) ? -1 : 1); + uintptr_t lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, (value == 0) ? 0 : 1); + if (!is_bool) { + lv_tag |= IMMORTALITY_BIT_MASK; + } + _PyObject_ASSERT(obj, long_value->lv_tag == lv_tag); + _PyObject_ASSERT(obj, long_value->ob_digit[0] == Py_ABS(value)); +} + + +static void +check_singleton_bytes(PyObject *obj, Py_ssize_t size, unsigned char ch) +{ + check_singleton(obj, &PyBytes_Type); + _PyObject_ASSERT(obj, PyBytes_GET_SIZE(obj) == size); + const unsigned char *str = (const unsigned char *)PyBytes_AS_STRING(obj); + _PyObject_ASSERT(obj, str[0] == ch); + if (size > 0) { + _PyObject_ASSERT(obj, str[1] == 0); + } +} + + +static void +check_singleton_unicode(PyObject *obj, Py_ssize_t length, Py_UCS4 ch) +{ + check_singleton(obj, &PyUnicode_Type); + _PyObject_ASSERT(obj, _PyUnicode_CheckConsistency(obj, 1)); + + _PyObject_ASSERT(obj, PyUnicode_GET_LENGTH(obj) == length); + + _PyObject_ASSERT(obj, PyUnicode_READ_CHAR(obj, 0) == ch); + if (length > 0) { + _PyObject_ASSERT(obj, PyUnicode_READ_CHAR(obj, 1) == 0); + } +} + + +// Check singletons consistency: try to detect if a C extension modified a +// singleton by mistake. +// +// Since the hash is computed lazily, don't check the hash, except for empty +// tuple. +void +_Py_CheckSingletons(void) +{ + assert(!PyErr_Occurred()); + PyObject *obj; + long ival; + + // None + obj = Py_None; + check_singleton(obj, &_PyNone_Type); + + // Ellipsis (...) + obj = Py_Ellipsis; + check_singleton(obj, &PyEllipsis_Type); + + // False, True + check_singleton_long(Py_False, 0, 1); + check_singleton_long(Py_True, 1, 1); + + // Small integers + for (ival=-_PY_NSMALLNEGINTS; ival < _PY_NSMALLPOSINTS; ival++) { + obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + ival]; + check_singleton_long(obj, ival, 0); + } + + // Empty bytes string (b'') + obj = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + check_singleton_bytes(obj, 0, '\0'); + + for (ival=0; ival <= 255; ival++) { + obj = (PyObject*)&_Py_SINGLETON(bytes_characters)[ival]; + check_singleton_bytes(obj, 1, ival); + } + + // Empty Unicode string ('') + obj = Py_GetConstant(Py_CONSTANT_EMPTY_STR); + check_singleton_unicode(obj, 0, 0); + + for (ival=0; ival <= 255; ival++) { + obj = _Py_LATIN1_CHR(ival); + check_singleton_unicode(obj, 1, ival); + } + + // Empty tuple (()) + obj = Py_GetConstant(Py_CONSTANT_EMPTY_TUPLE); + check_singleton(obj, &PyTuple_Type); + _PyObject_ASSERT(obj, PyTuple_GET_SIZE(obj) == 0); + _PyObject_ASSERT(obj, ((PyTupleObject*)obj)->ob_hash == _PyTuple_HASH_EMPTY); + + assert(!PyErr_Occurred()); +} +#endif diff --git a/Python/gc.c b/Python/gc.c index 201c621bcc3cb9b..6f1588cabfad1b4 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1643,6 +1643,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) invoke_gc_callback(tstate, "stop", generation, &stats); } +#ifndef NDEBUG + // Checking singletons consistency is unrelated to a garbage collection. + // Using a garbage collection to trigger this function is just convenient. + _Py_CheckSingletons(); +#endif + assert(!_PyErr_Occurred(tstate)); gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 99f1a1eb47e3ddc..8c43d5f04e065b0 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2313,6 +2313,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) invoke_gc_callback(tstate, "stop", generation, m, n, state.candidates, duration); } +#ifndef NDEBUG + // Checking singletons consistency is unrelated to a garbage collection. + // Using a garbage collection to trigger this function is just convenient. + _Py_CheckSingletons(); +#endif + assert(!_PyErr_Occurred(tstate)); gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); From 0608d2bbef09dd4be8b0354e246d215ebbadc3c0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Aug 2026 16:59:27 +0200 Subject: [PATCH 2/5] Add support.built_with_c_assertions() Fix also a compiler warning on Windows: Objects\object.c(3629,39): warning C4244: 'function': conversion from 'long' to 'un signed char', possible loss of data --- Lib/test/pythoninfo.py | 10 ++-------- Lib/test/support/__init__.py | 11 +++++++++++ Lib/test/test_capi/test_misc.py | 3 +++ Objects/object.c | 2 +- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index ea7edb798051567..90b32aaaaf9a40a 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -611,14 +611,6 @@ def collect_sysconfig(info_add): value = normalize_text(value) info_add('sysconfig[%s]' % name, value) - PY_CFLAGS = sysconfig.get_config_var('PY_CFLAGS') - NDEBUG = (PY_CFLAGS and '-DNDEBUG' in PY_CFLAGS) - if NDEBUG: - text = 'ignore assertions (macro defined)' - else: - text= 'build assertions (macro not defined)' - info_add('build.NDEBUG',text) - for name in ( 'WITH_DOC_STRINGS', 'WITH_DTRACE', @@ -844,6 +836,8 @@ def collect_support(info_add): support.check_sanitizer(memory=True)) info_add('support.check_sanitizer(ub=True)', support.check_sanitizer(ub=True)) + info_add('support.built_with_c_assertions', + support.built_with_c_assertions()) def collect_support_os_helper(info_add): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 3f2caebd21e336c..b0651f46e4de256 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3492,3 +3492,14 @@ def check_immutable_type(testcase, type): else: flags = type_getflags(type) testcase.assertTrue(flags & Py_TPFLAGS_IMMUTABLETYPE) + + +def built_with_c_assertions(): + # Check if Python was built in debug mode or using --with-assertions + if MS_WINDOWS: + Py_DEBUG = hasattr(sys, 'gettotalrefcount') + return Py_DEBUG + + PY_CFLAGS = sysconfig.get_config_var('PY_CFLAGS') + NDEBUG = (PY_CFLAGS and '-DNDEBUG' in PY_CFLAGS) + return (not NDEBUG) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index a37ebadc8efe740..c3c8edea35b2212 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -3094,6 +3094,9 @@ def test_ceval_decref(self): self.assertEqual(lines.count("DESTROY list"), 2) +# _Py_CheckSingletons() is only built if the NDEBUG macro is not defined +@unittest.skipUnless(support.built_with_c_assertions(), + 'Python built without C assertions') class TestCheckSingleton(unittest.TestCase): # Test _Py_CheckSingletons() which is called by gc.collect() # diff --git a/Objects/object.c b/Objects/object.c index 8acf80ed7245a16..0b81fc7426957fe 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3626,7 +3626,7 @@ _Py_CheckSingletons(void) for (ival=0; ival <= 255; ival++) { obj = (PyObject*)&_Py_SINGLETON(bytes_characters)[ival]; - check_singleton_bytes(obj, 1, ival); + check_singleton_bytes(obj, 1, (unsigned char)ival); } // Empty Unicode string ('') From c8dd5912eafac7f2aabac428709cc81a93524848 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Aug 2026 17:15:22 +0200 Subject: [PATCH 3/5] Py_DEBUG is already declared in test.support --- Lib/test/support/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b0651f46e4de256..c10064ace21d6cf 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3497,7 +3497,6 @@ def check_immutable_type(testcase, type): def built_with_c_assertions(): # Check if Python was built in debug mode or using --with-assertions if MS_WINDOWS: - Py_DEBUG = hasattr(sys, 'gettotalrefcount') return Py_DEBUG PY_CFLAGS = sysconfig.get_config_var('PY_CFLAGS') From 962ef4a1db0cf81d58376ab3783d109ff494f25b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Aug 2026 17:54:28 +0200 Subject: [PATCH 4/5] Test more singletons --- Objects/object.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Objects/object.c b/Objects/object.c index 0b81fc7426957fe..19f5a6e4242eca0 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3601,15 +3601,19 @@ _Py_CheckSingletons(void) assert(!PyErr_Occurred()); PyObject *obj; long ival; + PyInterpreterState *interp = _PyInterpreterState_GET(); // None obj = Py_None; check_singleton(obj, &_PyNone_Type); - // Ellipsis (...) + // Ellipsis (...), Py_NotImplemented obj = Py_Ellipsis; check_singleton(obj, &PyEllipsis_Type); + obj = Py_NotImplemented; + check_singleton(obj, &_PyNotImplemented_Type); + // False, True check_singleton_long(Py_False, 0, 1); check_singleton_long(Py_True, 1, 1); @@ -3624,6 +3628,7 @@ _Py_CheckSingletons(void) obj = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); check_singleton_bytes(obj, 0, '\0'); + // 1-character bytes strings for (ival=0; ival <= 255; ival++) { obj = (PyObject*)&_Py_SINGLETON(bytes_characters)[ival]; check_singleton_bytes(obj, 1, (unsigned char)ival); @@ -3633,6 +3638,9 @@ _Py_CheckSingletons(void) obj = Py_GetConstant(Py_CONSTANT_EMPTY_STR); check_singleton_unicode(obj, 0, 0); + // Do not tests _Py_STR() strings since there is no API to list them + + // 1-character Unicode strings for (ival=0; ival <= 255; ival++) { obj = _Py_LATIN1_CHR(ival); check_singleton_unicode(obj, 1, ival); @@ -3644,6 +3652,15 @@ _Py_CheckSingletons(void) _PyObject_ASSERT(obj, PyTuple_GET_SIZE(obj) == 0); _PyObject_ASSERT(obj, ((PyTupleObject*)obj)->ob_hash == _PyTuple_HASH_EMPTY); + // Do not test less commoly used singletons to reduce the overhead + // of calling _Py_CheckSingletons(): + // + // - PyDateTime_TimeZone_UTC + // - context_token_missing + // - hamt_bitmap_node_empty + // - interp hamt_empty + // - interp last_resort_memory_error + assert(!PyErr_Occurred()); } #endif From a6391761a05548745777fbc5ac0ecb359a5f4ce9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Aug 2026 18:07:59 +0200 Subject: [PATCH 5/5] Remove unused variable (interp) --- Objects/object.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/object.c b/Objects/object.c index 19f5a6e4242eca0..0837b8e81d79201 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3601,7 +3601,6 @@ _Py_CheckSingletons(void) assert(!PyErr_Occurred()); PyObject *obj; long ival; - PyInterpreterState *interp = _PyInterpreterState_GET(); // None obj = Py_None;