Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Include/internal/pycore_object.h
Original file line number Diff line number Diff line change
Expand Up @@ -1037,6 +1037,10 @@ static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op)
* references. */
PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg);

#ifndef NDEBUG
extern void _Py_CheckSingletons(void);
#endif

#ifdef __cplusplus
}
#endif
Expand Down
10 changes: 2 additions & 8 deletions Lib/test/pythoninfo.py
Original file line number Diff line number Diff line change
Expand Up @@ -611,14 +611,6 @@ def collect_sysconfig(info_add):
value = normalize_text(value)
info_add('sysconfig[%s]' % name, value)

PY_CFLAGS = sysconfig.get_config_var('PY_CFLAGS')
NDEBUG = (PY_CFLAGS and '-DNDEBUG' in PY_CFLAGS)
if NDEBUG:
text = 'ignore assertions (macro defined)'
else:
text= 'build assertions (macro not defined)'
info_add('build.NDEBUG',text)

for name in (
'WITH_DOC_STRINGS',
'WITH_DTRACE',
Expand Down Expand Up @@ -844,6 +836,8 @@ def collect_support(info_add):
support.check_sanitizer(memory=True))
info_add('support.check_sanitizer(ub=True)',
support.check_sanitizer(ub=True))
info_add('support.built_with_c_assertions',
support.built_with_c_assertions())


def collect_support_os_helper(info_add):
Expand Down
10 changes: 10 additions & 0 deletions Lib/test/support/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3492,3 +3492,13 @@ def check_immutable_type(testcase, type):
else:
flags = type_getflags(type)
testcase.assertTrue(flags & Py_TPFLAGS_IMMUTABLETYPE)


def built_with_c_assertions():
# Check if Python was built in debug mode or using --with-assertions
if MS_WINDOWS:
return Py_DEBUG

PY_CFLAGS = sysconfig.get_config_var('PY_CFLAGS')
NDEBUG = (PY_CFLAGS and '-DNDEBUG' in PY_CFLAGS)
return (not NDEBUG)
56 changes: 56 additions & 0 deletions Lib/test/test_capi/test_misc.py
Original file line number Diff line number Diff line change
Expand Up @@ -3094,5 +3094,61 @@ def test_ceval_decref(self):
self.assertEqual(lines.count("DESTROY list"), 2)


# _Py_CheckSingletons() is only built if the NDEBUG macro is not defined
@unittest.skipUnless(support.built_with_c_assertions(),
'Python built without C assertions')
class TestCheckSingleton(unittest.TestCase):
# Test _Py_CheckSingletons() which is called by gc.collect()
#
# Corrupt some singleton objects and make sure that the data corruption
# is detected.

def check(self, func):
code = f"""if 1:
import _testcapi
from test import support
support.SuppressCrashReport().__enter__()
_testcapi.{func}()
"""
proc = assert_python_failure("-c", code)
return proc.err

def test_corrupt_bytes_singleton(self):
stderr = self.check("corrupt_bytes_singleton")

# In fact, it's the character b'a' which is corrupted
self.assertIn(b"object repr : b'A'", stderr)
self.assertIn((b'check_singleton_bytes: '
b'Assertion "str[0] == ch" failed'),
stderr)

def test_corrupt_unicode_singleton(self):
stderr = self.check("corrupt_unicode_singleton")

# In fact, it's the character b'a' which is corrupted
self.assertIn(b"object repr : 'A'", stderr)
self.assertIn((b'check_singleton_unicode: Assertion '
b'"PyUnicode_READ_CHAR(((PyObject*)((obj))), (0))'
b' == ch" failed'),
stderr)

def test_corrupt_bool_singleton(self):
stderr = self.check("corrupt_bool_singleton")

self.assertIn(b"object repr : True", stderr)
self.assertIn((b'check_singleton_long: '
b'Assertion "compact == value" failed'),
stderr)

def test_corrupt_long_singleton(self):
stderr = self.check("corrupt_long_singleton")

# In fact, it's the number 5 which is corrupted
self.assertIn(b"object repr : 42", stderr)
self.assertIn((b'check_singleton_long: '
b'Assertion "compact == value" failed'),
stderr)


if __name__ == "__main__":
unittest.main()
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
If Python is built in debug mode or with assertions, the garbage collector
now checks singletons consistencty to detect data corruption in C extension.
Patch by Victor Stinner.
13 changes: 13 additions & 0 deletions Modules/_testcapi/bytes.c
Original file line number Diff line number Diff line change
Expand Up @@ -351,12 +351,25 @@ byteswriter_highlevel(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
}


static PyObject *
corrupt_bytes_singleton(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
{
PyObject *obj = PyBytes_FromStringAndSize("a", 1);
assert(obj != NULL);
PyBytes_AS_STRING(obj)[0] = 'A';

PyGC_Collect();
Py_RETURN_NONE;
}


static PyMethodDef test_methods[] = {
{"bytes_resize", bytes_resize, METH_VARARGS},
{"bytes_join", bytes_join, METH_VARARGS},
{"byteswriter_abc", byteswriter_abc, METH_NOARGS},
{"byteswriter_resize", byteswriter_resize, METH_NOARGS},
{"byteswriter_highlevel", byteswriter_highlevel, METH_NOARGS},
{"corrupt_bytes_singleton", corrupt_bytes_singleton, METH_NOARGS},
{NULL},
};

Expand Down
25 changes: 25 additions & 0 deletions Modules/_testcapi/long.c
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,29 @@ get_pylong_layout(PyObject *module, PyObject *Py_UNUSED(args))
}


static PyObject *
corrupt_bool_singleton(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
{
PyObject *obj = Py_True;
((PyLongObject*)obj)->long_value.ob_digit[0] = 0;

PyGC_Collect();
Py_RETURN_NONE;
}


static PyObject *
corrupt_long_singleton(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
{
PyObject *obj = PyLong_FromLong(5);
assert(obj != NULL);
((PyLongObject*)obj)->long_value.ob_digit[0] = 42;

PyGC_Collect();
Py_RETURN_NONE;
}


static PyMethodDef test_methods[] = {
_TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF
{"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS},
Expand All @@ -295,6 +318,8 @@ static PyMethodDef test_methods[] = {
{"pylong_ispositive", pylong_ispositive, METH_O},
{"pylong_isnegative", pylong_isnegative, METH_O},
{"pylong_iszero", pylong_iszero, METH_O},
{"corrupt_bool_singleton", corrupt_bool_singleton, METH_NOARGS},
{"corrupt_long_singleton", corrupt_long_singleton, METH_NOARGS},
{NULL},
};

Expand Down
14 changes: 14 additions & 0 deletions Modules/_testcapi/unicode.c
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,19 @@ static PyType_Spec Writer_spec = {
};


static PyObject *
corrupt_unicode_singleton(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
{
PyObject *obj = PyUnicode_FromOrdinal('a');
assert(obj != NULL);
assert(PyUnicode_KIND(obj) == PyUnicode_1BYTE_KIND);
PyUnicode_1BYTE_DATA(obj)[0] = 'A';

PyGC_Collect();
Py_RETURN_NONE;
}


static PyMethodDef TestMethods[] = {
{"unicode_new", unicode_new, METH_VARARGS},
{"unicode_fill", unicode_fill, METH_VARARGS},
Expand All @@ -572,6 +585,7 @@ static PyMethodDef TestMethods[] = {
{"unicode_asutf8", unicode_asutf8, METH_VARARGS},
{"unicode_copycharacters", unicode_copycharacters, METH_VARARGS},
{"unicode_GET_CACHED_HASH", unicode_GET_CACHED_HASH, METH_O},
{"corrupt_unicode_singleton", corrupt_unicode_singleton, METH_NOARGS},
{NULL},
};

Expand Down
137 changes: 137 additions & 0 deletions Objects/object.c
Original file line number Diff line number Diff line change
Expand Up @@ -3526,3 +3526,140 @@ Py_ssize_t Py_REFCNT(PyObject *ob) { return _Py_REFCNT(ob); }
Py_ssize_t Py_SIZE(PyObject *o) { return _Py_SIZE_impl(o); }
int Py_IS_TYPE(PyObject *o, PyTypeObject *t) { return _Py_IS_TYPE_impl(o, t); }
void Py_SET_SIZE(PyVarObject *o, Py_ssize_t s) { _Py_SET_SIZE_impl(o, s); }


#ifndef NDEBUG
static void
check_singleton(PyObject *obj, PyTypeObject *type)
{
// Check PyObject.ob_refcnt
_PyObject_ASSERT(obj, _Py_IsImmortal(obj));

// Check PyObject.ob_type
_PyObject_ASSERT(obj, Py_TYPE(obj) == type);
}


static void
check_singleton_long(PyObject *obj, long value, int is_bool)
{
PyTypeObject *type = is_bool ? &PyBool_Type : &PyLong_Type;
check_singleton(obj, type);

// Check _PyLong_CompactValue()
Py_ssize_t compact = _PyLong_CompactValue((const PyLongObject *)obj);
_PyObject_ASSERT(obj, compact == value);

// Check tv_tag and ob_digit[0]
_PyLongValue *long_value = &((PyLongObject*)obj)->long_value;
int sign = (value == 0) ? 0 : ((value < 0) ? -1 : 1);
uintptr_t lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, (value == 0) ? 0 : 1);
if (!is_bool) {
lv_tag |= IMMORTALITY_BIT_MASK;
}
_PyObject_ASSERT(obj, long_value->lv_tag == lv_tag);
_PyObject_ASSERT(obj, long_value->ob_digit[0] == Py_ABS(value));
}


static void
check_singleton_bytes(PyObject *obj, Py_ssize_t size, unsigned char ch)
{
check_singleton(obj, &PyBytes_Type);
_PyObject_ASSERT(obj, PyBytes_GET_SIZE(obj) == size);
const unsigned char *str = (const unsigned char *)PyBytes_AS_STRING(obj);
_PyObject_ASSERT(obj, str[0] == ch);
if (size > 0) {
_PyObject_ASSERT(obj, str[1] == 0);
}
}


static void
check_singleton_unicode(PyObject *obj, Py_ssize_t length, Py_UCS4 ch)
{
check_singleton(obj, &PyUnicode_Type);
_PyObject_ASSERT(obj, _PyUnicode_CheckConsistency(obj, 1));

_PyObject_ASSERT(obj, PyUnicode_GET_LENGTH(obj) == length);

_PyObject_ASSERT(obj, PyUnicode_READ_CHAR(obj, 0) == ch);
if (length > 0) {
_PyObject_ASSERT(obj, PyUnicode_READ_CHAR(obj, 1) == 0);
}
}


// Check singletons consistency: try to detect if a C extension modified a
// singleton by mistake.
//
// Since the hash is computed lazily, don't check the hash, except for empty
// tuple.
void
_Py_CheckSingletons(void)
{
assert(!PyErr_Occurred());
PyObject *obj;
long ival;

// None
obj = Py_None;
check_singleton(obj, &_PyNone_Type);

// Ellipsis (...), Py_NotImplemented
obj = Py_Ellipsis;
check_singleton(obj, &PyEllipsis_Type);

obj = Py_NotImplemented;
check_singleton(obj, &_PyNotImplemented_Type);

// False, True
check_singleton_long(Py_False, 0, 1);
check_singleton_long(Py_True, 1, 1);

// Small integers
for (ival=-_PY_NSMALLNEGINTS; ival < _PY_NSMALLPOSINTS; ival++) {
obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + ival];
check_singleton_long(obj, ival, 0);
}

// Empty bytes string (b'')
obj = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
check_singleton_bytes(obj, 0, '\0');

// 1-character bytes strings
for (ival=0; ival <= 255; ival++) {
obj = (PyObject*)&_Py_SINGLETON(bytes_characters)[ival];
check_singleton_bytes(obj, 1, (unsigned char)ival);
}

// Empty Unicode string ('')
obj = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
check_singleton_unicode(obj, 0, 0);

// Do not tests _Py_STR() strings since there is no API to list them

// 1-character Unicode strings
for (ival=0; ival <= 255; ival++) {
obj = _Py_LATIN1_CHR(ival);
check_singleton_unicode(obj, 1, ival);
}

// Empty tuple (())
obj = Py_GetConstant(Py_CONSTANT_EMPTY_TUPLE);
check_singleton(obj, &PyTuple_Type);
_PyObject_ASSERT(obj, PyTuple_GET_SIZE(obj) == 0);
_PyObject_ASSERT(obj, ((PyTupleObject*)obj)->ob_hash == _PyTuple_HASH_EMPTY);

// Do not test less commoly used singletons to reduce the overhead
// of calling _Py_CheckSingletons():
//
// - PyDateTime_TimeZone_UTC
// - context_token_missing
// - hamt_bitmap_node_empty
// - interp hamt_empty
// - interp last_resort_memory_error

assert(!PyErr_Occurred());
}
#endif
6 changes: 6 additions & 0 deletions Python/gc.c
Original file line number Diff line number Diff line change
Expand Up @@ -1643,6 +1643,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
invoke_gc_callback(tstate, "stop", generation, &stats);
}

#ifndef NDEBUG
// Checking singletons consistency is unrelated to a garbage collection.
// Using a garbage collection to trigger this function is just convenient.
_Py_CheckSingletons();
#endif

assert(!_PyErr_Occurred(tstate));
gcstate->frame = NULL;
_Py_atomic_store_int(&gcstate->collecting, 0);
Expand Down
6 changes: 6 additions & 0 deletions Python/gc_free_threading.c
Original file line number Diff line number Diff line change
Expand Up @@ -2313,6 +2313,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
invoke_gc_callback(tstate, "stop", generation, m, n, state.candidates, duration);
}

#ifndef NDEBUG
// Checking singletons consistency is unrelated to a garbage collection.
// Using a garbage collection to trigger this function is just convenient.
_Py_CheckSingletons();
#endif

assert(!_PyErr_Occurred(tstate));
gcstate->frame = NULL;
_Py_atomic_store_int(&gcstate->collecting, 0);
Expand Down
Loading