Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Include/internal/pycore_dict.h
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ extern void _PyDict_Clear_LockHeld(PyObject *op);

#ifdef Py_GIL_DISABLED
PyAPI_FUNC(void) _PyDict_EnsureSharedOnRead(PyDictObject *mp);
extern void _PyFrozenDict_ClearInternal(PyObject *op);
#endif

// Export for '_elementtree' shared extension
Expand Down
11 changes: 9 additions & 2 deletions Lib/test/test_ast/test_ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -2683,6 +2683,11 @@ def test_validation(self):
self.assertEqual(str(cm.exception),
"got an invalid type in Constant: list")

with self.assertRaises(TypeError) as cm:
self.compile_constant(frozendict({1: [2, 3]}))
self.assertEqual(str(cm.exception),
"got an invalid type in Constant: list")

def test_singletons(self):
for const in (None, False, True, Ellipsis, b''):
with self.subTest(const=const):
Expand All @@ -2692,13 +2697,15 @@ def test_singletons(self):
def test_values(self):
nested_tuple = (1,)
nested_frozenset = frozenset({1})
nested_frozendict = frozendict({1: 1})
for level in range(3):
nested_tuple = (nested_tuple, 2)
nested_frozenset = frozenset({nested_frozenset, 2})
nested_frozendict = frozendict({nested_frozendict: 2})
values = (123, 123.0, 123j,
"unicode", b'bytes',
tuple("tuple"), frozenset("frozenset"),
nested_tuple, nested_frozenset)
tuple("tuple"), frozenset("frozenset"), frozendict({"a": 1}),
nested_tuple, nested_frozenset, nested_frozendict)
for value in values:
with self.subTest(value=value):
result = self.compile_constant(value)
Expand Down
47 changes: 47 additions & 0 deletions Lib/test/test_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -776,6 +776,13 @@ def check_constant(self, func, expected):
self.fail("unable to find constant %r in %r"
% (expected, func.__code__.co_consts))

@staticmethod
def _frozen_dict_consts(*consts):
"""Use AST to make frozendict constants since it has no literal syntax"""
m = ast.Interactive([ast.Expr(ast.Constant(c)) for c in consts])
ast.fix_missing_locations(m)
return compile(m, "<test>", "single")

# Merging equal constants is not a strict requirement for the Python
# semantics, it's a more an implementation detail.
@support.cpython_only
Expand Down Expand Up @@ -820,6 +827,36 @@ def check_same_constant(const):
self.check_constant(f1, frozenset({0}))
self.assertTrue(f1(0))

# two identical frozendicts merge into one constant
c = self._frozen_dict_consts(frozendict({0: 1}), frozendict({0: 1}))
self.assertEqual(c.co_consts, (frozendict({0: 1}),))

# empty frozendicts also merge
c = self._frozen_dict_consts(frozendict(), frozendict())
self.assertEqual(c.co_consts, (frozendict(),))

# frozendicts containing a nested frozendict value merge
c = self._frozen_dict_consts(
frozendict({0: frozendict({1: 2})}),
frozendict({0: frozendict({1: 2})}),
)
self.assertEqual(c.co_consts, (frozendict({0: frozendict({1: 2})}),))

# A tuple value inside a frozendict is merged with the same
# constant used elsewhere. Use a variable to ensure the two tuple
# objects are distinct before they are merged.
name = "not a name"
t_standalone = (name,)
m = ast.Interactive([
ast.Expr(ast.Constant(t_standalone)),
ast.Expr(ast.Constant(frozendict({0: (name,)}))),
ast.Expr(ast.Constant(frozendict({(name,): 0}))),
])
ast.fix_missing_locations(m)
c = compile(m, "<test>", "single")
self.assertIs(c.co_consts[0], c.co_consts[1][0])
self.assertIs(c.co_consts[0], next(iter(c.co_consts[2])))

# Merging equal co_linetable is not a strict requirement
# for the Python semantics, it's a more an implementation detail.
@support.cpython_only
Expand Down Expand Up @@ -1033,6 +1070,16 @@ def check_different_constants(const1, const2):
self.assertTrue(f1(0))
self.assertTrue(f2(0.0))

# frozendicts with type-distinct keys must not merge (0 vs 0.0)
c = self._frozen_dict_consts(frozendict({0: 1}), frozendict({0.0: 1}))
self.assertEqual(c.co_consts, (frozendict({0: 1}), frozendict({0.0: 1})))
self.assertIsNot(c.co_consts[0], c.co_consts[1])

# frozendicts with type-distinct values must not merge (1 vs 1.0)
c = self._frozen_dict_consts(frozendict({0: 1}), frozendict({0: 1.0}))
self.assertEqual(c.co_consts, (frozendict({0: 1}), frozendict({0: 1.0})))
self.assertIsNot(c.co_consts[0], c.co_consts[1])

def test_path_like_objects(self):
# An implicit test for PyUnicode_FSDecoder().
compile("42", FakePath("test_compile_pathlike"), "single")
Expand Down
115 changes: 115 additions & 0 deletions Objects/codeobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "pycore_pymem.h" // _PyMem_FreeDelayed()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_setobject.h" // _PySet_NextEntry()
#include "pycore_dict.h" // _PyFrozenDict_ClearInternal()
#include "pycore_tuple.h" // _PyTuple_ITEMS()
#include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal()
#include "pycore_uniqueid.h" // _PyObject_AssignUniqueId()
Expand Down Expand Up @@ -167,6 +168,16 @@ should_immortalize_constant(PyObject *v)
}
return 1;
}
else if (PyFrozenDict_CheckExact(v)) {
Py_ssize_t pos = 0;
PyObject *key, *value;
while (PyDict_Next(v, &pos, &key, &value)) {
if (!_Py_IsImmortal(key) || !_Py_IsImmortal(value)) {
return 0;
}
}
return 1;
}
else if (PySlice_Check(v)) {
PySliceObject *slice = (PySliceObject *)v;
return (_Py_IsImmortal(slice->start) &&
Expand Down Expand Up @@ -248,6 +259,53 @@ intern_constants(PyObject *tuple, int *modified)
}
Py_DECREF(tmp);
}
else if (PyFrozenDict_CheckExact(v)) {
PyObject *w = v;
PyObject *tmp = PyTuple_New(2 * PyDict_GET_SIZE(v));
if (tmp == NULL) {
return -1;
}
Py_ssize_t pos = 0;
PyObject *k, *val;
Py_ssize_t j = 0;
while (PyDict_Next(v, &pos, &k, &val)) {
PyTuple_SET_ITEM(tmp, j++, Py_NewRef(k));
PyTuple_SET_ITEM(tmp, j++, Py_NewRef(val));
}
int tmp_modified = 0;
if (intern_constants(tmp, &tmp_modified) < 0) {
Py_DECREF(tmp);
return -1;
}
if (tmp_modified) {
PyObject *new_dict = PyDict_New();
if (new_dict == NULL) {
Py_DECREF(tmp);
return -1;
}
for (j = 0; j < PyTuple_GET_SIZE(tmp); j += 2) {
if (PyDict_SetItem(new_dict,
PyTuple_GET_ITEM(tmp, j),
PyTuple_GET_ITEM(tmp, j + 1)) < 0) {
Py_DECREF(tmp);
Py_DECREF(new_dict);
return -1;
}
}
v = PyFrozenDict_New(new_dict);
Py_DECREF(new_dict);
if (v == NULL) {
Py_DECREF(tmp);
return -1;
}
PyTuple_SET_ITEM(tuple, i, v);
Py_DECREF(w);
if (modified) {
*modified = 1;
}
}
Py_DECREF(tmp);
}
#ifdef Py_GIL_DISABLED
else if (PySlice_Check(v)) {
PySliceObject *slice = (PySliceObject *)v;
Expand Down Expand Up @@ -3039,6 +3097,45 @@ _PyCode_ConstantKey(PyObject *op)
Py_DECREF(set);
return key;
}
else if (PyFrozenDict_CheckExact(op)) {
PyObject *dict = PyDict_New();
if (dict == NULL) {
return NULL;
}

Py_ssize_t pos = 0;
PyObject *k_obj, *v_obj;
while (PyDict_Next(op, &pos, &k_obj, &v_obj)) {
PyObject *k_key = _PyCode_ConstantKey(k_obj);
if (k_key == NULL) {
Py_DECREF(dict);
return NULL;
}
PyObject *v_key = _PyCode_ConstantKey(v_obj);
if (v_key == NULL) {
Py_DECREF(k_key);
Py_DECREF(dict);
return NULL;
}
int res = PyDict_SetItem(dict, k_key, v_key);
Py_DECREF(k_key);
Py_DECREF(v_key);
if (res < 0) {
Py_DECREF(dict);
return NULL;
}
}

PyObject *fdict = PyFrozenDict_New(dict);
Py_DECREF(dict);
if (fdict == NULL) {
return NULL;
}

key = _PyTuple_FromPair(fdict, op);
Py_DECREF(fdict);
return key;
}
else if (PySlice_Check(op)) {
PySliceObject *slice = (PySliceObject *)op;
PyObject *start_key = NULL;
Expand Down Expand Up @@ -3160,6 +3257,21 @@ compare_constants(const void *key1, const void *key2)
}
return 1;
}
else if (PyFrozenDict_CheckExact(op1)) {
if (PyDict_GET_SIZE(op1) != PyDict_GET_SIZE(op2)) {
return 0;
}
Py_ssize_t pos1 = 0, pos2 = 0;
PyObject *k1, *k2, *v1, *v2;
while (PyDict_Next(op1, &pos1, &k1, &v1) &&
PyDict_Next(op2, &pos2, &k2, &v2))
{
if (k1 != k2 || v1 != v2) {
return 0;
}
}
return 1;
}
else if (PySlice_Check(op1)) {
PySliceObject *s1 = (PySliceObject *)op1;
PySliceObject *s2 = (PySliceObject *)op2;
Expand Down Expand Up @@ -3234,6 +3346,9 @@ clear_containers(_Py_hashtable_t *ht, const void *key, const void *value,
else if (PyFrozenSet_CheckExact(op)) {
_PySet_ClearInternal((PySetObject *)op);
}
else if (PyFrozenDict_CheckExact(op)) {
_PyFrozenDict_ClearInternal(op);
}
return 0;
}

Expand Down
28 changes: 28 additions & 0 deletions Objects/dictobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -3098,6 +3098,34 @@ _PyDict_Clear_LockHeld(PyObject *op) {
clear_lock_held(op);
}

#ifdef Py_GIL_DISABLED
void
_PyFrozenDict_ClearInternal(PyObject *op)
{
assert(PyFrozenDict_CheckExact(op));
PyDictObject *mp = (PyDictObject *)op;
PyDictKeysObject *keys = mp->ma_keys;
if (keys == Py_EMPTY_KEYS) {
return;
}
assert(mp->ma_values == NULL);
if (DK_IS_UNICODE(keys)) {
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
Py_CLEAR(entries[i].me_key);
Py_CLEAR(entries[i].me_value);
}
}
else {
PyDictKeyEntry *entries = DK_ENTRIES(keys);
for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
Py_CLEAR(entries[i].me_key);
Py_CLEAR(entries[i].me_value);
}
}
}
#endif

void
PyDict_Clear(PyObject *op)
{
Expand Down
17 changes: 17 additions & 0 deletions Python/ast.c
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,23 @@ validate_constant(PyObject *value)
return 1;
}

if (PyFrozenDict_CheckExact(value)) {
ENTER_RECURSIVE();

Py_ssize_t pos = 0;
PyObject *key, *val;

while (PyDict_Next(value, &pos, &key, &val)) {
if (!validate_constant(key) || !validate_constant(val)) {
LEAVE_RECURSIVE();
return 0;
}
}

LEAVE_RECURSIVE();
return 1;
}

if (!PyErr_Occurred()) {
PyErr_Format(PyExc_TypeError,
"got an invalid type in Constant: %s",
Expand Down
Loading
Loading