From b9d2e73d3962cf8901f06e9dc6a864ed94d23271 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Thu, 15 Oct 2026 14:37:13 +0900 Subject: [PATCH] Align JSON array hooks and libraries with Python 3.15 Generate the JSON library and selected tests with update_lib from CPython d220409a13b77de7a6c30abe16ea90db01528edc, preserving native bootstrap workarounds and test markers. Exclude generated cache directories at the library copy boundary. Assisted-by: OpenAI Codex:model version unavailable --- Lib/json/__init__.py | 42 ++++++++++-- Lib/json/decoder.py | 35 +++++----- Lib/json/encoder.py | 38 +++++------ Lib/json/scanner.py | 3 +- Lib/json/tool.py | 13 ++-- Lib/test/test_json/test_decode.py | 61 +++++++++++++++++ Lib/test/test_json/test_dump.py | 65 +++++++++++++++++++ .../test_json/test_encode_basestring_ascii.py | 7 ++ Lib/test/test_json/test_enum.py | 29 ++++++--- Lib/test/test_json/test_pickleable.py | 14 ++++ Lib/test/test_json/test_unicode.py | 8 ++- crates/stdlib/src/json.rs | 28 +++++++- scripts/update_lib/cmd_copy_lib.py | 10 ++- scripts/update_lib/tests/test_copy_lib.py | 39 +++++++++++ 14 files changed, 327 insertions(+), 65 deletions(-) create mode 100644 Lib/test/test_json/test_pickleable.py diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 800cc37f6af..37a86831ff9 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -95,7 +95,6 @@ $ echo '{ 1.2:3.4}' | python -m json Expecting property name enclosed in double quotes: line 1 column 3 (char 2) """ -__version__ = '2.0.9' __all__ = [ 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', @@ -242,7 +241,7 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, **kw).encode(obj) -_default_decoder = JSONDecoder(object_hook=None, object_pairs_hook=None) +_default_decoder = JSONDecoder() def detect_encoding(b): @@ -276,7 +275,8 @@ def detect_encoding(b): def load(fp, *, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, **kw): + parse_int=None, parse_constant=None, object_pairs_hook=None, + array_hook=None, **kw): """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing a JSON document) to a Python object. @@ -292,17 +292,27 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. + ``array_hook`` is an optional function that will be called with the + result of any literal array decode (a ``list``). The return value of + this function will be used instead of the ``list``. This feature can + be used along ``object_pairs_hook`` to customize the resulting data + structure - for example, by setting that to ``frozendict`` and + ``array_hook`` to ``tuple``, one can get a deep immutable data structure + from any JSON data. + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. """ return loads(fp.read(), cls=cls, object_hook=object_hook, parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw) + parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, + array_hook=array_hook, **kw) def loads(s, *, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, **kw): + parse_int=None, parse_constant=None, object_pairs_hook=None, + array_hook=None, **kw): """Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance containing a JSON document) to a Python object. @@ -318,6 +328,14 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. + ``array_hook`` is an optional function that will be called with the + result of any literal array decode (a ``list``). The return value of + this function will be used instead of the ``list``. This feature can + be used along ``object_pairs_hook`` to customize the resulting data + structure - for example, by setting that to ``frozendict`` and + ``array_hook`` to ``tuple``, one can get a deep immutable data structure + from any JSON data. + ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser @@ -348,7 +366,8 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, if (cls is None and object_hook is None and parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None and not kw): + parse_constant is None and object_pairs_hook is None + and array_hook is None and not kw): return _default_decoder.decode(s) if cls is None: cls = JSONDecoder @@ -356,6 +375,8 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, kw['object_hook'] = object_hook if object_pairs_hook is not None: kw['object_pairs_hook'] = object_pairs_hook + if array_hook is not None: + kw['array_hook'] = array_hook if parse_float is not None: kw['parse_float'] = parse_float if parse_int is not None: @@ -363,3 +384,12 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, if parse_constant is not None: kw['parse_constant'] = parse_constant return cls(**kw).decode(s) + + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "2.0.9" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index 04e000e0760..3150a29405f 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -27,20 +27,6 @@ class JSONDecodeError(ValueError): colno: The column corresponding to pos """ - # RUSTPYTHON SPECIFIC - @classmethod - def _from_serde(cls, msg, doc, line, col): - pos = 0 - # 0-indexed - line -= 1 - col -= 1 - while line > 0: - i = doc.index('\n', pos) - line -= 1 - pos = i - pos += col - return cls(msg, doc, pos) - # Note that this exception is used from _json def __init__(self, msg, doc, pos): lineno = doc.count('\n', 0, pos) + 1 @@ -57,11 +43,11 @@ def __reduce__(self): return self.__class__, (self.msg, self.doc, self.pos) -_CONSTANTS = { +_CONSTANTS = frozendict({ '-Infinity': NegInf, 'Infinity': PosInf, 'NaN': NaN, -} +}) HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS) @@ -232,7 +218,7 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook, pairs = object_hook(pairs) return pairs, end -def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): +def JSONArray(s_and_end, scan_once, array_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR): s, end = s_and_end values = [] nextchar = s[end:end + 1] @@ -241,6 +227,8 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): nextchar = s[end:end + 1] # Look-ahead for trivial empty array if nextchar == ']': + if array_hook is not None: + values = array_hook(values) return values, end + 1 _append = values.append while True: @@ -270,6 +258,8 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): if nextchar == ']': raise JSONDecodeError("Illegal trailing comma before end of array", s, comma_idx) + if array_hook is not None: + values = array_hook(values) return values, end @@ -305,7 +295,7 @@ class JSONDecoder(object): def __init__(self, *, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None): + object_pairs_hook=None, array_hook=None): """``object_hook``, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given ``dict``. This can be used to provide custom @@ -318,6 +308,14 @@ def __init__(self, *, object_hook=None, parse_float=None, If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. + ``array_hook`` is an optional function that will be called with the + result of any literal array decode (a ``list``). The return value of + this function will be used instead of the ``list``. This feature can + be used along ``object_pairs_hook`` to customize the resulting data + structure - for example, by setting that to ``frozendict`` and + ``array_hook`` to ``tuple``, one can get a deep immutable data + structute from any JSON data. + ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser @@ -344,6 +342,7 @@ def __init__(self, *, object_hook=None, parse_float=None, self.parse_constant = parse_constant or _CONSTANTS.__getitem__ self.strict = strict self.object_pairs_hook = object_pairs_hook + self.array_hook = array_hook self.parse_object = JSONObject self.parse_array = JSONArray self.parse_string = scanstring diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 5cf6d64f3ea..718b3254241 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -79,7 +79,7 @@ class JSONEncoder(object): +-------------------+---------------+ | Python | JSON | +===================+===============+ - | dict | object | + | dict, frozendict | object | +-------------------+---------------+ | list, tuple | array | +-------------------+---------------+ @@ -264,17 +264,6 @@ def floatstr(o, allow_nan=self.allow_nan, def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, - ## HACK: hand-optimized bytecode; turn globals into locals - ValueError=ValueError, - dict=dict, - float=float, - id=id, - int=int, - isinstance=isinstance, - list=list, - str=str, - tuple=tuple, - _intstr=int.__repr__, ): def _iterencode_list(lst, _current_indent_level): @@ -311,7 +300,7 @@ def _iterencode_list(lst, _current_indent_level): # Subclasses of int/float may override __repr__, but we still # want to encode them as integers/floats in JSON. One example # within the standard library is IntEnum. - yield buf + _intstr(value) + yield buf + int.__repr__(value) elif isinstance(value, float): # see comment above for int yield buf + _floatstr(value) @@ -319,7 +308,7 @@ def _iterencode_list(lst, _current_indent_level): yield buf if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): + elif isinstance(value, (dict, frozendict)): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) @@ -374,7 +363,7 @@ def _iterencode_dict(dct, _current_indent_level): key = 'null' elif isinstance(key, int): # see comment for int/float in _make_iterencode - key = _intstr(key) + key = int.__repr__(key) elif _skipkeys: continue else: @@ -399,14 +388,14 @@ def _iterencode_dict(dct, _current_indent_level): yield 'false' elif isinstance(value, int): # see comment for int/float in _make_iterencode - yield _intstr(value) + yield int.__repr__(value) elif isinstance(value, float): # see comment for int/float in _make_iterencode yield _floatstr(value) else: if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): + elif isinstance(value, (dict, frozendict)): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) @@ -434,13 +423,13 @@ def _iterencode(o, _current_indent_level): yield 'false' elif isinstance(o, int): # see comment for int/float in _make_iterencode - yield _intstr(o) + yield int.__repr__(o) elif isinstance(o, float): # see comment for int/float in _make_iterencode yield _floatstr(o) elif isinstance(o, (list, tuple)): yield from _iterencode_list(o, _current_indent_level) - elif isinstance(o, dict): + elif isinstance(o, (dict, frozendict)): yield from _iterencode_dict(o, _current_indent_level) else: if markers is not None: @@ -458,4 +447,13 @@ def _iterencode(o, _current_indent_level): raise if markers is not None: del markers[markerid] - return _iterencode + + def _iterencode_once(o, _current_indent_level): + nonlocal _iterencode, _iterencode_dict, _iterencode_list + try: + yield from _iterencode(o, _current_indent_level) + finally: + # Break reference cycles due to mutually recursive closures: + del _iterencode, _iterencode_dict, _iterencode_list + + return _iterencode_once diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py index 090897515fe..b484e00be0f 100644 --- a/Lib/json/scanner.py +++ b/Lib/json/scanner.py @@ -23,6 +23,7 @@ def py_make_scanner(context): parse_constant = context.parse_constant object_hook = context.object_hook object_pairs_hook = context.object_pairs_hook + array_hook = context.array_hook memo = context.memo def _scan_once(string, idx): @@ -37,7 +38,7 @@ def _scan_once(string, idx): return parse_object((string, idx + 1), strict, _scan_once, object_hook, object_pairs_hook, memo) elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) + return parse_array((string, idx + 1), _scan_once, array_hook) elif nextchar == 'n' and string[idx:idx + 4] == 'null': return None, idx + 4 elif nextchar == 't' and string[idx:idx + 4] == 'true': diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 0cabbdba85a..6385d971f73 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -22,13 +22,13 @@ (?Pnull) ''', re.VERBOSE) -_group_to_theme_color = { +_group_to_theme_color = frozendict({ "key": "definition", "string": "string", "number": "number", "boolean": "keyword", "null": "keyword", -} +}) def _colorize_json(json_str, theme): @@ -44,12 +44,13 @@ def _replace_match_callback(match): def main(): description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') - parser = argparse.ArgumentParser(description=description, color=True) + parser = argparse.ArgumentParser(description=description) parser.add_argument('infile', nargs='?', - help='a JSON file to be validated or pretty-printed', + help='a JSON file to be validated or pretty-printed; ' + 'defaults to `stdin`', default='-') parser.add_argument('outfile', nargs='?', - help='write the output of infile to outfile', + help='write the output of `infile` to `outfile`', default=None) parser.add_argument('--sort-keys', action='store_true', default=False, help='sort the output of dictionaries alphabetically by key') @@ -57,7 +58,7 @@ def main(): help='disable escaping of non-ASCII characters') parser.add_argument('--json-lines', action='store_true', default=False, help='parse input using the JSON Lines format. ' - 'Use with --no-indent or --compact to produce valid JSON Lines output.') + 'Use with `--no-indent` or `--compact` to produce valid JSON Lines output.') group = parser.add_mutually_exclusive_group() group.add_argument('--indent', default=4, type=int, help='separate items with newlines and use this number ' diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 6d6ca11f673..13938ed3316 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -1,4 +1,5 @@ import decimal +import unittest.mock from io import StringIO from collections import OrderedDict from test.test_json import PyTest, CTest @@ -49,6 +50,26 @@ def test_empty_objects(self): self.assertEqual(self.loads('[]'), []) self.assertEqual(self.loads('""'), "") + def test_object_hook(self): + s = '{"a":{"b":{}}}' + + expected_result = {"a":{"b":{"x":1}, "x":1}, "x":1} + expected_hook_arguments = [ + {}, {"b": {"x":1}}, {"a": {"b": {"x":1}, "x":1}} + ] + + hook_arguments = [] + + def hook(x): + hook_arguments.append(x) + return {**x, "x":1} + + result = self.loads(s, object_hook=hook) + + self.assertEqual(result, expected_result) + self.assertEqual(hook_arguments, expected_hook_arguments) + + def test_object_pairs_hook(self): s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), @@ -71,6 +92,31 @@ def test_object_pairs_hook(self): object_pairs_hook=OrderedDict), OrderedDict([('empty', OrderedDict())])) + def test_array_hook(self): + s = '[1, 2, 3]' + t = self.loads(s, array_hook=tuple) + self.assertEqual(t, (1, 2, 3)) + self.assertEqual(type(t), tuple) + + # Nested array in inner structure with object_hook + s = '{"xkd": [[1], [2], [3]]}' + p = frozendict(xkd=((1,), (2,), (3,))) + data = self.loads(s, object_hook=frozendict, array_hook=tuple) + self.assertEqual(data, p) + self.assertEqual(type(data), frozendict) + self.assertEqual(type(data["xkd"]), tuple) + for item in data["xkd"]: + self.assertEqual(type(item), tuple) + + self.assertEqual(self.loads('[]', array_hook=tuple), ()) + + def test_load_array_hook(self): + # json.load must forward array_hook to loads + fp = StringIO('[10, 20, 30]') + result = self.json.load(fp, array_hook=tuple) + self.assertEqual(result, (10, 20, 30)) + self.assertEqual(type(result), tuple) + def test_decoder_optimizations(self): # Several optimizations were made that skip over calls to # the whitespace regex, so this test is designed to try and @@ -125,6 +171,10 @@ def test_negative_index(self): d = self.json.JSONDecoder() self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000) + def test_unterminated_string(self): + d = self.json.JSONDecoder() + self.assertRaises(self.JSONDecodeError, d.raw_decode, '"\\') + def test_limit_int(self): maxdigits = 5000 with support.adjust_int_max_str_digits(maxdigits): @@ -132,9 +182,20 @@ def test_limit_int(self): with self.assertRaises(ValueError): self.loads('1' * (maxdigits + 1)) + def test_explicit_cls_skips_json_decoder_default(self): + class CustomDecoder: + pass + + with unittest.mock.patch.object( + CustomDecoder, 'decode', create=True) as mock_decode: + self.loads('{}', cls=CustomDecoder) + mock_decode.assert_called_once() + class TestPyDecode(TestDecode, PyTest): pass class TestCDecode(TestDecode, CTest): + pass + @unittest.expectedFailure # TODO: RUSTPYTHON def test_limit_int(self): return super().test_limit_int() diff --git a/Lib/test/test_json/test_dump.py b/Lib/test/test_json/test_dump.py index 39470754003..5bc03085e60 100644 --- a/Lib/test/test_json/test_dump.py +++ b/Lib/test/test_json/test_dump.py @@ -12,6 +12,18 @@ def test_dump(self): def test_dumps(self): self.assertEqual(self.dumps({}), '{}') + def test_dumps_dict(self): + self.assertEqual(self.dumps({'x': 1, 'y': 2}), + '{"x": 1, "y": 2}') + self.assertEqual(self.dumps(frozendict({'x': 1, 'y': 2})), + '{"x": 1, "y": 2}') + lst = [{'x': 1}, frozendict(y=2)] + self.assertEqual(self.dumps(lst), + '[{"x": 1}, {"y": 2}]') + data = {'x': dict(a=1), 'y': frozendict(b=2)} + self.assertEqual(self.dumps(data), + '{"x": {"a": 1}, "y": {"b": 2}}') + def test_dump_skipkeys(self): v = {b'invalid_key': False, 'valid_key': True} with self.assertRaises(TypeError): @@ -65,6 +77,59 @@ def __lt__(self, o): d[1337] = "true.dat" self.assertEqual(self.dumps(d, sort_keys=True), '{"1337": "true.dat"}') + # gh-145244: UAF on borrowed key when default callback mutates dict + def test_default_clears_dict_key_uaf(self): + class Evil: + pass + + class AlsoEvil: + pass + + # Use a non-interned string key so it can actually be freed + key = "A" * 100 + target = {key: Evil()} + del key + + def evil_default(obj): + if isinstance(obj, Evil): + target.clear() + return AlsoEvil() + raise TypeError("not serializable") + + with self.assertRaises(TypeError): + self.json.dumps(target, default=evil_default, + check_circular=False) + + def test_dumps_str_subclass(self): + # Don't call obj.__str__() on str subclasses + + # str subclass which returns a different string on str(obj) + class StrSubclass(str): + def __str__(self): + return "StrSubclass" + + obj = StrSubclass('ascii') + self.assertEqual(self.dumps(obj), '"ascii"') + self.assertEqual(self.dumps([obj]), '["ascii"]') + self.assertEqual(self.dumps({'key': obj}), '{"key": "ascii"}') + + obj = StrSubclass('escape\n') + self.assertEqual(self.dumps(obj), '"escape\\n"') + self.assertEqual(self.dumps([obj]), '["escape\\n"]') + self.assertEqual(self.dumps({'key': obj}), '{"key": "escape\\n"}') + + obj = StrSubclass('nonascii:é') + self.assertEqual(self.dumps(obj, ensure_ascii=False), + '"nonascii:é"') + self.assertEqual(self.dumps([obj], ensure_ascii=False), + '["nonascii:é"]') + self.assertEqual(self.dumps({'key': obj}, ensure_ascii=False), + '{"key": "nonascii:é"}') + self.assertEqual(self.dumps(obj), '"nonascii:\\u00e9"') + self.assertEqual(self.dumps([obj]), '["nonascii:\\u00e9"]') + self.assertEqual(self.dumps({'key': obj}), + '{"key": "nonascii:\\u00e9"}') + class TestPyDump(TestDump, PyTest): pass diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index c90d3e968e5..1b5dfcfde01 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -3,6 +3,11 @@ from test.support import bigaddrspacetest +# str subclass which returns a different string on str(obj) +class StrSubclass(str): + def __str__(self): + return "StrSubclass" + CASES = [ ('/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), @@ -14,6 +19,8 @@ ('\U0001d120', '"\\ud834\\udd20"'), ('\u03b1\u03a9', '"\\u03b1\\u03a9"'), ("`1~!@#$%^&*()_+-={':[,]}|;.?", '"`1~!@#$%^&*()_+-={\':[,]}|;.?"'), + # Don't call obj.__str__() on str subclasses + (StrSubclass('ascii'), '"ascii"'), ] class TestEncodeBasestringAscii: diff --git a/Lib/test/test_json/test_enum.py b/Lib/test/test_json/test_enum.py index 10f414898b8..518c3e11200 100644 --- a/Lib/test/test_json/test_enum.py +++ b/Lib/test/test_json/test_enum.py @@ -26,11 +26,14 @@ class FloatNum(float, Enum): NEG_INF = float('-inf') NAN = float('nan') -class WierdNum(float, Enum): +class WeirdNum(float, Enum): inf = INF neg_inf = NEG_INF nan = NAN +class StringEnum(str, Enum): + COLOR = "color" + class TestEnum: def test_floats(self): @@ -40,7 +43,7 @@ def test_floats(self): self.assertEqual(self.loads(self.dumps(enum)), enum) def test_weird_floats(self): - for enum, expected in zip(WierdNum, ('Infinity', '-Infinity', 'NaN')): + for enum, expected in zip(WeirdNum, ('Infinity', '-Infinity', 'NaN')): self.assertEqual(self.dumps(enum), expected) if not isnan(enum): self.assertEqual(float(self.dumps(enum)), enum) @@ -64,16 +67,16 @@ def test_list(self): str([E, PI, TAU])) self.assertEqual(self.loads(self.dumps(list(FloatNum))), list(FloatNum)) - self.assertEqual(self.dumps(list(WierdNum)), + self.assertEqual(self.dumps(list(WeirdNum)), '[Infinity, -Infinity, NaN]') - self.assertEqual(self.loads(self.dumps(list(WierdNum)))[:2], - list(WierdNum)[:2]) - self.assertTrue(isnan(self.loads(self.dumps(list(WierdNum)))[2])) + self.assertEqual(self.loads(self.dumps(list(WeirdNum)))[:2], + list(WeirdNum)[:2]) + self.assertTrue(isnan(self.loads(self.dumps(list(WeirdNum)))[2])) def test_dict_keys(self): s, b, h, r = BigNum e, p, t = FloatNum - i, j, n = WierdNum + i, j, n = WeirdNum d = { s:'tiny', b:'large', h:'larger', r:'largest', e:"Euler's number", p:'pi', t:'tau', @@ -100,9 +103,9 @@ def test_dict_values(self): e=FloatNum.e, pi=FloatNum.pi, tau=FloatNum.tau, - i=WierdNum.inf, - j=WierdNum.neg_inf, - n=WierdNum.nan, + i=WeirdNum.inf, + j=WeirdNum.neg_inf, + n=WeirdNum.nan, ) nd = self.loads(self.dumps(d)) self.assertEqual(nd['tiny'], SMALL) @@ -116,5 +119,11 @@ def test_dict_values(self): self.assertEqual(nd['j'], NEG_INF) self.assertTrue(isnan(nd['n'])) + def test_str_enum(self): + obj = StringEnum.COLOR + self.assertEqual(self.dumps(obj), '"color"') + self.assertEqual(self.dumps([obj]), '["color"]') + self.assertEqual(self.dumps({'key': obj}), '{"key": "color"}') + class TestPyEnum(TestEnum, PyTest): pass class TestCEnum(TestEnum, CTest): pass diff --git a/Lib/test/test_json/test_pickleable.py b/Lib/test/test_json/test_pickleable.py new file mode 100644 index 00000000000..517cfcdd41a --- /dev/null +++ b/Lib/test/test_json/test_pickleable.py @@ -0,0 +1,14 @@ +import json +import pickle +import unittest + +class TestPickleable(unittest.TestCase): + def test_json_decode_error_is_pickleable(self): + e = json.JSONDecodeError(msg="abc", doc="def", pos=7) + + pickled = pickle.dumps(e) + unpickled = pickle.loads(pickled) + + self.assertEqual(unpickled.msg, e.msg) + self.assertEqual(unpickled.doc, e.doc) + self.assertEqual(unpickled.pos, e.pos) diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 1aa9546dc46..291580cb047 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -44,7 +44,13 @@ def test_ascii_non_printable_decode(self): '\b\t\n\f\r') s = ''.join(map(chr, range(32))) for c in s: - self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"') + with self.subTest(control_character=ord(c)): + with self.assertRaises(self.JSONDecodeError) as cm: + self.loads(f'"a{c}b"') + error = cm.exception + self.assertEqual(error.pos, 2) + self.assertEqual(error.lineno, 1) + self.assertEqual(error.colno, 3) self.assertEqual(self.loads(f'"{s}"', strict=False), s) self.assertEqual(self.loads('"\x7f"'), '\x7f') diff --git a/crates/stdlib/src/json.rs b/crates/stdlib/src/json.rs index 58461f90c26..b962f457081 100644 --- a/crates/stdlib/src/json.rs +++ b/crates/stdlib/src/json.rs @@ -52,6 +52,7 @@ mod _json { strict: bool, object_hook: Option, object_pairs_hook: Option, + array_hook: Option, parse_float: Option, parse_int: Option, parse_constant: PyObjectRef, @@ -65,6 +66,7 @@ mod _json { let strict = ctx.get_attr("strict", vm)?.try_to_bool(vm)?; let object_hook = vm.option_if_none(ctx.get_attr("object_hook", vm)?); let object_pairs_hook = vm.option_if_none(ctx.get_attr("object_pairs_hook", vm)?); + let array_hook = vm.option_if_none(ctx.get_attr("array_hook", vm)?); let parse_float = ctx.get_attr("parse_float", vm)?; let parse_float = if vm.is_none(&parse_float) || parse_float.is(vm.ctx.types.float_type) { @@ -84,6 +86,7 @@ mod _json { strict, object_hook, object_pairs_hook, + array_hook, parse_float, parse_int, parse_constant, @@ -94,6 +97,11 @@ mod _json { #[pyclass(with(Callable, Constructor))] impl JsonScanner { + #[pygetset] + fn array_hook(zelf: &Py) -> Option { + zelf.array_hook.clone() + } + fn parse( &self, pystr: PyStrRef, @@ -438,7 +446,7 @@ mod _json { // Check for empty array if bytes.get(byte_idx) == Some(&b']') { - return Ok((vm.ctx.new_list(vec![]).into(), char_idx + 1, byte_idx + 1)); + return self.finalize_array(vec![], char_idx + 1, byte_idx + 1, vm); } let mut values: Vec = Vec::new(); @@ -494,7 +502,23 @@ mod _json { } } - Ok((vm.ctx.new_list(values).into(), char_idx, byte_idx)) + self.finalize_array(values, char_idx, byte_idx, vm) + } + + fn finalize_array( + &self, + values: Vec, + end_char_idx: usize, + end_byte_idx: usize, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, usize, usize)> { + let values = vm.ctx.new_list(values); + let result = if let Some(ref hook) = self.array_hook { + hook.call((values,), vm)? + } else { + values.into() + }; + Ok((result, end_char_idx, end_byte_idx)) } /// Finalize object construction with hooks. diff --git a/scripts/update_lib/cmd_copy_lib.py b/scripts/update_lib/cmd_copy_lib.py index 1b16497fc83..26b899d21de 100644 --- a/scripts/update_lib/cmd_copy_lib.py +++ b/scripts/update_lib/cmd_copy_lib.py @@ -16,6 +16,14 @@ import sys +def _ignore_bytecode_caches(directory: str, names: list[str]) -> set[str]: + return { + name + for name in names + if name == "__pycache__" and (pathlib.Path(directory) / name).is_dir() + } + + def _copy_single( src_path: pathlib.Path, lib_path: pathlib.Path, @@ -38,7 +46,7 @@ def _copy_single( if verbose: print(f"Copying directory: {src_path} -> {lib_path}") lib_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(src_path, lib_path) + shutil.copytree(src_path, lib_path, ignore=_ignore_bytecode_caches) else: if verbose: print(f"Copying file: {src_path} -> {lib_path}") diff --git a/scripts/update_lib/tests/test_copy_lib.py b/scripts/update_lib/tests/test_copy_lib.py index aca00cb18f3..2268a830cc9 100644 --- a/scripts/update_lib/tests/test_copy_lib.py +++ b/scripts/update_lib/tests/test_copy_lib.py @@ -57,6 +57,45 @@ def test_removes_existing_before_copy(self): self.assertEqual(dst.read_text(), "new content") + def test_directory_copy_excludes_bytecode_cache_directories(self): + from update_lib.cmd_copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + src = root / "source" + dst = root / "destination" + for relative in ("__pycache__", "package/__pycache__", "assets"): + (src / relative).mkdir(parents=True) + files = { + "__init__.py": b"source\n", + "package/module.py": b"nested source\n", + "assets/data.bin": b"\x00\xffasset", + "assets/__pycache__": b"ordinary file, not a directory", + "assets/fixture.pyc": b"canonical bytecode fixture", + "__pycache__/module.cpython-315.pyc": b"stale cache", + "package/__pycache__/module.cpython-315.pyc": b"nested cache", + } + for relative, content in files.items(): + (src / relative).write_bytes(content) + + _copy_single(src, dst, verbose=False) + + expected = { + relative: content + for relative, content in files.items() + if "__pycache__" not in pathlib.Path(relative).parts[:-1] + } + actual = { + path.relative_to(dst).as_posix(): path.read_bytes() + for path in dst.rglob("*") + if path.is_file() + } + self.assertEqual(actual, expected) + self.assertFalse((dst / "__pycache__").exists()) + self.assertFalse((dst / "package/__pycache__").exists()) + for relative, content in files.items(): + self.assertEqual((src / relative).read_bytes(), content) + class TestCopyLib(unittest.TestCase): """Tests for copy_lib function."""