From c547d6c990ccc698c2805df9f6ea5ca2da071296 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 13 Feb 2025 11:11:28 +0100 Subject: [PATCH 1/4] Reduce number of yield statements in json encoder --- Lib/json/encoder.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index b804224098e14f8..cfcf2f514880d6a 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -295,24 +295,24 @@ def _iterencode_list(lst, _current_indent_level): separator = _item_separator for i, value in enumerate(lst): if i: - buf = separator + buf += separator try: if isinstance(value, str): - yield buf + _encoder(value) + buf += _encoder(value) elif value is None: - yield buf + 'null' + buf += 'null' elif value is True: - yield buf + 'true' + buf += 'true' elif value is False: - yield buf + 'false' + buf += 'false' elif isinstance(value, int): # Subclasses of int/float may override __repr__, but we still # want to encode them as integers/floats in JSON. One example # within the standard library is IntEnum. - yield buf + _intstr(value) + buf += _intstr(value) elif isinstance(value, float): # see comment above for int - yield buf + _floatstr(value) + buf += _floatstr(value) else: yield buf if isinstance(value, (list, tuple)): @@ -322,11 +322,19 @@ def _iterencode_list(lst, _current_indent_level): else: chunks = _iterencode(value, _current_indent_level) yield from chunks + buf = '' + if len(buf)> 1024: + yield buf + buf = '' + except GeneratorExit: + yield buf raise except BaseException as exc: + yield buf exc.add_note(f'when serializing {type(lst).__name__} item {i}') raise + yield buf if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level From be96354f8b5e82c6b00a00707f5723f5de8c96a6 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 13 Feb 2025 12:37:39 +0100 Subject: [PATCH 2/4] add dict encoder --- Lib/json/encoder.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index cfcf2f514880d6a..0836efa1e3aa692 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -351,12 +351,12 @@ def _iterencode_dict(dct, _current_indent_level): if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = dct - yield '{' + buf = '{' if _indent is not None: _current_indent_level += 1 newline_indent = '\n' + _indent * _current_indent_level item_separator = _item_separator + newline_indent - yield newline_indent + buf += newline_indent else: newline_indent = None item_separator = _item_separator @@ -385,30 +385,32 @@ def _iterencode_dict(dct, _current_indent_level): elif _skipkeys: continue else: + yield buf raise TypeError(f'keys must be str, int, float, bool or None, ' f'not {key.__class__.__name__}') if first: first = False else: - yield item_separator - yield _encoder(key) - yield _key_separator + buf += item_separator + buf += _encoder(key) + buf += _key_separator try: if isinstance(value, str): - yield _encoder(value) + buf += _encoder(value) elif value is None: - yield 'null' + buf += 'null' elif value is True: - yield 'true' + buf += 'true' elif value is False: - yield 'false' + buf += 'false' elif isinstance(value, int): # see comment for int/float in _make_iterencode - yield _intstr(value) + buf += _intstr(value) elif isinstance(value, float): # see comment for int/float in _make_iterencode - yield _floatstr(value) + buf += _floatstr(value) else: + yield buf if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) elif isinstance(value, dict): @@ -416,11 +418,19 @@ def _iterencode_dict(dct, _current_indent_level): else: chunks = _iterencode(value, _current_indent_level) yield from chunks + buf = '' except GeneratorExit: + yield buf raise except BaseException as exc: exc.add_note(f'when serializing {type(dct).__name__} item {key!r}') + yield buf raise + if len(buf) > 1024: + yield buf + buf = '' + yield buf + if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level From f409afc90393a116f70be8c39ea88dc452a249ee Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 13 Feb 2025 21:00:18 +0100 Subject: [PATCH 3/4] wip --- Lib/json/encoder.py | 60 +++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 0836efa1e3aa692..1cf08cc62e536bb 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -260,6 +260,26 @@ def floatstr(o, allow_nan=self.allow_nan, self.skipkeys, _one_shot) return _iterencode(o, 0) +class _JsonBuffer(list): + + def __init__(self): + self.size=0 + + def add_json(self, j): + self.append(j) + self.size += len(j) + + def needs_yield(self): + return self.size > 10_000 or len(self) > 100 + + def value(self): + return ''.join(self) + def reset(self): + self.clear() + self.size=0 +buf = _JsonBuffer() + + def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, ## HACK: hand-optimized bytecode; turn globals into locals @@ -275,7 +295,9 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _intstr=int.__repr__, ): + def _iterencode_list(lst, _current_indent_level): + buf = _JsonBuffer() if not lst: yield '[]' return @@ -284,37 +306,37 @@ def _iterencode_list(lst, _current_indent_level): if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = lst - buf = '[' + buf.append('[') if _indent is not None: _current_indent_level += 1 newline_indent = '\n' + _indent * _current_indent_level separator = _item_separator + newline_indent - buf += newline_indent + buf.append(newline_indent) else: newline_indent = None separator = _item_separator for i, value in enumerate(lst): if i: - buf += separator + buf.append(separator) try: if isinstance(value, str): - buf += _encoder(value) + buf.append(_encoder(value)) elif value is None: - buf += 'null' + buf.append('null') elif value is True: - buf += 'true' + buf.append('true') elif value is False: - buf += 'false' + buf.append( 'false') elif isinstance(value, int): # Subclasses of int/float may override __repr__, but we still # want to encode them as integers/floats in JSON. One example # within the standard library is IntEnum. - buf += _intstr(value) + buf.append(_intstr(value)) elif isinstance(value, float): # see comment above for int - buf += _floatstr(value) + buf.append( _floatstr(value)) else: - yield buf + yield buf.value() if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) elif isinstance(value, dict): @@ -322,23 +344,23 @@ def _iterencode_list(lst, _current_indent_level): else: chunks = _iterencode(value, _current_indent_level) yield from chunks - buf = '' - if len(buf)> 1024: - yield buf - buf = '' + buf.reset() + if buf.needs_yield(): + yield buf.value() + buf.reset() except GeneratorExit: - yield buf + yield from buf raise except BaseException as exc: - yield buf + yield from buf exc.add_note(f'when serializing {type(lst).__name__} item {i}') raise - yield buf if newline_indent is not None: _current_indent_level -= 1 - yield '\n' + _indent * _current_indent_level - yield ']' + buf.append( '\n' + _indent * _current_indent_level) + buf.append(']') + yield from buf if markers is not None: del markers[markerid] From 8de90b5d06a81339363e04ed4d0ee86764c9bfab Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 13 Feb 2025 21:55:41 +0100 Subject: [PATCH 4/4] wip --- Lib/json/encoder.py | 114 +++++++++++++++++++++++++++++--------------- 1 file changed, 75 insertions(+), 39 deletions(-) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 1cf08cc62e536bb..cb48c5dd760d7ce 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -260,22 +260,29 @@ def floatstr(o, allow_nan=self.allow_nan, self.skipkeys, _one_shot) return _iterencode(o, 0) -class _JsonBuffer(list): +class _JsonBuffer: def __init__(self): - self.size=0 + self.chunks = [] + self.size = 0 - def add_json(self, j): - self.append(j) + def append(self, j): + self.chunks.append(j) self.size += len(j) + def __iadd__(self, x): + self.append(x) + return self + def needs_yield(self): - return self.size > 10_000 or len(self) > 100 + return self.size > 10_000 or len(self.chunks) >=1000 def value(self): - return ''.join(self) - def reset(self): - self.clear() + x= ''.join(self.chunks) + self._reset() + return x + def _reset(self): + self.chunks.clear() self.size=0 buf = _JsonBuffer() @@ -297,7 +304,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, def _iterencode_list(lst, _current_indent_level): - buf = _JsonBuffer() if not lst: yield '[]' return @@ -306,6 +312,7 @@ def _iterencode_list(lst, _current_indent_level): if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = lst + buf = _JsonBuffer() buf.append('[') if _indent is not None: _current_indent_level += 1 @@ -336,31 +343,30 @@ def _iterencode_list(lst, _current_indent_level): # see comment above for int buf.append( _floatstr(value)) else: - yield buf.value() if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) elif isinstance(value, dict): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) - yield from chunks - buf.reset() + for c in chunks: + buf.append(c) + if buf.needs_yield(): + yield buf.value() if buf.needs_yield(): yield buf.value() - buf.reset() - except GeneratorExit: - yield from buf + yield buf.value() raise except BaseException as exc: - yield from buf + yield buf.value() exc.add_note(f'when serializing {type(lst).__name__} item {i}') raise if newline_indent is not None: _current_indent_level -= 1 buf.append( '\n' + _indent * _current_indent_level) buf.append(']') - yield from buf + yield buf.value() if markers is not None: del markers[markerid] @@ -373,12 +379,16 @@ def _iterencode_dict(dct, _current_indent_level): if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = dct - buf = '{' + + #buf = '{' + buf = _JsonBuffer(); buf.append('{') + if _indent is not None: _current_indent_level += 1 newline_indent = '\n' + _indent * _current_indent_level item_separator = _item_separator + newline_indent - buf += newline_indent + buf +=(newline_indent) + #buf.append(newline_indent) else: newline_indent = None item_separator = _item_separator @@ -407,56 +417,79 @@ def _iterencode_dict(dct, _current_indent_level): elif _skipkeys: continue else: - yield buf + #yield buf + yield buf.value() raise TypeError(f'keys must be str, int, float, bool or None, ' f'not {key.__class__.__name__}') if first: first = False else: - buf += item_separator - buf += _encoder(key) - buf += _key_separator + buf +=item_separator + #buf.append(item_separator) + buf.append(_encoder(key) + _key_separator) + #buf.append(_key_separator) + #buf.append(_encoder(key)) + #buf.append(_key_separator) try: if isinstance(value, str): - buf += _encoder(value) + buf.append(_encoder(value)) + #buf.append(_encoder(value)) elif value is None: - buf += 'null' + buf +='null' + #buf.append('null') elif value is True: - buf += 'true' + buf +='true' + #buf.append('true') elif value is False: - buf += 'false' + buf.append('false') + #buf.append('false') elif isinstance(value, int): # see comment for int/float in _make_iterencode - buf += _intstr(value) + buf.append(_intstr(value)) + #buf.append(_intstr(value)) elif isinstance(value, float): # see comment for int/float in _make_iterencode - buf += _floatstr(value) + buf.append( _floatstr(value)) + #buf.append( _floatstr(value)) else: - yield buf + #yield buf + #yield buf.value() if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) + #chunks=list(chunks) + #print(f'_iterencode_list: returned {len(chunks)}') elif isinstance(value, dict): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) - yield from chunks - buf = '' + #yield from chunks + #buf = '' + #buf = _JsonBuffer() + for c in chunks: + buf.append(c) + if buf.needs_yield(): + yield buf.value() + # buf.reset() + except GeneratorExit: - yield buf + #yield buf + yield buf.value() raise except BaseException as exc: exc.add_note(f'when serializing {type(dct).__name__} item {key!r}') - yield buf + #yield buf + yield buf.value() raise - if len(buf) > 1024: - yield buf - buf = '' - yield buf + #yield buf + yield buf.value() if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level + #buf.append( '\n' + _indent * _current_indent_level) yield '}' + #buf.append('}') + #yield buf.value() if markers is not None: del markers[markerid] @@ -478,7 +511,10 @@ def _iterencode(o, _current_indent_level): elif isinstance(o, (list, tuple)): yield from _iterencode_list(o, _current_indent_level) elif isinstance(o, dict): - yield from _iterencode_dict(o, _current_indent_level) + z=_iterencode_dict(o, _current_indent_level) + #z=list(z); print(f'_iterencode_dict: {len(z)}') + #print(f' --> {list(map(len, z))} ') + yield from z else: if markers is not None: markerid = id(o)