Skip to content

Unpacker: Change max_buffer_size to 100MiB #391

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
methane merged 5 commits into from
Dec 9, 2019
Merged

Unpacker: Change max_buffer_size to 100MiB #391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1423e7a
Unpacker: Change max_buffer_size to 100MiB
methane Dec 6, 2019
6e95db8
black
methane Dec 6, 2019
f273c79
Change default value of max_(array|map)_len
methane Dec 6, 2019
60a66d4
fix unpavkb
methane Dec 6, 2019
89ba434
change default max_(array|map)_len
methane Dec 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 18 additions & 16 deletions msgpack/_unpacker.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,41 +236,42 @@ cdef class Unpacker(object):
(See also simplejson)

:param int max_buffer_size:
Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
Limits size of data waiting unpacked. 0 means system's INT_MAX.
The default value is 100*1024*1024 (100MiB).
Raises `BufferFull` exception when it is insufficient.
You should set this parameter when unpacking data from untrusted source.

:param int max_str_len:
Deprecated, use *max_buffer_size* instead.
Limits max length of str. (default: max_buffer_size or 1024*1024)
Limits max length of str. (default: max_buffer_size)

:param int max_bin_len:
Deprecated, use *max_buffer_size* instead.
Limits max length of bin. (default: max_buffer_size or 1024*1024)
Limits max length of bin. (default: max_buffer_size)

:param int max_array_len:
Limits max length of array. (default: max_buffer_size or 128*1024)
Limits max length of array. (default: max_buffer_size)

:param int max_map_len:
Limits max length of map. (default: max_buffer_size//2 or 32*1024)
Limits max length of map. (default: max_buffer_size//2)

:param int max_ext_len:
Deprecated, use *max_buffer_size* instead.
Limits max size of ext type. (default: max_buffer_size or 1024*1024)
Limits max size of ext type. (default: max_buffer_size)

:param str unicode_errors:
Error handler used for decoding str type. (default: `'strict'`)


Example of streaming deserialize from file-like object::

unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024)
unpacker = Unpacker(file_like)
for o in unpacker:
process(o)

Example of streaming deserialize from socket::

unpacker = Unpacker(max_buffer_size=10*1024*1024)
unpacker = Unpacker()
while True:
buf = sock.recv(1024**2)
if not buf:
Expand Down Expand Up @@ -307,7 +308,7 @@ cdef class Unpacker(object):
def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
bint use_list=True, bint raw=False, bint strict_map_key=False,
object object_hook=None, object object_pairs_hook=None, object list_hook=None,
unicode_errors=None, Py_ssize_t max_buffer_size=0,
unicode_errors=None, Py_ssize_t max_buffer_size=100*1024*1024,
object ext_hook=ExtType,
Py_ssize_t max_str_len=-1,
Py_ssize_t max_bin_len=-1,
Expand All @@ -327,23 +328,24 @@ cdef class Unpacker(object):
if not PyCallable_Check(self.file_like_read):
raise TypeError("`file_like.read` must be a callable.")

if not max_buffer_size:
max_buffer_size = INT_MAX
if max_str_len == -1:
max_str_len = max_buffer_size or 1024*1024
max_str_len = max_buffer_size
if max_bin_len == -1:
max_bin_len = max_buffer_size or 1024*1024
max_bin_len = max_buffer_size
if max_array_len == -1:
max_array_len = max_buffer_size or 128*1024
max_array_len = max_buffer_size
if max_map_len == -1:
max_map_len = max_buffer_size//2 or 32*1024
max_map_len = max_buffer_size//2
if max_ext_len == -1:
max_ext_len = max_buffer_size or 1024*1024
max_ext_len = max_buffer_size

if not max_buffer_size:
max_buffer_size = INT_MAX
if read_size > max_buffer_size:
raise ValueError("read_size should be less or equal to max_buffer_size")
if not read_size:
read_size = min(max_buffer_size, 1024**2)

self.max_buffer_size = max_buffer_size
self.read_size = read_size
self.buf = <char*>PyMem_Malloc(read_size)
Expand Down
33 changes: 18 additions & 15 deletions msgpack/fallback.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,39 +195,40 @@ class Unpacker(object):
contains invalid UTF-8 string.

:param int max_buffer_size:
Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
Limits size of data waiting unpacked. 0 means 2**32-1.
The default value is 100*1024*1024 (100MiB).
Raises `BufferFull` exception when it is insufficient.
You should set this parameter when unpacking data from untrusted source.

:param int max_str_len:
Deprecated, use *max_buffer_size* instead.
Limits max length of str. (default: max_buffer_size or 1024*1024)
Limits max length of str. (default: max_buffer_size)

:param int max_bin_len:
Deprecated, use *max_buffer_size* instead.
Limits max length of bin. (default: max_buffer_size or 1024*1024)
Limits max length of bin. (default: max_buffer_size)

:param int max_array_len:
Limits max length of array.
(default: max_buffer_size or 128*1024)
(default: max_buffer_size)

:param int max_map_len:
Limits max length of map.
(default: max_buffer_size//2 or 32*1024)
(default: max_buffer_size//2)

:param int max_ext_len:
Deprecated, use *max_buffer_size* instead.
Limits max size of ext type. (default: max_buffer_size or 1024*1024)
Limits max size of ext type. (default: max_buffer_size)

Example of streaming deserialize from file-like object::

unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024)
unpacker = Unpacker(file_like)
for o in unpacker:
process(o)

Example of streaming deserialize from socket::

unpacker = Unpacker(max_buffer_size=10*1024*1024)
unpacker = Unpacker(max_buffer_size)
while True:
buf = sock.recv(1024**2)
if not buf:
Expand All @@ -254,7 +255,7 @@ def __init__(
object_pairs_hook=None,
list_hook=None,
unicode_errors=None,
max_buffer_size=0,
max_buffer_size=100 * 1024 * 1024,
ext_hook=ExtType,
max_str_len=-1,
max_bin_len=-1,
Expand Down Expand Up @@ -287,18 +288,20 @@ def __init__(
# state, which _buf_checkpoint records.
self._buf_checkpoint = 0

if not max_buffer_size:
max_buffer_size = 2 ** 31 - 1
if max_str_len == -1:
max_str_len = max_buffer_size or 1024 * 1024
max_str_len = max_buffer_size
if max_bin_len == -1:
max_bin_len = max_buffer_size or 1024 * 1024
max_bin_len = max_buffer_size
if max_array_len == -1:
max_array_len = max_buffer_size or 128 * 1024
max_array_len = max_buffer_size
if max_map_len == -1:
max_map_len = max_buffer_size // 2 or 32 * 1024
max_map_len = max_buffer_size // 2
if max_ext_len == -1:
max_ext_len = max_buffer_size or 1024 * 1024
max_ext_len = max_buffer_size

self._max_buffer_size = max_buffer_size or 2 ** 31 - 1
self._max_buffer_size = max_buffer_size
if read_size > self._max_buffer_size:
raise ValueError("read_size must be smaller than max_buffer_size")
self._read_size = read_size or min(self._max_buffer_size, 16 * 1024)
Expand Down