From 1423e7a0a52722d9cc38f27baf40bd219e301105 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 21:38:40 +0900 Subject: [PATCH 1/5] Unpacker: Change max_buffer_size to 100MiB --- msgpack/_unpacker.pyx | 34 ++++++++++++++++++---------------- msgpack/fallback.py | 33 ++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index f10e99d7..c506fbd3 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -236,27 +236,28 @@ cdef class Unpacker(object): (See also simplejson) :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Limits size of data waiting unpacked. 0 means system's INT_MAX. + The default value is 100*1024*1024 (100MiB). Raises `BufferFull` exception when it is insufficient. You should set this parameter when unpacking data from untrusted source. :param int max_str_len: Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size or 1024*1024) + Limits max length of str. (default: max_buffer_size) :param int max_bin_len: Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size or 1024*1024) + Limits max length of bin. (default: max_buffer_size) :param int max_array_len: - Limits max length of array. (default: max_buffer_size or 128*1024) + Limits max length of array. (default: max_buffer_size) :param int max_map_len: - Limits max length of map. (default: max_buffer_size//2 or 32*1024) + Limits max length of map. (default: max_buffer_size//2) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size or 1024*1024) + Limits max size of ext type. (default: max_buffer_size) :param str unicode_errors: Error handler used for decoding str type. (default: `'strict'`) @@ -264,13 +265,13 @@ cdef class Unpacker(object): Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) + unpacker = Unpacker(file_like) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(max_buffer_size=10*1024*1024) + unpacker = Unpacker() while True: buf = sock.recv(1024**2) if not buf: @@ -307,7 +308,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, *, Py_ssize_t read_size=0, bint use_list=True, bint raw=False, bint strict_map_key=False, object object_hook=None, object object_pairs_hook=None, object list_hook=None, - unicode_errors=None, Py_ssize_t max_buffer_size=0, + unicode_errors=None, Py_ssize_t max_buffer_size=100*1024*1024, object ext_hook=ExtType, Py_ssize_t max_str_len=-1, Py_ssize_t max_bin_len=-1, @@ -327,23 +328,24 @@ cdef class Unpacker(object): if not PyCallable_Check(self.file_like_read): raise TypeError("`file_like.read` must be a callable.") + if not max_buffer_size: + max_buffer_size = INT_MAX if max_str_len == -1: - max_str_len = max_buffer_size or 1024*1024 + max_str_len = max_buffer_size if max_bin_len == -1: - max_bin_len = max_buffer_size or 1024*1024 + max_bin_len = max_buffer_size if max_array_len == -1: - max_array_len = max_buffer_size or 128*1024 + max_array_len = max_buffer_size if max_map_len == -1: - max_map_len = max_buffer_size//2 or 32*1024 + max_map_len = max_buffer_size//2 if max_ext_len == -1: - max_ext_len = max_buffer_size or 1024*1024 + max_ext_len = max_buffer_size - if not max_buffer_size: - max_buffer_size = INT_MAX if read_size > max_buffer_size: raise ValueError("read_size should be less or equal to max_buffer_size") if not read_size: read_size = min(max_buffer_size, 1024**2) + self.max_buffer_size = max_buffer_size self.read_size = read_size self.buf = PyMem_Malloc(read_size) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 85a711b6..6c8a03bc 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -195,39 +195,40 @@ class Unpacker(object): contains invalid UTF-8 string. :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Limits size of data waiting unpacked. 0 means 2**32-1. + The default value is 100*1024*1024 (100MiB). Raises `BufferFull` exception when it is insufficient. You should set this parameter when unpacking data from untrusted source. :param int max_str_len: Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size or 1024*1024) + Limits max length of str. (default: max_buffer_size) :param int max_bin_len: Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size or 1024*1024) + Limits max length of bin. (default: max_buffer_size) :param int max_array_len: Limits max length of array. - (default: max_buffer_size or 128*1024) + (default: max_buffer_size) :param int max_map_len: Limits max length of map. - (default: max_buffer_size//2 or 32*1024) + (default: max_buffer_size//2) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size or 1024*1024) + Limits max size of ext type. (default: max_buffer_size) Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) + unpacker = Unpacker(file_like) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(max_buffer_size=10*1024*1024) + unpacker = Unpacker(max_buffer_size) while True: buf = sock.recv(1024**2) if not buf: @@ -254,7 +255,7 @@ def __init__( object_pairs_hook=None, list_hook=None, unicode_errors=None, - max_buffer_size=0, + max_buffer_size=100*1024*1024, ext_hook=ExtType, max_str_len=-1, max_bin_len=-1, @@ -287,18 +288,20 @@ def __init__( # state, which _buf_checkpoint records. self._buf_checkpoint = 0 + if not max_buffer_size: + max_buffer_size = 2 ** 31 - 1 if max_str_len == -1: - max_str_len = max_buffer_size or 1024 * 1024 + max_str_len = max_buffer_size if max_bin_len == -1: - max_bin_len = max_buffer_size or 1024 * 1024 + max_bin_len = max_buffer_size if max_array_len == -1: - max_array_len = max_buffer_size or 128 * 1024 + max_array_len = max_buffer_size if max_map_len == -1: - max_map_len = max_buffer_size // 2 or 32 * 1024 + max_map_len = max_buffer_size // 2 if max_ext_len == -1: - max_ext_len = max_buffer_size or 1024 * 1024 + max_ext_len = max_buffer_size - self._max_buffer_size = max_buffer_size or 2 ** 31 - 1 + self._max_buffer_size = max_buffer_size if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") self._read_size = read_size or min(self._max_buffer_size, 16 * 1024) From 6e95db8d5d2383a3afabb7df2156f105ef08f8d5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 21:39:26 +0900 Subject: [PATCH 2/5] black --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 6c8a03bc..c78b1ee4 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -255,7 +255,7 @@ def __init__( object_pairs_hook=None, list_hook=None, unicode_errors=None, - max_buffer_size=100*1024*1024, + max_buffer_size=100 * 1024 * 1024, ext_hook=ExtType, max_str_len=-1, max_bin_len=-1, From f273c7957b6c183317cf84a5156c1d342890652c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 21:55:28 +0900 Subject: [PATCH 3/5] Change default value of max_(array|map)_len --- msgpack/_unpacker.pyx | 12 ++++++------ msgpack/fallback.py | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index c506fbd3..dc2dcca6 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -171,9 +171,9 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, if max_bin_len == -1: max_bin_len = buf_len if max_array_len == -1: - max_array_len = buf_len + max_array_len = buf_len//4 if max_map_len == -1: - max_map_len = buf_len//2 + max_map_len = buf_len//8 if max_ext_len == -1: max_ext_len = buf_len @@ -250,10 +250,10 @@ cdef class Unpacker(object): Limits max length of bin. (default: max_buffer_size) :param int max_array_len: - Limits max length of array. (default: max_buffer_size) + Limits max length of array. (default: max_buffer_size//4) :param int max_map_len: - Limits max length of map. (default: max_buffer_size//2) + Limits max length of map. (default: max_buffer_size//8) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. @@ -335,9 +335,9 @@ cdef class Unpacker(object): if max_bin_len == -1: max_bin_len = max_buffer_size if max_array_len == -1: - max_array_len = max_buffer_size + max_array_len = max_buffer_size//4 if max_map_len == -1: - max_map_len = max_buffer_size//2 + max_map_len = max_buffer_size//8 if max_ext_len == -1: max_ext_len = max_buffer_size diff --git a/msgpack/fallback.py b/msgpack/fallback.py index c78b1ee4..34dd3fdf 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -210,11 +210,11 @@ class Unpacker(object): :param int max_array_len: Limits max length of array. - (default: max_buffer_size) + (default: max_buffer_size//4) :param int max_map_len: Limits max length of map. - (default: max_buffer_size//2) + (default: max_buffer_size//8) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. @@ -295,9 +295,9 @@ def __init__( if max_bin_len == -1: max_bin_len = max_buffer_size if max_array_len == -1: - max_array_len = max_buffer_size + max_array_len = max_buffer_size // 4 if max_map_len == -1: - max_map_len = max_buffer_size // 2 + max_map_len = max_buffer_size // 8 if max_ext_len == -1: max_ext_len = max_buffer_size From 60a66d46f0a291fa0d68b6b50d808efb6850b327 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 22:22:39 +0900 Subject: [PATCH 4/5] fix unpavkb --- msgpack/_unpacker.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index dc2dcca6..02b494f6 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -171,9 +171,9 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, if max_bin_len == -1: max_bin_len = buf_len if max_array_len == -1: - max_array_len = buf_len//4 + max_array_len = buf_len if max_map_len == -1: - max_map_len = buf_len//8 + max_map_len = buf_len//2 if max_ext_len == -1: max_ext_len = buf_len From 89ba434073b19e699525ee5cb6a34303f050652c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 22:29:36 +0900 Subject: [PATCH 5/5] change default max_(array|map)_len --- msgpack/_unpacker.pyx | 8 ++++---- msgpack/fallback.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 02b494f6..c506fbd3 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -250,10 +250,10 @@ cdef class Unpacker(object): Limits max length of bin. (default: max_buffer_size) :param int max_array_len: - Limits max length of array. (default: max_buffer_size//4) + Limits max length of array. (default: max_buffer_size) :param int max_map_len: - Limits max length of map. (default: max_buffer_size//8) + Limits max length of map. (default: max_buffer_size//2) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. @@ -335,9 +335,9 @@ cdef class Unpacker(object): if max_bin_len == -1: max_bin_len = max_buffer_size if max_array_len == -1: - max_array_len = max_buffer_size//4 + max_array_len = max_buffer_size if max_map_len == -1: - max_map_len = max_buffer_size//8 + max_map_len = max_buffer_size//2 if max_ext_len == -1: max_ext_len = max_buffer_size diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 34dd3fdf..c78b1ee4 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -210,11 +210,11 @@ class Unpacker(object): :param int max_array_len: Limits max length of array. - (default: max_buffer_size//4) + (default: max_buffer_size) :param int max_map_len: Limits max length of map. - (default: max_buffer_size//8) + (default: max_buffer_size//2) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. @@ -295,9 +295,9 @@ def __init__( if max_bin_len == -1: max_bin_len = max_buffer_size if max_array_len == -1: - max_array_len = max_buffer_size // 4 + max_array_len = max_buffer_size if max_map_len == -1: - max_map_len = max_buffer_size // 8 + max_map_len = max_buffer_size // 2 if max_ext_len == -1: max_ext_len = max_buffer_size