From 604ac8cefe7428d4ef14ff0704de2d63e9e99b29 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 4 Apr 2018 18:07:42 +0100 Subject: [PATCH 001/280] Get version from pkg_resources as described in setuptools_scm manual --- docs/conf.py | 8 +++----- lz4/__init__.py | 8 +++++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f752fa81..1c92b8e5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,11 +55,9 @@ # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -import lz4 -version = lz4.__version__ -# The full version, including alpha/beta/rc tags. -release = lz4.__version__ +from pkg_resources import get_distribution +release = get_distribution('lz4').version +version = '.'.join(release.split('.')[:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/lz4/__init__.py b/lz4/__init__.py index 69da37eb..32e85f5a 100644 --- a/lz4/__init__.py +++ b/lz4/__init__.py @@ -1,5 +1,11 @@ # Package version info, generated on install -from .version import version as __version__ +from pkg_resources import get_distribution, DistributionNotFound +try: + __version__ = get_distribution(__name__).version +except DistributionNotFound: + # package is not installed + pass + VERSION = __version__ from ._version import ( From 3d08a2c9d1a14d8b05b84738e5fc279de19fdd6e Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 4 Apr 2018 18:12:14 +0100 Subject: [PATCH 002/280] Fix up documentation version and copyright --- docs/conf.py | 4 ++-- docs/lz4.rst | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1c92b8e5..51b6576f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,7 +49,7 @@ # General information about the project. project = u'python-lz4' -copyright = u'2016, python-lz4 developers' +copyright = u'2016, 2017, 2018 python-lz4 developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -57,7 +57,7 @@ # from pkg_resources import get_distribution release = get_distribution('lz4').version -version = '.'.join(release.split('.')[:2]) +version = release # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/lz4.rst b/docs/lz4.rst index be058f5a..fbf0ec62 100644 --- a/docs/lz4.rst +++ b/docs/lz4.rst @@ -12,5 +12,6 @@ Contents :members: library_version_number, library_version_string, - version + __version__ + From 8d836e35e15a748bd8b1cf664916cde3bc8c11c7 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 4 Apr 2018 18:39:40 +0100 Subject: [PATCH 003/280] Fix up badges in README.rst --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 4a7b51c6..3759059f 100644 --- a/README.rst +++ b/README.rst @@ -9,11 +9,11 @@ Status :target: https://travis-ci.org/python-lz4/python-lz4 :alt: Build Status -.. image:: https://ci.appveyor.com/api/projects/status/github/python-lz4/python-lz4?branch=master - :target: https://ci.appveyor.com/project/jonathanunderwood/python-lz4 +.. image:: https://ci.appveyor.com/api/projects/status/r2qvw9mlfo63lklo/branch/master?svg=true + :target: https://ci.appveyor.com/project/python-lz4/python-lz4 :alt: Build Status Windows -.. image:: https://readthedocs.org/projects/python-lz4/badge/?version=latest +.. image:: https://readthedocs.org/projects/python-lz4/badge/?version=stable :target: https://readthedocs.org/projects/python-lz4/ :alt: Documentation @@ -40,7 +40,7 @@ thread safe. An extensive test suite is included. Documenation ============ -.. image:: https://readthedocs.org/projects/python-lz4/badge/?version=latest +.. image:: https://readthedocs.org/projects/python-lz4/badge/?version=stable :target: https://readthedocs.org/projects/python-lz4/ :alt: Documentation From e917a2b8a2fc38d15d176aeb66b6049cc8fb5448 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 4 Apr 2018 18:42:07 +0100 Subject: [PATCH 004/280] Fix up AppVeyor badge link --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3759059f..fb5d2913 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ Status :alt: Build Status .. image:: https://ci.appveyor.com/api/projects/status/r2qvw9mlfo63lklo/branch/master?svg=true - :target: https://ci.appveyor.com/project/python-lz4/python-lz4 + :target: https://ci.appveyor.com/project/jonathanunderwood/python-lz4 :alt: Build Status Windows .. image:: https://readthedocs.org/projects/python-lz4/badge/?version=stable From 6772b656c5a88a68c9724a93a5127d6594040571 Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Thu, 24 May 2018 16:39:11 -0400 Subject: [PATCH 005/280] dictionary support for block.compress() --- lz4/block/_block.c | 57 ++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index b4b5da2f..06dc817c 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -93,6 +93,32 @@ typedef enum HIGH_COMPRESSION } compression_type; +static int +lz4_compress_generic (int comp, char* source, char* dest, size_t source_size, size_t dest_size, + char* dict, size_t dict_size, int acceleration, int compression) +{ + if (comp != HIGH_COMPRESSION) + { + LZ4_stream_t lz4_state; + LZ4_resetStream (&lz4_state); + if (dict) { + LZ4_loadDict (&lz4_state, dict, dict_size); + } + if (comp != FAST) + { + acceleration = 1; + } + return LZ4_compress_fast_continue (&lz4_state, source, dest, source_size, dest_size, acceleration); + } else { + LZ4_streamHC_t lz4_state; + LZ4_resetStreamHC (&lz4_state, compression); + if (dict) { + LZ4_loadDictHC (&lz4_state, dict, dict_size); + } + return LZ4_compress_HC_continue (&lz4_state, source, dest, source_size, dest_size); + } +} + static PyObject * compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) { @@ -108,6 +134,7 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) Py_buffer source; size_t source_size; int return_bytearray = 0; + Py_buffer dict = { NULL, NULL }; static char *argnames[] = { "source", "mode", @@ -115,23 +142,24 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) "acceleration", "compression", "return_bytearray", + "dict", NULL }; #if IS_PY3 - if (!PyArg_ParseTupleAndKeywords (args, kwargs, "y*|spiip", argnames, + if (!PyArg_ParseTupleAndKeywords (args, kwargs, "y*|spiipz*", argnames, &source, &mode, &store_size, &acceleration, &compression, - &return_bytearray)) + &return_bytearray, &dict)) { return NULL; } #else - if (!PyArg_ParseTupleAndKeywords (args, kwargs, "s*|siiii", argnames, + if (!PyArg_ParseTupleAndKeywords (args, kwargs, "s*|siiiiz*", argnames, &source, &mode, &store_size, &acceleration, &compression, - &return_bytearray)) + &return_bytearray, &dict)) { return NULL; } @@ -144,6 +172,7 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) if (store_size && source_size > UINT_MAX) { PyBuffer_Release(&source); + PyBuffer_Release(&dict); PyErr_Format(PyExc_OverflowError, "Input too large for storing size in 4 byte header"); return NULL; @@ -164,6 +193,7 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) else { PyBuffer_Release(&source); + PyBuffer_Release(&dict); PyErr_Format (PyExc_ValueError, "Invalid mode argument: %s. Must be one of: standard, fast, high_compression", mode); @@ -199,25 +229,14 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) dest_start = dest; } - switch (comp) - { - case DEFAULT: - output_size = LZ4_compress_default (source.buf, dest_start, source_size, - dest_size); - break; - case FAST: - output_size = LZ4_compress_fast (source.buf, dest_start, source_size, - dest_size, acceleration); - break; - case HIGH_COMPRESSION: - output_size = LZ4_compress_HC (source.buf, dest_start, source_size, - dest_size, compression); - break; - } + output_size = lz4_compress_generic (comp, source.buf, dest_start, source_size, + dest_size, dict.buf, dict.len, acceleration, + compression); Py_END_ALLOW_THREADS PyBuffer_Release(&source); + PyBuffer_Release(&dict); if (output_size <= 0) { From b44fbaabf5a8e3d669ef9b6c265bb7458213cffb Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Fri, 25 May 2018 11:03:49 -0400 Subject: [PATCH 006/280] dictionary support for block.decompress() --- lz4/block/_block.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 06dc817c..edf9daa8 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -281,24 +281,26 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) size_t dest_size; int uncompressed_size = -1; int return_bytearray = 0; + Py_buffer dict = { NULL, NULL }; static char *argnames[] = { "source", "uncompressed_size", "return_bytearray", + "dict", NULL }; #if IS_PY3 - if (!PyArg_ParseTupleAndKeywords (args, kwargs, "y*|ip", argnames, + if (!PyArg_ParseTupleAndKeywords (args, kwargs, "y*|ipz*", argnames, &source, &uncompressed_size, - &return_bytearray)) + &return_bytearray, &dict)) { return NULL; } #else - if (!PyArg_ParseTupleAndKeywords (args, kwargs, "s*|ii", argnames, + if (!PyArg_ParseTupleAndKeywords (args, kwargs, "s*|iiz*", argnames, &source, &uncompressed_size, - &return_bytearray)) + &return_bytearray, &dict)) { return NULL; } @@ -315,6 +317,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) if (source_size < hdr_size) { PyBuffer_Release(&source); + PyBuffer_Release(&dict); PyErr_SetString (PyExc_ValueError, "Input source data size too small"); return NULL; } @@ -326,6 +329,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) if (dest_size < 0 || dest_size > PY_SSIZE_T_MAX) { PyBuffer_Release(&source); + PyBuffer_Release(&dict); PyErr_Format (PyExc_ValueError, "Invalid size in header: 0x%zu", dest_size); return NULL; @@ -340,11 +344,13 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) Py_BEGIN_ALLOW_THREADS output_size = - LZ4_decompress_safe (source_start, dest, source_size, dest_size); + LZ4_decompress_safe_usingDict (source_start, dest, source_size, dest_size, + dict.buf, dict.len); Py_END_ALLOW_THREADS PyBuffer_Release(&source); + PyBuffer_Release(&dict); if (output_size < 0) { From c3f1a23ed240be86f9293799768bf6382347324e Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Fri, 25 May 2018 13:35:57 -0400 Subject: [PATCH 007/280] document the new 'dict' kwarg for block.compress() and block.decompress() --- lz4/block/_block.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index edf9daa8..d40a187b 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -415,6 +415,8 @@ PyDoc_STRVAR(compress__doc, " return_bytearray (bool): If ``False`` (the default) then the function\n" \ " will return a bytes object. If ``True``, then the function will\n" \ " return a bytearray object.\n\n" \ + " dict (str, bytes or buffer-compatible object): If specified, perform\n" \ + " compression using this initial dictionary.\n" \ "Returns:\n" \ " bytes or bytearray: Compressed data.\n"); @@ -433,6 +435,8 @@ PyDoc_STRVAR(decompress__doc, " return_bytearray (bool): If ``False`` (the default) then the function\n" \ " will return a bytes object. If ``True``, then the function will\n" \ " return a bytearray object.\n\n" \ + " dict (str, bytes or buffer-compatible object): If specified, perform\n" \ + " decompression using this initial dictionary.\n" \ "Returns:\n" \ " bytes or bytearray: Decompressed data.\n"); From e53ede13b9a2f802b0b9db7cb8e022c612b406a2 Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Fri, 25 May 2018 17:16:30 -0400 Subject: [PATCH 008/280] Added simple test cases for block.compress and block.decompress using dictionary --- tests/block/test_block.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/block/test_block.py b/tests/block/test_block.py index 658d94a3..047bf1f9 100644 --- a/tests/block/test_block.py +++ b/tests/block/test_block.py @@ -157,3 +157,27 @@ def test_memoryview(): compressed = lz4.block.compress(data) assert lz4.block.compress(memoryview(data)) == compressed assert lz4.block.decompress(memoryview(compressed)) == data + +def test_with_dict_none(): + input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 + for mode in ['default', 'high_compression']: + assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode, dict=None)) == input_data + assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode), dict=None) == input_data + assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode, dict=b'')) == input_data + assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode), dict=b'') == input_data + assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode, dict='')) == input_data + assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode), dict='') == input_data + +def test_with_dict(): + input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 + dict1 = input_data[10:30] + dict2 = input_data[20:40] + for mode in ['default', 'high_compression']: + compressed = lz4.block.compress(input_data, mode=mode, dict=dict1) + with pytest.raises(ValueError, message=r'Decompressor wrote \d+ bytes, but \d+ bytes expected from header)'): + lz4.block.decompress(compressed) + with pytest.raises(ValueError, message=r'Decompressor wrote \d+ bytes, but \d+ bytes expected from header)'): + lz4.block.decompress(compressed, dict=dict1[:2]) + assert lz4.block.decompress(compressed, dict=dict2) != input_data + assert lz4.block.decompress(compressed, dict=dict1) == input_data + assert lz4.block.decompress(lz4.block.compress(input_data), dict=dict1) == input_data From 29e28a3614310e0c53d244ec54619431ef1abd41 Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Fri, 25 May 2018 17:17:40 -0400 Subject: [PATCH 009/280] Added known test cases for block.decompress --- tests/block/test_block.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/block/test_block.py b/tests/block/test_block.py index 047bf1f9..ad1b1e06 100644 --- a/tests/block/test_block.py +++ b/tests/block/test_block.py @@ -181,3 +181,17 @@ def test_with_dict(): assert lz4.block.decompress(compressed, dict=dict2) != input_data assert lz4.block.decompress(compressed, dict=dict1) == input_data assert lz4.block.decompress(lz4.block.compress(input_data), dict=dict1) == input_data + +def test_known_decompress(): + assert(lz4.block.decompress( + b'\x00\x00\x00\x00\x00') == + b'') + assert(lz4.block.decompress( + b'\x01\x00\x00\x00\x10 ') == + b' ') + assert(lz4.block.decompress( + b'h\x00\x00\x00\xff\x0bLorem ipsum dolor sit amet\x1a\x006P amet') == + b'Lorem ipsum dolor sit amet' * 4) + assert(lz4.block.decompress( + b'\xb0\xb3\x00\x00\xff\x1fExcepteur sint occaecat cupidatat non proident.\x00' + (b'\xff' * 180) + b'\x1ePident') == + b'Excepteur sint occaecat cupidatat non proident' * 1000) From ee605c0aff2c84e506a14dab6885dc0fc29d59cf Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Fri, 25 May 2018 22:45:23 -0400 Subject: [PATCH 010/280] fix indentation --- lz4/block/_block.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index d40a187b..734a0f70 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -97,26 +97,26 @@ static int lz4_compress_generic (int comp, char* source, char* dest, size_t source_size, size_t dest_size, char* dict, size_t dict_size, int acceleration, int compression) { - if (comp != HIGH_COMPRESSION) - { - LZ4_stream_t lz4_state; - LZ4_resetStream (&lz4_state); - if (dict) { - LZ4_loadDict (&lz4_state, dict, dict_size); - } - if (comp != FAST) - { - acceleration = 1; - } - return LZ4_compress_fast_continue (&lz4_state, source, dest, source_size, dest_size, acceleration); - } else { - LZ4_streamHC_t lz4_state; - LZ4_resetStreamHC (&lz4_state, compression); - if (dict) { - LZ4_loadDictHC (&lz4_state, dict, dict_size); + if (comp != HIGH_COMPRESSION) + { + LZ4_stream_t lz4_state; + LZ4_resetStream (&lz4_state); + if (dict) { + LZ4_loadDict (&lz4_state, dict, dict_size); + } + if (comp != FAST) + { + acceleration = 1; } - return LZ4_compress_HC_continue (&lz4_state, source, dest, source_size, dest_size); + return LZ4_compress_fast_continue (&lz4_state, source, dest, source_size, dest_size, acceleration); + } else { + LZ4_streamHC_t lz4_state; + LZ4_resetStreamHC (&lz4_state, compression); + if (dict) { + LZ4_loadDictHC (&lz4_state, dict, dict_size); } + return LZ4_compress_HC_continue (&lz4_state, source, dest, source_size, dest_size); + } } static PyObject * From 66170beb476169d3487dbd828604a9c5c3812f85 Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Mon, 28 May 2018 13:16:13 -0400 Subject: [PATCH 011/280] bug fix: use signed comparison when checking return value for error --- lz4/block/_block.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 734a0f70..18675703 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -238,7 +238,7 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) PyBuffer_Release(&source); PyBuffer_Release(&dict); - if (output_size <= 0) + if ((ssize_t)output_size <= 0) { PyErr_SetString (PyExc_ValueError, "Compression failed"); PyMem_Free (dest); @@ -352,7 +352,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) PyBuffer_Release(&source); PyBuffer_Release(&dict); - if (output_size < 0) + if ((ssize_t)output_size < 0) { PyErr_Format (PyExc_ValueError, "Corrupt input at byte %zu", -output_size); PyMem_Free (dest); From f848a34629faa6dbf554bc94d367343eb30ceccf Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Mon, 28 May 2018 13:21:37 -0400 Subject: [PATCH 012/280] more comprehensive dictionary tests, various fixes --- tests/block/conftest.py | 12 ++++++++ tests/block/test_block.py | 63 +++++++++++++++++++++++---------------- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/tests/block/conftest.py b/tests/block/conftest.py index d95a96e8..9973bf32 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -81,3 +81,15 @@ def d_return_bytearray(return_bytearray): ) def mode(request): return request.param + +@pytest.fixture( + params=[ + None, + (0, 0), + (100, 200), + (0, 8 * 1024), + os.urandom(8 * 1024) + ] +) +def dictionary(request): + return request.param diff --git a/tests/block/test_block.py b/tests/block/test_block.py index ad1b1e06..d69bb55e 100644 --- a/tests/block/test_block.py +++ b/tests/block/test_block.py @@ -28,25 +28,39 @@ def get_stored_size(buff): # Test single threaded usage with all valid variations of input -def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray): +def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dictionary): kwargs = {} + if dictionary: + if isinstance(dictionary, tuple): + kwargs['dict'] = data[dictionary[0]:dictionary[1]] + else: + kwargs['dict'] = dictionary + + c_kwargs = dict(kwargs) + if mode[0] != None: - kwargs['mode'] = mode[0] + c_kwargs['mode'] = mode[0] if mode[1] != None: - kwargs.update(mode[1]) + c_kwargs.update(mode[1]) - kwargs.update(store_size) - kwargs.update(c_return_bytearray) + c_kwargs.update(store_size) + c_kwargs.update(c_return_bytearray) + + c = lz4.block.compress(data, **c_kwargs) - c = lz4.block.compress(data, **kwargs) + d_kwargs = dict(kwargs) if store_size['store_size']: assert get_stored_size(c) == len(data) - d = lz4.block.decompress(c, **d_return_bytearray) + d = lz4.block.decompress(c, **d_kwargs) else: - d = lz4.block.decompress(c, uncompressed_size=len(data), - **d_return_bytearray) + d_kwargs['uncompressed_size'] = len(data) + + d_kwargs.update(d_return_bytearray) + + d = lz4.block.decompress(c, **d_kwargs) + assert d == data if d_return_bytearray['return_bytearray']: assert isinstance(d, bytearray) @@ -89,9 +103,9 @@ def test_decompress_without_leak(): # Verify that hand-crafted packet does not leak uninitialized(?) memory. data = lz4.block.compress(b'A' * 64) message=r'^Decompressor wrote 64 bytes, but 79 bytes expected from header$' - with pytest.raises(ValueError, message=message): + with pytest.raises(ValueError, match=message): lz4.block.decompress(b'\x4f' + data[1:]) - with pytest.raises(ValueError, message=message): + with pytest.raises(ValueError, match=message): lz4.block.decompress(data[4:], uncompressed_size=79) @@ -103,28 +117,25 @@ def test_decompress_truncated(): # lz4.block.decompress(compressed[:i]) # except: # print(i, sys.exc_info()[0], sys.exc_info()[1]) - with pytest.raises(ValueError, message='Input source data size too small'): + with pytest.raises(ValueError, match='Input source data size too small'): lz4.block.decompress(compressed[:0]) - lz4.block.decompress(compressed[:1]) - with pytest.raises(ValueError, message=r'^Corrupt input at byte'): - lz4.block.decompress(compressed[:24]) - lz4.block.decompress(compressed[:25]) - lz4.block.decompress(compressed[:-2]) - with pytest.raises(ValueError, message=r'Decompressor wrote \d+ bytes, but \d+ bytes expected from header)'): - lz4.block.decompress(compressed[:27]) - lz4.block.decompress(compressed[:67]) - lz4.block.decompress(compressed[:85]) + for n in [0, 1]: + with pytest.raises(ValueError, match='Input source data size too small'): + lz4.block.decompress(compressed[:n]) + for n in [24, 25, -2, 27, 67, 85]: + with pytest.raises(ValueError, match=r'Corrupt input at byte \d+'): + lz4.block.decompress(compressed[:n]) def test_decompress_with_trailer(): data = b'A' * 64 comp = lz4.block.compress(data) message=r'^Corrupt input at byte' - with pytest.raises(ValueError, message=message): + with pytest.raises(ValueError, match=message): lz4.block.decompress(comp + b'A') - with pytest.raises(ValueError, message=message): + with pytest.raises(ValueError, match=message): lz4.block.decompress(comp + comp) - with pytest.raises(ValueError, message=message): + with pytest.raises(ValueError, match=message): lz4.block.decompress(comp + comp[4:]) @@ -174,9 +185,9 @@ def test_with_dict(): dict2 = input_data[20:40] for mode in ['default', 'high_compression']: compressed = lz4.block.compress(input_data, mode=mode, dict=dict1) - with pytest.raises(ValueError, message=r'Decompressor wrote \d+ bytes, but \d+ bytes expected from header)'): + with pytest.raises(ValueError, match=r'Corrupt input at byte \d+'): lz4.block.decompress(compressed) - with pytest.raises(ValueError, message=r'Decompressor wrote \d+ bytes, but \d+ bytes expected from header)'): + with pytest.raises(ValueError, match=r'Corrupt input at byte \d+'): lz4.block.decompress(compressed, dict=dict1[:2]) assert lz4.block.decompress(compressed, dict=dict2) != input_data assert lz4.block.decompress(compressed, dict=dict1) == input_data From 327cb8ab4c1d200ba7b13d838e8352df7307cf80 Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Mon, 28 May 2018 17:02:23 -0400 Subject: [PATCH 013/280] fixed indentation and code style mismatches --- lz4/block/_block.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 18675703..47e83a1e 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -95,26 +95,30 @@ typedef enum static int lz4_compress_generic (int comp, char* source, char* dest, size_t source_size, size_t dest_size, - char* dict, size_t dict_size, int acceleration, int compression) + char* dict, size_t dict_size, int acceleration, int compression) { if (comp != HIGH_COMPRESSION) { LZ4_stream_t lz4_state; LZ4_resetStream (&lz4_state); - if (dict) { - LZ4_loadDict (&lz4_state, dict, dict_size); - } + if (dict) + { + LZ4_loadDict (&lz4_state, dict, dict_size); + } if (comp != FAST) - { - acceleration = 1; - } + { + acceleration = 1; + } return LZ4_compress_fast_continue (&lz4_state, source, dest, source_size, dest_size, acceleration); - } else { + } + else + { LZ4_streamHC_t lz4_state; LZ4_resetStreamHC (&lz4_state, compression); - if (dict) { - LZ4_loadDictHC (&lz4_state, dict, dict_size); - } + if (dict) + { + LZ4_loadDictHC (&lz4_state, dict, dict_size); + } return LZ4_compress_HC_continue (&lz4_state, source, dest, source_size, dest_size); } } @@ -195,8 +199,8 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) PyBuffer_Release(&source); PyBuffer_Release(&dict); PyErr_Format (PyExc_ValueError, - "Invalid mode argument: %s. Must be one of: standard, fast, high_compression", - mode); + "Invalid mode argument: %s. Must be one of: standard, fast, high_compression", + mode); return NULL; } @@ -230,8 +234,8 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) } output_size = lz4_compress_generic (comp, source.buf, dest_start, source_size, - dest_size, dict.buf, dict.len, acceleration, - compression); + dest_size, dict.buf, dict.len, acceleration, + compression); Py_END_ALLOW_THREADS @@ -317,7 +321,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) if (source_size < hdr_size) { PyBuffer_Release(&source); - PyBuffer_Release(&dict); + PyBuffer_Release(&dict); PyErr_SetString (PyExc_ValueError, "Input source data size too small"); return NULL; } @@ -345,7 +349,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) output_size = LZ4_decompress_safe_usingDict (source_start, dest, source_size, dest_size, - dict.buf, dict.len); + dict.buf, dict.len); Py_END_ALLOW_THREADS From 84eb19c5294510f612ef60314c5cdbadf7027d9c Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Mon, 28 May 2018 19:04:16 -0400 Subject: [PATCH 014/280] Use int instead of size_t when interacting with LZ4 library API --- lz4/block/_block.c | 54 +++++++++++++++++++++++++++++---------- tests/block/test_block.py | 12 +++++++++ 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 47e83a1e..d48a02e8 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -94,8 +94,8 @@ typedef enum } compression_type; static int -lz4_compress_generic (int comp, char* source, char* dest, size_t source_size, size_t dest_size, - char* dict, size_t dict_size, int acceleration, int compression) +lz4_compress_generic (int comp, char* source, char* dest, int source_size, int dest_size, + char* dict, int dict_size, int acceleration, int compression) { if (comp != HIGH_COMPRESSION) { @@ -134,9 +134,9 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) PyObject *py_dest; char *dest, *dest_start; compression_type comp; - size_t output_size; + int output_size; Py_buffer source; - size_t source_size; + int source_size; int return_bytearray = 0; Py_buffer dict = { NULL, NULL }; static char *argnames[] = { @@ -169,19 +169,26 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) } #endif - source_size = (size_t) source.len; + if (source.len > INT_MAX) + { + PyBuffer_Release(&source); + PyBuffer_Release(&dict); + PyErr_Format(PyExc_OverflowError, + "Input too large for LZ4 API"); + return NULL; + } - /* We're using 4 bytes for the size of the content in the header. This means - we can store a size as large as the maximum value of an unsinged int. */ - if (store_size && source_size > UINT_MAX) + if (dict.len > INT_MAX) { PyBuffer_Release(&source); PyBuffer_Release(&dict); PyErr_Format(PyExc_OverflowError, - "Input too large for storing size in 4 byte header"); + "Dictionary too large for LZ4 API"); return NULL; } + source_size = source.len; + if (!strncmp (mode, "default", sizeof ("default"))) { comp = DEFAULT; @@ -242,7 +249,7 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) PyBuffer_Release(&source); PyBuffer_Release(&dict); - if ((ssize_t)output_size <= 0) + if (output_size <= 0) { PyErr_SetString (PyExc_ValueError, "Compression failed"); PyMem_Free (dest); @@ -278,10 +285,10 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) { Py_buffer source; const char * source_start; - size_t source_size; + int source_size; PyObject *py_dest; char *dest; - size_t output_size; + int output_size; size_t dest_size; int uncompressed_size = -1; int return_bytearray = 0; @@ -309,8 +316,27 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) return NULL; } #endif + + if (source.len > INT_MAX) + { + PyBuffer_Release(&source); + PyBuffer_Release(&dict); + PyErr_Format(PyExc_OverflowError, + "Input too large for LZ4 API"); + return NULL; + } + + if (dict.len > INT_MAX) + { + PyBuffer_Release(&source); + PyBuffer_Release(&dict); + PyErr_Format(PyExc_OverflowError, + "Dictionary too large for LZ4 API"); + return NULL; + } + source_start = (const char *) source.buf; - source_size = (size_t) source.len; + source_size = source.len; if (uncompressed_size >= 0) { @@ -356,7 +382,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) PyBuffer_Release(&source); PyBuffer_Release(&dict); - if ((ssize_t)output_size < 0) + if (output_size < 0) { PyErr_Format (PyExc_ValueError, "Corrupt input at byte %zu", -output_size); PyMem_Free (dest); diff --git a/tests/block/test_block.py b/tests/block/test_block.py index d69bb55e..7fa6a32a 100644 --- a/tests/block/test_block.py +++ b/tests/block/test_block.py @@ -206,3 +206,15 @@ def test_known_decompress(): assert(lz4.block.decompress( b'\xb0\xb3\x00\x00\xff\x1fExcepteur sint occaecat cupidatat non proident.\x00' + (b'\xff' * 180) + b'\x1ePident') == b'Excepteur sint occaecat cupidatat non proident' * 1000) + +#def test_huge(): +# if sys.maxsize > 0xffffffff: +# huge = b'\0' * 0x100000000 +# with pytest.raises(OverflowError, match='Input too large for LZ4 API'): +# lz4.block.compress(huge) +# with pytest.raises(OverflowError, match='Dictionary too large for LZ4 API'): +# lz4.block.compress(b'', dict=huge) +# with pytest.raises(OverflowError, match='Input too large for LZ4 API'): +# lz4.block.decompress(huge) +# with pytest.raises(OverflowError, match='Dictionary too large for LZ4 API'): +# lz4.block.decompress(b'', dict=huge) From e7a37f7b7839f87607baab08fa6db3597bdc49c1 Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Mon, 28 May 2018 23:32:21 -0400 Subject: [PATCH 015/280] allow for the possibility that truncated input may form a valid lz4 stream --- tests/block/test_block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/block/test_block.py b/tests/block/test_block.py index 7fa6a32a..91306739 100644 --- a/tests/block/test_block.py +++ b/tests/block/test_block.py @@ -123,7 +123,7 @@ def test_decompress_truncated(): with pytest.raises(ValueError, match='Input source data size too small'): lz4.block.decompress(compressed[:n]) for n in [24, 25, -2, 27, 67, 85]: - with pytest.raises(ValueError, match=r'Corrupt input at byte \d+'): + with pytest.raises(ValueError, match=r'Corrupt input at byte \d+|Decompressor wrote \d+ bytes, but \d+ bytes expected from header'): lz4.block.decompress(compressed[:n]) From 9f5fcf8a5ef3613949d1ae4c942ea6cc0143d97f Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Tue, 29 May 2018 14:29:18 -0400 Subject: [PATCH 016/280] use correct format specifier after change from size_t to int --- lz4/block/_block.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index d48a02e8..d3094773 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -384,7 +384,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) if (output_size < 0) { - PyErr_Format (PyExc_ValueError, "Corrupt input at byte %zu", -output_size); + PyErr_Format (PyExc_ValueError, "Corrupt input at byte %u", -output_size); PyMem_Free (dest); return NULL; } @@ -392,7 +392,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) { /* Better to fail explicitly than to allow fishy data to pass through. */ PyErr_Format (PyExc_ValueError, - "Decompressor wrote %zu bytes, but %zu bytes expected from header", + "Decompressor wrote %u bytes, but %zu bytes expected from header", output_size, dest_size); PyMem_Free (dest); return NULL; From f568abd531be360b16b4877dc30eb6113d12445d Mon Sep 17 00:00:00 2001 From: Michael Kuyper Date: Tue, 29 May 2018 14:32:38 -0400 Subject: [PATCH 017/280] share code between single- and multithreaded test, add dictionary fixture to multi-threaded test --- tests/block/test_block.py | 71 +++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/tests/block/test_block.py b/tests/block/test_block.py index 91306739..fb6cb4a7 100644 --- a/tests/block/test_block.py +++ b/tests/block/test_block.py @@ -1,6 +1,7 @@ import lz4.block import sys from multiprocessing.pool import ThreadPool +from functools import partial import os import pytest if sys.version_info <= (3, 2): @@ -27,17 +28,27 @@ def get_stored_size(buff): return struct.unpack(' 0xffffffff: -# huge = b'\0' * 0x100000000 +# huge = b'\0' * 0x100000000 # warning: this allocates 4GB of memory! # with pytest.raises(OverflowError, match='Input too large for LZ4 API'): # lz4.block.compress(huge) # with pytest.raises(OverflowError, match='Dictionary too large for LZ4 API'): From 4c0d5dbf60d9cf152da432575f56ed0ca545d3a8 Mon Sep 17 00:00:00 2001 From: jonathanunderwood Date: Sat, 2 Jun 2018 23:59:59 +0100 Subject: [PATCH 018/280] Refactor and add to block tests and clean up some problems * Fix compiler warning for size comparison * Enable or skip test_huge depending on system memory, and/or running on Travis * Make compress_generic inline * Fix some pep8 violations in test_block.py * Fix whitespace issues in conftest.py * Upgrade pip at start of travis builds * Refactor block tests into several files * Ensure psutil is installed for testing * Ensure environment vars are passed through to tox --- .travis.yml | 5 +- lz4/block/_block.c | 20 ++-- setup.py | 17 ++- tests/block/conftest.py | 9 +- tests/block/test_block_0.py | 89 ++++++++++++++ .../block/{test_block.py => test_block_1.py} | 110 +----------------- tests/block/test_block_2.py | 60 ++++++++++ tox.ini | 23 ++-- 8 files changed, 204 insertions(+), 129 deletions(-) create mode 100644 tests/block/test_block_0.py rename tests/block/{test_block.py => test_block_1.py} (62%) create mode 100644 tests/block/test_block_2.py diff --git a/.travis.yml b/.travis.yml index 023acd32..637cde96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,14 @@ python: - 3.5 - 3.6 install: + - pip install -U pip - pip install tox - pip install -r docs/requirements.txt script: + - pip install . + - pip install .[tests] + - pytest tests/block/test_block_2.py - tox - - python setup.py install - make -C docs doctest html deploy: - provider: pypi diff --git a/lz4/block/_block.c b/lz4/block/_block.c index d3094773..37d02dee 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -80,10 +80,6 @@ load_le32 (const char *c) return d[0] | (d[1] << 8) | (d[2] << 16) | (d[3] << 24); } -#ifdef inline -#undef inline -#endif - static const size_t hdr_size = sizeof (uint32_t); typedef enum @@ -93,7 +89,7 @@ typedef enum HIGH_COMPRESSION } compression_type; -static int +static inline int lz4_compress_generic (int comp, char* source, char* dest, int source_size, int dest_size, char* dict, int dict_size, int acceleration, int compression) { @@ -123,6 +119,10 @@ lz4_compress_generic (int comp, char* source, char* dest, int source_size, int d } } +#ifdef inline +#undef inline +#endif + static PyObject * compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) { @@ -285,7 +285,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) { Py_buffer source; const char * source_start; - int source_size; + size_t source_size; PyObject *py_dest; char *dest; int output_size; @@ -444,16 +444,16 @@ PyDoc_STRVAR(compress__doc, " block.\n" \ " return_bytearray (bool): If ``False`` (the default) then the function\n" \ " will return a bytes object. If ``True``, then the function will\n" \ - " return a bytearray object.\n\n" \ + " return a bytearray object.\n\n" \ " dict (str, bytes or buffer-compatible object): If specified, perform\n" \ - " compression using this initial dictionary.\n" \ + " compression using this initial dictionary.\n" \ "Returns:\n" \ " bytes or bytearray: Compressed data.\n"); PyDoc_STRVAR(decompress__doc, - "decompress(source, uncompressed_size=-1, return_bytearray=False)\n\n" \ + "decompress(source, uncompressed_size=-1, return_bytearray=False)\n\n" \ "Decompress source, returning the uncompressed data as a string.\n" \ - "Raises an exception if any error occurs.\n" \ + "Raises an exception if any error occurs.\n" \ "\n" \ "Args:\n" \ " source (str, bytes or buffer-compatible object): Data to decompress.\n" \ diff --git a/setup.py b/setup.py index 7dbc2331..fb2925d7 100644 --- a/setup.py +++ b/setup.py @@ -130,6 +130,16 @@ if sys.version_info < (3, 0): install_requires.append('future') + +# Dependencies for testing. We define a list here, so that we can +# refer to it for the tests_require and the extras_require arguments +# to setup below. The latter enables us to use pip install .[tests] to +# install testing dependencies. +tests_require=[ + 'pytest', + 'psutil', +], + # Finally call setup with the extension modules as defined above. setup( name='lz4', @@ -154,9 +164,10 @@ lz4block, lz4frame ], - tests_require=[ - 'pytest', - ], + tests_require=tests_require, + extras_require={ + 'tests': tests_require, + }, classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: BSD License', diff --git a/tests/block/conftest.py b/tests/block/conftest.py index 9973bf32..c05119cb 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -2,7 +2,8 @@ import os import sys -test_data=[ + +test_data = [ (b''), (os.urandom(8 * 1024)), (b'0' * 8 * 1024), @@ -15,6 +16,7 @@ (memoryview(os.urandom(8 * 1024))) ] + @pytest.fixture( params=test_data, ids=[ @@ -24,6 +26,7 @@ def data(request): return request.param + @pytest.fixture( params=[ ( @@ -58,14 +61,17 @@ def store_size(request): def return_bytearray(request): return request.param + @pytest.fixture def c_return_bytearray(return_bytearray): return return_bytearray + @pytest.fixture def d_return_bytearray(return_bytearray): return return_bytearray + @pytest.fixture( params=[ ('fast', None) @@ -82,6 +88,7 @@ def d_return_bytearray(return_bytearray): def mode(request): return request.param + @pytest.fixture( params=[ None, diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py new file mode 100644 index 00000000..c4bb40d9 --- /dev/null +++ b/tests/block/test_block_0.py @@ -0,0 +1,89 @@ +import lz4.block +from multiprocessing.pool import ThreadPool +import sys +from functools import partial +if sys.version_info <= (3, 2): + import struct + +def get_stored_size(buff): + if sys.version_info > (2, 7): + if isinstance(buff, memoryview): + b = buff.tobytes() + else: + b = bytes(buff) + else: + b = bytes(buff) + + if len(b) < 4: + return None + + if sys.version_info > (3, 2): + return int.from_bytes(b[:4], 'little') + else: + # This would not work on a memoryview object, hence buff.tobytes call + # above + return struct.unpack(' (2, 7): - if isinstance(buff, memoryview): - b = buff.tobytes() - else: - b = bytes(buff) - else: - b = bytes(buff) - - if len(b) < 4: - return None - - if sys.version_info > (3, 2): - return int.from_bytes(b[:4], 'little') - else: - # This would not work on a memoryview object, hence buff.tobytes call - # above - return struct.unpack(' 0xffffffff: -# huge = b'\0' * 0x100000000 # warning: this allocates 4GB of memory! -# with pytest.raises(OverflowError, match='Input too large for LZ4 API'): -# lz4.block.compress(huge) -# with pytest.raises(OverflowError, match='Dictionary too large for LZ4 API'): -# lz4.block.compress(b'', dict=huge) -# with pytest.raises(OverflowError, match='Input too large for LZ4 API'): -# lz4.block.decompress(huge) -# with pytest.raises(OverflowError, match='Dictionary too large for LZ4 API'): -# lz4.block.decompress(b'', dict=huge) diff --git a/tests/block/test_block_2.py b/tests/block/test_block_2.py new file mode 100644 index 00000000..363f0099 --- /dev/null +++ b/tests/block/test_block_2.py @@ -0,0 +1,60 @@ +import pytest +import sys +import lz4.block +import psutil +import os + +# This test requires allocating a big lump of memory. In order to +# avoid a massive memory allocation during byte compilation, we have +# to declare a variable for the size of the buffer we're going to +# create outside the scope of the function below. See: +# https://bugs.python.org/issue21074 +_4GB = 0x100000000 # 4GB + +# This test will be killed on Travis due to the 3GB memory limit +# there. Unfortunately psutil reports the host memory, not the memory +# available to the container, and so can't be used to detect available +# memory, so instead, as an ugly hack for detecting we're on Travis we +# check for the TRAVIS environment variable being set. This is quite +# fragile. +@pytest.mark.skipif( + os.environ.get('TRAVIS') is not None, + reason='Skipping test on Travis due to insufficient memory' +) +@pytest.mark.skipif( + sys.maxsize < 0xffffffff, + reason='Py_ssize_t too small for this test' +) +@pytest.mark.skipif( + psutil.virtual_memory().total < _4GB, + reason='Insufficient system memory for this test' +) +def test_huge(): + try: + huge = b'\0' * _4GB + except MemoryError: + pytest.skip('Insufficient system memory for this test') + + with pytest.raises( + OverflowError, match='Input too large for LZ4 API' + ): + lz4.block.compress(huge) + + with pytest.raises( + OverflowError, match='Dictionary too large for LZ4 API' + ): + lz4.block.compress(b'', dict=huge) + + with pytest.raises( + OverflowError, match='Input too large for LZ4 API' + ): + lz4.block.decompress(huge) + + with pytest.raises( + OverflowError, match='Dictionary too large for LZ4 API' + ): + lz4.block.decompress(b'', dict=huge) + + +def test_dummy(): + pass diff --git a/tox.ini b/tox.ini index a5d50251..43c08a91 100644 --- a/tox.ini +++ b/tox.ini @@ -2,16 +2,19 @@ envlist = py [testenv] -deps=pytest +deps=.[tests] +passenv = * # setenv = PYTHONMALLOC = pymalloc # PYTHONMALLOCSTATS = 'yes' commands= - pytest tests/block/test_block.py - pytest tests/frame/test_frame_0.py - pytest tests/frame/test_frame_1.py - pytest tests/frame/test_frame_2.py - pytest tests/frame/test_frame_3.py - pytest tests/frame/test_frame_4.py - pytest tests/frame/test_frame_5.py - pytest tests/frame/test_frame_6.py - pytest tests/frame/test_frame_7.py + pytest --tb=long -l tests/block/test_block_0.py + pytest --tb=long -l tests/block/test_block_1.py + pytest --tb=long -l tests/block/test_block_2.py + pytest --tb=long -l tests/frame/test_frame_0.py + pytest --tb=long -l tests/frame/test_frame_1.py + pytest --tb=long -l tests/frame/test_frame_2.py + pytest --tb=long -l tests/frame/test_frame_3.py + pytest --tb=long -l tests/frame/test_frame_4.py + pytest --tb=long -l tests/frame/test_frame_5.py + pytest --tb=long -l tests/frame/test_frame_6.py + pytest --tb=long -l tests/frame/test_frame_7.py From 5c943c762f982246615ac853c34193a6d34df283 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:06:07 +0100 Subject: [PATCH 019/280] Fix some flake8 complaints --- lz4/__init__.py | 4 ++-- lz4/block/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lz4/__init__.py b/lz4/__init__.py index 32e85f5a..15701bb8 100644 --- a/lz4/__init__.py +++ b/lz4/__init__.py @@ -1,4 +1,3 @@ -# Package version info, generated on install from pkg_resources import get_distribution, DistributionNotFound try: __version__ = get_distribution(__name__).version @@ -8,7 +7,8 @@ VERSION = __version__ -from ._version import ( + +from ._version import ( # noqa: F401 library_version_number, library_version_string, ) diff --git a/lz4/block/__init__.py b/lz4/block/__init__.py index b1b13f9d..2f4ff026 100644 --- a/lz4/block/__init__.py +++ b/lz4/block/__init__.py @@ -1 +1 @@ -from ._block import compress, decompress +from ._block import compress, decompress # noqa: F401 From 56a62283b23adc5bb087203852941dc6e1a8fa62 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:08:47 +0100 Subject: [PATCH 020/280] Run autopep8 over tests files --- tests/bench.py | 15 ++++++++++----- tests/block/conftest.py | 1 + tests/block/test_block_0.py | 7 +++++-- tests/block/test_block_1.py | 29 ++++++++++++++++++++--------- tests/block/test_block_2.py | 6 ++++-- tests/frame/conftest.py | 24 ++++++++++++++++++------ tests/frame/helpers.py | 2 +- tests/frame/test_frame_0.py | 19 ++++++++++++++++++- tests/frame/test_frame_1.py | 9 ++++++--- tests/frame/test_frame_2.py | 2 +- tests/frame/test_frame_3.py | 8 +++++--- tests/frame/test_frame_4.py | 8 ++++++-- tests/frame/test_frame_5.py | 5 ++--- tests/frame/test_frame_6.py | 17 ++++++++++++----- tests/frame/test_frame_7.py | 10 ++++++++-- 15 files changed, 117 insertions(+), 45 deletions(-) diff --git a/tests/bench.py b/tests/bench.py index f99151ba..58862c61 100644 --- a/tests/bench.py +++ b/tests/bench.py @@ -14,11 +14,16 @@ print("Data Size:") print(" Input: %d" % len(DATA)) print(" LZ4: %d (%.2f)" % (len(LZ4_DATA), len(LZ4_DATA) / float(len(DATA)))) -print(" Blosc: %d (%.2f)" % (len(BLOSC_DATA), len(BLOSC_DATA) / float(len(DATA)))) +print(" Blosc: %d (%.2f)" % + (len(BLOSC_DATA), len(BLOSC_DATA) / float(len(DATA)))) print(" LZ4 / Blosc: %f" % (float(len(LZ4_DATA)) / float(len(BLOSC_DATA)))) print("Benchmark: %d calls" % LOOPS) -print(" LZ4 Compression: %fs" % (Timer("lz4.block.compress(DATA)", "from __main__ import DATA; import lz4").timeit(number=LOOPS)/LOOPS)) -print(" Blosc Compression: %fs" % (Timer("blosc.compress(DATA, cname='lz4', clevel=5, shuffle=True)", "from __main__ import DATA; import blosc").timeit(number=LOOPS)/LOOPS)) -print(" LZ4 Decompression: %fs" % (Timer("lz4.block.decompress(LZ4_DATA)", "from __main__ import LZ4_DATA; import lz4").timeit(number=LOOPS)/LOOPS)) -print(" Blosc Decompression : %fs" % (Timer("blosc.decompress(BLOSC_DATA)", "from __main__ import BLOSC_DATA; import blosc").timeit(number=LOOPS)/LOOPS)) +print(" LZ4 Compression: %fs" % (Timer("lz4.block.compress(DATA)", + "from __main__ import DATA; import lz4").timeit(number=LOOPS)/LOOPS)) +print(" Blosc Compression: %fs" % (Timer("blosc.compress(DATA, cname='lz4', clevel=5, shuffle=True)", + "from __main__ import DATA; import blosc").timeit(number=LOOPS)/LOOPS)) +print(" LZ4 Decompression: %fs" % (Timer("lz4.block.decompress(LZ4_DATA)", + "from __main__ import LZ4_DATA; import lz4").timeit(number=LOOPS)/LOOPS)) +print(" Blosc Decompression : %fs" % (Timer("blosc.decompress(BLOSC_DATA)", + "from __main__ import BLOSC_DATA; import blosc").timeit(number=LOOPS)/LOOPS)) diff --git a/tests/block/conftest.py b/tests/block/conftest.py index c05119cb..8eab729d 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -44,6 +44,7 @@ def data(request): def store_size(request): return request.param + @pytest.fixture( params=[ ( diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index c4bb40d9..cca3e65b 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -5,6 +5,7 @@ if sys.version_info <= (3, 2): import struct + def get_stored_size(buff): if sys.version_info > (2, 7): if isinstance(buff, memoryview): @@ -67,7 +68,8 @@ def setup_kwargs(mode, store_size, c_return_bytearray=None, d_return_bytearray=N # Test single threaded usage with all valid variations of input def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dictionary): - (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size, c_return_bytearray, d_return_bytearray) + (c_kwargs, d_kwargs) = setup_kwargs( + mode, store_size, c_return_bytearray, d_return_bytearray) d = roundtrip(data, c_kwargs, d_kwargs, dictionary) @@ -83,7 +85,8 @@ def test_2(data, mode, store_size, dictionary): data_in = [data for i in range(32)] pool = ThreadPool(8) - rt = partial(roundtrip, c_kwargs=c_kwargs, d_kwargs=d_kwargs, dictionary=dictionary) + rt = partial(roundtrip, c_kwargs=c_kwargs, + d_kwargs=d_kwargs, dictionary=dictionary) data_out = pool.map(rt, data_in) pool.close() assert data_in == data_out diff --git a/tests/block/test_block_1.py b/tests/block/test_block_1.py index dc22da77..0f5de25e 100644 --- a/tests/block/test_block_1.py +++ b/tests/block/test_block_1.py @@ -3,6 +3,7 @@ import sys import os + def test_decompress_ui32_overflow(): data = lz4.block.compress(b'A' * 64) with pytest.raises(OverflowError): @@ -40,7 +41,7 @@ def test_decompress_truncated(): def test_decompress_with_trailer(): data = b'A' * 64 comp = lz4.block.compress(data) - message=r'^Corrupt input at byte' + message = r'^Corrupt input at byte' with pytest.raises(ValueError, match=message): lz4.block.decompress(comp + b'A') with pytest.raises(ValueError, match=message): @@ -54,11 +55,13 @@ def test_unicode(): return # skip DATA = b'x' with pytest.raises(TypeError): - lz4.block.compress (DATA.decode('latin1')) + lz4.block.compress(DATA.decode('latin1')) lz4.block.decompress(lz4.block.compress(DATA).decode('latin1')) # These next two are probably redundant given test_1 above but we'll keep them # for now + + def test_return_bytearray(): if sys.version_info < (3,): return # skip @@ -80,15 +83,22 @@ def test_memoryview(): assert lz4.block.compress(memoryview(data)) == compressed assert lz4.block.decompress(memoryview(compressed)) == data + def test_with_dict_none(): input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 for mode in ['default', 'high_compression']: - assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode, dict=None)) == input_data - assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode), dict=None) == input_data - assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode, dict=b'')) == input_data - assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode), dict=b'') == input_data - assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode, dict='')) == input_data - assert lz4.block.decompress(lz4.block.compress(input_data, mode=mode), dict='') == input_data + assert lz4.block.decompress(lz4.block.compress( + input_data, mode=mode, dict=None)) == input_data + assert lz4.block.decompress(lz4.block.compress( + input_data, mode=mode), dict=None) == input_data + assert lz4.block.decompress(lz4.block.compress( + input_data, mode=mode, dict=b'')) == input_data + assert lz4.block.decompress(lz4.block.compress( + input_data, mode=mode), dict=b'') == input_data + assert lz4.block.decompress(lz4.block.compress( + input_data, mode=mode, dict='')) == input_data + assert lz4.block.decompress(lz4.block.compress( + input_data, mode=mode), dict='') == input_data def test_with_dict(): @@ -103,7 +113,8 @@ def test_with_dict(): lz4.block.decompress(compressed, dict=dict1[:2]) assert lz4.block.decompress(compressed, dict=dict2) != input_data assert lz4.block.decompress(compressed, dict=dict1) == input_data - assert lz4.block.decompress(lz4.block.compress(input_data), dict=dict1) == input_data + assert lz4.block.decompress(lz4.block.compress( + input_data), dict=dict1) == input_data def test_known_decompress(): diff --git a/tests/block/test_block_2.py b/tests/block/test_block_2.py index 363f0099..87ceefb7 100644 --- a/tests/block/test_block_2.py +++ b/tests/block/test_block_2.py @@ -17,6 +17,8 @@ # memory, so instead, as an ugly hack for detecting we're on Travis we # check for the TRAVIS environment variable being set. This is quite # fragile. + + @pytest.mark.skipif( os.environ.get('TRAVIS') is not None, reason='Skipping test on Travis due to insufficient memory' @@ -34,7 +36,7 @@ def test_huge(): huge = b'\0' * _4GB except MemoryError: pytest.skip('Insufficient system memory for this test') - + with pytest.raises( OverflowError, match='Input too large for LZ4 API' ): @@ -53,7 +55,7 @@ def test_huge(): with pytest.raises( OverflowError, match='Dictionary too large for LZ4 API' ): - lz4.block.decompress(b'', dict=huge) + lz4.block.decompress(b'', dict=huge) def test_dummy(): diff --git a/tests/frame/conftest.py b/tests/frame/conftest.py index 3c721a07..5ab52c0a 100644 --- a/tests/frame/conftest.py +++ b/tests/frame/conftest.py @@ -2,9 +2,10 @@ import lz4.frame as lz4frame import lz4 + @pytest.fixture( params=[ -# (lz4frame.BLOCKSIZE_DEFAULT), + # (lz4frame.BLOCKSIZE_DEFAULT), (lz4frame.BLOCKSIZE_MAX64KB), (lz4frame.BLOCKSIZE_MAX256KB), (lz4frame.BLOCKSIZE_MAX1MB), @@ -14,6 +15,7 @@ def block_size(request): return request.param + @pytest.fixture( params=[ (True), @@ -23,6 +25,7 @@ def block_size(request): def block_linked(request): return request.param + @pytest.fixture( params=[ (True), @@ -32,29 +35,36 @@ def block_linked(request): def content_checksum(request): return request.param + if lz4.library_version_number() >= 10800: p = [True, False] else: p = [False, ] + + @pytest.fixture( params=[ (pp) for pp in p - ] + ] ) def block_checksum(request): return request.param + compression_levels = [ - (lz4frame.COMPRESSIONLEVEL_MIN), - (lz4frame.COMPRESSIONLEVEL_MINHC), - (lz4frame.COMPRESSIONLEVEL_MAX), - ] + (lz4frame.COMPRESSIONLEVEL_MIN), + (lz4frame.COMPRESSIONLEVEL_MINHC), + (lz4frame.COMPRESSIONLEVEL_MAX), +] + + @pytest.fixture( params=compression_levels ) def compression_level(request): return request.param + @pytest.fixture( params=[ (True), @@ -64,6 +74,7 @@ def compression_level(request): def auto_flush(request): return request.param + @pytest.fixture( params=[ (True), @@ -73,6 +84,7 @@ def auto_flush(request): def store_size(request): return request.param + @pytest.fixture( params=[ (True), diff --git a/tests/frame/helpers.py b/tests/frame/helpers.py index 11cf622b..efe18d91 100644 --- a/tests/frame/helpers.py +++ b/tests/frame/helpers.py @@ -34,7 +34,7 @@ def get_frame_info_check(compressed_data, def get_chunked(data, nchunks): size = len(data) - stride = int(math.ceil(float(size)/nchunks)) # no // on py 2.6 + stride = int(math.ceil(float(size)/nchunks)) # no // on py 2.6 start = 0 end = start + stride while end < size: diff --git a/tests/frame/test_frame_0.py b/tests/frame/test_frame_0.py index 7706a5c4..e65d27c9 100644 --- a/tests/frame/test_frame_0.py +++ b/tests/frame/test_frame_0.py @@ -2,26 +2,31 @@ import lz4 import re + def test_library_version_number(): v = lz4.library_version_number() assert isinstance(v, int) assert v > 10000 + def test_library_version_string(): v = lz4.library_version_string() assert isinstance(v, str) assert v.count('.') == 2 - r=re.compile(r'^[0-9]*\.[0-9]*\.[0-9]*$') + r = re.compile(r'^[0-9]*\.[0-9]*\.[0-9]*$') assert r.match(v) is not None + def test_create_compression_context(): context = lz4frame.create_compression_context() assert context is not None + def test_create_decompression_context(): context = lz4frame.create_decompression_context() assert context is not None + def test_reset_decompression_context_1(): if lz4.library_version_number() >= 10800: context = lz4frame.create_decompression_context() @@ -30,6 +35,7 @@ def test_reset_decompression_context_1(): else: pass + def test_reset_decompression_context_2(): if lz4.library_version_number() >= 10800: c = lz4frame.compress(b'1234', return_bytearray=False) @@ -49,14 +55,17 @@ def test_reset_decompression_context_2(): else: pass + def test_compress_return_type_1(): r = lz4frame.compress(b'', return_bytearray=False) assert isinstance(r, bytes) + def test_compress_return_type_2(): r = lz4frame.compress(b'', return_bytearray=True) assert isinstance(r, bytearray) + def test_decompress_return_type_1(): c = lz4frame.compress(b'', return_bytearray=False) r = lz4frame.decompress( @@ -66,6 +75,7 @@ def test_decompress_return_type_1(): ) assert isinstance(r, bytes) + def test_decompress_return_type_2(): c = lz4frame.compress(b'', return_bytearray=False) r = lz4frame.decompress( @@ -75,6 +85,7 @@ def test_decompress_return_type_2(): ) assert isinstance(r, bytearray) + def test_decompress_return_type_3(): c = lz4frame.compress(b'', return_bytearray=False) r = lz4frame.decompress( @@ -86,6 +97,7 @@ def test_decompress_return_type_3(): assert isinstance(r[0], bytes) assert isinstance(r[1], int) + def test_decompress_return_type_4(): c = lz4frame.compress(b'', return_bytearray=False) r = lz4frame.decompress( @@ -97,6 +109,7 @@ def test_decompress_return_type_4(): assert isinstance(r[0], bytearray) assert isinstance(r[1], int) + def test_decompress_chunk_return_type_1(): c = lz4frame.compress(b'', return_bytearray=False) d = lz4frame.create_decompression_context() @@ -109,6 +122,7 @@ def test_decompress_chunk_return_type_1(): assert isinstance(b, int) assert isinstance(e, bool) + def test_decompress_chunk_return_type_2(): c = lz4frame.compress(b'', return_bytearray=False) d = lz4frame.create_decompression_context() @@ -121,6 +135,7 @@ def test_decompress_chunk_return_type_2(): assert isinstance(b, int) assert isinstance(e, bool) + def test_decompress_chunk_return_type_3(): c = lz4frame.compress(b'', return_bytearray=False) d = lz4frame.create_decompression_context() @@ -134,6 +149,7 @@ def test_decompress_chunk_return_type_3(): assert isinstance(r[1], int) assert isinstance(r[2], bool) + def test_decompress_chunk_return_type_4(): c = lz4frame.compress(b'', return_bytearray=False) d = lz4frame.create_decompression_context() @@ -147,6 +163,7 @@ def test_decompress_chunk_return_type_4(): assert isinstance(r[1], int) assert isinstance(r[2], bool) + def test_block_size_constants(): assert lz4frame.BLOCKSIZE_DEFAULT == 0 assert lz4frame.BLOCKSIZE_MAX64KB == 4 diff --git a/tests/frame/test_frame_1.py b/tests/frame/test_frame_1.py index 444b6e8c..35110c44 100644 --- a/tests/frame/test_frame_1.py +++ b/tests/frame/test_frame_1.py @@ -5,7 +5,7 @@ from .helpers import get_frame_info_check -test_data=[ +test_data = [ (b''), (os.urandom(8 * 1024)), (b'0' * 8 * 1024), @@ -21,6 +21,7 @@ (memoryview(os.urandom(8 * 1024))) ] + @pytest.fixture( params=test_data, ids=[ @@ -59,7 +60,8 @@ def test_roundtrip_1( content_checksum, block_checksum, ) - decompressed, bytes_read = lz4frame.decompress(compressed, return_bytes_read=True) + decompressed, bytes_read = lz4frame.decompress( + compressed, return_bytes_read=True) assert bytes_read == len(compressed) assert decompressed == data @@ -103,6 +105,7 @@ def test_roundtrip_2(data, content_checksum, block_checksum, ) - decompressed, bytes_read = lz4frame.decompress(compressed, return_bytes_read=True) + decompressed, bytes_read = lz4frame.decompress( + compressed, return_bytes_read=True) assert bytes_read == len(compressed) assert decompressed == data diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index 3a0e6d6c..b47a1a72 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -8,7 +8,7 @@ ) -test_data=[ +test_data = [ (b'', 1, 1), (os.urandom(8 * 1024), 8, 1), (os.urandom(8 * 1024), 1, 8), diff --git a/tests/frame/test_frame_3.py b/tests/frame/test_frame_3.py index ee1a554a..aa7c782d 100644 --- a/tests/frame/test_frame_3.py +++ b/tests/frame/test_frame_3.py @@ -3,10 +3,11 @@ import os import struct -test_data=[ +test_data = [ (os.urandom(256 * 1024)), ] + @pytest.fixture( params=test_data, ids=[ @@ -24,8 +25,9 @@ def test_decompress_truncated(data): with pytest.raises(RuntimeError, message=message): lz4frame.decompress(compressed[:6]) - for i in range(16, len(compressed) - 1, 5): # 15 is the max size of the header - message = r'^Frame incomplete. LZ4F_decompress returned: {0}'.format(len(compressed) - i) + for i in range(16, len(compressed) - 1, 5): # 15 is the max size of the header + message = r'^Frame incomplete. LZ4F_decompress returned: {0}'.format( + len(compressed) - i) try: lz4frame.decompress(compressed[:i]) except RuntimeError as r: diff --git a/tests/frame/test_frame_4.py b/tests/frame/test_frame_4.py index 490f6376..7fa16547 100644 --- a/tests/frame/test_frame_4.py +++ b/tests/frame/test_frame_4.py @@ -6,7 +6,7 @@ get_chunked, ) -test_data=[ +test_data = [ b'', (128 * (32 * os.urandom(32))), (256 * (32 * os.urandom(32))), @@ -14,6 +14,7 @@ (1024 * (32 * os.urandom(32))), ] + @pytest.fixture( params=test_data, ids=[ @@ -23,6 +24,7 @@ def data(request): return request.param + @pytest.fixture( params=[ (True), @@ -32,6 +34,7 @@ def data(request): def reset(request): return request.param + @pytest.fixture( params=[ (1), @@ -86,7 +89,8 @@ def do_compress(): block_checksum, ) - decompressed, bytes_read = lz4frame.decompress(compressed, return_bytes_read=True) + decompressed, bytes_read = lz4frame.decompress( + compressed, return_bytes_read=True) assert data == decompressed assert bytes_read == len(compressed) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 9d5c72dc..d959edd8 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -2,10 +2,11 @@ import time import pytest -test_data=[ +test_data = [ (b'a' * 1024 * 1024), ] + @pytest.fixture( params=test_data, ids=[ @@ -88,5 +89,3 @@ def test_dummy_always_pass(): # If pytest finds all tests are skipped, then it exits with code 5 rather # than 0, which tox sees as an error. Here we add a dummy test that always passes. assert True - - diff --git a/tests/frame/test_frame_6.py b/tests/frame/test_frame_6.py index 8608f675..09e2acd6 100644 --- a/tests/frame/test_frame_6.py +++ b/tests/frame/test_frame_6.py @@ -2,12 +2,13 @@ import pytest import lz4.frame as lz4frame -test_data=[ +test_data = [ b'', (128 * (32 * os.urandom(32))), (5 * 128 * os.urandom(1024)), ] + @pytest.fixture( params=test_data, ids=[ @@ -17,11 +18,14 @@ def data(request): return request.param + compression_levels = [ - (lz4frame.COMPRESSIONLEVEL_MIN), - # (lz4frame.COMPRESSIONLEVEL_MINHC), - # (lz4frame.COMPRESSIONLEVEL_MAX), - ] + (lz4frame.COMPRESSIONLEVEL_MIN), + # (lz4frame.COMPRESSIONLEVEL_MINHC), + # (lz4frame.COMPRESSIONLEVEL_MAX), +] + + @pytest.fixture( params=compression_levels ) @@ -33,6 +37,7 @@ def test_lz4frame_open_write(data): with lz4frame.open('testfile', mode='wb') as fp: fp.write(data) + def test_lz4frame_open_write_read_defaults(data): with lz4frame.open('testfile', mode='wb') as fp: fp.write(data) @@ -40,6 +45,7 @@ def test_lz4frame_open_write_read_defaults(data): data_out = fp.read() assert data_out == data + def test_lz4frame_open_write_read_text(): data = u'This is a test string' with lz4frame.open('testfile', mode='wt') as fp: @@ -48,6 +54,7 @@ def test_lz4frame_open_write_read_text(): data_out = fp.read() assert data_out == data + def test_lz4frame_open_write_read( data, compression_level, diff --git a/tests/frame/test_frame_7.py b/tests/frame/test_frame_7.py index 1ec48942..73f7e64a 100644 --- a/tests/frame/test_frame_7.py +++ b/tests/frame/test_frame_7.py @@ -2,10 +2,11 @@ import pytest import os -test_data=[ +test_data = [ (os.urandom(32) * 256), ] + @pytest.fixture( params=test_data, ids=[ @@ -15,6 +16,7 @@ def data(request): return request.param + def test_roundtrip_multiframe_1(data): nframes = 4 @@ -29,6 +31,7 @@ def test_roundtrip_multiframe_1(data): assert len(decompressed) == nframes * len(data) assert data * nframes == decompressed + def test_roundtrip_multiframe_2(data): nframes = 4 @@ -46,6 +49,7 @@ def test_roundtrip_multiframe_2(data): assert len(decompressed) == nframes * len(data) assert data * nframes == decompressed + def test_roundtrip_multiframe_3(data): nframes = 4 @@ -67,6 +71,7 @@ def test_roundtrip_multiframe_3(data): assert len(decompressed) == nframes * len(data) assert data * nframes == decompressed + def test_roundtrip_multiframe_4(data): nframes = 4 @@ -90,7 +95,8 @@ def test_roundtrip_multiframe_4(data): if i == nframes - 1: assert decompressor.unused_data == None else: - assert len(decompressor.unused_data) == len(compressed) * (nframes - i - 1) / nframes + assert len(decompressor.unused_data) == len( + compressed) * (nframes - i - 1) / nframes assert len(decompressed) == nframes * len(data) assert data * nframes == decompressed From 4547f0608979d78a893d4f1fa227c32c1224a0a1 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:18:59 +0100 Subject: [PATCH 021/280] Fix some more pep8 errors --- tests/frame/helpers.py | 2 +- tests/frame/test_frame_0.py | 2 +- tests/frame/test_frame_2.py | 4 ++-- tests/frame/test_frame_5.py | 7 ++++--- tests/frame/test_frame_7.py | 8 ++++---- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/frame/helpers.py b/tests/frame/helpers.py index efe18d91..39debbe8 100644 --- a/tests/frame/helpers.py +++ b/tests/frame/helpers.py @@ -15,7 +15,7 @@ def get_frame_info_check(compressed_data, assert frame_info["content_checksum"] == content_checksum assert frame_info["block_checksum"] == block_checksum - assert frame_info["skippable"] == False + assert frame_info["skippable"] is False if store_size is True: assert frame_info["content_size"] == source_size diff --git a/tests/frame/test_frame_0.py b/tests/frame/test_frame_0.py index e65d27c9..23f1c97b 100644 --- a/tests/frame/test_frame_0.py +++ b/tests/frame/test_frame_0.py @@ -51,7 +51,7 @@ def test_reset_decompression_context_2(): d, bytes_read, eof = lz4frame.decompress_chunk(context, c) assert d == b'1234' assert bytes_read == len(c) - assert eof == True + assert eof is True else: pass diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index b47a1a72..80b44b87 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -103,5 +103,5 @@ def test_roundtrip_chunked(data, block_size, block_linked, assert bytes_read == len(compressed) assert decompressed == data - assert eofs[-1] == True - assert (True in eofs[:-2]) == False + assert eofs[-1] is True + assert (True in eofs[:-2]) is False diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index d959edd8..a2f11323 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -1,5 +1,4 @@ import lz4.frame -import time import pytest test_data = [ @@ -26,7 +25,7 @@ def test_frame_decompress_mem_usage(data): prev_snapshot = None for i in range(1000): - decompressed = lz4.frame.decompress(compressed) + decompressed = lz4.frame.decompress(compressed) # noqa: F841 if i % 100 == 0: snapshot = tracemalloc.take_snapshot() @@ -48,7 +47,9 @@ def test_frame_decompress_chunk_mem_usage(data): for i in range(1000): context = lz4.frame.create_decompression_context() - decompressed = lz4.frame.decompress_chunk(context, compressed) + decompressed = lz4.frame.decompress_chunk( # noqa: F841 + context, compressed + ) if i % 100 == 0: snapshot = tracemalloc.take_snapshot() diff --git a/tests/frame/test_frame_7.py b/tests/frame/test_frame_7.py index 73f7e64a..583f3fbb 100644 --- a/tests/frame/test_frame_7.py +++ b/tests/frame/test_frame_7.py @@ -65,7 +65,7 @@ def test_roundtrip_multiframe_3(data): for _ in range(nframes): d, bytes_read, eof = lz4frame.decompress_chunk(ctx, compressed) decompressed += d - assert eof == True + assert eof is True assert bytes_read == len(compressed) // nframes assert len(decompressed) == nframes * len(data) @@ -90,10 +90,10 @@ def test_roundtrip_multiframe_4(data): else: d = decompressor.unused_data decompressed += decompressor.decompress(d) - assert decompressor.eof == True - assert decompressor.needs_input == True + assert decompressor.eof is True + assert decompressor.needs_input is True if i == nframes - 1: - assert decompressor.unused_data == None + assert decompressor.unused_data is None else: assert len(decompressor.unused_data) == len( compressed) * (nframes - i - 1) / nframes From 0540145e4602535ed77a6002867d939f33de46dc Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:20:33 +0100 Subject: [PATCH 022/280] Remove bench.py --- tests/bench.py | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 tests/bench.py diff --git a/tests/bench.py b/tests/bench.py deleted file mode 100644 index 58862c61..00000000 --- a/tests/bench.py +++ /dev/null @@ -1,29 +0,0 @@ -import uuid -import timeit -import lz4 -import os -from timeit import Timer -import sys -import blosc - -DATA = open(sys.argv[1], "rb").read() -LZ4_DATA = lz4.block.compress(DATA) -BLOSC_DATA = blosc.compress(DATA, cname='lz4', clevel=5, shuffle=True) -LOOPS = 100 - -print("Data Size:") -print(" Input: %d" % len(DATA)) -print(" LZ4: %d (%.2f)" % (len(LZ4_DATA), len(LZ4_DATA) / float(len(DATA)))) -print(" Blosc: %d (%.2f)" % - (len(BLOSC_DATA), len(BLOSC_DATA) / float(len(DATA)))) -print(" LZ4 / Blosc: %f" % (float(len(LZ4_DATA)) / float(len(BLOSC_DATA)))) - -print("Benchmark: %d calls" % LOOPS) -print(" LZ4 Compression: %fs" % (Timer("lz4.block.compress(DATA)", - "from __main__ import DATA; import lz4").timeit(number=LOOPS)/LOOPS)) -print(" Blosc Compression: %fs" % (Timer("blosc.compress(DATA, cname='lz4', clevel=5, shuffle=True)", - "from __main__ import DATA; import blosc").timeit(number=LOOPS)/LOOPS)) -print(" LZ4 Decompression: %fs" % (Timer("lz4.block.decompress(LZ4_DATA)", - "from __main__ import LZ4_DATA; import lz4").timeit(number=LOOPS)/LOOPS)) -print(" Blosc Decompression : %fs" % (Timer("blosc.decompress(BLOSC_DATA)", - "from __main__ import BLOSC_DATA; import blosc").timeit(number=LOOPS)/LOOPS)) From b132a77facae039e66adc7380a7490ad01912be4 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:33:58 +0100 Subject: [PATCH 023/280] Fix some more pep8 errors --- tests/frame/helpers.py | 4 ++-- tests/frame/test_frame_0.py | 4 ++-- tests/frame/test_frame_5.py | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/frame/helpers.py b/tests/frame/helpers.py index 39debbe8..e6cb0c9e 100644 --- a/tests/frame/helpers.py +++ b/tests/frame/helpers.py @@ -1,5 +1,4 @@ import lz4.frame as lz4frame -import math def get_frame_info_check(compressed_data, @@ -34,7 +33,8 @@ def get_frame_info_check(compressed_data, def get_chunked(data, nchunks): size = len(data) - stride = int(math.ceil(float(size)/nchunks)) # no // on py 2.6 + # stride = int(math.ceil(float(size)/nchunks)) # no // on py 2.6 + stride = size // nchunks start = 0 end = start + stride while end < size: diff --git a/tests/frame/test_frame_0.py b/tests/frame/test_frame_0.py index 23f1c97b..f03431d4 100644 --- a/tests/frame/test_frame_0.py +++ b/tests/frame/test_frame_0.py @@ -42,8 +42,8 @@ def test_reset_decompression_context_2(): context = lz4frame.create_decompression_context() try: # Simulate an error by passing junk to decompress - d = lz4frame.decompress_chunk(context, c[1:3]) - except: + d = lz4frame.decompress_chunk(context, c[4:]) + except RuntimeError: pass r = lz4frame.reset_decompression_context(context) assert r is None diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index a2f11323..b61aec73 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -72,7 +72,7 @@ def test_frame_open_decompress_mem_usage(data): for i in range(1000): with lz4.frame.open('test.lz4', 'r') as f: - decompressed = f.read() + decompressed = f.read() # noqa: F841 if i % 100 == 0: snapshot = tracemalloc.take_snapshot() @@ -84,7 +84,8 @@ def test_frame_open_decompress_mem_usage(data): prev_snapshot = snapshot -# TODO: add many more memory usage tests along the lines of this one for other funcs +# TODO: add many more memory usage tests along the lines of this one +# for other funcs def test_dummy_always_pass(): # If pytest finds all tests are skipped, then it exits with code 5 rather From 3d35ce3a06ca89a90ea56203a859dc7c0e1c37d4 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:38:17 +0100 Subject: [PATCH 024/280] Run flake8 on files on travis builds --- .travis.yml | 6 ++++-- setup.py | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 637cde96..afe81006 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,13 @@ install: - pip install tox - pip install -r docs/requirements.txt script: + - tox - pip install . - pip install .[tests] - - pytest tests/block/test_block_2.py - - tox - make -C docs doctest html + - pip install .[flake8] + - flake8 --ignore=E501 lz4 + - flake8 --ignore=E501 tests deploy: - provider: pypi # server: https://test.pypi.org/legacy/ diff --git a/setup.py b/setup.py index fb2925d7..4e682a03 100644 --- a/setup.py +++ b/setup.py @@ -167,6 +167,9 @@ tests_require=tests_require, extras_require={ 'tests': tests_require, + 'flake8': [ + 'flake8', + ] }, classifiers=[ 'Development Status :: 5 - Production/Stable', From 63abd281f7995a41538e4b70f58a9e5d9881788d Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:45:54 +0100 Subject: [PATCH 025/280] Fix a flake8 error --- lz4/frame/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 28e8e45a..5fa03ce6 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -396,7 +396,7 @@ def decompress(self, data, max_length=-1): # noqa: F811 data, max_length=max_length, return_bytearray=self._return_bytearray, - ) + ) if bytes_read < len(data): if eoframe: From 49d549a9b058142e733561efb8ef2cb18380a744 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:46:53 +0100 Subject: [PATCH 026/280] Run flake8 checks ahead of tox --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index afe81006..42bce57b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,13 +9,13 @@ install: - pip install tox - pip install -r docs/requirements.txt script: + - pip install .[flake8] + - flake8 --ignore=E501 lz4 + - flake8 --ignore=E501 tests - tox - pip install . - pip install .[tests] - make -C docs doctest html - - pip install .[flake8] - - flake8 --ignore=E501 lz4 - - flake8 --ignore=E501 tests deploy: - provider: pypi # server: https://test.pypi.org/legacy/ From 95c8b6dd38240a0992c69f0107caa4ead344ee88 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:51:49 +0100 Subject: [PATCH 027/280] Fix some flake8 issue in setup.py --- setup.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 4e682a03..73a8e2ae 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ #!/usr/bin/env python from setuptools import setup, find_packages, Extension -import subprocess -import os import sys from distutils import ccompiler @@ -104,26 +102,23 @@ extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, libraries=libraries, - include_dirs=include_dirs, -) + include_dirs=include_dirs) lz4block = Extension('lz4.block._block', lz4block_sources, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, libraries=libraries, - include_dirs=include_dirs, -) + include_dirs=include_dirs) lz4frame = Extension('lz4.frame._frame', lz4frame_sources, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, libraries=libraries, - include_dirs=include_dirs, -) + include_dirs=include_dirs) -install_requires=[] +install_requires = [] # On Python earlier than 3.0 the builtins package isn't included, but it is # provided by the future package @@ -135,7 +130,7 @@ # refer to it for the tests_require and the extras_require arguments # to setup below. The latter enables us to use pip install .[tests] to # install testing dependencies. -tests_require=[ +tests_require = [ 'pytest', 'psutil', ], From 62b1ffa68bf6be34c8f61bce05545c7ed57f0a9c Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 3 Jun 2018 00:52:36 +0100 Subject: [PATCH 028/280] Run flake8 over setup.py on Travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 42bce57b..cc722345 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ script: - pip install .[flake8] - flake8 --ignore=E501 lz4 - flake8 --ignore=E501 tests + - flake8 --ignore=E501 setup.py - tox - pip install . - pip install .[tests] From d6133eb4f2b7c7338ce960ba58cd85bf2ef71bfc Mon Sep 17 00:00:00 2001 From: Carl Lei Date: Mon, 4 Jun 2018 11:34:06 +0800 Subject: [PATCH 029/280] Compatibility fix for 2.x memoryview objects --- lz4/frame/_compression.py | 11 +++++++++++ tests/frame/test_frame_6.py | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/lz4/frame/_compression.py b/lz4/frame/_compression.py index c6977449..7773ee44 100644 --- a/lz4/frame/_compression.py +++ b/lz4/frame/_compression.py @@ -4,6 +4,7 @@ """Internal classes used by the gzip, lzma and bz2 modules""" +import sys import io # Ensure super has Python 3 semantics even on Python 2 from builtins import super @@ -156,3 +157,13 @@ def seek(self, offset, whence=io.SEEK_SET): def tell(self): """Return the current file position.""" return self._pos + + +if sys.version_info < (3, 3): + # memoryview.cast is added in 3.3 + def readinto(self, b): + data = self.read(len(b)) + b[:len(data)] = data + return len(data) + + DecompressReader.readinto = readinto diff --git a/tests/frame/test_frame_6.py b/tests/frame/test_frame_6.py index 09e2acd6..335d09e4 100644 --- a/tests/frame/test_frame_6.py +++ b/tests/frame/test_frame_6.py @@ -55,6 +55,17 @@ def test_lz4frame_open_write_read_text(): assert data_out == data +def test_lz4frame_open_write_read_text_iter(): + data = u'This is a test string' + with lz4frame.open('testfile', mode='wt') as fp: + fp.write(data) + data_out = '' + with lz4frame.open('testfile', mode='rt') as fp: + for line in fp: + data_out += line + assert data_out == data + + def test_lz4frame_open_write_read( data, compression_level, From 791d72f1057da54031547313d0d09bb93789264c Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 6 Jun 2018 00:56:35 +0100 Subject: [PATCH 030/280] Update bundled lz4 libs to version 1.8.2 --- lz4libs/lz4.c | 1181 ++++++++++++++++++++++++------------- lz4libs/lz4.h | 294 ++++++--- lz4libs/lz4frame.c | 292 ++++++--- lz4libs/lz4frame.h | 192 +++++- lz4libs/lz4frame_static.h | 115 +--- lz4libs/lz4hc.c | 806 ++++++++++++++++++++----- lz4libs/lz4hc.h | 137 ++++- lz4libs/lz4opt.h | 356 ----------- lz4libs/xxhash.c | 2 +- 9 files changed, 2113 insertions(+), 1262 deletions(-) delete mode 100644 lz4libs/lz4opt.h diff --git a/lz4libs/lz4.c b/lz4libs/lz4.c index 213b0851..e51a3e0a 100644 --- a/lz4libs/lz4.c +++ b/lz4libs/lz4.c @@ -69,9 +69,11 @@ * Prefer these methods in priority order (0 > 1 > 2) */ #ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define LZ4_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) # define LZ4_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -80,7 +82,7 @@ * LZ4_FORCE_SW_BITCOUNT * Define this parameter if your target system or compiler does not support hardware bit count */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ # define LZ4_FORCE_SW_BITCOUNT #endif @@ -89,6 +91,8 @@ /*-************************************ * Dependency **************************************/ +#define LZ4_STATIC_LINKING_ONLY +#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ #include "lz4.h" /* see also "memory routines" below */ @@ -146,18 +150,23 @@ # define expect(expr,value) (expr) #endif +#ifndef likely #define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely #define unlikely(expr) expect((expr) != 0, 0) +#endif /*-************************************ * Memory routines **************************************/ #include /* malloc, calloc, free */ -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free +#define ALLOC(s) malloc(s) +#define ALLOC_AND_ZERO(s) calloc(1,s) +#define FREEMEM(p) free(p) #include /* memset, memcpy */ -#define MEM_INIT memset +#define MEM_INIT(p,v,s) memset((p),(v),(s)) /*-************************************ @@ -270,11 +279,6 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } } -static void LZ4_copy8(void* dst, const void* src) -{ - memcpy(dst,src,8); -} - /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ LZ4_FORCE_O2_INLINE_GCC_PPC64LE void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) @@ -283,7 +287,7 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) const BYTE* s = (const BYTE*)srcPtr; BYTE* const e = (BYTE*)dstEnd; - do { LZ4_copy8(d,s); d+=8; s+=8; } while (d=2) # include @@ -450,10 +454,33 @@ static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression ru /*-************************************ * Local Structures and types **************************************/ -typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; -typedef enum { byPtr, byU32, byU16 } tableType_t; - -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { notLimited = 0, limitedOutput = 1, fillOutput = 2 } limitedOutput_directive; +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; + +/** + * This enum distinguishes several different modes of accessing previous + * content in the stream. + * + * - noDict : There is no preceding content. + * - withPrefix64k : Table entries up to ctx->dictSize before the current blob + * blob being compressed are valid and refer to the preceding + * content (of length ctx->dictSize), which is available + * contiguously preceding in memory the content currently + * being compressed. + * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere + * else in memory, starting at ctx->dictionary with length + * ctx->dictSize. + * - usingDictCtx : Like usingExtDict, but everything concerning the preceding + * content is in a separate context, pointed to by + * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table + * entries in the current context that refer to positions + * preceding the beginning of the current compression are + * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx + * ->dictSize describe the location and size of the preceding + * content, and matches are found by looking in the ctx + * ->dictCtx->hashTable. + */ +typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; @@ -497,10 +524,25 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab return LZ4_hash4(LZ4_read32(p), tableType); } -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } + } +} + +static void LZ4_putPositionOnHash(const BYTE* p, U32 h, + void* tableBase, tableType_t const tableType, + const BYTE* srcBase) { switch (tableType) { + case clearedTable: { /* illegal! */ assert(0); return; } case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } @@ -513,19 +555,81 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_ LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32* const hashTable = (const U32*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16* const hashTable = (const U16*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-1))); + return hashTable[h]; + } + assert(0); return 0; /* forbidden case */ +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase) { - if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } - if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ + if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } + if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; } + { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType, + const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } +LZ4_FORCE_INLINE void LZ4_prepareTable( + LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is + * therefore safe to use no matter what mode we're in. Otherwise, we figure + * out if it's safe to leave as is or whether it needs to be reset. + */ + if (cctx->tableType != clearedTable) { + if (cctx->tableType != tableType + || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU) + || (tableType == byU32 && cctx->currentOffset > 1 GB) + || tableType == byPtr + || inputSize >= 4 KB) + { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); + MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + + /* Adding a gap, so all previous entries are > MAX_DISTANCE back, is faster + * than compressing without a gap. However, compressing with + * currentOffset == 0 is faster still, so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 KB; + } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ @@ -534,50 +638,70 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( const char* const source, char* const dest, const int inputSize, + int *inputConsumed, /* only written when outputLimited == fillOutput */ const int maxOutputSize, const limitedOutput_directive outputLimited, const tableType_t tableType, - const dict_directive dict, + const dict_directive dictDirective, const dictIssue_directive dictIssue, const U32 acceleration) { const BYTE* ip = (const BYTE*) source; - const BYTE* base; + + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*) source - startIndex; const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - cctx->dictSize; - const BYTE* const dictionary = cctx->dictionary; - const BYTE* const dictEnd = dictionary + cctx->dictSize; - const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; + + const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; + const BYTE* const dictionary = + dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; + const U32 dictSize = + dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */ + + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ + const BYTE* const dictEnd = dictionary + dictSize; const BYTE* anchor = (const BYTE*) source; const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; const BYTE* const matchlimit = iend - LASTLITERALS; + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE* dictBase = dictDirective == usingDictCtx ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; + U32 offset = 0; U32 forwardH; + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); /* Init conditions */ + if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - switch(dict) - { - case noDict: - default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; - case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; - case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ + assert(acceleration >= 1); + + lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; } - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (inputSizecurrentOffset += (U32)inputSize; + cctx->tableType = tableType; + + if (inputSizehashTable, tableType, base); @@ -585,12 +709,12 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Main Loop */ for ( ; ; ) { - ptrdiff_t refDelta = 0; const BYTE* match; BYTE* token; /* Find a match */ - { const BYTE* forwardIp = ip; + if (tableType == byPtr) { + const BYTE* forwardIp = ip; unsigned step = 1; unsigned searchMatchNb = acceleration << LZ4_skipTrigger; do { @@ -599,34 +723,89 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( forwardIp += step; step = (searchMatchNb++ >> LZ4_skipTrigger); - if (unlikely(forwardIp > mflimit)) goto _last_literals; + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( (match+MAX_DISTANCE < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + assert(startIndex - matchIndex >= MINMATCH); + match = dictBase + matchIndex; lowLimit = dictionary; } else { - refDelta = 0; + match = base + matchIndex; lowLimit = (const BYTE*)source; - } } + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */ + assert(matchIndex < current); + if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far */ + if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */ - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } + + } while(1); } /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } /* Encode Literals */ { unsigned const litLength = (unsigned)(ip - anchor); token = op++; - if ((outputLimited) && /* Check output buffer overflow */ + if ((outputLimited == limitedOutput) && /* Check output buffer overflow */ (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) return 0; + if ((outputLimited == fillOutput) && + (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { + op--; + goto _last_literals; + } if (litLength >= RUN_MASK) { int len = (int)litLength-RUN_MASK; *token = (RUN_MASK< olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= MAX_DISTANCE && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= MAX_DISTANCE); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } /* Encode MatchLength */ { unsigned matchCode; - if ((dict==usingExtDict) && (lowLimit==dictionary)) { - const BYTE* limit; - match += refDelta; - limit = ip + (dictEnd-match); + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); if (limit > matchlimit) limit = matchlimit; matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); ip += MINMATCH + matchCode; if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); matchCode += more; ip += more; } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); } else { matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); ip += MINMATCH + matchCode; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); } - if ( outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) - return 0; + if ((outputLimited) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) { + if (outputLimited == limitedOutput) + return 0; + if (outputLimited == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255; + ip -= matchCode - newMatchCode; + matchCode = newMatchCode; + } + } if (matchCode >= ML_MASK) { *token += ML_MASK; matchCode -= ML_MASK; @@ -685,37 +899,80 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( anchor = ip; /* Test end of chunk */ - if (ip > mflimit) break; + if (ip >= mflimitPlusOne) break; /* Fill table */ LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( (match+MAX_DISTANCE >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; + } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + assert(matchIndex < current); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; + } + } /* Prepare next loop */ forwardH = LZ4_hashPosition(++ip, tableType); + } _last_literals: /* Encode Last Literals */ - { size_t const lastRun = (size_t)(iend - anchor); + { size_t lastRun = (size_t)(iend - anchor); if ( (outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - return 0; + (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { + if (outputLimited == fillOutput) { + /* adapt lastRun to fill 'dst' */ + lastRun = (olimit-op) - 1; + lastRun -= (lastRun+240)/255; + } + if (outputLimited == limitedOutput) + return 0; + } if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; *op++ = RUN_MASK << ML_BITS; @@ -725,44 +982,97 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( *op++ = (BYTE)(lastRun<internal_donotuse; + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; LZ4_resetStream((LZ4_stream_t*)state); + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (inputSize < LZ4_64Klimit) {; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + +/** + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). + */ +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) +{ + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } } } int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { + int result; #if (LZ4_HEAPMODE) - void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctxPtr == NULL) return 0; #else LZ4_stream_t ctx; - void* const ctxPtr = &ctx; + LZ4_stream_t* const ctxPtr = &ctx; #endif - - int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); #if (LZ4_HEAPMODE) FREEMEM(ctxPtr); @@ -785,172 +1095,15 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m LZ4_resetStream(&ctx); if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); -} - - -/*-****************************** -* *_destSize() variant -********************************/ - -static int LZ4_compress_destSize_generic( - LZ4_stream_t_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) -{ - const BYTE* ip = (const BYTE*) src; - const BYTE* base = (const BYTE*) src; - const BYTE* lowLimit = (const BYTE*) src; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + targetDstSize; - BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; - BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); - BYTE* const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - - /* Init conditions */ - if (targetDstSize < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (*srcSizePtrhashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = 1 << LZ4_skipTrigger; - - do { - U32 h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - - } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - - /* Encode Literal length */ - { unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if (op + ((litLength+240)/255) + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength>=RUN_MASK) { - unsigned len = litLength - RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15-1) + (oMaxMatch-op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength>=ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; - } - else *token += (BYTE)(matchLength); - } - - anchor = ip; - - /* Test end of block */ - if (ip > mflimit) break; - if (op > oMaxSeq) break; - - /* Fill table */ - LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); - if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend-op) - 1; - lastRunSize -= (lastRunSize+240)/255; - } - ip = anchor + lastRunSize; - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1); + } else { + tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, tableType, noDict, noDictIssue, 1); + } } } int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (LZ4_HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctx == NULL) return 0; #else LZ4_stream_t ctxBody; LZ4_stream_t* ctx = &ctxBody; @@ -991,21 +1146,28 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe LZ4_stream_t* LZ4_createStream(void) { - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + DEBUGLOG(4, "LZ4_createStream %p", lz4s); + if (lz4s == NULL) return NULL; LZ4_resetStream(lz4s); return lz4s; } void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { - DEBUGLOG(4, "LZ4_resetStream"); + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); +} + int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { if (!LZ4_stream) return 0; /* support free on NULL */ + DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); FREEMEM(LZ4_stream); return (0); } @@ -1015,43 +1177,70 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; - if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); - if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; - return 0; - } + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; - base = p - dict->currentOffset; + base = dictEnd - 64 KB - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; + dict->currentOffset += 64 KB; + dict->tableType = tableType; + + if (dictSize < (int)HASH_UNIT) { + return 0; + } while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); + LZ4_putPosition(p, dict->hashTable, tableType, base); p+=3; } return dict->dictSize; } +void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { + if (dictionary_stream != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (working_stream->internal_donotuse.currentOffset == 0) { + working_stream->internal_donotuse.currentOffset = 64 KB; + } + working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse); + } else { + working_stream->internal_donotuse.dictCtx = NULL; + } +} + -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) { - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ + if (LZ4_dict->currentOffset + nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ /* rescale hash table */ U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; + DEBUGLOG(4, "LZ4_renormDictT"); for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; @@ -1065,15 +1254,25 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { + const tableType_t tableType = byU32; LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); - const BYTE* smallest = (const BYTE*) source; if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); + LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + /* invalidate tiny dictionaries */ + if ( (streamPtr->dictSize-1 < 4) /* intentional underflow */ + && (dictEnd != (const BYTE*)source) ) { + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)source; + dictEnd = (const BYTE*)source; + } + /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { @@ -1086,46 +1285,61 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* prefix mode : source data follows dictionary */ if (dictEnd == (const BYTE*)source) { - int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); } /* external dictionary mode */ { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t)); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); + } + } else { + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } + } streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; return result; } } -/* Hidden debug function, to force external dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) { LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT(streamPtr, smallest); + LZ4_renormDictT(streamPtr, srcSize); - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + } streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; + streamPtr->dictSize = (U32)srcSize; return result; } @@ -1196,41 +1410,72 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( const int safeDecode = (endOnInput==endOnInputSize); const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; + + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i)", srcSize); /* Special cases */ if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => just decode everything */ if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + if ((endOnInput) && unlikely(srcSize==0)) return -1; /* Main Loop : decode sequences */ while (1) { - size_t length; const BYTE* match; size_t offset; unsigned const token = *ip++; - - /* shortcut for common case : - * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes). - * this shortcut was tested on x86 and x64, where it improves decoding speed. - * it has not yet been benchmarked on ARM, Power, mips, etc. */ - if (((ip + 14 /*maxLL*/ + 2 /*offset*/ <= iend) - & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend)) - & ((token < (15<> ML_BITS; - size_t const off = LZ4_readLE16(ip+ll); - const BYTE* const matchPtr = op + ll - off; /* pointer underflow risk ? */ - if ((off >= 18) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) { - size_t const ml = (token & ML_MASK) + MINMATCH; - memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/; - memcpy(op, matchPtr, 18); op += ml; + size_t length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (endOnInput ? length != RUN_MASK : length <= 8) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { + /* Copy the literals */ + memcpy(op, ip, endOnInput ? 16 : 8); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + memcpy(op + 0, match + 0, 8); + memcpy(op + 8, match + 8, 8); + memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ continue; } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; } /* decode literal length */ - if ((length=(token>>ML_BITS)) == RUN_MASK) { + if (length == RUN_MASK) { unsigned s; + if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */ do { s = *ip++; length += s; @@ -1262,11 +1507,14 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( /* get offset */ offset = LZ4_readLE16(ip); ip+=2; match = op - offset; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ /* get matchlength */ length = token & ML_MASK; + +_copy_match: + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ + if (length == ML_MASK) { unsigned s; do { @@ -1313,7 +1561,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( match += inc32table[offset]; memcpy(op+4, match, 4); match -= dec64table[offset]; - } else { LZ4_copy8(op, match); match+=8; } + } else { memcpy(op, match, 8); match+=8; } op += 8; if (unlikely(cpy>oend-12)) { @@ -1326,7 +1574,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( } while (op16) LZ4_wildCopy(op+8, match+8, cpy); } op = cpy; /* correction */ @@ -1344,30 +1592,105 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( } +/*===== Instantiate the API decoding functions. =====*/ + LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + endOnInputSize, full, 0, noDict, + (BYTE*)dest, NULL, 0); } LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + endOnInputSize, partial, targetOutputSize, + noDict, (BYTE*)dest, NULL, 0); } LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); } +/*===== Instantiate a few more decoding cases, used more than once. =====*/ + +LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */ +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +/* Another obsolete API function, paired with the previous one. */ +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + /* LZ4_decompress_fast doesn't validate match offsets, + * and thus serves well with any prefixed dictionary. */ + return LZ4_decompress_fast(source, dest, originalSize); +} + +LZ4_FORCE_O2_GCC_PPC64LE +static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2_GCC_PPC64LE /* Exported under another name, for tests/fullbench.c */ +#define LZ4_decompress_safe_extDict LZ4_decompress_safe_forceExtDict +int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2_GCC_PPC64LE +static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +/* The "double dictionary" mode, for use with e.g. ring buffers: the first part + * of the dictionary is passed as prefix, and the second via dictStart + dictSize. + * These routines are used only once, in LZ4_decompress_*_continue(). + */ +LZ4_FORCE_INLINE +int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_INLINE +int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} /*===== streaming decompression functions =====*/ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); return lz4s; } @@ -1378,12 +1701,11 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) return 0; } -/*! - * LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * @return : 1 if OK, 0 if error */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { @@ -1395,6 +1717,25 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti return 1; } +/*! LZ4_decoderRingBufferSize() : + * when setting a ring buffer for streaming decompression (optional scenario), + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * Note : in a ring buffer scenario, + * blocks are presumed decompressed next to each other. + * When not enough space remains for next block (remainingSize < maxBlockSize), + * decoding resumes from beginning of ring buffer. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +int LZ4_decoderRingBufferSize(int maxBlockSize) +{ + if (maxBlockSize < 0) return 0; + if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; + if (maxBlockSize < 16) maxBlockSize = 16; + return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); +} + /* *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. @@ -1408,19 +1749,32 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (lz4sd->prefixSize == 0) { + /* The first call, no dictionary yet. */ + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (result <= 0) return result; + lz4sd->prefixSize = result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + /* They're rolling the current segment. */ + if (lz4sd->prefixSize >= 64 KB - 1) + result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + else if (lz4sd->extDictSize == 0) + result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize); + else + result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize += result; lz4sd->prefixEnd += result; } else { + /* The buffer wraps around, or they're switching to another buffer. */ lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + result = LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize = result; lz4sd->prefixEnd = (BYTE*)dest + result; @@ -1435,19 +1789,26 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (lz4sd->prefixSize == 0) { + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_fast(source, dest, originalSize); + if (result <= 0) return result; + lz4sd->prefixSize = originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0) + result = LZ4_decompress_fast(source, dest, originalSize); + else + result = LZ4_decompress_fast_doubleDict(source, dest, originalSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize += originalSize; lz4sd->prefixEnd += originalSize; } else { lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize = originalSize; lz4sd->prefixEnd = (BYTE*)dest + originalSize; @@ -1464,36 +1825,23 @@ Advanced decoding functions : the dictionary must be explicitly provided within parameters */ -LZ4_FORCE_O2_GCC_PPC64LE -LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (dictStart+dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + if (dictSize >= 64 KB - 1) + return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize); } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); + return LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize); } -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); -} - -LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); -} - -/* debug function */ -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); + if (dictSize==0 || dictStart+dictSize == dest) + return LZ4_decompress_fast(source, dest, originalSize); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, dictSize); } @@ -1501,64 +1849,67 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres * Obsolete Functions ***************************************************/ /* obsolete compression functions */ -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } -int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } -int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_default(source, dest, inputSize, maxOutputSize); +} +int LZ4_compress(const char* source, char* dest, int inputSize) +{ + return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); +} +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); +} +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); +} +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); +} +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); +} /* -These function names are deprecated and should no longer be used. +These decompression functions are deprecated and should no longer be used. They are only provided here for compatibility with older user programs. - LZ4_uncompress is totally equivalent to LZ4_decompress_fast - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe */ -int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - +int LZ4_uncompress (const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_fast(source, dest, outputSize); +} +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) +{ + return LZ4_decompress_safe(source, dest, isize, maxOutputSize); +} /* Obsolete Streaming functions */ int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } -static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) -{ - MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); - lz4ds->internal_donotuse.bufferStart = base; -} - int LZ4_resetStreamState(void* state, char* inputBuffer) { - if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); + (void)inputBuffer; + LZ4_resetStream((LZ4_stream_t*)state); return 0; } void* LZ4_create (char* inputBuffer) { - LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); - LZ4_init (lz4ds, (BYTE*)inputBuffer); - return lz4ds; + (void)inputBuffer; + return LZ4_createStream(); } -char* LZ4_slideInputBuffer (void* LZ4_Data) -{ - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; - int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); - return (char*)(ctx->bufferStart + dictSize); -} - -/* Obsolete streaming decompression functions */ - -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +char* LZ4_slideInputBuffer (void* state) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); + /* avoid const char * -> char * conversion warning */ + return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; } #endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/lz4libs/lz4.h b/lz4libs/lz4.h index a06b8a46..7d131221 100644 --- a/lz4libs/lz4.h +++ b/lz4libs/lz4.h @@ -93,7 +93,7 @@ extern "C" { /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -102,8 +102,8 @@ extern "C" { #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) -LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; to be used when checking dll version */ -LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; to be used when checking dll version */ +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; unseful to check dll version */ /*-************************************ @@ -113,7 +113,7 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; * LZ4_MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect + * Reduced memory usage may improve speed, thanks to cache effect * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #ifndef LZ4_MEMORY_USAGE @@ -128,12 +128,12 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; into already allocated 'dst' buffer of size 'dstCapacity'. Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). It also runs faster, so it's a recommended setting. - If the function cannot compress 'src' into a limited 'dst' budget, + If the function cannot compress 'src' into a more limited 'dst' budget, compression stops *immediately*, and the function result is zero. - As a consequence, 'dst' content is not valid. - This function never writes outside 'dst' buffer, nor read outside 'source' buffer. - srcSize : supported max value is LZ4_MAX_INPUT_VALUE - dstCapacity : full or partial size of buffer 'dst' (which must be already allocated) + Note : as a consequence, 'dst' content is not valid. + Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + dstCapacity : size of buffer 'dst' (which must be already allocated) return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) or 0 if compression fails */ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); @@ -144,8 +144,7 @@ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) If destination buffer is not large enough, decoding will stop and output an error code (negative value). If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, including malicious data packets. - It never writes outside output buffer, nor reads outside input buffer. + This function is protected against malicious data packets. */ LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); @@ -161,20 +160,20 @@ LZ4_compressBound() : Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) This function is primarily useful for memory allocation purposes (destination buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) + Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) inputSize : max supported value is LZ4_MAX_INPUT_SIZE return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) + or 0, if input size is incorrect (too large or negative) */ LZ4LIB_API int LZ4_compressBound(int inputSize); /*! LZ4_compress_fast() : - Same as LZ4_compress_default(), but allows to select an "acceleration" factor. + Same as LZ4_compress_default(), but allows selection of "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. + Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c). */ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -205,15 +204,19 @@ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePt /*! -LZ4_decompress_fast() : (unsafe!!) - originalSize : is the original uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes. - note : This function respects memory boundaries for *properly formed* compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). +LZ4_decompress_fast() : **unsafe!** +This function is a bit faster than LZ4_decompress_safe(), +but it may misbehave on malformed input because it doesn't perform full validation of compressed data. + originalSize : is the uncompressed size to regenerate + Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes. + return : number of bytes read from source buffer (== compressed size). + If the source stream is detected malformed, the function stops decoding and return a negative result. + note : This function is only usable if the originalSize of uncompressed data is known in advance. + The caller should also check that all the compressed input has been consumed properly, + i.e. that the return value matches the size of the buffer with compressed input. + The function never writes past the output buffer. However, since it doesn't know its 'src' size, + it may read past the intended input. Also, because match offsets are not validated during decoding, + reads from 'src' may underflow. Use this function in trusted environment **only**. */ LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); @@ -222,12 +225,12 @@ LZ4_decompress_safe_partial() : This function decompress a compressed block of size 'srcSize' at position 'src' into destination buffer 'dst' of size 'dstCapacity'. The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that. - However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity). + However, it's not accurate, and may write more than 'targetOutputSize' (but always <= dstCapacity). @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity) - Note : this number can be < 'targetOutputSize' should the compressed block contain less data. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets. + Note : this number can also be < targetOutputSize, if compressed block contains less data. + Therefore, always control how many bytes were decoded. + If source stream is detected malformed, function returns a negative result. + This function is protected against malicious data packets. */ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); @@ -235,7 +238,7 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcS /*-********************************************* * Streaming Compression Functions ***********************************************/ -typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ /*! LZ4_createStream() and LZ4_freeStream() : * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. @@ -259,67 +262,93 @@ LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); /*! LZ4_compress_fast_continue() : - * Compress content into 'src' using data from previously compressed blocks, improving compression ratio. + * Compress 'src' content using data from previously compressed blocks, for better compression ratio. * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. * - * Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! - * Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. - * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * Important : The previous 64KB of compressed data is assumed to remain present and unmodified in memory! + * + * Special 1 : When input is a double-buffer, they can have any size, including < 64 KB. + * Make sure that buffers are separated by at least one byte. + * This way, each block only depends on previous block. + * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. * * @return : size of compressed block - * or 0 if there is an error (typically, compressed data cannot fit into 'dst') + * or 0 if there is an error (typically, cannot fit into 'dst'). * After an error, the stream status is invalid, it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its current memory location, + * If last 64KB data cannot be guaranteed to remain available at its current memory location, * save it into a safer place (char* safeBuffer). - * Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable. - * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), + * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. + * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. */ -LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); +LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); /*-********************************************** * Streaming Decompression Functions * Bufferless synchronous API ************************************************/ -typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : - * creation / destruction of streaming decompression tracking structure. - * A tracking structure can be re-used multiple times sequentially. */ + * creation / destruction of streaming decompression tracking context. + * A tracking context can be re-used multiple times. + */ LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); /*! LZ4_setStreamDecode() : - * An LZ4_streamDecode_t structure can be allocated once and re-used multiple times. + * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. * Use this function to start decompression of a new stream of blocks. - * A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order. + * A dictionary can optionnally be set. Use NULL or size 0 for a reset order. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. * @return : 1 if OK, 0 if error */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); +/*! LZ4_decoderRingBufferSize() : v1.8.2 + * Note : in a ring buffer scenario (optional), + * blocks are presumed decompressed next to each other + * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), + * at which stage it resumes from beginning of ring buffer. + * When setting such a ring buffer for streaming decompression, + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); +#define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs)) /* for static allocation; mbs presumed valid */ + /*! LZ4_decompress_*_continue() : * These decoding functions allow decompression of consecutive blocks in "streaming" mode. * A block is an unsplittable entity, it must be presented entirely to a decompression function. - * Decompression functions only accept one block at a time. - * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB). + * Decompression functions only accepts one block at a time. + * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded. + * If less than 64KB of data has been decoded, all the data must be present. * - * Special : if application sets a ring buffer for decompression, it must respect one of the following conditions : - * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - * maxBlockSize is implementation dependent. It's the maximum size of any single block. + * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : + * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). + * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. + * In which case, encoding and decoding buffers do not need to be synchronized. + * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. + * - Synchronized mode : + * Decompression buffer size is _exactly_ the same as compression buffer size, + * and follows exactly same update rule (block boundaries at same positions), + * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. * In which case, encoding and decoding buffers do not need to be synchronized, * and encoding ring buffer can have any size, including small ones ( < 64 KB). - * - _At least_ 64 KB + 8 bytes + maxBlockSize. - * In which case, encoding and decoding buffers do not need to be synchronized, - * and encoding ring buffer can have any size, including larger than decoding buffer. - * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - * and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block. + * + * Whenever these conditions are not possible, + * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, + * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. */ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); @@ -329,6 +358,7 @@ LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecod * These decoding functions work the same as * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() * They are stand-alone, and don't need an LZ4_streamDecode_t structure. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. */ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); @@ -337,6 +367,94 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or /*^********************************************** * !!!!!! STATIC LINKING ONLY !!!!!! ***********************************************/ + +/*-************************************ + * Unstable declarations + ************************************** + * Declarations in this section should be considered unstable. + * Use at your own peril, etc., etc. + * They may be removed in the future. + * Their signatures may change. + **************************************/ + +#ifdef LZ4_STATIC_LINKING_ONLY + +/*! LZ4_resetStream_fast() : + * Use this, like LZ4_resetStream(), to prepare a context for a new chain of + * calls to a streaming API (e.g., LZ4_compress_fast_continue()). + * + * Note: + * Using this in advance of a non- streaming-compression function is redundant, + * and potentially bad for performance, since they all perform their own custom + * reset internally. + * + * Differences from LZ4_resetStream(): + * When an LZ4_stream_t is known to be in a internally coherent state, + * it can often be prepared for a new compression with almost no work, only + * sometimes falling back to the full, expensive reset that is always required + * when the stream is in an indeterminate state (i.e., the reset performed by + * LZ4_resetStream()). + * + * LZ4_streams are guaranteed to be in a valid state when: + * - returned from LZ4_createStream() + * - reset by LZ4_resetStream() + * - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged + * - the stream was in a valid state and was reset by LZ4_resetStream_fast() + * - the stream was in a valid state and was then used in any compression call + * that returned success + * - the stream was in an indeterminate state and was used in a compression + * call that fully reset the state (e.g., LZ4_compress_fast_extState()) and + * that returned success + * + * When a stream isn't known to be in a valid state, it is not safe to pass to + * any fastReset or streaming function. It must first be cleansed by the full + * LZ4_resetStream(). + */ +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); + +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly + * initialized"). From a high level, the difference is that this function + * initializes the provided state with a call to something like + * LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a + * call to LZ4_resetStream(). + */ +LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_attach_dictionary() : + * This is an experimental API that allows for the efficient use of a + * static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDict() should + * be expected to work. + * + * Alternatively, the provided dictionary stream pointer may be NULL, in which + * case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the first compression call on the stream. + */ +LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream); + +#endif + /*-************************************ * Private definitions ************************************** @@ -351,14 +469,16 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) #include -typedef struct { +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { uint32_t hashTable[LZ4_HASH_SIZE_U32]; uint32_t currentOffset; - uint32_t initCheck; + uint16_t initCheck; + uint16_t tableType; const uint8_t* dictionary; - uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ + const LZ4_stream_t_internal* dictCtx; uint32_t dictSize; -} LZ4_stream_t_internal; +}; typedef struct { const uint8_t* externalDict; @@ -369,14 +489,16 @@ typedef struct { #else -typedef struct { +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { unsigned int hashTable[LZ4_HASH_SIZE_U32]; unsigned int currentOffset; - unsigned int initCheck; + unsigned short initCheck; + unsigned short tableType; const unsigned char* dictionary; - unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ + const LZ4_stream_t_internal* dictCtx; unsigned int dictSize; -} LZ4_stream_t_internal; +}; typedef struct { const unsigned char* externalDict; @@ -433,11 +555,9 @@ union LZ4_streamDecode_u { # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */ -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) +# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) # elif (LZ4_GCC_VERSION >= 301) # define LZ4_DEPRECATED(message) __attribute__((deprecated)) @@ -450,26 +570,34 @@ union LZ4_streamDecode_u { #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Obsolete decompression functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); -/* Obsolete streaming functions; use new streaming interface whenever possible */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); +/* Obsolete streaming functions; degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, they don't + * actually retain any history between compression calls. The compression ratio + * achieved will therefore be no better than compressing each chunk + * independently. + */ +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); /* Obsolete streaming decoding functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); #endif /* LZ4_H_2983827168210 */ diff --git a/lz4libs/lz4frame.c b/lz4libs/lz4frame.c index 0b26f75a..e1d0b1d0 100644 --- a/lz4libs/lz4frame.c +++ b/lz4libs/lz4frame.c @@ -46,11 +46,25 @@ You can contact the author at : #endif +/*-************************************ +* Tuning parameters +**************************************/ +/* + * LZ4F_HEAPMODE : + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef LZ4F_HEAPMODE +# define LZ4F_HEAPMODE 0 +#endif + + /*-************************************ * Memory routines **************************************/ #include /* malloc, calloc, free */ -#define ALLOCATOR(s) calloc(1,s) +#define ALLOC(s) malloc(s) +#define ALLOC_AND_ZERO(s) calloc(1,s) #define FREEMEM free #include /* memset, memcpy, memmove */ #define MEM_INIT memset @@ -59,7 +73,9 @@ You can contact the author at : /*-************************************ * Includes **************************************/ -#include "lz4frame_static.h" +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" +#define LZ4_STATIC_LINKING_ONLY #include "lz4.h" #define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" @@ -80,6 +96,19 @@ You can contact the author at : #define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) && !defined(DEBUGLOG) +# include +static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + /*-************************************ * Basic Types @@ -188,7 +217,8 @@ typedef struct LZ4F_cctx_s U64 totalInSize; XXH32_state_t xxh; void* lz4CtxPtr; - U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ + U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ + U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ } LZ4F_cctx_t; @@ -279,7 +309,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, size_t alreadyBuffered) { LZ4F_preferences_t prefsNull; - memset(&prefsNull, 0, sizeof(prefsNull)); + MEM_INIT(&prefsNull, 0, sizeof(prefsNull)); prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; U32 const flush = prefsPtr->autoFlush | (srcSize==0); @@ -308,7 +338,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere size_t const headerSize = maxFHSize; /* max header size, including optional fields */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; - else memset(&prefs, 0, sizeof(prefs)); + else MEM_INIT(&prefs, 0, sizeof(prefs)); prefs.autoFlush = 1; return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);; @@ -324,27 +354,22 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere * @return : number of bytes written into dstBuffer, * or an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity, +size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_CDict* cdict, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_cctx_t cctxI; - LZ4_stream_t lz4ctx; /* pretty large on stack */ LZ4F_preferences_t prefs; LZ4F_compressOptions_t options; BYTE* const dstStart = (BYTE*) dstBuffer; BYTE* dstPtr = dstStart; BYTE* const dstEnd = dstStart + dstCapacity; - memset(&cctxI, 0, sizeof(cctxI)); - cctxI.version = LZ4F_VERSION; - cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ - if (preferencesPtr!=NULL) prefs = *preferencesPtr; else - memset(&prefs, 0, sizeof(prefs)); + MEM_INIT(&prefs, 0, sizeof(prefs)); if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ @@ -353,32 +378,24 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity, if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */ - if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - cctxI.lz4CtxPtr = &lz4ctx; - cctxI.lz4CtxLevel = 1; - } /* fast compression context pre-created on stack */ - - memset(&options, 0, sizeof(options)); + MEM_INIT(&options, 0, sizeof(options)); options.stableSrc = 1; if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */ return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs); /* write header */ + { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */ if (LZ4F_isError(headerSize)) return headerSize; dstPtr += headerSize; /* header size */ } - { size_t const cSize = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); + { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); if (LZ4F_isError(cSize)) return cSize; dstPtr += cSize; } - { size_t const tailSize = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ + { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */ - FREEMEM(cctxI.lz4CtxPtr); - return (dstPtr - dstStart); } @@ -394,9 +411,44 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity, - srcBuffer, srcSize, - NULL, preferencesPtr); + size_t result; +#if (LZ4F_HEAPMODE) + LZ4F_cctx_t *cctxPtr; + result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION); + if (LZ4F_isError(result)) return result; +#else + LZ4F_cctx_t cctx; + LZ4_stream_t lz4ctx; + LZ4F_cctx_t *cctxPtr = &cctx; + + DEBUGLOG(4, "LZ4F_compressFrame"); + MEM_INIT(&cctx, 0, sizeof(cctx)); + cctx.version = LZ4F_VERSION; + cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ + if (preferencesPtr == NULL || + preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN) + { + LZ4_resetStream(&lz4ctx); + cctxPtr->lz4CtxPtr = &lz4ctx; + cctxPtr->lz4CtxAlloc = 1; + cctxPtr->lz4CtxState = 1; + } +#endif + + result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity, + srcBuffer, srcSize, + NULL, preferencesPtr); + +#if (LZ4F_HEAPMODE) + LZ4F_freeCompressionContext(cctxPtr); +#else + if (preferencesPtr != NULL && + preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) + { + FREEMEM(cctxPtr->lz4CtxPtr); + } +#endif + return result; } @@ -419,13 +471,14 @@ struct LZ4F_CDict_s { LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) { const char* dictStart = (const char*)dictBuffer; - LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict)); + LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict)); + DEBUGLOG(4, "LZ4F_createCDict"); if (!cdict) return NULL; if (dictSize > 64 KB) { dictStart += dictSize - 64 KB; dictSize = 64 KB; } - cdict->dictContent = ALLOCATOR(dictSize); + cdict->dictContent = ALLOC(dictSize); cdict->fastCtx = LZ4_createStream(); cdict->HCCtx = LZ4_createStreamHC(); if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { @@ -433,9 +486,8 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) return NULL; } memcpy(cdict->dictContent, dictStart, dictSize); - LZ4_resetStream(cdict->fastCtx); LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize); - LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT); + LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT); LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize); return cdict; } @@ -464,7 +516,7 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict) */ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) { - LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t)); + LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t)); if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed); cctxPtr->version = version; @@ -490,6 +542,36 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp } +/** + * This function prepares the internal LZ4(HC) stream for a new compression, + * resetting the context and attaching the dictionary, if there is one. + * + * It needs to be called at the beginning of each independent compression + * stream (i.e., at the beginning of a frame in blockLinked mode, or at the + * beginning of each block in blockIndependent mode). + */ +static void LZ4F_initStream(void* ctx, + const LZ4F_CDict* cdict, + int level, + LZ4F_blockMode_t blockMode) { + if (level < LZ4HC_CLEVEL_MIN) { + if (cdict != NULL || blockMode == LZ4F_blockLinked) { + /* In these cases, we will call LZ4_compress_fast_continue(), + * which needs an already reset context. Otherwise, we'll call a + * one-shot API. The non-continued APIs internally perform their own + * resets at the beginning of their calls, where they know what + * tableType they need the context to be in. So in that case this + * would be misguided / wasted work. */ + LZ4_resetStream_fast((LZ4_stream_t*)ctx); + } + LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL); + } else { + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level); + LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL); + } +} + + /*! LZ4F_compressBegin_usingCDict() : * init streaming compression and writes frame header into dstBuffer. * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. @@ -507,21 +589,33 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, BYTE* headerStart; if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - memset(&prefNull, 0, sizeof(prefNull)); + MEM_INIT(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; /* Ctx Management */ - { U32 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ - if (cctxPtr->lz4CtxLevel < ctxTypeID) { + { U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; + if (cctxPtr->lz4CtxAlloc < ctxTypeID) { FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); - else + } else { cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); + } if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed); - cctxPtr->lz4CtxLevel = ctxTypeID; - } } + cctxPtr->lz4CtxAlloc = ctxTypeID; + cctxPtr->lz4CtxState = ctxTypeID; + } else if (cctxPtr->lz4CtxState != ctxTypeID) { + /* otherwise, a sufficient buffer is allocated, but we need to + * reset it to the correct context type */ + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr); + } else { + LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + } + cctxPtr->lz4CtxState = ctxTypeID; + } + } /* Buffer Management */ if (cctxPtr->prefs.frameInfo.blockSizeID == 0) @@ -535,7 +629,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, if (cctxPtr->maxBufferSize < requiredBuffSize) { cctxPtr->maxBufferSize = 0; FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); + cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize); if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed); cctxPtr->maxBufferSize = requiredBuffSize; } } @@ -547,19 +641,10 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, cctxPtr->cdict = cdict; if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) { /* frame init only for blockLinked : blockIndependent will be init at each block */ - if (cdict) { - if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx)); - } else { - memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx)); - LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); - } - } else { - if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) - LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); - else - LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); - } + LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked); + } + if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) { + LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); } /* Magic Number */ @@ -654,11 +739,12 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { int const acceleration = (level < -1) ? -level : 1; + LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent); if (cdict) { - memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx)); return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); + } else { + return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration); } - return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration); } static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) @@ -670,12 +756,11 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { + LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent); if (cdict) { - memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx)); - LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level); return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); } - return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level); + return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level); } static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) @@ -724,10 +809,12 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, LZ4F_lastBlockStatus lastBlockCompressed = notDone; compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); + DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize); if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - memset(&cOptionsNull, 0, sizeof(cOptionsNull)); + if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) + return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; /* complete tmp buffer */ @@ -931,7 +1018,7 @@ struct LZ4F_dctx_s { */ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { - LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx)); + LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx)); if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC); dctx->version = versionNumber; @@ -1003,7 +1090,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize /* need to decode header to get frameInfo */ if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */ - memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo)); + MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo)); /* special case : skippable frames */ if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) { @@ -1136,24 +1223,31 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP /* LZ4F_updateDict() : * only used for LZ4F_blockLinked mode */ -static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) +static void LZ4F_updateDict(LZ4F_dctx* dctx, + const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart, + unsigned withinTmp) { if (dctx->dictSize==0) dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ - if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity */ + if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity, directly within dstBuffer */ dctx->dictSize += dstSize; return; } - if (dstPtr - dstPtr0 + dstSize >= 64 KB) { /* dstBuffer large enough to become dictionary */ - dctx->dict = (const BYTE*)dstPtr0; - dctx->dictSize = dstPtr - dstPtr0 + dstSize; + if (dstPtr - dstBufferStart + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */ + dctx->dict = (const BYTE*)dstBufferStart; + dctx->dictSize = dstPtr - dstBufferStart + dstSize; return; } - if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { - /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */ + assert(dstSize < 64 KB); /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */ + + /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOut */ + + if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */ + /* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */ + assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart); dctx->dictSize += dstSize; return; } @@ -1174,7 +1268,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */ if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */ - size_t const preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ + size_t const preserveSize = 64 KB - dstSize; memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); dctx->dictSize = preserveSize; } @@ -1184,7 +1278,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, } /* join dict & dest into tmp */ - { size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ + { size_t preserveSize = 64 KB - dstSize; if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize; memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize); @@ -1230,7 +1324,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, size_t nextSrcSizeHint = 1; - memset(&optionsNull, 0, sizeof(optionsNull)); + MEM_INIT(&optionsNull, 0, sizeof(optionsNull)); if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; *srcSizePtr = 0; *dstSizePtr = 0; @@ -1251,7 +1345,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } dctx->tmpInSize = 0; if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */ - dctx->tmpInTarget = minFHSize; /* minimum to attempt decode */ + dctx->tmpInTarget = minFHSize; /* minimum size to decode header */ dctx->dStage = dstage_storeFrameHeader; /* fall-through */ @@ -1279,11 +1373,11 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ FREEMEM(dctx->tmpIn); - dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize + 4 /* block checksum */); + dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + 4 /* block checksum */); if (dctx->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed); FREEMEM(dctx->tmpOutBuffer); - dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded); + dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded); if (dctx->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed); dctx->maxBufferSize = bufferNeeded; @@ -1408,8 +1502,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); if (readCRC != calcCRC) return err0r(LZ4F_ERROR_blockChecksum_invalid); - } - } + } } dctx->dStage = dstage_getBlockHeader; /* new block */ break; @@ -1450,11 +1543,19 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } } if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) { + const char* dict = (const char*)dctx->dict; + size_t dictSize = dctx->dictSize; + int decodedSize; + if (dict && dictSize > 1 GB) { + /* the dictSize param is an int, avoid truncation / sign issues */ + dict += dictSize - 64 KB; + dictSize = 64 KB; + } /* enough capacity in `dst` to decompress directly there */ - int const decodedSize = LZ4_decompress_safe_usingDict( + decodedSize = LZ4_decompress_safe_usingDict( (const char*)selectedIn, (char*)dstPtr, (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, - (const char*)dctx->dict, (int)dctx->dictSize); + dict, (int)dictSize); if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */ if (dctx->frameInfo.contentChecksumFlag) XXH32_update(&(dctx->xxh), dstPtr, decodedSize); @@ -1482,14 +1583,21 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } else { /* dict not within tmp */ size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB); dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace; - } - } + } } /* Decode block */ - { int const decodedSize = LZ4_decompress_safe_usingDict( + { const char* dict = (const char*)dctx->dict; + size_t dictSize = dctx->dictSize; + int decodedSize; + if (dict && dictSize > 1 GB) { + /* the dictSize param is an int, avoid truncation / sign issues */ + dict += dictSize - 64 KB; + dictSize = 64 KB; + } + decodedSize = LZ4_decompress_safe_usingDict( (const char*)selectedIn, (char*)dctx->tmpOut, (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, - (const char*)dctx->dict, (int)dctx->dictSize); + dict, (int)dictSize); if (decodedSize < 0) /* decompression failed */ return err0r(LZ4F_ERROR_decompressionFailed); if (dctx->frameInfo.contentChecksumFlag) @@ -1507,8 +1615,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy); /* dictionary management */ - if (dctx->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1); + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1 /*withinTmp*/); dctx->tmpOutStart += sizeToCopy; dstPtr += sizeToCopy; @@ -1517,8 +1625,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, dctx->dStage = dstage_getBlockHeader; /* get next block */ break; } + /* could not flush everything : stop there, just request a block header */ + doAnotherStage = 0; nextSrcSizeHint = BHSize; - doAnotherStage = 0; /* still some data to flush */ break; } @@ -1555,7 +1664,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, selectedIn = dctx->tmpIn; } /* if (dctx->dStage == dstage_storeSuffix) */ - /* case dstage_checkSuffix: */ /* no direct call, avoid scan-build warning */ + /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */ { U32 const readCRC = LZ4F_readLE32(selectedIn); U32 const resultCRC = XXH32_digest(&(dctx->xxh)); if (readCRC != resultCRC) @@ -1579,8 +1688,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, if (dctx->dStage == dstage_storeSFrameSize) case dstage_storeSFrameSize: - { - size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, + { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr) ); memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; @@ -1594,7 +1702,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, selectedIn = dctx->header + 4; } /* if (dctx->dStage == dstage_storeSFrameSize) */ - /* case dstage_decodeSFrameSize: */ /* no direct access */ + /* case dstage_decodeSFrameSize: */ /* no direct entry */ { size_t const SFrameSize = LZ4F_readLE32(selectedIn); dctx->frameInfo.contentSize = SFrameSize; dctx->tmpInTarget = SFrameSize; @@ -1613,7 +1721,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, LZ4F_resetDecompressionContext(dctx); break; } - } + } /* switch (dctx->dStage) */ } /* while (doAnotherStage) */ /* preserve history within tmp whenever necessary */ diff --git a/lz4libs/lz4frame.h b/lz4libs/lz4frame.h index eb55e45e..fb434ff7 100644 --- a/lz4libs/lz4frame.h +++ b/lz4libs/lz4frame.h @@ -93,8 +93,8 @@ extern "C" { **************************************/ typedef size_t LZ4F_errorCode_t; -LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells if a `LZ4F_errorCode_t` function result is an error code */ -LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; useful for debugging */ +LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells when a function result is an error code */ +LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; for debugging */ /*-************************************ @@ -162,24 +162,25 @@ typedef LZ4F_contentChecksum_t contentChecksum_t; * It's not required to set all fields, as long as the structure was initially memset() to zero. * For all fields, 0 sets it to default value */ typedef struct { - LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */ - LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */ - LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */ - unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */ - unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */ - LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */ + LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default */ + LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */ + LZ4F_contentChecksum_t contentChecksumFlag; /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */ + LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */ + unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */ + unsigned dictID; /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */ + LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */ } LZ4F_frameInfo_t; /*! LZ4F_preferences_t : * makes it possible to supply detailed compression parameters to the stream interface. - * It's not required to set all fields, as long as the structure was initially memset() to zero. + * Structure is presumed initially memset() to zero, representing default settings. * All reserved fields must be set to zero. */ typedef struct { LZ4F_frameInfo_t frameInfo; - int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */ - unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */ - unsigned reserved[4]; /* must be zero for forward compatibility */ + int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */ + unsigned autoFlush; /* 1: always flush, to reduce usage of internal buffers */ + unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */ /* >= v1.8.2 */ + unsigned reserved[3]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; LZ4FLIB_API int LZ4F_compressionLevel_max(void); @@ -189,8 +190,10 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void); * Simple compression function ***********************************/ /*! LZ4F_compressFrameBound() : - * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. - * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. + * Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences. + * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences. + * Note : this result is only usable with LZ4F_compressFrame(). + * It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed. */ LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); @@ -235,7 +238,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /*---- Compression ----*/ -#define LZ4F_HEADER_SIZE_MAX 19 +#define LZ4F_HEADER_SIZE_MAX 19 /* LZ4 Frame header size can vary from 7 to 19 bytes */ /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. @@ -248,8 +251,13 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, const LZ4F_preferences_t* prefsPtr); /*! LZ4F_compressBound() : - * Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case situations. + * Provides minimum dstCapacity required to guarantee compression success + * given a srcSize and preferences, covering worst case scenario. * prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. + * Estimation is valid for either LZ4F_compressUpdate(), LZ4F_flush() or LZ4F_compressEnd(), + * Estimation includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. + * It also includes frame footer (ending + checksum), which would have to be generated by LZ4F_compressEnd(). + * Estimation doesn't include frame header, as it was already generated by LZ4F_compressBegin(). * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. */ @@ -257,36 +265,44 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* /*! LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. + * Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations. * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). - * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. + * LZ4F_compressUpdate() doesn't guarantee error recovery. + * When an error occurs, compression context must be freed or resized. * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); +LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); /*! LZ4F_flush() : * When data must be generated and sent immediately, without waiting for a block to be completely filled, * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. * `dstCapacity` must be large enough to ensure the operation will be successful. * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. - * @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx) + * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx) * or an error code if it fails (which can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); +LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* cOptPtr); /*! LZ4F_compressEnd() : * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). * It will flush whatever data remained within `cctx` (like LZ4_flush()) * and properly finalize the frame, with an endMark and a checksum. * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. - * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) + * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark), * or an error code if it fails (which can be tested using LZ4F_isError()) * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. */ -LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); +LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* cOptPtr); /*-********************************* @@ -296,7 +312,7 @@ typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */ typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */ typedef struct { - unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */ + unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */ unsigned reserved[3]; /* must be set to zero for forward compatibility */ } LZ4F_decompressOptions_t; @@ -309,7 +325,7 @@ typedef struct { * The function provides a pointer to an allocated and initialized LZ4F_dctx object. * The result is an errorCode, which can be tested using LZ4F_isError(). * dctx memory can be released using LZ4F_freeDecompressionContext(); - * The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. + * Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released. * That is, it should be == 0 if decompression has been completed fully and correctly. */ LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); @@ -350,8 +366,8 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, * The function will read up to *srcSizePtr bytes from srcBuffer, * and decompress data into dstBuffer, of capacity *dstSizePtr. * - * The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value). - * The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value). + * The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value). + * The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value). * * The function does not necessarily read all input bytes, so always check value in *srcSizePtr. * Unconsumed source data must be presented again in subsequent invocations. @@ -394,3 +410,123 @@ LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always su #endif #endif /* LZ4F_H_09782039843 */ + +#if defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) +#define LZ4F_H_STATIC_09782039843 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* These declarations are not stable and may change in the future. They are + * therefore only safe to depend on when the caller is statically linked + * against the library. To access their declarations, define + * LZ4F_STATIC_LINKING_ONLY. + * + * There is a further protection mechanism where these symbols aren't published + * into shared/dynamic libraries. You can override this behavior and force + * them to be published by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. Use at + * your own risk. + */ +#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS +#define LZ4FLIB_STATIC_API LZ4FLIB_API +#else +#define LZ4FLIB_STATIC_API +#endif + + +/* --- Error List --- */ +#define LZ4F_LIST_ERRORS(ITEM) \ + ITEM(OK_NoError) \ + ITEM(ERROR_GENERIC) \ + ITEM(ERROR_maxBlockSize_invalid) \ + ITEM(ERROR_blockMode_invalid) \ + ITEM(ERROR_contentChecksumFlag_invalid) \ + ITEM(ERROR_compressionLevel_invalid) \ + ITEM(ERROR_headerVersion_wrong) \ + ITEM(ERROR_blockChecksum_invalid) \ + ITEM(ERROR_reservedFlag_set) \ + ITEM(ERROR_allocation_failed) \ + ITEM(ERROR_srcSize_tooLarge) \ + ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_frameHeader_incomplete) \ + ITEM(ERROR_frameType_unknown) \ + ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_srcPtr_wrong) \ + ITEM(ERROR_decompressionFailed) \ + ITEM(ERROR_headerChecksum_invalid) \ + ITEM(ERROR_contentChecksum_invalid) \ + ITEM(ERROR_frameDecoding_alreadyStarted) \ + ITEM(ERROR_maxCode) + +#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, + +/* enum list is exposed, to handle specific errors */ +typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; + +LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + + + +/********************************** + * Bulk processing dictionary API + *********************************/ +typedef struct LZ4F_CDict_s LZ4F_CDict; + +/*! LZ4_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ +LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); +LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict); + + +/*! LZ4_compressFrame_usingCDict() : + * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary. + * cctx must point to a context created by LZ4F_createCompressionContext(). + * If cdict==NULL, compress without a dictionary. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * If this condition is not respected, function will fail (@return an errorCode). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * but it's not recommended, as it's the only way to provide dictID in the frame header. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) */ +LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict( + LZ4F_cctx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr); + + +/*! LZ4F_compressBegin_usingCDict() : + * Inits streaming dictionary compression, and writes the frame header into dstBuffer. + * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * `prefsPtr` is optional : you may provide NULL as argument, + * however, it's the only way to provide dictID in the frame header. + * @return : number of bytes written into dstBuffer for the header, + * or an error code (which can be tested using LZ4F_isError()) */ +LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict( + LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* prefsPtr); + + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. + * It must remain accessible throughout the entire frame decoding. */ +LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict( + LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr); + +#if defined (__cplusplus) +} +#endif + +#endif /* defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) */ diff --git a/lz4libs/lz4frame_static.h b/lz4libs/lz4frame_static.h index a59b94b5..925a2c5c 100644 --- a/lz4libs/lz4frame_static.h +++ b/lz4libs/lz4frame_static.h @@ -36,119 +36,12 @@ #ifndef LZ4FRAME_STATIC_H_0398209384 #define LZ4FRAME_STATIC_H_0398209384 -#if defined (__cplusplus) -extern "C" { -#endif - -/* lz4frame_static.h should be used solely in the context of static linking. - * It contains definitions which are not stable and may change in the future. - * Never use it in the context of DLL linking. - * - * Defining LZ4F_PUBLISH_STATIC_FUNCTIONS allows one to override this. Use at - * your own risk. +/* The declarations that formerly were made here have been merged into + * lz4frame.h, protected by the LZ4F_STATIC_LINKING_ONLY macro. Going forward, + * it is recommended to simply include that header directly. */ -#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS -#define LZ4FLIB_STATIC_API LZ4FLIB_API -#else -#define LZ4FLIB_STATIC_API -#endif - -/* --- Dependency --- */ +#define LZ4F_STATIC_LINKING_ONLY #include "lz4frame.h" - -/* --- Error List --- */ -#define LZ4F_LIST_ERRORS(ITEM) \ - ITEM(OK_NoError) \ - ITEM(ERROR_GENERIC) \ - ITEM(ERROR_maxBlockSize_invalid) \ - ITEM(ERROR_blockMode_invalid) \ - ITEM(ERROR_contentChecksumFlag_invalid) \ - ITEM(ERROR_compressionLevel_invalid) \ - ITEM(ERROR_headerVersion_wrong) \ - ITEM(ERROR_blockChecksum_invalid) \ - ITEM(ERROR_reservedFlag_set) \ - ITEM(ERROR_allocation_failed) \ - ITEM(ERROR_srcSize_tooLarge) \ - ITEM(ERROR_dstMaxSize_tooSmall) \ - ITEM(ERROR_frameHeader_incomplete) \ - ITEM(ERROR_frameType_unknown) \ - ITEM(ERROR_frameSize_wrong) \ - ITEM(ERROR_srcPtr_wrong) \ - ITEM(ERROR_decompressionFailed) \ - ITEM(ERROR_headerChecksum_invalid) \ - ITEM(ERROR_contentChecksum_invalid) \ - ITEM(ERROR_frameDecoding_alreadyStarted) \ - ITEM(ERROR_maxCode) - -#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, - -/* enum list is exposed, to handle specific errors */ -typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; - -LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); - - - -/********************************** - * Bulk processing dictionary API - *********************************/ -typedef struct LZ4F_CDict_s LZ4F_CDict; - -/*! LZ4_createCDict() : - * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. - * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. - * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ -LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); -LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict); - - -/*! LZ4_compressFrame_usingCDict() : - * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary. - * If cdict==NULL, compress without a dictionary. - * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). - * If this condition is not respected, function will fail (@return an errorCode). - * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, - * but it's not recommended, as it's the only way to provide dictID in the frame header. - * @return : number of bytes written into dstBuffer. - * or an error code if it fails (can be tested using LZ4F_isError()) */ -LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict( - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* preferencesPtr); - - -/*! LZ4F_compressBegin_usingCDict() : - * Inits streaming dictionary compression, and writes the frame header into dstBuffer. - * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. - * `prefsPtr` is optional : you may provide NULL as argument, - * however, it's the only way to provide dictID in the frame header. - * @return : number of bytes written into dstBuffer for the header, - * or an error code (which can be tested using LZ4F_isError()) */ -LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict( - LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* prefsPtr); - - -/*! LZ4F_decompress_usingDict() : - * Same as LZ4F_decompress(), using a predefined dictionary. - * Dictionary is used "in place", without any preprocessing. - * It must remain accessible throughout the entire frame decoding. */ -LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict( - LZ4F_dctx* dctxPtr, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const void* dict, size_t dictSize, - const LZ4F_decompressOptions_t* decompressOptionsPtr); - - -#if defined (__cplusplus) -} -#endif - #endif /* LZ4FRAME_STATIC_H_0398209384 */ diff --git a/lz4libs/lz4hc.c b/lz4libs/lz4hc.c index f2c2566f..8108ea01 100644 --- a/lz4libs/lz4hc.c +++ b/lz4libs/lz4hc.c @@ -67,6 +67,7 @@ /*=== Constants ===*/ #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) +#define LZ4_OPT_NUM (1<<12) /*=== Macros ===*/ @@ -78,21 +79,33 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtx } dictCtx_directive; /************************************** * HC Compression **************************************/ -static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) +static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = 64 KB; - hc4->base = start - 64 KB; +} + +static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) +{ + uptrval startingOffset = hc4->end - hc4->base; + if (startingOffset > 1 GB) { + LZ4HC_clearTables(hc4); + startingOffset = 0; + } + startingOffset += 64 KB; + hc4->nextToUpdate = (U32) startingOffset; + hc4->base = start - startingOffset; hc4->end = start; - hc4->dictBase = start - 64 KB; - hc4->dictLimit = 64 KB; - hc4->lowLimit = 64 KB; + hc4->dictBase = start - startingOffset; + hc4->dictLimit = (U32) startingOffset; + hc4->lowLimit = (U32) startingOffset; } @@ -123,17 +136,21 @@ LZ4_FORCE_INLINE int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, const BYTE* const iMin, const BYTE* const mMin) { - int back=0; - while ( (ip+back > iMin) - && (match+back > mMin) - && (ip[back-1] == match[back-1])) + int back = 0; + int const min = (int)MAX(iMin - ip, mMin - match); + assert(min <= 0); + assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31)); + assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31)); + while ( (back > min) + && (ip[back-1] == match[back-1]) ) back--; return back; } /* LZ4HC_countPattern() : * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ -static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) +static unsigned +LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) { const BYTE* const iStart = ip; reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32; @@ -165,7 +182,8 @@ static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 c /* LZ4HC_reverseCountPattern() : * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) * read using natural platform endianess */ -static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) +static unsigned +LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) { const BYTE* const iStart = ip; @@ -182,8 +200,10 @@ static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow } typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; +typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; -LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( +LZ4_FORCE_INLINE int +LZ4HC_InsertAndGetWiderMatch ( LZ4HC_CCtx_internal* hc4, const BYTE* const ip, const BYTE* const iLowLimit, @@ -192,19 +212,26 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts, - const int patternAnalysis) + const int patternAnalysis, + const int chainSwap, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; + const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx; const BYTE* const base = hc4->base; const U32 dictLimit = hc4->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE; + const U32 ipIndex = (U32)(ip - base); + const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE; const BYTE* const dictBase = hc4->dictBase; - int const delta = (int)(ip-iLowLimit); + int const lookBackLength = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; + int matchChainPos = 0; U32 const pattern = LZ4_read32(ip); U32 matchIndex; + U32 dictMatchIndex; repeat_state_e repeat = rep_untested; size_t srcPatternLength = 0; @@ -212,86 +239,141 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( /* First Match */ LZ4HC_Insert(hc4, ip); matchIndex = HashTable[LZ4HC_hashPtr(ip)]; - DEBUGLOG(7, "First match at index %u / %u (lowLimit)", - matchIndex, lowLimit); + DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)", + matchIndex, lowestMatchIndex); - while ((matchIndex>=lowLimit) && (nbAttempts)) { - DEBUGLOG(7, "remaining attempts : %i", nbAttempts); + while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) { + int matchLength=0; nbAttempts--; - if (matchIndex >= dictLimit) { + assert(matchIndex < ipIndex); + if (favorDecSpeed && (ipIndex - matchIndex < 8)) { + /* do nothing */ + } else if (matchIndex >= dictLimit) { /* within current Prefix */ const BYTE* const matchPtr = base + matchIndex; - if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) { + assert(matchPtr >= lowPrefixPtr); + assert(matchPtr < ip); + assert(longest >= 1); + if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) { if (LZ4_read32(matchPtr) == pattern) { - int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); - #if 0 - /* more generic but unfortunately slower on clang */ - int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr); - #else - int back = 0; - while ( (ip+back > iLowLimit) - && (matchPtr+back > lowPrefixPtr) - && (ip[back-1] == matchPtr[back-1])) { - back--; - } - #endif - mlt -= back; - - if (mlt > longest) { - longest = mlt; - *matchpos = matchPtr+back; - *startpos = ip+back; - } } - } - } else { /* matchIndex < dictLimit */ + int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0; + matchLength = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + *matchpos = matchPtr + back; + *startpos = ip + back; + } } } + } else { /* lowestMatchIndex <= matchIndex < dictLimit */ const BYTE* const matchPtr = dictBase + matchIndex; if (LZ4_read32(matchPtr) == pattern) { - int mlt; + const BYTE* const dictStart = dictBase + hc4->lowLimit; int back = 0; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - while ( (ip+back > iLowLimit) - && (matchIndex+back > lowLimit) - && (ip[back-1] == matchPtr[back-1])) - back--; - mlt -= back; - if (mlt > longest) { - longest = mlt; - *matchpos = base + matchIndex + back; + matchLength = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) + matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit); + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0; + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + *matchpos = base + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */ *startpos = ip + back; } } } - { U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex); - matchIndex -= nextOffset; - if (patternAnalysis && nextOffset==1) { + if (chainSwap && matchLength==longest) { /* better match => select a better chain */ + assert(lookBackLength==0); /* search forward only */ + if (matchIndex + longest <= ipIndex) { + U32 distanceToNextMatch = 1; + int pos; + for (pos = 0; pos <= longest - MINMATCH; pos++) { + U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos); + if (candidateDist > distanceToNextMatch) { + distanceToNextMatch = candidateDist; + matchChainPos = pos; + } } + if (distanceToNextMatch > 1) { + if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ + matchIndex -= distanceToNextMatch; + continue; + } } } + + { U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex); + if (patternAnalysis && distNextMatch==1 && matchChainPos==0) { + U32 const matchCandidateIdx = matchIndex-1; /* may be a repeated pattern */ if (repeat == rep_untested) { if ( ((pattern & 0xFFFF) == (pattern >> 16)) & ((pattern & 0xFF) == (pattern >> 24)) ) { repeat = rep_confirmed; - srcPatternLength = LZ4HC_countPattern(ip+4, iHighLimit, pattern) + 4; + srcPatternLength = LZ4HC_countPattern(ip+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); } else { repeat = rep_not; } } if ( (repeat == rep_confirmed) - && (matchIndex >= dictLimit) ) { /* same segment only */ - const BYTE* const matchPtr = base + matchIndex; + && (matchCandidateIdx >= dictLimit) ) { /* same segment only */ + const BYTE* const matchPtr = base + matchCandidateIdx; if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); - const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE; - size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern); + const BYTE* const lowestMatchPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE; + size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); size_t const currentSegmentLength = backLength + forwardPatternLength; if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ - matchIndex += (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ + matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ } else { - matchIndex -= (U32)backLength; /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */ - } - } } } } - } /* while ((matchIndex>=lowLimit) && (nbAttempts)) */ + matchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ + if (lookBackLength==0) { /* no back possible */ + size_t const maxML = MIN(currentSegmentLength, srcPatternLength); + if ((size_t)longest < maxML) { + assert(maxML < 2 GB); + longest = (int)maxML; + *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ + *startpos = ip; + } + { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); + if (distToNextPattern > matchIndex) break; /* avoid overflow */ + matchIndex -= distToNextPattern; + } } } + continue; + } } + } } /* PA optimization */ + + /* follow current chain */ + matchIndex -= DELTANEXTU16(chainTable, matchIndex+matchChainPos); + + } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */ + + if (dict == usingDictCtx && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) { + size_t const dictEndOffset = dictCtx->end - dictCtx->base; + assert(dictEndOffset <= 1 GB); + dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; + matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; + while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) { + const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; + + if (LZ4_read32(matchPtr) == pattern) { + int mlt; + int back = 0; + const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0; + mlt -= back; + if (mlt > longest) { + longest = mlt; + *matchpos = base + matchIndex + back; + *startpos = ip + back; + } + } + + { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); + dictMatchIndex -= nextOffset; + matchIndex -= nextOffset; + } + } + } return longest; } @@ -301,13 +383,14 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index tabl const BYTE* const ip, const BYTE* const iLimit, const BYTE** matchpos, const int maxNbAttempts, - const int patternAnalysis) + const int patternAnalysis, + const dictCtx_directive dict) { const BYTE* uselessPtr = ip; /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), * but this won't be the case here, as we define iLowLimit==ip, * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ - return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis); + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio); } @@ -333,7 +416,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( size_t length; BYTE* const token = (*op)++; -#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2) +#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6) static const BYTE* start = NULL; static U32 totalCost = 0; U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start); @@ -342,8 +425,8 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; U32 const cost = 1 + llAdd + ll + 2 + mlAdd; if (start==NULL) start = *anchor; /* only works for single segment */ - //g_debuglog_enable = (pos >= 2228) & (pos <= 2262); - DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u", + /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ + DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u", pos, (U32)(*ip - *anchor), matchLength, (U32)(*ip-match), cost, totalCost); @@ -390,22 +473,19 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( return 0; } -/* btopt */ -#include "lz4opt.h" - - -static int LZ4HC_compress_hashChain ( +LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( LZ4HC_CCtx_internal* const ctx, const char* const source, char* const dest, int* srcSizePtr, int const maxOutputSize, unsigned maxNbAttempts, - limitedOutput_directive limit + const limitedOutput_directive limit, + const dictCtx_directive dict ) { const int inputSize = *srcSizePtr; - const int patternAnalysis = (maxNbAttempts > 64); /* levels 8+ */ + const int patternAnalysis = (maxNbAttempts > 128); /* levels 9+ */ const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; @@ -417,14 +497,14 @@ static int LZ4HC_compress_hashChain ( BYTE* op = (BYTE*) dest; BYTE* oend = op + maxOutputSize; - int ml, ml2, ml3, ml0; + int ml0, ml, ml2, ml3; + const BYTE* start0; + const BYTE* ref0; const BYTE* ref = NULL; const BYTE* start2 = NULL; const BYTE* ref2 = NULL; const BYTE* start3 = NULL; const BYTE* ref3 = NULL; - const BYTE* start0; - const BYTE* ref0; /* init */ *srcSizePtr = 0; @@ -432,36 +512,32 @@ static int LZ4HC_compress_hashChain ( if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* Main Loop */ - while (ip < mflimit) { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis); + while (ip <= mflimit) { + ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict); if (ml encode ML1 */ optr = op; if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; continue; } - if (start0 < ip) { - if (start2 < ip + ml0) { /* empirical */ - ip = start0; - ref = ref0; - ml = ml0; - } - } + if (start0 < ip) { /* first match was skipped at least once */ + if (start2 < ip + ml0) { /* squeezing ML1 between ML0(original ML1) and ML2 */ + ip = start0; ref = ref0; ml = ml0; /* restore initial ML1 */ + } } /* Here, start0==ip */ if ((start2 - ip) < 3) { /* First Match too small : removed */ @@ -489,14 +565,15 @@ static int LZ4HC_compress_hashChain ( } /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ - if (start2 + ml2 < mflimit) + if (start2 + ml2 <= mflimit) { ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, - maxNbAttempts, patternAnalysis); - else + maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio); + } else { ml3 = ml2; + } - if (ml3 == ml2) { /* No better match : 2 sequences to encode */ + if (ml3 == ml2) { /* No better match => encode ML1 and ML2 */ /* ip & ref are known; Now for ml */ if (start2 < ip+ml) ml = (int)(start2 - ip); /* Now, encode 2 sequences */ @@ -541,11 +618,12 @@ static int LZ4HC_compress_hashChain ( } /* - * OK, now we have 3 ascending matches; let's write at least the first one - * ip & ref are known; Now for ml + * OK, now we have 3 ascending matches; + * let's write the first one ML1. + * ip & ref are known; Now decide ml. */ if (start2 < ip+ml) { - if ((start2 - ip) < (int)ML_MASK) { + if ((start2 - ip) < OPTIMAL_ML) { int correction; if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; @@ -562,14 +640,13 @@ static int LZ4HC_compress_hashChain ( optr = op; if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; - ip = start2; - ref = ref2; - ml = ml2; + /* ML2 becomes ML1 */ + ip = start2; ref = ref2; ml = ml2; - start2 = start3; - ref2 = ref3; - ml2 = ml3; + /* ML3 becomes ML2 */ + start2 = start3; ref2 = ref3; ml2 = ml3; + /* let's find a new ML3 */ goto _Search3; } @@ -613,14 +690,24 @@ static int LZ4HC_compress_hashChain ( } -static int LZ4HC_compress_generic ( +static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx, + const char* const source, char* dst, + int* srcSizePtr, int dstCapacity, + int const nbSearches, size_t sufficient_len, + const limitedOutput_directive limit, int const fullUpdate, + const dictCtx_directive dict, + HCfavor_e favorDecSpeed); + + +LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( LZ4HC_CCtx_internal* const ctx, const char* const src, char* const dst, int* const srcSizePtr, int const dstCapacity, int cLevel, - limitedOutput_directive limit + const limitedOutput_directive limit, + const dictCtx_directive dict ) { typedef enum { lz4hc, lz4opt } lz4hc_strat_e; @@ -630,49 +717,110 @@ static int LZ4HC_compress_generic ( U32 targetLength; } cParams_t; static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = { - { lz4hc, 2, 16 }, /* 0, unused */ - { lz4hc, 2, 16 }, /* 1, unused */ - { lz4hc, 2, 16 }, /* 2, unused */ - { lz4hc, 4, 16 }, /* 3 */ - { lz4hc, 8, 16 }, /* 4 */ - { lz4hc, 16, 16 }, /* 5 */ - { lz4hc, 32, 16 }, /* 6 */ - { lz4hc, 64, 16 }, /* 7 */ - { lz4hc, 128, 16 }, /* 8 */ - { lz4hc, 256, 16 }, /* 9 */ - { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ - { lz4opt, 512,128 }, /*11 */ - { lz4opt,8192, LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ + { lz4hc, 2, 16 }, /* 0, unused */ + { lz4hc, 2, 16 }, /* 1, unused */ + { lz4hc, 2, 16 }, /* 2, unused */ + { lz4hc, 4, 16 }, /* 3 */ + { lz4hc, 8, 16 }, /* 4 */ + { lz4hc, 16, 16 }, /* 5 */ + { lz4hc, 32, 16 }, /* 6 */ + { lz4hc, 64, 16 }, /* 7 */ + { lz4hc, 128, 16 }, /* 8 */ + { lz4hc, 256, 16 }, /* 9 */ + { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ + { lz4opt, 512,128 }, /*11 */ + { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ }; + DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr); + if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ ctx->end += *srcSizePtr; if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */ cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); - assert(cLevel >= 0); - assert(cLevel <= LZ4HC_CLEVEL_MAX); { cParams_t const cParam = clTable[cLevel]; + HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio; if (cParam.strat == lz4hc) return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, - cParam.nbSearches, limit); + cParam.nbSearches, limit, dict); assert(cParam.strat == lz4opt); return LZ4HC_compress_optimal(ctx, src, dst, srcSizePtr, dstCapacity, cParam.nbSearches, cParam.targetLength, limit, - cLevel == LZ4HC_CLEVEL_MAX); /* ultra mode */ + cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ + dict, favor); + } +} + +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock); + +static int LZ4HC_compress_generic_noDictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + assert(ctx->dictCtx == NULL); + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx); +} + +static int LZ4HC_compress_generic_dictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + const size_t position = ctx->end - ctx->base - ctx->lowLimit; + assert(ctx->dictCtx != NULL); + if (position >= 64 KB) { + ctx->dictCtx = NULL; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else if (position == 0 && *srcSizePtr > 4 KB) { + memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); + LZ4HC_setExternalDict(ctx, (const BYTE *)src); + ctx->compressionLevel = (short)cLevel; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); + } +} + +static int LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + if (ctx->dictCtx == NULL) { + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); } } int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } -int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel); LZ4HC_init (ctx, (const BYTE*)src); if (dstCapacity < LZ4_compressBound(srcSize)) return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); @@ -680,10 +828,17 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit); } +int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + LZ4_resetStreamHC ((LZ4_streamHC_t*)state, compressionLevel); + return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel); +} + int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 - LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); + LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); #else LZ4_streamHC_t state; LZ4_streamHC_t* const statePtr = &state; @@ -700,6 +855,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; + LZ4_resetStreamHC((LZ4_streamHC_t*)LZ4HC_Data, cLevel); LZ4HC_init(ctx, (const BYTE*) source); return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize); } @@ -710,8 +866,15 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i * Streaming Functions **************************************/ /* allocation */ -LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { +LZ4_streamHC_t* LZ4_createStreamHC(void) { + LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); + if (LZ4_streamHCPtr==NULL) return NULL; + LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); + return LZ4_streamHCPtr; +} + +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { + DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr); if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ free(LZ4_streamHCPtr); return 0; @@ -722,36 +885,61 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ + DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel); + LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0; + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} + +void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); + LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { - if (compressionLevel < 1) compressionLevel = 1; + if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; - LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel; +} + +void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor) +{ + LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0); } int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize); if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; } + LZ4_resetStreamHC(LZ4_streamHCPtr, ctxPtr->compressionLevel); LZ4HC_init (ctxPtr, (const BYTE*)dictionary); ctxPtr->end = (const BYTE*)dictionary + dictSize; if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; } +void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) { + working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL; +} /* compression */ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); + if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; @@ -768,6 +956,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, limitedOutput_directive limit) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(4, "LZ4_compressHC_continue_generic(%p, %p, %d)", LZ4_streamHCPtr, src, *srcSizePtr); /* auto-init if forgotten */ if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src); @@ -816,6 +1005,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS { LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); + DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; @@ -855,17 +1045,17 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer) { LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ + LZ4_resetStreamHC((LZ4_streamHC_t*)state, ((LZ4_streamHC_t*)state)->internal_donotuse.compressionLevel); LZ4HC_init(ctx, (const BYTE*)inputBuffer); - ctx->inputBuffer = (BYTE*)inputBuffer; return 0; } -void* LZ4_createHC (char* inputBuffer) +void* LZ4_createHC (const char* inputBuffer) { - LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t)); + LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); if (hc4 == NULL) return NULL; /* not enough memory */ + LZ4_resetStreamHC(hc4, 0 /* compressionLevel */); LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); - hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer; return hc4; } @@ -887,7 +1077,333 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, c char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { - LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; - int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); - return (char*)(hc4->inputBuffer + dictSize); + LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data; + const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit; + LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel); + /* avoid const char * -> char * conversion warning :( */ + return (char *)(uptrval)bufferStart; +} + + +/* ================================================ + * LZ4 Optimal parser (levels 10-12) + * ===============================================*/ +typedef struct { + int price; + int off; + int mlen; + int litlen; +} LZ4HC_optimal_t; + +/* price in bytes */ +LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) +{ + int price = litlen; + if (litlen >= (int)RUN_MASK) + price += 1 + (litlen-RUN_MASK)/255; + return price; +} + + +/* requires mlen >= MINMATCH */ +LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) +{ + int price = 1 + 2 ; /* token + 16-bit offset */ + + price += LZ4HC_literalsPrice(litlen); + + if (mlen >= (int)(ML_MASK+MINMATCH)) + price += 1 + (mlen-(ML_MASK+MINMATCH))/255; + + return price; +} + + +typedef struct { + int off; + int len; +} LZ4HC_match_t; + +LZ4_FORCE_INLINE LZ4HC_match_t +LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, + const BYTE* ip, const BYTE* const iHighLimit, + int minLen, int nbSearches, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + LZ4HC_match_t match = { 0 , 0 }; + const BYTE* matchPtr = NULL; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); + if (matchLength <= minLen) return match; + if (favorDecSpeed) { + if ((matchLength>18) & (matchLength<=36)) matchLength=18; /* favor shortcut */ + } + match.len = matchLength; + match.off = (int)(ip-matchPtr); + return match; } + + +static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, + const char* const source, + char* dst, + int* srcSizePtr, + int dstCapacity, + int const nbSearches, + size_t sufficient_len, + const limitedOutput_directive limit, + int const fullUpdate, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ +#define TRAILING_LITERALS 3 + LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */ + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + BYTE* op = (BYTE*) dst; + BYTE* opSaved = (BYTE*) dst; + BYTE* oend = op + dstCapacity; + + /* init */ + DEBUGLOG(5, "LZ4HC_compress_optimal"); + *srcSizePtr = 0; + if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; + + /* Main Loop */ + assert(ip - anchor < LZ4_MAX_INPUT_SIZE); + while (ip <= mflimit) { + int const llen = (int)(ip - anchor); + int best_mlen, best_off; + int cur, last_match_pos = 0; + + LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + if (firstMatch.len==0) { ip++; continue; } + + if ((size_t)firstMatch.len > sufficient_len) { + /* good enough solution : immediate encoding */ + int const firstML = firstMatch.len; + const BYTE* const matchPos = ip - firstMatch.off; + opSaved = op; + if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ + goto _dest_overflow; + continue; + } + + /* set prices for first positions (literals) */ + { int rPos; + for (rPos = 0 ; rPos < MINMATCH ; rPos++) { + int const cost = LZ4HC_literalsPrice(llen + rPos); + opt[rPos].mlen = 1; + opt[rPos].off = 0; + opt[rPos].litlen = llen + rPos; + opt[rPos].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + rPos, cost, opt[rPos].litlen); + } } + /* set prices using initial match */ + { int mlen = MINMATCH; + int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ + int const offset = firstMatch.off; + assert(matchML < LZ4_OPT_NUM); + for ( ; mlen <= matchML ; mlen++) { + int const cost = LZ4HC_sequencePrice(llen, mlen); + opt[mlen].mlen = mlen; + opt[mlen].off = offset; + opt[mlen].litlen = llen; + opt[mlen].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", + mlen, cost, mlen); + } } + last_match_pos = firstMatch.len; + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + + /* check further positions */ + for (cur = 1; cur < last_match_pos; cur++) { + const BYTE* const curPtr = ip + cur; + LZ4HC_match_t newMatch; + + if (curPtr > mflimit) break; + DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", + cur, opt[cur].price, opt[cur+1].price, cur+1); + if (fullUpdate) { + /* not useful to search here if next position has same (or lower) cost */ + if ( (opt[cur+1].price <= opt[cur].price) + /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ + && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) ) + continue; + } else { + /* not useful to search here if next position has same (or lower) cost */ + if (opt[cur+1].price <= opt[cur].price) continue; + } + + DEBUGLOG(7, "search at rPos:%u", cur); + if (fullUpdate) + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + else + /* only test matches of minimum length; slightly faster, but misses a few bytes */ + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed); + if (!newMatch.len) continue; + + if ( ((size_t)newMatch.len > sufficient_len) + || (newMatch.len + cur >= LZ4_OPT_NUM) ) { + /* immediate encoding */ + best_mlen = newMatch.len; + best_off = newMatch.off; + last_match_pos = cur + 1; + goto encode; + } + + /* before match : set price with literals at beginning */ + { int const baseLitlen = opt[cur].litlen; + int litlen; + for (litlen = 1; litlen < MINMATCH; litlen++) { + int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen); + int const pos = cur + litlen; + if (price < opt[pos].price) { + opt[pos].mlen = 1; /* literal */ + opt[pos].off = 0; + opt[pos].litlen = baseLitlen+litlen; + opt[pos].price = price; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", + pos, price, opt[pos].litlen); + } } } + + /* set prices using match at position = cur */ + { int const matchML = newMatch.len; + int ml = MINMATCH; + + assert(cur + newMatch.len < LZ4_OPT_NUM); + for ( ; ml <= matchML ; ml++) { + int const pos = cur + ml; + int const offset = newMatch.off; + int price; + int ll; + DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)", + pos, last_match_pos); + if (opt[cur].mlen == 1) { + ll = opt[cur].litlen; + price = ((cur > ll) ? opt[cur - ll].price : 0) + + LZ4HC_sequencePrice(ll, ml); + } else { + ll = 0; + price = opt[cur].price + LZ4HC_sequencePrice(0, ml); + } + + assert((U32)favorDecSpeed <= 1); + if (pos > last_match_pos+TRAILING_LITERALS + || price <= opt[pos].price - (int)favorDecSpeed) { + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", + pos, price, ml); + assert(pos < LZ4_OPT_NUM); + if ( (ml == matchML) /* last pos of last match */ + && (last_match_pos < pos) ) + last_match_pos = pos; + opt[pos].mlen = ml; + opt[pos].off = offset; + opt[pos].litlen = ll; + opt[pos].price = price; + } } } + /* complete following positions with literals */ + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + } /* for (cur = 1; cur <= last_match_pos; cur++) */ + + best_mlen = opt[last_match_pos].mlen; + best_off = opt[last_match_pos].off; + cur = last_match_pos - best_mlen; + + encode: /* cur, last_match_pos, best_mlen, best_off must be set */ + assert(cur < LZ4_OPT_NUM); + assert(last_match_pos >= 1); /* == 1 when only one candidate */ + DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos); + { int candidate_pos = cur; + int selected_matchLength = best_mlen; + int selected_offset = best_off; + while (1) { /* from end to beginning */ + int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ + int const next_offset = opt[candidate_pos].off; + DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength); + opt[candidate_pos].mlen = selected_matchLength; + opt[candidate_pos].off = selected_offset; + selected_matchLength = next_matchLength; + selected_offset = next_offset; + if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */ + assert(next_matchLength > 0); /* can be 1, means literal */ + candidate_pos -= next_matchLength; + } } + + /* encode all recorded sequences in order */ + { int rPos = 0; /* relative position (to ip) */ + while (rPos < last_match_pos) { + int const ml = opt[rPos].mlen; + int const offset = opt[rPos].off; + if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ + rPos += ml; + assert(ml >= MINMATCH); + assert((offset >= 1) && (offset <= MAX_DISTANCE)); + opSaved = op; + if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ + goto _dest_overflow; + } } + } /* while (ip <= mflimit) */ + + _last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + litLength + lastRunSize; + if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; /* Check output limit */ + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (size_t)(oend - op) - 1; + litLength = (lastRunSize + 255 - RUN_MASK) / 255; + lastRunSize -= litLength; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + return (int) ((char*)op-dst); + + _dest_overflow: + if (limit == limitedDestSize) { + op = opSaved; /* restore correct out pointer */ + goto _last_literals; + } + return 0; + } diff --git a/lz4libs/lz4hc.h b/lz4libs/lz4hc.h index d41bf420..bb5e0737 100644 --- a/lz4libs/lz4hc.h +++ b/lz4libs/lz4hc.h @@ -141,35 +141,39 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) #include -typedef struct +typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; +struct LZ4HC_CCtx_internal { uint32_t hashTable[LZ4HC_HASHTABLESIZE]; uint16_t chainTable[LZ4HC_MAXD]; const uint8_t* end; /* next block here to continue on current prefix */ const uint8_t* base; /* All index relative to this position */ const uint8_t* dictBase; /* alternate base for extDict */ - uint8_t* inputBuffer; /* deprecated */ uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ - int compressionLevel; -} LZ4HC_CCtx_internal; + short compressionLevel; + short favorDecSpeed; + const LZ4HC_CCtx_internal* dictCtx; +}; #else -typedef struct +typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; +struct LZ4HC_CCtx_internal { unsigned int hashTable[LZ4HC_HASHTABLESIZE]; unsigned short chainTable[LZ4HC_MAXD]; const unsigned char* end; /* next block here to continue on current prefix */ const unsigned char* base; /* All index relative to this position */ const unsigned char* dictBase; /* alternate base for extDict */ - unsigned char* inputBuffer; /* deprecated */ unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ - int compressionLevel; -} LZ4HC_CCtx_internal; + short compressionLevel; + short favorDecSpeed; + const LZ4HC_CCtx_internal* dictCtx; +}; #endif @@ -195,25 +199,32 @@ union LZ4_streamHC_u { /* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ /* deprecated compression functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); - -/* Deprecated Streaming functions using older model; should no longer be used */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC (const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* Obsolete streaming functions; degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, use of + * LZ4_slideInputBufferHC() will truncate the history of the stream, rather + * than preserve a window-sized chunk of history. + */ +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer); +LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data); +LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int LZ4_sizeofStreamStateHC(void); +LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API int LZ4_resetStreamStateHC(void* state, char* inputBuffer); #if defined (__cplusplus) @@ -244,9 +255,9 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStr * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` */ int LZ4_compress_HC_destSize(void* LZ4HC_Data, - const char* src, char* dst, - int* srcSizePtr, int targetDstSize, - int compressionLevel); + const char* src, char* dst, + int* srcSizePtr, int targetDstSize, + int compressionLevel); /*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental) * Similar as LZ4_compress_HC_continue(), @@ -266,7 +277,71 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, */ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); - +/*! LZ4_favorDecompressionSpeed() : v1.8.2 (experimental) + * Parser will select decisions favoring decompression over compression ratio. + * Only work at highest compression settings (level >= LZ4HC_CLEVEL_OPT_MIN) + */ +void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor); + +/*! LZ4_resetStreamHC_fast() : + * When an LZ4_streamHC_t is known to be in a internally coherent state, + * it can often be prepared for a new compression with almost no work, only + * sometimes falling back to the full, expensive reset that is always required + * when the stream is in an indeterminate state (i.e., the reset performed by + * LZ4_resetStreamHC()). + * + * LZ4_streamHCs are guaranteed to be in a valid state when: + * - returned from LZ4_createStreamHC() + * - reset by LZ4_resetStreamHC() + * - memset(stream, 0, sizeof(LZ4_streamHC_t)) + * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast() + * - the stream was in a valid state and was then used in any compression call + * that returned success + * - the stream was in an indeterminate state and was used in a compression + * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that + * returned success + */ +void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); + +/*! LZ4_compress_HC_extStateHC_fastReset() : + * A variant of LZ4_compress_HC_extStateHC(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStreamHC_fast() for a definition of + * "correctly initialized"). From a high level, the difference is that this + * function initializes the provided state with a call to + * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a + * call to LZ4_resetStreamHC(). + */ +int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); + +/*! LZ4_attach_HC_dictionary() : + * This is an experimental API that allows for the efficient use of a + * static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a + * working LZ4_streamHC_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDictHC() should + * be expected to work. + * + * Alternatively, the provided dictionary stream pointer may be NULL, in which + * case any existing dictionary stream is unset. + * + * A dictionary should only be attached to a stream without any history (i.e., + * a stream that has just been reset). + * + * The dictionary will remain attached to the working stream only for the + * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the + * dictionary context association from the working stream. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the lifetime of the stream session. + */ +LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream); #endif /* LZ4_HC_SLO_098092834 */ #endif /* LZ4_HC_STATIC_LINKING_ONLY */ diff --git a/lz4libs/lz4opt.h b/lz4libs/lz4opt.h deleted file mode 100644 index 5a8438c1..00000000 --- a/lz4libs/lz4opt.h +++ /dev/null @@ -1,356 +0,0 @@ -/* - lz4opt.h - Optimal Mode of LZ4 - Copyright (C) 2015-2017, Przemyslaw Skibinski - Note : this file is intended to be included within lz4hc.c - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -#define LZ4_OPT_NUM (1<<12) - -typedef struct { - int price; - int off; - int mlen; - int litlen; -} LZ4HC_optimal_t; - - -/* price in bytes */ -LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) -{ - int price = litlen; - if (litlen >= (int)RUN_MASK) - price += 1 + (litlen-RUN_MASK)/255; - return price; -} - - -/* requires mlen >= MINMATCH */ -LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) -{ - int price = 1 + 2 ; /* token + 16-bit offset */ - - price += LZ4HC_literalsPrice(litlen); - - if (mlen >= (int)(ML_MASK+MINMATCH)) - price += 1 + (mlen-(ML_MASK+MINMATCH))/255; - - return price; -} - - -/*-************************************* -* Match finder -***************************************/ -typedef struct { - int off; - int len; -} LZ4HC_match_t; - -LZ4_FORCE_INLINE -LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, - const BYTE* ip, const BYTE* const iHighLimit, - int minLen, int nbSearches) -{ - LZ4HC_match_t match = { 0 , 0 }; - const BYTE* matchPtr = NULL; - /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), - * but this won't be the case here, as we define iLowLimit==ip, - * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ - int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, - ip, ip, iHighLimit, minLen, &matchPtr, &ip, - nbSearches, 1 /* patternAnalysis */); - if (matchLength <= minLen) return match; - match.len = matchLength; - match.off = (int)(ip-matchPtr); - return match; -} - - -static int LZ4HC_compress_optimal ( - LZ4HC_CCtx_internal* ctx, - const char* const source, - char* dst, - int* srcSizePtr, - int dstCapacity, - int const nbSearches, - size_t sufficient_len, - limitedOutput_directive limit, - int const fullUpdate - ) -{ -#define TRAILING_LITERALS 3 - LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* this uses a bit too much stack memory to my taste ... */ - - const BYTE* ip = (const BYTE*) source; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - BYTE* op = (BYTE*) dst; - BYTE* opSaved = (BYTE*) dst; - BYTE* oend = op + dstCapacity; - - /* init */ - DEBUGLOG(5, "LZ4HC_compress_optimal"); - *srcSizePtr = 0; - if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ - if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; - - /* Main Loop */ - assert(ip - anchor < LZ4_MAX_INPUT_SIZE); - while (ip < mflimit) { - int const llen = (int)(ip - anchor); - int best_mlen, best_off; - int cur, last_match_pos = 0; - - LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches); - if (firstMatch.len==0) { ip++; continue; } - - if ((size_t)firstMatch.len > sufficient_len) { - /* good enough solution : immediate encoding */ - int const firstML = firstMatch.len; - const BYTE* const matchPos = ip - firstMatch.off; - opSaved = op; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ - goto _dest_overflow; - continue; - } - - /* set prices for first positions (literals) */ - { int rPos; - for (rPos = 0 ; rPos < MINMATCH ; rPos++) { - int const cost = LZ4HC_literalsPrice(llen + rPos); - opt[rPos].mlen = 1; - opt[rPos].off = 0; - opt[rPos].litlen = llen + rPos; - opt[rPos].price = cost; - DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", - rPos, cost, opt[rPos].litlen); - } } - /* set prices using initial match */ - { int mlen = MINMATCH; - int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ - int const offset = firstMatch.off; - assert(matchML < LZ4_OPT_NUM); - for ( ; mlen <= matchML ; mlen++) { - int const cost = LZ4HC_sequencePrice(llen, mlen); - opt[mlen].mlen = mlen; - opt[mlen].off = offset; - opt[mlen].litlen = llen; - opt[mlen].price = cost; - DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", - mlen, cost, mlen); - } } - last_match_pos = firstMatch.len; - { int addLit; - for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { - opt[last_match_pos+addLit].mlen = 1; /* literal */ - opt[last_match_pos+addLit].off = 0; - opt[last_match_pos+addLit].litlen = addLit; - opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); - DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", - last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); - } } - - /* check further positions */ - for (cur = 1; cur < last_match_pos; cur++) { - const BYTE* const curPtr = ip + cur; - LZ4HC_match_t newMatch; - - if (curPtr >= mflimit) break; - DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", - cur, opt[cur].price, opt[cur+1].price, cur+1); - if (fullUpdate) { - /* not useful to search here if next position has same (or lower) cost */ - if ( (opt[cur+1].price <= opt[cur].price) - /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ - && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) ) - continue; - } else { - /* not useful to search here if next position has same (or lower) cost */ - if (opt[cur+1].price <= opt[cur].price) continue; - } - - DEBUGLOG(7, "search at rPos:%u", cur); - if (fullUpdate) - newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches); - else - /* only test matches of minimum length; slightly faster, but misses a few bytes */ - newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches); - if (!newMatch.len) continue; - - if ( ((size_t)newMatch.len > sufficient_len) - || (newMatch.len + cur >= LZ4_OPT_NUM) ) { - /* immediate encoding */ - best_mlen = newMatch.len; - best_off = newMatch.off; - last_match_pos = cur + 1; - goto encode; - } - - /* before match : set price with literals at beginning */ - { int const baseLitlen = opt[cur].litlen; - int litlen; - for (litlen = 1; litlen < MINMATCH; litlen++) { - int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen); - int const pos = cur + litlen; - if (price < opt[pos].price) { - opt[pos].mlen = 1; /* literal */ - opt[pos].off = 0; - opt[pos].litlen = baseLitlen+litlen; - opt[pos].price = price; - DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", - pos, price, opt[pos].litlen); - } } } - - /* set prices using match at position = cur */ - { int const matchML = newMatch.len; - int ml = MINMATCH; - - assert(cur + newMatch.len < LZ4_OPT_NUM); - for ( ; ml <= matchML ; ml++) { - int const pos = cur + ml; - int const offset = newMatch.off; - int price; - int ll; - DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)", - pos, last_match_pos); - if (opt[cur].mlen == 1) { - ll = opt[cur].litlen; - price = ((cur > ll) ? opt[cur - ll].price : 0) - + LZ4HC_sequencePrice(ll, ml); - } else { - ll = 0; - price = opt[cur].price + LZ4HC_sequencePrice(0, ml); - } - - if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) { - DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", - pos, price, ml); - assert(pos < LZ4_OPT_NUM); - if ( (ml == matchML) /* last pos of last match */ - && (last_match_pos < pos) ) - last_match_pos = pos; - opt[pos].mlen = ml; - opt[pos].off = offset; - opt[pos].litlen = ll; - opt[pos].price = price; - } } } - /* complete following positions with literals */ - { int addLit; - for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { - opt[last_match_pos+addLit].mlen = 1; /* literal */ - opt[last_match_pos+addLit].off = 0; - opt[last_match_pos+addLit].litlen = addLit; - opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); - DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); - } } - } /* for (cur = 1; cur <= last_match_pos; cur++) */ - - best_mlen = opt[last_match_pos].mlen; - best_off = opt[last_match_pos].off; - cur = last_match_pos - best_mlen; - -encode: /* cur, last_match_pos, best_mlen, best_off must be set */ - assert(cur < LZ4_OPT_NUM); - assert(last_match_pos >= 1); /* == 1 when only one candidate */ - DEBUGLOG(6, "reverse traversal, looking for shortest path") - DEBUGLOG(6, "last_match_pos = %i", last_match_pos); - { int candidate_pos = cur; - int selected_matchLength = best_mlen; - int selected_offset = best_off; - while (1) { /* from end to beginning */ - int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ - int const next_offset = opt[candidate_pos].off; - DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength); - opt[candidate_pos].mlen = selected_matchLength; - opt[candidate_pos].off = selected_offset; - selected_matchLength = next_matchLength; - selected_offset = next_offset; - if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */ - assert(next_matchLength > 0); /* can be 1, means literal */ - candidate_pos -= next_matchLength; - } } - - /* encode all recorded sequences in order */ - { int rPos = 0; /* relative position (to ip) */ - while (rPos < last_match_pos) { - int const ml = opt[rPos].mlen; - int const offset = opt[rPos].off; - if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ - rPos += ml; - assert(ml >= MINMATCH); - assert((offset >= 1) && (offset <= MAX_DISTANCE)); - opSaved = op; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ - goto _dest_overflow; - } } - } /* while (ip < mflimit) */ - -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ - size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; - size_t const totalSize = 1 + litLength + lastRunSize; - if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ - if (limit && (op + totalSize > oend)) { - if (limit == limitedOutput) return 0; /* Check output limit */ - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (size_t)(oend - op) - 1; - litLength = (lastRunSize + 255 - RUN_MASK) / 255; - lastRunSize -= litLength; - } - ip = anchor + lastRunSize; - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = (RUN_MASK << ML_BITS); - for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize << ML_BITS); - } - memcpy(op, anchor, lastRunSize); - op += lastRunSize; - } - - /* End */ - *srcSizePtr = (int) (((const char*)ip) - source); - return (int) ((char*)op-dst); - -_dest_overflow: - if (limit == limitedDestSize) { - op = opSaved; /* restore correct out pointer */ - goto _last_literals; - } - return 0; -} diff --git a/lz4libs/xxhash.c b/lz4libs/xxhash.c index bcf1f1db..3fc97fd2 100644 --- a/lz4libs/xxhash.c +++ b/lz4libs/xxhash.c @@ -52,7 +52,7 @@ #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define XXH_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) # define XXH_FORCE_MEMORY_ACCESS 1 # endif From 3aa6135d8104cac474326d915f290ecbb5014fde Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 9 Jun 2018 08:34:29 +0100 Subject: [PATCH 031/280] Add test which checks for growing memory usage for block funcs --- tests/block/test_block_3.py | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/block/test_block_3.py diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py new file mode 100644 index 00000000..0c3fb082 --- /dev/null +++ b/tests/block/test_block_3.py @@ -0,0 +1,38 @@ +import lz4.block +import pytest + + +test_data = [ + (b'a' * 1024 * 1024), +] + + +@pytest.fixture( + params=test_data, + ids=[ + 'data' + str(i) for i in range(len(test_data)) + ] +) +def data(request): + return request.param + + +def test_block_decompress_mem_usage(data): + tracemalloc = pytest.importorskip('tracemalloc') + + tracemalloc.start() + + compressed = lz4.block.compress(data) + prev_snapshot = None + + for i in range(1000): + decompressed = lz4.block.decompress(compressed) # noqa: F841 + + if i % 100 == 0: + snapshot = tracemalloc.take_snapshot() + + if prev_snapshot: + stats = snapshot.compare_to(prev_snapshot, 'lineno') + assert stats[0].size_diff < (1024 * 4) + + prev_snapshot = snapshot From 2ffcefd17c21c9d16a4ea17c20e7d2134b923e8d Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sat, 9 Jun 2018 19:24:22 -0700 Subject: [PATCH 032/280] Update pypi.python.org URL to pypi.org For details on the new PyPI, see the blog post: https://pythoninsider.blogspot.ca/2018/04/new-pypi-launched-legacy-pypi-shutting.html --- docs/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.rst b/docs/install.rst index 1258b119..270be536 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -1,7 +1,7 @@ Install ======= -The package is hosted on `PyPI `_ and so can be +The package is hosted on `PyPI `_ and so can be installed via pip:: $ pip install lz4 From 38d641c6125d9affdd1b7a061d085c60557f67bd Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 21 Jun 2018 23:26:23 +0100 Subject: [PATCH 033/280] Increase memory use tolerance in test_frame_5.py The original values allowed the tests to succeed when each test file was run under pytest individually (e.g. via tox), but when run using `python setup.py test` they would fail. This is because the latter invocation collects all tests together, increasing the python process memory usage, and increasing the variance of memory usage. --- tests/frame/test_frame_5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index b61aec73..274a4181 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -32,7 +32,7 @@ def test_frame_decompress_mem_usage(data): if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 4) + assert stats[0].size_diff < (1024 * 15) prev_snapshot = snapshot @@ -56,7 +56,7 @@ def test_frame_decompress_chunk_mem_usage(data): if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 10) + assert stats[0].size_diff < (1024 * 15) prev_snapshot = snapshot @@ -79,7 +79,7 @@ def test_frame_open_decompress_mem_usage(data): if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 10) + assert stats[0].size_diff < (1024 * 15) prev_snapshot = snapshot From db7bfa279dc0c1ab6266e3cf03f79a3e806df1b7 Mon Sep 17 00:00:00 2001 From: Nikita Chepanov Date: Fri, 22 Jun 2018 15:49:15 -0400 Subject: [PATCH 034/280] Gracefully handle old versions of liblz4 Fixes: #136 Handles ValueError from pkgconfig for older versions of liblz4 that don't follow conventional X.Y.Z versioning schema. Falls back to using bundled version of liblz4 --- setup.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 73a8e2ae..08e8fdb7 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,21 @@ # pkgconfig is not installed. It will be installed by setup_requires. pass else: - try: - liblz4_found = pkgconfig.installed('liblz4', LZ4_REQUIRED_VERSION) - py3c_found = pkgconfig.installed('py3c', PY3C_REQUIRED_VERSION) - except EnvironmentError: - # Windows, no pkg-config present - pass + def pkgconfig_installed(lib, required_version, default): + installed = default + try: + installed = pkgconfig.installed(lib, required_version) + except EnvironmentError: + # Windows, no pkg-config present + pass + except ValueError: + # pkgconfig was unable to determine if + # required version of liblz4 is available + # Bundled version of liblz4 will be used + pass + return installed + liblz4_found = pkgconfig_installed('liblz4', LZ4_REQUIRED_VERSION, default=False) + py3c_found = pkgconfig_installed('py3c', PY3C_REQUIRED_VERSION, default=False) # Set up the extension modules. If a system wide lz4 library is found, and is From 10bffabb8791e301fb94b8ffadaf7a770fb5f20f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 7 Jul 2018 10:30:30 +0100 Subject: [PATCH 035/280] Fix up decompress_chunk docstring --- lz4/frame/_frame.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 5e8dbcac..2beb72c8 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1644,12 +1644,12 @@ PyDoc_STRVAR PyDoc_STRVAR ( decompress_chunk__doc, - "decompress(context, data)\n" \ + "decompress_chunk(context, data)\n" \ "\n" \ - "Decompresses part of a frame of compressed data.\n" \ - "\n" \ - "The returned uncompressed data should be concatenated with the data returned\n" \ - "from previous calls to `lz4.frame.decompress_chunk`\n" \ + "Decompresses part of a frame of compressed data.\n" \ + "\n" \ + "The returned uncompressed data should be concatenated with the data\n" \ + "returned from previous calls to `lz4.frame.decompress_chunk`\n" \ "\n" \ "Args:\n" \ " context (dCtx): decompression context\n" \ @@ -1664,7 +1664,7 @@ PyDoc_STRVAR " default is ``False``.\n" \ "\n" \ "Returns:\n" \ - " tuple: uncompressed data, bytes read, end of frame indicator\n" \ + " tuple: uncompressed data, bytes read, end of frame indicator\n" \ "\n" \ " This function returns a tuple consisting of:\n" \ "\n" \ @@ -1672,8 +1672,8 @@ PyDoc_STRVAR " - The number of bytes consumed from input ``data`` as an ``int``\n" \ " - The end of frame indicator as a ``bool``.\n" \ "\n" - "The end of frame indicator is ``True`` if the end of the compressed frame\n" \ - "has been reached, or ``False`` otherwise\n" + "The end of frame indicator is ``True`` if the end of the compressed\n" \ + "frame has been reached, or ``False`` otherwise\n" ); static PyMethodDef module_methods[] = From a09b965b39cd6af01f04059f4023619a936efba4 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 7 Jul 2018 17:19:46 +0100 Subject: [PATCH 036/280] Set options.stableDest=0 for max_length=-1 (#140) If max_length is -1, then there is potential for a PyMem_Realloc to occur later. In that case, the LZ4 library can't rely on the pointer it holds to the last 64 bytes to point to allocated memory, since we've potentially moved the data in memory. --- lz4/frame/_frame.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 2beb72c8..e1faa8ae 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1082,7 +1082,7 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size, Py_UNBLOCK_THREADS - if (full_frame) + if (full_frame && max_length >= 0) { options.stableDst = 1; } From 02ffd7fa86bf7511264b240e64a073a5f3a515aa Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 7 Jul 2018 17:36:06 +0100 Subject: [PATCH 037/280] Add a comment about stableDst to code --- lz4/frame/_frame.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index e1faa8ae..fded17ed 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1082,7 +1082,10 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size, Py_UNBLOCK_THREADS - if (full_frame && max_length >= 0) + /* Only set stableDst = 1 if we are sure no PyMem_Realloc will be called since + when stableDst = 1 the LZ4 library stores a pointer to the last compressed + data, which may be invalid after a PyMem_Realloc. */ + if (full_frame && max_length >= 0) { options.stableDst = 1; } From cd87d1bcb586a072e0712b0094e0a82e838a2cc0 Mon Sep 17 00:00:00 2001 From: Yann Diorcet Date: Mon, 2 Jul 2018 17:59:10 +0200 Subject: [PATCH 038/280] Frame: Correctly fill to 0 the structures --- lz4/frame/_frame.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index fded17ed..10e1281f 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -471,6 +471,8 @@ compress_chunk (PyObject * Py_UNUSED (self), PyObject * args, NULL }; + memset (&compress_options, 0, sizeof compress_options); + #if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, keywds, "Oy*|p", kwlist, &py_context, @@ -594,6 +596,9 @@ compress_flush (PyObject * Py_UNUSED (self), PyObject * args, PyObject * keywds) "return_bytearray", NULL }; + + memset (&compress_options, 0, sizeof compress_options); + #if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, keywds, "O|pp", kwlist, &py_context, @@ -1017,6 +1022,8 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size, LZ4F_decompressOptions_t options; int end_of_frame = 0; + memset(&options, 0, sizeof options); + Py_BEGIN_ALLOW_THREADS source_cursor = source; From defc784416137b0fb0b3b006262dfa6acba15bf5 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Mon, 30 Jul 2018 19:10:56 +0100 Subject: [PATCH 039/280] Enable Python 3.7 builds --- .travis.yml | 1 + appveyor.yml | 4 +++- setup.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cc722345..508a9b91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - 3.4 - 3.5 - 3.6 + - 3.7 install: - pip install -U pip - pip install tox diff --git a/appveyor.yml b/appveyor.yml index 6399da20..84449c9a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,10 +16,12 @@ environment: DISTUTILS_USE_SDK: "1" - PYTHON: "C:\\Python35-x64" - PYTHON: "C:\\Python36-x64" - - PYTHON: "C:\\Python36" + - PYTHON: "C:\\Python37-x64" - PYTHON: "C:\\Python27" - PYTHON: "C:\\Python34" - PYTHON: "C:\\Python35" + - PYTHON: "C:\\Python36" + - PYTHON: "C:\\Python37" matrix: # Immediately finish build once one of the jobs fails. diff --git a/setup.py b/setup.py index 08e8fdb7..80665fdf 100644 --- a/setup.py +++ b/setup.py @@ -185,5 +185,6 @@ def pkgconfig_installed(lib, required_version, default): 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ], ) From 626b602511cc959ee95f3dfaa28734b4fcf6384e Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Mon, 30 Jul 2018 16:08:26 +0100 Subject: [PATCH 040/280] Define new LZ4BlockError exception and relax error checking This commit changes the handling of errors for block decompression when uncompressed_size > 0. In this case, we no longer check that the uncompressedd data has exactly the size specified by uncompressed_size: this argument is now used as an upper bound. In addition, we introduce a new exception: LZ4BlockError which is raised whenever the LZ4 library fails. These two changes together allow "guessing" of the uncompressed data size: simply set uncompressed_size to a number and try decompress, and catch LZ4BlockError. If LZ4BlockError is raised, increase uncompressed_size and try again. --- lz4/block/__init__.py | 2 +- lz4/block/_block.c | 39 ++++++++++++++++++++++++++++--------- tests/block/test_block_1.py | 28 ++++++++++++++++---------- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/lz4/block/__init__.py b/lz4/block/__init__.py index 2f4ff026..6662bab4 100644 --- a/lz4/block/__init__.py +++ b/lz4/block/__init__.py @@ -1 +1 @@ -from ._block import compress, decompress # noqa: F401 +from ._block import compress, decompress, LZ4BlockError # noqa: F401 diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 37d02dee..61bafe52 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -89,6 +89,8 @@ typedef enum HIGH_COMPRESSION } compression_type; +static PyObject * LZ4BlockError; + static inline int lz4_compress_generic (int comp, char* source, char* dest, int source_size, int dest_size, char* dict, int dict_size, int acceleration, int compression) @@ -251,7 +253,7 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) if (output_size <= 0) { - PyErr_SetString (PyExc_ValueError, "Compression failed"); + PyErr_SetString (LZ4BlockError, "Compression failed"); PyMem_Free (dest); return NULL; } @@ -360,7 +362,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) { PyBuffer_Release(&source); PyBuffer_Release(&dict); - PyErr_Format (PyExc_ValueError, "Invalid size in header: 0x%zu", + PyErr_Format (PyExc_ValueError, "Invalid size: 0x%zu", dest_size); return NULL; } @@ -384,14 +386,15 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) if (output_size < 0) { - PyErr_Format (PyExc_ValueError, "Corrupt input at byte %u", -output_size); + PyErr_Format (LZ4BlockError, + "Decompression failed: corrupt input or insufficient space in destination buffer. Error code: %u", + -output_size); PyMem_Free (dest); return NULL; } - else if ((size_t)output_size != dest_size) + else if (((size_t)output_size != dest_size) && (uncompressed_size < 0)) { - /* Better to fail explicitly than to allow fishy data to pass through. */ - PyErr_Format (PyExc_ValueError, + PyErr_Format (LZ4BlockError, "Decompressor wrote %u bytes, but %zu bytes expected from header", output_size, dest_size); PyMem_Free (dest); @@ -461,14 +464,24 @@ PyDoc_STRVAR(decompress__doc, "Keyword Args:\n" \ " uncompressed_size (int): If not specified or negative, the uncompressed\n" \ " data size is read from the start of the source block. If specified,\n" \ - " it is assumed that the full source data is compressed data.\n" \ + " it is assumed that the full source data is compressed data. If this\n" \ + " argument is specified, it is considered to be a maximum possible size\n" \ + " for the buffer used to hold the uncompressed data, and so less data\n" \ + " may be returned. If `uncompressed_size` is too small, `LZ4BlockError`\n" \ + " will be raised. By catching `LZ4BlockError` it is possible to increase\n" \ + " `uncompressed_size` and try again.\n" \ " return_bytearray (bool): If ``False`` (the default) then the function\n" \ " will return a bytes object. If ``True``, then the function will\n" \ " return a bytearray object.\n\n" \ " dict (str, bytes or buffer-compatible object): If specified, perform\n" \ - " decompression using this initial dictionary.\n" \ + " decompression using this initial dictionary.\n" \ + "\n" \ "Returns:\n" \ - " bytes or bytearray: Decompressed data.\n"); + " bytes or bytearray: Decompressed data.\n" \ + "\n" \ + "Raises:\n" \ + " LZ4BlockError: raised if the call to the LZ4 library fails. This can be\n" \ + " caused by `uncompressed_size` being too small, or invalid data.\n"); PyDoc_STRVAR(lz4block__doc, "A Python wrapper for the LZ4 block protocol" @@ -517,5 +530,13 @@ MODULE_INIT_FUNC (_block) PyModule_AddIntConstant (module, "HC_LEVEL_OPT_MIN", LZ4HC_CLEVEL_OPT_MIN); PyModule_AddIntConstant (module, "HC_LEVEL_MAX", LZ4HC_CLEVEL_MAX); + LZ4BlockError = PyErr_NewExceptionWithDoc("_block.LZ4BlockError", "Call to LZ4 library failed.", NULL, NULL); + if (LZ4BlockError == NULL) + { + return NULL; + } + Py_INCREF(LZ4BlockError); + PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError); + return module; } diff --git a/tests/block/test_block_1.py b/tests/block/test_block_1.py index 0f5de25e..4ba105ae 100644 --- a/tests/block/test_block_1.py +++ b/tests/block/test_block_1.py @@ -14,10 +14,17 @@ def test_decompress_without_leak(): # Verify that hand-crafted packet does not leak uninitialized(?) memory. data = lz4.block.compress(b'A' * 64) message = r'^Decompressor wrote 64 bytes, but 79 bytes expected from header$' - with pytest.raises(ValueError, match=message): + with pytest.raises(lz4.block.LZ4BlockError, match=message): lz4.block.decompress(b'\x4f' + data[1:]) - with pytest.raises(ValueError, match=message): - lz4.block.decompress(data[4:], uncompressed_size=79) + + +def test_decompress_with_small_buffer(): + data = lz4.block.compress(b'A' * 64, store_size=False) + message = r'^Decompression failed: corrupt input or insufficient space in destination buffer. Error code: \d+$' + with pytest.raises(lz4.block.LZ4BlockError, match=message): + lz4.block.decompress(data[4:], uncompressed_size=64) + with pytest.raises(lz4.block.LZ4BlockError, match=message): + lz4.block.decompress(data, uncompressed_size=60) def test_decompress_truncated(): @@ -34,19 +41,19 @@ def test_decompress_truncated(): with pytest.raises(ValueError, match='Input source data size too small'): lz4.block.decompress(compressed[:n]) for n in [24, 25, -2, 27, 67, 85]: - with pytest.raises(ValueError, match=r'Corrupt input at byte \d+|Decompressor wrote \d+ bytes, but \d+ bytes expected from header'): + with pytest.raises(lz4.block.LZ4BlockError): lz4.block.decompress(compressed[:n]) def test_decompress_with_trailer(): data = b'A' * 64 comp = lz4.block.compress(data) - message = r'^Corrupt input at byte' - with pytest.raises(ValueError, match=message): + message = r'^Decompression failed: corrupt input or insufficient space in destination buffer. Error code: \d+$' + with pytest.raises(lz4.block.LZ4BlockError, match=message): lz4.block.decompress(comp + b'A') - with pytest.raises(ValueError, match=message): + with pytest.raises(lz4.block.LZ4BlockError, match=message): lz4.block.decompress(comp + comp) - with pytest.raises(ValueError, match=message): + with pytest.raises(lz4.block.LZ4BlockError, match=message): lz4.block.decompress(comp + comp[4:]) @@ -105,11 +112,12 @@ def test_with_dict(): input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 dict1 = input_data[10:30] dict2 = input_data[20:40] + message = r'^Decompression failed: corrupt input or insufficient space in destination buffer. Error code: \d+$' for mode in ['default', 'high_compression']: compressed = lz4.block.compress(input_data, mode=mode, dict=dict1) - with pytest.raises(ValueError, match=r'Corrupt input at byte \d+'): + with pytest.raises(lz4.block.LZ4BlockError, match=message): lz4.block.decompress(compressed) - with pytest.raises(ValueError, match=r'Corrupt input at byte \d+'): + with pytest.raises(lz4.block.LZ4BlockError, match=message): lz4.block.decompress(compressed, dict=dict1[:2]) assert lz4.block.decompress(compressed, dict=dict2) != input_data assert lz4.block.decompress(compressed, dict=dict1) == input_data From c3512517119048aba48e1d1e65351a6ff671d22c Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Mon, 30 Jul 2018 20:00:21 +0100 Subject: [PATCH 041/280] Remove Python 3.7 build on travis (#144) --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 508a9b91..cc722345 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: - 3.4 - 3.5 - 3.6 - - 3.7 install: - pip install -U pip - pip install tox From f16962a48a5ca60c1520e73a2e2d381d8bd856a4 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 31 Jul 2018 12:28:19 +0100 Subject: [PATCH 042/280] Enhance documentation for block module --- docs/lz4.block.rst | 91 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/docs/lz4.block.rst b/docs/lz4.block.rst index d50ead59..0b2ef103 100644 --- a/docs/lz4.block.rst +++ b/docs/lz4.block.rst @@ -5,30 +5,95 @@ lz4.block sub-package ===================== This sub-package provides the capability to compress and decompress data using -the `block specification `_. +the `block specification `_. -Because the LZ4 block format doesn't define a container format, the Python -bindings will by default insert the original data size as an integer at the -start of the compressed payload, like most other bindings do (Java...). However, -it is possible to disable this functionality. +Because the LZ4 block format doesn't define a container format, the +Python bindings will by default insert the original data size as an +integer at the start of the compressed payload. However, it is +possible to disable this functionality, and you may wish to do so for +compatibility with other language bindings, such as the `Java bindings +`_. Example usage ------------- -To use the lz4 block format bindings is straightforward:: +To use the lz4 block format bindings is straightforward: .. doctest:: - >>> import lz4.block - >>> import os - >>> input_data = 20 * 128 * os.urandom(1024) # Read 20 * 128kb - >>> compressed_data = lz4.block.compress(input_data) - >>> output_data = lz4.block.decompress(compressed_data) - >>> input_data == output_data - True + >>> import lz4.block + >>> import os + >>> input_data = 20 * 128 * os.urandom(1024) # Read 20 * 128kb + >>> compressed_data = lz4.block.compress(input_data) + >>> output_data = lz4.block.decompress(compressed_data) + >>> input_data == output_data + True +In this simple example, the size of the uncompressed data is stored in +the compressed data, and this size is then utilized when uncompressing +the data in order to correctly size the buffer. Instead, you may want +to not store the size of the uncompressed data to ensure compatibility +with the `Java bindings `_. The +example below demonstrates how to use the block format without storing +the size of the uncompressed data. +.. doctest:: + + >>> import lz4.block + >>> data = b'0' * 255 + >>> compressed = lz4.block.compress(data, store_size=False) + >>> decompressed = lz4.block.decompress(compressed, uncompressed_size=255) + >>> decompressed == data + True + +The `uncompressed_size` argument specifies an upper bound on the size +of the uncompressed data size rather than an absolute value, such that +the following example also works. + +.. doctest:: + + >>> import lz4.block + >>> data = b'0' * 255 + >>> compressed = lz4.block.compress(data, store_size=False) + >>> decompressed = lz4.block.decompress(compressed, uncompressed_size=2048) + >>> decompressed == data + True + +A common situation is not knowing the size of the uncompressed data at +decompression time. The following example illustrates a strategy that +can be used in this case. + +.. doctest:: + + >>> import lz4.block + >>> data = b'0' * 2048 + >>> compressed = lz4.block.compress(data, store_size=False) + >>> usize = 255 + >>> max_size = 4096 + >>> while True: + ... try: + ... decompressed = lz4.block.decompress(compressed, uncompressed_size=usize) + ... break + ... except lz4.block.LZ4BlockError: + ... usize *= 2 + ... if usize > max_size: + ... print('Error: data too large or corrupt') + ... break + >>> decompressed == data + True + +In this example we are catching the `lz4.block.LZ4BlockError` +exception. This exception is raisedd if the LZ4 library call fails, +which can be caused by either the buffer used to store the +uncompressed data (as set by `usize`) being too small, or the input +compressed data being invalid - it is not possible to distinguish the +two cases, and this is why we set an absolute upper bound (`max_size`) +on the memory that can be allocated for the uncompressed data. If we +did not take this precaution, the code, if ppassed invalid compressed +data would continuously try to allocate a larger and larger buffer for +decompression until the system ran out of memory. + Contents ---------------- From 54c17a584b830565f6fa12e292d3110aea142080 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 16 Sep 2018 13:32:28 -0700 Subject: [PATCH 043/280] Correct capitalization of PyPI As spelled on https://pypi.org/. --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 84449c9a..174ce6a5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -60,7 +60,7 @@ install: # We need wheel installed to build wheels, and this isn't pulled in by setup.py - "build.cmd python -m pip install wheel" - # We need twine installed to push wheels to PyPi + # We need twine installed to push wheels to PyPI - "build.cmd python -m pip install twine" # We need tox to run tests From c4e16e27971707d84f5436c19dbe2520cc041d35 Mon Sep 17 00:00:00 2001 From: A Holyoake Date: Fri, 12 Oct 2018 18:12:59 +0100 Subject: [PATCH 044/280] update bundled lz4 to 1.8.3 (with previously problematic numpy array) --- lz4libs/lz4.c | 178 ++++++++++++++++++++----------- lz4libs/lz4.h | 128 +++++++++++++--------- lz4libs/lz4frame.c | 4 +- lz4libs/lz4frame.h | 29 +++-- lz4libs/lz4hc.c | 3 + lz4libs/lz4hc.h | 8 ++ tests/block/conftest.py | 2 + tests/block/numpy_byte_array.bin | Bin 0 -> 8552 bytes 8 files changed, 226 insertions(+), 126 deletions(-) create mode 100644 tests/block/numpy_byte_array.bin diff --git a/lz4libs/lz4.c b/lz4libs/lz4.c index e51a3e0a..4046102e 100644 --- a/lz4libs/lz4.c +++ b/lz4libs/lz4.c @@ -1,6 +1,6 @@ /* LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2017, Yann Collet. + Copyright (C) 2011-present, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -297,8 +297,9 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) #define MINMATCH 4 #define WILDCOPYLENGTH 8 -#define LASTLITERALS 5 -#define MFLIMIT (WILDCOPYLENGTH+MINMATCH) +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ static const int LZ4_minLength = (MFLIMIT+1); #define KB *(1 <<10) @@ -483,9 +484,6 @@ typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - /*-************************************ * Local Utils @@ -496,6 +494,21 @@ int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_sizeofState() { return LZ4_STREAMSIZE; } +/*-************************************ +* Internal Definitions used in Tests +**************************************/ +#if defined (__cplusplus) +extern "C" { +#endif + +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); + +int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize); + +#if defined (__cplusplus) +} +#endif + /*-****************************** * Compression functions ********************************/ @@ -669,9 +682,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* the dictCtx currentOffset is indexed on the start of the dictionary, * while a dictionary in the current context precedes the currentOffset */ - const BYTE* dictBase = dictDirective == usingDictCtx ? - dictionary + dictSize - dictCtx->currentOffset : - dictionary + dictSize - startIndex; + const BYTE* dictBase = (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; @@ -699,7 +712,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( cctx->dictSize += (U32)inputSize; } cctx->currentOffset += (U32)inputSize; - cctx->tableType = tableType; + cctx->tableType = (U16)tableType; if (inputSize oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => just decode everything */ + assert(lowPrefix <= op); + assert(src != NULL); if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); if ((endOnInput) && unlikely(srcSize==0)) return -1; /* Main Loop : decode sequences */ @@ -1428,7 +1447,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( size_t offset; unsigned const token = *ip++; - size_t length = token >> ML_BITS; /* literal length */ + size_t length = token >> ML_BITS; /* literal length */ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ @@ -1453,6 +1472,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( length = token & ML_MASK; /* match length */ offset = LZ4_readLE16(ip); ip += 2; match = op - offset; + assert(match <= op); /* check overflow */ /* Do not deal with overlapping matches. */ if ( (length != ML_MASK) @@ -1486,11 +1506,12 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( /* copy literals */ cpy = op+length; - if ( ((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) { if (partialDecoding) { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if (cpy > oend) { cpy = oend; length = oend-op; } /* Partial decoding : stop in the middle of literal segment */ if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ } else { if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ @@ -1499,10 +1520,15 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( memcpy(op, ip, length); ip += length; op += length; - break; /* Necessarily EOF, due to parsing restrictions */ + if (!partialDecoding || (cpy == oend)) { + /* Necessarily EOF, due to parsing restrictions */ + break; + } + + } else { + LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ + ip += length; op = cpy; } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; /* get offset */ offset = LZ4_readLE16(ip); ip+=2; @@ -1513,7 +1539,11 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( _copy_match: if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ + if (!partialDecoding) { + assert(oend > op); + assert(oend - op >= 4); + LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ + } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ if (length == ML_MASK) { unsigned s; @@ -1526,21 +1556,24 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( } length += MINMATCH; - /* check external dictionary */ + /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } if (length <= (size_t)(lowPrefix-match)) { - /* match can be copied as a single segment from external dictionary */ + /* match fits entirely within external dictionary : just copy */ memmove(op, dictEnd - (lowPrefix-match), length); op += length; } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); size_t const restSize = length - copySize; memcpy(op, dictEnd - copySize, copySize); op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ BYTE* const endOfMatch = op + restSize; const BYTE* copyFrom = lowPrefix; while (op < endOfMatch) *op++ = *copyFrom++; @@ -1553,6 +1586,23 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( /* copy match within block */ cpy = op + length; + + /* partialDecoding : may not respect endBlock parsing restrictions */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) *op++ = *match++; + } else { + memcpy(op, match, mlen); + } + op = copyEnd; + if (op==oend) break; + continue; + } + if (unlikely(offset<8)) { op[0] = match[0]; op[1] = match[1]; @@ -1561,23 +1611,26 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic( match += inc32table[offset]; memcpy(op+4, match, 4); match -= dec64table[offset]; - } else { memcpy(op, match, 8); match+=8; } + } else { + memcpy(op, match, 8); + match += 8; + } op += 8; - if (unlikely(cpy>oend-12)) { - BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ if (op < oCopyLimit) { LZ4_wildCopy(op, match, oCopyLimit); match += oCopyLimit - op; op = oCopyLimit; } - while (op16) LZ4_wildCopy(op+8, match+8, cpy); + if (length > 16) LZ4_wildCopy(op+8, match+8, cpy); } - op = cpy; /* correction */ + op = cpy; /* wildcopy correction */ } /* end of decoding */ @@ -1598,23 +1651,24 @@ LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, - endOnInputSize, full, 0, noDict, + endOnInputSize, decode_full_block, noDict, (BYTE*)dest, NULL, 0); } LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) +int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, - endOnInputSize, partial, targetOutputSize, - noDict, (BYTE*)dest, NULL, 0); + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, + endOnInputSize, partial_decode, + noDict, (BYTE*)dst, NULL, 0); } LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, withPrefix64k, + endOnOutputSize, decode_full_block, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 0); } @@ -1624,7 +1678,7 @@ LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, withPrefix64k, + endOnInputSize, decode_full_block, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 0); } @@ -1641,17 +1695,17 @@ static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, i size_t prefixSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, noDict, + endOnInputSize, decode_full_block, noDict, (BYTE*)dest-prefixSize, NULL, 0); } -LZ4_FORCE_O2_GCC_PPC64LE /* Exported under another name, for tests/fullbench.c */ -#define LZ4_decompress_safe_extDict LZ4_decompress_safe_forceExtDict -int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSize, int maxOutputSize, - const void* dictStart, size_t dictSize) +LZ4_FORCE_O2_GCC_PPC64LE +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, usingExtDict, + endOnInputSize, decode_full_block, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } @@ -1660,7 +1714,7 @@ static int LZ4_decompress_fast_extDict(const char* source, char* dest, int origi const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, usingExtDict, + endOnOutputSize, decode_full_block, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } @@ -1673,7 +1727,7 @@ int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compresse size_t prefixSize, const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, usingExtDict, + endOnInputSize, decode_full_block, usingExtDict, (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); } @@ -1682,7 +1736,7 @@ int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalS size_t prefixSize, const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, usingExtDict, + endOnOutputSize, decode_full_block, usingExtDict, (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); } @@ -1773,8 +1827,8 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch /* The buffer wraps around, or they're switching to another buffer. */ lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, - lz4sd->externalDict, lz4sd->extDictSize); + result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize = result; lz4sd->prefixEnd = (BYTE*)dest + result; @@ -1834,7 +1888,7 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize); } - return LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) diff --git a/lz4libs/lz4.h b/lz4libs/lz4.h index 7d131221..059ef7c1 100644 --- a/lz4libs/lz4.h +++ b/lz4libs/lz4.h @@ -1,7 +1,7 @@ /* * LZ4 - Fast LZ compression algorithm * Header File - * Copyright (C) 2011-2017, Yann Collet. + * Copyright (C) 2011-present, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -46,7 +46,7 @@ extern "C" { /** Introduction - LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, + LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core, scalable with multi-cores CPU. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. @@ -62,8 +62,8 @@ extern "C" { An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), take care of encoding standard metadata alongside LZ4-compressed blocks. - If your application requires interoperability, it's recommended to use it. - A library is provided to take care of it, see lz4frame.h. + Frame format is required for interoperability. + It is delivered through a companion API, declared in lz4frame.h. */ /*^*************************************************************** @@ -93,7 +93,7 @@ extern "C" { /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -183,55 +183,72 @@ LZ4_compress_fast_extState() : Same compression function, just using an externally allocated memory space to store compression state. Use LZ4_sizeofState() to know how much memory must be allocated, and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. + Then, provide this buffer as 'void* state' to compression function. */ LZ4LIB_API int LZ4_sizeofState(void); LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -/*! -LZ4_compress_destSize() : - Reverse the logic : compresses as much data as possible from 'src' buffer - into already allocated buffer 'dst' of size 'targetDestSize'. - This function either compresses the entire 'src' content into 'dst' if it's large enough, - or fill 'dst' buffer completely with as much data as possible from 'src'. - *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. - New value is necessarily <= old value. - return : Nb bytes written into 'dst' (necessarily <= targetDestSize) - or 0 if compression fails +/*! LZ4_compress_destSize() : + * Reverse the logic : compresses as much data as possible from 'src' buffer + * into already allocated buffer 'dst', of size >= 'targetDestSize'. + * This function either compresses the entire 'src' content into 'dst' if it's large enough, + * or fill 'dst' buffer completely with as much data as possible from 'src'. + * note: acceleration parameter is fixed to "default". + * + * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + * New value is necessarily <= input value. + * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) + * or 0 if compression fails. */ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); -/*! -LZ4_decompress_fast() : **unsafe!** -This function is a bit faster than LZ4_decompress_safe(), -but it may misbehave on malformed input because it doesn't perform full validation of compressed data. - originalSize : is the uncompressed size to regenerate - Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes. - return : number of bytes read from source buffer (== compressed size). - If the source stream is detected malformed, the function stops decoding and return a negative result. - note : This function is only usable if the originalSize of uncompressed data is known in advance. - The caller should also check that all the compressed input has been consumed properly, - i.e. that the return value matches the size of the buffer with compressed input. - The function never writes past the output buffer. However, since it doesn't know its 'src' size, - it may read past the intended input. Also, because match offsets are not validated during decoding, - reads from 'src' may underflow. Use this function in trusted environment **only**. -*/ +/*! LZ4_decompress_fast() : **unsafe!** + * This function used to be a bit faster than LZ4_decompress_safe(), + * though situation has changed in recent versions, + * and now `LZ4_decompress_safe()` can be as fast and sometimes faster than `LZ4_decompress_fast()`. + * Moreover, LZ4_decompress_fast() is not protected vs malformed input, as it doesn't perform full validation of compressed data. + * As a consequence, this function is no longer recommended, and may be deprecated in future versions. + * It's only remaining specificity is that it can decompress data without knowing its compressed size. + * + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : This function requires uncompressed originalSize to be known in advance. + * The function never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read past the intended input. + * Also, because match offsets are not validated during decoding, + * reads from 'src' may underflow. + * Use this function in trusted environment **only**. + */ LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); -/*! -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'srcSize' at position 'src' - into destination buffer 'dst' of size 'dstCapacity'. - The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that. - However, it's not accurate, and may write more than 'targetOutputSize' (but always <= dstCapacity). - @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity) - Note : this number can also be < targetOutputSize, if compressed block contains less data. - Therefore, always control how many bytes were decoded. - If source stream is detected malformed, function returns a negative result. - This function is protected against malicious data packets. -*/ +/*! LZ4_decompress_safe_partial() : + * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', + * into destination buffer 'dst' of size 'dstCapacity'. + * Up to 'targetOutputSize' bytes will be decoded. + * The function stops decoding on reaching this objective, + * which can boost performance when only the beginning of a block is required. + * + * @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity) + * If source stream is detected malformed, function returns a negative result. + * + * Note : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity, + * and expects targetOutputSize <= dstCapacity. + * It effectively stops decoding on reaching targetOutputSize, + * so dstCapacity is kind of redundant. + * This is because in a previous version of this function, + * decoding operation would not "break" a sequence in the middle. + * As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize, + * it could write more bytes, though only up to dstCapacity. + * Some "margin" used to be required for this operation to work properly. + * This is no longer necessary. + * The function nonetheless keeps its signature, in an effort to not break API. + */ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); @@ -266,16 +283,23 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. * - * Important : The previous 64KB of compressed data is assumed to remain present and unmodified in memory! - * - * Special 1 : When input is a double-buffer, they can have any size, including < 64 KB. - * Make sure that buffers are separated by at least one byte. - * This way, each block only depends on previous block. - * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. - * * @return : size of compressed block * or 0 if there is an error (typically, cannot fit into 'dst'). - * After an error, the stream status is invalid, it can only be reset or freed. + * + * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. + * Each block has precise boundaries. + * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. + * Each block must be decompressed separately, calling LZ4_decompress_*() with associated metadata. + * + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory! + * + * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. + * Make sure that buffers are separated, by at least one byte. + * This construction ensures that each block only depends on previous block. + * + * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * Note 5 : After an error, the stream status is invalid, it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -305,7 +329,7 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str /*! LZ4_setStreamDecode() : * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. * Use this function to start decompression of a new stream of blocks. - * A dictionary can optionnally be set. Use NULL or size 0 for a reset order. + * A dictionary can optionally be set. Use NULL or size 0 for a reset order. * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. * @return : 1 if OK, 0 if error */ diff --git a/lz4libs/lz4frame.c b/lz4libs/lz4frame.c index e1d0b1d0..08bf0fae 100644 --- a/lz4libs/lz4frame.c +++ b/lz4libs/lz4frame.c @@ -738,7 +738,7 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { - int const acceleration = (level < -1) ? -level : 1; + int const acceleration = (level < 0) ? -level + 1 : 1; LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent); if (cdict) { return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); @@ -749,7 +749,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { - int const acceleration = (level < -1) ? -level : 1; + int const acceleration = (level < 0) ? -level + 1 : 1; (void)cdict; /* init once at beginning of frame */ return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); } diff --git a/lz4libs/lz4frame.h b/lz4libs/lz4frame.h index fb434ff7..75f1fd91 100644 --- a/lz4libs/lz4frame.h +++ b/lz4libs/lz4frame.h @@ -33,9 +33,10 @@ */ /* LZ4F is a stand-alone API to create LZ4-compressed frames - * conformant with specification v1.5.1. + * conformant with specification v1.6.1. * It also offers streaming capabilities. - * lz4.h is not required when using lz4frame.h. + * lz4.h is not required when using lz4frame.h, + * except to get constant such as LZ4_VERSION_NUMBER. * */ #ifndef LZ4F_H_09782039843 @@ -159,8 +160,9 @@ typedef LZ4F_contentChecksum_t contentChecksum_t; /*! LZ4F_frameInfo_t : * makes it possible to set or read frame parameters. - * It's not required to set all fields, as long as the structure was initially memset() to zero. - * For all fields, 0 sets it to default value */ + * Structure must be first init to 0, using memset() or LZ4F_INIT_FRAMEINFO, + * setting all parameters to default. + * It's then possible to update selectively some parameters */ typedef struct { LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default */ LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */ @@ -171,24 +173,30 @@ typedef struct { LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */ } LZ4F_frameInfo_t; +#define LZ4F_INIT_FRAMEINFO { 0, 0, 0, 0, 0, 0, 0 } /* v1.8.3+ */ + /*! LZ4F_preferences_t : - * makes it possible to supply detailed compression parameters to the stream interface. - * Structure is presumed initially memset() to zero, representing default settings. + * makes it possible to supply advanced compression instructions to streaming interface. + * Structure must be first init to 0, using memset() or LZ4F_INIT_PREFERENCES, + * setting all parameters to default. * All reserved fields must be set to zero. */ typedef struct { LZ4F_frameInfo_t frameInfo; int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */ - unsigned autoFlush; /* 1: always flush, to reduce usage of internal buffers */ - unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */ /* >= v1.8.2 */ + unsigned autoFlush; /* 1: always flush; reduces usage of internal buffers */ + unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4HC_CLEVEL_OPT_MIN) */ /* v1.8.2+ */ unsigned reserved[3]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; -LZ4FLIB_API int LZ4F_compressionLevel_max(void); +#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0, 0, { 0, 0, 0 } } /* v1.8.3+ */ /*-********************************* * Simple compression function ***********************************/ + +LZ4FLIB_API int LZ4F_compressionLevel_max(void); + /*! LZ4F_compressFrameBound() : * Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences. * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences. @@ -222,8 +230,9 @@ typedef struct { /*--- Resource Management ---*/ -#define LZ4F_VERSION 100 +#define LZ4F_VERSION 100 /* This number can be used to check for an incompatible API breaking change */ LZ4FLIB_API unsigned LZ4F_getVersion(void); + /*! LZ4F_createCompressionContext() : * The first thing to do is to create a compressionContext object, which will be used in all compression operations. * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. diff --git a/lz4libs/lz4hc.c b/lz4libs/lz4hc.c index 8108ea01..e913ee7b 100644 --- a/lz4libs/lz4hc.c +++ b/lz4libs/lz4hc.c @@ -327,6 +327,8 @@ LZ4HC_InsertAndGetWiderMatch ( if (lookBackLength==0) { /* no back possible */ size_t const maxML = MIN(currentSegmentLength, srcPatternLength); if ((size_t)longest < maxML) { + assert(base + matchIndex < ip); + if (ip - (base+matchIndex) > MAX_DISTANCE) break; assert(maxML < 2 GB); longest = (int)maxML; *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ @@ -450,6 +452,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( *op += length; /* Encode Offset */ + assert( (*ip - match) <= MAX_DISTANCE ); /* note : consider providing offset as a value, rather than as a pointer difference */ LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; /* Encode MatchLength */ diff --git a/lz4libs/lz4hc.h b/lz4libs/lz4hc.h index bb5e0737..970fa396 100644 --- a/lz4libs/lz4hc.h +++ b/lz4libs/lz4hc.h @@ -246,6 +246,10 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API int LZ4_resetStr #ifndef LZ4_HC_SLO_098092834 #define LZ4_HC_SLO_098092834 +#if defined (__cplusplus) +extern "C" { +#endif + /*! LZ4_compress_HC_destSize() : v1.8.0 (experimental) * Will try to compress as much data from `src` as possible * that can fit into `targetDstSize` budget. @@ -343,5 +347,9 @@ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* ds */ LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream); +#if defined (__cplusplus) +} +#endif + #endif /* LZ4_HC_SLO_098092834 */ #endif /* LZ4_HC_STATIC_LINKING_ONLY */ diff --git a/tests/block/conftest.py b/tests/block/conftest.py index 8eab729d..845eac03 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -9,7 +9,9 @@ (b'0' * 8 * 1024), (bytearray(b'')), (bytearray(os.urandom(8 * 1024))), + (bytearray(open(os.path.join(os.path.dirname(__file__), 'numpy_byte_array.bin'), 'rb').read())) ] + if sys.version_info > (2, 7): test_data += [ (memoryview(b'')), diff --git a/tests/block/numpy_byte_array.bin b/tests/block/numpy_byte_array.bin new file mode 100644 index 0000000000000000000000000000000000000000..49537e2d90a5161ad8fcc847c696b7101b1cdd4c GIT binary patch literal 8552 zcmeI2T}V@57{_f9VFX2V6}0rCQo__^L>F})V;>MjC`<>5w!#Vw6#If!1W`n3IlC*0 z0)?ODQxY0lw)^`{VH^Bu91 zR>s!Fz<#vWnT?b8k-W}Et(V^OMjx@~#vSY5=SP+qKL@NkC|ZAR=Q5Aub}n;&kvv*= ze7AXE2dO2lqm`bO_Z|L|5h>ZsiM=9Hs&q*R88O>Cm%m1!Vul0zxFP}8FFVOEI z5#6T27Kd!to1*I*t=Fqht$k>%2iFtD!wzyk=+l+FA1;gf;(lB{)zr1TWXejZN@wG` z+!?QvDz=XnhC*)L+46S0()8LzV5-g~TbH)`R_pg0nLh8@z_aHkf>uhDd`_Z#dH#u2 zi$hl8ln(d~zxZk;&M);vhpx0cNiBJNqB5_u`P!n}-xkQUK36N(uP~eZ z({5T_+7a9&zn`ZHYcIVzxJEu_1b061dfJ?EpNE!~-P+O`u#*yl-48#%o3axK|NM;& z1y#ik0vC3j8M;~GAWqmp*pEzH>@F|0ky>7DI5rn8wh<@dxgIj?pw$2I(Yg0_Qp>#Z z@cuU+#Qu8QQ75hg^3Sh)U%)fy}97=l)z4#X}GB zbC&3N%>9&PQJzHgqWdOO^tk^sr5~gF!2CoM)dPQ#Jze#DpQ`OcDK1ev>QKr9KPB=! cT;}n~{v8DQ Date: Sun, 4 Nov 2018 09:53:54 +0000 Subject: [PATCH 045/280] Stop using pkg_resources to get version (issue #154) --- lz4/__init__.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lz4/__init__.py b/lz4/__init__.py index 15701bb8..af0fa05b 100644 --- a/lz4/__init__.py +++ b/lz4/__init__.py @@ -1,10 +1,16 @@ -from pkg_resources import get_distribution, DistributionNotFound -try: - __version__ = get_distribution(__name__).version -except DistributionNotFound: - # package is not installed - pass +# Although the canonical way to get the package version is using pkg_resources +# as below, this turns out to be very slow on systems with lots of packages. +# So, until that is remedied, we'll import the version from a local file +# created by setuptools_scm. +# from pkg_resources import get_distribution, DistributionNotFound +# try: +# __version__ = get_distribution(__name__).version +# except DistributionNotFound: +# # package is not installed +# pass + +from .version import version as __version__ VERSION = __version__ From 56201f2649a85555a4794f3ba25eee16390511f0 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 4 Nov 2018 10:34:50 +0000 Subject: [PATCH 046/280] Refactor and cleanup flake8 problems with test_block_1.py --- tests/block/test_block_1.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/tests/block/test_block_1.py b/tests/block/test_block_1.py index 4ba105ae..4392bb33 100644 --- a/tests/block/test_block_1.py +++ b/tests/block/test_block_1.py @@ -125,16 +125,25 @@ def test_with_dict(): input_data), dict=dict1) == input_data -def test_known_decompress(): - assert(lz4.block.decompress( - b'\x00\x00\x00\x00\x00') == - b'') - assert(lz4.block.decompress( - b'\x01\x00\x00\x00\x10 ') == - b' ') - assert(lz4.block.decompress( - b'h\x00\x00\x00\xff\x0bLorem ipsum dolor sit amet\x1a\x006P amet') == - b'Lorem ipsum dolor sit amet' * 4) - assert(lz4.block.decompress( - b'\xb0\xb3\x00\x00\xff\x1fExcepteur sint occaecat cupidatat non proident.\x00' + (b'\xff' * 180) + b'\x1ePident') == - b'Excepteur sint occaecat cupidatat non proident' * 1000) +def test_known_decompress_1(): + input = b'\x00\x00\x00\x00\x00' + output = b'' + assert lz4.block.decompress(input) == output + + +def test_known_decompress_2(): + input = b'\x01\x00\x00\x00\x10 ' + output = b' ' + assert lz4.block.decompress(input) == output + + +def test_known_decompress_3(): + input = b'h\x00\x00\x00\xff\x0bLorem ipsum dolor sit amet\x1a\x006P amet' + output = b'Lorem ipsum dolor sit amet' * 4 + assert lz4.block.decompress(input) == output + + +def test_known_decompress_4(): + input = b'\xb0\xb3\x00\x00\xff\x1fExcepteur sint occaecat cupidatat non proident.\x00' + (b'\xff' * 180) + b'\x1ePident' + output = b'Excepteur sint occaecat cupidatat non proident' * 1000 + assert lz4.block.decompress(input) == output From bc8e66e24b1422d06afdb015a0ff25fcaf167b04 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 4 Nov 2018 10:42:06 +0000 Subject: [PATCH 047/280] Increase memory margins in test_frame_5.py Intermittently these tests fail as memory usage might tip over the 15 kB increase due to pytest objects etc. --- tests/frame/test_frame_5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 274a4181..41e31855 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -32,7 +32,7 @@ def test_frame_decompress_mem_usage(data): if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 15) + assert stats[0].size_diff < (1024 * 20) prev_snapshot = snapshot @@ -56,7 +56,7 @@ def test_frame_decompress_chunk_mem_usage(data): if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 15) + assert stats[0].size_diff < (1024 * 20) prev_snapshot = snapshot @@ -79,7 +79,7 @@ def test_frame_open_decompress_mem_usage(data): if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 15) + assert stats[0].size_diff < (1024 * 20) prev_snapshot = snapshot From a2f5bd79e6bc1c7e3beea9e56b675c4bb93c9430 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 4 Nov 2018 10:47:08 +0000 Subject: [PATCH 048/280] Fix flake8 W504 in frame/_compression.py --- lz4/frame/_compression.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lz4/frame/_compression.py b/lz4/frame/_compression.py index 7773ee44..3c68904b 100644 --- a/lz4/frame/_compression.py +++ b/lz4/frame/_compression.py @@ -87,8 +87,9 @@ def read(self, size=-1): # return any data. In this case, try again after reading another block. while True: if self._decompressor.eof: - rawblock = (self._decompressor.unused_data or - self._fp.read(BUFFER_SIZE)) + rawblock = ( + self._decompressor.unused_data or self._fp.read(BUFFER_SIZE) + ) if not rawblock: break # Continue to next stream. From 7240a252585d324495073d4fb4fd3b6d0aa2e363 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 11 Nov 2018 18:45:16 -0800 Subject: [PATCH 049/280] Simplify tox.ini configuration with the extras option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://tox.readthedocs.io/en/latest/config.html#conf-extras > A list of “extras” to be installed with the sdist or develop install. > For example, extras = testing is equivalent to [testing] in a pip > install command. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 43c08a91..6b865a75 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,9 @@ [tox] envlist = py +minversion = 2.4.0 [testenv] -deps=.[tests] +extras = tests passenv = * # setenv = PYTHONMALLOC = pymalloc # PYTHONMALLOCSTATS = 'yes' From 0b02380a6fccaa53995fce0deed452f7e0fcecc9 Mon Sep 17 00:00:00 2001 From: Raymond Ehlers Date: Tue, 11 Dec 2018 19:27:48 -0500 Subject: [PATCH 050/280] Explicitly import needed pkgconfig function Addresses #158 --- setup.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 80665fdf..fed831b2 100644 --- a/setup.py +++ b/setup.py @@ -13,15 +13,17 @@ liblz4_found = False py3c_found = False try: - import pkgconfig + from pkgconfig import installed as pkgconfig_installed + from pkgconfig import cflags as pkgconfig_cflags + from pkgconfig import libs as pkgconfig_libs except ImportError: # pkgconfig is not installed. It will be installed by setup_requires. pass else: - def pkgconfig_installed(lib, required_version, default): + def pkgconfig_installed_check(lib, required_version, default): installed = default try: - installed = pkgconfig.installed(lib, required_version) + installed = pkgconfig_installed(lib, required_version) except EnvironmentError: # Windows, no pkg-config present pass @@ -31,8 +33,8 @@ def pkgconfig_installed(lib, required_version, default): # Bundled version of liblz4 will be used pass return installed - liblz4_found = pkgconfig_installed('liblz4', LZ4_REQUIRED_VERSION, default=False) - py3c_found = pkgconfig_installed('py3c', PY3C_REQUIRED_VERSION, default=False) + liblz4_found = pkgconfig_installed_check('liblz4', LZ4_REQUIRED_VERSION, default=False) + py3c_found = pkgconfig_installed_check('py3c', PY3C_REQUIRED_VERSION, default=False) # Set up the extension modules. If a system wide lz4 library is found, and is @@ -93,9 +95,9 @@ def pkgconfig_installed(lib, required_version, default): extra_compile_args = ['/Ot', '/Wall'] elif compiler in ('unix', 'mingw32'): if liblz4_found: - extra_link_args.append(pkgconfig.libs('liblz4')) - if pkgconfig.cflags('liblz4'): - extra_compile_args.append(pkgconfig.cflags('liblz4')) + extra_link_args.append(pkgconfig_libs('liblz4')) + if pkgconfig_cflags('liblz4'): + extra_compile_args.append(pkgconfig_cflags('liblz4')) else: extra_compile_args = [ '-O3', From af58fcda96644fbab00145d32ba60669b300515f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 16 Dec 2018 20:24:14 +0000 Subject: [PATCH 051/280] Only require pytest-runner if tests are being run (#161) --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index fed831b2..738a4cde 100644 --- a/setup.py +++ b/setup.py @@ -146,6 +146,10 @@ def pkgconfig_installed_check(lib, required_version, default): 'psutil', ], +# Only require pytest-runner if actually running the tests +needs_pytest = {'pytest', 'test', 'ptr'}.intersection(sys.argv) +pytest_runner = ['pytest-runner'] if needs_pytest else [] + # Finally call setup with the extension modules as defined above. setup( name='lz4', @@ -156,8 +160,7 @@ def pkgconfig_installed_check(lib, required_version, default): setup_requires=[ 'setuptools_scm', 'pkgconfig', - 'pytest-runner', - ], + ] + pytest_runner, install_requires=install_requires, description="LZ4 Bindings for Python", long_description=open('README.rst', 'r').read(), From b05072fc7ed2d3c02b3f75f3f79c16c7c36ec731 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 13 Dec 2018 23:49:40 +0000 Subject: [PATCH 052/280] Integrate cibuildwheel into .travis.yml --- .travis.yml | 125 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 113 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc722345..adb227bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,123 @@ language: python -python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 +cache: pip +matrix: + include: + - name: linux-py27 + python: 2.7 + sudo: required + services: + - docker + env: + - PIP=pip + - CIBW_BUILD=cp27-* + - name: linux-py34 + python: 3.4 + sudo: required + services: + - docker + env: + - PIP=pip + - CIBW_BUILD=cp34-* + - name: linux-py35 + python: 3.5 + sudo: required + services: + - docker + env: + - PIP=pip3 + - CIBW_BUILD=cp35-* + - name: linux-py36 + python: 3.6 + sudo: required + services: + - docker + env: + - PIP=pip3 + - CIBW_BUILD=cp36-* + - name: linux-py37 + python: 3.7 + dist: xenial + sudo: required + services: + - docker + env: + - PIP=pip3 + - CIBW_BUILD=cp37-* + - name: osx-py27 + os: osx + language: generic + env: + - PIP=pip2 + - CIBW_BUILD=cp27-* + - CIBW_TEST_COMMAND="pytest {project}/tests" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + install: + - ${PIP} install cibuildwheel==0.10.0 + script: + - cibuildwheel --output-dir dist + - name: osx-py34 + os: osx + language: generic + env: + - PIP=pip2 + - CIBW_BUILD=cp34-* + - CIBW_TEST_COMMAND="pytest {project}/tests" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + install: + - ${PIP} install cibuildwheel==0.10.0 + script: + - cibuildwheel --output-dir dist + - name: osx-py35 + os: osx + language: generic + env: + - PIP=pip2 + - CIBW_BUILD=cp35-* + - CIBW_TEST_COMMAND="pytest {project}/tests" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + install: + - ${PIP} install cibuildwheel==0.10.0 + script: + - cibuildwheel --output-dir dist + - name: osx-py36 + os: osx + language: generic + env: + - PIP=pip2 + - CIBW_BUILD=cp36-* + - CIBW_TEST_COMMAND="pytest {project}/tests" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + install: + - ${PIP} install cibuildwheel==0.10.0 + script: + - cibuildwheel --output-dir dist + - name: osx-py37 + os: osx + language: generic + env: + - PIP=pip2 + - CIBW_BUILD=cp37-* + - CIBW_TEST_COMMAND="pytest {project}/tests" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + install: + - ${PIP} install cibuildwheel==0.10.0 + script: + - cibuildwheel --output-dir dist install: - - pip install -U pip - - pip install tox - - pip install -r docs/requirements.txt + - ${PIP} install -U pip + - ${PIP} install tox + - ${PIP} install -r docs/requirements.txt + - ${PIP} install cibuildwheel==0.10.0 script: - - pip install .[flake8] + - ${PIP} install .[flake8] - flake8 --ignore=E501 lz4 - flake8 --ignore=E501 tests - flake8 --ignore=E501 setup.py + - ${PIP} install . + - ${PIP} install .[tests] - tox - - pip install . - - pip install .[tests] - make -C docs doctest html + - cibuildwheel --output-dir dist deploy: - provider: pypi # server: https://test.pypi.org/legacy/ @@ -27,4 +128,4 @@ deploy: on: branch: master tags: true - python: '3.6' + From 81fdaa43661e8ec9981817b969828372e719d785 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Mon, 17 Dec 2018 18:37:06 +0000 Subject: [PATCH 053/280] Prevent the use of pytest 3.3.0 for testing as it contains a bug https://github.com/pytest-dev/pytest/issues/2957 --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fed831b2..47338822 100644 --- a/setup.py +++ b/setup.py @@ -141,8 +141,10 @@ def pkgconfig_installed_check(lib, required_version, default): # refer to it for the tests_require and the extras_require arguments # to setup below. The latter enables us to use pip install .[tests] to # install testing dependencies. +# Note: pytest 3.3.0 contains a bug with null bytes in parameter IDs: +# https://github.com/pytest-dev/pytest/issues/2957 tests_require = [ - 'pytest', + 'pytest!=3.3.0', 'psutil', ], From 27becc3581f93c20ebb084c89f51c73081bb2be2 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Mon, 17 Dec 2018 18:38:01 +0000 Subject: [PATCH 054/280] Use pytest instead of tox for tests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index adb227bf..1be74a62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -cache: pip + matrix: include: - name: linux-py27 @@ -115,7 +115,7 @@ script: - flake8 --ignore=E501 setup.py - ${PIP} install . - ${PIP} install .[tests] - - tox + - pytest - make -C docs doctest html - cibuildwheel --output-dir dist deploy: From 283feb277fd5d8783cb8a0b17990e0526869d81e Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Mon, 17 Dec 2018 19:07:33 +0000 Subject: [PATCH 055/280] Add ids to dictionary fixture for block tests --- tests/block/conftest.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/block/conftest.py b/tests/block/conftest.py index 845eac03..0b3578ea 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -92,13 +92,19 @@ def mode(request): return request.param +dictionary = [ + None, + (0, 0), + (100, 200), + (0, 8 * 1024), + os.urandom(8 * 1024) +] + + @pytest.fixture( - params=[ - None, - (0, 0), - (100, 200), - (0, 8 * 1024), - os.urandom(8 * 1024) + params=dictionary, + ids=[ + 'dictionary' + str(i) for i in range(len(dictionary)) ] ) def dictionary(request): From 6193753b698e67934325a98766fcfd3be3d35f25 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Mon, 17 Dec 2018 19:45:16 +0000 Subject: [PATCH 056/280] Specify laguage in each travis matrix entry for linux --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1be74a62..e948e2e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,7 @@ -language: python - matrix: include: - name: linux-py27 + language: python python: 2.7 sudo: required services: @@ -11,6 +10,7 @@ matrix: - PIP=pip - CIBW_BUILD=cp27-* - name: linux-py34 + language: python python: 3.4 sudo: required services: @@ -19,6 +19,7 @@ matrix: - PIP=pip - CIBW_BUILD=cp34-* - name: linux-py35 + language: python python: 3.5 sudo: required services: @@ -27,6 +28,7 @@ matrix: - PIP=pip3 - CIBW_BUILD=cp35-* - name: linux-py36 + language: python python: 3.6 sudo: required services: @@ -35,6 +37,7 @@ matrix: - PIP=pip3 - CIBW_BUILD=cp36-* - name: linux-py37 + language: python python: 3.7 dist: xenial sudo: required @@ -105,11 +108,10 @@ matrix: - cibuildwheel --output-dir dist install: - ${PIP} install -U pip - - ${PIP} install tox - ${PIP} install -r docs/requirements.txt - ${PIP} install cibuildwheel==0.10.0 -script: - ${PIP} install .[flake8] +script: - flake8 --ignore=E501 lz4 - flake8 --ignore=E501 tests - flake8 --ignore=E501 setup.py From 0764604ddb41e9c9705689b19f8bd02d46b6643f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 02:08:22 +0000 Subject: [PATCH 057/280] Update PyPi password and add skip_existing and skip_cleanup to deploy --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e948e2e1..09aabcea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,8 +125,10 @@ deploy: # server: https://test.pypi.org/legacy/ user: jgu password: - secure: SS5kSQtsfHIYG5gLKC9jQ23bnNBgoejVgdrDxfaccNqojrIy5CtuUo+a7/P16bObWp1jCnlwUyX6mQj1S+abDsUHr+2+pNjl5MP8pWdREu19fBTcwMkZLXlUdSuWwcBjmZjaVpuKQRCKwajHrGjOE4s3vO1h/IOLETEX0d1rmuavSUftqYcea/TO/MNzDuxnUe7pkGsPkfbMxHY5oXNmx5zOvtG2h0eZowjwgE5YVZfTZ82fqfCfbqmoytwmQKd3rTmo/hpYvNcGweHizKouzLArw0+DDtkJt/jT459zM3lnUEh+TfaCSeqd+cTWmEZDiMR8MxnSJkpN3RGHcvxreF+HZjCAKhvAbZcWIpRC0cNp4VxWyVGZN/dLZA0WM3gJJKCjgCA70eslC+ccZh6WyG93effiVLhHYB9vyt7SfwsUzDrZo3++Czi6drCVNyI6pBdb587yfkuJYdomNIQ2DCesUKjuoFFt34VZC5bJOK82F81jMXYGaQn3BCWPdfMTmqJ3maimfji7KXts6bbyQUzgc+610Yd8eIRGvDQwmeBjovRT5uyYDJtXfDOIYXiv3n79wtDt+ygZzLOXNMvEt6/Wsy/xrWEsHFfA5bknSenurfgQoYCkLOjgKJEqxDV4nXdwhJxwbC740Fea/cGwem5L0mfz8SGtd7giFHDpgkg= + secure: "K1HppGu970i6WBKfnjQbCo0A0cYPih2rfM/rIDRzQXS8t+ZckyBywiKq/+4pkNJAfaHhQ8bHSz2J1GCq3Wz6DloGFHqEKAoHy5OgJxaK5nEp6avYvyHBSxrcZ6UfwFGMMMTAy4vfhnoJ3ARHaUFE4EIWYCVvP8/yCsE/6ir8AfozTssdYTUioJaYjSnowJ38nqa8H3hK3K5gke8aeclB+lnnWcyakgot65MfL2ov2mkkpWeiEn44ldV2iRKcl9a7Utgq3tJ1A634eUiNvIPpBYxnxPe++VRHA/mov9BWX0XfTmDuiVr+zDgMWdt3BK7f21r6P5DndpCZb8C2WzBFMKVz+rbuspbVJTol9Nt/9sVvSS6/fEBpXooRHRr9nP9EIt8Jlyv4CZ3iX/+T/912s84ZQCgqrhFNnRvECpmiVtHg49GecnMtJqC4jW57527c//oQPFyv/84Fd7fZ5jZg4LwhCl1C04Sqjmx8MFuMCR829CcZXJ2dAa5hQ/1v44SJAOKCQ0zDSdTqAy6x9RK/KNxnoQ9Nbn6ZFBUcjKx6rofyvXlwIcsQsjuzLw2mqmavZPduavVRi6cxK1WCEnaBVOXODWUSDatWchch1OcBm46X9UwwnHTsDHCVh6Bw+BDXhZuNw/72UA8z0sU/Val8iXBeThu33PMh3fo+uoqDf1g=" distributions: "sdist" + skip_existing: true + skip_cleanup: true on: branch: master tags: true From 26775cba2c18905cf589364b4e3782cdb43d8547 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 02:35:36 +0000 Subject: [PATCH 058/280] Update encrypted Appveyor credentials --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 174ce6a5..dd89f49a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,7 +6,7 @@ environment: PYPI_PASSWORD: - secure: LGlG50Ta854XC2pxoPRxxQ== + secure: Sopzu1BZfXmABwhzM7mYSQ== matrix: From 92b70934f18d15c6d3ebc03f4ce8c7c0a6744e22 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 02:44:51 +0000 Subject: [PATCH 059/280] Run tests with pytest rather than via tox in Appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index dd89f49a..fb9cc7ce 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -75,7 +75,7 @@ test_script: # due to memory fragmentation resulting from many alloc/free cycles # ps: $env:is64bit = $env:PYTHON.Contains("-x64") # IF %is64bit% == True build.cmd tox - - "build.cmd tox" + - "build.cmd pytest tests" after_test: # If tests are successful, create binary packages for the project. From 26682e9ed1df90763a90a1b38fa5ee07a78a1adc Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 13:15:01 +0000 Subject: [PATCH 060/280] Install pytest in Appveyor environment --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index fb9cc7ce..7b6e9b27 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -64,7 +64,7 @@ install: - "build.cmd python -m pip install twine" # We need tox to run tests - - "build.cmd python -m pip install tox" + - "build.cmd python -m pip install pytest" build_script: # Build the compiled extension From cd5abf65b1fbec82a15c031e8d1723d920d943a8 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 13:23:40 +0000 Subject: [PATCH 061/280] Install package before running tests in Appveyor --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 7b6e9b27..5c63aa33 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -75,6 +75,7 @@ test_script: # due to memory fragmentation resulting from many alloc/free cycles # ps: $env:is64bit = $env:PYTHON.Contains("-x64") # IF %is64bit% == True build.cmd tox + - "build.cmd python -m pip install -e ." - "build.cmd pytest tests" after_test: From 8e4fd9a07ddb088aaeed36debae30da26a2de449 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 13:39:43 +0000 Subject: [PATCH 062/280] Install test requirements before running tests in Appveyor --- appveyor.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 5c63aa33..7430f747 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -63,9 +63,6 @@ install: # We need twine installed to push wheels to PyPI - "build.cmd python -m pip install twine" - # We need tox to run tests - - "build.cmd python -m pip install pytest" - build_script: # Build the compiled extension - "build.cmd python setup.py build" @@ -76,6 +73,7 @@ test_script: # ps: $env:is64bit = $env:PYTHON.Contains("-x64") # IF %is64bit% == True build.cmd tox - "build.cmd python -m pip install -e ." + - "build.cmd python -m pip install -e .[tests]" - "build.cmd pytest tests" after_test: From 95198c952e9ab0775a3b1ee64863535d4e2e2e9e Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 17:55:35 +0000 Subject: [PATCH 063/280] Set PATH for osx Python 2.7 builds on Travis to fix the deployment stage Because cibuildwheel doesn't install a new Python for Python 2.7, the PATH is never set to point to the location where pip installed packages install scripts. This results in twine not being found in the deploy stage. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 09aabcea..0b2e489b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,8 @@ matrix: os: osx language: generic env: - - PIP=pip2 + - PATH="/Library/Frameworks/Python.framework/Versions/2.7/bin:$PATH" + - PIP=pip - CIBW_BUILD=cp27-* - CIBW_TEST_COMMAND="pytest {project}/tests" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" From 6ec563fbf456a8dae85aa564c7828adbd23a0ac8 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 19:40:39 +0000 Subject: [PATCH 064/280] Add codecov support --- .travis.yml | 4 +++- README.rst | 5 +++++ setup.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0b2e489b..66ebee38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -112,14 +112,16 @@ install: - ${PIP} install -r docs/requirements.txt - ${PIP} install cibuildwheel==0.10.0 - ${PIP} install .[flake8] + - ${PIP} install codecov script: - flake8 --ignore=E501 lz4 - flake8 --ignore=E501 tests - flake8 --ignore=E501 setup.py - ${PIP} install . - ${PIP} install .[tests] - - pytest + - pytest --cov=./lz4 - make -C docs doctest html + - codecov - cibuildwheel --output-dir dist deploy: - provider: pypi diff --git a/README.rst b/README.rst index fb5d2913..178b5ed9 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,11 @@ Status :target: https://readthedocs.org/projects/python-lz4/ :alt: Documentation +.. image:: https://codecov.io/gh/python-lz4/python-lz4/branch/codecov/graph/badge.svg + :target: https://codecov.io/gh/python-lz4/python-lz4 + :alt: CodeCov + + Introduction ============ This package provides python bindings for the `LZ4 compression library diff --git a/setup.py b/setup.py index b402e708..d7d48d18 100644 --- a/setup.py +++ b/setup.py @@ -146,6 +146,7 @@ def pkgconfig_installed_check(lib, required_version, default): tests_require = [ 'pytest!=3.3.0', 'psutil', + 'pytest-cov', ], # Only require pytest-runner if actually running the tests From 4e6667ea1035ab5aa5ea6f75298650ab58a79d02 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 21:00:05 +0000 Subject: [PATCH 065/280] Use coverage directly rather than pytest-cov --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66ebee38..71165817 100644 --- a/.travis.yml +++ b/.travis.yml @@ -117,9 +117,9 @@ script: - flake8 --ignore=E501 lz4 - flake8 --ignore=E501 tests - flake8 --ignore=E501 setup.py - - ${PIP} install . + - ${PIP} install -e . - ${PIP} install .[tests] - - pytest --cov=./lz4 + - coverage run --source=lz4 -m pytest tests - make -C docs doctest html - codecov - cibuildwheel --output-dir dist From 1efa378a4f87e86f79efed55c3e3dcd2cd2d9d60 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 23:05:10 +0000 Subject: [PATCH 066/280] Increase memory size increase limit in test_frame_5.py Using the coverage package increased the incremental memory usage slightly, requiring this limit to increase. --- tests/frame/test_frame_5.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 41e31855..4805c838 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -1,6 +1,8 @@ import lz4.frame import pytest +MEM_INCREASE_LIMIT = (1024 * 25) + test_data = [ (b'a' * 1024 * 1024), ] @@ -32,7 +34,7 @@ def test_frame_decompress_mem_usage(data): if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 20) + assert stats[0].size_diff < MEM_INCREASE_LIMIT prev_snapshot = snapshot @@ -56,7 +58,7 @@ def test_frame_decompress_chunk_mem_usage(data): if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 20) + assert stats[0].size_diff < MEM_INCREASE_LIMIT prev_snapshot = snapshot @@ -79,7 +81,7 @@ def test_frame_open_decompress_mem_usage(data): if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 20) + assert stats[0].size_diff < MEM_INCREASE_LIMIT prev_snapshot = snapshot From 5ebabbd95bdaefd5d42c7693fd03fcb0a5a33cd2 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 23:06:46 +0000 Subject: [PATCH 067/280] Revert to using pytest-cov for coverage reporting --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 71165817..e8da2db8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -119,7 +119,7 @@ script: - flake8 --ignore=E501 setup.py - ${PIP} install -e . - ${PIP} install .[tests] - - coverage run --source=lz4 -m pytest tests + - pytest --cov=lz4 tests # coverage run --source=lz4 -m pytest tests - make -C docs doctest html - codecov - cibuildwheel --output-dir dist From 9744f3d37c5b4773e82aa23f5bbd6fff78eeb04f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 23:17:24 +0000 Subject: [PATCH 068/280] Ensure package is installed immediately before coverage tests It's unclear why, but if "pip install -e ." is followed by "pip install .[tests]" before the coverage check is run, the coverage report is empty. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e8da2db8..cf9fd522 100644 --- a/.travis.yml +++ b/.travis.yml @@ -117,8 +117,8 @@ script: - flake8 --ignore=E501 lz4 - flake8 --ignore=E501 tests - flake8 --ignore=E501 setup.py - - ${PIP} install -e . - ${PIP} install .[tests] + - ${PIP} install -e . # This needs to be IMMEDIATELY before coverage tests - pytest --cov=lz4 tests # coverage run --source=lz4 -m pytest tests - make -C docs doctest html - codecov From 08d8f22c2ec7a90bc46c15d1e1c0edde11962935 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 18 Dec 2018 23:21:16 +0000 Subject: [PATCH 069/280] Remove any generated files before running cibuildwheel For some reason, running cibuildwheel after the coverage tests results in auditwheel failures during cibuildwheel. So, we revert to a clean state before calling cibuildwheel. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index cf9fd522..f480bc4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -122,6 +122,7 @@ script: - pytest --cov=lz4 tests # coverage run --source=lz4 -m pytest tests - make -C docs doctest html - codecov + - git stash --all # Restore fresh checkout - cibuildwheel --output-dir dist deploy: - provider: pypi From 89b3c966c90209e6d7d5862437ccf49b93c652f7 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 10:32:37 +0000 Subject: [PATCH 070/280] Update tox.ini to use coverage --- tox.ini | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/tox.ini b/tox.ini index 6b865a75..de394bd5 100644 --- a/tox.ini +++ b/tox.ini @@ -7,15 +7,6 @@ extras = tests passenv = * # setenv = PYTHONMALLOC = pymalloc # PYTHONMALLOCSTATS = 'yes' +usedevelop = True commands= - pytest --tb=long -l tests/block/test_block_0.py - pytest --tb=long -l tests/block/test_block_1.py - pytest --tb=long -l tests/block/test_block_2.py - pytest --tb=long -l tests/frame/test_frame_0.py - pytest --tb=long -l tests/frame/test_frame_1.py - pytest --tb=long -l tests/frame/test_frame_2.py - pytest --tb=long -l tests/frame/test_frame_3.py - pytest --tb=long -l tests/frame/test_frame_4.py - pytest --tb=long -l tests/frame/test_frame_5.py - pytest --tb=long -l tests/frame/test_frame_6.py - pytest --tb=long -l tests/frame/test_frame_7.py + pytest --cov=lz4 --tb=long {posargs} From 4d1434859a39d09f72bc2dfa7ad572d98a8606ad Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 10:42:04 +0000 Subject: [PATCH 071/280] Use tox for testing on travis --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f480bc4d..1225b829 100644 --- a/.travis.yml +++ b/.travis.yml @@ -112,14 +112,12 @@ install: - ${PIP} install -r docs/requirements.txt - ${PIP} install cibuildwheel==0.10.0 - ${PIP} install .[flake8] - - ${PIP} install codecov + - ${PIP} install codecov tox script: - flake8 --ignore=E501 lz4 - flake8 --ignore=E501 tests - flake8 --ignore=E501 setup.py - - ${PIP} install .[tests] - - ${PIP} install -e . # This needs to be IMMEDIATELY before coverage tests - - pytest --cov=lz4 tests # coverage run --source=lz4 -m pytest tests + - tox tests - make -C docs doctest html - codecov - git stash --all # Restore fresh checkout From 824be78552d6b277d6311a1d4f3c0d70df8acfba Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 10:42:24 +0000 Subject: [PATCH 072/280] Use tox for testing on Appveyor --- appveyor.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 7430f747..c23e4bd0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -60,6 +60,9 @@ install: # We need wheel installed to build wheels, and this isn't pulled in by setup.py - "build.cmd python -m pip install wheel" + # Install tox for testing + - "build.cmd python -m pip install tox" + # We need twine installed to push wheels to PyPI - "build.cmd python -m pip install twine" @@ -68,13 +71,7 @@ build_script: - "build.cmd python setup.py build" test_script: - # Run the project tests. Test only for 64 bit Python as the 32 bit tests fail - # due to memory fragmentation resulting from many alloc/free cycles - # ps: $env:is64bit = $env:PYTHON.Contains("-x64") - # IF %is64bit% == True build.cmd tox - - "build.cmd python -m pip install -e ." - - "build.cmd python -m pip install -e .[tests]" - - "build.cmd pytest tests" + - "build.cmd tox tests" after_test: # If tests are successful, create binary packages for the project. From 6042f9b68326734a243ba926ca444f957287e936 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 14:36:12 +0000 Subject: [PATCH 073/280] Add flake8 and docs envs to tox.ini --- tox.ini | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index de394bd5..c161d225 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py +envlist = py,flake8,docs minversion = 2.4.0 [testenv] @@ -10,3 +10,18 @@ passenv = * usedevelop = True commands= pytest --cov=lz4 --tb=long {posargs} + +[testenv:flake8] +extras = flake8 +passenv = * +commands = + flake8 lz4 setup.py tests + +[flake8] +ignore = E501 + +[testenv:docs] +usedevelop = True +extras = docs +commands = + make -C docs doctest html \ No newline at end of file From b180388bb20e9e819dea89a1f77dd7144eb53e93 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 14:36:40 +0000 Subject: [PATCH 074/280] Add docs requires section to setup.py --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index d7d48d18..3c82c8e0 100644 --- a/setup.py +++ b/setup.py @@ -179,6 +179,10 @@ def pkgconfig_installed_check(lib, required_version, default): tests_require=tests_require, extras_require={ 'tests': tests_require, + 'docs': [ + 'sphinx >= 1.6.0', + 'sphinx_bootstrap_theme', + ], 'flake8': [ 'flake8', ] From 719c83b9c6851c40d200906b3e1a7e8edaedb495 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 14:38:01 +0000 Subject: [PATCH 075/280] Use tox for flake8 and docs tests on Travis --- .travis.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1225b829..50984556 100644 --- a/.travis.yml +++ b/.travis.yml @@ -109,17 +109,13 @@ matrix: - cibuildwheel --output-dir dist install: - ${PIP} install -U pip - - ${PIP} install -r docs/requirements.txt - ${PIP} install cibuildwheel==0.10.0 - - ${PIP} install .[flake8] - ${PIP} install codecov tox script: - - flake8 --ignore=E501 lz4 - - flake8 --ignore=E501 tests - - flake8 --ignore=E501 setup.py + - tox -e flake8 - tox tests - - make -C docs doctest html - codecov + - tox -e docs - git stash --all # Restore fresh checkout - cibuildwheel --output-dir dist deploy: From b9df6e226cd798fa4be54714fa10ec439aabbc2d Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 14:40:34 +0000 Subject: [PATCH 076/280] Use tox for OSX tests on Travis --- .travis.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 50984556..b0c8e7f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,8 +53,8 @@ matrix: - PATH="/Library/Frameworks/Python.framework/Versions/2.7/bin:$PATH" - PIP=pip - CIBW_BUILD=cp27-* - - CIBW_TEST_COMMAND="pytest {project}/tests" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" install: - ${PIP} install cibuildwheel==0.10.0 script: @@ -65,8 +65,8 @@ matrix: env: - PIP=pip2 - CIBW_BUILD=cp34-* - - CIBW_TEST_COMMAND="pytest {project}/tests" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" install: - ${PIP} install cibuildwheel==0.10.0 script: @@ -77,8 +77,8 @@ matrix: env: - PIP=pip2 - CIBW_BUILD=cp35-* - - CIBW_TEST_COMMAND="pytest {project}/tests" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" install: - ${PIP} install cibuildwheel==0.10.0 script: @@ -89,8 +89,8 @@ matrix: env: - PIP=pip2 - CIBW_BUILD=cp36-* - - CIBW_TEST_COMMAND="pytest {project}/tests" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" install: - ${PIP} install cibuildwheel==0.10.0 script: @@ -101,8 +101,8 @@ matrix: env: - PIP=pip2 - CIBW_BUILD=cp37-* - - CIBW_TEST_COMMAND="pytest {project}/tests" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install .; $PIP install .[tests]" + - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" install: - ${PIP} install cibuildwheel==0.10.0 script: From 19ec8fe9adaf6623f101def9ba1fa2fd39ddc0b9 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 15:21:32 +0000 Subject: [PATCH 077/280] Whitelist make and add pytest options in tox.ini --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index c161d225..57d526d7 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,10 @@ usedevelop = True commands= pytest --cov=lz4 --tb=long {posargs} +[pytest] +addopts = -x --tb=long --showlocals + + [testenv:flake8] extras = flake8 passenv = * @@ -22,6 +26,7 @@ ignore = E501 [testenv:docs] usedevelop = True +whitelist_externals=make extras = docs commands = make -C docs doctest html \ No newline at end of file From 9cd62f3e038838ae975ab276b2deb3f43dfb11ba Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 15:21:52 +0000 Subject: [PATCH 078/280] Remove duplicated tox tests from .travis.yml --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b0c8e7f1..98b6076f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -112,10 +112,8 @@ install: - ${PIP} install cibuildwheel==0.10.0 - ${PIP} install codecov tox script: - - tox -e flake8 - tox tests - codecov - - tox -e docs - git stash --all # Restore fresh checkout - cibuildwheel --output-dir dist deploy: From 7196aa13d97fe5ee9bea1f397b286021d773e0b5 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 19 Dec 2018 16:16:56 +0000 Subject: [PATCH 079/280] Don't build docs or run flake8 on Appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index c23e4bd0..932b9def 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -71,7 +71,7 @@ build_script: - "build.cmd python setup.py build" test_script: - - "build.cmd tox tests" + - "build.cmd tox -e py tests" after_test: # If tests are successful, create binary packages for the project. From f87138185ce357b23df5d772f9cab18ac197ae6c Mon Sep 17 00:00:00 2001 From: Pratyush Das Date: Fri, 19 Apr 2019 19:52:31 +0530 Subject: [PATCH 080/280] Fixed broken links --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 178b5ed9..18fb95f1 100644 --- a/README.rst +++ b/README.rst @@ -28,8 +28,8 @@ This package provides python bindings for the `LZ4 compression library `_. The bindings provided in this package cover the `frame format -`_ and the `block format -`_ specifications. The frame +`_ and the `block format +`_ specifications. The frame format bindings are the recommended ones to use, as this guarantees interoperability with other implementations and language bindings. From 5d8110a3c605278e619782ddab39f06e4b9e585a Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 12 May 2019 11:02:41 +0100 Subject: [PATCH 081/280] Replace uses of message with match in tests --- tests/frame/test_frame_3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/frame/test_frame_3.py b/tests/frame/test_frame_3.py index aa7c782d..6d195888 100644 --- a/tests/frame/test_frame_3.py +++ b/tests/frame/test_frame_3.py @@ -22,7 +22,7 @@ def test_decompress_truncated(data): compressed = lz4frame.compress(data) message = r'^LZ4F_getFrameInfo failed with code: ERROR_frameHeader_incomplete' - with pytest.raises(RuntimeError, message=message): + with pytest.raises(RuntimeError, match=message): lz4frame.decompress(compressed[:6]) for i in range(16, len(compressed) - 1, 5): # 15 is the max size of the header @@ -32,14 +32,14 @@ def test_decompress_truncated(data): lz4frame.decompress(compressed[:i]) except RuntimeError as r: print(r) - with pytest.raises(RuntimeError, message=message): + with pytest.raises(RuntimeError, match=message): lz4frame.decompress(compressed[:i]) def test_content_checksum_failure(data): compressed = lz4frame.compress(data, content_checksum=True) message = r'^LZ4F_decompress failed with code: ERROR_contentChecksum_invalid$' - with pytest.raises(RuntimeError, message=message): + with pytest.raises(RuntimeError, match=message): last = struct.unpack('B', compressed[-1:])[0] lz4frame.decompress(compressed[:-1] + struct.pack('B', last ^ 0x42)) @@ -53,6 +53,6 @@ def test_block_checksum_failure(data): ) message = r'^LZ4F_decompress failed with code: ERROR_blockChecksum_invalid$' if len(compressed) > 32: - with pytest.raises(RuntimeError, message=message): + with pytest.raises(RuntimeError, match=message): compressed[18] = compressed[18] ^ 0x42 lz4frame.decompress(compressed) From 663c547fe9759e8f82742272a93f543d6abb35ad Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 1 Jun 2019 15:32:12 +0100 Subject: [PATCH 082/280] Suppress warnings with MSVC: 4711, 4820 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3c82c8e0..9fa31318 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ def pkgconfig_installed_check(lib, required_version, default): extra_compile_args = [] if compiler == 'msvc': - extra_compile_args = ['/Ot', '/Wall'] + extra_compile_args = ['/Ot', '/Wall', '/wd4711', '/wd4820'] elif compiler in ('unix', 'mingw32'): if liblz4_found: extra_link_args.append(pkgconfig_libs('liblz4')) From 3035f5fab6f35293a67a839010795c6efcbc3f2b Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 1 Jun 2019 15:38:27 +0100 Subject: [PATCH 083/280] Fix tests in test_frame_3.py --- tests/frame/test_frame_3.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/frame/test_frame_3.py b/tests/frame/test_frame_3.py index 6d195888..a7835a46 100644 --- a/tests/frame/test_frame_3.py +++ b/tests/frame/test_frame_3.py @@ -26,8 +26,7 @@ def test_decompress_truncated(data): lz4frame.decompress(compressed[:6]) for i in range(16, len(compressed) - 1, 5): # 15 is the max size of the header - message = r'^Frame incomplete. LZ4F_decompress returned: {0}'.format( - len(compressed) - i) + message = r'^Frame incomplete. LZ4F_decompress returned:' try: lz4frame.decompress(compressed[:i]) except RuntimeError as r: @@ -54,5 +53,5 @@ def test_block_checksum_failure(data): message = r'^LZ4F_decompress failed with code: ERROR_blockChecksum_invalid$' if len(compressed) > 32: with pytest.raises(RuntimeError, match=message): - compressed[18] = compressed[18] ^ 0x42 + compressed[22] = compressed[18] ^ 0x42 lz4frame.decompress(compressed) From d72043e3063f3d20927fa9aa8fe92d13b94ed571 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 1 Jun 2019 15:39:49 +0100 Subject: [PATCH 084/280] Fix handling of Py_ssize_t on MSVC (#172) --- lz4/frame/_frame.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 10e1281f..4367c1ec 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -290,7 +290,7 @@ compress_begin (PyObject * Py_UNUSED (self), PyObject * args, PyObject * keywds) { PyObject *py_context = NULL; - Py_ssize_t source_size = 0; + Py_ssize_t source_size = (Py_ssize_t) 0; int return_bytearray = 0; int content_checksum = 0; int block_checksum = 0; @@ -1067,9 +1067,9 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size, } else { - if (max_length >= 0) + if (max_length >= (Py_ssize_t) 0) { - destination_size = max_length; + destination_size = (size_t) max_length; } else { @@ -1092,7 +1092,7 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size, /* Only set stableDst = 1 if we are sure no PyMem_Realloc will be called since when stableDst = 1 the LZ4 library stores a pointer to the last compressed data, which may be invalid after a PyMem_Realloc. */ - if (full_frame && max_length >= 0) + if (full_frame && max_length >= (Py_ssize_t) 0) { options.stableDst = 1; } @@ -1157,7 +1157,7 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size, /* Destination buffer is full. So, stop decompressing if max_length is set. Otherwise expand the destination buffer. */ - if (max_length >= 0) + if (max_length >= (Py_ssize_t) 0) { break; } @@ -1339,7 +1339,7 @@ decompress_chunk (PyObject * Py_UNUSED (self), PyObject * args, Py_buffer py_source; char * source; size_t source_size; - Py_ssize_t max_length = -1; + Py_ssize_t max_length = (Py_ssize_t) -1; int return_bytearray = 0; static char *kwlist[] = { "context", "data", @@ -1349,7 +1349,7 @@ decompress_chunk (PyObject * Py_UNUSED (self), PyObject * args, }; #if IS_PY3 - if (!PyArg_ParseTupleAndKeywords (args, keywds, "Oy*|kp", kwlist, + if (!PyArg_ParseTupleAndKeywords (args, keywds, "Oy*|np", kwlist, &py_context, &py_source, &max_length, @@ -1359,7 +1359,7 @@ decompress_chunk (PyObject * Py_UNUSED (self), PyObject * args, return NULL; } #else - if (!PyArg_ParseTupleAndKeywords (args, keywds, "Os*|ki", kwlist, + if (!PyArg_ParseTupleAndKeywords (args, keywds, "Os*|ni", kwlist, &py_context, &py_source, &max_length, From 78b66226b4b695fe9ba916ca739ea7775addef38 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 1 Jun 2019 15:40:50 +0100 Subject: [PATCH 085/280] Fix docstrings for frame compression functions --- lz4/frame/_frame.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 4367c1ec..99ebe02d 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1654,7 +1654,7 @@ PyDoc_STRVAR PyDoc_STRVAR ( decompress_chunk__doc, - "decompress_chunk(context, data)\n" \ + "decompress_chunk(context, data, max_length=-1)\n" \ "\n" \ "Decompresses part of a frame of compressed data.\n" \ "\n" \ @@ -1667,7 +1667,7 @@ PyDoc_STRVAR " frame of compressed data\n" \ "\n" \ "Keyword Args:\n" \ - " max_length (int): if non-negative this specifies the maximum number" \ + " max_length (int): if non-negative this specifies the maximum number\n" \ " of bytes of uncompressed data to return. Default is ``-1``.\n" \ " return_bytearray (bool): If ``True`` a bytearray object will be\n" \ " returned.If ``False``, a string of bytes is returned. The\n" \ From 2b640b639877da4608f502b75a96dd3fdff56d0a Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 1 Jun 2019 15:43:43 +0100 Subject: [PATCH 086/280] Add test_frame_9.py to test for issue #172 --- tests/frame/test_frame_9.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/frame/test_frame_9.py diff --git a/tests/frame/test_frame_9.py b/tests/frame/test_frame_9.py new file mode 100644 index 00000000..27d61607 --- /dev/null +++ b/tests/frame/test_frame_9.py @@ -0,0 +1,44 @@ +import os +import lz4.frame + + +def test_issue_172_1(): + """Test reproducer for issue 172 + + Issue 172 is a reported failure occurring on Windows 10 only. This bug was + due to incorrect handling of Py_ssize_t types when doing comparisons and + using them as a size when allocating memory. + + """ + input_data = 8 * os.urandom(1024) + with lz4.frame.open('testfile_small', 'wb') as fp: + bytes_written = fp.write(input_data) # noqa: F841 + + with lz4.frame.open('testfile_small', 'rb') as fp: + data = fp.read(10) + assert len(data) == 10 + + +def test_issue_172_2(): + input_data = 9 * os.urandom(1024) + with lz4.frame.open('testfile_small', 'w') as fp: + bytes_written = fp.write(input_data) # noqa: F841 + + with lz4.frame.open('testfile_small', 'r') as fp: + data = fp.read(10) + assert len(data) == 10 + + +def test_issue_172_3(): + input_data = 9 * os.urandom(1024) + with lz4.frame.open('testfile_small', 'wb') as fp: + bytes_written = fp.write(input_data) # noqa: F841 + + with lz4.frame.open('testfile_small', 'rb') as fp: + data = fp.read(10) + assert len(data) == 10 + + with lz4.frame.open('testfile_small', 'rb') as fp: + data = fp.read(16 * 1024 - 1) + assert len(data) == 9 * 1024 + assert data == input_data From a3fe8d173d680188d638ef851180a264348e06dc Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 1 Jun 2019 15:44:36 +0100 Subject: [PATCH 087/280] Add test_frame_8.py --- tests/frame/test_frame_8.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/frame/test_frame_8.py diff --git a/tests/frame/test_frame_8.py b/tests/frame/test_frame_8.py new file mode 100644 index 00000000..159534ae --- /dev/null +++ b/tests/frame/test_frame_8.py @@ -0,0 +1,12 @@ +import lz4.frame as lz4frame + + +def test_lz4frame_open_write_read_text_iter(): + data = u'This is a test string' + with lz4frame.open('testfile', mode='wt') as fp: + fp.write(data) + data_out = '' + with lz4frame.open('testfile', mode='rt') as fp: + for line in fp: + data_out += line + assert data_out == data From a4ea513b87e311fa74f85193334a90211985c719 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 2 Jun 2019 08:06:05 +0100 Subject: [PATCH 088/280] Ensure that twine is installed on travis --- .travis.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98b6076f..a3163046 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ matrix: - PIP=pip - CIBW_BUILD=cp27-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - ${PIP} install cibuildwheel==0.10.0 script: @@ -66,7 +66,7 @@ matrix: - PIP=pip2 - CIBW_BUILD=cp34-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - ${PIP} install cibuildwheel==0.10.0 script: @@ -78,7 +78,7 @@ matrix: - PIP=pip2 - CIBW_BUILD=cp35-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - ${PIP} install cibuildwheel==0.10.0 script: @@ -90,7 +90,7 @@ matrix: - PIP=pip2 - CIBW_BUILD=cp36-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - ${PIP} install cibuildwheel==0.10.0 script: @@ -102,7 +102,7 @@ matrix: - PIP=pip2 - CIBW_BUILD=cp37-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox" + - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - ${PIP} install cibuildwheel==0.10.0 script: @@ -110,7 +110,7 @@ matrix: install: - ${PIP} install -U pip - ${PIP} install cibuildwheel==0.10.0 - - ${PIP} install codecov tox + - ${PIP} install codecov tox twine script: - tox tests - codecov @@ -128,4 +128,3 @@ deploy: on: branch: master tags: true - From 1747401344d4b8016e32cb470a9eb770fb75676f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 2 Jun 2019 08:34:20 +0100 Subject: [PATCH 089/280] Invoke garbage collector in test_frame_5.py --- tests/frame/test_frame_5.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 4805c838..05daf283 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -1,5 +1,6 @@ import lz4.frame import pytest +import gc MEM_INCREASE_LIMIT = (1024 * 25) @@ -30,6 +31,7 @@ def test_frame_decompress_mem_usage(data): decompressed = lz4.frame.decompress(compressed) # noqa: F841 if i % 100 == 0: + gc.collect() snapshot = tracemalloc.take_snapshot() if prev_snapshot: @@ -54,6 +56,7 @@ def test_frame_decompress_chunk_mem_usage(data): ) if i % 100 == 0: + gc.collect() snapshot = tracemalloc.take_snapshot() if prev_snapshot: @@ -77,6 +80,7 @@ def test_frame_open_decompress_mem_usage(data): decompressed = f.read() # noqa: F841 if i % 100 == 0: + gc.collect() snapshot = tracemalloc.take_snapshot() if prev_snapshot: From 545dd860ee6d9a449c45d32305132fb7ba125c85 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 2 Jun 2019 08:39:01 +0100 Subject: [PATCH 090/280] Update bundled lz4 libs to version 1.9.1 --- lz4libs/lz4.c | 884 +++++++++++++++++++++++++++++++-------------- lz4libs/lz4.h | 371 +++++++++++-------- lz4libs/lz4frame.c | 428 ++++++++++++---------- lz4libs/lz4frame.h | 147 +++++--- lz4libs/lz4hc.c | 358 ++++++++++-------- lz4libs/lz4hc.h | 244 ++++++++----- lz4libs/xxhash.c | 614 +++++++++++++++++++------------ lz4libs/xxhash.h | 221 +++++++----- 8 files changed, 2042 insertions(+), 1225 deletions(-) diff --git a/lz4libs/lz4.c b/lz4libs/lz4.c index 4046102e..e614c457 100644 --- a/lz4libs/lz4.c +++ b/lz4libs/lz4.c @@ -32,7 +32,6 @@ - LZ4 source repository : https://github.com/lz4/lz4 */ - /*-************************************ * Tuning parameters **************************************/ @@ -91,8 +90,22 @@ /*-************************************ * Dependency **************************************/ +/* + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included + */ +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 +#endif + +#ifndef LZ4_STATIC_LINKING_ONLY #define LZ4_STATIC_LINKING_ONLY +#endif + +#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS #define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ +#endif + #include "lz4.h" /* see also "memory routines" below */ @@ -123,7 +136,7 @@ #endif /* LZ4_FORCE_INLINE */ /* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE - * Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy, + * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, * together with a simple 8-byte copy loop as a fall-back path. * However, this optimization hurts the decompression speed by >30%, * because the execution does not go to the optimized loop @@ -131,12 +144,12 @@ * before going to the fall-back path become useless overhead. * This optimization happens only with the -O3 flag, and -O2 generates * a simple 8-byte copy loop. - * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 * functions are annotated with __attribute__((optimize("O2"))), - * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute - * of LZ4_wildCopy does not affect the compression speed. + * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy8 does not affect the compression speed. */ -#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) # define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) # define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE #else @@ -170,7 +183,7 @@ /*-************************************ -* Basic Types +* Types **************************************/ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # include @@ -195,6 +208,13 @@ typedef size_t reg_t; /* 32-bits in x32 mode */ #endif +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; + + /*-************************************ * Reading and writing into memory **************************************/ @@ -228,7 +248,7 @@ static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArc static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -#else /* safe and portable access through memcpy() */ +#else /* safe and portable access using memcpy() */ static U16 LZ4_read16(const void* memPtr) { @@ -281,7 +301,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value) /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ LZ4_FORCE_O2_INLINE_GCC_PPC64LE -void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; @@ -290,6 +310,86 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) do { memcpy(d,s,8); d+=8; s+=8; } while (d= 16. */ +LZ4_FORCE_O2_INLINE_GCC_PPC64LE void +LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d 65535) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif #define ML_BITS 4 #define ML_MASK ((1U< compression ru /*-************************************ * Local Structures and types **************************************/ -typedef enum { notLimited = 0, limitedOutput = 1, fillOutput = 2 } limitedOutput_directive; typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; /** @@ -522,13 +627,14 @@ static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { - static const U64 prime5bytes = 889523592379ULL; - static const U64 prime8bytes = 11400714785074694791ULL; const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - else + } else { + const U64 prime8bytes = 11400714785074694791ULL; return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } } LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) @@ -609,6 +715,15 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( LZ4_stream_t_internal* const cctx, const int inputSize, const tableType_t tableType) { + /* If compression failed during the previous step, then the context + * is marked as dirty, therefore, it has to be fully reset. + */ + if (cctx->dirty) { + DEBUGLOG(5, "LZ4_prepareTable: Full reset for %p", cctx); + MEM_INIT(cctx, 0, sizeof(LZ4_stream_t_internal)); + return; + } + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is * therefore safe to use no matter what mode we're in. Otherwise, we figure * out if it's safe to leave as is or whether it needs to be reset. @@ -629,7 +744,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( } } - /* Adding a gap, so all previous entries are > MAX_DISTANCE back, is faster + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster * than compressing without a gap. However, compressing with * currentOffset == 0 is faster still, so we preserve that case. */ @@ -651,14 +766,15 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( const char* const source, char* const dest, const int inputSize, - int *inputConsumed, /* only written when outputLimited == fillOutput */ + int *inputConsumed, /* only written when outputDirective == fillOutput */ const int maxOutputSize, - const limitedOutput_directive outputLimited, + const limitedOutput_directive outputDirective, const tableType_t tableType, const dict_directive dictDirective, const dictIssue_directive dictIssue, - const U32 acceleration) + const int acceleration) { + int result; const BYTE* ip = (const BYTE*) source; U32 const startIndex = cctx->currentOffset; @@ -693,9 +809,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( U32 forwardH; DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); - /* Init conditions */ - if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ + /* If init conditions are not met, we don't have to mark stream + * as having dirty context, since no action was taken yet */ + if (outputDirective == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ assert(acceleration >= 1); @@ -728,8 +845,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Find a match */ if (tableType == byPtr) { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; do { U32 const h = forwardH; ip = forwardIp; @@ -743,14 +860,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - } while ( (match+MAX_DISTANCE < ip) + } while ( (match+LZ4_DISTANCE_MAX < ip) || (LZ4_read32(match) != LZ4_read32(ip)) ); } else { /* byU32, byU16 */ const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; do { U32 const h = forwardH; U32 const current = (U32)(forwardIp - base); @@ -794,8 +911,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */ assert(matchIndex < current); - if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far */ - if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */ + if ((tableType != byU16) && (matchIndex+LZ4_DISTANCE_MAX < current)) continue; /* too far */ + if (tableType == byU16) assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* too_far presumed impossible with byU16 */ if (LZ4_read32(match) == LZ4_read32(ip)) { if (maybe_extMem) offset = current - matchIndex; @@ -811,16 +928,17 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Encode Literals */ { unsigned const litLength = (unsigned)(ip - anchor); token = op++; - if ((outputLimited == limitedOutput) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; - if ((outputLimited == fillOutput) && + if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + + if ((outputDirective == fillOutput) && (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { op--; goto _last_literals; } if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; + int len = (int)(litLength - RUN_MASK); *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; *op++ = (BYTE)len; @@ -828,7 +946,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( else *token = (BYTE)(litLength< olimit)) { /* the match was too close to the end, rewind and go to last literals */ op = token; @@ -853,11 +971,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Encode Offset */ if (maybe_extMem) { /* static test */ DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); - assert(offset <= MAX_DISTANCE && offset > 0); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); LZ4_writeLE16(op, (U16)offset); op+=2; } else { DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); - assert(ip-match <= MAX_DISTANCE); + assert(ip-match <= LZ4_DISTANCE_MAX); LZ4_writeLE16(op, (U16)(ip - match)); op+=2; } @@ -870,7 +988,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( assert(dictEnd > match); if (limit > matchlimit) limit = matchlimit; matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; + ip += (size_t)matchCode + MINMATCH; if (ip==limit) { unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); matchCode += more; @@ -879,19 +997,20 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); } else { matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; + ip += (size_t)matchCode + MINMATCH; DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); } - if ((outputLimited) && /* Check output buffer overflow */ + if ((outputDirective) && /* Check output buffer overflow */ (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) { - if (outputLimited == limitedOutput) - return 0; - if (outputLimited == fillOutput) { + if (outputDirective == fillOutput) { /* Match description too long : reduce it */ U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255; ip -= matchCode - newMatchCode; matchCode = newMatchCode; + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ } } if (matchCode >= ML_MASK) { @@ -922,7 +1041,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE >= ip) + if ( (match+LZ4_DISTANCE_MAX >= ip) && (LZ4_read32(match) == LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } @@ -957,7 +1076,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); assert(matchIndex < current); if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) - && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current)) + && ((tableType==byU16) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) && (LZ4_read32(match) == LZ4_read32(ip)) ) { token=op++; *token=0; @@ -976,15 +1095,17 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( _last_literals: /* Encode Last Literals */ { size_t lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ + if ( (outputDirective) && /* Check output buffer overflow */ (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { - if (outputLimited == fillOutput) { + if (outputDirective == fillOutput) { /* adapt lastRun to fill 'dst' */ - lastRun = (olimit-op) - 1; + assert(olimit >= op); + lastRun = (size_t)(olimit-op) - 1; lastRun -= (lastRun+240)/255; + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ } - if (outputLimited == limitedOutput) - return 0; } if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; @@ -999,31 +1120,33 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( op += lastRun; } - if (outputLimited == fillOutput) { + if (outputDirective == fillOutput) { *inputConsumed = (int) (((const char*)ip)-source); } DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, (int)(((char*)op) - dest)); - return (int)(((char*)op) - dest); + result = (int)(((char*)op) - dest); + assert(result > 0); + return result; } int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; + LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; + assert(ctx != NULL); if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - LZ4_resetStream((LZ4_stream_t*)state); if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) { return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { if (inputSize < LZ4_64Klimit) {; return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } @@ -1053,7 +1176,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; LZ4_prepareTable(ctx, srcSize, tableType); return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } @@ -1067,7 +1190,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; LZ4_prepareTable(ctx, srcSize, tableType); return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } @@ -1094,23 +1217,25 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp } -int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) +int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputSize) { - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); + return LZ4_compress_fast(src, dst, srcSize, maxOutputSize, 1); } /* hidden debug function */ /* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { LZ4_stream_t ctx; - LZ4_resetStream(&ctx); + LZ4_initStream(&ctx, sizeof(ctx)); - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (srcSize < LZ4_64Klimit) { + return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + tableType_t const addrMode = (sizeof(void*) > 4) ? byU32 : byPtr; + return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, addrMode, noDict, noDictIssue, acceleration); + } } @@ -1119,7 +1244,8 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m * _continue() call without resetting it. */ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) { - LZ4_resetStream(state); + void* const s = LZ4_initStream(state, sizeof (*state)); + assert(s != NULL); (void)s; if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); @@ -1127,8 +1253,8 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, if (*srcSizePtr < LZ4_64Klimit) { return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1); } else { - tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; - return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, tableType, noDict, noDictIssue, 1); + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1); } } } @@ -1159,14 +1285,40 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe LZ4_stream_t* LZ4_createStream(void) { - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ DEBUGLOG(4, "LZ4_createStream %p", lz4s); if (lz4s == NULL) return NULL; - LZ4_resetStream(lz4s); + LZ4_initStream(lz4s, sizeof(*lz4s)); return lz4s; } +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + it reports an aligment of 8-bytes, + while actually aligning LZ4_stream_t on 4 bytes. */ +static size_t LZ4_stream_t_alignment(void) +{ + struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(t_a.t); +} +#endif + +LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) return NULL; + if (size < sizeof(LZ4_stream_t)) return NULL; +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + it reports an aligment of 8-bytes, + while actually aligning LZ4_stream_t on 4 bytes. */ + if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) return NULL; /* alignment check */ +#endif + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); + return (LZ4_stream_t*)buffer; +} + +/* resetStream is now deprecated, + * prefer initStream() which is more general */ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); @@ -1226,10 +1378,16 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) p+=3; } - return dict->dictSize; + return (int)dict->dictSize; } void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { + /* Calling LZ4_resetStream_fast() here makes sure that changes will not be + * erased by subsequent calls to LZ4_resetStream_fast() in case stream was + * marked as having dirty context, e.g. requiring full reset. + */ + LZ4_resetStream_fast(working_stream); + if (dictionary_stream != NULL) { /* If the current offset is zero, we will never look in the * external dictionary context, since there is no value a table @@ -1248,7 +1406,8 @@ void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dic static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) { - if (LZ4_dict->currentOffset + nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ /* rescale hash table */ U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; @@ -1265,7 +1424,10 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) } -int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, + const char* source, char* dest, + int inputSize, int maxOutputSize, + int acceleration) { const tableType_t tableType = byU32; LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; @@ -1273,12 +1435,12 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if (streamPtr->dirty) return 0; /* Uninitialized structure detected */ LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; /* invalidate tiny dictionaries */ - if ( (streamPtr->dictSize-1 < 4) /* intentional underflow */ + if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */ && (dictEnd != (const BYTE*)source) ) { DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); streamPtr->dictSize = 0; @@ -1371,7 +1533,7 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + if ((U32)dictSize > dict->dictSize) dictSize = (int)dict->dictSize; memmove(safeBuffer, previousDictEnd - dictSize, dictSize); @@ -1393,6 +1555,37 @@ typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; #undef MIN #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +/* Read the variable-length literal or match length. + * + * ip - pointer to use as input. + * lencheck - end ip. Return an error if ip advances >= lencheck. + * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so. + * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so. + * error (output) - error code. Should be set to 0 before call. + */ +typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error; +LZ4_FORCE_INLINE unsigned +read_variable_length(const BYTE**ip, const BYTE* lencheck, int loop_check, int initial_check, variable_length_error* error) +{ + unsigned length = 0; + unsigned s; + if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = initial_error; + return length; + } + do { + s = **ip; + (*ip)++; + length += s; + if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = loop_error; + return length; + } + } while (s==255); + + return length; +} + /*! LZ4_decompress_generic() : * This generic decompression function covers all use cases. * It shall be instantiated several times, using different sets of directives. @@ -1414,234 +1607,369 @@ LZ4_decompress_generic( const size_t dictSize /* note : = 0 if noDict */ ) { - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; + if (src == NULL) return -1; - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + outputSize; - BYTE* cpy; + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; - const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; - /* Set up the "end" pointers for the shortcut. */ - const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; - const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); - /* Special cases */ - assert(lowPrefix <= op); - assert(src != NULL); - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); - if ((endOnInput) && unlikely(srcSize==0)) return -1; + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; - /* Main Loop : decode sequences */ - while (1) { const BYTE* match; size_t offset; + unsigned token; + size_t length; - unsigned const token = *ip++; - size_t length = token >> ML_BITS; /* literal length */ - assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); - /* A two-stage shortcut for the most common case: - * 1) If the literal length is 0..14, and there is enough space, - * enter the shortcut and copy 16 bytes on behalf of the literals - * (in the fast mode, only 8 bytes can be safely copied this way). - * 2) Further if the match length is 4..18, copy 18 bytes in a similar - * manner; but we ensure that there's enough space in the output for - * those 18 bytes earlier, upon entering the shortcut (in other words, - * there is a combined check for both stages). - */ - if ( (endOnInput ? length != RUN_MASK : length <= 8) - /* strictly "less than" on input, to re-enter the loop with at least one byte */ - && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { - /* Copy the literals */ - memcpy(op, ip, endOnInput ? 16 : 8); - op += length; ip += length; - - /* The second stage: prepare for match copying, decode full info. - * If it doesn't work out, the info won't be wasted. */ - length = token & ML_MASK; /* match length */ - offset = LZ4_readLE16(ip); ip += 2; + /* Special cases */ + assert(lowPrefix <= op); + if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); + if ((endOnInput) && unlikely(srcSize==0)) return -1; + + /* Currently the fast loop shows a regression on qualcomm arm chips. */ +#if LZ4_FAST_DEC_LOOP + if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(6, "skip fast decode loop"); + goto safe_decode; + } + + /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */ + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + if (endOnInput) assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + if (error == initial_error) goto _output_error; + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ + + /* copy literals */ + cpy = op+length; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if (endOnInput) { /* LZ4_decompress_safe() */ + if ((cpy>oend-32) || (ip+length>iend-32)) goto safe_literal_copy; + LZ4_wildCopy32(op, ip, cpy); + } else { /* LZ4_decompress_fast() */ + if (cpy>oend-8) goto safe_literal_copy; + LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and only relies on end-of-block properties */ + } + ip += length; op = cpy; + } else { + cpy = op+length; + if (endOnInput) { /* LZ4_decompress_safe() */ + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* We don't need to check oend, since we check it once for each loop below */ + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) goto safe_literal_copy; + /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ + memcpy(op, ip, 16); + } else { /* LZ4_decompress_fast() */ + /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and relies on end-of-block properties */ + memcpy(op, ip, 8); + if (length > 8) memcpy(op+8, ip+8, 8); + } + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; match = op - offset; - assert(match <= op); /* check overflow */ - - /* Do not deal with overlapping matches. */ - if ( (length != ML_MASK) - && (offset >= 8) - && (dict==withPrefix64k || match >= lowPrefix) ) { - /* Copy the match. */ - memcpy(op + 0, match + 0, 8); - memcpy(op + 8, match + 8, 8); - memcpy(op +16, match +16, 2); - op += length + MINMATCH; - /* Both stages worked, load the next token. */ + + /* get matchlength */ + length = token & ML_MASK; + + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + + if (length == ML_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); + if (error != ok) goto _output_error; + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + + /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ + if (!(dict == usingExtDict) || (match >= lowPrefix)) { + if (offset >= 8) { + memcpy(op, match, 8); + memcpy(op+8, match+8, 8); + memcpy(op+16, match+16, 2); + op += length; + continue; + } } } + + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } continue; } - /* The second stage didn't work out, but the info is ready. - * Propel it right to the point of match copying. */ - goto _copy_match; - } + /* copy match within block */ + cpy = op + length; - /* decode literal length */ - if (length == RUN_MASK) { - unsigned s; - if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */ - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip= 32)); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ } + safe_decode: +#endif - /* copy literals */ - cpy = op+length; - LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); - if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) { cpy = oend; length = oend-op; } /* Partial decoding : stop in the middle of literal segment */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + while (1) { + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (endOnInput ? length != RUN_MASK : length <= 8) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { + /* Copy the literals */ + memcpy(op, ip, endOnInput ? 16 : 8); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + memcpy(op + 0, match + 0, 8); + memcpy(op + 8, match + 8, 8); + memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; } - memcpy(op, ip, length); - ip += length; - op += length; - if (!partialDecoding || (cpy == oend)) { - /* Necessarily EOF, due to parsing restrictions */ - break; + + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + if (error == initial_error) goto _output_error; + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ } - } else { - LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ - ip += length; op = cpy; - } + /* copy literals */ + cpy = op+length; +#if LZ4_FAST_DEC_LOOP + safe_literal_copy: +#endif + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + if (partialDecoding) { + if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); } /* Partial decoding : stop in the middle of literal segment */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } else { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + if (!partialDecoding || (cpy == oend)) { + /* Necessarily EOF, due to parsing restrictions */ + break; + } - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; + } else { + LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ + ip += length; op = cpy; + } - /* get matchlength */ - length = token & ML_MASK; + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; -_copy_match: - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - if (!partialDecoding) { - assert(oend > op); - assert(oend - op >= 4); - LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ - } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ + /* get matchlength */ + length = token & ML_MASK; + + _copy_match: + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + if (!partialDecoding) { + assert(oend > op); + assert(oend - op >= 4); + LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ + } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ + + if (length == ML_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); + if (error != ok) goto _output_error; + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; +#if LZ4_FAST_DEC_LOOP + safe_match_copy: +#endif + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } - /* match starting within external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) { - if (partialDecoding) length = MIN(length, (size_t)(oend-op)); - else goto _output_error; /* doesn't respect parsing restriction */ + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; } - if (length <= (size_t)(lowPrefix-match)) { - /* match fits entirely within external dictionary : just copy */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match stretches into both external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix - match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) *op++ = *match++; } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } + memcpy(op, match, mlen); + } + op = copyEnd; + if (op==oend) break; + continue; + } - /* copy match within block */ - cpy = op + length; - - /* partialDecoding : may not respect endBlock parsing restrictions */ - assert(op<=oend); - if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - size_t const mlen = MIN(length, (size_t)(oend-op)); - const BYTE* const matchEnd = match + mlen; - BYTE* const copyEnd = op + mlen; - if (matchEnd > op) { /* overlap copy */ - while (op < copyEnd) *op++ = *match++; + if (unlikely(offset<8)) { + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + memcpy(op+4, match, 4); + match -= dec64table[offset]; } else { - memcpy(op, match, mlen); + memcpy(op, match, 8); + match += 8; } - op = copyEnd; - if (op==oend) break; - continue; - } - - if (unlikely(offset<8)) { - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += inc32table[offset]; - memcpy(op+4, match, 4); - match -= dec64table[offset]; - } else { - memcpy(op, match, 8); - match += 8; - } - op += 8; - - if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) *op++ = *match++; + } else { + memcpy(op, match, 8); + if (length > 16) LZ4_wildCopy8(op+8, match+8, cpy); } - while (op < cpy) *op++ = *match++; - } else { - memcpy(op, match, 8); - if (length > 16) LZ4_wildCopy(op+8, match+8, cpy); + op = cpy; /* wildcopy correction */ } - op = cpy; /* wildcopy correction */ - } - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-src); /* Nb of input bytes read */ + /* end of decoding */ + if (endOnInput) + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + else + return (int) (((const char*)ip)-src); /* Nb of input bytes read */ - /* Overflow error detected */ -_output_error: - return (int) (-(((const char*)ip)-src))-1; + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } } @@ -1745,12 +2073,13 @@ int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalS LZ4_streamDecode_t* LZ4_createStreamDecode(void) { LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); + LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */ return lz4s; } int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { - if (!LZ4_stream) return 0; /* support free on NULL */ + if (LZ4_stream == NULL) return 0; /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } @@ -1808,7 +2137,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch assert(lz4sd->extDictSize == 0); result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (result <= 0) return result; - lz4sd->prefixSize = result; + lz4sd->prefixSize = (size_t)result; lz4sd->prefixEnd = (BYTE*)dest + result; } else if (lz4sd->prefixEnd == (BYTE*)dest) { /* They're rolling the current segment. */ @@ -1821,7 +2150,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize += result; + lz4sd->prefixSize += (size_t)result; lz4sd->prefixEnd += result; } else { /* The buffer wraps around, or they're switching to another buffer. */ @@ -1830,7 +2159,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize = result; + lz4sd->prefixSize = (size_t)result; lz4sd->prefixEnd = (BYTE*)dest + result; } @@ -1842,12 +2171,13 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; + assert(originalSize >= 0); if (lz4sd->prefixSize == 0) { assert(lz4sd->extDictSize == 0); result = LZ4_decompress_fast(source, dest, originalSize); if (result <= 0) return result; - lz4sd->prefixSize = originalSize; + lz4sd->prefixSize = (size_t)originalSize; lz4sd->prefixEnd = (BYTE*)dest + originalSize; } else if (lz4sd->prefixEnd == (BYTE*)dest) { if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0) @@ -1856,7 +2186,7 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch result = LZ4_decompress_fast_doubleDict(source, dest, originalSize, lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize += originalSize; + lz4sd->prefixSize += (size_t)originalSize; lz4sd->prefixEnd += originalSize; } else { lz4sd->extDictSize = lz4sd->prefixSize; @@ -1864,7 +2194,7 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch result = LZ4_decompress_fast_extDict(source, dest, originalSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize = originalSize; + lz4sd->prefixSize = (size_t)originalSize; lz4sd->prefixEnd = (BYTE*)dest + originalSize; } diff --git a/lz4libs/lz4.h b/lz4libs/lz4.h index 059ef7c1..a9c932cc 100644 --- a/lz4libs/lz4.h +++ b/lz4libs/lz4.h @@ -51,19 +51,23 @@ extern "C" { multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. The LZ4 compression library provides in-memory compression and decompression functions. + It gives full buffer control to user. Compression can be done in: - a single step (described as Simple Functions) - a single step, reusing a context (described in Advanced Functions) - unbounded multiple steps (described as Streaming compression) - lz4.h provides block compression functions. It gives full buffer control to user. - Decompressing an lz4-compressed block also requires metadata (such as compressed size). - Each application is free to encode such metadata in whichever way it wants. + lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). + Decompressing a block requires additional metadata, such as its compressed size. + Each application is free to encode and pass such metadata in whichever way it wants. - An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), - take care of encoding standard metadata alongside LZ4-compressed blocks. - Frame format is required for interoperability. - It is delivered through a companion API, declared in lz4frame.h. + lz4.h only handle blocks, it can not generate Frames. + + Blocks are different from Frames (doc/lz4_Frame_format.md). + Frames bundle both blocks and metadata in a specified manner. + This are required for compressed data to be self-contained and portable. + Frame format is delivered through a companion API, declared in lz4frame.h. + Note that the `lz4` CLI can only manage frames. */ /*^*************************************************************** @@ -92,8 +96,8 @@ extern "C" { /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -103,7 +107,7 @@ extern "C" { #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */ -LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; unseful to check dll version */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */ /*-************************************ @@ -112,14 +116,15 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; /*! * LZ4_MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage may improve speed, thanks to cache effect + * Increasing memory usage improves compression ratio. + * Reduced memory usage may improve speed, thanks to better cache locality. * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #ifndef LZ4_MEMORY_USAGE # define LZ4_MEMORY_USAGE 14 #endif + /*-************************************ * Simple Functions **************************************/ @@ -130,21 +135,22 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; It also runs faster, so it's a recommended setting. If the function cannot compress 'src' into a more limited 'dst' budget, compression stops *immediately*, and the function result is zero. - Note : as a consequence, 'dst' content is not valid. - Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + In which case, 'dst' content is undefined (invalid). srcSize : max supported value is LZ4_MAX_INPUT_SIZE. dstCapacity : size of buffer 'dst' (which must be already allocated) - return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) - or 0 if compression fails */ + @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + or 0 if compression fails + Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). +*/ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : compressedSize : is the exact complete size of the compressed block. dstCapacity : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) If destination buffer is not large enough, decoding will stop and output an error code (negative value). If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against malicious data packets. + Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer). */ LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); @@ -155,8 +161,7 @@ LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSi #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) -/*! -LZ4_compressBound() : +/*! LZ4_compressBound() : Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) This function is primarily useful for memory allocation purposes (destination buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). @@ -167,8 +172,7 @@ LZ4_compressBound() : */ LZ4LIB_API int LZ4_compressBound(int inputSize); -/*! -LZ4_compress_fast() : +/*! LZ4_compress_fast() : Same as LZ4_compress_default(), but allows selection of "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. @@ -178,13 +182,12 @@ LZ4_compress_fast() : LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -/*! -LZ4_compress_fast_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide this buffer as 'void* state' to compression function. -*/ +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ LZ4LIB_API int LZ4_sizeofState(void); LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -204,27 +207,6 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* d LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); -/*! LZ4_decompress_fast() : **unsafe!** - * This function used to be a bit faster than LZ4_decompress_safe(), - * though situation has changed in recent versions, - * and now `LZ4_decompress_safe()` can be as fast and sometimes faster than `LZ4_decompress_fast()`. - * Moreover, LZ4_decompress_fast() is not protected vs malformed input, as it doesn't perform full validation of compressed data. - * As a consequence, this function is no longer recommended, and may be deprecated in future versions. - * It's only remaining specificity is that it can decompress data without knowing its compressed size. - * - * originalSize : is the uncompressed size to regenerate. - * `dst` must be already allocated, its size must be >= 'originalSize' bytes. - * @return : number of bytes read from source buffer (== compressed size). - * If the source stream is detected malformed, the function stops decoding and returns a negative result. - * note : This function requires uncompressed originalSize to be known in advance. - * The function never writes past the output buffer. - * However, since it doesn't know its 'src' size, it may read past the intended input. - * Also, because match offsets are not validated during decoding, - * reads from 'src' may underflow. - * Use this function in trusted environment **only**. - */ -LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); - /*! LZ4_decompress_safe_partial() : * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', * into destination buffer 'dst' of size 'dstCapacity'. @@ -257,30 +239,49 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcS ***********************************************/ typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ -/*! LZ4_createStream() and LZ4_freeStream() : - * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. - * LZ4_freeStream() releases its memory. - */ LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); -/*! LZ4_resetStream() : - * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to start compressing a new stream. +/*! LZ4_resetStream_fast() : v1.9.0+ + * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks + * (e.g., LZ4_compress_fast_continue()). + * + * An LZ4_stream_t must be initialized once before usage. + * This is automatically done when created by LZ4_createStream(). + * However, should the LZ4_stream_t be simply declared on stack (for example), + * it's necessary to initialize it first, using LZ4_initStream(). + * + * After init, start any new stream with LZ4_resetStream_fast(). + * A same LZ4_stream_t can be re-used multiple times consecutively + * and compress multiple streams, + * provided that it starts each new stream with LZ4_resetStream_fast(). + * + * LZ4_resetStream_fast() is much faster than LZ4_initStream(), + * but is not compatible with memory regions containing garbage data. + * + * Note: it's only useful to call LZ4_resetStream_fast() + * in the context of streaming compression. + * The *extState* functions perform their own resets. + * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. */ -LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); /*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream_t. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Use this function to reference a static dictionary into LZ4_stream_t. + * The dictionary must remain available during compression. + * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. + * The same dictionary will have to be loaded on decompression side for successful decoding. + * Dictionary are useful for better compression of small data (KB range). + * While LZ4 accept any input as dictionary, + * results are generally better when using Zstandard's Dictionary Builder. * Loading a size of 0 is allowed, and is the same as reset. - * @return : dictionary size, in bytes (necessarily <= 64 KB) + * @return : loaded dictionary size, in bytes (necessarily <= 64 KB) */ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); /*! LZ4_compress_fast_continue() : * Compress 'src' content using data from previously compressed blocks, for better compression ratio. - * 'dst' buffer must be already allocated. + * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. * * @return : size of compressed block @@ -288,10 +289,10 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in * * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. - * Each block must be decompressed separately, calling LZ4_decompress_*() with associated metadata. * - * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory! + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! * * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. * Make sure that buffers are separated, by at least one byte. @@ -299,7 +300,7 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in * * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. * - * Note 5 : After an error, the stream status is invalid, it can only be reset or freed. + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -335,7 +336,7 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); -/*! LZ4_decoderRingBufferSize() : v1.8.2 +/*! LZ4_decoderRingBufferSize() : v1.8.2+ * Note : in a ring buffer scenario (optional), * blocks are presumed decompressed next to each other * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), @@ -347,7 +348,7 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const * or 0 if there is an error (invalid maxBlockSize). */ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); -#define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs)) /* for static allocation; mbs presumed valid */ +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ /*! LZ4_decompress_*_continue() : * These decoding functions allow decompression of consecutive blocks in "streaming" mode. @@ -375,83 +376,67 @@ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. */ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); -LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); /*! LZ4_decompress_*_usingDict() : * These decoding functions work the same as * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() * They are stand-alone, and don't need an LZ4_streamDecode_t structure. - * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. */ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); -/*^********************************************** +/*^************************************* * !!!!!! STATIC LINKING ONLY !!!!!! - ***********************************************/ + ***************************************/ -/*-************************************ - * Unstable declarations - ************************************** - * Declarations in this section should be considered unstable. - * Use at your own peril, etc., etc. - * They may be removed in the future. - * Their signatures may change. - **************************************/ +/*-**************************************************************************** + * Experimental section + * + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, + * the definitions are hidden by default + * when building LZ4 as a shared/dynamic library. + * + * In order to access these declarations, + * define LZ4_STATIC_LINKING_ONLY in your application + * before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +#define LZ4LIB_STATIC_API LZ4LIB_API +#else +#define LZ4LIB_STATIC_API +#endif #ifdef LZ4_STATIC_LINKING_ONLY -/*! LZ4_resetStream_fast() : - * Use this, like LZ4_resetStream(), to prepare a context for a new chain of - * calls to a streaming API (e.g., LZ4_compress_fast_continue()). - * - * Note: - * Using this in advance of a non- streaming-compression function is redundant, - * and potentially bad for performance, since they all perform their own custom - * reset internally. - * - * Differences from LZ4_resetStream(): - * When an LZ4_stream_t is known to be in a internally coherent state, - * it can often be prepared for a new compression with almost no work, only - * sometimes falling back to the full, expensive reset that is always required - * when the stream is in an indeterminate state (i.e., the reset performed by - * LZ4_resetStream()). - * - * LZ4_streams are guaranteed to be in a valid state when: - * - returned from LZ4_createStream() - * - reset by LZ4_resetStream() - * - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged - * - the stream was in a valid state and was reset by LZ4_resetStream_fast() - * - the stream was in a valid state and was then used in any compression call - * that returned success - * - the stream was in an indeterminate state and was used in a compression - * call that fully reset the state (e.g., LZ4_compress_fast_extState()) and - * that returned success - * - * When a stream isn't known to be in a valid state, it is not safe to pass to - * any fastReset or streaming function. It must first be cleansed by the full - * LZ4_resetStream(). - */ -LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); /*! LZ4_compress_fast_extState_fastReset() : * A variant of LZ4_compress_fast_extState(). * - * Using this variant avoids an expensive initialization step. It is only safe - * to call if the state buffer is known to be correctly initialized already - * (see above comment on LZ4_resetStream_fast() for a definition of "correctly - * initialized"). From a high level, the difference is that this function - * initializes the provided state with a call to something like - * LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a - * call to LZ4_resetStream(). + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). */ -LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_attach_dictionary() : - * This is an experimental API that allows for the efficient use of a - * static dictionary many times. + * This is an experimental API that allows + * efficient use of a static dictionary many times. * * Rather than re-loading the dictionary buffer into a working context before * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a @@ -462,8 +447,8 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr * Currently, only streams which have been prepared by LZ4_loadDict() should * be expected to work. * - * Alternatively, the provided dictionary stream pointer may be NULL, in which - * case any existing dictionary stream is unset. + * Alternatively, the provided dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. * * If a dictionary is provided, it replaces any pre-existing stream history. * The dictionary contents are the only history that can be referenced and @@ -475,17 +460,18 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr * stream (and source buffer) must remain in-place / accessible / unchanged * through the completion of the first compression call on the stream. */ -LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream); +LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); #endif -/*-************************************ - * Private definitions - ************************************** - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Using these definitions will expose code to API and/or ABI break in future versions of the library. - **************************************/ + +/*-************************************************************ + * PRIVATE DEFINITIONS + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose code to API and/or ABI break in future versions of the library. + **************************************************************/ #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ @@ -497,7 +483,7 @@ typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; struct LZ4_stream_t_internal { uint32_t hashTable[LZ4_HASH_SIZE_U32]; uint32_t currentOffset; - uint16_t initCheck; + uint16_t dirty; uint16_t tableType; const uint8_t* dictionary; const LZ4_stream_t_internal* dictCtx; @@ -517,7 +503,7 @@ typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; struct LZ4_stream_t_internal { unsigned int hashTable[LZ4_HASH_SIZE_U32]; unsigned int currentOffset; - unsigned short initCheck; + unsigned short dirty; unsigned short tableType; const unsigned char* dictionary; const LZ4_stream_t_internal* dictCtx; @@ -526,38 +512,54 @@ struct LZ4_stream_t_internal { typedef struct { const unsigned char* externalDict; - size_t extDictSize; const unsigned char* prefixEnd; + size_t extDictSize; size_t prefixSize; } LZ4_streamDecode_t_internal; #endif -/*! - * LZ4_stream_t : - * information structure to track an LZ4 stream. - * init this structure before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * it may change in a future version ! +/*! LZ4_stream_t : + * information structure to track an LZ4 stream. + * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended. + * The structure definition can be convenient for static allocation + * (on stack, or as part of larger structure). + * Init this structure with LZ4_initStream() before first use. + * note : only use this definition in association with static linking ! + * this definition is not API/ABI safe, and may change in a future version. */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ ) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) union LZ4_stream_u { unsigned long long table[LZ4_STREAMSIZE_U64]; LZ4_stream_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_stream_t */ +/*! LZ4_initStream() : v1.9.0+ + * An LZ4_stream_t structure must be initialized at least once. + * This is automatically done when invoking LZ4_createStream(), + * but it's not when the structure is simply declared on stack (for example). + * + * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. + * It can also initialize any arbitrary buffer of sufficient size, + * and will @return a pointer of proper type upon initialization. + * + * Note : initialization fails if size and alignment conditions are not respected. + * In which case, the function will @return NULL. + * Note2: An LZ4_stream_t structure guarantees correct alignment and size. + * Note3: Before v1.9.0, use LZ4_resetStream() instead + */ +LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size); -/*! - * LZ4_streamDecode_t : - * information structure to track an LZ4 stream during decompression. - * init this structure using LZ4_setStreamDecode (or memset()) before first use - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! + +/*! LZ4_streamDecode_t : + * information structure to track an LZ4 stream during decompression. + * init this structure using LZ4_setStreamDecode() before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! */ -#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ ) #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) union LZ4_streamDecode_u { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; @@ -570,11 +572,16 @@ union LZ4_streamDecode_u { **************************************/ /*! Deprecation warnings - Should deprecation warnings be a problem, - it is generally possible to disable them, - typically with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual. - Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ + * + * Deprecated functions make the compiler generate a warning when invoked. + * This is meant to invite users to update their source code. + * Should deprecation warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * + * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS + * before including the header file. + */ #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else @@ -594,8 +601,8 @@ union LZ4_streamDecode_u { #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); @@ -616,13 +623,57 @@ LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompres */ LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); -LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); /* Obsolete streaming decoding functions */ LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); +/*! LZ4_decompress_fast() : **unsafe!** + * These functions used to be faster than LZ4_decompress_safe(), + * but it has changed, and they are now slower than LZ4_decompress_safe(). + * This is because LZ4_decompress_fast() doesn't know the input size, + * and therefore must progress more cautiously in the input buffer to not read beyond the end of block. + * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. + * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. + * + * The last remaining LZ4_decompress_fast() specificity is that + * it can decompress a block without knowing its compressed size. + * Such functionality could be achieved in a more secure manner, + * by also providing the maximum size of input buffer, + * but it would require new prototypes, and adaptation of the implementation to this new use case. + * + * Parameters: + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * The function expects to finish at block's end exactly. + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. + * Also, since match offsets are not validated, match reads from 'src' may underflow too. + * These issues never happen if input (compressed) data is correct. + * But they may happen if input data is invalid (error or intentional tampering). + * As a consequence, use these functions in trusted environments with trusted data **only**. + */ + +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); + +/*! LZ4_resetStream() : + * An LZ4_stream_t structure must be initialized at least once. + * This is done with LZ4_initStream(), or LZ4_resetStream(). + * Consider switching to LZ4_initStream(), + * invoking LZ4_resetStream() will trigger deprecation warnings in the future. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + + #endif /* LZ4_H_2983827168210 */ diff --git a/lz4libs/lz4frame.c b/lz4libs/lz4frame.c index 08bf0fae..a10e4af0 100644 --- a/lz4libs/lz4frame.c +++ b/lz4libs/lz4frame.c @@ -1,41 +1,44 @@ /* -LZ4 auto-framing library -Copyright (C) 2011-2016, Yann Collet. - -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- LZ4 homepage : http://www.lz4.org -- LZ4 source repository : https://github.com/lz4/lz4 -*/ + * LZ4 auto-framing library + * Copyright (C) 2011-2016, Yann Collet. + * + * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at : + * - LZ4 homepage : http://www.lz4.org + * - LZ4 source repository : https://github.com/lz4/lz4 + */ /* LZ4F is a stand-alone API to create LZ4-compressed Frames -* in full conformance with specification v1.5.0 -* All related operations, including memory management, are handled by the library. -* */ + * in full conformance with specification v1.6.1 . + * This library rely upon memory management capabilities (malloc, free) + * provided either by , + * or redirected towards another library of user's choice + * (see Memory Routines below). + */ /*-************************************ @@ -62,16 +65,27 @@ You can contact the author at : /*-************************************ * Memory routines **************************************/ +/* + * User may redirect invocations of + * malloc(), calloc() and free() + * towards another library or solution of their choice + * by modifying below section. + */ #include /* malloc, calloc, free */ -#define ALLOC(s) malloc(s) -#define ALLOC_AND_ZERO(s) calloc(1,s) -#define FREEMEM free +#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,(s)) +# define FREEMEM(p) free(p) +#endif + #include /* memset, memcpy, memmove */ -#define MEM_INIT memset +#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# define MEM_INIT(p,v,s) memset((p),(v),(s)) +#endif /*-************************************ -* Includes +* Library declarations **************************************/ #define LZ4F_STATIC_LINKING_ONLY #include "lz4frame.h" @@ -134,8 +148,8 @@ static U32 LZ4F_readLE32 (const void* src) { const BYTE* const srcPtr = (const BYTE*)src; U32 value32 = srcPtr[0]; - value32 += (srcPtr[1]<<8); - value32 += (srcPtr[2]<<16); + value32 += ((U32)srcPtr[1])<< 8; + value32 += ((U32)srcPtr[2])<<16; value32 += ((U32)srcPtr[3])<<24; return value32; } @@ -180,9 +194,11 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) /*-************************************ * Constants **************************************/ -#define KB *(1<<10) -#define MB *(1<<20) -#define GB *(1<<30) +#ifndef LZ4_SRC_INCLUDED /* avoid double definition */ +# define KB *(1<<10) +# define MB *(1<<20) +# define GB *(1<<30) +#endif #define _1BIT 0x01 #define _2BITS 0x03 @@ -195,9 +211,10 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB -static const size_t minFHSize = 7; +static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */ static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ -static const size_t BHSize = 4; +static const size_t BHSize = 4; /* block header : size, and compress flag */ +static const size_t BFSize = 4; /* block footer : checksum (optional) */ /*-************************************ @@ -258,22 +275,22 @@ unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } - -/*-************************************ -* Private functions -**************************************/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -static size_t LZ4F_getBlockSize(unsigned blockSizeID) +size_t LZ4F_getBlockSize(unsigned blockSizeID) { static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; - blockSizeID -= 4; - if (blockSizeID > 3) return err0r(LZ4F_ERROR_maxBlockSize_invalid); + if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB) + return err0r(LZ4F_ERROR_maxBlockSize_invalid); + blockSizeID -= LZ4F_max64KB; return blockSizes[blockSizeID]; } +/*-************************************ +* Private functions +**************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + static BYTE LZ4F_headerChecksum (const void* header, size_t length) { U32 const xxh = XXH32(header, length, 0); @@ -308,8 +325,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) { - LZ4F_preferences_t prefsNull; - MEM_INIT(&prefsNull, 0, sizeof(prefsNull)); + LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES; prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; U32 const flush = prefsPtr->autoFlush | (srcSize==0); @@ -323,11 +339,10 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, size_t const lastBlockSize = flush ? partialBlockSize : 0; unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); - size_t const blockHeaderSize = 4; - size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag; - size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + size_t const blockCRCSize = BFSize * prefsPtr->frameInfo.blockChecksumFlag; + size_t const frameEnd = BHSize + (prefsPtr->frameInfo.contentChecksumFlag*BFSize); - return ((blockHeaderSize + blockCRCSize) * nbBlocks) + + return ((BHSize + blockCRCSize) * nbBlocks) + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd; } } @@ -388,15 +403,18 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, if (LZ4F_isError(headerSize)) return headerSize; dstPtr += headerSize; /* header size */ } - { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); + assert(dstEnd >= dstPtr); + { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options); if (LZ4F_isError(cSize)) return cSize; dstPtr += cSize; } - { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ + assert(dstEnd >= dstPtr); + { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */ if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - return (dstPtr - dstStart); + assert(dstEnd >= dstStart); + return (size_t)(dstPtr - dstStart); } @@ -428,7 +446,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, if (preferencesPtr == NULL || preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN) { - LZ4_resetStream(&lz4ctx); + LZ4_initStream(&lz4ctx, sizeof(lz4ctx)); cctxPtr->lz4CtxPtr = &lz4ctx; cctxPtr->lz4CtxAlloc = 1; cctxPtr->lz4CtxState = 1; @@ -598,20 +616,22 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, if (cctxPtr->lz4CtxAlloc < ctxTypeID) { FREEMEM(cctxPtr->lz4CtxPtr); if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); + cctxPtr->lz4CtxPtr = LZ4_createStream(); } else { - cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); + cctxPtr->lz4CtxPtr = LZ4_createStreamHC(); } - if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed); + if (cctxPtr->lz4CtxPtr == NULL) + return err0r(LZ4F_ERROR_allocation_failed); cctxPtr->lz4CtxAlloc = ctxTypeID; cctxPtr->lz4CtxState = ctxTypeID; } else if (cctxPtr->lz4CtxState != ctxTypeID) { /* otherwise, a sufficient buffer is allocated, but we need to * reset it to the correct context type */ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr); + LZ4_initStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr, sizeof (LZ4_stream_t)); } else { - LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + LZ4_initStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t)); + LZ4_setCompressionLevel((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); } cctxPtr->lz4CtxState = ctxTypeID; } @@ -623,8 +643,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); { size_t const requiredBuffSize = preferencesPtr->autoFlush ? - (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */ - cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 64 KB : 0) : /* only needs past data up to window size */ + cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 128 KB : 0); if (cctxPtr->maxBufferSize < requiredBuffSize) { cctxPtr->maxBufferSize = 0; @@ -635,7 +655,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, } } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; - XXH32_reset(&(cctxPtr->xxh), 0); + (void)XXH32_reset(&(cctxPtr->xxh), 0); /* context init */ cctxPtr->cdict = cdict; @@ -644,7 +664,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked); } if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) { - LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); + LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); } /* Magic Number */ @@ -656,7 +676,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) - + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3) + + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3) + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) + (cctxPtr->prefs.frameInfo.dictID > 0) ); /* BD Byte */ @@ -673,11 +693,11 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, dstPtr += 4; } /* Header CRC Byte */ - *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); + *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart)); dstPtr++; cctxPtr->cStage = 1; /* header written, now request input data block */ - return (dstPtr - dstStart); + return (size_t)(dstPtr - dstStart); } @@ -686,7 +706,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. * preferencesPtr can be NULL, in which case default parameters are selected. * @return : number of bytes written into dstBuffer for the header - * or an error code (can be tested using LZ4F_isError()) + * or an error code (can be tested using LZ4F_isError()) */ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, @@ -712,27 +732,31 @@ typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize /*! LZ4F_makeBlock(): - * compress a single block, add header and checksum - * assumption : dst buffer capacity is >= srcSize */ -static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, + * compress a single block, add header and optional checksum. + * assumption : dst buffer capacity is >= BHSize + srcSize + crcSize + */ +static size_t LZ4F_makeBlock(void* dst, + const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level, - const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag) + const LZ4F_CDict* cdict, + LZ4F_blockChecksum_t crcFlag) { BYTE* const cSizePtr = (BYTE*)dst; - U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), + U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), (int)(srcSize), (int)(srcSize-1), level, cdict); - LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) { /* compression failed */ cSize = (U32)srcSize; LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); - memcpy(cSizePtr+4, src, srcSize); + memcpy(cSizePtr+BHSize, src, srcSize); + } else { + LZ4F_writeLE32(cSizePtr, cSize); } if (crcFlag) { - U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */ - LZ4F_writeLE32(cSizePtr+4+cSize, crc32); + U32 const crc32 = XXH32(cSizePtr+BHSize, cSize, 0); /* checksum of compressed data */ + LZ4F_writeLE32(cSizePtr+BHSize+cSize, crc32); } - return 4 + cSize + ((U32)crcFlag)*4; + return BHSize + cSize + ((U32)crcFlag)*BFSize; } @@ -832,9 +856,11 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; - dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, + dstPtr += LZ4F_makeBlock(dstPtr, + cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; @@ -844,18 +870,22 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, while ((size_t)(srcEnd - srcPtr) >= blockSize) { /* compress full blocks */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, + dstPtr += LZ4F_makeBlock(dstPtr, + srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); srcPtr += blockSize; } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { /* compress remaining input < blockSize */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, + dstPtr += LZ4F_makeBlock(dstPtr, + srcPtr, (size_t)(srcEnd - srcPtr), compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); srcPtr = srcEnd; } @@ -881,28 +911,30 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, /* some input data left, necessarily < blockSize */ if (srcPtr < srcEnd) { /* fill tmp buffer */ - size_t const sizeToCopy = srcEnd - srcPtr; + size_t const sizeToCopy = (size_t)(srcEnd - srcPtr); memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); cctxPtr->tmpInSize = sizeToCopy; } if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) - XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); cctxPtr->totalInSize += srcSize; - return dstPtr - dstStart; + return (size_t)(dstPtr - dstStart); } /*! LZ4F_flush() : - * Should you need to create compressed data immediately, without waiting for a block to be filled, - * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. - * The result of the function is the number of bytes written into dstBuffer - * (it can be zero, this means there was no data left within compressionContext) + * When compressed data must be sent immediately, without waiting for a block to be filled, + * invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx. + * The result of the function is the number of bytes written into dstBuffer. + * It can be zero, this means there was no data left within LZ4F_cctx. * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * LZ4F_compressOptions_t* is optional. NULL is a valid argument. */ -size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_flush(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -910,52 +942,65 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +4 : block header(4) */ + if (dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize)) + return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); /* compress tmp buffer */ - dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, + dstPtr += LZ4F_makeBlock(dstPtr, + cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); - if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity)); + + if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) + cctxPtr->tmpIn += cctxPtr->tmpInSize; cctxPtr->tmpInSize = 0; /* keep tmpIn within limits */ if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */ - int realDictSize = LZ4F_localSaveDict(cctxPtr); + int const realDictSize = LZ4F_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } - return dstPtr - dstStart; + return (size_t)(dstPtr - dstStart); } /*! LZ4F_compressEnd() : - * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). - * It will flush whatever data remained within compressionContext (like LZ4_flush()) - * but also properly finalize the frame, with an endMark and a checksum. - * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. + * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ4_flush()) + * but also properly finalize the frame, with an endMark and an (optional) checksum. + * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * @return: the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) + * or an error code if it fails (can be tested using LZ4F_isError()) + * The context can then be used again to compress a new frame, starting with LZ4F_compressBegin(). */ -size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; - size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr); + size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); if (LZ4F_isError(flushSize)) return flushSize; dstPtr += flushSize; + assert(flushSize <= dstCapacity); + dstCapacity -= flushSize; + + if (dstCapacity < 4) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); LZ4F_writeLE32(dstPtr, 0); - dstPtr+=4; /* endMark */ + dstPtr += 4; /* endMark */ if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) { U32 const xxh = XXH32_digest(&(cctxPtr->xxh)); + if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); LZ4F_writeLE32(dstPtr, xxh); dstPtr+=4; /* content Checksum */ } @@ -968,7 +1013,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, return err0r(LZ4F_ERROR_frameSize_wrong); } - return dstPtr - dstStart; + return (size_t)(dstPtr - dstStart); } @@ -1019,7 +1064,10 @@ struct LZ4F_dctx_s { LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx)); - if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC); + if (dctx == NULL) { /* failed allocation */ + *LZ4F_decompressionContextPtr = NULL; + return err0r(LZ4F_ERROR_allocation_failed); + } dctx->version = versionNumber; *LZ4F_decompressionContextPtr = dctx; @@ -1049,31 +1097,6 @@ void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) } -/*! LZ4F_headerSize() : - * @return : size of frame header - * or an error code, which can be tested using LZ4F_isError() - */ -static size_t LZ4F_headerSize(const void* src, size_t srcSize) -{ - /* minimal srcSize to determine header size */ - if (srcSize < 5) return err0r(LZ4F_ERROR_frameHeader_incomplete); - - /* special case : skippable frames */ - if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8; - - /* control magic number */ - if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) - return err0r(LZ4F_ERROR_frameType_unknown); - - /* Frame Header Size */ - { BYTE const FLG = ((const BYTE*)src)[4]; - U32 const contentSizeFlag = (FLG>>3) & _1BIT; - U32 const dictIDFlag = FLG & _1BIT; - return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); - } -} - - /*! LZ4F_decodeHeader() : * input : `src` points at the **beginning of the frame** * output : set internal values of dctx, such as @@ -1125,7 +1148,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize } /* Frame Header Size */ - frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); + frameHeaderSize = minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0); if (srcSize < frameHeaderSize) { /* not enough input to fully decode frame header */ @@ -1146,6 +1169,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize } /* check header */ + assert(frameHeaderSize > 5); { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); @@ -1169,6 +1193,34 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize } +/*! LZ4F_headerSize() : + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + */ +size_t LZ4F_headerSize(const void* src, size_t srcSize) +{ + if (src == NULL) return err0r(LZ4F_ERROR_srcPtr_wrong); + + /* minimal srcSize to determine header size */ + if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH) + return err0r(LZ4F_ERROR_frameHeader_incomplete); + + /* special case : skippable frames */ + if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) + return 8; + + /* control magic number */ + if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) + return err0r(LZ4F_ERROR_frameType_unknown); + + /* Frame Header Size */ + { BYTE const FLG = ((const BYTE*)src)[4]; + U32 const contentSizeFlag = (FLG>>3) & _1BIT; + U32 const dictIDFlag = FLG & _1BIT; + return minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0); + } +} + /*! LZ4F_getFrameInfo() : * This function extracts frame parameters (max blockSize, frame checksum, etc.). * Usage is optional. Objective is to provide relevant information for allocation purposes. @@ -1184,10 +1236,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. */ -LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, - const void* srcBuffer, size_t* srcSizePtr) +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr) { - if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */ + LZ4F_STATIC_ASSERT(dstage_getFrameHeader < dstage_storeFrameHeader); + if (dctx->dStage > dstage_storeFrameHeader) { /* frameInfo already decoded */ size_t o=0, i=0; *srcSizePtr = 0; @@ -1200,7 +1254,6 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP *srcSizePtr = 0; return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted); } else { - size_t decodeResult; size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; } if (*srcSizePtr < hSize) { @@ -1208,16 +1261,16 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP return err0r(LZ4F_ERROR_frameHeader_incomplete); } - decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); - if (LZ4F_isError(decodeResult)) { - *srcSizePtr = 0; - } else { - *srcSizePtr = decodeResult; - decodeResult = BHSize; /* block header size */ - } - *frameInfoPtr = dctx->frameInfo; - return decodeResult; - } } + { size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); + if (LZ4F_isError(decodeResult)) { + *srcSizePtr = 0; + } else { + *srcSizePtr = decodeResult; + decodeResult = BHSize; /* block header size */ + } + *frameInfoPtr = dctx->frameInfo; + return decodeResult; + } } } } @@ -1235,9 +1288,10 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, return; } - if (dstPtr - dstBufferStart + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */ + assert(dstPtr >= dstBufferStart); + if ((size_t)(dstPtr - dstBufferStart) + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */ dctx->dict = (const BYTE*)dstBufferStart; - dctx->dictSize = dstPtr - dstBufferStart + dstSize; + dctx->dictSize = (size_t)(dstPtr - dstBufferStart) + dstSize; return; } @@ -1253,7 +1307,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, } if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ - size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; + size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer); size_t copySize = 64 KB - dctx->tmpOutSize; const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; if (dctx->tmpOutSize > 64 KB) copySize = 0; @@ -1338,7 +1392,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, case dstage_getFrameHeader: if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ - size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */ + size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */ if (LZ4F_isError(hSize)) return hSize; srcPtr += hSize; break; @@ -1366,14 +1420,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; case dstage_init: - if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0); + if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0); /* internal buffers allocation */ { size_t const bufferNeeded = dctx->maxBlockSize - + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); + + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0); if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ FREEMEM(dctx->tmpIn); - dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + 4 /* block checksum */); + dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + BFSize /* block checksum */); if (dctx->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed); FREEMEM(dctx->tmpOutBuffer); @@ -1420,7 +1474,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* decode block header */ { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; - size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4; + size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize; if (nextCBlockSize==0) { /* frameEnd signal, no more block */ dctx->dStage = dstage_getSuffix; break; @@ -1431,7 +1485,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* next block is uncompressed */ dctx->tmpInTarget = nextCBlockSize; if (dctx->frameInfo.blockChecksumFlag) { - XXH32_reset(&dctx->blockChecksum, 0); + (void)XXH32_reset(&dctx->blockChecksum, 0); } dctx->dStage = dstage_copyDirect; break; @@ -1440,7 +1494,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, dctx->tmpInTarget = nextCBlockSize + crcSize; dctx->dStage = dstage_getCBlock; if (dstPtr==dstEnd) { - nextSrcSizeHint = nextCBlockSize + crcSize + BHSize; + nextSrcSizeHint = BHSize + nextCBlockSize + crcSize; doAnotherStage = 0; } break; @@ -1451,10 +1505,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); memcpy(dstPtr, srcPtr, sizeToCopy); if (dctx->frameInfo.blockChecksumFlag) { - XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); } if (dctx->frameInfo.contentChecksumFlag) - XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); + (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); if (dctx->frameInfo.contentSize) dctx->frameRemainingSize -= sizeToCopy; @@ -1474,7 +1528,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } dctx->tmpInTarget -= sizeToCopy; /* need to copy more */ nextSrcSizeHint = dctx->tmpInTarget + - + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */ + +(dctx->frameInfo.blockChecksumFlag ? BFSize : 0) + BHSize /* next header size */; doAnotherStage = 0; break; @@ -1525,8 +1579,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, dctx->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */ - nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; - doAnotherStage=0; + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + + (dctx->frameInfo.blockChecksumFlag ? BFSize : 0) + + BHSize /* next header size */; + doAnotherStage = 0; break; } selectedIn = dctx->tmpIn; @@ -1558,13 +1614,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, dict, (int)dictSize); if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */ if (dctx->frameInfo.contentChecksumFlag) - XXH32_update(&(dctx->xxh), dstPtr, decodedSize); + XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize); if (dctx->frameInfo.contentSize) - dctx->frameRemainingSize -= decodedSize; + dctx->frameRemainingSize -= (size_t)decodedSize; /* dictionary management */ if (dctx->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0); + LZ4F_updateDict(dctx, dstPtr, (size_t)decodedSize, dstStart, 0); dstPtr += decodedSize; dctx->dStage = dstage_getBlockHeader; @@ -1601,10 +1657,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, if (decodedSize < 0) /* decompression failed */ return err0r(LZ4F_ERROR_decompressionFailed); if (dctx->frameInfo.contentChecksumFlag) - XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize); + XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize); if (dctx->frameInfo.contentSize) - dctx->frameRemainingSize -= decodedSize; - dctx->tmpOutSize = decodedSize; + dctx->frameRemainingSize -= (size_t)decodedSize; + dctx->tmpOutSize = (size_t)decodedSize; dctx->tmpOutStart = 0; dctx->dStage = dstage_flushOut; } @@ -1732,7 +1788,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */ { if (dctx->dStage == dstage_flushOut) { - size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; + size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer); size_t copySize = 64 KB - dctx->tmpOutSize; const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; if (dctx->tmpOutSize > 64 KB) copySize = 0; @@ -1756,8 +1812,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } } - *srcSizePtr = (srcPtr - srcStart); - *dstSizePtr = (dstPtr - dstStart); + *srcSizePtr = (size_t)(srcPtr - srcStart); + *dstSizePtr = (size_t)(dstPtr - dstStart); return nextSrcSizeHint; } diff --git a/lz4libs/lz4frame.h b/lz4libs/lz4frame.h index 75f1fd91..742c2528 100644 --- a/lz4libs/lz4frame.h +++ b/lz4libs/lz4frame.h @@ -32,11 +32,14 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -/* LZ4F is a stand-alone API to create LZ4-compressed frames - * conformant with specification v1.6.1. - * It also offers streaming capabilities. +/* LZ4F is a stand-alone API able to create and decode LZ4 frames + * conformant with specification v1.6.1 in doc/lz4_Frame_format.md . + * Generated frames are compatible with `lz4` CLI. + * + * LZ4F also offers streaming capabilities. + * * lz4.h is not required when using lz4frame.h, - * except to get constant such as LZ4_VERSION_NUMBER. + * except to extract common constant such as LZ4_VERSION_NUMBER. * */ #ifndef LZ4F_H_09782039843 @@ -173,7 +176,7 @@ typedef struct { LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */ } LZ4F_frameInfo_t; -#define LZ4F_INIT_FRAMEINFO { 0, 0, 0, 0, 0, 0, 0 } /* v1.8.3+ */ +#define LZ4F_INIT_FRAMEINFO { LZ4F_default, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0ULL, 0U, LZ4F_noBlockChecksum } /* v1.8.3+ */ /*! LZ4F_preferences_t : * makes it possible to supply advanced compression instructions to streaming interface. @@ -188,14 +191,14 @@ typedef struct { unsigned reserved[3]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; -#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0, 0, { 0, 0, 0 } } /* v1.8.3+ */ +#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0u, 0u, { 0u, 0u, 0u } } /* v1.8.3+ */ /*-********************************* * Simple compression function ***********************************/ -LZ4FLIB_API int LZ4F_compressionLevel_max(void); +LZ4FLIB_API int LZ4F_compressionLevel_max(void); /* v1.8.0+ */ /*! LZ4F_compressFrameBound() : * Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences. @@ -247,7 +250,9 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /*---- Compression ----*/ -#define LZ4F_HEADER_SIZE_MAX 19 /* LZ4 Frame header size can vary from 7 to 19 bytes */ +#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */ +#define LZ4F_HEADER_SIZE_MAX 19 + /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. @@ -260,15 +265,19 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, const LZ4F_preferences_t* prefsPtr); /*! LZ4F_compressBound() : - * Provides minimum dstCapacity required to guarantee compression success - * given a srcSize and preferences, covering worst case scenario. + * Provides minimum dstCapacity required to guarantee success of + * LZ4F_compressUpdate(), given a srcSize and preferences, for a worst case scenario. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() instead. + * Note that the result is only valid for a single invocation of LZ4F_compressUpdate(). + * When invoking LZ4F_compressUpdate() multiple times, + * if the output buffer is gradually filled up instead of emptied and re-used from its start, + * one must check if there is enough remaining capacity before each invocation, using LZ4F_compressBound(). + * @return is always the same for a srcSize and prefsPtr. * prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. - * Estimation is valid for either LZ4F_compressUpdate(), LZ4F_flush() or LZ4F_compressEnd(), - * Estimation includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. - * It also includes frame footer (ending + checksum), which would have to be generated by LZ4F_compressEnd(). - * Estimation doesn't include frame header, as it was already generated by LZ4F_compressBegin(). - * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. - * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + * tech details : + * @return includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. + * It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd(). + * @return doesn't include frame header, as it was already generated by LZ4F_compressBegin(). */ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); @@ -295,6 +304,7 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx) * or an error code if it fails (which can be tested using LZ4F_isError()) + * Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). */ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, @@ -307,6 +317,7 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark), * or an error code if it fails (which can be tested using LZ4F_isError()) + * Note : LZ4F_compressEnd() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. */ LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, @@ -345,23 +356,58 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); * Streaming decompression functions *************************************/ +#define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5 + +/*! LZ4F_headerSize() : v1.9.0+ + * Provide the header size of a frame starting at `src`. + * `srcSize` must be >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH, + * which is enough to decode the header length. + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + * note : Frame header size is variable, but is guaranteed to be + * >= LZ4F_HEADER_SIZE_MIN bytes, and <= LZ4F_HEADER_SIZE_MAX bytes. + */ +size_t LZ4F_headerSize(const void* src, size_t srcSize); + /*! LZ4F_getFrameInfo() : * This function extracts frame parameters (max blockSize, dictID, etc.). - * Its usage is optional. - * Extracted information is typically useful for allocation and dictionary. - * This function works in 2 situations : - * - At the beginning of a new frame, in which case - * it will decode information from `srcBuffer`, starting the decoding process. - * Input size must be large enough to successfully decode the entire frame header. - * Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. - * It's allowed to provide more input data than this minimum. - * - After decoding has been started. - * In which case, no input is read, frame parameters are extracted from dctx. - * - If decoding has barely started, but not yet extracted information from header, + * Its usage is optional: user can call LZ4F_decompress() directly. + * + * Extracted information will fill an existing LZ4F_frameInfo_t structure. + * This can be useful for allocation and dictionary identification purposes. + * + * LZ4F_getFrameInfo() can work in the following situations : + * + * 1) At the beginning of a new frame, before any invocation of LZ4F_decompress(). + * It will decode header from `srcBuffer`, + * consuming the header and starting the decoding process. + * + * Input size must be large enough to contain the full frame header. + * Frame header size can be known beforehand by LZ4F_headerSize(). + * Frame header size is variable, but is guaranteed to be >= LZ4F_HEADER_SIZE_MIN bytes, + * and not more than <= LZ4F_HEADER_SIZE_MAX bytes. + * Hence, blindly providing LZ4F_HEADER_SIZE_MAX bytes or more will always work. + * It's allowed to provide more input data than the header size, + * LZ4F_getFrameInfo() will only consume the header. + * + * If input size is not large enough, + * aka if it's smaller than header size, + * function will fail and return an error code. + * + * 2) After decoding has been started, + * it's possible to invoke LZ4F_getFrameInfo() anytime + * to extract already decoded frame parameters stored within dctx. + * + * Note that, if decoding has barely started, + * and not yet read enough information to decode the header, * LZ4F_getFrameInfo() will fail. - * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * Decompression must resume from (srcBuffer + *srcSizePtr). - * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, + * + * The number of bytes consumed from srcBuffer will be updated in *srcSizePtr (necessarily <= original value). + * LZ4F_getFrameInfo() only consumes bytes when decoding has not yet started, + * and when decoding the header has been successful. + * Decompression must then resume from (srcBuffer + *srcSizePtr). + * + * @return : a hint about how many srcSize bytes LZ4F_decompress() expects for next call, * or an error code which can be tested using LZ4F_isError(). * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. @@ -427,15 +473,15 @@ LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always su extern "C" { #endif -/* These declarations are not stable and may change in the future. They are - * therefore only safe to depend on when the caller is statically linked - * against the library. To access their declarations, define - * LZ4F_STATIC_LINKING_ONLY. +/* These declarations are not stable and may change in the future. + * They are therefore only safe to depend on + * when the caller is statically linked against the library. + * To access their declarations, define LZ4F_STATIC_LINKING_ONLY. * - * There is a further protection mechanism where these symbols aren't published - * into shared/dynamic libraries. You can override this behavior and force - * them to be published by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. Use at - * your own risk. + * By default, these symbols aren't published into shared/dynamic libraries. + * You can override this behavior and force them to be published + * by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. + * Use at your own risk. */ #ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS #define LZ4FLIB_STATIC_API LZ4FLIB_API @@ -471,19 +517,38 @@ extern "C" { #define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, /* enum list is exposed, to handle specific errors */ -typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; +typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) + _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes; LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); - +LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned); /********************************** * Bulk processing dictionary API *********************************/ + +/* A Dictionary is useful for the compression of small messages (KB range). + * It dramatically improves compression efficiency. + * + * LZ4 can ingest any input as dictionary, though only the last 64 KB are useful. + * Best results are generally achieved by using Zstandard's Dictionary Builder + * to generate a high-quality dictionary from a set of samples. + * + * Loading a dictionary has a cost, since it involves construction of tables. + * The Bulk processing dictionary API makes it possible to share this cost + * over an arbitrary number of compression jobs, even concurrently, + * markedly improving compression latency for these cases. + * + * The same dictionary will have to be used on the decompression side + * for decoding to be successful. + * To help identify the correct dictionary at decoding stage, + * the frame header allows optional embedding of a dictID field. + */ typedef struct LZ4F_CDict_s LZ4F_CDict; /*! LZ4_createCDict() : - * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it just once. * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ diff --git a/lz4libs/lz4hc.c b/lz4libs/lz4hc.c index e913ee7b..936f7396 100644 --- a/lz4libs/lz4hc.c +++ b/lz4libs/lz4hc.c @@ -61,9 +61,14 @@ # pragma clang diagnostic ignored "-Wunused-function" #endif +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; + + #define LZ4_COMMONDEFS_ONLY +#ifndef LZ4_SRC_INCLUDED #include "lz4.c" /* LZ4_count, constants, mem */ - +#endif /*=== Constants ===*/ #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) @@ -76,12 +81,11 @@ #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) #define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ +/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */ +#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } -/*=== Enums ===*/ -typedef enum { noDictCtx, usingDictCtx } dictCtx_directive; - /************************************** * HC Compression @@ -92,9 +96,9 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); } -static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) +static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start) { - uptrval startingOffset = hc4->end - hc4->base; + uptrval startingOffset = (uptrval)(hc4->end - hc4->base); if (startingOffset > 1 GB) { LZ4HC_clearTables(hc4); startingOffset = 0; @@ -121,7 +125,7 @@ LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) while (idx < target) { U32 const h = LZ4HC_hashPtr(base+idx); size_t delta = idx - hashTable[h]; - if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; + if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX; DELTANEXTU16(chainTable, idx) = (U16)delta; hashTable[h] = idx; idx++; @@ -224,14 +228,13 @@ LZ4HC_InsertAndGetWiderMatch ( const U32 dictLimit = hc4->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; const U32 ipIndex = (U32)(ip - base); - const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE; + const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; const BYTE* const dictBase = hc4->dictBase; int const lookBackLength = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; - int matchChainPos = 0; + U32 matchChainPos = 0; U32 const pattern = LZ4_read32(ip); U32 matchIndex; - U32 dictMatchIndex; repeat_state_e repeat = rep_untested; size_t srcPatternLength = 0; @@ -256,7 +259,7 @@ LZ4HC_InsertAndGetWiderMatch ( if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) { if (LZ4_read32(matchPtr) == pattern) { int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0; - matchLength = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); + matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); matchLength -= back; if (matchLength > longest) { longest = matchLength; @@ -270,7 +273,7 @@ LZ4HC_InsertAndGetWiderMatch ( int back = 0; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; - matchLength = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit); back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0; @@ -283,14 +286,14 @@ LZ4HC_InsertAndGetWiderMatch ( if (chainSwap && matchLength==longest) { /* better match => select a better chain */ assert(lookBackLength==0); /* search forward only */ - if (matchIndex + longest <= ipIndex) { + if (matchIndex + (U32)longest <= ipIndex) { U32 distanceToNextMatch = 1; int pos; for (pos = 0; pos <= longest - MINMATCH; pos++) { - U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos); + U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); if (candidateDist > distanceToNextMatch) { distanceToNextMatch = candidateDist; - matchChainPos = pos; + matchChainPos = (U32)pos; } } if (distanceToNextMatch > 1) { if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ @@ -315,7 +318,7 @@ LZ4HC_InsertAndGetWiderMatch ( const BYTE* const matchPtr = base + matchCandidateIdx; if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); - const BYTE* const lowestMatchPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE; + const BYTE* const lowestMatchPtr = (lowPrefixPtr + LZ4_DISTANCE_MAX >= ip) ? lowPrefixPtr : ip - LZ4_DISTANCE_MAX; size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); size_t const currentSegmentLength = backLength + forwardPatternLength; @@ -328,7 +331,7 @@ LZ4HC_InsertAndGetWiderMatch ( size_t const maxML = MIN(currentSegmentLength, srcPatternLength); if ((size_t)longest < maxML) { assert(base + matchIndex < ip); - if (ip - (base+matchIndex) > MAX_DISTANCE) break; + if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break; assert(maxML < 2 GB); longest = (int)maxML; *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ @@ -343,16 +346,18 @@ LZ4HC_InsertAndGetWiderMatch ( } } /* PA optimization */ /* follow current chain */ - matchIndex -= DELTANEXTU16(chainTable, matchIndex+matchChainPos); + matchIndex -= DELTANEXTU16(chainTable, matchIndex + matchChainPos); } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */ - if (dict == usingDictCtx && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) { - size_t const dictEndOffset = dictCtx->end - dictCtx->base; + if ( dict == usingDictCtxHc + && nbAttempts + && ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) { + size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->base); + U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; assert(dictEndOffset <= 1 GB); - dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; - while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) { + while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; if (LZ4_read32(matchPtr) == pattern) { @@ -360,22 +365,19 @@ LZ4HC_InsertAndGetWiderMatch ( int back = 0; const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0; mlt -= back; if (mlt > longest) { longest = mlt; *matchpos = base + matchIndex + back; *startpos = ip + back; - } - } + } } { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); dictMatchIndex -= nextOffset; matchIndex -= nextOffset; - } - } - } + } } } return longest; } @@ -395,14 +397,6 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index tabl return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio); } - - -typedef enum { - noLimit = 0, - limitedOutput = 1, - limitedDestSize = 2, -} limitedOutput_directive; - /* LZ4HC_encodeSequence() : * @return : 0 if ok, * 1 if buffer issue detected */ @@ -437,7 +431,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( /* Encode Literal length */ length = (size_t)(*ip - *anchor); - if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ + if ((limit) && ((*op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ if (length >= RUN_MASK) { size_t len = length - RUN_MASK; *token = (RUN_MASK << ML_BITS); @@ -448,17 +442,17 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( } /* Copy Literals */ - LZ4_wildCopy(*op, *anchor, (*op) + length); + LZ4_wildCopy8(*op, *anchor, (*op) + length); *op += length; /* Encode Offset */ - assert( (*ip - match) <= MAX_DISTANCE ); /* note : consider providing offset as a value, rather than as a pointer difference */ + assert( (*ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */ LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; /* Encode MatchLength */ assert(matchLength >= MINMATCH); - length = (size_t)(matchLength - MINMATCH); - if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ + length = (size_t)matchLength - MINMATCH; + if ((limit) && (*op + (length / 255) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ if (length >= ML_MASK) { *token += ML_MASK; length -= ML_MASK; @@ -511,12 +505,12 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( /* init */ *srcSizePtr = 0; - if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* Main Loop */ while (ip <= mflimit) { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict); + ml = LZ4HC_InsertAndFindBestMatch(ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict); if (ml encode ML1 */ optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; continue; } @@ -581,10 +575,10 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( if (start2 < ip+ml) ml = (int)(start2 - ip); /* Now, encode 2 sequences */ optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; ip = start2; optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) goto _dest_overflow; continue; } @@ -603,7 +597,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( } optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; ip = start3; ref = ref3; ml = ml3; @@ -641,7 +635,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( } } optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; /* ML2 becomes ML1 */ ip = start2; ref = ref2; ml = ml2; @@ -658,7 +652,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; size_t const totalSize = 1 + litLength + lastRunSize; - if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ if (limit && (op + totalSize > oend)) { if (limit == limitedOutput) return 0; /* Check output limit */ /* adapt lastRunSize to fill 'dest' */ @@ -685,7 +679,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( return (int) (((char*)op)-dest); _dest_overflow: - if (limit == limitedDestSize) { + if (limit == fillOutput) { op = optr; /* restore correct out pointer */ goto _last_literals; } @@ -735,56 +729,64 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ }; - DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr); + DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d)", ctx, src, *srcSizePtr); - if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ + if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ ctx->end += *srcSizePtr; if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */ cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); { cParams_t const cParam = clTable[cLevel]; HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio; - if (cParam.strat == lz4hc) - return LZ4HC_compress_hashChain(ctx, + int result; + + if (cParam.strat == lz4hc) { + result = LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, cParam.nbSearches, limit, dict); - assert(cParam.strat == lz4opt); - return LZ4HC_compress_optimal(ctx, - src, dst, srcSizePtr, dstCapacity, - cParam.nbSearches, cParam.targetLength, limit, - cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ - dict, favor); + } else { + assert(cParam.strat == lz4opt); + result = LZ4HC_compress_optimal(ctx, + src, dst, srcSizePtr, dstCapacity, + (int)cParam.nbSearches, cParam.targetLength, limit, + cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ + dict, favor); + } + if (result <= 0) ctx->dirty = 1; + return result; } } static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock); -static int LZ4HC_compress_generic_noDictCtx ( - LZ4HC_CCtx_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - int const dstCapacity, - int cLevel, - limitedOutput_directive limit - ) +static int +LZ4HC_compress_generic_noDictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) { assert(ctx->dictCtx == NULL); return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx); } -static int LZ4HC_compress_generic_dictCtx ( - LZ4HC_CCtx_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - int const dstCapacity, - int cLevel, - limitedOutput_directive limit - ) +static int +LZ4HC_compress_generic_dictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) { - const size_t position = ctx->end - ctx->base - ctx->lowLimit; + const size_t position = (size_t)(ctx->end - ctx->base) - ctx->lowLimit; assert(ctx->dictCtx != NULL); if (position >= 64 KB) { ctx->dictCtx = NULL; @@ -795,19 +797,20 @@ static int LZ4HC_compress_generic_dictCtx ( ctx->compressionLevel = (short)cLevel; return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); } else { - return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtxHc); } } -static int LZ4HC_compress_generic ( - LZ4HC_CCtx_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - int const dstCapacity, - int cLevel, - limitedOutput_directive limit - ) +static int +LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) { if (ctx->dictCtx == NULL) { return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); @@ -817,24 +820,41 @@ static int LZ4HC_compress_generic ( } -int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } +int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); } +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + * it reports an aligment of 8-bytes, + * while actually aligning LZ4_streamHC_t on 4 bytes. */ +static size_t LZ4_streamHC_t_alignment(void) +{ + struct { char c; LZ4_streamHC_t t; } t_a; + return sizeof(t_a) - sizeof(t_a.t); +} +#endif + +/* state is presumed correctly initialized, + * in which case its size and alignment have already been validate */ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + * it reports an aligment of 8-bytes, + * while actually aligning LZ4_streamHC_t on 4 bytes. */ + assert(((size_t)state & (LZ4_streamHC_t_alignment() - 1)) == 0); /* check alignment */ +#endif if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel); - LZ4HC_init (ctx, (const BYTE*)src); + LZ4HC_init_internal (ctx, (const BYTE*)src); if (dstCapacity < LZ4_compressBound(srcSize)) return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); else - return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit); + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, notLimited); } int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4_resetStreamHC ((LZ4_streamHC_t*)state, compressionLevel); + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel); } @@ -848,19 +868,19 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in #endif int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 - free(statePtr); + FREEMEM(statePtr); #endif return cSize; } -/* LZ4_compress_HC_destSize() : - * only compatible with regular HC parser */ -int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) +/* state is presumed sized correctly (>= sizeof(LZ4_streamHC_t)) */ +int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) { - LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; - LZ4_resetStreamHC((LZ4_streamHC_t*)LZ4HC_Data, cLevel); - LZ4HC_init(ctx, (const BYTE*) source); - return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize); + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ + LZ4HC_init_internal(&ctx->internal_donotuse, (const BYTE*) source); + LZ4_setCompressionLevel(ctx, cLevel); + return LZ4HC_compress_generic(&ctx->internal_donotuse, source, dest, sourceSizePtr, targetDestSize, cLevel, fillOutput); } @@ -869,44 +889,70 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i * Streaming Functions **************************************/ /* allocation */ -LZ4_streamHC_t* LZ4_createStreamHC(void) { +LZ4_streamHC_t* LZ4_createStreamHC(void) +{ LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); if (LZ4_streamHCPtr==NULL) return NULL; - LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); /* full initialization, malloc'ed buffer can be full of garbage */ return LZ4_streamHCPtr; } -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) +{ DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr); if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ - free(LZ4_streamHCPtr); + FREEMEM(LZ4_streamHCPtr); return 0; } -/* initialization */ -void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) { - LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ - DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel); + LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer; + if (buffer == NULL) return NULL; + if (size < sizeof(LZ4_streamHC_t)) return NULL; +#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : + * it reports an aligment of 8-bytes, + * while actually aligning LZ4_streamHC_t on 4 bytes. */ + if (((size_t)buffer) & (LZ4_streamHC_t_alignment() - 1)) return NULL; /* alignment check */ +#endif + /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ + LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE); + DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", LZ4_streamHCPtr, (unsigned)size); + /* end-base will trigger a clearTable on starting compression */ LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0; + LZ4_streamHCPtr->internal_donotuse.dirty = 0; + LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); + return LZ4_streamHCPtr; +} + +/* just a stub */ +void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); - LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; - LZ4_streamHCPtr->internal_donotuse.base = NULL; - LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + if (LZ4_streamHCPtr->internal_donotuse.dirty) { + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + } else { + /* preserve end - base : can trigger clearTable's threshold */ + LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + } LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { + DEBUGLOG(5, "LZ4_setCompressionLevel(%p, %d)", LZ4_streamHCPtr, compressionLevel); if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel; @@ -917,16 +963,24 @@ void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor) LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0); } -int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) +/* LZ4_loadDictHC() : + * LZ4_streamHCPtr is presumed properly initialized */ +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize); + assert(LZ4_streamHCPtr != NULL); if (dictSize > 64 KB) { - dictionary += dictSize - 64 KB; + dictionary += (size_t)dictSize - 64 KB; dictSize = 64 KB; } - LZ4_resetStreamHC(LZ4_streamHCPtr, ctxPtr->compressionLevel); - LZ4HC_init (ctxPtr, (const BYTE*)dictionary); + /* need a full initialization, there are bad side-effects when using resetFast() */ + { int const cLevel = ctxPtr->compressionLevel; + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_setCompressionLevel(LZ4_streamHCPtr, cLevel); + } + LZ4HC_init_internal (ctxPtr, (const BYTE*)dictionary); ctxPtr->end = (const BYTE*)dictionary + dictSize; if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; @@ -959,9 +1013,11 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, limitedOutput_directive limit) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - DEBUGLOG(4, "LZ4_compressHC_continue_generic(%p, %p, %d)", LZ4_streamHCPtr, src, *srcSizePtr); + DEBUGLOG(4, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d)", + LZ4_streamHCPtr, src, *srcSizePtr); + assert(ctxPtr != NULL); /* auto-init if forgotten */ - if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src); + if (ctxPtr->base == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src); /* Check overflow */ if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { @@ -971,7 +1027,8 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, } /* Check if blocks follow each other */ - if ((const BYTE*)src != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src); + if ((const BYTE*)src != ctxPtr->end) + LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src); /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr; @@ -992,12 +1049,12 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, if (dstCapacity < LZ4_compressBound(srcSize)) return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput); else - return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, noLimit); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, notLimited); } int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize) { - return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize); + return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput); } @@ -1016,19 +1073,21 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base); streamPtr->end = (const BYTE*)safeBuffer + dictSize; streamPtr->base = streamPtr->end - endIndex; - streamPtr->dictLimit = endIndex - dictSize; - streamPtr->lowLimit = endIndex - dictSize; + streamPtr->dictLimit = endIndex - (U32)dictSize; + streamPtr->lowLimit = endIndex - (U32)dictSize; if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; } return dictSize; } -/*********************************** +/*************************************************** * Deprecated Functions -***********************************/ +***************************************************/ + /* These functions currently generate deprecation warnings */ -/* Deprecated compression functions */ + +/* Wrappers for deprecated compression functions */ int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } @@ -1044,25 +1103,26 @@ int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, /* Deprecated streaming functions */ int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } +/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t) + * @return : 0 on success, !=0 if error */ int LZ4_resetStreamStateHC(void* state, char* inputBuffer) { - LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; - if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ - LZ4_resetStreamHC((LZ4_streamHC_t*)state, ((LZ4_streamHC_t*)state)->internal_donotuse.compressionLevel); - LZ4HC_init(ctx, (const BYTE*)inputBuffer); + LZ4_streamHC_t* const hc4 = LZ4_initStreamHC(state, sizeof(*hc4)); + if (hc4 == NULL) return 1; /* init failed */ + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); return 0; } void* LZ4_createHC (const char* inputBuffer) { - LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); + LZ4_streamHC_t* const hc4 = LZ4_createStreamHC(); if (hc4 == NULL) return NULL; /* not enough memory */ - LZ4_resetStreamHC(hc4, 0 /* compressionLevel */); - LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); return hc4; } -int LZ4_freeHC (void* LZ4HC_Data) { +int LZ4_freeHC (void* LZ4HC_Data) +{ if (!LZ4HC_Data) return 0; /* support free on NULL */ FREEMEM(LZ4HC_Data); return 0; @@ -1070,7 +1130,7 @@ int LZ4_freeHC (void* LZ4HC_Data) { int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) { - return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, noLimit); + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, notLimited); } int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel) @@ -1089,7 +1149,7 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data) /* ================================================ - * LZ4 Optimal parser (levels 10-12) + * LZ4 Optimal parser (levels [LZ4HC_CLEVEL_OPT_MIN - LZ4HC_CLEVEL_MAX]) * ===============================================*/ typedef struct { int price; @@ -1102,8 +1162,9 @@ typedef struct { LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) { int price = litlen; + assert(litlen >= 0); if (litlen >= (int)RUN_MASK) - price += 1 + (litlen-RUN_MASK)/255; + price += 1 + ((litlen-(int)RUN_MASK) / 255); return price; } @@ -1112,11 +1173,13 @@ LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) { int price = 1 + 2 ; /* token + 16-bit offset */ + assert(litlen >= 0); + assert(mlen >= MINMATCH); price += LZ4HC_literalsPrice(litlen); if (mlen >= (int)(ML_MASK+MINMATCH)) - price += 1 + (mlen-(ML_MASK+MINMATCH))/255; + price += 1 + ((mlen-(int)(ML_MASK+MINMATCH)) / 255); return price; } @@ -1175,9 +1238,9 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, BYTE* oend = op + dstCapacity; /* init */ - DEBUGLOG(5, "LZ4HC_compress_optimal"); + DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity); *srcSizePtr = 0; - if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; /* Main Loop */ @@ -1195,7 +1258,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, int const firstML = firstMatch.len; const BYTE* const matchPos = ip - firstMatch.off; opSaved = op; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ goto _dest_overflow; continue; } @@ -1333,6 +1396,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, } } } /* for (cur = 1; cur <= last_match_pos; cur++) */ + assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS); best_mlen = opt[last_match_pos].mlen; best_off = opt[last_match_pos].off; cur = last_match_pos - best_mlen; @@ -1365,9 +1429,9 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ rPos += ml; assert(ml >= MINMATCH); - assert((offset >= 1) && (offset <= MAX_DISTANCE)); + assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX)); opSaved = op; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ goto _dest_overflow; } } } /* while (ip <= mflimit) */ @@ -1377,7 +1441,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; size_t const totalSize = 1 + litLength + lastRunSize; - if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ if (limit && (op + totalSize > oend)) { if (limit == limitedOutput) return 0; /* Check output limit */ /* adapt lastRunSize to fill 'dst' */ @@ -1404,7 +1468,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, return (int) ((char*)op-dst); _dest_overflow: - if (limit == limitedDestSize) { + if (limit == fillOutput) { op = opSaved; /* restore correct out pointer */ goto _last_literals; } diff --git a/lz4libs/lz4hc.h b/lz4libs/lz4hc.h index 970fa396..cdc6d895 100644 --- a/lz4libs/lz4hc.h +++ b/lz4libs/lz4hc.h @@ -54,7 +54,7 @@ extern "C" { * Block Compression **************************************/ /*! LZ4_compress_HC() : - * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. + * Compress data from `src` into `dst`, using the powerful but slower "HC" algorithm. * `dst` must be already allocated. * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") @@ -77,7 +77,21 @@ LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dst * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly). */ LZ4LIB_API int LZ4_sizeofStateHC(void); -LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); +LZ4LIB_API int LZ4_compress_HC_extStateHC(void* stateHC, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + + +/*! LZ4_compress_HC_destSize() : v1.9.0+ + * Will compress as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided in 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr is updated to indicate how much bytes were read from `src` + */ +LZ4LIB_API int LZ4_compress_HC_destSize(void* stateHC, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize, + int compressionLevel); /*-************************************ @@ -89,46 +103,92 @@ LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* ds /*! LZ4_createStreamHC() and LZ4_freeStreamHC() : * These functions create and release memory for LZ4 HC streaming state. * Newly created states are automatically initialized. - * Existing states can be re-used several times, using LZ4_resetStreamHC(). - * These methods are API and ABI stable, they can be used in combination with a DLL. + * A same state can be used multiple times consecutively, + * starting with LZ4_resetStreamHC_fast() to start a new stream of blocks. */ LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); -LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); -LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); - -LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); - -LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); - /* - These functions compress data in successive blocks of any size, using previous blocks as dictionary. + These functions compress data in successive blocks of any size, + using previous blocks as dictionary, to improve compression ratio. One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. There is an exception for ring buffers, which can be smaller than 64 KB. - Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue(). + Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue(). + + Before starting compression, state must be allocated and properly initialized. + LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT. + + Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream) + or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental). + LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once, + which is automatically the case when state is created using LZ4_createStreamHC(). + + After reset, a first "fictional block" can be designated as initial dictionary, + using LZ4_loadDictHC() (Optional). + + Invoke LZ4_compress_HC_continue() to compress each successive block. + The number of blocks is unlimited. + Previous input blocks, including initial dictionary when present, + must remain accessible and unmodified during compression. + + It's allowed to update compression level anytime between blocks, + using LZ4_setCompressionLevel() (experimental). + + 'dst' buffer should be sized to handle worst case scenarios + (see LZ4_compressBound(), it ensures compression success). + In case of failure, the API does not guarantee recovery, + so the state _must_ be reset. + To ensure compression success + whenever `dst` buffer size cannot be made >= LZ4_compressBound(), + consider using LZ4_compress_HC_continue_destSize(). + + Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks, + it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC(). + Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB) + + After completing a streaming compression, + it's possible to start a new stream of blocks, using the same LZ4_streamHC_t state, + just by resetting it, using LZ4_resetStreamHC_fast(). +*/ - Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). - A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). +LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel); /* v1.9.0+ */ +LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); + +LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, + const char* src, char* dst, + int srcSize, int maxDstSize); + +/*! LZ4_compress_HC_continue_destSize() : v1.9.0+ + * Similar to LZ4_compress_HC_continue(), + * but will read as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided into 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`. + * Note that this function may not consume the entire input. + */ +LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize); + +LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); - Then, use LZ4_compress_HC_continue() to compress each successive block. - Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. - 'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success. - Because in case of failure, the API does not guarantee context recovery, and context will have to be reset. - If `dst` buffer budget cannot be >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead. - If, for any reason, previous data block can't be preserved unmodified in memory for next compression block, - you can save it to a more stable memory space, using LZ4_saveDictHC(). - Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. -*/ +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ -/*-************************************************************** +/*-****************************************************************** * PRIVATE DEFINITIONS : - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_streamHC_t`. - * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 - ****************************************************************/ + * Do not use these definitions directly. + * They are merely exposed to allow static allocation of `LZ4_streamHC_t`. + * Declare an `LZ4_streamHC_t` directly, rather than any type below. + * Even then, only do so in the context of static linking, as definitions may change between versions. + ********************************************************************/ + #define LZ4HC_DICTIONARY_LOGSIZE 16 #define LZ4HC_MAXD (1<= LZ4HC_CLEVEL_OPT_MIN) +/*! LZ4_favorDecompressionSpeed() : v1.8.2+ (experimental) + * Opt. Parser will favor decompression speed over compression ratio. + * Only applicable to levels >= LZ4HC_CLEVEL_OPT_MIN. */ -void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor); +LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed( + LZ4_streamHC_t* LZ4_streamHCPtr, int favor); -/*! LZ4_resetStreamHC_fast() : +/*! LZ4_resetStreamHC_fast() : v1.9.0+ * When an LZ4_streamHC_t is known to be in a internally coherent state, * it can often be prepared for a new compression with almost no work, only * sometimes falling back to the full, expensive reset that is always required @@ -304,8 +372,14 @@ void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor); * - the stream was in an indeterminate state and was used in a compression * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that * returned success + * + * Note: + * A stream that was last used in a compression call that returned an error + * may be passed to this function. However, it will be fully reset, which will + * clear any existing history and settings from the context. */ -void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast( + LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); /*! LZ4_compress_HC_extStateHC_fastReset() : * A variant of LZ4_compress_HC_extStateHC(). @@ -318,7 +392,11 @@ void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLeve * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a * call to LZ4_resetStreamHC(). */ -int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); +LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset ( + void* state, + const char* src, char* dst, + int srcSize, int dstCapacity, + int compressionLevel); /*! LZ4_attach_HC_dictionary() : * This is an experimental API that allows for the efficient use of a @@ -345,7 +423,9 @@ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* ds * stream (and source buffer) must remain in-place / accessible / unchanged * through the lifetime of the stream session. */ -LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream); +LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary( + LZ4_streamHC_t *working_stream, + const LZ4_streamHC_t *dictionary_stream); #if defined (__cplusplus) } diff --git a/lz4libs/xxhash.c b/lz4libs/xxhash.c index 3fc97fd2..ff28749e 100644 --- a/lz4libs/xxhash.c +++ b/lz4libs/xxhash.c @@ -50,20 +50,26 @@ * Prefer these methods in priority order (0 > 1 > 2) */ #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ + || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \ + || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define XXH_FORCE_MEMORY_ACCESS 2 # elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \ + || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \ + || defined(__ARM_ARCH_7S__) )) # define XXH_FORCE_MEMORY_ACCESS 1 # endif #endif /*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : + * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault. + * When this macro is enabled, xxHash actively checks input for null pointer. + * It it is, result for null input pointers is the same as a null-length input. */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ +#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ +# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +#endif /*!XXH_FORCE_NATIVE_FORMAT : * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. @@ -80,8 +86,9 @@ /*!XXH_FORCE_ALIGN_CHECK : * This is a minor performance trick, only useful with lots of very small keys. * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. + * The check costs one initial branch per hash; + * set it to 0 when the input is guaranteed to be aligned, + * or when alignment doesn't matter for performance. */ #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ # if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) @@ -104,6 +111,8 @@ static void XXH_free (void* p) { free(p); } #include static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } +#include /* assert */ + #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" @@ -113,40 +122,35 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp ***************************************/ #ifdef _MSC_VER /* Visual Studio */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - -#ifndef XXH_FORCE_INLINE -# ifdef _MSC_VER /* Visual Studio */ -# define XXH_FORCE_INLINE static __forceinline -# else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define XXH_FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define XXH_FORCE_INLINE static inline -# endif +# define FORCE_INLINE static __forceinline +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) # else -# define XXH_FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -# endif /* _MSC_VER */ -#endif /* XXH_FORCE_INLINE */ +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif /* ************************************* * Basic Types ***************************************/ #ifndef MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; - typedef int32_t S32; # else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; - typedef signed int S32; # endif #endif @@ -213,8 +217,12 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ #ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +static int XXH_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() #endif @@ -223,7 +231,7 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; *****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; -XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); @@ -231,7 +239,7 @@ XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, X return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); } -XXH_FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } @@ -245,12 +253,12 @@ static U32 XXH_readBE32(const void* ptr) /* ************************************* * Macros ***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#define XXH_STATIC_ASSERT(c) { enum { XXH_sa = 1/(int)(!!(c)) }; } /* use after variable declarations */ XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } /* ******************************************************************* -* 32-bits hash functions +* 32-bit hash functions *********************************************************************/ static const U32 PRIME32_1 = 2654435761U; static const U32 PRIME32_2 = 2246822519U; @@ -266,14 +274,89 @@ static U32 XXH32_round(U32 seed, U32 input) return seed; } -XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +/* mix all bits */ +static U32 XXH32_avalanche(U32 h32) +{ + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +static U32 +XXH32_finalize(U32 h32, const void* ptr, size_t len, + XXH_endianess endian, XXH_alignment align) + +{ + const BYTE* p = (const BYTE*)ptr; + +#define PROCESS1 \ + h32 += (*p++) * PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + +#define PROCESS4 \ + h32 += XXH_get32bits(p) * PRIME32_3; \ + p+=4; \ + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + + switch(len&15) /* or switch(bEnd - p) */ + { + case 12: PROCESS4; + /* fallthrough */ + case 8: PROCESS4; + /* fallthrough */ + case 4: PROCESS4; + return XXH32_avalanche(h32); + + case 13: PROCESS4; + /* fallthrough */ + case 9: PROCESS4; + /* fallthrough */ + case 5: PROCESS4; + PROCESS1; + return XXH32_avalanche(h32); + + case 14: PROCESS4; + /* fallthrough */ + case 10: PROCESS4; + /* fallthrough */ + case 6: PROCESS4; + PROCESS1; + PROCESS1; + return XXH32_avalanche(h32); + + case 15: PROCESS4; + /* fallthrough */ + case 11: PROCESS4; + /* fallthrough */ + case 7: PROCESS4; + /* fallthrough */ + case 3: PROCESS1; + /* fallthrough */ + case 2: PROCESS1; + /* fallthrough */ + case 1: PROCESS1; + /* fallthrough */ + case 0: return XXH32_avalanche(h32); + } + assert(0); + return h32; /* reaching this point is deemed impossible */ +} + + +FORCE_INLINE U32 +XXH32_endian_align(const void* input, size_t len, U32 seed, + XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)16; @@ -281,7 +364,7 @@ XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, #endif if (len>=16) { - const BYTE* const limit = bEnd - 16; + const BYTE* const limit = bEnd - 15; U32 v1 = seed + PRIME32_1 + PRIME32_2; U32 v2 = seed + PRIME32_2; U32 v3 = seed + 0; @@ -292,34 +375,17 @@ XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); + } while (p < limit); - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); } else { h32 = seed + PRIME32_5; } - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; + h32 += (U32)len; - return h32; + return XXH32_finalize(h32, p, len&15, endian, align); } @@ -371,74 +437,81 @@ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) { XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ + memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME32_1 + PRIME32_2; state.v2 = seed + PRIME32_2; state.v3 = seed + 0; state.v4 = seed - PRIME32_1; - memcpy(statePtr, &state, sizeof(state)); + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); return XXH_OK; } -XXH_FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +FORCE_INLINE XXH_errorcode +XXH32_update_endian(XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) { - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; #endif - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); + { const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } + state->total_len_32 += (unsigned)len; + state->large_len |= (len>=16) | (state->total_len_32>=16); - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); + } + p += 16-state->memsize; + state->memsize = 0; + } - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } } return XXH_OK; } + XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -450,40 +523,23 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* } - -XXH_FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +FORCE_INLINE U32 +XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) { - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; U32 h32; if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + h32 = XXH_rotl32(state->v1, 1) + + XXH_rotl32(state->v2, 7) + + XXH_rotl32(state->v3, 12) + + XXH_rotl32(state->v4, 18); } else { h32 = state->v3 /* == seed */ + PRIME32_5; } h32 += state->total_len_32; - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; + return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned); } @@ -503,7 +559,7 @@ XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) /*! Default XXH result types are basic unsigned 32 and 64 bits. * The canonical representation follows human-readable write convention, aka big-endian (large digits first). * These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. +* This way, hash values can be written into a file or buffer, remaining comparable across different systems. */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) @@ -522,18 +578,21 @@ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src #ifndef XXH_NO_LONG_LONG /* ******************************************************************* -* 64-bits hash functions +* 64-bit hash functions *********************************************************************/ /*====== Memory access ======*/ #ifndef MEM_MODULE # define MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint64_t U64; # else - typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ + /* if compiler doesn't support unsigned long long, replace by another 64-bit type */ + typedef unsigned long long U64; # endif #endif @@ -583,7 +642,7 @@ static U64 XXH_swap64 (U64 x) } #endif -XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); @@ -591,7 +650,7 @@ XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, X return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); } -XXH_FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) { return XXH_readLE64_align(ptr, endian, XXH_unaligned); } @@ -626,14 +685,137 @@ static U64 XXH64_mergeRound(U64 acc, U64 val) return acc; } -XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +static U64 XXH64_avalanche(U64 h64) +{ + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +static U64 +XXH64_finalize(U64 h64, const void* ptr, size_t len, + XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)ptr; + +#define PROCESS1_64 \ + h64 ^= (*p++) * PRIME64_5; \ + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + +#define PROCESS4_64 \ + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \ + p+=4; \ + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + +#define PROCESS8_64 { \ + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \ + p+=8; \ + h64 ^= k1; \ + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \ +} + + switch(len&31) { + case 24: PROCESS8_64; + /* fallthrough */ + case 16: PROCESS8_64; + /* fallthrough */ + case 8: PROCESS8_64; + return XXH64_avalanche(h64); + + case 28: PROCESS8_64; + /* fallthrough */ + case 20: PROCESS8_64; + /* fallthrough */ + case 12: PROCESS8_64; + /* fallthrough */ + case 4: PROCESS4_64; + return XXH64_avalanche(h64); + + case 25: PROCESS8_64; + /* fallthrough */ + case 17: PROCESS8_64; + /* fallthrough */ + case 9: PROCESS8_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 29: PROCESS8_64; + /* fallthrough */ + case 21: PROCESS8_64; + /* fallthrough */ + case 13: PROCESS8_64; + /* fallthrough */ + case 5: PROCESS4_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 26: PROCESS8_64; + /* fallthrough */ + case 18: PROCESS8_64; + /* fallthrough */ + case 10: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 30: PROCESS8_64; + /* fallthrough */ + case 22: PROCESS8_64; + /* fallthrough */ + case 14: PROCESS8_64; + /* fallthrough */ + case 6: PROCESS4_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 27: PROCESS8_64; + /* fallthrough */ + case 19: PROCESS8_64; + /* fallthrough */ + case 11: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 31: PROCESS8_64; + /* fallthrough */ + case 23: PROCESS8_64; + /* fallthrough */ + case 15: PROCESS8_64; + /* fallthrough */ + case 7: PROCESS4_64; + /* fallthrough */ + case 3: PROCESS1_64; + /* fallthrough */ + case 2: PROCESS1_64; + /* fallthrough */ + case 1: PROCESS1_64; + /* fallthrough */ + case 0: return XXH64_avalanche(h64); + } + + /* impossible to reach */ + assert(0); + return 0; /* unreachable, but some compilers complain without it */ +} + +FORCE_INLINE U64 +XXH64_endian_align(const void* input, size_t len, U64 seed, + XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; + const BYTE* bEnd = p + len; U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)32; @@ -666,32 +848,7 @@ XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, h64 += (U64) len; - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; + return XXH64_finalize(h64, p, len, endian, align); } @@ -741,65 +898,71 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) { XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ + memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME64_1 + PRIME64_2; state.v2 = seed + PRIME64_2; state.v3 = seed + 0; state.v4 = seed - PRIME64_1; - memcpy(statePtr, &state, sizeof(state)); + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); return XXH_OK; } -XXH_FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +FORCE_INLINE XXH_errorcode +XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) { - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; #endif - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } + { const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } + state->total_len += len; - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } } return XXH_OK; @@ -815,10 +978,8 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* return XXH64_update_endian(state_in, input, len, XXH_bigEndian); } -XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) { - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; U64 h64; if (state->total_len >= 32) { @@ -833,37 +994,12 @@ XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endian h64 = XXH64_mergeRound(h64, v3); h64 = XXH64_mergeRound(h64, v4); } else { - h64 = state->v3 + PRIME64_5; + h64 = state->v3 /*seed*/ + PRIME64_5; } h64 += (U64) state->total_len; - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; + return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned); } XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) diff --git a/lz4libs/xxhash.h b/lz4libs/xxhash.h index 870a6d91..d6bad943 100644 --- a/lz4libs/xxhash.h +++ b/lz4libs/xxhash.h @@ -57,8 +57,8 @@ Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. Name Speed on 64 bits Speed on 32 bits XXH64 13.8 GB/s 1.9 GB/s XXH32 6.8 GB/s 6.0 GB/s @@ -80,18 +80,19 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; /* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module. -*/ -#ifdef XXH_PRIVATE_API + * API modifier + ******************************/ +/** XXH_INLINE_ALL (and XXH_PRIVATE_API) + * This is useful to include xxhash functions in `static` mode + * in order to inline them, and remove their symbol from the public list. + * Inlining can offer dramatic performance improvement on small keys. + * Methodology : + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * `xxhash.c` is automatically included. + * It's not useful to compile and link it as a separate module. + */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) # ifndef XXH_STATIC_LINKING_ONLY # define XXH_STATIC_LINKING_ONLY # endif @@ -102,23 +103,24 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; # elif defined(_MSC_VER) # define XXH_PUBLIC_API static __inline # else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ + /* this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static # endif #else # define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ - -/*!XXH_NAMESPACE, aka Namespace Emulation : - -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with other libraries which may also include xxHash, - -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). - -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/*! XXH_NAMESPACE, aka Namespace Emulation : + * + * If you want to include _and expose_ xxHash functions from within your own library, + * but also want to avoid symbol collisions with other libraries which may also include xxHash, + * + * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library + * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). + * + * Note that no change is required within the calling program as long as it includes `xxhash.h` : + * regular symbol name will be automatically translated by this header. + */ #ifdef XXH_NAMESPACE # define XXH_CAT(A,B) A##B # define XXH_NAME2(A,B) XXH_CAT(A,B) @@ -149,18 +151,18 @@ regular symbol name will be automatically translated by this header. ***************************************/ #define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 2 +#define XXH_VERSION_RELEASE 5 #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) XXH_PUBLIC_API unsigned XXH_versionNumber (void); /*-********************************************************************** -* 32-bits hash +* 32-bit hash ************************************************************************/ -typedef unsigned int XXH32_hash_t; +typedef unsigned int XXH32_hash_t; /*! XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input". The memory between input & input+length must be valid (allocated and read-accessible). "seed" can be used to alter the result predictably. Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ @@ -177,26 +179,25 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); /* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . - -XXH state must first be allocated, using XXH*_createState() . - -Start a new hash by initializing state with a seed, using XXH*_reset(). - -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. - -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). - -When done, free XXH state space if it was allocated dynamically. -*/ + * Streaming functions generate the xxHash of an input provided in multiple segments. + * Note that, for small input, they are slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * XXH state must first be allocated, using XXH*_createState() . + * + * Start a new hash by initializing state with a seed, using XXH*_reset(). + * + * Then, feed the hash state by calling XXH*_update() as many times as necessary. + * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using XXH*_digest(). + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a digest, + * and generate some new hashes later on, by calling again XXH*_digest(). + * + * When done, free XXH state space if it was allocated dynamically. + */ /*====== Canonical representation ======*/ @@ -205,22 +206,22 @@ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); /* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ + * The canonical representation uses human-readable write convention, aka big-endian (large digits first). + * These functions allow transformation of hash result into and from its canonical format. + * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. + */ #ifndef XXH_NO_LONG_LONG /*-********************************************************************** -* 64-bits hash +* 64-bit hash ************************************************************************/ typedef unsigned long long XXH64_hash_t; /*! XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + Calculate the 64-bit hash of sequence of length "len" stored at memory address "input". "seed" can be used to alter the result predictably. - This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). + This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark). */ XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); @@ -241,48 +242,82 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src #endif /* XXH_NO_LONG_LONG */ + #ifdef XXH_STATIC_LINKING_ONLY /* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. + This section contains declarations which are not guaranteed to remain stable. They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! + These declarations should only be used with static linking. + Never use them in association with dynamic linking ! =================================================================================================== */ -/* These definitions are only meant to allow allocation of XXH state - statically, on stack, or in a struct for example. - Do not use members directly. */ - - struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH32_state_t */ - -#ifndef XXH_NO_LONG_LONG - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH64_state_t */ +/* These definitions are only present to allow + * static allocation of XXH state, on stack or in a struct for example. + * Never **ever** use members directly. */ + +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + +struct XXH32_state_s { + uint32_t total_len_32; + uint32_t large_len; + uint32_t v1; + uint32_t v2; + uint32_t v3; + uint32_t v4; + uint32_t mem32[4]; + uint32_t memsize; + uint32_t reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +struct XXH64_state_s { + uint64_t total_len; + uint64_t v1; + uint64_t v2; + uint64_t v3; + uint64_t v4; + uint64_t mem64[4]; + uint32_t memsize; + uint32_t reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ + +# else + +struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; + unsigned memsize; + unsigned reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +# ifndef XXH_NO_LONG_LONG /* remove 64-bit support */ +struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; + unsigned memsize; + unsigned reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ +# endif + +# endif + + +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ #endif -# ifdef XXH_PRIVATE_API -# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ -# endif - #endif /* XXH_STATIC_LINKING_ONLY */ From 16e0fc924b02911de659219f2f63827a8ea0b412 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 8 Jun 2019 16:59:48 +0100 Subject: [PATCH 091/280] Fix some compiler warnings for _block.c --- lz4/block/_block.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 61bafe52..d55b1d92 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -140,7 +140,7 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) Py_buffer source; int source_size; int return_bytearray = 0; - Py_buffer dict = { NULL, NULL }; + Py_buffer dict = {0}; static char *argnames[] = { "source", "mode", @@ -189,7 +189,7 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) return NULL; } - source_size = source.len; + source_size = (int) source.len; if (!strncmp (mode, "default", sizeof ("default"))) { @@ -243,8 +243,8 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) } output_size = lz4_compress_generic (comp, source.buf, dest_start, source_size, - dest_size, dict.buf, dict.len, acceleration, - compression); + (int) dest_size, dict.buf, (int) dict.len, + acceleration, compression); Py_END_ALLOW_THREADS @@ -260,7 +260,7 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) if (store_size) { - output_size += hdr_size; + output_size += (int) hdr_size; } if (return_bytearray) @@ -294,7 +294,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) size_t dest_size; int uncompressed_size = -1; int return_bytearray = 0; - Py_buffer dict = { NULL, NULL }; + Py_buffer dict = {0}; static char *argnames[] = { "source", "uncompressed_size", @@ -338,7 +338,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) } source_start = (const char *) source.buf; - source_size = source.len; + source_size = (int) source.len; if (uncompressed_size >= 0) { @@ -358,7 +358,7 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) source_size -= hdr_size; } - if (dest_size < 0 || dest_size > PY_SSIZE_T_MAX) + if (dest_size > INT_MAX) { PyBuffer_Release(&source); PyBuffer_Release(&dict); @@ -376,8 +376,8 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) Py_BEGIN_ALLOW_THREADS output_size = - LZ4_decompress_safe_usingDict (source_start, dest, source_size, dest_size, - dict.buf, dict.len); + LZ4_decompress_safe_usingDict (source_start, dest, source_size, (int) dest_size, + dict.buf, (int) dict.len); Py_END_ALLOW_THREADS From c14fad6802ab4fefbdd498cb9a16c78e0d9b8675 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 9 Jun 2019 08:33:43 +0100 Subject: [PATCH 092/280] Upgrade cibuildwheel from 0.10.0 to 0.11.1 --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a3163046..bc9f2359 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,7 +56,7 @@ matrix: - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - - ${PIP} install cibuildwheel==0.10.0 + - ${PIP} install cibuildwheel==0.11.1 script: - cibuildwheel --output-dir dist - name: osx-py34 @@ -68,7 +68,7 @@ matrix: - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - - ${PIP} install cibuildwheel==0.10.0 + - ${PIP} install cibuildwheel==0.11.1 script: - cibuildwheel --output-dir dist - name: osx-py35 @@ -80,7 +80,7 @@ matrix: - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - - ${PIP} install cibuildwheel==0.10.0 + - ${PIP} install cibuildwheel==0.11.1 script: - cibuildwheel --output-dir dist - name: osx-py36 @@ -92,7 +92,7 @@ matrix: - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - - ${PIP} install cibuildwheel==0.10.0 + - ${PIP} install cibuildwheel==0.11.1 script: - cibuildwheel --output-dir dist - name: osx-py37 @@ -104,12 +104,12 @@ matrix: - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - - ${PIP} install cibuildwheel==0.10.0 + - ${PIP} install cibuildwheel==0.11.1 script: - cibuildwheel --output-dir dist install: - ${PIP} install -U pip - - ${PIP} install cibuildwheel==0.10.0 + - ${PIP} install cibuildwheel==0.11.1 - ${PIP} install codecov tox twine script: - tox tests From 4e42fd2121882b9d977e1f5d2c2df4a6ccd1ef41 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 9 Jun 2019 10:05:18 +0100 Subject: [PATCH 093/280] Add more detail to the install section of docs --- docs/install.rst | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 270be536..3d531f7b 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -1,22 +1,49 @@ Install ======= -The package is hosted on `PyPI `_ and so can be -installed via pip:: +The bindings to the LZ4 compression library provided by this package are in the +form of a Python extension module written in C. These extension modules need to +be compiled against the LZ4 library and the Python + +Installing from pre-built wheels +-------------------------------- + +The package is hosted on `PyPI `_ and pre-built +wheels are available for Linux, OSX and Windows. Installation using a pre-built +wheel can be achieved by:: $ pip install lz4 + +Installing from source +---------------------- + The LZ4 bindings require linking to the LZ4 library, and so if there is not a pre-compiled wheel available for your platform you will need to have a suitable -C compiler available, as well as the Python development header files. +C compiler available, as well as the Python development header files. On +Debian/Ubuntu based systems the header files for Python are found in the +distribution package ``pythonX.Y-dev`` e.g. ``python3.7-dev``. On Fedora/Red Hat +based systems, the Python header files are found in the distribution package +``python-devel``. The LZ4 library bindings provided by this package require the LZ4 library. If the system already has an LZ4 library and development header files present, and the library is a recent enough version the package will build against that. Otherwise, the package will use a bundled version of the library files to link -against. The package currently requires LZ4 version 1.7.5 or later. +against. The package currently requires LZ4 version 1.7.5 or later. + +On a system for which there are no pre-built wheels available on PyPi, running +this command will result in the extension modules being compiled from source:: + + $ pip install lz4 + +On systems for which pre-built wheels are available, the following command will +force a local compilation of the extension modules from source:: + + $ pip install --no-binary --no-cache-dir lz4 -The package can also be installed manually:: +The package can also be installed manually from a checkout of the source code +git repository:: $ python setup.py install From 9afeaf1a220ab881aba29ff0679b3dac79432b12 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Mon, 26 Aug 2019 22:43:31 +0100 Subject: [PATCH 094/280] Update bundled LZ4 library to version 1.9.2 --- lz4libs/lz4.c | 425 ++++++++++++++++++++++++++++----------------- lz4libs/lz4.h | 142 +++++++++++---- lz4libs/lz4frame.c | 28 ++- lz4libs/lz4frame.h | 9 + lz4libs/lz4hc.c | 120 +++++++++---- lz4libs/lz4hc.h | 3 + 6 files changed, 502 insertions(+), 225 deletions(-) diff --git a/lz4libs/lz4.c b/lz4libs/lz4.c index e614c457..9808d70a 100644 --- a/lz4libs/lz4.c +++ b/lz4libs/lz4.c @@ -106,6 +106,7 @@ #define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ #endif +#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ #include "lz4.h" /* see also "memory routines" below */ @@ -182,6 +183,60 @@ #define MEM_INIT(p,v,s) memset((p),(v),(s)) +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif + +#define ML_BITS 4 +#define ML_MASK ((1U<=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include +static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + /*-************************************ * Types **************************************/ @@ -317,6 +372,11 @@ static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; #ifndef LZ4_FAST_DEC_LOOP # if defined(__i386__) || defined(__x86_64__) # define LZ4_FAST_DEC_LOOP 1 +# elif defined(__aarch64__) && !defined(__clang__) + /* On aarch64, we disable this optimization for clang because on certain + * mobile chipsets and clang, it reduces performance. For more information + * refer to https://github.com/lz4/lz4/pull/707. */ +# define LZ4_FAST_DEC_LOOP 1 # else # define LZ4_FAST_DEC_LOOP 0 # endif @@ -358,29 +418,35 @@ LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH + * - there is at least 8 bytes available to write after dstEnd */ LZ4_FORCE_O2_INLINE_GCC_PPC64LE void LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) { BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */ + switch(offset) { case 1: memset(v, *srcPtr, 8); - goto copy_loop; + break; case 2: memcpy(v, srcPtr, 2); memcpy(&v[2], srcPtr, 2); memcpy(&v[4], &v[0], 4); - goto copy_loop; + break; case 4: memcpy(v, srcPtr, 4); memcpy(&v[4], srcPtr, 4); - goto copy_loop; + break; default: LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); return; } - copy_loop: memcpy(dstPtr, v, 8); dstPtr += 8; while (dstPtr < dstEnd) { @@ -391,63 +457,6 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si #endif -/*-************************************ -* Common Constants -**************************************/ -#define MINMATCH 4 - -#define WILDCOPYLENGTH 8 -#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ -#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ -#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ -#define FASTLOOP_SAFE_DISTANCE 64 -static const int LZ4_minLength = (MFLIMIT+1); - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#ifndef LZ4_DISTANCE_MAX /* can be user - defined at compile time */ -# define LZ4_DISTANCE_MAX 65535 -#endif - -#if (LZ4_DISTANCE_MAX > 65535) /* max supported by LZ4 format */ -# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" -#endif - -#define ML_BITS 4 -#define ML_MASK ((1U<=1) -# include -#else -# ifndef assert -# define assert(condition) ((void)0) -# endif -#endif - -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ - -#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) -# include -static int g_debuglog_enable = 1; -# define DEBUGLOG(l, ...) { \ - if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ - } } -#else -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif - - /*-************************************ * Common functions **************************************/ @@ -460,7 +469,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanForward64( &r, (U64)val ); return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); + return (unsigned)__builtin_ctzll((U64)val) >> 3; # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, @@ -478,7 +487,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanForward( &r, (U32)val ); return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); + return (unsigned)__builtin_ctz((U32)val) >> 3; # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, @@ -494,7 +503,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanReverse64( &r, val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); + return (unsigned)__builtin_clzll((U64)val) >> 3; # else static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. @@ -511,7 +520,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); + return (unsigned)__builtin_clz((U32)val) >> 3; # else unsigned r; if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } @@ -606,9 +615,11 @@ int LZ4_sizeofState() { return LZ4_STREAMSIZE; } extern "C" { #endif -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); -int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize); +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); #if defined (__cplusplus) } @@ -643,6 +654,18 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab return LZ4_hash4(LZ4_read32(p), tableType); } +static void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} + static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) { switch (tableType) @@ -703,18 +726,19 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, - const void* tableBase, tableType_t tableType, - const BYTE* srcBase) +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType, + const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } -LZ4_FORCE_INLINE void LZ4_prepareTable( - LZ4_stream_t_internal* const cctx, - const int inputSize, - const tableType_t tableType) { +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { /* If compression failed during the previous step, then the context * is marked as dirty, therefore, it has to be fully reset. */ @@ -729,9 +753,10 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( * out if it's safe to leave as is or whether it needs to be reset. */ if (cctx->tableType != clearedTable) { + assert(inputSize >= 0); if (cctx->tableType != tableType - || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU) - || (tableType == byU32 && cctx->currentOffset > 1 GB) + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) || tableType == byPtr || inputSize >= 4 KB) { @@ -811,9 +836,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); /* If init conditions are not met, we don't have to mark stream * as having dirty context, since no action was taken yet */ - if (outputDirective == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */ + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ assert(acceleration >= 1); @@ -841,6 +866,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( for ( ; ; ) { const BYTE* match; BYTE* token; + const BYTE* filledIp; /* Find a match */ if (tableType == byPtr) { @@ -909,10 +935,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); - if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */ + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ assert(matchIndex < current); - if ((tableType != byU16) && (matchIndex+LZ4_DISTANCE_MAX < current)) continue; /* too far */ - if (tableType == byU16) assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* too_far presumed impossible with byU16 */ + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ if (LZ4_read32(match) == LZ4_read32(ip)) { if (maybe_extMem) offset = current - matchIndex; @@ -923,15 +953,16 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( } /* Catch up */ + filledIp = ip; while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } /* Encode Literals */ { unsigned const litLength = (unsigned)(ip - anchor); token = op++; if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ - + } if ((outputDirective == fillOutput) && (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { op--; @@ -1002,12 +1033,26 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( } if ((outputDirective) && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) { + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { if (outputDirective == fillOutput) { /* Match description too long : reduce it */ - U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255; + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } } else { assert(outputDirective == limitedOutput); return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ @@ -1027,6 +1072,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( } else *token += (BYTE)(matchCode); } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); anchor = ip; @@ -1076,7 +1123,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); assert(matchIndex < current); if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) - && ((tableType==byU16) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) && (LZ4_read32(match) == LZ4_read32(ip)) ) { token=op++; *token=0; @@ -1143,7 +1190,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { - if (inputSize < LZ4_64Klimit) {; + if (inputSize < LZ4_64Klimit) { return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); } else { const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; @@ -1306,12 +1353,12 @@ static size_t LZ4_stream_t_alignment(void) LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) { DEBUGLOG(5, "LZ4_initStream"); - if (buffer == NULL) return NULL; - if (size < sizeof(LZ4_stream_t)) return NULL; + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } #ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : it reports an aligment of 8-bytes, while actually aligning LZ4_stream_t on 4 bytes. */ - if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) return NULL; /* alignment check */ + if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */ #endif MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); return (LZ4_stream_t*)buffer; @@ -1361,18 +1408,18 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) * there are only valid offsets in the window, which allows an optimization * in LZ4_compress_fast_continue() where it uses noDictIssue even when the * dictionary isn't a full 64k. */ - - if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - base = dictEnd - 64 KB - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); dict->currentOffset += 64 KB; - dict->tableType = tableType; if (dictSize < (int)HASH_UNIT) { return 0; } + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + base = dictEnd - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->tableType = tableType; + while (p <= dictEnd-HASH_UNIT) { LZ4_putPosition(p, dict->hashTable, tableType, base); p+=3; @@ -1381,26 +1428,37 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) return (int)dict->dictSize; } -void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) { + const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + workingStream, dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + /* Calling LZ4_resetStream_fast() here makes sure that changes will not be * erased by subsequent calls to LZ4_resetStream_fast() in case stream was * marked as having dirty context, e.g. requiring full reset. */ - LZ4_resetStream_fast(working_stream); + LZ4_resetStream_fast(workingStream); - if (dictionary_stream != NULL) { + if (dictCtx != NULL) { /* If the current offset is zero, we will never look in the * external dictionary context, since there is no value a table * entry can take that indicate a miss. In that case, we need * to bump the offset to something non-zero. */ - if (working_stream->internal_donotuse.currentOffset == 0) { - working_stream->internal_donotuse.currentOffset = 64 KB; + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; } - working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse); - } else { - working_stream->internal_donotuse.dictCtx = NULL; } + workingStream->internal_donotuse.dictCtx = dictCtx; } @@ -1435,7 +1493,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); - if (streamPtr->dirty) return 0; /* Uninitialized structure detected */ + if (streamPtr->dirty) { return 0; } /* Uninitialized structure detected */ LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; @@ -1532,8 +1590,8 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = (int)dict->dictSize; + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } memmove(safeBuffer, previousDictEnd - dictSize, dictSize); @@ -1607,7 +1665,7 @@ LZ4_decompress_generic( const size_t dictSize /* note : = 0 if noDict */ ) { - if (src == NULL) return -1; + if (src == NULL) { return -1; } { const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; @@ -1636,9 +1694,13 @@ LZ4_decompress_generic( /* Special cases */ assert(lowPrefix <= op); - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); - if ((endOnInput) && unlikely(srcSize==0)) return -1; + if ((endOnInput) && (unlikely(outputSize==0))) { + /* Empty output buffer */ + if (partialDecoding) return 0; + return ((srcSize==1) && (*ip==0)) ? 0 : -1; + } + if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); } + if ((endOnInput) && unlikely(srcSize==0)) { return -1; } /* Currently the fast loop shows a regression on qualcomm arm chips. */ #if LZ4_FAST_DEC_LOOP @@ -1651,7 +1713,7 @@ LZ4_decompress_generic( while (1) { /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ assert(oend - op >= FASTLOOP_SAFE_DISTANCE); - if (endOnInput) assert(ip < iend); + if (endOnInput) { assert(ip < iend); } token = *ip++; length = token >> ML_BITS; /* literal length */ @@ -1661,18 +1723,18 @@ LZ4_decompress_generic( if (length == RUN_MASK) { variable_length_error error = ok; length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); - if (error == initial_error) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ /* copy literals */ cpy = op+length; LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); if (endOnInput) { /* LZ4_decompress_safe() */ - if ((cpy>oend-32) || (ip+length>iend-32)) goto safe_literal_copy; + if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } LZ4_wildCopy32(op, ip, cpy); } else { /* LZ4_decompress_fast() */ - if (cpy>oend-8) goto safe_literal_copy; + if (cpy>oend-8) { goto safe_literal_copy; } LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : * it doesn't know input length, and only relies on end-of-block properties */ } @@ -1682,14 +1744,14 @@ LZ4_decompress_generic( if (endOnInput) { /* LZ4_decompress_safe() */ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); /* We don't need to check oend, since we check it once for each loop below */ - if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) goto safe_literal_copy; + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ memcpy(op, ip, 16); } else { /* LZ4_decompress_fast() */ /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : * it doesn't know input length, and relies on end-of-block properties */ memcpy(op, ip, 8); - if (length > 8) memcpy(op+8, ip+8, 8); + if (length > 8) { memcpy(op+8, ip+8, 8); } } ip += length; op = cpy; } @@ -1697,17 +1759,17 @@ LZ4_decompress_generic( /* get offset */ offset = LZ4_readLE16(ip); ip+=2; match = op - offset; + assert(match <= op); /* get matchlength */ length = token & ML_MASK; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - if (length == ML_MASK) { variable_length_error error = ok; + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); - if (error != ok) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + if (error != ok) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ length += MINMATCH; if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { goto safe_match_copy; @@ -1719,8 +1781,12 @@ LZ4_decompress_generic( } /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ - if (!(dict == usingExtDict) || (match >= lowPrefix)) { + if ((dict == withPrefix64k) || (match >= lowPrefix)) { if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + memcpy(op, match, 8); memcpy(op+8, match+8, 8); memcpy(op+16, match+16, 2); @@ -1728,12 +1794,15 @@ LZ4_decompress_generic( continue; } } } + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { - if (partialDecoding) length = MIN(length, (size_t)(oend-op)); - else goto _output_error; /* doesn't respect parsing restriction */ - } + if (partialDecoding) { + length = MIN(length, (size_t)(oend-op)); /* reach end of buffer */ + } else { + goto _output_error; /* end-of-block condition violated */ + } } if (length <= (size_t)(lowPrefix-match)) { /* match fits entirely within external dictionary : just copy */ @@ -1748,7 +1817,7 @@ LZ4_decompress_generic( if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ BYTE* const endOfMatch = op + restSize; const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; + while (op < endOfMatch) { *op++ = *copyFrom++; } } else { memcpy(op, lowPrefix, restSize); op += restSize; @@ -1821,11 +1890,11 @@ LZ4_decompress_generic( /* decode literal length */ if (length == RUN_MASK) { - variable_length_error error = ok; - length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); - if (error == initial_error) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ } /* copy literals */ @@ -1837,21 +1906,50 @@ LZ4_decompress_generic( if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) { + /* We've either hit the input parsing restriction or the output parsing restriction. + * If we've hit the input parsing condition then this must be the last sequence. + * If we've hit the output parsing condition then we are either using partialDecoding + * or we've hit the output parsing condition. + */ if (partialDecoding) { - if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); } /* Partial decoding : stop in the middle of literal segment */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + assert(endOnInput); + /* If we're in this block because of the input parsing condition, then we must be on the + * last sequence (or invalid), so we must check that we exactly consume the input. + */ + if ((ip+length>iend-(2+1+LASTLITERALS)) && (ip+length != iend)) { goto _output_error; } + assert(ip+length <= iend); + /* We are finishing in the middle of a literals segment. + * Break after the copy. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + assert(ip+length <= iend); } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + /* We must be on the last sequence because of the parsing limitations so check + * that we exactly regenerate the original size (must be exact when !endOnInput). + */ + if ((!endOnInput) && (cpy != oend)) { goto _output_error; } + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; } } - memcpy(op, ip, length); + memmove(op, ip, length); /* supports overlapping memory regions, which only matters for in-place decompression scenarios */ ip += length; op += length; - if (!partialDecoding || (cpy == oend)) { - /* Necessarily EOF, due to parsing restrictions */ + /* Necessarily EOF when !partialDecoding. When partialDecoding + * it is EOF if we've either filled the output buffer or hit + * the input parsing restriction. + */ + if (!partialDecoding || (cpy == oend) || (ip == iend)) { break; } - } else { LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ ip += length; op = cpy; @@ -1865,13 +1963,6 @@ LZ4_decompress_generic( length = token & ML_MASK; _copy_match: - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - if (!partialDecoding) { - assert(oend > op); - assert(oend - op >= 4); - LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ - } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ - if (length == ML_MASK) { variable_length_error error = ok; length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); @@ -1883,6 +1974,7 @@ LZ4_decompress_generic( #if LZ4_FAST_DEC_LOOP safe_match_copy: #endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { @@ -1910,6 +2002,7 @@ LZ4_decompress_generic( } } continue; } + assert(match >= lowPrefix); /* copy match within block */ cpy = op + length; @@ -1921,16 +2014,17 @@ LZ4_decompress_generic( const BYTE* const matchEnd = match + mlen; BYTE* const copyEnd = op + mlen; if (matchEnd > op) { /* overlap copy */ - while (op < copyEnd) *op++ = *match++; + while (op < copyEnd) { *op++ = *match++; } } else { memcpy(op, match, mlen); } op = copyEnd; - if (op==oend) break; + if (op == oend) { break; } continue; } if (unlikely(offset<8)) { + LZ4_write32(op, 0); /* silence msan warning when offset==0 */ op[0] = match[0]; op[1] = match[1]; op[2] = match[2]; @@ -1946,25 +2040,26 @@ LZ4_decompress_generic( if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ if (op < oCopyLimit) { LZ4_wildCopy8(op, match, oCopyLimit); match += oCopyLimit - op; op = oCopyLimit; } - while (op < cpy) *op++ = *match++; + while (op < cpy) { *op++ = *match++; } } else { memcpy(op, match, 8); - if (length > 16) LZ4_wildCopy8(op+8, match+8, cpy); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } } op = cpy; /* wildcopy correction */ } /* end of decoding */ - if (endOnInput) + if (endOnInput) { return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else + } else { return (int) (((const char*)ip)-src); /* Nb of input bytes read */ + } /* Overflow error detected */ _output_error: @@ -2079,7 +2174,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void) int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { - if (LZ4_stream == NULL) return 0; /* support free on NULL */ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } @@ -2214,18 +2309,22 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed if (dictSize==0) return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (dictStart+dictSize == dest) { - if (dictSize >= 64 KB - 1) + if (dictSize >= 64 KB - 1) { return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); - return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); } - return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize); + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { if (dictSize==0 || dictStart+dictSize == dest) return LZ4_decompress_fast(source, dest, originalSize); - return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, dictSize); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); } @@ -2237,9 +2336,9 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) +int LZ4_compress(const char* src, char* dest, int srcSize) { - return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); } int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { diff --git a/lz4libs/lz4.h b/lz4libs/lz4.h index a9c932cc..32108e23 100644 --- a/lz4libs/lz4.h +++ b/lz4libs/lz4.h @@ -46,7 +46,7 @@ extern "C" { /** Introduction - LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core, + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, scalable with multi-cores CPU. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. @@ -58,16 +58,19 @@ extern "C" { - unbounded multiple steps (described as Streaming compression) lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). - Decompressing a block requires additional metadata, such as its compressed size. + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. Each application is free to encode and pass such metadata in whichever way it wants. lz4.h only handle blocks, it can not generate Frames. Blocks are different from Frames (doc/lz4_Frame_format.md). Frames bundle both blocks and metadata in a specified manner. - This are required for compressed data to be self-contained and portable. + Embedding metadata is required for compressed data to be self-contained and portable. Frame format is delivered through a companion API, declared in lz4frame.h. - Note that the `lz4` CLI can only manage frames. + The `lz4` CLI can only manage frames. */ /*^*************************************************************** @@ -97,7 +100,7 @@ extern "C" { /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -129,29 +132,35 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; * Simple Functions **************************************/ /*! LZ4_compress_default() : - Compresses 'srcSize' bytes from buffer 'src' - into already allocated 'dst' buffer of size 'dstCapacity'. - Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'src' into a more limited 'dst' budget, - compression stops *immediately*, and the function result is zero. - In which case, 'dst' content is undefined (invalid). - srcSize : max supported value is LZ4_MAX_INPUT_SIZE. - dstCapacity : size of buffer 'dst' (which must be already allocated) - @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) - or 0 if compression fails - Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). -*/ + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : - compressedSize : is the exact complete size of the compressed block. - dstCapacity : is the size of destination buffer, which must be already allocated. - @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) - If destination buffer is not large enough, decoding will stop and output an error code (negative value). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer). -*/ + * compressedSize : is the exact complete size of the compressed block. + * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); @@ -388,6 +397,8 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod */ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); +#endif /* LZ4_H_2983827168210 */ + /*^************************************* * !!!!!! STATIC LINKING ONLY !!!!!! @@ -413,14 +424,17 @@ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int sr * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. ******************************************************************************/ +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + #ifdef LZ4_PUBLISH_STATIC_FUNCTIONS #define LZ4LIB_STATIC_API LZ4LIB_API #else #define LZ4LIB_STATIC_API #endif -#ifdef LZ4_STATIC_LINKING_ONLY - /*! LZ4_compress_fast_extState_fastReset() : * A variant of LZ4_compress_fast_extState(). @@ -462,8 +476,75 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c */ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly contrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ #endif +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 /*-************************************************************ * PRIVATE DEFINITIONS @@ -567,6 +648,7 @@ union LZ4_streamDecode_u { } ; /* previously typedef'd to LZ4_streamDecode_t */ + /*-************************************ * Obsolete Functions **************************************/ @@ -601,8 +683,8 @@ union LZ4_streamDecode_u { #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); @@ -674,7 +756,7 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); -#endif /* LZ4_H_2983827168210 */ +#endif /* LZ4_H_98237428734687 */ #if defined (__cplusplus) diff --git a/lz4libs/lz4frame.c b/lz4libs/lz4frame.c index a10e4af0..c9f630d6 100644 --- a/lz4libs/lz4frame.c +++ b/lz4libs/lz4frame.c @@ -213,8 +213,8 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */ static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ -static const size_t BHSize = 4; /* block header : size, and compress flag */ -static const size_t BFSize = 4; /* block footer : checksum (optional) */ +static const size_t BHSize = LZ4F_BLOCK_HEADER_SIZE; /* block header : size, and compress flag */ +static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checksum (optional) */ /*-************************************ @@ -327,6 +327,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, { LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES; prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ + prefsNull.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; U32 const flush = prefsPtr->autoFlush | (srcSize==0); LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID; @@ -1130,8 +1131,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize } /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); +#endif dctx->frameInfo.frameType = LZ4F_frame; /* Flags */ @@ -1170,10 +1173,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize /* check header */ assert(frameHeaderSize > 5); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); } +#endif /* save */ dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; @@ -1210,8 +1215,10 @@ size_t LZ4F_headerSize(const void* src, size_t srcSize) return 8; /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); +#endif /* Frame Header Size */ { BYTE const FLG = ((const BYTE*)src)[4]; @@ -1493,7 +1500,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* next block is a compressed block */ dctx->tmpInTarget = nextCBlockSize + crcSize; dctx->dStage = dstage_getCBlock; - if (dstPtr==dstEnd) { + if (dstPtr==dstEnd || srcPtr==srcEnd) { nextSrcSizeHint = BHSize + nextCBlockSize + crcSize; doAnotherStage = 0; } @@ -1554,8 +1561,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } { U32 const readCRC = LZ4F_readLE32(crcSrc); U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (readCRC != calcCRC) return err0r(LZ4F_ERROR_blockChecksum_invalid); +#else + (void)readCRC; + (void)calcCRC; +#endif } } dctx->dStage = dstage_getBlockHeader; /* new block */ break; @@ -1594,8 +1606,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */ { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (readBlockCrc != calcBlockCrc) return err0r(LZ4F_ERROR_blockChecksum_invalid); +#else + (void)readBlockCrc; + (void)calcBlockCrc; +#endif } } if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) { @@ -1723,8 +1740,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */ { U32 const readCRC = LZ4F_readLE32(selectedIn); U32 const resultCRC = XXH32_digest(&(dctx->xxh)); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid); +#else + (void)readCRC; + (void)resultCRC; +#endif nextSrcSizeHint = 0; LZ4F_resetDecompressionContext(dctx); doAnotherStage = 0; diff --git a/lz4libs/lz4frame.h b/lz4libs/lz4frame.h index 742c2528..391e4840 100644 --- a/lz4libs/lz4frame.h +++ b/lz4libs/lz4frame.h @@ -253,6 +253,15 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); #define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */ #define LZ4F_HEADER_SIZE_MAX 19 +/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */ +#define LZ4F_BLOCK_HEADER_SIZE 4 + +/* Size in bytes of a block checksum footer in little-endian format. */ +#define LZ4F_BLOCK_CHECKSUM_SIZE 4 + +/* Size in bytes of the content checksum. */ +#define LZ4F_CONTENT_CHECKSUM_SIZE 4 + /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. diff --git a/lz4libs/lz4hc.c b/lz4libs/lz4hc.c index 936f7396..5922ed7b 100644 --- a/lz4libs/lz4hc.c +++ b/lz4libs/lz4hc.c @@ -151,6 +151,21 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, return back; } +#if defined(_MSC_VER) +# define LZ4HC_rotl32(x,r) _rotl(x,r) +#else +# define LZ4HC_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) +{ + size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; + if (bitsToRotate == 0) + return pattern; + return LZ4HC_rotl32(pattern, (int)bitsToRotate); +} + /* LZ4HC_countPattern() : * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ static unsigned @@ -203,6 +218,16 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) return (unsigned)(iStart - ip); } +/* LZ4HC_protectDictEnd() : + * Checks if the match is in the last 3 bytes of the dictionary, so reading the + * 4 byte MINMATCH would overflow. + * @returns true if the match index is okay. + */ +static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) +{ + return ((U32)((dictLimit - 1) - matchIndex) >= 3); +} + typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; @@ -228,7 +253,7 @@ LZ4HC_InsertAndGetWiderMatch ( const U32 dictLimit = hc4->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; const U32 ipIndex = (U32)(ip - base); - const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; + const U32 lowestMatchIndex = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; const BYTE* const dictBase = hc4->dictBase; int const lookBackLength = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; @@ -287,14 +312,21 @@ LZ4HC_InsertAndGetWiderMatch ( if (chainSwap && matchLength==longest) { /* better match => select a better chain */ assert(lookBackLength==0); /* search forward only */ if (matchIndex + (U32)longest <= ipIndex) { + int const kTrigger = 4; U32 distanceToNextMatch = 1; + int const end = longest - MINMATCH + 1; + int step = 1; + int accel = 1 << kTrigger; int pos; - for (pos = 0; pos <= longest - MINMATCH; pos++) { + for (pos = 0; pos < end; pos += step) { U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); + step = (accel++ >> kTrigger); if (candidateDist > distanceToNextMatch) { distanceToNextMatch = candidateDist; matchChainPos = (U32)pos; - } } + accel = 1 << kTrigger; + } + } if (distanceToNextMatch > 1) { if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ matchIndex -= distanceToNextMatch; @@ -313,34 +345,61 @@ LZ4HC_InsertAndGetWiderMatch ( } else { repeat = rep_not; } } - if ( (repeat == rep_confirmed) - && (matchCandidateIdx >= dictLimit) ) { /* same segment only */ - const BYTE* const matchPtr = base + matchCandidateIdx; + if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) + && LZ4HC_protectDictEnd(dictLimit, matchCandidateIdx) ) { + const int extDict = matchCandidateIdx < dictLimit; + const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx; if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ - size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); - const BYTE* const lowestMatchPtr = (lowPrefixPtr + LZ4_DISTANCE_MAX >= ip) ? lowPrefixPtr : ip - LZ4_DISTANCE_MAX; - size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); - size_t const currentSegmentLength = backLength + forwardPatternLength; - - if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ - && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ - matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ - } else { - matchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ - if (lookBackLength==0) { /* no back possible */ - size_t const maxML = MIN(currentSegmentLength, srcPatternLength); - if ((size_t)longest < maxML) { - assert(base + matchIndex < ip); - if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break; - assert(maxML < 2 GB); - longest = (int)maxML; - *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ - *startpos = ip; + const BYTE* const dictStart = dictBase + hc4->lowLimit; + const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit; + size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); + if (extDict && matchPtr + forwardPatternLength == iLimit) { + U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern); + forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern); + } + { const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr; + size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); + size_t currentSegmentLength; + if (!extDict && matchPtr - backLength == lowPrefixPtr && hc4->lowLimit < dictLimit) { + U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern); + backLength += LZ4HC_reverseCountPattern(dictBase + dictLimit, dictStart, rotatedPattern); + } + /* Limit backLength not go further than lowestMatchIndex */ + backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex); + assert(matchCandidateIdx - backLength >= lowestMatchIndex); + currentSegmentLength = backLength + forwardPatternLength; + /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */ + if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ + && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ + U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ + if (LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) + matchIndex = newMatchIndex; + else { + /* Can only happen if started in the prefix */ + assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict); + matchIndex = dictLimit; } - { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); - if (distToNextPattern > matchIndex) break; /* avoid overflow */ - matchIndex -= distToNextPattern; - } } } + } else { + U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ + if (!LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) { + assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict); + matchIndex = dictLimit; + } else { + matchIndex = newMatchIndex; + if (lookBackLength==0) { /* no back possible */ + size_t const maxML = MIN(currentSegmentLength, srcPatternLength); + if ((size_t)longest < maxML) { + assert(base + matchIndex < ip); + if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break; + assert(maxML < 2 GB); + longest = (int)maxML; + *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ + *startpos = ip; + } + { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); + if (distToNextPattern > matchIndex) break; /* avoid overflow */ + matchIndex -= distToNextPattern; + } } } } } continue; } } } } /* PA optimization */ @@ -1005,6 +1064,9 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->end = newBlock; ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + + /* cannot reference an extDict and a dictCtx at the same time */ + ctxPtr->dictCtx = NULL; } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, diff --git a/lz4libs/lz4hc.h b/lz4libs/lz4hc.h index cdc6d895..44e35bbf 100644 --- a/lz4libs/lz4hc.h +++ b/lz4libs/lz4hc.h @@ -336,6 +336,9 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL #ifndef LZ4_HC_SLO_098092834 #define LZ4_HC_SLO_098092834 +#define LZ4_STATIC_LINKING_ONLY /* LZ4LIB_STATIC_API */ +#include "lz4.h" + #if defined (__cplusplus) extern "C" { #endif From 3db542fb4ca7c1a6c9238fe3affa274e2a052525 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 15 Sep 2019 11:05:50 +0100 Subject: [PATCH 095/280] Specify virtualenv version used by tox This forces the version of pip to be recent. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 57d526d7..9dde1650 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] envlist = py,flake8,docs minversion = 2.4.0 +requires = virtualenv >= 16.4.0 [testenv] extras = tests From d668ab6f859a2adeb63b34ce652c772121981653 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 15 Sep 2019 11:58:51 +0100 Subject: [PATCH 096/280] Pin more-itertools to a version supporting Python 2.7 Also add a comment about why we're specifying versions of tox deps for Python 2.7. --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index 9dde1650..50942dee 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,11 @@ [tox] envlist = py,flake8,docs minversion = 2.4.0 +# Pinning more-itertools and specifying virtualenv minimum version are +# workaround hacks for Python 2.7 and can be dropped when we drop support for +# Python 2.7. requires = virtualenv >= 16.4.0 + more-itertools==4.2.0 [testenv] extras = tests From 9c3031575157d656d0e6ee647ebbac12f69234e4 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 15 Sep 2019 13:57:06 +0100 Subject: [PATCH 097/280] Add pathlib2 dependency --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 50942dee..2992d4e9 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ minversion = 2.4.0 # Python 2.7. requires = virtualenv >= 16.4.0 more-itertools==4.2.0 + pathlib2 [testenv] extras = tests From d077ecc959c6c9c27040723e3c188b9ae1f5a438 Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Mon, 24 Jun 2019 14:41:28 +0200 Subject: [PATCH 098/280] Fix tox.ini end-of-file --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2992d4e9..c6de5abb 100644 --- a/tox.ini +++ b/tox.ini @@ -35,4 +35,4 @@ usedevelop = True whitelist_externals=make extras = docs commands = - make -C docs doctest html \ No newline at end of file + make -C docs doctest html From 9439cc8dc2526da814d71f3b4bc52075fe521279 Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Tue, 30 Apr 2019 10:34:20 +0200 Subject: [PATCH 099/280] Add bindings for LZ4 stream (double-buffer mode) --- lz4/stream/__init__.py | 200 +++++ lz4/stream/_stream.c | 1685 ++++++++++++++++++++++++++++++++++++++++ setup.py | 20 +- 3 files changed, 1904 insertions(+), 1 deletion(-) create mode 100644 lz4/stream/__init__.py create mode 100644 lz4/stream/_stream.c diff --git a/lz4/stream/__init__.py b/lz4/stream/__init__.py new file mode 100644 index 00000000..60a119ea --- /dev/null +++ b/lz4/stream/__init__.py @@ -0,0 +1,200 @@ +from ._stream import _create_context, _compress, _decompress, _get_block +from ._stream import LZ4StreamError, _compress_bound, _input_bound, LZ4_MAX_INPUT_SIZE # noqa: F401 + + +__doc__ = """\ +A Python wrapper for the LZ4 stream protocol. + +""" + + +class LZ4StreamDecompressor: + """ LZ4 stream decompression context. + + """ + def __init__(self, strategy, buffer_size, return_bytearray=False, store_comp_size=4, dictionary=""): + """ Instantiates and initializes a LZ4 stream decompression context. + + Args: + strategy (str): Buffer management strategy. Can be: ``double_buffer``. + buffer_size (int): Size of one buffer of the double-buffer used + internally for stream decompression in the case of ``double_buffer`` + strategy. + + Keyword Args: + return_bytearray (bool): If ``False`` (the default) then the function + will return a ``bytes`` object. If ``True``, then the function will + return a ``bytearray`` object. + store_comp_size (int): Specify the size in bytes of the following + compressed block. Can be: ``1``, ``2`` or ``4`` (default: ``4``). + dictionary (str, bytes or buffer-compatible object): If specified, + perform decompression using this initial dictionary. + + Raises: + Exceptions occuring during the context initialization. + + OverflowError: raised if the ``dictionary`` parameter is too large + for the LZ4 context. + ValueError: raised if some parameters are invalid. + MemoryError: raised if some internal resources cannot be allocated. + RuntimeError: raised if some internal resources cannot be initialized. + + """ + return_bytearray = 1 if return_bytearray else 0 + + self._context = _create_context(strategy, "decompress", buffer_size, + return_bytearray=return_bytearray, + store_comp_size=store_comp_size, + dictionary=dictionary) + + def __enter__(self): + """ Enter the LZ4 stream context. + + """ + return self + + def __exit__(self, exc_type, exc, exc_tb): + """ Exit the LZ4 stream context. + + """ + pass + + def decompress(self, chunk): + """ Decompress streamed compressed data. + + Decompress the given ``chunk``, using the given LZ4 stream context, + Raises an exception if any error occurs. + + Args: + chunk (str, bytes or buffer-compatible object): Data to decompress + + Returns: + bytes or bytearray: Decompressed data. + + Raises: + Exceptions occuring during decompression. + + ValueError: raised if the source is inconsistent with a finite LZ4 + stream block chain. + MemoryError: raised if the work output buffer cannot be allocated. + OverflowError: raised if the source is too large for being decompressed + in the given context. + LZ4StreamError: raised if the call to the LZ4 library fails. This can be + caused by ``decompressed_size`` being too small, or invalid data. + + """ + return _decompress(self._context, chunk) + + def get_block(self, stream): + """ Return the first LZ4 compressed block from ``stream``. + + Args: + stream (str, bytes or buffer-compatible object): LZ4 compressed stream. + + Returns: + bytes or bytearray: LZ4 compressed data block. + + Raises: + Exceptions occuring while getting the first block from ``stream``. + + BufferError: raised if the function cannot return a complete LZ4 + compressed block from the stream (i.e. the stream does not hold + a complete block). + MemoryError: raised if the output buffer cannot be allocated. + OverflowError: raised if the source is too large for being handled by + the given context. + + """ + return _get_block(self._context, stream) + + +class LZ4StreamCompressor: + """ LZ4 stream compressing context. + + """ + def __init__(self, strategy, buffer_size, mode="default", acceleration=True, compression_level=9, + return_bytearray=False, store_comp_size=4, dictionary=""): + """ Instantiates and initializes a LZ4 stream compression context. + + Args: + strategy (str): Buffer management strategy. Can be: ``double_buffer``. + buffer_size (int): Base size of the buffer(s) used internally for stream + compression/decompression. In the ``double_buffer`` strategy case, + this is the size of each buffer of the double-buffer. + + Keyword Args: + mode (str): If ``default`` or unspecified use the default LZ4 + compression mode. Set to ``fast`` to use the fast compression + LZ4 mode at the expense of compression. Set to + ``high_compression`` to use the LZ4 high-compression mode at + the expense of speed. + acceleration (int): When mode is set to ``fast`` this argument + specifies the acceleration. The larger the acceleration, the + faster the but the lower the compression. The default + compression corresponds to a value of ``1``. + compression_level (int): When mode is set to ``high_compression`` this + argument specifies the compression. Valid values are between + ``1`` and ``12``. Values between ``4-9`` are recommended, and + ``9`` is the default. Only relevant if ``mode`` is + ``high_compression``. + return_bytearray (bool): If ``False`` (the default) then the function + will return a bytes object. If ``True``, then the function will + return a bytearray object. + store_comp_size (int): Specify the size in bytes of the following + compressed block. Can be: ``1``, ``2`` or ``4`` (default: ``4``). + dictionary (str, bytes or buffer-compatible object): If specified, + perform compression using this initial dictionary. + + Raises: + Exceptions occuring during the context initialization. + + OverflowError: raised if the ``dictionary`` parameter is too large + for the LZ4 context. + ValueError: raised if some parameters are invalid. + MemoryError: raised if some internal resources cannot be allocated. + RuntimeError: raised if some internal resources cannot be initialized. + + """ + return_bytearray = 1 if return_bytearray else 0 + + self._context = _create_context(strategy, "compress", buffer_size, + mode=mode, + acceleration=acceleration, + compression_level=compression_level, + return_bytearray=return_bytearray, + store_comp_size=store_comp_size, + dictionary=dictionary) + + def __enter__(self): + """ Enter the LZ4 stream context. + + """ + return self + + def __exit__(self, exc_type, exc, exc_tb): + """ Exit the LZ4 stream context. + + """ + pass + + def compress(self, chunk): + """ Stream compress given ``chunk`` of data. + + Compress the given ``chunk``, using the given LZ4 stream context, + returning the compressed data as a ``bytearray`` or as a ``bytes`` object. + + Args: + chunk (str, bytes or buffer-compatible object): Data to compress + + Returns: + bytes or bytearray: Compressed data. + + Raises: + Exceptions occuring during compression. + + OverflowError: raised if the source is too large for being compressed in + the given context. + LZ4StreamError: raised if the call to the LZ4 library fails. + + """ + return _compress(self._context, chunk) diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c new file mode 100644 index 00000000..330f94bc --- /dev/null +++ b/lz4/stream/_stream.c @@ -0,0 +1,1685 @@ +/* + * Copyright (c) 2019, Samuel Martin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#if defined(_WIN32) && defined(_MSC_VER) +#define inline __inline +#elif defined(__SUNPRO_C) || defined(__hpux) || defined(_AIX) +#define inline +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) && defined(_MSC_VER) +#if _MSC_VER >= 1600 +#include +#else /* _MSC_VER >= 1600 */ +typedef signed __int8 int8_t; +typedef signed __int16 int16_t; +typedef signed __int32 int32_t; +typedef signed __int64 int64_t; +typedef unsigned __int8 uint8_t; +typedef unsigned __int16 uint16_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int64 uint64_t; + +#if !defined(UINT8_MAX) +#define UINT8_MAX 0xff +#endif +#if !defined(UINT16_MAX) +#define UINT16_MAX 0xffff +#endif +#if !defined(UINT32_MAX) +#define UINT32_MAX 0xffffffff +#endif +#if !defined(INT32_MAX) +#define INT32_MAX 0x7fffffff +#endif +#endif /* _MSC_VER >= 1600 */ + +#if !defined(__CHAR_BIT__) +#define __CHAR_BIT__ 8 +#endif /* __CHAR_BIT__ */ +#endif /* _WIN32 && _MSC_VER */ + +#ifndef Py_UNUSED /* This is already defined for Python 3.4 onwards */ +#ifdef __GNUC__ +#define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) +#else +#define Py_UNUSED(name) _unused_ ## name +#endif +#endif + +#define LZ4_VERSION_NUMBER_1_9_0 10900 + +static const char * stream_context_capsule_name = "_stream.LZ4S_ctx"; + +typedef enum { + DOUBLE_BUFFER, + RING_BUFFER, + + BUFFER_STRATEGY_COUNT /* must be the last entry */ +} buffer_strategy_e; + +typedef enum { + COMPRESS, + DECOMPRESS, +} direction_e; + +typedef enum { + DEFAULT, + FAST, + HIGH_COMPRESSION, +} compression_type_e; + + +/* Forward declarations */ +static PyObject * LZ4StreamError; + +#define DOUBLE_BUFFER_PAGE_COUNT (2) + +#define DOUBLE_BUFFER_INDEX_MIN (0) +#define DOUBLE_BUFFER_INDEX_INVALID (-1) + +#define _GET_MAX_UINT(byte_depth, type) (type)( ( 1ULL << ( __CHAR_BIT__ * (byte_depth) ) ) - 1 ) +#define _GET_MAX_UINT32(byte_depth) _GET_MAX_UINT((byte_depth), uint32_t) + +typedef struct { + char * buf; + unsigned int len; +} buffer_t; + +/* forward declaration */ +typedef struct stream_context_t stream_context_t; + +typedef struct { + /** + * Release buffer strategy's resources. + * + * \param[inout] context Stream context. + */ + void (*release_resources) (stream_context_t *context); + + /** + * Reserve buffer strategy's resources. + * + * \param[inout] context Stream context. + * \param[in] buffer_size Base buffer size to allocate and initialize. + * + * \return 0 on success, non-0 otherwise + */ + int (*reserve_resources) (stream_context_t * context, unsigned int buffer_size); + + /** + * Return a pointer on the work buffer. + * + * \param[inout] context Stream context. + * + * \return A pointer on the work buffer. + */ + char * (*get_work_buffer) (const stream_context_t * context); + + /** + * Return the length of (available space in) the work buffer. + * + * \param[inout] context Stream context. + * + * \return The length of the work buffer. + */ + unsigned int (*get_work_buffer_size) (const stream_context_t * context); + + /** + * Return the length of the output buffer. + * + * \param[inout] context Stream context. + * + * \return The length the output buffer. + */ + unsigned int (*get_dest_buffer_size) (const stream_context_t * context); + + /** + * Update the stream context at the end of the LZ4 operation (a block compression or + * decompression). + * + * \param[inout] context Stream context. + * + * \return 0 on success, non-0 otherwise + */ + int (*update_context_after_process) (stream_context_t * context); +} strategy_ops_t; + +struct stream_context_t { + /* Buffer strategy resources */ + struct { + strategy_ops_t * ops; + union { + /* Double-buffer */ + struct { + char * buf; + unsigned int page_size; + char * pages[DOUBLE_BUFFER_PAGE_COUNT]; + int index; + } double_buffer; + + /* Ring-buffer (not implemented) */ + struct { + char * buf; + unsigned int size; + } ring_buffer; + } data; + } strategy; + + buffer_t output; + + /* LZ4 state */ + union { + union { + LZ4_stream_t * fast; + LZ4_streamHC_t * hc; + } compress; + LZ4_streamDecode_t * decompress; + void * context; + } lz4_state; + + /* LZ4 configuration */ + struct { + int acceleration; + int compression_level; + int store_comp_size; + int return_bytearray; + direction_e direction; + compression_type_e comp; + } config; +}; + + +#ifndef PyCapsule_Type +#define _PyCapsule_get_context(py_ctx) \ + ((stream_context_t *) PyCapsule_GetPointer((py_ctx), stream_context_capsule_name)) +#else +/* Compatibility with 2.6 via capsulethunk. */ +#define _PyCapsule_get_context(py_ctx) \ + ((stream_context_t *) (py_ctx)) +#endif + + +static inline void +store_le8 (char * c, uint8_t x) +{ + c[0] = x & 0xff; +} + + +/************************* + * block content helpers * + *************************/ +static inline uint8_t +load_le8 (const char * c) +{ + const uint8_t * d = (const uint8_t *) c; + return d[0]; +} + +static inline void +store_le16 (char * c, uint16_t x) +{ + c[0] = x & 0xff; + c[1] = (x >> 8) & 0xff; +} + +static inline uint16_t +load_le16 (const char * c) +{ + const uint8_t * d = (const uint8_t *) c; + return (d[0] | (d[1] << 8)); +} + +static inline void +store_le32 (char * c, uint32_t x) +{ + c[0] = x & 0xff; + c[1] = (x >> 8) & 0xff; + c[2] = (x >> 16) & 0xff; + c[3] = (x >> 24) & 0xff; +} + +static inline uint32_t +load_le32 (const char * c) +{ + const uint8_t * d = (const uint8_t *) c; + return (d[0] | (d[1] << 8) | (d[2] << 16) | (d[3] << 24)); +} + +static inline int +load_block_length (int block_length_size, const char *buf) +{ + int block_length = -1; + switch (block_length_size) + { + case 1: + block_length = load_le8 (buf); + break; + + case 2: + block_length = load_le16 (buf); + break; + + case 4: + block_length = load_le32 (buf); + break; + + case 0: + /* fallthrough */ + default: + break; + } + + return block_length; +} + +static inline int +store_block_length (int block_length, int block_length_size, char * buf) +{ + int status = 1; + + switch (block_length_size) + { + case 0: /* do nothing */ + break; + + case 1: + { + if (block_length > UINT8_MAX) + { + status = 0; + break; + } + store_le8 (buf, (uint8_t) (block_length & UINT8_MAX)); + } + break; + + case 2: + { + if (block_length > UINT16_MAX) + { + status = 0; + break; + } + store_le16 (buf, (uint16_t) (block_length & UINT16_MAX)); + } + break; + + case 4: + { + if (block_length > INT32_MAX) + { + status = 0; + break; + } + store_le32 (buf, (uint32_t) (block_length & UINT32_MAX)); + } + break; + + default: /* unsupported cases */ + status = 0; + break; + } + + if (status != 1) + { + PyErr_SetString (LZ4StreamError, "Compressed stream size too large"); + } + + return status; +} + + +/*************** + * LZ4 helpers * + ***************/ +static inline uint32_t +get_compress_bound(uint32_t input_size) +{ + /* result of LZ4_compressBound is null or positive */ + return (uint32_t)LZ4_compressBound(input_size); +} + + +static inline uint32_t +get_input_bound(uint32_t compress_max_size) +{ + uint64_t isize = 0; + uint64_t csize = (uint64_t) compress_max_size; + + /* Reversing the LZ4_COMPRESSBOUND macro gives: + * isize = ((csize - 16) * 255) / 256 + * = (((csize - 16) * 256) - (csize - 16)) / 256 + * = ((csize * 256) - (16 * 256) - (csize - 16)) / 256 + * = ((csize << 8) - (16 << 8) - csize + 16) >> 8 + * = ((csize << 8) - csize + 16 - (16 << 8)) >> 8 + * = ((csize << 8) - csize - 4080) >> 8 + * + * Notes: + * - Using 64-bit long integer for intermediate computation to avoid any + * truncation when shifting left large csize values. + * - Due to the round integer approximation, running the following + * calculation can give a non-null result: + * result = n - _LZ4_inputBound( _LZ4_compressBound( n ) ) + * but in all cases, this difference is between 0 and 1. + * Thus, the valid maximal input size returned by this funtcion is + * incremented by 1 to avoid any buffer overflow in case of decompression + * in a dynamically allocated buffer. + * - For small compressed length (shorter than 16 bytes), make sure a + * non-null size is returned. + */ + if (csize < 16) + { + csize = 17; + } + + if (csize <= get_compress_bound (LZ4_MAX_INPUT_SIZE)) + { + isize = ((csize << 8) - csize - 4080) >> 8; + + if (isize > (uint32_t)LZ4_MAX_INPUT_SIZE) + { + isize = 0; + } + else + { + isize += 1; + } + } + + return (uint32_t)(isize & UINT32_MAX); +} + + +/************************************** + * LZ4 version-compatibility wrappers * + **************************************/ +#if defined (__GNUC__) +/* Runtime detection of the support of new functions in the LZ4 API + * (old functions remain available but are deprecated and will trigger + * compilation warnings. + * + * Declare weak symbols on the required functions provided by recent versions + * of the library. + * This way, these symbols will always be available - NULL if the (old version + * of the) library does not define them. + */ + +/* Function introduced in LZ4 >= 1.9.0 */ +__attribute__ ((weak)) void +LZ4_resetStreamHC_fast (LZ4_streamHC_t* streamHCPtr, int compressionLevel); + +/* Function introduced in LZ4 >= 1.8.2 */ +__attribute__ ((weak)) void +LZ4_resetStream_fast (LZ4_stream_t* streamPtr); + +#else +/* Assuming the bundled LZ4 library sources are always used, so meet the + * LZ4 minimal version requirements. + */ +#endif + +static inline void reset_stream (LZ4_stream_t* streamPtr) +{ + if (LZ4_versionNumber () >= LZ4_VERSION_NUMBER_1_9_0) + { + if (LZ4_resetStream_fast) + { + LZ4_resetStream_fast (streamPtr); + } + else + { + PyErr_SetString (PyExc_RuntimeError, + "Inconsistent LZ4 library version/available APIs"); + } + } + else + { + LZ4_resetStream (streamPtr); + } +} + + +static inline void reset_stream_hc (LZ4_streamHC_t* streamHCPtr, int compressionLevel) +{ + if (LZ4_versionNumber () >= LZ4_VERSION_NUMBER_1_9_0) + { + if (LZ4_resetStreamHC_fast) + { + LZ4_resetStreamHC_fast (streamHCPtr, compressionLevel); + } + else + { + PyErr_SetString (PyExc_RuntimeError, + "Inconsistent LZ4 library version/available APIs"); + } + } + else + { + LZ4_resetStreamHC (streamHCPtr, compressionLevel); + } +} + + +/************************* + * Double-buffer helpers * + *************************/ +static int +double_buffer_update_index (stream_context_t * context) +{ +#if DOUBLE_BUFFER_PAGE_COUNT != 2 +#error "DOUBLE_BUFFER_PAGE_COUNT must be 2." +#endif /* DOUBLE_BUFFER_PAGE_COUNT != 2 */ + context->strategy.data.double_buffer.index = (context->strategy.data.double_buffer.index + 1) & 0x1; /* modulo 2 */ + + return 0; +} + +static char * +double_buffer_get_compression_page (const stream_context_t * context) +{ + return context->strategy.data.double_buffer.pages[context->strategy.data.double_buffer.index]; +} + +static void +double_buffer_release_resources (stream_context_t * context) +{ + unsigned int i; + for (i = DOUBLE_BUFFER_INDEX_MIN; + i < (DOUBLE_BUFFER_INDEX_MIN + DOUBLE_BUFFER_PAGE_COUNT); + ++i) + { + context->strategy.data.double_buffer.pages[i] = NULL; + } + + if (context->strategy.data.double_buffer.buf != NULL) + { + memset (context->strategy.data.double_buffer.buf, + 0x0, + context->strategy.data.double_buffer.page_size * DOUBLE_BUFFER_PAGE_COUNT); + PyMem_Free (context->strategy.data.double_buffer.buf); + } + context->strategy.data.double_buffer.buf = NULL; + context->strategy.data.double_buffer.index = DOUBLE_BUFFER_INDEX_INVALID; + context->strategy.data.double_buffer.page_size = 0; +} + +static int +double_buffer_reserve_resources (stream_context_t * context, unsigned int buffer_size) +{ + int status = 0; + unsigned int i; + + context->strategy.data.double_buffer.page_size = buffer_size; + context->strategy.data.double_buffer.buf = PyMem_Malloc (buffer_size * DOUBLE_BUFFER_PAGE_COUNT); + + if (context->strategy.data.double_buffer.buf == NULL) + { + PyErr_Format (PyExc_MemoryError, + "Could not allocate double-buffer"); + status = -1; + goto exit_now; + } + + memset (context->strategy.data.double_buffer.buf, + 0x0, + context->strategy.data.double_buffer.page_size * DOUBLE_BUFFER_PAGE_COUNT); + + for (i = DOUBLE_BUFFER_INDEX_MIN; + i < (DOUBLE_BUFFER_INDEX_MIN + DOUBLE_BUFFER_PAGE_COUNT); + ++i) + { + context->strategy.data.double_buffer.pages[i] = context->strategy.data.double_buffer.buf + + (i * buffer_size); + } + + context->strategy.data.double_buffer.index = DOUBLE_BUFFER_INDEX_MIN; + +exit_now: + return status; +} + +static unsigned int +double_buffer_get_work_buffer_size (const stream_context_t * context) +{ + return context->strategy.data.double_buffer.page_size; +} + +static unsigned int +double_buffer_get_dest_buffer_size (const stream_context_t * context) +{ + unsigned int len; + + if (context->config.direction == COMPRESS) + { + len = context->output.len; + } + else + { + len = context->strategy.data.double_buffer.page_size; + } + + return len; +} + + +/**************************************** + * Ring-buffer helpers: Not implemented * + ****************************************/ +static void +ring_buffer_release_resources (stream_context_t * context) +{ + (void) context; /* unused */ + + /* Not implemented (yet) */ + PyErr_Format (PyExc_NotImplementedError, + "Buffer strategy not implemented: ring_buffer"); + + return; +} + +static int +ring_buffer_reserve_resources (stream_context_t * context, unsigned int buffer_size) +{ + (void) context; /* unused */ + (void) buffer_size; /* unused */ + + /* Not implemented (yet) */ + PyErr_Format (PyExc_NotImplementedError, + "Buffer strategy not implemented: ring_buffer"); + return -1; +} + +static unsigned int +ring_buffer_get_dest_buffer_size (const stream_context_t * context) +{ + (void) context; /* unused */ + + /* Not implemented (yet) */ + PyErr_Format (PyExc_NotImplementedError, + "Buffer strategy not implemented: ring_buffer"); + + return 0; +} + +static unsigned int +ring_buffer_get_work_buffer_size (const stream_context_t * context) +{ + (void) context; /* unused */ + + /* Not implemented (yet) */ + PyErr_Format (PyExc_NotImplementedError, + "Buffer strategy not implemented: ring_buffer"); + + return 0; +} + +static char * +ring_buffer_get_buffer_position (const stream_context_t * context) +{ + (void) context; /* unused */ + + /* Not implemented (yet) */ + PyErr_Format (PyExc_NotImplementedError, + "Buffer strategy not implemented: ring_buffer"); + + return NULL; +} + +static int +ring_buffer_update_context (stream_context_t * context) +{ + (void) context; /* unused */ + + /* Not implemented (yet) */ + PyErr_Format (PyExc_NotImplementedError, + "Buffer strategy not implemented: ring_buffer"); + + return -1; +} + + +/********************** + * strategy operators * + **********************/ +static strategy_ops_t strategy_ops[BUFFER_STRATEGY_COUNT] = { + /* [DOUBLE_BUFFER] = */ + { + /* .release_resources */ double_buffer_release_resources, + /* .reserve_resources */ double_buffer_reserve_resources, + /* .get_work_buffer */ double_buffer_get_compression_page, + /* .get_work_buffer_size */ double_buffer_get_work_buffer_size, + /* .get_dest_buffer_size */ double_buffer_get_dest_buffer_size, + /* .update_context_after_process */ double_buffer_update_index, + }, + /* [RING_BUFFER] = */ + { + /* .release_resources */ ring_buffer_release_resources, + /* .reserve_resources */ ring_buffer_reserve_resources, + /* .get_work_buffer */ ring_buffer_get_buffer_position, + /* .get_work_buffer_size */ ring_buffer_get_work_buffer_size, + /* .get_dest_buffer_size */ ring_buffer_get_dest_buffer_size, + /* .update_context_after_process */ ring_buffer_update_context, + }, +}; + + +/******************* + * generic helpers * + *******************/ +static void +destroy_context (stream_context_t * context) +{ + if (context == NULL) + { + return; + } + + /* Release lz4 state */ + Py_BEGIN_ALLOW_THREADS + if (context->lz4_state.context != NULL) + { + if (context->config.direction == COMPRESS) + { + if (context->config.comp == HIGH_COMPRESSION) + { + LZ4_freeStreamHC (context->lz4_state.compress.hc); + } + else + { + LZ4_freeStream (context->lz4_state.compress.fast); + } + } + else /* context->config.direction == DECOMPRESS */ + { + LZ4_freeStreamDecode (context->lz4_state.decompress); + } + } + Py_END_ALLOW_THREADS + context->lz4_state.context = NULL; + + /* Release strategy resources */ + if (context->strategy.ops != NULL) + { + context->strategy.ops->release_resources (context); + } + context->strategy.ops = NULL; + + /* Release output buffer */ + if (context->output.buf != NULL) + { + memset (context->output.buf, 0x0, context->output.len); + PyMem_Free (context->output.buf); + } + context->output.buf = NULL; + context->output.len = 0; + + /* Release python memory */ + memset (context, 0x0, sizeof (stream_context_t)); + PyMem_Free (context); +} + +static void +destroy_py_context (PyObject * py_context) +{ + if (py_context == NULL) + { + return; + } + + destroy_context (_PyCapsule_get_context (py_context)); +} + + +/************** + * Python API * + **************/ +static PyObject * +_create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) +{ + stream_context_t * context = NULL; + + const char * direction = ""; + const char * strategy_name = ""; + unsigned int buffer_size; + + buffer_strategy_e strategy = BUFFER_STRATEGY_COUNT; + const char * mode = "default"; + int acceleration = 1; + int compression_level = 9; + int store_comp_size = 4; + int return_bytearray = 0; + Py_buffer dict = { NULL, NULL, }; + + int status = 0; + int total_size = 0; + + static char * argnames[] = { + "strategy", + "direction", + "buffer_size", + "mode", + "acceleration", + "compression_level", + "return_bytearray", + "store_comp_size", + "dictionary", + NULL + }; + +#if IS_PY3 +#define _ARG_FMT "ssI|sIIpIz*" +#else +#define _ARG_FMT "ssI|sIIiIz*" +#endif + if (!PyArg_ParseTupleAndKeywords (args, kwds, _ARG_FMT, argnames, + &strategy_name, &direction, &buffer_size, + &mode, &acceleration, &compression_level, &return_bytearray, + &store_comp_size, &dict)) + { + goto abort_now; + } +#undef _ARG_FMT + + /* Sanity checks on arguments */ + if (dict.len > INT_MAX) + { + PyErr_Format (PyExc_OverflowError, + "Dictionary too large for LZ4 API"); + goto abort_now; + } + + /* Input max length limited to 0x7E000000 (2 113 929 216 bytes < 2GiB). + * https://github.com/lz4/lz4/blob/dev/lib/lz4.h#L161 + * + * So, restrict the block length bitwise to 32 (signed 32 bit integer). */ + if ((store_comp_size != 1) && (store_comp_size != 2) && (store_comp_size != 4)) + { + PyErr_Format (PyExc_ValueError, + "Invalid store_comp_size, valid values: 1, 2 or 4"); + goto abort_now; + } + + context = (stream_context_t *) PyMem_Malloc (sizeof (stream_context_t)); + if (context == NULL) + { + PyErr_NoMemory (); + goto abort_now; + } + + memset (context, 0x00, sizeof (stream_context_t)); + + /* Set buffer strategy */ + if (!strncmp (strategy_name, "double_buffer", sizeof ("double_buffer"))) + { + strategy = DOUBLE_BUFFER; + } + else if (!strncmp (strategy_name, "ring_buffer", sizeof ("ring_buffer"))) + { + strategy = RING_BUFFER; + } + else + { + PyErr_Format (PyExc_ValueError, + "Invalid strategy argument: %s. Must be one of: double_buffer, ring_buffer", + strategy_name); + goto abort_now; + } + + /* Set direction */ + if (!strncmp (direction, "compress", sizeof ("compress"))) + { + context->config.direction = COMPRESS; + } + else if (!strncmp (direction, "decompress", sizeof ("decompress"))) + { + context->config.direction = DECOMPRESS; + } + else + { + PyErr_Format (PyExc_ValueError, + "Invalid direction argument: %s. Must be one of: compress, decompress", + direction); + goto abort_now; + } + + /* Set compression mode */ + if (!strncmp (mode, "default", sizeof ("default"))) + { + context->config.comp = DEFAULT; + } + else if (!strncmp (mode, "fast", sizeof ("fast"))) + { + context->config.comp = FAST; + } + else if (!strncmp (mode, "high_compression", sizeof ("high_compression"))) + { + context->config.comp = HIGH_COMPRESSION; + } + else + { + PyErr_Format (PyExc_ValueError, + "Invalid mode argument: %s. Must be one of: default, fast, high_compression", + mode); + goto abort_now; + } + + /* Initialize the output buffer */ + if (context->config.direction == COMPRESS) + { + uint32_t store_max_size = _GET_MAX_UINT32(store_comp_size); + + context->output.len = get_compress_bound (buffer_size); + total_size = context->output.len + context->config.store_comp_size; + + if (context->output.len == 0) + { + PyErr_Format (PyExc_ValueError, + "Invalid buffer_size argument: %u. Cannot define output buffer size. " + "Must be lesser or equal to %u", + buffer_size, LZ4_MAX_INPUT_SIZE); + goto abort_now; + } + + /* Assert the output buffer size and the store_comp_size values are consistent */ + if (context->output.len > store_max_size) + { + /* The maximal/"worst case" compressed data length cannot fit in the + * store_comp_size bytes. */ + PyErr_Format (LZ4StreamError, + "Inconsistent buffer_size/store_comp_size values. " + "Maximal compressed length (%u) cannot fit in a %u byte-long integer", + buffer_size, store_comp_size); + goto abort_now; + } + } + else /* context->config.direction == DECOMPRESS */ + { + uint32_t store_max_size = _GET_MAX_UINT32(store_comp_size); + if (store_max_size > LZ4_MAX_INPUT_SIZE) + { + store_max_size = LZ4_MAX_INPUT_SIZE; + } + + context->output.len = buffer_size; + total_size = context->output.len; + + /* Here we cannot assert the maximal theorical decompressed chunk length + * will fit in one page of the double_buffer, i.e.: + * assert( !(double_buffer.page_size < _LZ4_inputBound(store_max_size)) ) + * + * Doing such a check would require aligning the page_size on the maximal + * value of the store_comp_size prefix, i.e.: + * page_size = 256B if store_comp_size == 1 + * page_size = 64KB if store_comp_size == 2 + * page_size = 4GB if store_comp_size == 4 + * + * This constraint is too strict, so is not implemented. + * + * On the other hand, the compression logic tells the page size cannot be + * larger than the maximal value fitting in store_comp_size bytes. + * So here, the check could be checking the page_size is smaller or equal + * to the maximal decompressed chunk length, i.e.: + * assert( !(double_buffer.page_size > _LZ4_inputBound(store_max_size)) ) + * + * But this check is not really relevant and could bring other limitations. + * + * So, on the decompression case, no check regarding the page_size and the + * store_comp_size values can reliably be done during the LZ4 context + * initialization, they will be deferred in the decompression process. + */ + } + + /* Set all remaining settings in the context */ + context->config.store_comp_size = store_comp_size; + context->config.acceleration = acceleration; + context->config.compression_level = compression_level; + context->config.return_bytearray = !!return_bytearray; + + /* Set internal resources related to the buffer strategy */ + context->strategy.ops = &strategy_ops[strategy]; + + status = context->strategy.ops->reserve_resources (context, buffer_size); + if (status != 0) + { + /* Python exception already set in the strategy's resource creation helper */ + goto abort_now; + } + + /* Set output buffer */ + context->output.buf = PyMem_Malloc (total_size * sizeof (* (context->output.buf))); + if (context->output.buf == NULL) + { + PyErr_Format (PyExc_MemoryError, + "Could not allocate output buffer"); + goto abort_now; + } + + /* Initialize lz4 state */ + if (context->config.direction == COMPRESS) + { + if (context->config.comp == HIGH_COMPRESSION) + { + context->lz4_state.compress.hc = LZ4_createStreamHC (); + if (context->lz4_state.compress.hc == NULL) + { + PyErr_Format (PyExc_MemoryError, + "Could not create LZ4 state"); + goto abort_now; + } + + reset_stream_hc (context->lz4_state.compress.hc, context->config.compression_level); + + if (dict.len > 0) + { + LZ4_loadDictHC (context->lz4_state.compress.hc, dict.buf, dict.len); + } + } + else + { + context->lz4_state.compress.fast = LZ4_createStream (); + if (context->lz4_state.compress.fast == NULL) + { + PyErr_Format (PyExc_MemoryError, + "Could not create LZ4 state"); + goto abort_now; + } + + reset_stream (context->lz4_state.compress.fast); + + if (dict.len > 0) + { + LZ4_loadDict (context->lz4_state.compress.fast, dict.buf, dict.len); + } + } + } + else /* context->config.direction == DECOMPRESS */ + { + context->lz4_state.decompress = LZ4_createStreamDecode (); + if (context->lz4_state.decompress == NULL) + { + PyErr_Format (PyExc_MemoryError, + "Could not create LZ4 state"); + goto abort_now; + } + + if (!LZ4_setStreamDecode (context->lz4_state.decompress, dict.buf, dict.len)) + { + PyErr_Format (PyExc_RuntimeError, + "Could not initialize LZ4 state"); + LZ4_freeStreamDecode (context->lz4_state.decompress); + goto abort_now; + } + } + + PyBuffer_Release (&dict); + + return PyCapsule_New (context, stream_context_capsule_name, destroy_py_context); + +abort_now: + if (dict.buf != NULL) + { + PyBuffer_Release (&dict); + } + destroy_context (context); + + return NULL; +} + +static PyObject * +_compress_bound (PyObject * Py_UNUSED (self), PyObject * args) +{ + PyObject * py_dest = NULL; + uint32_t input_size; + + + /* Positional arguments: input_size + * Keyword arguments : none + */ +#if IS_PY3 +#define _ARG_FMT "OI" +#else +#define _ARG_FMT "OI" +#endif + if (!PyArg_ParseTuple (args, _ARG_FMT, + &input_size)) + { + goto exit_now; + } +#undef _ARG_FMT + + py_dest = PyLong_FromUnsignedLong (get_compress_bound (input_size)); + + if (py_dest == NULL) + { + PyErr_NoMemory (); + } + +exit_now: + return py_dest; +} + +static PyObject * +_input_bound (PyObject * Py_UNUSED (self), PyObject * args) +{ + PyObject * py_dest = NULL; + uint32_t compress_max_size; + + /* Positional arguments: compress_max_size + * Keyword arguments : none + */ +#if IS_PY3 +#define _ARG_FMT "I" +#else +#define _ARG_FMT "I" +#endif + if (!PyArg_ParseTuple (args, _ARG_FMT, + &compress_max_size)) + { + goto exit_now; + } +#undef _ARG_FMT + + py_dest = PyLong_FromUnsignedLong (get_input_bound (compress_max_size)); + + if (py_dest == NULL) + { + PyErr_NoMemory (); + } + +exit_now: + return py_dest; +} + +static inline int +_compress_generic (stream_context_t * lz4_ctxt, char * source, int source_size, + char * dest, int dest_size) +{ + int comp_len; + + if (lz4_ctxt->config.comp == HIGH_COMPRESSION) + { + LZ4_streamHC_t * lz4_state = lz4_ctxt->lz4_state.compress.hc; + + comp_len = LZ4_compress_HC_continue (lz4_state, source, dest, source_size, dest_size); + } + else + { + LZ4_stream_t * lz4_state = lz4_ctxt->lz4_state.compress.fast; + int acceleration = (lz4_ctxt->config.comp != FAST) + ? 1 /* defaults */ + : lz4_ctxt->config.acceleration; + + comp_len = LZ4_compress_fast_continue (lz4_state, source, dest, source_size, dest_size, + acceleration); + } + + return comp_len; +} + +#ifdef inline +#undef inline +#endif + +static PyObject * +_compress (PyObject * Py_UNUSED (self), PyObject * args) +{ + stream_context_t * context = NULL; + PyObject * py_context = NULL; + PyObject * py_dest = NULL; + int output_size; + Py_buffer source = { NULL, NULL, }; + + /* Positional arguments: capsule_context, source + * Keyword arguments : none + */ +#if IS_PY3 +#define _ARG_FMT "Oy*" +#else +#define _ARG_FMT "Os*" +#endif + if (!PyArg_ParseTuple (args, _ARG_FMT, + &py_context, &source)) + { + goto exit_now; + } +#undef _ARG_FMT + + context = _PyCapsule_get_context (py_context); + if ((context == NULL) || (context->lz4_state.context == NULL)) + { + PyErr_SetString (PyExc_ValueError, "No valid LZ4 stream context supplied"); + goto exit_now; + } + + if (source.len > context->strategy.ops->get_work_buffer_size (context)) + { + PyErr_SetString (PyExc_OverflowError, + "Input too large for LZ4 API"); + goto exit_now; + } + + memcpy (context->strategy.ops->get_work_buffer (context), source.buf, source.len); + + Py_BEGIN_ALLOW_THREADS + + output_size = _compress_generic (context, + context->strategy.ops->get_work_buffer (context), + source.len, + context->output.buf + context->config.store_comp_size, + context->output.len); + + Py_END_ALLOW_THREADS + + if (output_size <= 0) + { + /* No error code set in output_size! */ + PyErr_SetString (LZ4StreamError, + "Compression failed"); + goto exit_now; + } + + if (!store_block_length (output_size, context->config.store_comp_size, context->output.buf)) + { + PyErr_SetString (LZ4StreamError, + "Compressed stream size too large"); + goto exit_now; + } + + output_size += context->config.store_comp_size; + + if (context->config.return_bytearray) + { + py_dest = PyByteArray_FromStringAndSize (context->output.buf, (Py_ssize_t) output_size); + } + else + { + py_dest = PyBytes_FromStringAndSize (context->output.buf, (Py_ssize_t) output_size); + } + + if (py_dest == NULL) + { + PyErr_NoMemory (); + goto exit_now; + } + + if (context->strategy.ops->update_context_after_process (context) != 0) + { + PyErr_Format (PyExc_RuntimeError, "Internal error"); + goto exit_now; + } + +exit_now: + if (source.buf != NULL) + { + PyBuffer_Release (&source); + } + + return py_dest; +} + +static PyObject * +_get_block (PyObject * Py_UNUSED (self), PyObject * args) +{ + stream_context_t * context = NULL; + PyObject * py_context = NULL; + PyObject * py_dest = NULL; + Py_buffer source = { NULL, NULL, }; + buffer_t block = { NULL, 0, }; + + /* Positional arguments: capsule_context, source + * Keyword arguments : none + */ + +#if IS_PY3 +#define _ARG_FMT "Oy*" +#else +#define _ARG_FMT "Os*" +#endif + if (!PyArg_ParseTuple (args, _ARG_FMT, + &py_context, &source)) + { + goto exit_now; + } +#undef _ARG_FMT + + context = _PyCapsule_get_context (py_context); + if ((context == NULL) || (context->lz4_state.context == NULL)) + { + PyErr_SetString (PyExc_ValueError, "No valid LZ4 stream context supplied"); + goto exit_now; + } + + if (source.len > INT_MAX) + { + PyErr_Format (PyExc_OverflowError, + "Input too large for LZ4 API"); + goto exit_now; + } + + if (source.len < context->config.store_comp_size) + { + PyErr_Format (LZ4StreamError, + "Invalid source, too small for holding any block"); + goto exit_now; + } + + block.buf = (char *) source.buf + context->config.store_comp_size; + block.len = load_block_length (context->config.store_comp_size, source.buf); + + if ((source.len - context->config.store_comp_size) < block.len) + { + PyErr_Format (LZ4StreamError, + "Requested input size (%d) larger than source size (%ld)", + block.len, (source.len - context->config.store_comp_size)); + goto exit_now; + } + + if (context->config.return_bytearray) + { + py_dest = PyByteArray_FromStringAndSize (block.buf, (Py_ssize_t) block.len); + } + else + { + py_dest = PyBytes_FromStringAndSize (block.buf, (Py_ssize_t) block.len); + } + + if (py_dest == NULL) + { + PyErr_NoMemory (); + } + +exit_now: + if (source.buf != NULL) + { + PyBuffer_Release (&source); + } + + return py_dest; +} + +static PyObject * +_decompress (PyObject * Py_UNUSED (self), PyObject * args) +{ + stream_context_t * context = NULL; + PyObject * py_context = NULL; + PyObject * py_dest = NULL; + int output_size = 0; + Py_buffer source = { NULL, NULL, }; + + /* Positional arguments: capsule_context, source + * Keyword arguments : none + */ +#if IS_PY3 +#define _ARG_FMT "Oy*" +#else +#define _ARG_FMT "Os*" +#endif + if (!PyArg_ParseTuple (args, _ARG_FMT, + &py_context, &source)) + { + goto exit_now; + } +#undef _ARG_FMT + + context = _PyCapsule_get_context (py_context); + if ((context == NULL) || (context->lz4_state.context == NULL)) + { + PyErr_SetString (PyExc_ValueError, "No valid LZ4 stream context supplied"); + goto exit_now; + } + + if (source.len > _GET_MAX_UINT32(context->config.store_comp_size)) + { + PyErr_Format (PyExc_OverflowError, + "Source length (%ld) too large for LZ4 store_comp_size (%d) value", + source.len, context->config.store_comp_size); + goto exit_now; + } + + if ((get_input_bound (source.len) == 0) || + (get_input_bound (source.len) > context->strategy.ops->get_dest_buffer_size (context))) + { + PyErr_Format (LZ4StreamError, + "Maximal decompressed data (%d) cannot fit in LZ4 internal buffer (%u)", + get_input_bound (source.len), + context->strategy.ops->get_dest_buffer_size (context)); + goto exit_now; + } + + Py_BEGIN_ALLOW_THREADS + + output_size = LZ4_decompress_safe_continue (context->lz4_state.decompress, + (const char *) source.buf, + context->strategy.ops->get_work_buffer (context), + source.len, + context->strategy.ops->get_dest_buffer_size (context)); + + Py_END_ALLOW_THREADS + + if (output_size < 0) + { + /* In case of LZ4 decompression error, output_size holds the error code */ + PyErr_Format (LZ4StreamError, + "Decompression failed. error: %d", + -output_size); + goto exit_now; + } + + if ((unsigned int) output_size > context->output.len) + { + output_size = -1; + PyErr_Format (PyExc_OverflowError, + "Decompressed stream too large for LZ4 API"); + goto exit_now; + } + + memcpy (context->output.buf, + context->strategy.ops->get_work_buffer (context), + output_size); + + if ( context->strategy.ops->update_context_after_process (context) != 0) + { + PyErr_Format (PyExc_RuntimeError, "Internal error"); + goto exit_now; + } + + if (context->config.return_bytearray) + { + py_dest = PyByteArray_FromStringAndSize (context->output.buf, (Py_ssize_t) output_size); + } + else + { + py_dest = PyBytes_FromStringAndSize (context->output.buf, (Py_ssize_t) output_size); + } + + if (py_dest == NULL) + { + PyErr_NoMemory (); + } + +exit_now: + if (source.buf != NULL) + { + PyBuffer_Release (&source); + } + + return py_dest; +} + + +PyDoc_STRVAR (_compress_bound__doc, + "_compress_bound(input_size)\n" \ + "\n" \ + "Provides the maximum size that LZ4 compression may output in a \"worst case\"\n" \ + "scenario (input data not compressible).\n" \ + "This function is primarily useful for memory allocation purposes (destination\n" \ + "buffer size).\n" \ + "\n" \ + "Args:\n" \ + " input_size (int): Input data size.\n" \ + "\n" \ + "Returns:\n" \ + " int: Maximal (worst case) size of the compressed data;\n" \ + " or 0 if the input size is greater than 2 113 929 216.\n"); + +PyDoc_STRVAR (_input_bound__doc, + "_input_bound(compress_max_size)\n" \ + "\n" \ + "Provides the maximum size that LZ4 decompression may output in a \"worst case\"\n" \ + "scenario (compressed data with null compression ratio).\n" \ + "This function is primarily useful for memory allocation purposes (destination\n" \ + "buffer size).\n" \ + "\n" \ + "Args:\n" \ + " compress_max_size (int): Compressed data size.\n" \ + "\n" \ + "Returns:\n" \ + " int: Maximal (worst case) size of the input data;\n" \ + " or 0 if the compressed maximal size is lower than the LZ4 compression\n" \ + " minimal overhead, or if the computed input size is greater than\n" \ + " 2 113 929 216.\n"); + +PyDoc_STRVAR (_compress__doc, + "_compress(context, source)\n" \ + "\n" \ + "Compress source, using the given LZ4 stream context, returning the compressed\n" \ + "data as a bytearray or as a bytes object.\n" \ + "Raises an exception if any error occurs.\n" \ + "\n" \ + "Args:\n" \ + " context (ctx): LZ4 stream context.\n" \ + " source (str, bytes or buffer-compatible object): Data to compress.\n" \ + "\n" \ + "Returns:\n" \ + " bytes or bytearray: Compressed data.\n" \ + "\n" \ + "Raises:\n" \ + " OverflowError: raised if the source is too large for being compressed in\n" \ + " the given context.\n" \ + " RuntimeError: raised if some internal resources cannot be updated.\n" \ + " LZ4StreamError: raised if the call to the LZ4 library fails.\n"); + +PyDoc_STRVAR (_get_block__doc, + "_get_block(context, source)\n" \ + "\n" \ + "Return the first LZ4 compressed block from ``'source'``. \n" \ + "\n" \ + "Args:\n" \ + " context (ctx): LZ4 stream context.\n" \ + " source (str, bytes or buffer-compatible object): LZ4 compressed stream.\n" \ + "\n" \ + "Returns:\n" \ + " bytes or bytearray: LZ4 compressed data block.\n" \ + "\n" \ + "Raises:\n" \ + " MemoryError: raised if the output buffer cannot be allocated.\n" \ + " OverflowError: raised if the source is too large for being handled by \n" \ + " the given context.\n"); + +PyDoc_STRVAR (_decompress__doc, + "_decompress(context, source)\n" \ + "\n" \ + "Decompress source, using the given LZ4 stream context, returning the\n" \ + "uncompressed data as a bytearray or as a bytes object.\n" \ + "Raises an exception if any error occurs.\n" \ + "\n" \ + "Args:\n" \ + " context (obj): LZ4 stream context.\n" \ + " source (str, bytes or buffer-compatible object): Data to uncompress.\n" \ + "\n" \ + "Returns:\n" \ + " bytes or bytearray: Uncompressed data.\n" \ + "\n" \ + "Raises:\n" \ + " ValueError: raised if the source is inconsistent with a finite LZ4\n" \ + " stream block chain.\n" \ + " MemoryError: raised if the work output buffer cannot be allocated.\n" \ + " OverflowError: raised if the source is too large for being decompressed\n" \ + " in the given context.\n" \ + " RuntimeError: raised if some internal resources cannot be updated.\n" \ + " LZ4StreamError: raised if the call to the LZ4 library fails.\n"); + +PyDoc_STRVAR (_create_context__doc, + "_create_context(strategy, direction, buffer_size,\n" \ + " mode='default', acceleration=1, compression_level=9,\n" \ + " return_bytearray=0, store_comp_size=4, dict=None)\n" \ + "\n" \ + "Instantiates and initializes a LZ4 stream context.\n" \ + "Raises an exception if any error occurs.\n" \ + "\n" \ + "Args:\n" \ + " strategy (str): Can be ``'double_buffer'``.\n" \ + " Only ``'double_buffer'`` is currently implemented.\n" \ + " direction (str): Can be ``'compress'`` or ``'decompress'``.\n" \ + " buffer_size (int): Base size of the buffer(s) used internally for stream\n" \ + " compression/decompression.\n" \ + " For the ``'double_buffer'`` strategy, this is the size of each buffer\n" \ + " of the double-buffer.\n" \ + "\n" \ + "Keyword Args:\n" \ + " mode (str): If ``'default'`` or unspecified use the default LZ4\n" \ + " compression mode. Set to ``'fast'`` to use the fast compression\n" \ + " LZ4 mode at the expense of compression. Set to\n" \ + " ``'high_compression'`` to use the LZ4 high-compression mode at\n" \ + " the expense of speed.\n" \ + " acceleration (int): When mode is set to ``'fast'`` this argument\n" \ + " specifies the acceleration. The larger the acceleration, the\n" \ + " faster the but the lower the compression. The default\n" \ + " compression corresponds to a value of ``1``.\n" \ + " Only relevant if ``'direction'`` is ``'compress'``.\n" \ + " compression_level (int): When mode is set to ``high_compression`` this\n" \ + " argument specifies the compression. Valid values are between\n" \ + " ``1`` and ``12``. Values between ``4-9`` are recommended, and\n" \ + " ``9`` is the default.\n" \ + " Only relevant if ``'direction'`` is ``'compress'`` and ``'mode'`` .\n" \ + " is ``'high_compression'``.\n" \ + " return_bytearray (bool): If ``False`` (the default) then the function\n" \ + " will return a bytes object. If ``True``, then the function will\n" \ + " return a bytearray object.\n" \ + " store_comp_size (int): Specify the size in bytes of the following\n" \ + " compressed block. Can be: ``1``, ``2`` or ``4`` (default: ``4``).\n" \ + " dict (str, bytes or buffer-compatible object): If specified, perform\n" \ + " compression using this initial dictionary.\n" \ + "\n" \ + "Returns:\n" \ + " lz4_ctx: A LZ4 stream context.\n" \ + "\n" \ + "Raises:\n" \ + " OverflowError: raised if the ``dict`` parameter is too large for the\n" \ + " LZ4 context.\n" \ + " ValueError: raised if some parameters are invalid.\n" \ + " MemoryError: raised if some internal resources cannot be allocated.\n" \ + " RuntimeError: raised if some internal resources cannot be initialized.\n"); + +PyDoc_STRVAR (lz4stream__doc, + "A Python wrapper for the LZ4 stream protocol" + ); + +static PyMethodDef module_methods[] = { + { + "_create_context", (PyCFunction) _create_context, + METH_VARARGS | METH_KEYWORDS, + _create_context__doc + }, + + { + "_compress", + (PyCFunction) _compress, + METH_VARARGS, + _compress__doc + }, + { + "_decompress", + (PyCFunction) _decompress, + METH_VARARGS, + _decompress__doc + }, + { + "_get_block", + (PyCFunction) _get_block, + METH_VARARGS, + _get_block__doc + }, + { + "_compress_bound", + (PyCFunction) _compress_bound, + METH_VARARGS, + _compress_bound__doc + }, + { + "_input_bound", + (PyCFunction) _input_bound, + METH_VARARGS, + _input_bound__doc + }, + { + /* Sentinel */ + NULL, + NULL, + 0, + NULL + } +}; + +static PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + /* m_name */ "_stream", + /* m_doc */ lz4stream__doc, + /* m_size */ -1, + /* m_methods */ module_methods, +}; + + +MODULE_INIT_FUNC (_stream) +{ + PyObject * module = NULL; + +#if IS_PY3 + module = PyModule_Create (&moduledef); +#else + (void) moduledef; /* unused in python2 */ + module = Py_InitModule("_stream", module_methods); +#endif + + if (module == NULL) + { + return NULL; + } + + PyModule_AddIntConstant (module, "HC_LEVEL_MIN", LZ4HC_CLEVEL_MIN); + PyModule_AddIntConstant (module, "HC_LEVEL_DEFAULT", LZ4HC_CLEVEL_DEFAULT); + PyModule_AddIntConstant (module, "HC_LEVEL_OPT_MIN", LZ4HC_CLEVEL_OPT_MIN); + PyModule_AddIntConstant (module, "HC_LEVEL_MAX", LZ4HC_CLEVEL_MAX); + PyModule_AddIntConstant (module, "LZ4_MAX_INPUT_SIZE", LZ4_MAX_INPUT_SIZE); + + LZ4StreamError = PyErr_NewExceptionWithDoc ("_stream.LZ4StreamError", + "Call to LZ4 library failed.", + NULL, NULL); + if (LZ4StreamError == NULL) + { + return NULL; + } + Py_INCREF (LZ4StreamError); + PyModule_AddObject (module, "LZ4StreamError", LZ4StreamError); + + return module; +} diff --git a/setup.py b/setup.py index 9fa31318..95b479d3 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,10 @@ def pkgconfig_installed_check(lib, required_version, default): 'lz4/frame/_frame.c' ] +lz4stream_sources = [ + 'lz4/stream/_stream.c' +] + if liblz4_found is True: libraries.append('lz4') else: @@ -82,6 +86,12 @@ def pkgconfig_installed_check(lib, required_version, default): 'lz4libs/xxhash.c', ] ) + lz4stream_sources.extend( + [ + 'lz4libs/lz4.c', + 'lz4libs/lz4hc.c', + ] + ) if py3c_found is False: include_dirs.append('py3c') @@ -129,6 +139,13 @@ def pkgconfig_installed_check(lib, required_version, default): libraries=libraries, include_dirs=include_dirs) +lz4stream = Extension('lz4.stream._stream', + lz4stream_sources, + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + libraries=libraries, + include_dirs=include_dirs) + install_requires = [] # On Python earlier than 3.0 the builtins package isn't included, but it is @@ -174,7 +191,8 @@ def pkgconfig_installed_check(lib, required_version, default): ext_modules=[ lz4version, lz4block, - lz4frame + lz4frame, + lz4stream ], tests_require=tests_require, extras_require={ From 14e19749ae2697e1c68b501359a569deac82c585 Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Mon, 13 May 2019 17:47:55 +0200 Subject: [PATCH 100/280] Add documentation for stream module --- docs/intro.rst | 4 ++-- docs/lz4.rst | 6 ++--- docs/lz4.stream.rst | 57 +++++++++++++++++++++++++++++++++++++++++++++ docs/quickstart.rst | 40 ++++++++++++++++++++++++++++++- docs/userguide.rst | 1 + 5 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 docs/lz4.stream.rst diff --git a/docs/intro.rst b/docs/intro.rst index 4bf7e2af..766950f5 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -11,8 +11,8 @@ The LZ4 library provides support for three specifications: * The `block `_ format * The `stream `_ format -This Python interface currently supports the frame and block formats. Support -for the streaming format will be available in a future release. +This Python interface currently supports the frame, block and double-buffer +stream formats. For most applications, the frame format is what you should use as this guarantees interoperability with other bindings. The frame format defines a diff --git a/docs/lz4.rst b/docs/lz4.rst index fbf0ec62..709ba17c 100644 --- a/docs/lz4.rst +++ b/docs/lz4.rst @@ -2,8 +2,8 @@ lz4 package =========== -Most of the functionality of this package is found in the :py:mod:`lz4.frame` -and the :py:mod:`lz4.block` sub-packages. +Most of the functionality of this package is found in the :py:mod:`lz4.frame`, +the :py:mod:`lz4.block` and the :py:mod:`lz4.stream` sub-packages. Contents -------- @@ -13,5 +13,3 @@ Contents library_version_number, library_version_string, __version__ - - diff --git a/docs/lz4.stream.rst b/docs/lz4.stream.rst new file mode 100644 index 00000000..dc190a55 --- /dev/null +++ b/docs/lz4.stream.rst @@ -0,0 +1,57 @@ +.. default-role:: obj + + +lz4.stream sub-package +====================== + +This sub-package provides the capability to compress and decompress data using +the `stream specification +`_, +especially the `stream specification based on a double buffer +`_. + +Because the LZ4 stream format does not define a container format, the +Python bindings will by default insert the compressed data size as an +integer at the start of the compressed payload. However, it is +possible to set the bit depth of this compressed data size. + +So far, only the double-buffer based approach is implemented. + +Example usage +------------- +To use the lz4 stream format bindings is straightforward: + +.. doctest:: + + >>> from lz4.stream import LZ4StreamCompressor, LZ4StreamDecompressor + >>> import os + >>> block_size_length = 2 # LZ4 compressed block size stored on 2 bytes + >>> page_size = 8192 # LZ4 context double buffer page size + >>> origin_stream = 10 * 1024 * os.urandom(1024) # 10MiB + >>> # LZ4 stream compression of origin_stream into compressed_stream: + >>> compressed_stream = bytearray() + >>> with LZ4StreamCompressor("double_buffer", page_size, store_comp_size=block_size_length) as proc: + ... offset = 0 + ... while offset < len(origin_stream): + ... chunk = origin_stream[offset:offset + page_size] + ... block = proc.compress(chunk) + ... compressed_stream.extend(block) + ... offset += page_size + >>> # LZ4 stream decompression of compressed_stream into decompressed_stream: + >>> decompressed_stream = bytearray() + >>> with LZ4StreamDecompressor("double_buffer", page_size, store_comp_size=block_size_length) as proc: + ... offset = 0 + ... while offset < len(compressed_stream): + ... block = proc.get_block(compressed_stream[offset:]) + ... chunk = proc.decompress(block) + ... decompressed_stream.extend(chunk) + ... offset += block_size_length + len(block) + >>> decompressed_stream == origin_stream + True + + +Contents +---------------- + +.. automodule:: lz4.stream + :members: LZ4StreamCompressor, LZ4StreamDecompressor diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 8c4e3a52..a39cc3b2 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -218,7 +218,7 @@ This is the default. When ``auto_flush`` is ``True``, the compression functions will return compressed data immediately. - + Availability: :py:func:`lz4.frame.compress()`, :py:func:`lz4.frame.compress_begin()`, :py:func:`lz4.frame.open()`, :py:class:`lz4.frame.LZ4FrameCompressor`, :py:class:`lz4.frame.LZ4FrameFile`. @@ -244,3 +244,41 @@ header. Availability of ``source_size``: :py:meth:`lz4.frame.LZ4FrameCompressor.begin()`, :py:func:`lz4.frame.compress_begin()`, :py:func:`lz4.frame.open()`, :py:class:`lz4.frame.LZ4FrameFile`. + + +Working with streamed compressed data +------------------------------------- + +The stream bindings provide capability for working with stream compressed LZ4 +data. This functionality is based on the usage of a ring-buffer (not implemented +yet) or a double-buffer, with the length of each block preceding the compressed +payload in the stream. + +The stream compression reuses a context between each processed block for +performance gain. + +Most of the arguments used to initialize the LZ4 stream context are shared with +the block API. Hereafter, those specific to the LZ4 stream API are detailed. + + +Controlling the buffer size +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``buffer_size`` argument represents the base buffer size used internally for +memory allocation: + +* In the case of the double-buffer strategy, this is the size of each buffer of + the double-buffer. + +When compressing, this size is the maximal length of the input uncompressed +chunks. + +When decompressing, this size is the maximal length of the decompressed data. + + +Storing the compressed data size in the block +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``store_comp_size`` argument allows tuning of the size (in bytes) of the +compressed block, which is prepended to the actual LZ4 compressed payload. +This size can be either on ``1``, ``2`` or ``4`` bytes. diff --git a/docs/userguide.rst b/docs/userguide.rst index a8786354..9ec4b41d 100644 --- a/docs/userguide.rst +++ b/docs/userguide.rst @@ -6,3 +6,4 @@ User Guide lz4 lz4.frame lz4.block + lz4.stream From ec4a9164daa988c7100485fa7c7220240d2533bd Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Wed, 15 May 2019 14:22:25 +0200 Subject: [PATCH 101/280] Add tests for stream bindings --- tests/stream/conftest.py | 154 ++++++++ tests/stream/numpy_byte_array.bin | Bin 0 -> 8552 bytes tests/stream/test_stream_0.py | 116 ++++++ tests/stream/test_stream_1.py | 617 ++++++++++++++++++++++++++++++ tests/stream/test_stream_2.py | 148 +++++++ tests/stream/test_stream_3.py | 137 +++++++ 6 files changed, 1172 insertions(+) create mode 100644 tests/stream/conftest.py create mode 100644 tests/stream/numpy_byte_array.bin create mode 100644 tests/stream/test_stream_0.py create mode 100644 tests/stream/test_stream_1.py create mode 100644 tests/stream/test_stream_2.py create mode 100644 tests/stream/test_stream_3.py diff --git a/tests/stream/conftest.py b/tests/stream/conftest.py new file mode 100644 index 00000000..ea9fa83d --- /dev/null +++ b/tests/stream/conftest.py @@ -0,0 +1,154 @@ +import pytest +import os +import sys + +test_data = [ + (b''), + (os.urandom(8 * 1024)), + (b'0' * 8 * 1024), + (bytearray(b'')), + (bytearray(os.urandom(8 * 1024))), + (bytearray(open(os.path.join(os.path.dirname(__file__), 'numpy_byte_array.bin'), 'rb').read())) +] + +if sys.version_info > (2, 7): + test_data += [ + (memoryview(b'')), + (memoryview(os.urandom(8 * 1024))) + ] + + +@pytest.fixture( + params=test_data, + ids=[ + 'data' + str(i) for i in range(len(test_data)) + ] +) +def data(request): + return request.param + + +@pytest.fixture( + params=[ + ("double_buffer"), + # ("ring_buffer"), # not implemented + ] +) +def strategy(request): + return request.param + + +test_buffer_size = sorted( + [1, + # 4, + # 8, + # 64, + 256, + 941, + 1 * 1024, + # 4 * 1024, + # 8 * 1024, + # 16 * 1024, + # 32 * 1024, + 64 * 1024, + 128 * 1024] +) + + +@pytest.fixture( + params=test_buffer_size, + ids=[ + 'buffer_size' + str(i) for i in range(len(test_buffer_size)) + ] +) +def buffer_size(request): + return request.param + + +@pytest.fixture( + params=[ + ( + { + 'store_comp_size': 1 + } + ), + ( + { + 'store_comp_size': 2 + } + ), + ( + { + 'store_comp_size': 4 + } + ), + ] +) +def store_comp_size(request): + return request.param + + +@pytest.fixture( + params=[ + ( + { + 'return_bytearray': True + } + ), + ( + { + 'return_bytearray': False + } + ), + ] +) +def return_bytearray(request): + return request.param + + +@pytest.fixture +def c_return_bytearray(return_bytearray): + return return_bytearray + + +@pytest.fixture +def d_return_bytearray(return_bytearray): + return return_bytearray + + +@pytest.fixture( + params=[ + ('default', None) + ] + [ + ('fast', None) + ] + [ + ('fast', {'acceleration': s}) for s in range(10) + ] + [ + ('high_compression', None) + ] + [ + ('high_compression', {'compression_level': s}) for s in range(17) + ] + [ + (None, None) + ] +) +def mode(request): + return request.param + + +dictionary = [ + None, + (0, 0), + (100, 200), + (0, 8 * 1024), + os.urandom(8 * 1024) +] + + +@pytest.fixture( + params=dictionary, + ids=[ + 'dictionary' + str(i) for i in range(len(dictionary)) + ] +) +def dictionary(request): + return request.param diff --git a/tests/stream/numpy_byte_array.bin b/tests/stream/numpy_byte_array.bin new file mode 100644 index 0000000000000000000000000000000000000000..49537e2d90a5161ad8fcc847c696b7101b1cdd4c GIT binary patch literal 8552 zcmeI2T}V@57{_f9VFX2V6}0rCQo__^L>F})V;>MjC`<>5w!#Vw6#If!1W`n3IlC*0 z0)?ODQxY0lw)^`{VH^Bu91 zR>s!Fz<#vWnT?b8k-W}Et(V^OMjx@~#vSY5=SP+qKL@NkC|ZAR=Q5Aub}n;&kvv*= ze7AXE2dO2lqm`bO_Z|L|5h>ZsiM=9Hs&q*R88O>Cm%m1!Vul0zxFP}8FFVOEI z5#6T27Kd!to1*I*t=Fqht$k>%2iFtD!wzyk=+l+FA1;gf;(lB{)zr1TWXejZN@wG` z+!?QvDz=XnhC*)L+46S0()8LzV5-g~TbH)`R_pg0nLh8@z_aHkf>uhDd`_Z#dH#u2 zi$hl8ln(d~zxZk;&M);vhpx0cNiBJNqB5_u`P!n}-xkQUK36N(uP~eZ z({5T_+7a9&zn`ZHYcIVzxJEu_1b061dfJ?EpNE!~-P+O`u#*yl-48#%o3axK|NM;& z1y#ik0vC3j8M;~GAWqmp*pEzH>@F|0ky>7DI5rn8wh<@dxgIj?pw$2I(Yg0_Qp>#Z z@cuU+#Qu8QQ75hg^3Sh)U%)fy}97=l)z4#X}GB zbC&3N%>9&PQJzHgqWdOO^tk^sr5~gF!2CoM)dPQ#Jze#DpQ`OcDK1ev>QKr9KPB=! cT;}n~{v8DQ (2, 7): + if isinstance(buff, memoryview): + b = buff.tobytes() + else: + b = bytes(buff) + else: + b = bytes(buff) + + if len(b) < block_length_size: + return None + + if sys.version_info > (3, 2): + return int.from_bytes(b[:block_length_size], 'little') + else: + # This would not work on a memoryview object, hence buff.tobytes call + # above + fmt = {1: 'B', 2: 'H', 4: 'I', } + return struct.unpack('<' + fmt[block_length_size], b[:block_length_size])[0] + + +def roundtrip(x, c_kwargs, d_kwargs, dictionary): + if dictionary: + if isinstance(dictionary, tuple): + dict_ = x[dictionary[0]:dictionary[1]] + else: + dict_ = dictionary + c_kwargs['dictionary'] = dict_ + d_kwargs['dictionary'] = dict_ + + c = bytes() + with lz4.stream.LZ4StreamCompressor(**c_kwargs) as proc: + for start in range(0, len(x), c_kwargs['buffer_size']): + chunk = x[start:start + c_kwargs['buffer_size']] + assert len(chunk) <= c_kwargs['buffer_size'] + block = proc.compress(chunk) + if c_kwargs.get('return_bytearray'): + assert isinstance(block, bytearray) + if start == 0: + c = block + else: + c += block + assert get_stored_size(block, c_kwargs['store_comp_size']) == \ + (len(block) - c_kwargs['store_comp_size']) + + d = bytes() + with lz4.stream.LZ4StreamDecompressor(**d_kwargs) as proc: + start = 0 + while start < len(c): + block = proc.get_block(c[start:]) + chunk = proc.decompress(block) + if d_kwargs.get('return_bytearray'): + assert isinstance(chunk, bytearray) + if start == 0: + d = chunk + else: + d += chunk + start += d_kwargs['store_comp_size'] + len(block) + + return d + + +def setup_kwargs(strategy, mode, buffer_size, store_comp_size, + c_return_bytearray=None, d_return_bytearray=None): + c_kwargs = {} + + if mode[0] is not None: + c_kwargs['mode'] = mode[0] + if mode[1] is not None: + c_kwargs.update(mode[1]) + + c_kwargs['strategy'] = strategy + c_kwargs['buffer_size'] = buffer_size + c_kwargs.update(store_comp_size) + + if(c_return_bytearray): + c_kwargs.update(c_return_bytearray) + + d_kwargs = {} + + if(d_return_bytearray): + d_kwargs.update(d_return_bytearray) + + d_kwargs['strategy'] = strategy + d_kwargs['buffer_size'] = buffer_size + d_kwargs.update(store_comp_size) + + return (c_kwargs, d_kwargs) + + +# Test single threaded usage with all valid variations of input +def test_1(data, strategy, mode, buffer_size, store_comp_size, + c_return_bytearray, d_return_bytearray, dictionary): + if buffer_size >= (1 << (8 * store_comp_size['store_comp_size'])): + pytest.skip("Invalid case: buffer_size too large for the block length area") + + (c_kwargs, d_kwargs) = setup_kwargs( + strategy, mode, buffer_size, store_comp_size, c_return_bytearray, d_return_bytearray) + + d = roundtrip(data, c_kwargs, d_kwargs, dictionary) + + assert d == data + + +# Test multi threaded: +# Not relevant in the lz4.stream case (the process is highly sequential, +# and re-use/share the same context from one input chunk to the next one). +def test_2(data, strategy, mode, buffer_size, store_comp_size, dictionary): # noqa + pass diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py new file mode 100644 index 00000000..3386b050 --- /dev/null +++ b/tests/stream/test_stream_1.py @@ -0,0 +1,617 @@ +import lz4.stream +import pytest +import sys +import os +import psutil + +if sys.version_info < (3, ): + from struct import pack, unpack + + def _get_format(length, byteorder, signed): + _order = {'l': '<', 'b': '>'} + _fmt = {1: 'b', 2: 'h', 4: 'i', 8: 'q'} + _sign = {True: lambda x: x.lower(), False: lambda x: x.upper()} + return _sign[signed](_order[byteorder[0].lower()] + _fmt[length]) + + def int_to_bytes(value, length=4, byteorder='little', signed=False): + return bytearray(pack(_get_format(length, byteorder, signed), value)) + + def int_from_bytes(bytes, byteorder='little', signed=False): + return unpack(_get_format(len(bytes), byteorder, signed), bytes)[0] + +else: + def int_to_bytes(value, length=4, byteorder='little', signed=False): + return value.to_bytes(length, byteorder, signed=signed) + + def int_from_bytes(bytes, byteorder='little', signed=False): + return int.from_bytes(bytes, byteorder, signed=signed) + + +# This test requires allocating a big lump of memory. In order to +# avoid a massive memory allocation during byte compilation, we have +# to declare a variable for the size of the buffer we're going to +# create outside the scope of the function below. See: +# https://bugs.python.org/issue21074 +_4GB = 0x100000000 # 4GB + +# This test will be killed on Travis due to the 3GB memory limit +# there. Unfortunately psutil reports the host memory, not the memory +# available to the container, and so can't be used to detect available +# memory, so instead, as an ugly hack for detecting we're on Travis we +# check for the TRAVIS environment variable being set. This is quite +# fragile. + + +def compress(x, c_kwargs, return_block_offset=False, check_block_type=False): + o = [0, ] + if c_kwargs.get('return_bytearray', False): + c = bytearray() + else: + c = bytes() + with lz4.stream.LZ4StreamCompressor(**c_kwargs) as proc: + for start in range(0, len(x), c_kwargs['buffer_size']): + chunk = x[start:start + c_kwargs['buffer_size']] + block = proc.compress(chunk) + c += block + if return_block_offset: + o.append(len(c)) + if check_block_type: + assert isinstance(block, c.__class__) + if return_block_offset: + return c, o + else: + return c + + +def decompress(x, d_kwargs, check_chunk_type=False): + if d_kwargs.get('return_bytearray', False): + d = bytearray() + else: + d = bytes() + with lz4.stream.LZ4StreamDecompressor(**d_kwargs) as proc: + start = 0 + while start < len(x): + block = proc.get_block(x[start:]) + chunk = proc.decompress(block) + d += chunk + start += d_kwargs['store_comp_size'] + len(block) + if check_chunk_type: + assert isinstance(chunk, d.__class__) + return d + + +def test_invalid_config_c_1(): + c_kwargs = {} + c_kwargs['strategy'] = "ring_buffer" + c_kwargs['buffer_size'] = 1024 + + with pytest.raises(NotImplementedError): + lz4.stream.LZ4StreamCompressor(**c_kwargs) + + +def test_invalid_config_d_1(store_comp_size): + d_kwargs = {} + d_kwargs['strategy'] = "ring_buffer" + d_kwargs['buffer_size'] = 1024 + + with pytest.raises(NotImplementedError): + lz4.stream.LZ4StreamDecompressor(**d_kwargs) + + +def test_invalid_config_c_2(): + c_kwargs = {} + c_kwargs['strategy'] = "foo" + c_kwargs['buffer_size'] = 1024 + + with pytest.raises(ValueError): + lz4.stream.LZ4StreamCompressor(**c_kwargs) + + +def test_invalid_config_d_2(store_comp_size): + d_kwargs = {} + d_kwargs['strategy'] = "foo" + d_kwargs['buffer_size'] = 1024 + + with pytest.raises(ValueError): + lz4.stream.LZ4StreamDecompressor(**d_kwargs) + + +def test_invalid_config_c_3(store_comp_size): + c_kwargs = {} + c_kwargs['strategy'] = "double_buffer" + c_kwargs['buffer_size'] = 1024 + c_kwargs['store_comp_size'] = store_comp_size['store_comp_size'] + 5 + + with pytest.raises(ValueError): + lz4.stream.LZ4StreamCompressor(**c_kwargs) + + +def test_invalid_config_d_3(store_comp_size): + d_kwargs = {} + d_kwargs['strategy'] = "double_buffer" + d_kwargs['buffer_size'] = 1024 + d_kwargs['store_comp_size'] = store_comp_size['store_comp_size'] + 5 + + with pytest.raises(ValueError): + lz4.stream.LZ4StreamDecompressor(**d_kwargs) + + +def test_invalid_config_c_4(store_comp_size): + c_kwargs = {} + c_kwargs['strategy'] = "double_buffer" + c_kwargs['buffer_size'] = 1 << (8 * store_comp_size['store_comp_size']) + c_kwargs.update(store_comp_size) + + if store_comp_size['store_comp_size'] >= 4: + + if os.environ.get('TRAVIS') is not None: + pytest.skip('Skipping test on Travis due to insufficient memory') + + if sys.maxsize < 0xffffffff: + pytest.skip('Py_ssize_t too small for this test') + + if psutil.virtual_memory().total < 3 * c_kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer) + pytest.skip('Insufficient system memory for this test') + + # Make sure the page size is larger than what the input bound will be, + # but still fit in 4 bytes + c_kwargs['buffer_size'] -= 1 + + if c_kwargs['buffer_size'] > lz4.stream.LZ4_MAX_INPUT_SIZE: + message = r"^Invalid buffer_size argument: \d+. Cannot define output buffer size. Must be lesser or equal to 2113929216$" # noqa + err_class = ValueError + else: + message = r"^Inconsistent buffer_size/store_comp_size values. Maximal compressed length \(\d+\) cannot fit in a \d+ byte-long integer$" # noqa + err_class = lz4.stream.LZ4StreamError + + with pytest.raises(err_class, match=message): + lz4.stream.LZ4StreamCompressor(**c_kwargs) + + +def test_invalid_config_d_4(store_comp_size): + d_kwargs = {} + d_kwargs['strategy'] = "double_buffer" + d_kwargs['buffer_size'] = 1 << (8 * store_comp_size['store_comp_size']) + d_kwargs.update(store_comp_size) + + if store_comp_size['store_comp_size'] >= 4: + + if os.environ.get('TRAVIS') is not None: + pytest.skip('Skipping test on Travis due to insufficient memory') + + if sys.maxsize < 0xffffffff: + pytest.skip('Py_ssize_t too small for this test') + + if psutil.virtual_memory().available < 4 * d_kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer), so round up to 4 buffer_size. + pytest.skip('Insufficient system memory for this test') + + # Make sure the page size is larger than what the input bound will be, + # but still fit in 4 bytes + d_kwargs['buffer_size'] -= 1 + + # No failure expected during instanciation/initialization + lz4.stream.LZ4StreamDecompressor(**d_kwargs) + + +def test_invalid_config_c_5(): + c_kwargs = {} + c_kwargs['strategy'] = "double_buffer" + c_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE + + if os.environ.get('TRAVIS') is not None: + pytest.skip('Skipping test on Travis due to insufficient memory') + + if sys.maxsize < 0xffffffff: + pytest.skip('Py_ssize_t too small for this test') + + if psutil.virtual_memory().total < 3 * c_kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer) + pytest.skip('Insufficient system memory for this test') + + # No failure expected + lz4.stream.LZ4StreamCompressor(**c_kwargs) + + c_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE + 1 + with pytest.raises(ValueError): + lz4.stream.LZ4StreamCompressor(**c_kwargs) + + # Make sure the page size is larger than what the input bound will be, + # but still fit in 4 bytes + c_kwargs['buffer_size'] = _4GB - 1 # 4GB - 1 (to fit in 4 bytes) + with pytest.raises(ValueError): + lz4.stream.LZ4StreamCompressor(**c_kwargs) + + +def test_invalid_config_d_5(): + d_kwargs = {} + d_kwargs['strategy'] = "double_buffer" + + # No failure expected during instanciation/initialization + d_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE + + if os.environ.get('TRAVIS') is not None: + pytest.skip('Skipping test on Travis due to insufficient memory') + + if sys.maxsize < 0xffffffff: + pytest.skip('Py_ssize_t too small for this test') + + if psutil.virtual_memory().total < 3 * d_kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer) + pytest.skip('Insufficient system memory for this test') + + lz4.stream.LZ4StreamDecompressor(**d_kwargs) + + # No failure expected during instanciation/initialization + d_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE + 1 + + if os.environ.get('TRAVIS') is not None: + pytest.skip('Skipping test on Travis due to insufficient memory') + + if sys.maxsize < 0xffffffff: + pytest.skip('Py_ssize_t too small for this test') + + if psutil.virtual_memory().total < 3 * d_kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer) + pytest.skip('Insufficient system memory for this test') + + lz4.stream.LZ4StreamDecompressor(**d_kwargs) + + # No failure expected during instanciation/initialization + d_kwargs['buffer_size'] = _4GB - 1 # 4GB - 1 (to fit in 4 bytes) + + if os.environ.get('TRAVIS') is not None: + pytest.skip('Skipping test on Travis due to insufficient memory') + + if sys.maxsize < 0xffffffff: + pytest.skip('Py_ssize_t too small for this test') + + if psutil.virtual_memory().available < 4 * d_kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer), so round up to 4 buffer_size. + pytest.skip('Insufficient system memory for this test') + + lz4.stream.LZ4StreamDecompressor(**d_kwargs) + + +def test_decompress_corrupted_input_1(): + c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + d_kwargs = {} + d_kwargs.update(c_kwargs) + + data = compress(b'A' * 512, c_kwargs) + decompress(data, d_kwargs) + + message = r"^Requested input size \(\d+\) larger than source size \(\d+\)$" + + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(data[4:], d_kwargs) + + +def test_decompress_corrupted_input_2(): + c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + d_kwargs = {} + d_kwargs.update(c_kwargs) + + data = compress(b'A' * 512, c_kwargs) + decompress(data, d_kwargs) + + message = r"^Decompression failed. error: \d+$" + + # Block size corruption in the first block + + # Block size longer than actual: + data = int_to_bytes(int_from_bytes(data[:4], 'little') + 1, 4, 'little') + data[4:] + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(data, d_kwargs) + + # Block size shorter than actual: + data = int_to_bytes(int_from_bytes(data[:4], 'little') - 2, 4, 'little') + data[4:] + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(data, d_kwargs) + + +def test_decompress_corrupted_input_3(): + c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + d_kwargs = {} + d_kwargs.update(c_kwargs) + + data = compress(b'A' * 512, c_kwargs) + decompress(data, d_kwargs) + + message = r"^Decompression failed. error: \d+$" + + # Block size corruption in a block in the middle of the stream + offset = 4 + int_from_bytes(data[:4], 'little') + + # Block size longer than actual: + block_len = int_from_bytes(data[offset:offset + 4], 'little') + 1 + data = data[:offset] + int_to_bytes(block_len, 4, 'little') + data[offset + 4:] + + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(data, d_kwargs) + + # Block size shorter than actual: + block_len = int_from_bytes(data[offset:offset + 4], 'little') - 2 + data = data[:offset] + int_to_bytes(block_len, 4, 'little') + data[offset + 4:] + + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(data, d_kwargs) + + +def test_decompress_corrupted_input_4(): + c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + d_kwargs = {} + d_kwargs.update(c_kwargs) + + data = compress(b'A' * 256, c_kwargs) + decompress(data, d_kwargs) + + # Block size corruption in the last block of the stream + offset = 4 + int_from_bytes(data[:4], 'little') + + # Block size longer than actual: + block_len = int_from_bytes(data[offset:offset + 4], 'little') + 1 + data = data[:offset] + int_to_bytes(block_len, 4, 'little') + data[offset + 4:] + + message = r"^Requested input size \(\d+\) larger than source size \(\d+\)$" + + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(data, d_kwargs) + + # Block size shorter than actual: + block_len = int_from_bytes(data[offset:offset + 4], 'little') - 2 + data = data[:offset] + int_to_bytes(block_len, 4, 'little') + data[offset + 4:] + + message = r"^Decompression failed. error: \d+$" + + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(data, d_kwargs) + + +def test_decompress_truncated(): + c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + d_kwargs = {} + d_kwargs.update(c_kwargs) + + input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 + compressed, block_offsets = compress(input_data, c_kwargs, return_block_offset=True) + + last_block_offset = 0 + for n in range(len(compressed)): + if n in block_offsets: + # end of input matches end of block, so decompression must succeed + last_block_offset = n + decompress(compressed[:n], d_kwargs) + + else: + # end of input does not match end of block, so decompression failure is expected + if n - last_block_offset < c_kwargs['store_comp_size']: + message = "^Invalid source, too small for holding any block$" + else: + message = r"^Requested input size \(\d+\) larger than source size \(\d+\)$" + + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(compressed[:n], d_kwargs) + + +# This next is probably redundant given test_decompress_truncated above since +# the trailing bytes will be considered as the truncated last block, but we'll +# keep them for now + + +def test_decompress_with_trailer(): + c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + d_kwargs = {} + d_kwargs.update(c_kwargs) + + data = b'A' * 64 + comp = compress(data, c_kwargs) + + message = "^Invalid source, too small for holding any block$" + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(comp + b'A', d_kwargs) + + message = r"^Requested input size \(\d+\) larger than source size \(\d+\)$" + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(comp + b'A' * 10, d_kwargs) + + for n in range(1, 10): + if n < d_kwargs['store_comp_size']: + message = "^Invalid source, too small for holding any block$" + else: + message = r"^Decompression failed. error: \d+$" + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(comp + b'\x00' * n, d_kwargs) + + +def test_unicode(): + if sys.version_info < (3,): + return # skip + + c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + d_kwargs = {} + d_kwargs.update(c_kwargs) + + DATA = b'x' + with pytest.raises(TypeError): + compress(DATA.decode('latin1'), c_kwargs) + decompress(compress(DATA, c_kwargs).decode('latin1'), d_kwargs) + + +# These next two are probably redundant given test_1 above but we'll keep them +# for now + + +def test_return_bytearray(): + if sys.version_info < (3,): + return # skip + + c_kwargs_r = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + c_kwargs = {'return_bytearray': True} + c_kwargs.update(c_kwargs_r) + + d_kwargs = {} + d_kwargs.update(c_kwargs) + + data = os.urandom(128 * 1024) # Read 128kb + compressed = compress(data, c_kwargs_r, check_block_type=True) + b = compress(data, c_kwargs, check_block_type=True) + assert isinstance(b, bytearray) + assert bytes(b) == compressed + b = decompress(compressed, d_kwargs, check_chunk_type=True) + assert isinstance(b, bytearray) + assert bytes(b) == data + + +def test_memoryview(): + if sys.version_info < (2, 7): + return # skip + + c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + d_kwargs = {} + d_kwargs.update(c_kwargs) + + data = os.urandom(128 * 1024) # Read 128kb + compressed = compress(data, c_kwargs) + assert compress(memoryview(data), c_kwargs) == compressed + assert decompress(memoryview(compressed), d_kwargs) == data + + +def test_with_dict_none(): + kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 + for mode in ['default', 'high_compression']: + c_kwargs = {'mode': mode, 'dictionary': None} + c_kwargs.update(kwargs) + d_kwargs = {} + d_kwargs.update(kwargs) + assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data + + c_kwargs = {'mode': mode} + c_kwargs.update(kwargs) + d_kwargs = {'dictionary': None} + d_kwargs.update(kwargs) + assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data + + c_kwargs = {'mode': mode, 'dictionary': b''} + c_kwargs.update(kwargs) + d_kwargs = {} + d_kwargs.update(kwargs) + assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data + + c_kwargs = {'mode': mode} + c_kwargs.update(kwargs) + d_kwargs = {'dictionary': b''} + d_kwargs.update(kwargs) + assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data + + c_kwargs = {'mode': mode, 'dictionary': ''} + c_kwargs.update(kwargs) + d_kwargs = {} + d_kwargs.update(kwargs) + assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data + + c_kwargs = {'mode': mode} + c_kwargs.update(kwargs) + d_kwargs = {'dictionary': ''} + d_kwargs.update(kwargs) + assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data + + +def test_with_dict(): + kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 + dict1 = input_data[10:30] + dict2 = input_data[20:40] + message = r"^Decompression failed. error: \d+$" + + for mode in ['default', 'high_compression']: + c_kwargs = {'mode': mode, 'dictionary': dict1} + c_kwargs.update(kwargs) + compressed = compress(input_data, c_kwargs) + + d_kwargs = {} + d_kwargs.update(kwargs) + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(compressed, d_kwargs) + + d_kwargs = {'dictionary': dict1[:2]} + d_kwargs.update(kwargs) + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(compressed, d_kwargs) + + d_kwargs = {'dictionary': dict2} + d_kwargs.update(kwargs) + assert decompress(compressed, d_kwargs) != input_data + + d_kwargs = {'dictionary': dict1} + d_kwargs.update(kwargs) + assert decompress(compressed, d_kwargs) == input_data + + c_kwargs = {} + c_kwargs.update(kwargs) + d_kwargs = {'dictionary': dict1} + d_kwargs.update(kwargs) + assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data + + +def test_known_decompress_1(): + d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + output = b'' + + input = b'\x00\x00\x00\x00' + message = "^Decompression failed. error: 1$" + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + decompress(input, d_kwargs) + + input = b'\x01\x00\x00\x00\x00' + assert decompress(input, d_kwargs) == output + + +def test_known_decompress_2(): + d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + input = b'\x02\x00\x00\x00\x10 ' + output = b' ' + assert decompress(input, d_kwargs) == output + + +def test_known_decompress_3(): + d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + # uncompressed data size smaller than buffer_size + input = b'%\x00\x00\x00\xff\x0bLorem ipsum dolor sit amet\x1a\x006P amet' + output = b'Lorem ipsum dolor sit amet' * 4 + assert decompress(input, d_kwargs) == output + + +def test_known_decompress_4(): + d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} + + input = b'%\x00\x00\x00\xff\x0bLorem ipsum dolor sit amet\x1a\x00NPit am\n\x00\x00\x00\x0fh\x00hP sit \x05\x00\x00\x00@amet' + output = b'Lorem ipsum dolor sit amet' * 10 + assert decompress(input, d_kwargs) == output diff --git a/tests/stream/test_stream_2.py b/tests/stream/test_stream_2.py new file mode 100644 index 00000000..10e148ec --- /dev/null +++ b/tests/stream/test_stream_2.py @@ -0,0 +1,148 @@ +import pytest +import sys +import lz4.stream +import psutil +import os + + +# This test requires allocating a big lump of memory. In order to +# avoid a massive memory allocation during byte compilation, we have +# to declare a variable for the size of the buffer we're going to +# create outside the scope of the function below. See: +# https://bugs.python.org/issue21074 + +_4GB = 0xffffffff # actually 4GB - 1B, the maximum size on 4 bytes. + +# This test will be killed on Travis due to the 3GB memory limit +# there. Unfortunately psutil reports the host memory, not the memory +# available to the container, and so can't be used to detect available +# memory, so instead, as an ugly hack for detecting we're on Travis we +# check for the TRAVIS environment variable being set. This is quite +# fragile. + +try: + huge = b'\0' * _4GB +except (MemoryError, OverflowError): + huge = None + + +@pytest.mark.skipif( + os.environ.get('TRAVIS') is not None, + reason='Skipping test on Travis due to insufficient memory' +) +@pytest.mark.skipif( + sys.maxsize < _4GB, + reason='Py_ssize_t too small for this test' +) +@pytest.mark.skipif( + psutil.virtual_memory().total < _4GB or huge is None, + reason='Insufficient system memory for this test' +) +def test_huge_1(): + data = b'' + kwargs = { + 'strategy': "double_buffer", + 'buffer_size': lz4.stream.LZ4_MAX_INPUT_SIZE, + 'store_comp_size': 4, + 'dictionary': huge, + } + + if psutil.virtual_memory().total < 3 * kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer) + pytest.skip('Insufficient system memory for this test') + + # Triggering overflow error + message = r'^Dictionary too large for LZ4 API$' + + with pytest.raises(OverflowError, match=message): + with lz4.stream.LZ4StreamCompressor(**kwargs) as proc: + proc.compress(data) + + with pytest.raises(OverflowError, match=message): + with lz4.stream.LZ4StreamDecompressor(**kwargs) as proc: + proc.decompress(data) + + +@pytest.mark.skipif( + os.environ.get('TRAVIS') is not None, + reason='Skipping test on Travis due to insufficient memory' +) +@pytest.mark.skipif( + sys.maxsize < 0xffffffff, + reason='Py_ssize_t too small for this test' +) +@pytest.mark.skipif( + psutil.virtual_memory().total < _4GB or huge is None, + reason='Insufficient system memory for this test' +) +def test_huge_2(): + data = huge + kwargs = { + 'strategy': "double_buffer", + 'buffer_size': lz4.stream.LZ4_MAX_INPUT_SIZE, + 'store_comp_size': 4, + 'dictionary': b'', + } + + if psutil.virtual_memory().total < 3 * kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer) + pytest.skip('Insufficient system memory for this test') + + # Raising overflow error + message = r'^Input too large for LZ4 API$' + + with pytest.raises(OverflowError, match=message): + with lz4.stream.LZ4StreamCompressor(**kwargs) as proc: + proc.compress(data) + + # On decompression, too large input will raise LZ4StreamError + with pytest.raises(lz4.stream.LZ4StreamError): + with lz4.stream.LZ4StreamDecompressor(**kwargs) as proc: + proc.decompress(data) + + +@pytest.mark.skipif( + os.environ.get('TRAVIS') is not None, + reason='Skipping test on Travis due to insufficient memory' +) +@pytest.mark.skipif( + sys.maxsize < 0xffffffff, + reason='Py_ssize_t too small for this test' +) +@pytest.mark.skipif( + psutil.virtual_memory().total < _4GB or huge is None, + reason='Insufficient system memory for this test' +) +def test_huge_3(): + data = huge + kwargs = { + 'strategy': "double_buffer", + 'buffer_size': lz4.stream.LZ4_MAX_INPUT_SIZE, + 'store_comp_size': 4, + 'dictionary': huge, + } + + if psutil.virtual_memory().total < 3 * kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer) + pytest.skip('Insufficient system memory for this test') + + # Raising overflow error (during initialization because of the dictionary parameter) + message = r'^Dictionary too large for LZ4 API$' + + with pytest.raises(OverflowError, match=message): + with lz4.stream.LZ4StreamCompressor(**kwargs) as proc: + proc.compress(data) + + with pytest.raises(OverflowError, match=message): + with lz4.stream.LZ4StreamDecompressor(**kwargs) as proc: + proc.decompress(data) + + +def test_dummy(): + pass diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py new file mode 100644 index 00000000..74793ad6 --- /dev/null +++ b/tests/stream/test_stream_3.py @@ -0,0 +1,137 @@ +import lz4.stream +import pytest +import sys +import os +import psutil + + +_1KB = 1024 +_1MB = _1KB * 1024 +_1GB = _1MB * 1024 + + +def compress(x, c_kwargs): + if c_kwargs.get('return_bytearray', False): + c = bytearray() + else: + c = bytes() + with lz4.stream.LZ4StreamCompressor(**c_kwargs) as proc: + for start in range(0, len(x), c_kwargs['buffer_size']): + chunk = x[start:start + c_kwargs['buffer_size']] + block = proc.compress(chunk) + c += block + return c + + +def decompress(x, d_kwargs): + if d_kwargs.get('return_bytearray', False): + d = bytearray() + else: + d = bytes() + with lz4.stream.LZ4StreamDecompressor(**d_kwargs) as proc: + start = 0 + while start < len(x): + block = proc.get_block(x[start:]) + chunk = proc.decompress(block) + d += chunk + start += d_kwargs['store_comp_size'] + len(block) + return d + + +test_buffer_size = sorted( + [256, + 1 * _1KB, + 64 * _1KB, + 1 * _1MB, + 1 * _1GB, + lz4.stream.LZ4_MAX_INPUT_SIZE] +) + + +@pytest.fixture( + params=test_buffer_size, + ids=[ + 'buffer_size' + str(i) for i in range(len(test_buffer_size)) + ] +) +def buffer_size(request): + return request.param + + +test_data = [ + (b'a' * _1MB), +] + + +@pytest.fixture( + params=test_data, + ids=[ + 'data' + str(i) for i in range(len(test_data)) + ] +) +def data(request): + return request.param + + +def test_block_decompress_mem_usage(data, buffer_size): + kwargs = { + 'strategy': "double_buffer", + 'buffer_size': buffer_size, + 'store_comp_size': 4, + } + + if os.environ.get('TRAVIS') is not None: + pytest.skip('Skipping test on Travis due to insufficient memory') + + if os.environ.get('APPVEYOR') is not None: + pytest.skip('Skipping test on AppVeyor due to insufficient resources') + + if sys.maxsize < 0xffffffff: + pytest.skip('Py_ssize_t too small for this test') + + if psutil.virtual_memory().total < 3 * kwargs['buffer_size']: + # The internal LZ4 context will request at least 3 times buffer_size + # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size + # for the output buffer) + pytest.skip('Insufficient system memory for this test') + + tracemalloc = pytest.importorskip('tracemalloc') + + # Trace memory usage on compression + tracemalloc.start() + prev_snapshot = None + + for i in range(1000): + compressed = compress(data, kwargs) + + if i % 100 == 0: + snapshot = tracemalloc.take_snapshot() + + if prev_snapshot: + # Filter on lz4.stream module'a allocations + stats = [x for x in snapshot.compare_to(prev_snapshot, 'lineno') + if lz4.stream.__file__ in x.traceback._frames[0][0]] + assert sum(map(lambda x: x.size_diff, stats)) < (1024 * 4) + + prev_snapshot = snapshot + + tracemalloc.stop() + + tracemalloc.start() + prev_snapshot = None + + for i in range(1000): + decompressed = decompress(compressed, kwargs) # noqa: F841 + + if i % 100 == 0: + snapshot = tracemalloc.take_snapshot() + + if prev_snapshot: + # Filter on lz4.stream module'a allocations + stats = [x for x in snapshot.compare_to(prev_snapshot, 'lineno') + if lz4.stream.__file__ in x.traceback._frames[0][0]] + assert sum(map(lambda x: x.size_diff, stats)) < (1024 * 4) + + prev_snapshot = snapshot + + tracemalloc.stop() From 6be649ff8811a66bd6d5502968b9eb3146cdbb31 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 10 Nov 2019 08:37:19 +0000 Subject: [PATCH 102/280] Remove support for Python < 3.5 from CICD and setup.py --- .travis.yml | 55 ++++++++++++++++------------------------------------ appveyor.yml | 8 ++------ setup.py | 5 ++--- 3 files changed, 21 insertions(+), 47 deletions(-) diff --git a/.travis.yml b/.travis.yml index bc9f2359..16c9e40b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,5 @@ matrix: include: - - name: linux-py27 - language: python - python: 2.7 - sudo: required - services: - - docker - env: - - PIP=pip - - CIBW_BUILD=cp27-* - - name: linux-py34 - language: python - python: 3.4 - sudo: required - services: - - docker - env: - - PIP=pip - - CIBW_BUILD=cp34-* - name: linux-py35 language: python python: 3.5 @@ -46,61 +28,58 @@ matrix: env: - PIP=pip3 - CIBW_BUILD=cp37-* - - name: osx-py27 - os: osx - language: generic + - name: linux-py38 + language: python + python: 3.8 + dist: xenial + sudo: required + services: + - docker env: - - PATH="/Library/Frameworks/Python.framework/Versions/2.7/bin:$PATH" - - PIP=pip - - CIBW_BUILD=cp27-* - - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" - install: - - ${PIP} install cibuildwheel==0.11.1 - script: - - cibuildwheel --output-dir dist - - name: osx-py34 + - PIP=pip3 + - CIBW_BUILD=cp38-* + - name: osx-py35 os: osx language: generic env: - PIP=pip2 - - CIBW_BUILD=cp34-* + - CIBW_BUILD=cp35-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - ${PIP} install cibuildwheel==0.11.1 script: - cibuildwheel --output-dir dist - - name: osx-py35 + - name: osx-py36 os: osx language: generic env: - PIP=pip2 - - CIBW_BUILD=cp35-* + - CIBW_BUILD=cp36-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - ${PIP} install cibuildwheel==0.11.1 script: - cibuildwheel --output-dir dist - - name: osx-py36 + - name: osx-py37 os: osx language: generic env: - PIP=pip2 - - CIBW_BUILD=cp36-* + - CIBW_BUILD=cp37-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - ${PIP} install cibuildwheel==0.11.1 script: - cibuildwheel --output-dir dist - - name: osx-py37 + - name: osx-py38 os: osx language: generic env: - PIP=pip2 - - CIBW_BUILD=cp37-* + - CIBW_BUILD=cp38-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: diff --git a/appveyor.yml b/appveyor.yml index 932b9def..45a3a029 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,18 +10,14 @@ environment: matrix: - - PYTHON: "C:\\Python27-x64" - DISTUTILS_USE_SDK: "1" - - PYTHON: "C:\\Python34-x64" - DISTUTILS_USE_SDK: "1" - PYTHON: "C:\\Python35-x64" - PYTHON: "C:\\Python36-x64" - PYTHON: "C:\\Python37-x64" - - PYTHON: "C:\\Python27" - - PYTHON: "C:\\Python34" + - PYTHON: "C:\\Python38-x64" - PYTHON: "C:\\Python35" - PYTHON: "C:\\Python36" - PYTHON: "C:\\Python37" + - PYTHON: "C:\\Python38" matrix: # Immediately finish build once one of the jobs fails. diff --git a/setup.py b/setup.py index 95b479d3..c13e9623 100644 --- a/setup.py +++ b/setup.py @@ -176,7 +176,7 @@ def pkgconfig_installed_check(lib, required_version, default): use_scm_version={ 'write_to': "lz4/version.py", }, - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + python_requires=">=3.5", setup_requires=[ 'setuptools_scm', 'pkgconfig', @@ -211,10 +211,9 @@ def pkgconfig_installed_check(lib, required_version, default): 'Intended Audience :: Developers', 'Programming Language :: C', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ], ) From 96200065f5c18476e3e39c0b2f85c6ed9bc72f69 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 10 Nov 2019 08:39:33 +0000 Subject: [PATCH 103/280] Remove deprecation package from requirements.txt (#181) --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3ff12fe6..65529ded 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,5 @@ tox pytest pytest-runner setuptools_scm -deprecation pkgconfig future - From dc1bc2497624f82ec59a69c70fc996128b59aff4 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 09:19:31 +0000 Subject: [PATCH 104/280] Remove frame/_compression.py --- lz4/frame/_compression.py | 170 -------------------------------------- 1 file changed, 170 deletions(-) delete mode 100644 lz4/frame/_compression.py diff --git a/lz4/frame/_compression.py b/lz4/frame/_compression.py deleted file mode 100644 index 3c68904b..00000000 --- a/lz4/frame/_compression.py +++ /dev/null @@ -1,170 +0,0 @@ -# Local python-lz4 copy of this file taken from the CPython standard library -# for earlier Python versions that don't ship with this file. This file has -# been modified to work on Python < 3.0. - -"""Internal classes used by the gzip, lzma and bz2 modules""" - -import sys -import io -# Ensure super has Python 3 semantics even on Python 2 -from builtins import super - - -BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size - - -class BaseStream(io.BufferedIOBase): - """Mode-checking helper functions.""" - - def _check_not_closed(self): - if self.closed: - raise ValueError("I/O operation on closed file") - - def _check_can_read(self): - if not self.readable(): - raise io.UnsupportedOperation("File not open for reading") - - def _check_can_write(self): - if not self.writable(): - raise io.UnsupportedOperation("File not open for writing") - - def _check_can_seek(self): - if not self.readable(): - raise io.UnsupportedOperation("Seeking is only supported " - "on files open for reading") - if not self.seekable(): - raise io.UnsupportedOperation("The underlying file object " - "does not support seeking") - - -class DecompressReader(io.RawIOBase): - """Adapts the decompressor API to a RawIOBase reader API""" - - def readable(self): - return True - - def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args): - self._fp = fp - self._eof = False - self._pos = 0 # Current offset in decompressed stream - - # Set to size of decompressed stream once it is known, for SEEK_END - self._size = -1 - - # Save the decompressor factory and arguments. - # If the file contains multiple compressed streams, each - # stream will need a separate decompressor object. A new decompressor - # object is also needed when implementing a backwards seek(). - self._decomp_factory = decomp_factory - self._decomp_args = decomp_args - self._decompressor = self._decomp_factory(**self._decomp_args) - - # Exception class to catch from decompressor signifying invalid - # trailing data to ignore - self._trailing_error = trailing_error - - def close(self): - self._decompressor = None - return super().close() - - def seekable(self): - return self._fp.seekable() - - def readinto(self, b): - with memoryview(b) as view, view.cast("B") as byte_view: - data = self.read(len(byte_view)) - byte_view[:len(data)] = data - return len(data) - - def read(self, size=-1): - if size < 0: - return self.readall() - - if not size or self._eof: - return b"" - data = None # Default if EOF is encountered - # Depending on the input data, our call to the decompressor may not - # return any data. In this case, try again after reading another block. - while True: - if self._decompressor.eof: - rawblock = ( - self._decompressor.unused_data or self._fp.read(BUFFER_SIZE) - ) - if not rawblock: - break - # Continue to next stream. - self._decompressor = self._decomp_factory( - **self._decomp_args) - try: - data = self._decompressor.decompress(rawblock, size) - except self._trailing_error: - # Trailing data isn't a valid compressed stream; ignore it. - break - else: - if self._decompressor.needs_input: - rawblock = self._fp.read(BUFFER_SIZE) - if not rawblock: - raise EOFError("Compressed file ended before the " - "end-of-stream marker was reached") - else: - rawblock = b"" - data = self._decompressor.decompress(rawblock, size) - if data: - break - if not data: - self._eof = True - self._size = self._pos - return b"" - self._pos += len(data) - return data - - # Rewind the file to the beginning of the data stream. - def _rewind(self): - self._fp.seek(0) - self._eof = False - self._pos = 0 - self._decompressor = self._decomp_factory(**self._decomp_args) - - def seek(self, offset, whence=io.SEEK_SET): - # Recalculate offset as an absolute file position. - if whence == io.SEEK_SET: - pass - elif whence == io.SEEK_CUR: - offset = self._pos + offset - elif whence == io.SEEK_END: - # Seeking relative to EOF - we need to know the file's size. - if self._size < 0: - while self.read(io.DEFAULT_BUFFER_SIZE): - pass - offset = self._size + offset - else: - raise ValueError("Invalid value for whence: {}".format(whence)) - - # Make it so that offset is the number of bytes to skip forward. - if offset < self._pos: - self._rewind() - else: - offset -= self._pos - - # Read and discard data until we reach the desired position. - while offset > 0: - data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset)) - if not data: - break - offset -= len(data) - - return self._pos - - def tell(self): - """Return the current file position.""" - return self._pos - - -if sys.version_info < (3, 3): - # memoryview.cast is added in 3.3 - def readinto(self, b): - data = self.read(len(b)) - b[:len(data)] = data - return len(data) - - DecompressReader.readinto = readinto From 5f7ed45f41cf49dc15603d984f75ba3140ed7d69 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 10:57:25 +0000 Subject: [PATCH 105/280] Fix poor performance of frame decompression (#184) Previously we were growing the destination buffer by 3 times the return code of LZ4_decompress, which is simply an indication of the remaining bytes in the current source block. As such this was growing the destination buffer too slowly, and so we spent a lot of time resizing. --- lz4/frame/_frame.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 99ebe02d..b57e5287 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1021,6 +1021,7 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size, LZ4F_frameInfo_t frame_info; LZ4F_decompressOptions_t options; int end_of_frame = 0; + int resize_factor = 1; memset(&options, 0, sizeof options); @@ -1163,11 +1164,18 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size, } else { - /* Expand destination buffer. result is an indication of number of - source bytes remaining, so we'll use this to estimate the new - size of the destination buffer. */ + /* Expand the destination buffer. We've tried various strategies + here to estimate the compression ratio so far and adjust the + buffer size accordingly. However, that grows the buffer too + slowly. The best choices found were to either double the buffer + size each time, or to grow faster by multiplying the buffer + size by 2^N, where N is the number of resizes. We take the + latter approach, though the former approach may actually be + good enough in practice. */ char * buff; - destination_size += 3 * result; + + resize_factor *= 2; + destination_size *= resize_factor; Py_BLOCK_THREADS buff = PyMem_Realloc (destination, destination_size); From dd22b0d553fc1f8e3ecbb975c6b30c2bde6afbf4 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 15:24:14 +0000 Subject: [PATCH 106/280] Update cibuildwheel on travis to 1.1.0 --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 16c9e40b..d29fd322 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ matrix: - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - - ${PIP} install cibuildwheel==0.11.1 + - ${PIP} install cibuildwheel==1.1.0 script: - cibuildwheel --output-dir dist - name: osx-py36 @@ -59,7 +59,7 @@ matrix: - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - - ${PIP} install cibuildwheel==0.11.1 + - ${PIP} install cibuildwheel==1.1.0 script: - cibuildwheel --output-dir dist - name: osx-py37 @@ -71,7 +71,7 @@ matrix: - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - - ${PIP} install cibuildwheel==0.11.1 + - ${PIP} install cibuildwheel==1.1.0 script: - cibuildwheel --output-dir dist - name: osx-py38 @@ -83,12 +83,12 @@ matrix: - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" install: - - ${PIP} install cibuildwheel==0.11.1 + - ${PIP} install cibuildwheel==1.1.0 script: - cibuildwheel --output-dir dist install: - ${PIP} install -U pip - - ${PIP} install cibuildwheel==0.11.1 + - ${PIP} install cibuildwheel==1.1.0 - ${PIP} install codecov tox twine script: - tox tests From 55c4f4544a588d888e76de9fda8097a75ce8d33b Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 16:04:26 +0000 Subject: [PATCH 107/280] Invoke pip via module on OSX --- .travis.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index d29fd322..810cb907 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,54 +42,50 @@ matrix: os: osx language: generic env: - - PIP=pip2 - CIBW_BUILD=cp35-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" + - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - - ${PIP} install cibuildwheel==1.1.0 + - python -m pip install cibuildwheel==1.1.0 script: - cibuildwheel --output-dir dist - name: osx-py36 os: osx language: generic env: - - PIP=pip2 - CIBW_BUILD=cp36-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" + - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - - ${PIP} install cibuildwheel==1.1.0 + - python -m pip install cibuildwheel==1.1.0 script: - cibuildwheel --output-dir dist - name: osx-py37 os: osx language: generic env: - - PIP=pip2 - CIBW_BUILD=cp37-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" + - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - - ${PIP} install cibuildwheel==1.1.0 + - python -m pip install cibuildwheel==1.1.0 script: - cibuildwheel --output-dir dist - name: osx-py38 os: osx language: generic env: - - PIP=pip2 - CIBW_BUILD=cp38-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="${PIP} install -U pip; ${PIP} install tox twine" + - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - - ${PIP} install cibuildwheel==1.1.0 + - python -m pip install cibuildwheel==1.1.0 script: - cibuildwheel --output-dir dist install: - - ${PIP} install -U pip - - ${PIP} install cibuildwheel==1.1.0 - - ${PIP} install codecov tox twine + - python -m pip install -U pip + - python -m pip install cibuildwheel==1.1.0 + - python -m pip install codecov tox twine script: - tox tests - codecov From 594120777a3ba01f880b7d0078f977de727be9f0 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 16:14:08 +0000 Subject: [PATCH 108/280] Remove unused PIP env var on travis linux builds --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 810cb907..c1f361be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ matrix: services: - docker env: - - PIP=pip3 - CIBW_BUILD=cp35-* - name: linux-py36 language: python @@ -16,7 +15,6 @@ matrix: services: - docker env: - - PIP=pip3 - CIBW_BUILD=cp36-* - name: linux-py37 language: python @@ -26,7 +24,6 @@ matrix: services: - docker env: - - PIP=pip3 - CIBW_BUILD=cp37-* - name: linux-py38 language: python @@ -36,7 +33,6 @@ matrix: services: - docker env: - - PIP=pip3 - CIBW_BUILD=cp38-* - name: osx-py35 os: osx From b72a4d60b85f1f04a789e361cbd28faf082499dc Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 18:52:13 +0000 Subject: [PATCH 109/280] Ensure twine is available for OSX travis builds --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index c1f361be..b4705634 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,7 @@ matrix: - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - python -m pip install cibuildwheel==1.1.0 + - python -m pip install twine script: - cibuildwheel --output-dir dist - name: osx-py36 @@ -54,6 +55,7 @@ matrix: - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - python -m pip install cibuildwheel==1.1.0 + - python -m pip install twine script: - cibuildwheel --output-dir dist - name: osx-py37 @@ -65,6 +67,7 @@ matrix: - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - python -m pip install cibuildwheel==1.1.0 + - python -m pip install twine script: - cibuildwheel --output-dir dist - name: osx-py38 @@ -76,6 +79,7 @@ matrix: - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - python -m pip install cibuildwheel==1.1.0 + - python -m pip install twine script: - cibuildwheel --output-dir dist install: From e3c7d19a3adfada6d27679eda299e17c4b231003 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 19:26:29 +0000 Subject: [PATCH 110/280] Remove py3c from _block.c --- lz4/block/_block.c | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index d55b1d92..3e904a03 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -35,8 +35,7 @@ #define inline #endif -#include -#include +#include #include #include @@ -152,8 +151,6 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) NULL }; - -#if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, kwargs, "y*|spiipz*", argnames, &source, &mode, &store_size, &acceleration, &compression, @@ -161,15 +158,6 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) { return NULL; } -#else - if (!PyArg_ParseTupleAndKeywords (args, kwargs, "s*|siiiiz*", argnames, - &source, - &mode, &store_size, &acceleration, &compression, - &return_bytearray, &dict)) - { - return NULL; - } -#endif if (source.len > INT_MAX) { @@ -303,21 +291,12 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs) NULL }; -#if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, kwargs, "y*|ipz*", argnames, &source, &uncompressed_size, &return_bytearray, &dict)) { return NULL; } -#else - if (!PyArg_ParseTupleAndKeywords (args, kwargs, "s*|iiz*", argnames, - &source, &uncompressed_size, - &return_bytearray, &dict)) - { - return NULL; - } -#endif if (source.len > INT_MAX) { @@ -518,7 +497,8 @@ static struct PyModuleDef moduledef = module_methods }; -MODULE_INIT_FUNC (_block) +PyMODINIT_FUNC +PyInit__block(void) { PyObject *module = PyModule_Create (&moduledef); @@ -537,6 +517,6 @@ MODULE_INIT_FUNC (_block) } Py_INCREF(LZ4BlockError); PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError); - + return module; } From 4b26dca06d9d89ff7cac496c85397158d60e2cee Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 19:32:04 +0000 Subject: [PATCH 111/280] Remove Py_UNUSED macro from _block.c --- lz4/block/_block.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 3e904a03..a2774e5d 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -42,14 +42,6 @@ #include #include -#ifndef Py_UNUSED /* This is already defined for Python 3.4 onwards */ -#ifdef __GNUC__ -#define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) -#else -#define Py_UNUSED(name) _unused_ ## name -#endif -#endif - #if defined(_WIN32) && defined(_MSC_VER) #if _MSC_VER >= 1600 #include From 577729cba99cdd4b372c8836f48142ba9e88d644 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 19:37:22 +0000 Subject: [PATCH 112/280] Remove py3c from _frame.c --- lz4/frame/_frame.c | 91 +++------------------------------------------- 1 file changed, 5 insertions(+), 86 deletions(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index b57e5287..8aaa287e 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -35,8 +35,7 @@ #define inline #endif -#include -#include +#include #include #include /* Needed for LZ4_VERSION_NUMBER only. */ @@ -149,7 +148,6 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, memset (&preferences, 0, sizeof preferences); -#if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, keywds, "y*|iippppp", kwlist, &source, &preferences.compressionLevel, @@ -162,20 +160,6 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, { return NULL; } -#else - if (!PyArg_ParseTupleAndKeywords (args, keywds, "s*|iiiiiii", kwlist, - &source, - &preferences.compressionLevel, - &preferences.frameInfo.blockSizeID, - &content_checksum, - &block_checksum, - &block_linked, - &store_size, - &return_bytearray)) - { - return NULL; - } -#endif if (content_checksum) { @@ -318,7 +302,6 @@ compress_begin (PyObject * Py_UNUSED (self), PyObject * args, memset (&preferences, 0, sizeof preferences); -#if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, keywds, "O|kiippppp", kwlist, &py_context, &source_size, @@ -333,22 +316,7 @@ compress_begin (PyObject * Py_UNUSED (self), PyObject * args, { return NULL; } -#else - if (!PyArg_ParseTupleAndKeywords (args, keywds, "O|kiiiiiii", kwlist, - &py_context, - &source_size, - &preferences.compressionLevel, - &preferences.frameInfo.blockSizeID, - &content_checksum, - &block_checksum, - &block_linked, - &preferences.autoFlush, - &return_bytearray - )) - { - return NULL; - } -#endif + if (content_checksum) { preferences.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; @@ -473,7 +441,6 @@ compress_chunk (PyObject * Py_UNUSED (self), PyObject * args, memset (&compress_options, 0, sizeof compress_options); -#if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, keywds, "Oy*|p", kwlist, &py_context, &source, @@ -481,15 +448,6 @@ compress_chunk (PyObject * Py_UNUSED (self), PyObject * args, { return NULL; } -#else - if (!PyArg_ParseTupleAndKeywords (args, keywds, "Os*|i", kwlist, - &py_context, - &source, - &return_bytearray)) - { - return NULL; - } -#endif source_size = source.len; @@ -599,7 +557,6 @@ compress_flush (PyObject * Py_UNUSED (self), PyObject * args, PyObject * keywds) memset (&compress_options, 0, sizeof compress_options); -#if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, keywds, "O|pp", kwlist, &py_context, &end_frame, @@ -607,15 +564,7 @@ compress_flush (PyObject * Py_UNUSED (self), PyObject * args, PyObject * keywds) { return NULL; } -#else - if (!PyArg_ParseTupleAndKeywords (args, keywds, "O|ii", kwlist, - &py_context, - &end_frame, - &return_bytearray)) - { - return NULL; - } -#endif + if (!end_frame && LZ4_versionNumber() < 10800) { PyErr_SetString (PyExc_RuntimeError, @@ -717,19 +666,11 @@ get_frame_info (PyObject * Py_UNUSED (self), PyObject * args, NULL }; -#if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, keywds, "y*", kwlist, &py_source)) { return NULL; } -#else - if (!PyArg_ParseTupleAndKeywords (args, keywds, "s*", kwlist, - &py_source)) - { - return NULL; - } -#endif Py_BEGIN_ALLOW_THREADS @@ -1279,7 +1220,6 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, NULL }; -#if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, keywds, "y*|pp", kwlist, &py_source, &return_bytearray, @@ -1288,16 +1228,6 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args, { return NULL; } -#else - if (!PyArg_ParseTupleAndKeywords (args, keywds, "s*|ii", kwlist, - &py_source, - &return_bytearray, - &return_bytes_read - )) - { - return NULL; - } -#endif Py_BEGIN_ALLOW_THREADS result = LZ4F_createDecompressionContext (&context, LZ4F_VERSION); @@ -1356,7 +1286,6 @@ decompress_chunk (PyObject * Py_UNUSED (self), PyObject * args, NULL }; -#if IS_PY3 if (!PyArg_ParseTupleAndKeywords (args, keywds, "Oy*|np", kwlist, &py_context, &py_source, @@ -1366,17 +1295,6 @@ decompress_chunk (PyObject * Py_UNUSED (self), PyObject * args, { return NULL; } -#else - if (!PyArg_ParseTupleAndKeywords (args, keywds, "Os*|ni", kwlist, - &py_context, - &py_source, - &max_length, - &return_bytearray - )) - { - return NULL; - } -#endif context = (LZ4F_dctx *) PyCapsule_GetPointer (py_context, decompression_context_capsule_name); @@ -1752,7 +1670,8 @@ static struct PyModuleDef moduledef = module_methods }; -MODULE_INIT_FUNC (_frame) +PyMODINIT_FUNC +PyInit__frame(void) { PyObject *module = PyModule_Create (&moduledef); From 1a00e24f2c1bfe374ca0a69b8e085513d1ec582f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 19:38:02 +0000 Subject: [PATCH 113/280] Remove Py_UNUSED macro from _frame.c --- lz4/frame/_frame.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 8aaa287e..4dc42f75 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -41,14 +41,6 @@ #include /* Needed for LZ4_VERSION_NUMBER only. */ #include -#ifndef Py_UNUSED /* This is already defined for Python 3.4 onwards */ -#ifdef __GNUC__ -#define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) -#else -#define Py_UNUSED(name) _unused_ ## name -#endif -#endif - static const char * compression_context_capsule_name = "_frame.LZ4F_cctx"; static const char * decompression_context_capsule_name = "_frame.LZ4F_dctx"; From 2321c591d03fabbf8a65d874e96db11e73ae723c Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 19:59:12 +0000 Subject: [PATCH 114/280] Remove py3c from _stream.c --- lz4/stream/_stream.c | 69 +++++++------------------------------------- 1 file changed, 11 insertions(+), 58 deletions(-) diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 330f94bc..269f43ba 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -35,8 +35,8 @@ #define inline #endif -#include -#include +#include + #include #include #include @@ -805,19 +805,13 @@ _create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) NULL }; -#if IS_PY3 -#define _ARG_FMT "ssI|sIIpIz*" -#else -#define _ARG_FMT "ssI|sIIiIz*" -#endif - if (!PyArg_ParseTupleAndKeywords (args, kwds, _ARG_FMT, argnames, + if (!PyArg_ParseTupleAndKeywords (args, kwds, "ssI|sIIpIz*", argnames, &strategy_name, &direction, &buffer_size, &mode, &acceleration, &compression_level, &return_bytearray, &store_comp_size, &dict)) { goto abort_now; } -#undef _ARG_FMT /* Sanity checks on arguments */ if (dict.len > INT_MAX) @@ -1074,17 +1068,10 @@ _compress_bound (PyObject * Py_UNUSED (self), PyObject * args) /* Positional arguments: input_size * Keyword arguments : none */ -#if IS_PY3 -#define _ARG_FMT "OI" -#else -#define _ARG_FMT "OI" -#endif - if (!PyArg_ParseTuple (args, _ARG_FMT, - &input_size)) + if (!PyArg_ParseTuple (args, "OI", &input_size)) { goto exit_now; } -#undef _ARG_FMT py_dest = PyLong_FromUnsignedLong (get_compress_bound (input_size)); @@ -1106,17 +1093,10 @@ _input_bound (PyObject * Py_UNUSED (self), PyObject * args) /* Positional arguments: compress_max_size * Keyword arguments : none */ -#if IS_PY3 -#define _ARG_FMT "I" -#else -#define _ARG_FMT "I" -#endif - if (!PyArg_ParseTuple (args, _ARG_FMT, - &compress_max_size)) + if (!PyArg_ParseTuple (args, "I", &compress_max_size)) { goto exit_now; } -#undef _ARG_FMT py_dest = PyLong_FromUnsignedLong (get_input_bound (compress_max_size)); @@ -1171,17 +1151,10 @@ _compress (PyObject * Py_UNUSED (self), PyObject * args) /* Positional arguments: capsule_context, source * Keyword arguments : none */ -#if IS_PY3 -#define _ARG_FMT "Oy*" -#else -#define _ARG_FMT "Os*" -#endif - if (!PyArg_ParseTuple (args, _ARG_FMT, - &py_context, &source)) + if (!PyArg_ParseTuple (args, "Oy*", &py_context, &source)) { goto exit_now; } -#undef _ARG_FMT context = _PyCapsule_get_context (py_context); if ((context == NULL) || (context->lz4_state.context == NULL)) @@ -1269,17 +1242,10 @@ _get_block (PyObject * Py_UNUSED (self), PyObject * args) * Keyword arguments : none */ -#if IS_PY3 -#define _ARG_FMT "Oy*" -#else -#define _ARG_FMT "Os*" -#endif - if (!PyArg_ParseTuple (args, _ARG_FMT, - &py_context, &source)) + if (!PyArg_ParseTuple (args, "Oy*", &py_context, &source)) { goto exit_now; } -#undef _ARG_FMT context = _PyCapsule_get_context (py_context); if ((context == NULL) || (context->lz4_state.context == NULL)) @@ -1348,17 +1314,10 @@ _decompress (PyObject * Py_UNUSED (self), PyObject * args) /* Positional arguments: capsule_context, source * Keyword arguments : none */ -#if IS_PY3 -#define _ARG_FMT "Oy*" -#else -#define _ARG_FMT "Os*" -#endif - if (!PyArg_ParseTuple (args, _ARG_FMT, - &py_context, &source)) + if (!PyArg_ParseTuple (args, "Oy*", &py_context, &source)) { goto exit_now; } -#undef _ARG_FMT context = _PyCapsule_get_context (py_context); if ((context == NULL) || (context->lz4_state.context == NULL)) @@ -1649,16 +1608,10 @@ static PyModuleDef moduledef = { }; -MODULE_INIT_FUNC (_stream) +PyMODINIT_FUNC +PyInit__stream(void) { - PyObject * module = NULL; - -#if IS_PY3 - module = PyModule_Create (&moduledef); -#else - (void) moduledef; /* unused in python2 */ - module = Py_InitModule("_stream", module_methods); -#endif + PyObject * module = PyModule_Create (&moduledef); if (module == NULL) { From a2e1babe718e1dba0f8515b048e8400fe2abb389 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 19:59:38 +0000 Subject: [PATCH 115/280] Remove Py_UNUSED macro from _stream.c --- lz4/stream/_stream.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 269f43ba..a88a0432 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -76,14 +76,6 @@ typedef unsigned __int64 uint64_t; #endif /* __CHAR_BIT__ */ #endif /* _WIN32 && _MSC_VER */ -#ifndef Py_UNUSED /* This is already defined for Python 3.4 onwards */ -#ifdef __GNUC__ -#define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) -#else -#define Py_UNUSED(name) _unused_ ## name -#endif -#endif - #define LZ4_VERSION_NUMBER_1_9_0 10900 static const char * stream_context_capsule_name = "_stream.LZ4S_ctx"; From d41a42ceded80fa9a799d77a260e4789a59cc3b6 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 20:02:17 +0000 Subject: [PATCH 116/280] Remove py3c --- py3c/py3c.h | 34 ---------- py3c/py3c/capsulethunk.h | 141 --------------------------------------- py3c/py3c/comparison.h | 45 ------------- py3c/py3c/compat.h | 139 -------------------------------------- py3c/py3c/fileshim.h | 49 -------------- py3c/py3c/py3shims.h | 44 ------------ py3c/py3c/tpflags.h | 52 --------------- setup.py | 12 +--- 8 files changed, 3 insertions(+), 513 deletions(-) delete mode 100644 py3c/py3c.h delete mode 100644 py3c/py3c/capsulethunk.h delete mode 100644 py3c/py3c/comparison.h delete mode 100644 py3c/py3c/compat.h delete mode 100644 py3c/py3c/fileshim.h delete mode 100644 py3c/py3c/py3shims.h delete mode 100644 py3c/py3c/tpflags.h diff --git a/py3c/py3c.h b/py3c/py3c.h deleted file mode 100644 index b86324b3..00000000 --- a/py3c/py3c.h +++ /dev/null @@ -1,34 +0,0 @@ -/* -The MIT License (MIT) - -Copyright (c) 2015, Red Hat, Inc. and/or its affiliates - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#ifndef _PY3COMPAT_H_ -#define _PY3COMPAT_H_ - -#include - -#include -#include -#include - -#endif diff --git a/py3c/py3c/capsulethunk.h b/py3c/py3c/capsulethunk.h deleted file mode 100644 index 6ad70041..00000000 --- a/py3c/py3c/capsulethunk.h +++ /dev/null @@ -1,141 +0,0 @@ -/* Copyright (c) 2011, Larry Hastings - * Copyright (c) 2015, py3c contributors - * Licensed under the MIT license; see py3c.h - * - * (Note: Relicensed from PSF: http://bugs.python.org/issue24937#msg250191 ) - */ - -#ifndef __CAPSULETHUNK_H -#define __CAPSULETHUNK_H - -#if ( (PY_VERSION_HEX < 0x02070000) \ - || ((PY_VERSION_HEX >= 0x03000000) \ - && (PY_VERSION_HEX < 0x03010000)) ) - -#define __PyCapsule_GetField(capsule, field, error_value) \ - ( PyCapsule_CheckExact(capsule) \ - ? (((PyCObject *)capsule)->field) \ - : (PyErr_SetString(PyExc_TypeError, "CObject required"), (error_value)) \ - ) \ - -#define __PyCapsule_SetField(capsule, field, value) \ - ( PyCapsule_CheckExact(capsule) \ - ? (((PyCObject *)capsule)->field = value), 0 \ - : (PyErr_SetString(PyExc_TypeError, "CObject required"), 1) \ - ) \ - - -#define PyCapsule_Type PyCObject_Type - -#define PyCapsule_CheckExact(capsule) (PyCObject_Check(capsule)) -#define PyCapsule_IsValid(capsule, name) (PyCObject_Check(capsule)) - - -#define PyCapsule_New(pointer, name, destructor) \ - (PyCObject_FromVoidPtr(pointer, (void (*)(void*)) (destructor))) - - -#define PyCapsule_GetPointer(capsule, name) \ - (PyCObject_AsVoidPtr(capsule)) - -/* Don't call PyCObject_SetPointer here, it fails if there's a destructor */ -#define PyCapsule_SetPointer(capsule, pointer) \ - __PyCapsule_SetField(capsule, cobject, pointer) - - -#define PyCapsule_GetDestructor(capsule) \ - __PyCapsule_GetField(capsule, destructor, (void (*)(void*)) NULL) - -#define PyCapsule_SetDestructor(capsule, dtor) \ - __PyCapsule_SetField(capsule, destructor, (void (*)(void*)) dtor) - - -/* - * Sorry, there's simply no place - * to store a Capsule "name" in a CObject. - */ -#define PyCapsule_GetName(capsule) NULL - -static int -PyCapsule_SetName(PyObject *capsule, const char *unused) -{ - unused = unused; - PyErr_SetString(PyExc_NotImplementedError, - "can't use PyCapsule_SetName with CObjects"); - return 1; -} - - - -#define PyCapsule_GetContext(capsule) \ - __PyCapsule_GetField(capsule, desc, (void*) NULL) - -#define PyCapsule_SetContext(capsule, context) \ - __PyCapsule_SetField(capsule, desc, context) - - -static void * -PyCapsule_Import(const char *name, int no_block) -{ - PyObject *object = NULL; - void *return_value = NULL; - char *trace; - size_t name_length = (strlen(name) + 1) * sizeof(char); - char *name_dup = (char *)PyMem_MALLOC(name_length); - - if (!name_dup) { - return NULL; - } - - memcpy(name_dup, name, name_length); - - trace = name_dup; - while (trace) { - char *dot = strchr(trace, '.'); - if (dot) { - *dot++ = '\0'; - } - - if (object == NULL) { - if (no_block) { - object = PyImport_ImportModuleNoBlock(trace); - } else { - object = PyImport_ImportModule(trace); - if (!object) { - PyErr_Format(PyExc_ImportError, - "PyCapsule_Import could not " - "import module \"%s\"", trace); - } - } - } else { - PyObject *object2 = PyObject_GetAttrString(object, trace); - Py_DECREF(object); - object = object2; - } - if (!object) { - goto EXIT; - } - - trace = dot; - } - - if (PyCObject_Check(object)) { - PyCObject *cobject = (PyCObject *)object; - return_value = cobject->cobject; - } else { - PyErr_Format(PyExc_AttributeError, - "PyCapsule_Import \"%s\" is not valid", - name); - } - -EXIT: - Py_XDECREF(object); - if (name_dup) { - PyMem_FREE(name_dup); - } - return return_value; -} - -#endif /* #if PY_VERSION_HEX < 0x02070000 */ - -#endif /* __CAPSULETHUNK_H */ diff --git a/py3c/py3c/comparison.h b/py3c/py3c/comparison.h deleted file mode 100644 index 9d3765a4..00000000 --- a/py3c/py3c/comparison.h +++ /dev/null @@ -1,45 +0,0 @@ -/* Copyright (c) 2015, Red Hat, Inc. and/or its affiliates - * Licensed under the MIT license; see py3c.h - */ - -#ifndef _PY3C_COMPARISON_H_ -#define _PY3C_COMPARISON_H_ -#include - -/* Rich comparisons */ - -#ifndef Py_RETURN_NOTIMPLEMENTED -#define Py_RETURN_NOTIMPLEMENTED \ - return Py_INCREF(Py_NotImplemented), Py_NotImplemented -#endif - -#ifndef Py_UNREACHABLE -#define Py_UNREACHABLE() abort() -#endif - -#ifndef Py_RETURN_RICHCOMPARE -#define Py_RETURN_RICHCOMPARE(val1, val2, op) \ - do { \ - switch (op) { \ - case Py_EQ: if ((val1) == (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ - case Py_NE: if ((val1) != (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ - case Py_LT: if ((val1) < (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ - case Py_GT: if ((val1) > (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ - case Py_LE: if ((val1) <= (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ - case Py_GE: if ((val1) >= (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ - default: \ - Py_UNREACHABLE(); \ - } \ - } while (0) -#endif - -#define PY3C_RICHCMP(val1, val2, op) \ - ((op) == Py_EQ) ? PyBool_FromLong((val1) == (val2)) : \ - ((op) == Py_NE) ? PyBool_FromLong((val1) != (val2)) : \ - ((op) == Py_LT) ? PyBool_FromLong((val1) < (val2)) : \ - ((op) == Py_GT) ? PyBool_FromLong((val1) > (val2)) : \ - ((op) == Py_LE) ? PyBool_FromLong((val1) <= (val2)) : \ - ((op) == Py_GE) ? PyBool_FromLong((val1) >= (val2)) : \ - (Py_INCREF(Py_NotImplemented), Py_NotImplemented) - -#endif diff --git a/py3c/py3c/compat.h b/py3c/py3c/compat.h deleted file mode 100644 index 15d32def..00000000 --- a/py3c/py3c/compat.h +++ /dev/null @@ -1,139 +0,0 @@ -/* Copyright (c) 2015, Red Hat, Inc. and/or its affiliates - * Licensed under the MIT license; see py3c.h - */ - -#ifndef _PY3C_COMPAT_H_ -#define _PY3C_COMPAT_H_ -#include - -#if PY_MAJOR_VERSION >= 3 - -/***** Python 3 *****/ - -#define IS_PY3 1 - -/* Strings */ - -#define PyStr_Type PyUnicode_Type -#define PyStr_Check PyUnicode_Check -#define PyStr_CheckExact PyUnicode_CheckExact -#define PyStr_FromString PyUnicode_FromString -#define PyStr_FromStringAndSize PyUnicode_FromStringAndSize -#define PyStr_FromFormat PyUnicode_FromFormat -#define PyStr_FromFormatV PyUnicode_FromFormatV -#define PyStr_AsString PyUnicode_AsUTF8 -#define PyStr_Concat PyUnicode_Concat -#define PyStr_Format PyUnicode_Format -#define PyStr_InternInPlace PyUnicode_InternInPlace -#define PyStr_InternFromString PyUnicode_InternFromString -#define PyStr_Decode PyUnicode_Decode - -#define PyStr_AsUTF8String PyUnicode_AsUTF8String /* returns PyBytes */ -#define PyStr_AsUTF8 PyUnicode_AsUTF8 -#define PyStr_AsUTF8AndSize PyUnicode_AsUTF8AndSize - -/* Ints */ - -#define PyInt_Type PyLong_Type -#define PyInt_Check PyLong_Check -#define PyInt_CheckExact PyLong_CheckExact -#define PyInt_FromString PyLong_FromString -#define PyInt_FromLong PyLong_FromLong -#define PyInt_FromSsize_t PyLong_FromSsize_t -#define PyInt_FromSize_t PyLong_FromSize_t -#define PyInt_AsLong PyLong_AsLong -#define PyInt_AS_LONG PyLong_AS_LONG -#define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask -#define PyInt_AsSsize_t PyLong_AsSsize_t - -/* Module init */ - -#define MODULE_INIT_FUNC(name) \ - PyMODINIT_FUNC PyInit_ ## name(void); \ - PyMODINIT_FUNC PyInit_ ## name(void) - -#else - -/***** Python 2 *****/ - -#define IS_PY3 0 - -/* Strings */ - -#define PyStr_Type PyString_Type -#define PyStr_Check PyString_Check -#define PyStr_CheckExact PyString_CheckExact -#define PyStr_FromString PyString_FromString -#define PyStr_FromStringAndSize PyString_FromStringAndSize -#define PyStr_FromFormat PyString_FromFormat -#define PyStr_FromFormatV PyString_FromFormatV -#define PyStr_AsString PyString_AsString -#define PyStr_Format PyString_Format -#define PyStr_InternInPlace PyString_InternInPlace -#define PyStr_InternFromString PyString_InternFromString -#define PyStr_Decode PyString_Decode - -#ifdef __GNUC__ -static PyObject *PyStr_Concat(PyObject *left, PyObject *right) __attribute__ ((unused)); -#endif -static PyObject *PyStr_Concat(PyObject *left, PyObject *right) { - PyObject *str = left; - Py_INCREF(left); /* reference to old left will be stolen */ - PyString_Concat(&str, right); - if (str) { - return str; - } else { - return NULL; - } -} - -#define PyStr_AsUTF8String(str) (Py_INCREF(str), (str)) -#define PyStr_AsUTF8 PyString_AsString -#define PyStr_AsUTF8AndSize(pystr, sizeptr) \ - ((*sizeptr=PyString_Size(pystr)), PyString_AsString(pystr)) - -#define PyBytes_Type PyString_Type -#define PyBytes_Check PyString_Check -#define PyBytes_CheckExact PyString_CheckExact -#define PyBytes_FromString PyString_FromString -#define PyBytes_FromStringAndSize PyString_FromStringAndSize -#define PyBytes_FromFormat PyString_FromFormat -#define PyBytes_FromFormatV PyString_FromFormatV -#define PyBytes_Size PyString_Size -#define PyBytes_GET_SIZE PyString_GET_SIZE -#define PyBytes_AsString PyString_AsString -#define PyBytes_AS_STRING PyString_AS_STRING -#define PyBytes_AsStringAndSize PyString_AsStringAndSize -#define PyBytes_Concat PyString_Concat -#define PyBytes_ConcatAndDel PyString_ConcatAndDel -#define _PyBytes_Resize _PyString_Resize - -/* Floats */ - -#define PyFloat_FromString(str) PyFloat_FromString(str, NULL) - -/* Module init */ - -#define PyModuleDef_HEAD_INIT 0 - -typedef struct PyModuleDef { - int m_base; - const char* m_name; - const char* m_doc; - Py_ssize_t m_size; - PyMethodDef *m_methods; -} PyModuleDef; - -#define PyModule_Create(def) \ - Py_InitModule3((def)->m_name, (def)->m_methods, (def)->m_doc) - -#define MODULE_INIT_FUNC(name) \ - static PyObject *PyInit_ ## name(void); \ - PyMODINIT_FUNC init ## name(void); \ - PyMODINIT_FUNC init ## name(void) { PyInit_ ## name(); } \ - static PyObject *PyInit_ ## name(void) - - -#endif - -#endif diff --git a/py3c/py3c/fileshim.h b/py3c/py3c/fileshim.h deleted file mode 100644 index ecf31f40..00000000 --- a/py3c/py3c/fileshim.h +++ /dev/null @@ -1,49 +0,0 @@ -/* Copyright (c) 2015, Red Hat, Inc. and/or its affiliates - * Licensed under the MIT license; see py3c.h - */ - -#ifndef _PY3C_FILESHIM_H_ -#define _PY3C_FILESHIM_H_ -#include - -/* - -For debugging purposes only. -Caveats: - * Only works on file-like objects backed by an actual file - * All C-level writes should be done before additional - Python-level writes are allowed (e.g. by running Python code). - * Though the function tries to flush, there is no guarantee that - writes will be reordered due to different layers of buffering. - -*/ - -static char FLUSH[] = "flush"; -static char EMPTY_STRING[] = ""; - -static FILE* py3c_PyFile_AsFileWithMode(PyObject *py_file, const char *mode) { - FILE *f; - PyObject *ret; - int fd; - - ret = PyObject_CallMethod(py_file, FLUSH, EMPTY_STRING); - if (ret == NULL) { - return NULL; - } - Py_DECREF(ret); - - fd = PyObject_AsFileDescriptor(py_file); - if (fd == -1) { - return NULL; - } - - f = fdopen(fd, mode); - if (f == NULL) { - PyErr_SetFromErrno(PyExc_OSError); - return NULL; - } - - return f; -} - -#endif /* _PY3C_FILESHIM_H_ */ diff --git a/py3c/py3c/py3shims.h b/py3c/py3c/py3shims.h deleted file mode 100644 index 947595e9..00000000 --- a/py3c/py3c/py3shims.h +++ /dev/null @@ -1,44 +0,0 @@ -/* Copyright (c) 2016, Red Hat, Inc. and/or its affiliates - * Licensed under the MIT license; see py3c.h - */ - -/* - * Shims for new functionality from in Python 3.3+ - * - * See https://docs.python.org/3/c-api/memory.html#raw-memory-interface - */ - -#ifndef _PY3C_RAWMALLOC_H_ -#define _PY3C_RAWMALLOC_H_ -#include -#include - - -/* Py_UNUSED - added in Python 3.4, documneted in 3.7 */ - -#ifndef Py_UNUSED -#ifdef __GNUC__ -#define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) -#else -#define Py_UNUSED(name) _unused_ ## name -#endif -#endif - - -/* PyMem_Raw{Malloc,Realloc,Free} - added in Python 3.4 */ - -#if PY_MAJOR_VERSION < 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 4) -#define PyMem_RawMalloc(n) malloc((n) || 1) -#define PyMem_RawRealloc(p, n) realloc(p, (n) || 1) -#define PyMem_RawFree(p) free(p) -#endif /* version < 3.4 */ - - -/* PyMem_RawCalloc - added in Python 3.5 */ - -#if PY_MAJOR_VERSION < 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 5) -#define PyMem_RawCalloc(n, s) calloc((n) || 1, (s) || 1) -#endif /* version < 3.5 */ - - -#endif /* _PY3C_RAWMALLOC_H_ */ diff --git a/py3c/py3c/tpflags.h b/py3c/py3c/tpflags.h deleted file mode 100644 index aa570352..00000000 --- a/py3c/py3c/tpflags.h +++ /dev/null @@ -1,52 +0,0 @@ -/* Copyright (c) 2015, Red Hat, Inc. and/or its affiliates - * Licensed under the MIT license; see py3c.h - */ - -/* - * WARNING: These flags are only to be used in class definitions. - * - * Before including this header file, check that you do not use - * these flags with with PyType_HasFeature. Example command: - * grep -r PyType_HasFeature . - * - * In Python 3, *all objects* have the features corresponding to removed flags. - */ - -#ifndef _PY3C_TPFLAGS_H_ -#define _PY3C_TPFLAGS_H_ -#include - -#if PY_MAJOR_VERSION >= 3 - -#define Py_TPFLAGS_HAVE_GETCHARBUFFER 0 -#define Py_TPFLAGS_HAVE_SEQUENCE_IN 0 -#define Py_TPFLAGS_HAVE_INPLACEOPS 0 -#define Py_TPFLAGS_CHECKTYPES 0 -#define Py_TPFLAGS_HAVE_RICHCOMPARE 0 -#define Py_TPFLAGS_HAVE_WEAKREFS 0 -#define Py_TPFLAGS_HAVE_ITER 0 -#define Py_TPFLAGS_HAVE_CLASS 0 -/* Py_TPFLAGS_HEAPTYPE is still optional in py3 */ -/* Py_TPFLAGS_BASETYPE is still optional in py3 */ -/* Py_TPFLAGS_READY is still useful in py3 */ -/* Py_TPFLAGS_READYING is still useful in py3 */ -/* Py_TPFLAGS_HAVE_GC is still optional in py3 */ -/* Py_TPFLAGS_HAVE_STACKLESS_EXTENSION is still optional in py3 */ -#define Py_TPFLAGS_HAVE_INDEX 0 -/* Py_TPFLAGS_HAVE_VERSION_TAG is still optional in py3 */ -/* Py_TPFLAGS_VALID_VERSION_TAG is still optional in py3 */ -/* Py_TPFLAGS_IS_ABSTRACT is still optional in py3 */ -#define Py_TPFLAGS_HAVE_NEWBUFFER 0 -/* Py_TPFLAGS_INT_SUBCLASS is still optional in py3 */ -/* Py_TPFLAGS_LONG_SUBCLASS is still optional in py3 */ -/* Py_TPFLAGS_LIST_SUBCLASS is still optional in py3 */ -/* Py_TPFLAGS_TUPLE_SUBCLASS is still optional in py3 */ -/* Py_TPFLAGS_STRING_SUBCLASS is still optional in py3 */ -/* Py_TPFLAGS_UNICODE_SUBCLASS is still optional in py3 */ -/* Py_TPFLAGS_DICT_SUBCLASS is still optional in py3 */ -/* Py_TPFLAGS_BASE_EXC_SUBCLASS is still optional in py3 */ -/* Py_TPFLAGS_TYPE_SUBCLASS is still optional in py3 */ - -/* py 3.4 adds Py_TPFLAGS_HAVE_FINALIZE */ -#endif -#endif /* _PY3C_TPFLAGS_H_ */ diff --git a/setup.py b/setup.py index c13e9623..fedf9fdd 100644 --- a/setup.py +++ b/setup.py @@ -6,12 +6,11 @@ # Note: if updating LZ4_REQUIRED_VERSION you need to update docs/install.rst as # well. LZ4_REQUIRED_VERSION = '>= 1.7.5' -PY3C_REQUIRED_VERSION = '>= 1.0' -# Check to see if we have a lz4 and py3c libraries installed on the system, and -# of suitable versions, and use if so. If not, we'll use the bundled libraries. +# Check to see if we have a suitable lz4 library installed on the system and +# use if so. If not, we'll use the bundled libraries. liblz4_found = False -py3c_found = False + try: from pkgconfig import installed as pkgconfig_installed from pkgconfig import cflags as pkgconfig_cflags @@ -34,8 +33,6 @@ def pkgconfig_installed_check(lib, required_version, default): pass return installed liblz4_found = pkgconfig_installed_check('liblz4', LZ4_REQUIRED_VERSION, default=False) - py3c_found = pkgconfig_installed_check('py3c', PY3C_REQUIRED_VERSION, default=False) - # Set up the extension modules. If a system wide lz4 library is found, and is # recent enough, we'll use that. Otherwise we'll build with the bundled one. If @@ -93,9 +90,6 @@ def pkgconfig_installed_check(lib, required_version, default): ] ) -if py3c_found is False: - include_dirs.append('py3c') - compiler = ccompiler.get_default_compiler() extra_link_args = [] From 967356e2bcf6bdb0dbd007ed5db7965e367215be Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 20:24:59 +0000 Subject: [PATCH 117/280] Remove py3c from _version.c --- lz4/_version.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lz4/_version.c b/lz4/_version.c index d477add6..592869b3 100644 --- a/lz4/_version.c +++ b/lz4/_version.c @@ -35,8 +35,7 @@ #define inline #endif -#include -#include +#include #include #include @@ -120,7 +119,8 @@ static struct PyModuleDef moduledef = module_methods }; -MODULE_INIT_FUNC (_version) +PyMODINIT_FUNC +PyInit__version(void) { PyObject *module = PyModule_Create (&moduledef); From 0c8fc15046cc2a9f948ac81bb1c1b45328c8004a Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 20:25:28 +0000 Subject: [PATCH 118/280] Remove PyUNUSED from _version.c --- lz4/_version.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lz4/_version.c b/lz4/_version.c index 592869b3..9f2a4833 100644 --- a/lz4/_version.c +++ b/lz4/_version.c @@ -41,14 +41,6 @@ #include #include -#ifndef Py_UNUSED /* This is already defined for Python 3.4 onwards */ -#ifdef __GNUC__ -#define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) -#else -#define Py_UNUSED(name) _unused_ ## name -#endif -#endif - static PyObject * library_version_number (PyObject * Py_UNUSED (self), PyObject * Py_UNUSED (args)) { From 6b0df7982770c971d50a0a7e9c41f6f2b657eb9f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 20:25:47 +0000 Subject: [PATCH 119/280] Remove unused macros from _version.c --- lz4/_version.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lz4/_version.c b/lz4/_version.c index 9f2a4833..c611f0b3 100644 --- a/lz4/_version.c +++ b/lz4/_version.c @@ -29,12 +29,6 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#if defined(_WIN32) && defined(_MSC_VER) -#define inline __inline -#elif defined(__SUNPRO_C) || defined(__hpux) || defined(_AIX) -#define inline -#endif - #include #include From 44e2863732a4b06e35f826f1af9b39bc30e82222 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 23:45:17 +0000 Subject: [PATCH 120/280] Update README to mention streaming support --- README.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 18fb95f1..ae335e80 100644 --- a/README.rst +++ b/README.rst @@ -28,10 +28,13 @@ This package provides python bindings for the `LZ4 compression library `_. The bindings provided in this package cover the `frame format -`_ and the `block format -`_ specifications. The frame -format bindings are the recommended ones to use, as this guarantees -interoperability with other implementations and language bindings. +`_, the `block +format `_, and the +`streaming +format`_ +specifications. The frame format bindings are the recommended ones to use, as +this guarantees interoperability with other implementations and language +bindings. The API provided by the frame format bindings follows that of the LZMA, zlib, gzip and bzip2 compression libraries which are provided with the Python standard From e20c74680ce5e369aa348d45c92d5604b1ae4bce Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 28 Dec 2019 23:46:28 +0000 Subject: [PATCH 121/280] Fix link in README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ae335e80..50ed6303 100644 --- a/README.rst +++ b/README.rst @@ -30,8 +30,8 @@ This package provides python bindings for the `LZ4 compression library The bindings provided in this package cover the `frame format `_, the `block format `_, and the -`streaming -format`_ +`streaming format +`_ specifications. The frame format bindings are the recommended ones to use, as this guarantees interoperability with other implementations and language bindings. From 270f7d664c3faeede48ecb76a6248cfda1ae64fe Mon Sep 17 00:00:00 2001 From: nelzas Date: Wed, 5 Feb 2020 23:04:11 +0200 Subject: [PATCH 122/280] Fix docstring of compress_begin rename block_mode to block_linked as this the actual name of the argument --- lz4/frame/_frame.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 4dc42f75..ebf84a0f 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1396,7 +1396,7 @@ PyDoc_STRVAR ( compress_begin__doc, "compress_begin(context, source_size=0, compression_level=0, block_size=0,\n" \ - "content_checksum=0, content_size=1, block_mode=0, frame_type=0,\n" \ + "content_checksum=0, content_size=1, block_linked=0, frame_type=0,\n" \ "auto_flush=1)\n" \ "\n" \ "Creates a frame header from a compression context.\n\n" \ From b9cb055cfabd29cd65b44800c8e85a3fb157b846 Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Tue, 31 Mar 2020 14:13:13 +0200 Subject: [PATCH 123/280] Check for available memory instead of its total amount in tests This change prevents the python interpreter from triggering the OOM-killer while trying to use/allocate more memory than available on the system. --- tests/block/test_block_2.py | 2 +- tests/stream/test_stream_1.py | 8 ++++---- tests/stream/test_stream_2.py | 12 ++++++------ tests/stream/test_stream_3.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/block/test_block_2.py b/tests/block/test_block_2.py index 87ceefb7..a2aea4da 100644 --- a/tests/block/test_block_2.py +++ b/tests/block/test_block_2.py @@ -28,7 +28,7 @@ reason='Py_ssize_t too small for this test' ) @pytest.mark.skipif( - psutil.virtual_memory().total < _4GB, + psutil.virtual_memory().available < _4GB, reason='Insufficient system memory for this test' ) def test_huge(): diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py index 3386b050..abb87649 100644 --- a/tests/stream/test_stream_1.py +++ b/tests/stream/test_stream_1.py @@ -150,7 +150,7 @@ def test_invalid_config_c_4(store_comp_size): if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().total < 3 * c_kwargs['buffer_size']: + if psutil.virtual_memory().available < 3 * c_kwargs['buffer_size']: # The internal LZ4 context will request at least 3 times buffer_size # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size # for the output buffer) @@ -210,7 +210,7 @@ def test_invalid_config_c_5(): if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().total < 3 * c_kwargs['buffer_size']: + if psutil.virtual_memory().available < 3 * c_kwargs['buffer_size']: # The internal LZ4 context will request at least 3 times buffer_size # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size # for the output buffer) @@ -243,7 +243,7 @@ def test_invalid_config_d_5(): if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().total < 3 * d_kwargs['buffer_size']: + if psutil.virtual_memory().available < 3 * d_kwargs['buffer_size']: # The internal LZ4 context will request at least 3 times buffer_size # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size # for the output buffer) @@ -260,7 +260,7 @@ def test_invalid_config_d_5(): if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().total < 3 * d_kwargs['buffer_size']: + if psutil.virtual_memory().available < 3 * d_kwargs['buffer_size']: # The internal LZ4 context will request at least 3 times buffer_size # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size # for the output buffer) diff --git a/tests/stream/test_stream_2.py b/tests/stream/test_stream_2.py index 10e148ec..d6db6ec1 100644 --- a/tests/stream/test_stream_2.py +++ b/tests/stream/test_stream_2.py @@ -35,7 +35,7 @@ reason='Py_ssize_t too small for this test' ) @pytest.mark.skipif( - psutil.virtual_memory().total < _4GB or huge is None, + psutil.virtual_memory().available < _4GB or huge is None, reason='Insufficient system memory for this test' ) def test_huge_1(): @@ -47,7 +47,7 @@ def test_huge_1(): 'dictionary': huge, } - if psutil.virtual_memory().total < 3 * kwargs['buffer_size']: + if psutil.virtual_memory().available < 3 * kwargs['buffer_size']: # The internal LZ4 context will request at least 3 times buffer_size # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size # for the output buffer) @@ -74,7 +74,7 @@ def test_huge_1(): reason='Py_ssize_t too small for this test' ) @pytest.mark.skipif( - psutil.virtual_memory().total < _4GB or huge is None, + psutil.virtual_memory().available < _4GB or huge is None, reason='Insufficient system memory for this test' ) def test_huge_2(): @@ -86,7 +86,7 @@ def test_huge_2(): 'dictionary': b'', } - if psutil.virtual_memory().total < 3 * kwargs['buffer_size']: + if psutil.virtual_memory().available < 3 * kwargs['buffer_size']: # The internal LZ4 context will request at least 3 times buffer_size # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size # for the output buffer) @@ -114,7 +114,7 @@ def test_huge_2(): reason='Py_ssize_t too small for this test' ) @pytest.mark.skipif( - psutil.virtual_memory().total < _4GB or huge is None, + psutil.virtual_memory().available < _4GB or huge is None, reason='Insufficient system memory for this test' ) def test_huge_3(): @@ -126,7 +126,7 @@ def test_huge_3(): 'dictionary': huge, } - if psutil.virtual_memory().total < 3 * kwargs['buffer_size']: + if psutil.virtual_memory().available < 3 * kwargs['buffer_size']: # The internal LZ4 context will request at least 3 times buffer_size # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size # for the output buffer) diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index 74793ad6..60e0ffb0 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -89,7 +89,7 @@ def test_block_decompress_mem_usage(data, buffer_size): if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().total < 3 * kwargs['buffer_size']: + if psutil.virtual_memory().available < 3 * kwargs['buffer_size']: # The internal LZ4 context will request at least 3 times buffer_size # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size # for the output buffer) From afbfd24aee324ee486855f84ae0e99f73cb11b8c Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Tue, 31 Mar 2020 14:16:07 +0200 Subject: [PATCH 124/280] Avoid allocating memory that will not be used in the test This change avoids triggering the OOM-killer while loading this test. --- tests/stream/test_stream_2.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/stream/test_stream_2.py b/tests/stream/test_stream_2.py index d6db6ec1..5578f832 100644 --- a/tests/stream/test_stream_2.py +++ b/tests/stream/test_stream_2.py @@ -20,10 +20,14 @@ # check for the TRAVIS environment variable being set. This is quite # fragile. -try: - huge = b'\0' * _4GB -except (MemoryError, OverflowError): +if os.environ.get('TRAVIS') is not None or sys.maxsize < _4GB or \ + psutil.virtual_memory().available < _4GB: huge = None +else: + try: + huge = b'\0' * _4GB + except (MemoryError, OverflowError): + huge = None @pytest.mark.skipif( From 5fdda91832fe1039fb90279a08dbf6abffb762ac Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Tue, 31 Mar 2020 14:18:11 +0200 Subject: [PATCH 125/280] Reduce the number of test cases for the stream bindings This changes aims to avoid reaching the 50-minute limit in Travis jobs. --- tests/stream/conftest.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/stream/conftest.py b/tests/stream/conftest.py index ea9fa83d..c290ce0a 100644 --- a/tests/stream/conftest.py +++ b/tests/stream/conftest.py @@ -5,9 +5,9 @@ test_data = [ (b''), (os.urandom(8 * 1024)), - (b'0' * 8 * 1024), - (bytearray(b'')), - (bytearray(os.urandom(8 * 1024))), + # (b'0' * 8 * 1024), + # (bytearray(b'')), + # (bytearray(os.urandom(8 * 1024))), (bytearray(open(os.path.join(os.path.dirname(__file__), 'numpy_byte_array.bin'), 'rb').read())) ] @@ -43,15 +43,16 @@ def strategy(request): # 4, # 8, # 64, - 256, + # 256, 941, - 1 * 1024, + # 1 * 1024, # 4 * 1024, # 8 * 1024, # 16 * 1024, # 32 * 1024, 64 * 1024, - 128 * 1024] + # 128 * 1024 + ] ) @@ -122,11 +123,11 @@ def d_return_bytearray(return_bytearray): ] + [ ('fast', None) ] + [ - ('fast', {'acceleration': s}) for s in range(10) + ('fast', {'acceleration': 2 * s}) for s in range(5) ] + [ ('high_compression', None) ] + [ - ('high_compression', {'compression_level': s}) for s in range(17) + ('high_compression', {'compression_level': 2 * s}) for s in range(9) ] + [ (None, None) ] From 2055ace0cfd8d1d1468faf6d4538b5b0ea2ea6dc Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Tue, 31 Mar 2020 16:51:39 +0200 Subject: [PATCH 126/280] Make _get_block API raising an error in case of invalid call In the stream API, the _get_block function usage is restricted to the cases when the compressed block size is recorded in-band, i.e. leading the actual block data. --- lz4/stream/_stream.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index a88a0432..d80e57d7 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -1253,6 +1253,13 @@ _get_block (PyObject * Py_UNUSED (self), PyObject * args) goto exit_now; } + if (context->config.store_comp_size == 0) + { + PyErr_Format (LZ4StreamError, + "LZ4 context is configured for storing block size out-of-band"); + goto exit_now; + } + if (source.len < context->config.store_comp_size) { PyErr_Format (LZ4StreamError, From 791c63f58a38772f85e75b7601bc35503a8cf1a5 Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Tue, 31 Mar 2020 16:52:25 +0200 Subject: [PATCH 127/280] Add out-of-band block size record support to the stream bindings --- lz4/stream/__init__.py | 8 +++++--- lz4/stream/_stream.c | 39 ++++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/lz4/stream/__init__.py b/lz4/stream/__init__.py index 60a119ea..7458d9c0 100644 --- a/lz4/stream/__init__.py +++ b/lz4/stream/__init__.py @@ -26,7 +26,8 @@ def __init__(self, strategy, buffer_size, return_bytearray=False, store_comp_siz will return a ``bytes`` object. If ``True``, then the function will return a ``bytearray`` object. store_comp_size (int): Specify the size in bytes of the following - compressed block. Can be: ``1``, ``2`` or ``4`` (default: ``4``). + compressed block. Can be: ``0`` (meaning out-of-band block size), + ``1``, ``2`` or ``4`` (default: ``4``). dictionary (str, bytes or buffer-compatible object): If specified, perform decompression using this initial dictionary. @@ -140,8 +141,9 @@ def __init__(self, strategy, buffer_size, mode="default", acceleration=True, com return_bytearray (bool): If ``False`` (the default) then the function will return a bytes object. If ``True``, then the function will return a bytearray object. - store_comp_size (int): Specify the size in bytes of the following - compressed block. Can be: ``1``, ``2`` or ``4`` (default: ``4``). + store_comp_size (int): Specify the size in bytes of the following + compressed block. Can be: ``0`` (meaning out-of-band block size), + ``1``, ``2`` or ``4`` (default: ``4``). dictionary (str, bytes or buffer-compatible object): If specified, perform compression using this initial dictionary. diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index d80e57d7..ad5149c2 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -783,6 +783,7 @@ _create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) int status = 0; int total_size = 0; + uint32_t store_max_size; static char * argnames[] = { "strategy", @@ -817,10 +818,11 @@ _create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) * https://github.com/lz4/lz4/blob/dev/lib/lz4.h#L161 * * So, restrict the block length bitwise to 32 (signed 32 bit integer). */ - if ((store_comp_size != 1) && (store_comp_size != 2) && (store_comp_size != 4)) + if ((store_comp_size != 0) && (store_comp_size != 1) && + (store_comp_size != 2) && (store_comp_size != 4)) { PyErr_Format (PyExc_ValueError, - "Invalid store_comp_size, valid values: 1, 2 or 4"); + "Invalid store_comp_size, valid values: 0, 1, 2 or 4"); goto abort_now; } @@ -888,11 +890,22 @@ _create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) goto abort_now; } - /* Initialize the output buffer */ - if (context->config.direction == COMPRESS) + /* Initialize the output buffer + * + * In out-of-band block size case, use a best-effort strategy for scaling + * buffers. + */ + if (store_comp_size == 0) { - uint32_t store_max_size = _GET_MAX_UINT32(store_comp_size); + store_max_size = _GET_MAX_UINT32(4); + } + else + { + store_max_size = _GET_MAX_UINT32(store_comp_size); + } + if (context->config.direction == COMPRESS) + { context->output.len = get_compress_bound (buffer_size); total_size = context->output.len + context->config.store_comp_size; @@ -919,7 +932,6 @@ _create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) } else /* context->config.direction == DECOMPRESS */ { - uint32_t store_max_size = _GET_MAX_UINT32(store_comp_size); if (store_max_size > LZ4_MAX_INPUT_SIZE) { store_max_size = LZ4_MAX_INPUT_SIZE; @@ -1308,6 +1320,7 @@ _decompress (PyObject * Py_UNUSED (self), PyObject * args) PyObject * py_context = NULL; PyObject * py_dest = NULL; int output_size = 0; + uint32_t source_size_max = 0; Py_buffer source = { NULL, NULL, }; /* Positional arguments: capsule_context, source @@ -1325,7 +1338,19 @@ _decompress (PyObject * Py_UNUSED (self), PyObject * args) goto exit_now; } - if (source.len > _GET_MAX_UINT32(context->config.store_comp_size)) + /* In out-of-band block size case, use a best-effort strategy for scaling + * buffers. + */ + if (context->config.store_comp_size == 0) + { + source_size_max = _GET_MAX_UINT32(4); + } + else + { + source_size_max = _GET_MAX_UINT32(context->config.store_comp_size); + } + + if (source.len > source_size_max) { PyErr_Format (PyExc_OverflowError, "Source length (%ld) too large for LZ4 store_comp_size (%d) value", From f426ed40fb70f90f968fa028fbc7550505496501 Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Tue, 31 Mar 2020 18:00:55 +0200 Subject: [PATCH 128/280] Add documentation for the stream binding out-of-band block size record feature --- docs/lz4.stream.rst | 33 +++++++++++++++++++++++++++++++++ docs/quickstart.rst | 3 ++- lz4/stream/__init__.py | 2 ++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/lz4.stream.rst b/docs/lz4.stream.rst index dc190a55..d7b8c3af 100644 --- a/docs/lz4.stream.rst +++ b/docs/lz4.stream.rst @@ -49,6 +49,39 @@ To use the lz4 stream format bindings is straightforward: >>> decompressed_stream == origin_stream True +Out-of-band block size record example +------------------------------------- +.. doctest:: + + >>> from lz4.stream import LZ4StreamCompressor, LZ4StreamDecompressor + >>> import os + >>> page_size = 8192 # LZ4 context double buffer page size + >>> out_of_band_block_sizes = [] # Store the block sizes + >>> origin_stream = 10 * 1024 * os.urandom(1024) # 10MiB + >>> # LZ4 stream compression of origin_stream into compressed_stream: + >>> compressed_stream = bytearray() + >>> with LZ4StreamCompressor("double_buffer", page_size, store_comp_size=0) as proc: + ... offset = 0 + ... while offset < len(origin_stream): + ... chunk = origin_stream[offset:offset + page_size] + ... block = proc.compress(chunk) + ... out_of_band_block_sizes.append(len(block)) + ... compressed_stream.extend(block) + ... offset += page_size + >>> # LZ4 stream decompression of compressed_stream into decompressed_stream: + >>> decompressed_stream = bytearray() + >>> with LZ4StreamDecompressor("double_buffer", page_size, store_comp_size=0) as proc: + ... offset = 0 + ... for block_len in out_of_band_block_sizes: + ... # Sanity check: + ... if offset >= len(compressed_stream): + ... raise LZ4StreamError("Truncated stream") + ... block = compressed_stream[offset:offset + block_len] + ... chunk = proc.decompress(block) + ... decompressed_stream.extend(chunk) + ... offset += block_len + >>> decompressed_stream == origin_stream + True Contents ---------------- diff --git a/docs/quickstart.rst b/docs/quickstart.rst index a39cc3b2..046b9724 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -281,4 +281,5 @@ Storing the compressed data size in the block The ``store_comp_size`` argument allows tuning of the size (in bytes) of the compressed block, which is prepended to the actual LZ4 compressed payload. -This size can be either on ``1``, ``2`` or ``4`` bytes. +This size can be either on ``1``, ``2`` or ``4`` bytes, or ``0`` for out-of-band +block size record. diff --git a/lz4/stream/__init__.py b/lz4/stream/__init__.py index 7458d9c0..4b007464 100644 --- a/lz4/stream/__init__.py +++ b/lz4/stream/__init__.py @@ -104,6 +104,8 @@ def get_block(self, stream): MemoryError: raised if the output buffer cannot be allocated. OverflowError: raised if the source is too large for being handled by the given context. + LZ4StreamError: raised if used while in an out-of-band block size record + configuration. """ return _get_block(self._context, stream) From 23abfaab708482dbb9ec546df2d416862b60f2af Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Wed, 1 Apr 2020 00:00:08 +0200 Subject: [PATCH 129/280] Add tests for the stream binding out-of-band block size record feature --- tests/stream/test_stream_4.py | 160 ++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tests/stream/test_stream_4.py diff --git a/tests/stream/test_stream_4.py b/tests/stream/test_stream_4.py new file mode 100644 index 00000000..5128350d --- /dev/null +++ b/tests/stream/test_stream_4.py @@ -0,0 +1,160 @@ +import lz4.stream +import pytest +import sys +import os +import gc + + +def run_gc(func): + if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + else: + def wrapper(*args, **kwargs): + gc.collect() + try: + result = func(*args, **kwargs) + finally: + gc.collect() + return result + + wrapper.__name__ = func.__name__ + return wrapper + + +if sys.version_info < (3, ): + from struct import pack, unpack + + def _get_format(length, byteorder, signed): + _order = {'l': '<', 'b': '>'} + _fmt = {1: 'b', 2: 'h', 4: 'i', 8: 'q'} + _sign = {True: lambda x: x.lower(), False: lambda x: x.upper()} + return _sign[signed](_order[byteorder[0].lower()] + _fmt[length]) + + def int_to_bytes(value, length=4, byteorder='little', signed=False): + return bytearray(pack(_get_format(length, byteorder, signed), value)) + + def int_from_bytes(bytes, byteorder='little', signed=False): + return unpack(_get_format(len(bytes), byteorder, signed), bytes)[0] + +else: + def int_to_bytes(value, length=4, byteorder='little', signed=False): + return value.to_bytes(length, byteorder, signed=signed) + + def int_from_bytes(bytes, byteorder='little', signed=False): + return int.from_bytes(bytes, byteorder, signed=signed) + +# Out-of-band block size record tests + + +@run_gc +def test_round_trip(): + data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 + kwargs = {'strategy': "double_buffer", 'buffer_size': 256, 'store_comp_size': 4} + + oob_kwargs = {} + oob_kwargs.update(kwargs) + oob_kwargs['store_comp_size'] = 0 + + ib_cstream = bytearray() + oob_cstream = bytearray() + oob_sizes = [] + + with lz4.stream.LZ4StreamCompressor(**kwargs) as ib_proc, \ + lz4.stream.LZ4StreamCompressor(**oob_kwargs) as oob_proc: + for start in range(0, len(data), kwargs['buffer_size']): + chunk = data[start:start + kwargs['buffer_size']] + ib_block = ib_proc.compress(chunk) + oob_block = oob_proc.compress(chunk) + + assert (len(ib_block) == (len(oob_block) + kwargs['store_comp_size'])), \ + "Blocks size mismatch: " \ + "{}/{}".format(len(ib_block), len(oob_block) + kwargs['store_comp_size']) + + assert (int_from_bytes(ib_block[:kwargs['store_comp_size']]) == len(oob_block)), \ + "Blocks size record mismatch: got {}, expected {}".format( + int_from_bytes(ib_block[:kwargs['store_comp_size']]), + len(oob_block)) + + assert (ib_block[kwargs['store_comp_size']:] == oob_block), "Blocks data mismatch" + + ib_cstream += ib_block + oob_cstream += oob_block + oob_sizes.append(len(oob_block)) + + ib_dstream = bytearray() + oob_dstream = bytearray() + + with lz4.stream.LZ4StreamDecompressor(**kwargs) as ib_proc, \ + lz4.stream.LZ4StreamDecompressor(**oob_kwargs) as oob_proc: + ib_offset = 0 + oob_index = 0 + oob_offset = 0 + while ib_offset < len(ib_cstream) and oob_index < len(oob_sizes): + ib_block = ib_proc.get_block(ib_cstream[ib_offset:]) + oob_block = oob_cstream[oob_offset:oob_offset + oob_sizes[oob_index]] + + assert (len(ib_block) == len(oob_block)), \ + "Blocks size mismatch: {}/{}".format(len(ib_block), len(oob_block)) + + assert (ib_block == oob_block), "Blocks data mismatch" + + ib_chunk = ib_proc.decompress(ib_block) + oob_chunk = oob_proc.decompress(oob_block) + + assert (len(ib_chunk) == len(oob_chunk)), \ + "Chunks size mismatch: {}/{}".format(len(ib_chunk), len(oob_chunk)) + + assert (ib_chunk == oob_chunk), "Chunks data mismatch" + + ib_dstream += ib_chunk + oob_dstream += oob_chunk + + ib_offset += kwargs['store_comp_size'] + len(ib_block) + oob_offset += oob_sizes[oob_index] + oob_index += 1 + + assert (len(ib_dstream) == len(oob_dstream)), "Decompressed streams length mismatch" + + assert (len(data) == len(ib_dstream)), "Decompressed streams length mismatch" + + assert (len(data) == len(oob_dstream)), "Decompressed streams length mismatch" + + assert (ib_dstream == oob_dstream), "Decompressed streams mismatch" + + assert (data == ib_dstream), "Decompressed streams mismatch" + + assert (data == oob_dstream), "Decompressed streams mismatch" + + +@run_gc +def test_invalid_usage(): + data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 + kwargs = {'strategy': "double_buffer", 'buffer_size': 256, 'store_comp_size': 0} + + cstream = bytearray() + oob_sizes = [] + + with lz4.stream.LZ4StreamCompressor(**kwargs) as proc: + for start in range(0, len(data), kwargs['buffer_size']): + chunk = data[start:start + kwargs['buffer_size']] + block = proc.compress(chunk) + cstream += block + oob_sizes.append(len(block)) + + message = r"^LZ4 context is configured for storing block size out-of-band$" + + with pytest.raises(lz4.stream.LZ4StreamError, match=message): + dstream = bytearray() + + with lz4.stream.LZ4StreamDecompressor(**kwargs) as proc: + offset = 0 + index = 0 + while offset < len(cstream): + block = proc.get_block(cstream[offset:]) + chunk = proc.decompress(block) + + dstream += chunk + + offset += kwargs['store_comp_size'] + len(block) + index += 1 From bfa2a4fb8f45e1154899b157c20d5712cd07f553 Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Tue, 31 Mar 2020 15:27:03 +0200 Subject: [PATCH 130/280] Bug fix: Fix output buffer size calculation The store_comp_size field from the context is not yet set. --- lz4/stream/_stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index ad5149c2..11fd3748 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -907,7 +907,7 @@ _create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) if (context->config.direction == COMPRESS) { context->output.len = get_compress_bound (buffer_size); - total_size = context->output.len + context->config.store_comp_size; + total_size = context->output.len + store_comp_size; if (context->output.len == 0) { From 35fc6c5900d6df58f0df0193cf19e8c7ccacfaf9 Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Tue, 31 Mar 2020 18:01:37 +0200 Subject: [PATCH 131/280] Fix typo in README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 50ed6303..3050f485 100644 --- a/README.rst +++ b/README.rst @@ -45,8 +45,8 @@ and file handler support. The bindings drop the GIL when calling in to the underlying LZ4 library, and is thread safe. An extensive test suite is included. -Documenation -============ +Documentation +============= .. image:: https://readthedocs.org/projects/python-lz4/badge/?version=stable :target: https://readthedocs.org/projects/python-lz4/ From 2dc29fb4f38f495194acfaa8854e224cddead7ed Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Thu, 2 Apr 2020 14:27:40 +0200 Subject: [PATCH 132/280] Bug fix: Allow output buffer size to be larger than 2GB Using unsigned int for output buffer size avoids size over 2GB being considered as negative, which making PyMem_Malloc() calls fail. Fixes #192 --- lz4/stream/_stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 11fd3748..05d7db1e 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -782,7 +782,7 @@ _create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) Py_buffer dict = { NULL, NULL, }; int status = 0; - int total_size = 0; + unsigned int total_size = 0; uint32_t store_max_size; static char * argnames[] = { From 651c06ad3e9437d616a13b07913b45a290be031a Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Thu, 2 Apr 2020 15:14:28 +0200 Subject: [PATCH 133/280] Remove unneeded parameter in stream tests --- tests/stream/test_stream_1.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py index abb87649..f61d38cd 100644 --- a/tests/stream/test_stream_1.py +++ b/tests/stream/test_stream_1.py @@ -89,7 +89,7 @@ def test_invalid_config_c_1(): lz4.stream.LZ4StreamCompressor(**c_kwargs) -def test_invalid_config_d_1(store_comp_size): +def test_invalid_config_d_1(): d_kwargs = {} d_kwargs['strategy'] = "ring_buffer" d_kwargs['buffer_size'] = 1024 @@ -107,7 +107,7 @@ def test_invalid_config_c_2(): lz4.stream.LZ4StreamCompressor(**c_kwargs) -def test_invalid_config_d_2(store_comp_size): +def test_invalid_config_d_2(): d_kwargs = {} d_kwargs['strategy'] = "foo" d_kwargs['buffer_size'] = 1024 From 23a06b77b5bf4e4adfadad4d20c6e0853d3ec0cd Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Thu, 2 Apr 2020 15:22:37 +0200 Subject: [PATCH 134/280] Do not skip tests that can run --- tests/stream/test_stream_1.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py index f61d38cd..f22e6429 100644 --- a/tests/stream/test_stream_1.py +++ b/tests/stream/test_stream_1.py @@ -143,18 +143,8 @@ def test_invalid_config_c_4(store_comp_size): c_kwargs.update(store_comp_size) if store_comp_size['store_comp_size'] >= 4: - - if os.environ.get('TRAVIS') is not None: - pytest.skip('Skipping test on Travis due to insufficient memory') - - if sys.maxsize < 0xffffffff: - pytest.skip('Py_ssize_t too small for this test') - - if psutil.virtual_memory().available < 3 * c_kwargs['buffer_size']: - # The internal LZ4 context will request at least 3 times buffer_size - # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size - # for the output buffer) - pytest.skip('Insufficient system memory for this test') + # No need for skiping this test case, since arguments check is + # expecting to raise an error. # Make sure the page size is larger than what the input bound will be, # but still fit in 4 bytes From 29b8cd0180674a325567d3623ffc1bcfa993aaee Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Thu, 2 Apr 2020 15:23:28 +0200 Subject: [PATCH 135/280] Review comments in test_stream_1.py --- tests/stream/test_stream_1.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py index f22e6429..92af9abb 100644 --- a/tests/stream/test_stream_1.py +++ b/tests/stream/test_stream_1.py @@ -402,9 +402,9 @@ def test_decompress_truncated(): decompress(compressed[:n], d_kwargs) -# This next is probably redundant given test_decompress_truncated above since -# the trailing bytes will be considered as the truncated last block, but we'll -# keep them for now +# This next test is probably redundant given test_decompress_truncated above +# since the trailing bytes will be considered as the truncated last block, but +# we will keep them for now def test_decompress_with_trailer(): From 84b110bc95d6a100081821c0daab9bbca4ad9794 Mon Sep 17 00:00:00 2001 From: Samuel Martin Date: Thu, 2 Apr 2020 15:30:40 +0200 Subject: [PATCH 136/280] Reduce pressure on memory in stream tests This change runs the python garbage collector before and after each stream test. The garbage collector is disabled in the CI since it has a significant impact on the duration of the jobs (which are time constrained in the CI). --- tests/stream/test_stream_0.py | 20 ++++++++++++ tests/stream/test_stream_1.py | 61 +++++++++++++++++++++++++++++++++++ tests/stream/test_stream_2.py | 21 ++++++++++++ tests/stream/test_stream_3.py | 19 +++++++++++ 4 files changed, 121 insertions(+) diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index fd2354ef..c7d34bcc 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -1,10 +1,29 @@ import lz4.stream import sys import pytest +import gc +import os if sys.version_info <= (3, 2): import struct +def run_gc(func): + if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + else: + def wrapper(*args, **kwargs): + gc.collect() + try: + result = func(*args, **kwargs) + finally: + gc.collect() + return result + + wrapper.__name__ = func.__name__ + return wrapper + + def get_stored_size(buff, block_length_size): if sys.version_info > (2, 7): if isinstance(buff, memoryview): @@ -26,6 +45,7 @@ def get_stored_size(buff, block_length_size): return struct.unpack('<' + fmt[block_length_size], b[:block_length_size])[0] +@run_gc def roundtrip(x, c_kwargs, d_kwargs, dictionary): if dictionary: if isinstance(dictionary, tuple): diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py index 92af9abb..50c558b9 100644 --- a/tests/stream/test_stream_1.py +++ b/tests/stream/test_stream_1.py @@ -3,6 +3,8 @@ import sys import os import psutil +import gc + if sys.version_info < (3, ): from struct import pack, unpack @@ -42,6 +44,40 @@ def int_from_bytes(bytes, byteorder='little', signed=False): # fragile. +def run_gc(func): + if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + else: + def wrapper(*args, **kwargs): + gc.collect() + try: + result = func(*args, **kwargs) + finally: + gc.collect() + return result + + wrapper.__name__ = func.__name__ + return wrapper + + +def run_gc_param_store_comp_size(func): + if os.environ.get('TRAVIS') is not None: + def wrapper(store_comp_size, *args, **kwargs): + return func(store_comp_size, *args, **kwargs) + else: + def wrapper(store_comp_size, *args, **kwargs): + gc.collect() + try: + result = func(store_comp_size, *args, **kwargs) + finally: + gc.collect() + return result + + wrapper.__name__ = func.__name__ + return wrapper + + def compress(x, c_kwargs, return_block_offset=False, check_block_type=False): o = [0, ] if c_kwargs.get('return_bytearray', False): @@ -80,6 +116,7 @@ def decompress(x, d_kwargs, check_chunk_type=False): return d +@run_gc def test_invalid_config_c_1(): c_kwargs = {} c_kwargs['strategy'] = "ring_buffer" @@ -89,6 +126,7 @@ def test_invalid_config_c_1(): lz4.stream.LZ4StreamCompressor(**c_kwargs) +@run_gc def test_invalid_config_d_1(): d_kwargs = {} d_kwargs['strategy'] = "ring_buffer" @@ -98,6 +136,7 @@ def test_invalid_config_d_1(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) +@run_gc def test_invalid_config_c_2(): c_kwargs = {} c_kwargs['strategy'] = "foo" @@ -107,6 +146,7 @@ def test_invalid_config_c_2(): lz4.stream.LZ4StreamCompressor(**c_kwargs) +@run_gc def test_invalid_config_d_2(): d_kwargs = {} d_kwargs['strategy'] = "foo" @@ -116,6 +156,7 @@ def test_invalid_config_d_2(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) +@run_gc_param_store_comp_size def test_invalid_config_c_3(store_comp_size): c_kwargs = {} c_kwargs['strategy'] = "double_buffer" @@ -126,6 +167,7 @@ def test_invalid_config_c_3(store_comp_size): lz4.stream.LZ4StreamCompressor(**c_kwargs) +@run_gc_param_store_comp_size def test_invalid_config_d_3(store_comp_size): d_kwargs = {} d_kwargs['strategy'] = "double_buffer" @@ -136,6 +178,7 @@ def test_invalid_config_d_3(store_comp_size): lz4.stream.LZ4StreamDecompressor(**d_kwargs) +@run_gc_param_store_comp_size def test_invalid_config_c_4(store_comp_size): c_kwargs = {} c_kwargs['strategy'] = "double_buffer" @@ -161,6 +204,7 @@ def test_invalid_config_c_4(store_comp_size): lz4.stream.LZ4StreamCompressor(**c_kwargs) +@run_gc_param_store_comp_size def test_invalid_config_d_4(store_comp_size): d_kwargs = {} d_kwargs['strategy'] = "double_buffer" @@ -189,6 +233,7 @@ def test_invalid_config_d_4(store_comp_size): lz4.stream.LZ4StreamDecompressor(**d_kwargs) +@run_gc def test_invalid_config_c_5(): c_kwargs = {} c_kwargs['strategy'] = "double_buffer" @@ -220,6 +265,7 @@ def test_invalid_config_c_5(): lz4.stream.LZ4StreamCompressor(**c_kwargs) +@run_gc def test_invalid_config_d_5(): d_kwargs = {} d_kwargs['strategy'] = "double_buffer" @@ -276,6 +322,7 @@ def test_invalid_config_d_5(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) +@run_gc def test_decompress_corrupted_input_1(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -291,6 +338,7 @@ def test_decompress_corrupted_input_1(): decompress(data[4:], d_kwargs) +@run_gc def test_decompress_corrupted_input_2(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -315,6 +363,7 @@ def test_decompress_corrupted_input_2(): decompress(data, d_kwargs) +@run_gc def test_decompress_corrupted_input_3(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -344,6 +393,7 @@ def test_decompress_corrupted_input_3(): decompress(data, d_kwargs) +@run_gc def test_decompress_corrupted_input_4(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -375,6 +425,7 @@ def test_decompress_corrupted_input_4(): decompress(data, d_kwargs) +@run_gc def test_decompress_truncated(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -407,6 +458,7 @@ def test_decompress_truncated(): # we will keep them for now +@run_gc def test_decompress_with_trailer(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -433,6 +485,7 @@ def test_decompress_with_trailer(): decompress(comp + b'\x00' * n, d_kwargs) +@run_gc def test_unicode(): if sys.version_info < (3,): return # skip @@ -452,6 +505,7 @@ def test_unicode(): # for now +@run_gc def test_return_bytearray(): if sys.version_info < (3,): return # skip @@ -473,6 +527,7 @@ def test_return_bytearray(): assert bytes(b) == data +@run_gc def test_memoryview(): if sys.version_info < (2, 7): return # skip @@ -488,6 +543,7 @@ def test_memoryview(): assert decompress(memoryview(compressed), d_kwargs) == data +@run_gc def test_with_dict_none(): kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -530,6 +586,7 @@ def test_with_dict_none(): assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data +@run_gc def test_with_dict(): kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -568,6 +625,7 @@ def test_with_dict(): assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data +@run_gc def test_known_decompress_1(): d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -582,6 +640,7 @@ def test_known_decompress_1(): assert decompress(input, d_kwargs) == output +@run_gc def test_known_decompress_2(): d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -590,6 +649,7 @@ def test_known_decompress_2(): assert decompress(input, d_kwargs) == output +@run_gc def test_known_decompress_3(): d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -599,6 +659,7 @@ def test_known_decompress_3(): assert decompress(input, d_kwargs) == output +@run_gc def test_known_decompress_4(): d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} diff --git a/tests/stream/test_stream_2.py b/tests/stream/test_stream_2.py index 5578f832..de0693a4 100644 --- a/tests/stream/test_stream_2.py +++ b/tests/stream/test_stream_2.py @@ -3,6 +3,24 @@ import lz4.stream import psutil import os +import gc + + +def run_gc(func): + if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + else: + def wrapper(*args, **kwargs): + gc.collect() + try: + result = func(*args, **kwargs) + finally: + gc.collect() + return result + + wrapper.__name__ = func.__name__ + return wrapper # This test requires allocating a big lump of memory. In order to @@ -42,6 +60,7 @@ psutil.virtual_memory().available < _4GB or huge is None, reason='Insufficient system memory for this test' ) +@run_gc def test_huge_1(): data = b'' kwargs = { @@ -81,6 +100,7 @@ def test_huge_1(): psutil.virtual_memory().available < _4GB or huge is None, reason='Insufficient system memory for this test' ) +@run_gc def test_huge_2(): data = huge kwargs = { @@ -121,6 +141,7 @@ def test_huge_2(): psutil.virtual_memory().available < _4GB or huge is None, reason='Insufficient system memory for this test' ) +@run_gc def test_huge_3(): data = huge kwargs = { diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index 60e0ffb0..352efbdc 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -3,6 +3,7 @@ import sys import os import psutil +import gc _1KB = 1024 @@ -10,6 +11,23 @@ _1GB = _1MB * 1024 +def run_gc_param_data_buffer_size(func): + if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: + def wrapper(data, buffer_size, *args, **kwargs): + return func(data, buffer_size, *args, **kwargs) + else: + def wrapper(data, buffer_size, *args, **kwargs): + gc.collect() + try: + result = func(data, buffer_size, *args, **kwargs) + finally: + gc.collect() + return result + + wrapper.__name__ = func.__name__ + return wrapper + + def compress(x, c_kwargs): if c_kwargs.get('return_bytearray', False): c = bytearray() @@ -73,6 +91,7 @@ def data(request): return request.param +@run_gc_param_data_buffer_size def test_block_decompress_mem_usage(data, buffer_size): kwargs = { 'strategy': "double_buffer", From a11fa74d7fe3e31491d41161ddfc7bb81603b504 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 9 Jun 2020 22:59:00 +0100 Subject: [PATCH 137/280] Fix a flake8 error in lz4/__init__.py --- lz4/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lz4/__init__.py b/lz4/__init__.py index af0fa05b..6f2f2430 100644 --- a/lz4/__init__.py +++ b/lz4/__init__.py @@ -11,10 +11,9 @@ # pass from .version import version as __version__ -VERSION = __version__ - - from ._version import ( # noqa: F401 library_version_number, library_version_string, ) + +VERSION = __version__ From 048390910606d49aef05ee5a412ee0595c6ff6c4 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Tue, 9 Jun 2020 23:27:42 +0100 Subject: [PATCH 138/280] Revert "Remove Py_UNUSED macro from _block.c" This reverts commit 4b26dca06d9d89ff7cac496c85397158d60e2cee. This is needed to support PyPy, which lacks Py_UNUSED. Reported to PyPy project: https://foss.heptapod.net/pypy/pypy/issues/3245 --- lz4/block/_block.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lz4/block/_block.c b/lz4/block/_block.c index a2774e5d..3e904a03 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -42,6 +42,14 @@ #include #include +#ifndef Py_UNUSED /* This is already defined for Python 3.4 onwards */ +#ifdef __GNUC__ +#define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) +#else +#define Py_UNUSED(name) _unused_ ## name +#endif +#endif + #if defined(_WIN32) && defined(_MSC_VER) #if _MSC_VER >= 1600 #include From a7c0b49a7292a5a2250e7b8458ec313fdb6a7a26 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 6 Oct 2020 20:35:07 -0400 Subject: [PATCH 139/280] setup.py: Handle multiple args from pkgconfig The pkgconfig_cflags and pkgconfig_libs functions return the arguments to add to the compiler and linker command line as a space-separated string. Handle this correctly by splitting the string apart on spaces and appending each individual argument to the respective command line. --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index fedf9fdd..e8e473c5 100644 --- a/setup.py +++ b/setup.py @@ -99,9 +99,8 @@ def pkgconfig_installed_check(lib, required_version, default): extra_compile_args = ['/Ot', '/Wall', '/wd4711', '/wd4820'] elif compiler in ('unix', 'mingw32'): if liblz4_found: - extra_link_args.append(pkgconfig_libs('liblz4')) - if pkgconfig_cflags('liblz4'): - extra_compile_args.append(pkgconfig_cflags('liblz4')) + extra_link_args.extend(pkgconfig_libs('liblz4').split()) + extra_compile_args.extend(pkgconfig_cflags('liblz4').split()) else: extra_compile_args = [ '-O3', From df8b31cdfa0784657a1b5b8e44cee3429fb00241 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Sun, 11 Oct 2020 19:05:57 -0400 Subject: [PATCH 140/280] setup.py: Pass Extension build flags with **kwargs This commit doesn't change the behavior at all, but sets things up to make it easier to integrate with pkgconfig.parse() in a future commit. --- setup.py | 43 ++++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index e8e473c5..7742534b 100644 --- a/setup.py +++ b/setup.py @@ -41,8 +41,7 @@ def pkgconfig_installed_check(lib, required_version, default): # against the bundled lz4 files, we'll set the compiler flags to be consistent # with what upstream lz4 recommends. -include_dirs = [] -libraries = [] +extension_kwargs = {} lz4version_sources = [ 'lz4/_version.c' @@ -61,9 +60,9 @@ def pkgconfig_installed_check(lib, required_version, default): ] if liblz4_found is True: - libraries.append('lz4') + extension_kwargs['libraries'] = ['lz4'] else: - include_dirs.append('lz4libs') + extension_kwargs['include_dirs'] = ['lz4libs'] lz4version_sources.extend( [ 'lz4libs/lz4.c', @@ -92,17 +91,19 @@ def pkgconfig_installed_check(lib, required_version, default): compiler = ccompiler.get_default_compiler() -extra_link_args = [] -extra_compile_args = [] - if compiler == 'msvc': - extra_compile_args = ['/Ot', '/Wall', '/wd4711', '/wd4820'] + extension_kwargs['extra_compile_args'] = [ + '/Ot', + '/Wall', + '/wd4711', + '/wd4820', + ] elif compiler in ('unix', 'mingw32'): if liblz4_found: - extra_link_args.extend(pkgconfig_libs('liblz4').split()) - extra_compile_args.extend(pkgconfig_cflags('liblz4').split()) + extension_kwargs['extra_link_args'] = pkgconfig_libs('liblz4').split() + extension_kwargs['extra_compile_args'] = pkgconfig_cflags('liblz4').split() else: - extra_compile_args = [ + extension_kwargs['extra_compile_args'] = [ '-O3', '-Wall', '-Wundef' @@ -113,31 +114,19 @@ def pkgconfig_installed_check(lib, required_version, default): lz4version = Extension('lz4._version', lz4version_sources, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - libraries=libraries, - include_dirs=include_dirs) + **extension_kwargs) lz4block = Extension('lz4.block._block', lz4block_sources, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - libraries=libraries, - include_dirs=include_dirs) + **extension_kwargs) lz4frame = Extension('lz4.frame._frame', lz4frame_sources, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - libraries=libraries, - include_dirs=include_dirs) + **extension_kwargs) lz4stream = Extension('lz4.stream._stream', lz4stream_sources, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - libraries=libraries, - include_dirs=include_dirs) + **extension_kwargs) install_requires = [] From d4ce0defe72c72cb4858346be76fd49c07f5956c Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Sun, 11 Oct 2020 19:28:41 -0400 Subject: [PATCH 141/280] setup.py: Use pkgconfig.parse() Rather than using pkgconfig.libs() and pkgconfig.cflags() directly, use pkgconfig.parse(), which returns a dictionary suitable for passing as keyword arguments to the Extension constructor. --- setup.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 7742534b..0e1f8df5 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,7 @@ try: from pkgconfig import installed as pkgconfig_installed - from pkgconfig import cflags as pkgconfig_cflags - from pkgconfig import libs as pkgconfig_libs + from pkgconfig import parse as pkgconfig_parse except ImportError: # pkgconfig is not installed. It will be installed by setup_requires. pass @@ -100,8 +99,7 @@ def pkgconfig_installed_check(lib, required_version, default): ] elif compiler in ('unix', 'mingw32'): if liblz4_found: - extension_kwargs['extra_link_args'] = pkgconfig_libs('liblz4').split() - extension_kwargs['extra_compile_args'] = pkgconfig_cflags('liblz4').split() + extension_kwargs = pkgconfig_parse('liblz4') else: extension_kwargs['extra_compile_args'] = [ '-O3', From ac0703ff20c3cc7e3d5dc22782ea1fbd13838113 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 6 Oct 2020 20:26:59 -0400 Subject: [PATCH 142/280] Use CHAR_BIT rather than __CHAR_BIT__ This macro is required by C99 onward, but if it is missing (as it is for MSVC 2008 and earlier), default to 8. --- lz4/stream/_stream.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 05d7db1e..65ca3d68 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -44,10 +44,8 @@ #include #include -#if defined(_WIN32) && defined(_MSC_VER) -#if _MSC_VER >= 1600 -#include -#else /* _MSC_VER >= 1600 */ +#if defined(_WIN32) && defined(_MSC_VER) && _MSC_VER < 1600 +/* MSVC 2008 and earlier lacks stdint.h */ typedef signed __int8 int8_t; typedef signed __int16 int16_t; typedef signed __int32 int32_t; @@ -69,12 +67,14 @@ typedef unsigned __int64 uint64_t; #if !defined(INT32_MAX) #define INT32_MAX 0x7fffffff #endif -#endif /* _MSC_VER >= 1600 */ +#if !defined(CHAR_BIT) +#define CHAR_BIT 8 +#endif -#if !defined(__CHAR_BIT__) -#define __CHAR_BIT__ 8 -#endif /* __CHAR_BIT__ */ -#endif /* _WIN32 && _MSC_VER */ +#else +/* Not MSVC, or MSVC 2010 or higher */ +#include +#endif /* _WIN32 && _MSC_VER && _MSC_VER < 1600 */ #define LZ4_VERSION_NUMBER_1_9_0 10900 @@ -107,7 +107,7 @@ static PyObject * LZ4StreamError; #define DOUBLE_BUFFER_INDEX_MIN (0) #define DOUBLE_BUFFER_INDEX_INVALID (-1) -#define _GET_MAX_UINT(byte_depth, type) (type)( ( 1ULL << ( __CHAR_BIT__ * (byte_depth) ) ) - 1 ) +#define _GET_MAX_UINT(byte_depth, type) (type)( ( 1ULL << ( CHAR_BIT * (byte_depth) ) ) - 1 ) #define _GET_MAX_UINT32(byte_depth) _GET_MAX_UINT((byte_depth), uint32_t) typedef struct { From 462c1038223597ac7c2a56218fccf86d54602aff Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 17 Dec 2020 21:30:29 +0000 Subject: [PATCH 143/280] Remove unnecessary memory clears during stream buffer allocation and release --- lz4/stream/_stream.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 65ca3d68..522fdeda 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -524,9 +524,6 @@ double_buffer_release_resources (stream_context_t * context) if (context->strategy.data.double_buffer.buf != NULL) { - memset (context->strategy.data.double_buffer.buf, - 0x0, - context->strategy.data.double_buffer.page_size * DOUBLE_BUFFER_PAGE_COUNT); PyMem_Free (context->strategy.data.double_buffer.buf); } context->strategy.data.double_buffer.buf = NULL; @@ -551,10 +548,6 @@ double_buffer_reserve_resources (stream_context_t * context, unsigned int buffer goto exit_now; } - memset (context->strategy.data.double_buffer.buf, - 0x0, - context->strategy.data.double_buffer.page_size * DOUBLE_BUFFER_PAGE_COUNT); - for (i = DOUBLE_BUFFER_INDEX_MIN; i < (DOUBLE_BUFFER_INDEX_MIN + DOUBLE_BUFFER_PAGE_COUNT); ++i) @@ -738,14 +731,12 @@ destroy_context (stream_context_t * context) /* Release output buffer */ if (context->output.buf != NULL) { - memset (context->output.buf, 0x0, context->output.len); PyMem_Free (context->output.buf); } context->output.buf = NULL; context->output.len = 0; /* Release python memory */ - memset (context, 0x0, sizeof (stream_context_t)); PyMem_Free (context); } From 420165316f937d76da20492a07150ed8747da01b Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 17 Dec 2020 22:40:55 +0000 Subject: [PATCH 144/280] Avoid quadratic cost when concatenating chunks --- tests/stream/test_stream_3.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index 352efbdc..a10885ec 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -29,31 +29,31 @@ def wrapper(data, buffer_size, *args, **kwargs): def compress(x, c_kwargs): - if c_kwargs.get('return_bytearray', False): - c = bytearray() - else: - c = bytes() + c = [] with lz4.stream.LZ4StreamCompressor(**c_kwargs) as proc: for start in range(0, len(x), c_kwargs['buffer_size']): chunk = x[start:start + c_kwargs['buffer_size']] block = proc.compress(chunk) - c += block - return c + c.append(block) + if c_kwargs.get('return_bytearray', False): + return bytearray().join(c) + else: + return bytes().join(c) def decompress(x, d_kwargs): - if d_kwargs.get('return_bytearray', False): - d = bytearray() - else: - d = bytes() + d = [] with lz4.stream.LZ4StreamDecompressor(**d_kwargs) as proc: start = 0 while start < len(x): block = proc.get_block(x[start:]) chunk = proc.decompress(block) - d += chunk + d.append(chunk) start += d_kwargs['store_comp_size'] + len(block) - return d + if d_kwargs.get('return_bytearray', False): + return bytearray().join(d) + else: + return bytes().join(d) test_buffer_size = sorted( From f10213eaf47a52925f440da1ada8f3c111eadffb Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 17 Dec 2020 22:55:49 +0000 Subject: [PATCH 145/280] Remove calls to garbage collector during stream tests --- tests/stream/test_stream_0.py | 20 ------------ tests/stream/test_stream_1.py | 60 ----------------------------------- tests/stream/test_stream_2.py | 21 ------------ tests/stream/test_stream_3.py | 19 ----------- tests/stream/test_stream_4.py | 21 ------------ 5 files changed, 141 deletions(-) diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index c7d34bcc..fd2354ef 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -1,29 +1,10 @@ import lz4.stream import sys import pytest -import gc -import os if sys.version_info <= (3, 2): import struct -def run_gc(func): - if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - else: - def wrapper(*args, **kwargs): - gc.collect() - try: - result = func(*args, **kwargs) - finally: - gc.collect() - return result - - wrapper.__name__ = func.__name__ - return wrapper - - def get_stored_size(buff, block_length_size): if sys.version_info > (2, 7): if isinstance(buff, memoryview): @@ -45,7 +26,6 @@ def get_stored_size(buff, block_length_size): return struct.unpack('<' + fmt[block_length_size], b[:block_length_size])[0] -@run_gc def roundtrip(x, c_kwargs, d_kwargs, dictionary): if dictionary: if isinstance(dictionary, tuple): diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py index 50c558b9..a0f24e78 100644 --- a/tests/stream/test_stream_1.py +++ b/tests/stream/test_stream_1.py @@ -3,7 +3,6 @@ import sys import os import psutil -import gc if sys.version_info < (3, ): @@ -44,40 +43,6 @@ def int_from_bytes(bytes, byteorder='little', signed=False): # fragile. -def run_gc(func): - if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - else: - def wrapper(*args, **kwargs): - gc.collect() - try: - result = func(*args, **kwargs) - finally: - gc.collect() - return result - - wrapper.__name__ = func.__name__ - return wrapper - - -def run_gc_param_store_comp_size(func): - if os.environ.get('TRAVIS') is not None: - def wrapper(store_comp_size, *args, **kwargs): - return func(store_comp_size, *args, **kwargs) - else: - def wrapper(store_comp_size, *args, **kwargs): - gc.collect() - try: - result = func(store_comp_size, *args, **kwargs) - finally: - gc.collect() - return result - - wrapper.__name__ = func.__name__ - return wrapper - - def compress(x, c_kwargs, return_block_offset=False, check_block_type=False): o = [0, ] if c_kwargs.get('return_bytearray', False): @@ -116,7 +81,6 @@ def decompress(x, d_kwargs, check_chunk_type=False): return d -@run_gc def test_invalid_config_c_1(): c_kwargs = {} c_kwargs['strategy'] = "ring_buffer" @@ -126,7 +90,6 @@ def test_invalid_config_c_1(): lz4.stream.LZ4StreamCompressor(**c_kwargs) -@run_gc def test_invalid_config_d_1(): d_kwargs = {} d_kwargs['strategy'] = "ring_buffer" @@ -136,7 +99,6 @@ def test_invalid_config_d_1(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) -@run_gc def test_invalid_config_c_2(): c_kwargs = {} c_kwargs['strategy'] = "foo" @@ -146,7 +108,6 @@ def test_invalid_config_c_2(): lz4.stream.LZ4StreamCompressor(**c_kwargs) -@run_gc def test_invalid_config_d_2(): d_kwargs = {} d_kwargs['strategy'] = "foo" @@ -156,7 +117,6 @@ def test_invalid_config_d_2(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) -@run_gc_param_store_comp_size def test_invalid_config_c_3(store_comp_size): c_kwargs = {} c_kwargs['strategy'] = "double_buffer" @@ -167,7 +127,6 @@ def test_invalid_config_c_3(store_comp_size): lz4.stream.LZ4StreamCompressor(**c_kwargs) -@run_gc_param_store_comp_size def test_invalid_config_d_3(store_comp_size): d_kwargs = {} d_kwargs['strategy'] = "double_buffer" @@ -178,7 +137,6 @@ def test_invalid_config_d_3(store_comp_size): lz4.stream.LZ4StreamDecompressor(**d_kwargs) -@run_gc_param_store_comp_size def test_invalid_config_c_4(store_comp_size): c_kwargs = {} c_kwargs['strategy'] = "double_buffer" @@ -204,7 +162,6 @@ def test_invalid_config_c_4(store_comp_size): lz4.stream.LZ4StreamCompressor(**c_kwargs) -@run_gc_param_store_comp_size def test_invalid_config_d_4(store_comp_size): d_kwargs = {} d_kwargs['strategy'] = "double_buffer" @@ -233,7 +190,6 @@ def test_invalid_config_d_4(store_comp_size): lz4.stream.LZ4StreamDecompressor(**d_kwargs) -@run_gc def test_invalid_config_c_5(): c_kwargs = {} c_kwargs['strategy'] = "double_buffer" @@ -265,7 +221,6 @@ def test_invalid_config_c_5(): lz4.stream.LZ4StreamCompressor(**c_kwargs) -@run_gc def test_invalid_config_d_5(): d_kwargs = {} d_kwargs['strategy'] = "double_buffer" @@ -322,7 +277,6 @@ def test_invalid_config_d_5(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) -@run_gc def test_decompress_corrupted_input_1(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -338,7 +292,6 @@ def test_decompress_corrupted_input_1(): decompress(data[4:], d_kwargs) -@run_gc def test_decompress_corrupted_input_2(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -363,7 +316,6 @@ def test_decompress_corrupted_input_2(): decompress(data, d_kwargs) -@run_gc def test_decompress_corrupted_input_3(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -393,7 +345,6 @@ def test_decompress_corrupted_input_3(): decompress(data, d_kwargs) -@run_gc def test_decompress_corrupted_input_4(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -425,7 +376,6 @@ def test_decompress_corrupted_input_4(): decompress(data, d_kwargs) -@run_gc def test_decompress_truncated(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -458,7 +408,6 @@ def test_decompress_truncated(): # we will keep them for now -@run_gc def test_decompress_with_trailer(): c_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -485,7 +434,6 @@ def test_decompress_with_trailer(): decompress(comp + b'\x00' * n, d_kwargs) -@run_gc def test_unicode(): if sys.version_info < (3,): return # skip @@ -505,7 +453,6 @@ def test_unicode(): # for now -@run_gc def test_return_bytearray(): if sys.version_info < (3,): return # skip @@ -527,7 +474,6 @@ def test_return_bytearray(): assert bytes(b) == data -@run_gc def test_memoryview(): if sys.version_info < (2, 7): return # skip @@ -543,7 +489,6 @@ def test_memoryview(): assert decompress(memoryview(compressed), d_kwargs) == data -@run_gc def test_with_dict_none(): kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -586,7 +531,6 @@ def test_with_dict_none(): assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data -@run_gc def test_with_dict(): kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -625,7 +569,6 @@ def test_with_dict(): assert decompress(compress(input_data, c_kwargs), d_kwargs) == input_data -@run_gc def test_known_decompress_1(): d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -640,7 +583,6 @@ def test_known_decompress_1(): assert decompress(input, d_kwargs) == output -@run_gc def test_known_decompress_2(): d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -649,7 +591,6 @@ def test_known_decompress_2(): assert decompress(input, d_kwargs) == output -@run_gc def test_known_decompress_3(): d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} @@ -659,7 +600,6 @@ def test_known_decompress_3(): assert decompress(input, d_kwargs) == output -@run_gc def test_known_decompress_4(): d_kwargs = {'strategy': "double_buffer", 'buffer_size': 128, 'store_comp_size': 4} diff --git a/tests/stream/test_stream_2.py b/tests/stream/test_stream_2.py index de0693a4..5578f832 100644 --- a/tests/stream/test_stream_2.py +++ b/tests/stream/test_stream_2.py @@ -3,24 +3,6 @@ import lz4.stream import psutil import os -import gc - - -def run_gc(func): - if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - else: - def wrapper(*args, **kwargs): - gc.collect() - try: - result = func(*args, **kwargs) - finally: - gc.collect() - return result - - wrapper.__name__ = func.__name__ - return wrapper # This test requires allocating a big lump of memory. In order to @@ -60,7 +42,6 @@ def wrapper(*args, **kwargs): psutil.virtual_memory().available < _4GB or huge is None, reason='Insufficient system memory for this test' ) -@run_gc def test_huge_1(): data = b'' kwargs = { @@ -100,7 +81,6 @@ def test_huge_1(): psutil.virtual_memory().available < _4GB or huge is None, reason='Insufficient system memory for this test' ) -@run_gc def test_huge_2(): data = huge kwargs = { @@ -141,7 +121,6 @@ def test_huge_2(): psutil.virtual_memory().available < _4GB or huge is None, reason='Insufficient system memory for this test' ) -@run_gc def test_huge_3(): data = huge kwargs = { diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index a10885ec..f535aae3 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -3,7 +3,6 @@ import sys import os import psutil -import gc _1KB = 1024 @@ -11,23 +10,6 @@ _1GB = _1MB * 1024 -def run_gc_param_data_buffer_size(func): - if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: - def wrapper(data, buffer_size, *args, **kwargs): - return func(data, buffer_size, *args, **kwargs) - else: - def wrapper(data, buffer_size, *args, **kwargs): - gc.collect() - try: - result = func(data, buffer_size, *args, **kwargs) - finally: - gc.collect() - return result - - wrapper.__name__ = func.__name__ - return wrapper - - def compress(x, c_kwargs): c = [] with lz4.stream.LZ4StreamCompressor(**c_kwargs) as proc: @@ -91,7 +73,6 @@ def data(request): return request.param -@run_gc_param_data_buffer_size def test_block_decompress_mem_usage(data, buffer_size): kwargs = { 'strategy': "double_buffer", diff --git a/tests/stream/test_stream_4.py b/tests/stream/test_stream_4.py index 5128350d..3d139a02 100644 --- a/tests/stream/test_stream_4.py +++ b/tests/stream/test_stream_4.py @@ -1,25 +1,6 @@ import lz4.stream import pytest import sys -import os -import gc - - -def run_gc(func): - if os.environ.get('TRAVIS') is not None or os.environ.get('APPVEYOR') is not None: - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - else: - def wrapper(*args, **kwargs): - gc.collect() - try: - result = func(*args, **kwargs) - finally: - gc.collect() - return result - - wrapper.__name__ = func.__name__ - return wrapper if sys.version_info < (3, ): @@ -47,7 +28,6 @@ def int_from_bytes(bytes, byteorder='little', signed=False): # Out-of-band block size record tests -@run_gc def test_round_trip(): data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 kwargs = {'strategy': "double_buffer", 'buffer_size': 256, 'store_comp_size': 4} @@ -127,7 +107,6 @@ def test_round_trip(): assert (data == oob_dstream), "Decompressed streams mismatch" -@run_gc def test_invalid_usage(): data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123" * 24 kwargs = {'strategy': "double_buffer", 'buffer_size': 256, 'store_comp_size': 0} From 508709148fd194c6b3835d3caf94b258d67e8d30 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 17 Dec 2020 23:17:53 +0000 Subject: [PATCH 146/280] Remove unnecessary resource checks --- tests/stream/test_stream_1.py | 53 ----------------------------------- tests/stream/test_stream_3.py | 14 --------- 2 files changed, 67 deletions(-) diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py index a0f24e78..6b49267e 100644 --- a/tests/stream/test_stream_1.py +++ b/tests/stream/test_stream_1.py @@ -2,7 +2,6 @@ import pytest import sys import os -import psutil if sys.version_info < (3, ): @@ -35,13 +34,6 @@ def int_from_bytes(bytes, byteorder='little', signed=False): # https://bugs.python.org/issue21074 _4GB = 0x100000000 # 4GB -# This test will be killed on Travis due to the 3GB memory limit -# there. Unfortunately psutil reports the host memory, not the memory -# available to the container, and so can't be used to detect available -# memory, so instead, as an ugly hack for detecting we're on Travis we -# check for the TRAVIS environment variable being set. This is quite -# fragile. - def compress(x, c_kwargs, return_block_offset=False, check_block_type=False): o = [0, ] @@ -170,18 +162,9 @@ def test_invalid_config_d_4(store_comp_size): if store_comp_size['store_comp_size'] >= 4: - if os.environ.get('TRAVIS') is not None: - pytest.skip('Skipping test on Travis due to insufficient memory') - if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().available < 4 * d_kwargs['buffer_size']: - # The internal LZ4 context will request at least 3 times buffer_size - # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size - # for the output buffer), so round up to 4 buffer_size. - pytest.skip('Insufficient system memory for this test') - # Make sure the page size is larger than what the input bound will be, # but still fit in 4 bytes d_kwargs['buffer_size'] -= 1 @@ -195,18 +178,9 @@ def test_invalid_config_c_5(): c_kwargs['strategy'] = "double_buffer" c_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE - if os.environ.get('TRAVIS') is not None: - pytest.skip('Skipping test on Travis due to insufficient memory') - if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().available < 3 * c_kwargs['buffer_size']: - # The internal LZ4 context will request at least 3 times buffer_size - # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size - # for the output buffer) - pytest.skip('Insufficient system memory for this test') - # No failure expected lz4.stream.LZ4StreamCompressor(**c_kwargs) @@ -228,52 +202,25 @@ def test_invalid_config_d_5(): # No failure expected during instanciation/initialization d_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE - if os.environ.get('TRAVIS') is not None: - pytest.skip('Skipping test on Travis due to insufficient memory') - if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().available < 3 * d_kwargs['buffer_size']: - # The internal LZ4 context will request at least 3 times buffer_size - # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size - # for the output buffer) - pytest.skip('Insufficient system memory for this test') - lz4.stream.LZ4StreamDecompressor(**d_kwargs) # No failure expected during instanciation/initialization d_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE + 1 - if os.environ.get('TRAVIS') is not None: - pytest.skip('Skipping test on Travis due to insufficient memory') - if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().available < 3 * d_kwargs['buffer_size']: - # The internal LZ4 context will request at least 3 times buffer_size - # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size - # for the output buffer) - pytest.skip('Insufficient system memory for this test') - lz4.stream.LZ4StreamDecompressor(**d_kwargs) # No failure expected during instanciation/initialization d_kwargs['buffer_size'] = _4GB - 1 # 4GB - 1 (to fit in 4 bytes) - if os.environ.get('TRAVIS') is not None: - pytest.skip('Skipping test on Travis due to insufficient memory') - if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().available < 4 * d_kwargs['buffer_size']: - # The internal LZ4 context will request at least 3 times buffer_size - # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size - # for the output buffer), so round up to 4 buffer_size. - pytest.skip('Insufficient system memory for this test') - lz4.stream.LZ4StreamDecompressor(**d_kwargs) diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index f535aae3..2b52d6b5 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -1,8 +1,6 @@ import lz4.stream import pytest import sys -import os -import psutil _1KB = 1024 @@ -80,21 +78,9 @@ def test_block_decompress_mem_usage(data, buffer_size): 'store_comp_size': 4, } - if os.environ.get('TRAVIS') is not None: - pytest.skip('Skipping test on Travis due to insufficient memory') - - if os.environ.get('APPVEYOR') is not None: - pytest.skip('Skipping test on AppVeyor due to insufficient resources') - if sys.maxsize < 0xffffffff: pytest.skip('Py_ssize_t too small for this test') - if psutil.virtual_memory().available < 3 * kwargs['buffer_size']: - # The internal LZ4 context will request at least 3 times buffer_size - # as memory (2 buffer_size for the double-buffer, and 1.x buffer_size - # for the output buffer) - pytest.skip('Insufficient system memory for this test') - tracemalloc = pytest.importorskip('tracemalloc') # Trace memory usage on compression From aedfd1666411ed5baf9dd0b21dc3616e92dc7685 Mon Sep 17 00:00:00 2001 From: ossdev07 Date: Fri, 3 Jan 2020 09:11:51 +0000 Subject: [PATCH 147/280] Added arm64 jobs in Travis-CI Signed-off-by: ossdev07 --- .travis.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.travis.yml b/.travis.yml index b4705634..2c06ddfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,28 @@ matrix: - docker env: - CIBW_BUILD=cp37-* + - name: linux-py37 + arch: arm64 + language: python + python: 3.7 + dist: xenial + sudo: required + services: + - docker + env: + - PIP_NO_CACHE_DIR=off + - CIBW_BUILD=cp37-manylinux_aarch64 + install: + - python -m pip install -U pip + - python -m pip install cibuildwheel==1.7.4 + - python -m pip install codecov tox twine + script: + - tox tests + - codecov + - git config --global user.email "junk@junk.com" + - git config --global user.name "junk" + - git stash --all # Restore fresh checkout + - cibuildwheel --output-dir dist - name: linux-py38 language: python python: 3.8 @@ -34,6 +56,28 @@ matrix: - docker env: - CIBW_BUILD=cp38-* + - name: linux-py38 + arch: arm64 + language: python + python: 3.8 + dist: xenial + sudo: required + services: + - docker + env: + - PIP_NO_CACHE_DIR=off + - CIBW_BUILD=cp38-manylinux_aarch64 + install: + - python -m pip install -U pip + - python -m pip install cibuildwheel==1.7.4 + - python -m pip install codecov tox twine + script: + - tox tests + - codecov + - git config --global user.email "junk@junk.com" + - git config --global user.name "junk" + - git stash --all # Restore fresh checkout + - cibuildwheel --output-dir dist - name: osx-py35 os: osx language: generic @@ -89,6 +133,8 @@ install: script: - tox tests - codecov + - git config --global user.email "junk@junk.com" + - git config --global user.name "junk" - git stash --all # Restore fresh checkout - cibuildwheel --output-dir dist deploy: From 3d971ed28fa2cd119807b3570c59aa36be22fecd Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 16 Jan 2021 13:41:22 +0000 Subject: [PATCH 148/280] Update bundled lz4 libs to 1.9.3 release --- lz4libs/lz4.c | 557 ++++++++++++++++++++++++++------------------- lz4libs/lz4.h | 150 ++++++------ lz4libs/lz4frame.c | 147 +++++++----- lz4libs/lz4frame.h | 36 +-- lz4libs/lz4hc.c | 307 +++++++++++++++---------- lz4libs/lz4hc.h | 55 ++--- 6 files changed, 729 insertions(+), 523 deletions(-) diff --git a/lz4libs/lz4.c b/lz4libs/lz4.c index 9808d70a..9f5e9bfa 100644 --- a/lz4libs/lz4.c +++ b/lz4libs/lz4.c @@ -45,10 +45,16 @@ #endif /* - * ACCELERATION_DEFAULT : + * LZ4_ACCELERATION_DEFAULT : * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 */ -#define ACCELERATION_DEFAULT 1 +#define LZ4_ACCELERATION_DEFAULT 1 +/* + * LZ4_ACCELERATION_MAX : + * Any "acceleration" value higher than this threshold + * get treated as LZ4_ACCELERATION_MAX instead (fix #876) + */ +#define LZ4_ACCELERATION_MAX 65537 /*-************************************ @@ -82,6 +88,7 @@ * Define this parameter if your target system or compiler does not support hardware bit count */ #if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ +# undef LZ4_FORCE_SW_BITCOUNT /* avoid double def */ # define LZ4_FORCE_SW_BITCOUNT #endif @@ -114,10 +121,9 @@ /*-************************************ * Compiler Options **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ +# include /* only present in VS2005+ */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif /* _MSC_VER */ #ifndef LZ4_FORCE_INLINE @@ -136,7 +142,7 @@ # endif /* _MSC_VER */ #endif /* LZ4_FORCE_INLINE */ -/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE +/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, * together with a simple 8-byte copy loop as a fall-back path. * However, this optimization hurts the decompression speed by >30%, @@ -151,11 +157,11 @@ * of LZ4_wildCopy8 does not affect the compression speed. */ #if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) -# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) -# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE +# define LZ4_FORCE_O2 __attribute__((optimize("O2"))) +# undef LZ4_FORCE_INLINE +# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline)) #else -# define LZ4_FORCE_O2_GCC_PPC64LE -# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static +# define LZ4_FORCE_O2 #endif #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) @@ -171,14 +177,33 @@ #define unlikely(expr) expect((expr) != 0, 0) #endif +/* Should the alignment test prove unreliable, for some reason, + * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ +#ifndef LZ4_ALIGN_TEST /* can be externally provided */ +# define LZ4_ALIGN_TEST 1 +#endif + /*-************************************ * Memory routines **************************************/ -#include /* malloc, calloc, free */ -#define ALLOC(s) malloc(s) -#define ALLOC_AND_ZERO(s) calloc(1,s) -#define FREEMEM(p) free(p) +#ifdef LZ4_USER_MEMORY_FUNCTIONS +/* memory management functions can be customized by user project. + * Below functions must exist somewhere in the Project + * and be available at link time */ +void* LZ4_malloc(size_t s); +void* LZ4_calloc(size_t n, size_t s); +void LZ4_free(void* p); +# define ALLOC(s) LZ4_malloc(s) +# define ALLOC_AND_ZERO(s) LZ4_calloc(1,s) +# define FREEMEM(p) LZ4_free(p) +#else +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,s) +# define FREEMEM(p) free(p) +#endif + #include /* memset, memcpy */ #define MEM_INIT(p,v,s) memset((p),(v),(s)) @@ -225,21 +250,27 @@ static const int LZ4_minLength = (MFLIMIT+1); #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) # include -static int g_debuglog_enable = 1; -# define DEBUGLOG(l, ...) { \ - if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ - } } + static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } #else -# define DEBUGLOG(l, ...) {} /* disabled */ +# define DEBUGLOG(l, ...) {} /* disabled */ #endif +static int LZ4_isAligned(const void* ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment -1)) == 0; +} + /*-************************************ * Types **************************************/ +#include #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # include typedef uint8_t BYTE; @@ -249,6 +280,9 @@ static int g_debuglog_enable = 1; typedef uint64_t U64; typedef uintptr_t uptrval; #else +# if UINT_MAX != 4294967295UL +# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4" +# endif typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; @@ -273,6 +307,21 @@ typedef enum { /*-************************************ * Reading and writing into memory **************************************/ + +/** + * LZ4 relies on memcpy with a constant size being inlined. In freestanding + * environments, the compiler can't assume the implementation of memcpy() is + * standard compliant, so it can't apply its specialized memcpy() inlining + * logic. When possible, use __builtin_memcpy() to tell the compiler to analyze + * memcpy() as if it were standard compliant, so it can inline it in freestanding + * environments. This is needed when decompressing the Linux Kernel, for example. + */ +#if defined(__GNUC__) && (__GNUC__ >= 4) +#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +#else +#define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +#endif + static unsigned LZ4_isLittleEndian(void) { const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ @@ -307,27 +356,27 @@ static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = val static U16 LZ4_read16(const void* memPtr) { - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; + U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } static U32 LZ4_read32(const void* memPtr) { - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; + U32 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } static reg_t LZ4_read_ARCH(const void* memPtr) { - reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; + reg_t val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } static void LZ4_write16(void* memPtr, U16 value) { - memcpy(memPtr, &value, sizeof(value)); + LZ4_memcpy(memPtr, &value, sizeof(value)); } static void LZ4_write32(void* memPtr, U32 value) { - memcpy(memPtr, &value, sizeof(value)); + LZ4_memcpy(memPtr, &value, sizeof(value)); } #endif /* LZ4_FORCE_MEMORY_ACCESS */ @@ -355,14 +404,14 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -LZ4_FORCE_O2_INLINE_GCC_PPC64LE +LZ4_FORCE_INLINE void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; BYTE* const e = (BYTE*)dstEnd; - do { memcpy(d,s,8); d+=8; s+=8; } while (d= 16. */ -LZ4_FORCE_O2_INLINE_GCC_PPC64LE void +LZ4_FORCE_INLINE void LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; BYTE* const e = (BYTE*)dstEnd; - do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH * - there is at least 8 bytes available to write after dstEnd */ -LZ4_FORCE_O2_INLINE_GCC_PPC64LE void +LZ4_FORCE_INLINE void LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) { BYTE v[8]; assert(dstEnd >= dstPtr + MINMATCH); - LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */ switch(offset) { case 1: - memset(v, *srcPtr, 8); + MEM_INIT(v, *srcPtr, 8); break; case 2: - memcpy(v, srcPtr, 2); - memcpy(&v[2], srcPtr, 2); - memcpy(&v[4], &v[0], 4); + LZ4_memcpy(v, srcPtr, 2); + LZ4_memcpy(&v[2], srcPtr, 2); + LZ4_memcpy(&v[4], v, 4); break; case 4: - memcpy(v, srcPtr, 4); - memcpy(&v[4], srcPtr, 4); + LZ4_memcpy(v, srcPtr, 4); + LZ4_memcpy(&v[4], srcPtr, 4); break; default: LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); return; } - memcpy(dstPtr, v, 8); + LZ4_memcpy(dstPtr, v, 8); dstPtr += 8; while (dstPtr < dstEnd) { - memcpy(dstPtr, v, 8); + LZ4_memcpy(dstPtr, v, 8); dstPtr += 8; } } @@ -462,75 +512,92 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si **************************************/ static unsigned LZ4_NbCommonBytes (reg_t val) { + assert(val != 0); if (LZ4_isLittleEndian()) { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + if (sizeof(val) == 8) { +# if defined(_MSC_VER) && (_MSC_VER >= 1800) && defined(_M_AMD64) && !defined(LZ4_FORCE_SW_BITCOUNT) + /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ + return (unsigned)_tzcnt_u64(val) >> 3; +# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + _BitScanForward64(&r, (U64)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) return (unsigned)__builtin_ctzll((U64)val) >> 3; # else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, - 0, 3, 1, 3, 1, 4, 2, 7, - 0, 2, 3, 6, 1, 5, 3, 5, - 1, 3, 4, 4, 2, 5, 6, 7, - 7, 0, 1, 2, 3, 3, 4, 6, - 2, 6, 5, 5, 3, 4, 5, 6, - 7, 1, 2, 4, 6, 4, 4, 5, - 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; + const U64 m = 0x0101010101010101ULL; + val ^= val - 1; + return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56); # endif } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + _BitScanForward(&r, (U32)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) return (unsigned)__builtin_ctz((U32)val) >> 3; # else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, - 3, 2, 2, 1, 3, 2, 0, 1, - 3, 3, 1, 2, 2, 2, 2, 0, - 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; # endif } } else /* Big Endian CPU */ { - if (sizeof(val)==8) { /* 64-bits */ -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + if (sizeof(val)==8) { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) return (unsigned)__builtin_clzll((U64)val) >> 3; # else +#if 1 + /* this method is probably faster, + * but adds a 128 bytes lookup table */ + static const unsigned char ctz7_tab[128] = { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + }; + U64 const mask = 0x0101010101010101ULL; + U64 const t = (((val >> 8) - mask) | val) & mask; + return ctz7_tab[(t * 0x0080402010080402ULL) >> 57]; +#else + /* this method doesn't consume memory space like the previous one, + * but it contains several branches, + * that may end up slowing execution */ static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. - Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. - Note that this code path is never triggered in 32-bits mode. */ + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ unsigned r; if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } r += (!val); return r; +#endif # endif } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) return (unsigned)__builtin_clz((U32)val) >> 3; # else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; # endif } } } + #define STEPSIZE sizeof(reg_t) LZ4_FORCE_INLINE unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) @@ -605,7 +672,7 @@ typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } +int LZ4_sizeofState(void) { return LZ4_STREAMSIZE; } /*-************************************ @@ -628,7 +695,7 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, /*-****************************** * Compression functions ********************************/ -static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); @@ -636,7 +703,7 @@ static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; if (LZ4_isLittleEndian()) { @@ -654,7 +721,7 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab return LZ4_hash4(LZ4_read32(p), tableType); } -static void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) { switch (tableType) { @@ -666,7 +733,7 @@ static void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) } } -static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) { switch (tableType) { @@ -678,7 +745,7 @@ static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t cons } } -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, +LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { @@ -703,7 +770,7 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_ * Assumption 1 : only valid if tableType == byU32 or byU16. * Assumption 2 : h is presumed valid (within limits of hash table) */ -static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) { LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); if (tableType == byU32) { @@ -739,22 +806,13 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(LZ4_stream_t_internal* const cctx, const int inputSize, const tableType_t tableType) { - /* If compression failed during the previous step, then the context - * is marked as dirty, therefore, it has to be fully reset. - */ - if (cctx->dirty) { - DEBUGLOG(5, "LZ4_prepareTable: Full reset for %p", cctx); - MEM_INIT(cctx, 0, sizeof(LZ4_stream_t_internal)); - return; - } - /* If the table hasn't been used, it's guaranteed to be zeroed out, and is * therefore safe to use no matter what mode we're in. Otherwise, we figure * out if it's safe to leave as is or whether it needs to be reset. */ - if (cctx->tableType != clearedTable) { + if ((tableType_t)cctx->tableType != clearedTable) { assert(inputSize >= 0); - if (cctx->tableType != tableType + if ((tableType_t)cctx->tableType != tableType || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) || ((tableType == byU32) && cctx->currentOffset > 1 GB) || tableType == byPtr @@ -763,7 +821,7 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx, DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); cctx->currentOffset = 0; - cctx->tableType = clearedTable; + cctx->tableType = (U32)clearedTable; } else { DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); } @@ -785,8 +843,12 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx, } /** LZ4_compress_generic() : - inlined, to ensure branches are decided at compilation time */ -LZ4_FORCE_INLINE int LZ4_compress_generic( + * inlined, to ensure branches are decided at compilation time. + * Presumed already validated at this stage: + * - source != NULL + * - inputSize > 0 + */ +LZ4_FORCE_INLINE int LZ4_compress_generic_validated( LZ4_stream_t_internal* const cctx, const char* const source, char* const dest, @@ -815,7 +877,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ - const BYTE* const dictEnd = dictionary + dictSize; + const BYTE* const dictEnd = dictionary ? dictionary + dictSize : dictionary; const BYTE* anchor = (const BYTE*) source; const BYTE* const iend = ip + inputSize; const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; @@ -823,7 +885,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* the dictCtx currentOffset is indexed on the start of the dictionary, * while a dictionary in the current context precedes the currentOffset */ - const BYTE* dictBase = (dictDirective == usingDictCtx) ? + const BYTE* dictBase = !dictionary ? NULL : (dictDirective == usingDictCtx) ? dictionary + dictSize - dictCtx->currentOffset : dictionary + dictSize - startIndex; @@ -833,11 +895,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( U32 offset = 0; U32 forwardH; - DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); + DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); + assert(ip != NULL); /* If init conditions are not met, we don't have to mark stream * as having dirty context, since no action was taken yet */ if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */ if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ assert(acceleration >= 1); @@ -854,7 +916,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( cctx->dictSize += (U32)inputSize; } cctx->currentOffset += (U32)inputSize; - cctx->tableType = (U16)tableType; + cctx->tableType = (U32)tableType; if (inputSize= op); - lastRun = (size_t)(olimit-op) - 1; - lastRun -= (lastRun+240)/255; + lastRun = (size_t)(olimit-op) - 1/*token*/; + lastRun -= (lastRun + 256 - RUN_MASK) / 256; /*additional length tokens*/ } else { assert(outputDirective == limitedOutput); return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ } } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; *op++ = RUN_MASK << ML_BITS; @@ -1162,7 +1225,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( } else { *op++ = (BYTE)(lastRun< 0); + DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, result); return result; } +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time; + * takes care of src == (NULL, 0) + * and forward the rest to LZ4_compress_generic_validated */ +LZ4_FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const src, + char* const dst, + const int srcSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int dstCapacity, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", + srcSize, dstCapacity); + + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + if (outputDirective != notLimited && dstCapacity <= 0) return 0; /* no output, can't write anything */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; + if (outputDirective == fillOutput) { + assert (inputConsumed != NULL); + *inputConsumed = 0; + } + return 1; + } + assert(src != NULL); + + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, + tableType, dictDirective, dictIssue, acceleration); +} + int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; assert(ctx != NULL); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) { return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); @@ -1211,7 +1316,8 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; if (dstCapacity >= LZ4_compressBound(srcSize)) { if (srcSize < LZ4_64Klimit) { @@ -1270,22 +1376,6 @@ int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputS } -/* hidden debug function */ -/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) -{ - LZ4_stream_t ctx; - LZ4_initStream(&ctx, sizeof(ctx)); - - if (srcSize < LZ4_64Klimit) { - return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, byU16, noDict, noDictIssue, acceleration); - } else { - tableType_t const addrMode = (sizeof(void*) > 4) ? byU32 : byPtr; - return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, addrMode, noDict, noDictIssue, acceleration); - } -} - - /* Note!: This function leaves the stream in an unclean/broken state! * It is not safe to subsequently use the same state with a _fastReset() or * _continue() call without resetting it. */ @@ -1340,27 +1430,23 @@ LZ4_stream_t* LZ4_createStream(void) return lz4s; } -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - it reports an aligment of 8-bytes, - while actually aligning LZ4_stream_t on 4 bytes. */ static size_t LZ4_stream_t_alignment(void) { - struct { char c; LZ4_stream_t t; } t_a; - return sizeof(t_a) - sizeof(t_a.t); -} +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_stream_t); +#else + return 1; /* effectively disabled */ #endif +} LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) { DEBUGLOG(5, "LZ4_initStream"); if (buffer == NULL) { return NULL; } if (size < sizeof(LZ4_stream_t)) { return NULL; } -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - it reports an aligment of 8-bytes, - while actually aligning LZ4_stream_t on 4 bytes. */ - if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */ -#endif - MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); return (LZ4_stream_t*)buffer; } @@ -1369,7 +1455,7 @@ LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); } void LZ4_resetStream_fast(LZ4_stream_t* ctx) { @@ -1418,7 +1504,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) base = dictEnd - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); - dict->tableType = tableType; + dict->tableType = (U32)tableType; while (p <= dictEnd-HASH_UNIT) { LZ4_putPosition(p, dict->hashTable, tableType, base); @@ -1436,12 +1522,6 @@ void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dict workingStream, dictionaryStream, dictCtx != NULL ? dictCtx->dictSize : 0); - /* Calling LZ4_resetStream_fast() here makes sure that changes will not be - * erased by subsequent calls to LZ4_resetStream_fast() in case stream was - * marked as having dirty context, e.g. requiring full reset. - */ - LZ4_resetStream_fast(workingStream); - if (dictCtx != NULL) { /* If the current offset is zero, we will never look in the * external dictionary context, since there is no value a table @@ -1493,9 +1573,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); - if (streamPtr->dirty) { return 0; } /* Uninitialized structure detected */ LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; /* invalidate tiny dictionaries */ if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */ @@ -1538,7 +1618,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, * cost to copy the dictionary's tables into the active context, * so that the compression loop is only looking into one table. */ - memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t)); + LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(*streamPtr)); result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); } else { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); @@ -1593,7 +1673,9 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); dict->dictionary = (const BYTE*)safeBuffer; dict->dictSize = (U32)dictSize; @@ -1623,25 +1705,27 @@ typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; */ typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error; LZ4_FORCE_INLINE unsigned -read_variable_length(const BYTE**ip, const BYTE* lencheck, int loop_check, int initial_check, variable_length_error* error) -{ - unsigned length = 0; - unsigned s; - if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ - *error = initial_error; - return length; - } - do { - s = **ip; - (*ip)++; - length += s; - if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ - *error = loop_error; - return length; +read_variable_length(const BYTE**ip, const BYTE* lencheck, + int loop_check, int initial_check, + variable_length_error* error) +{ + U32 length = 0; + U32 s; + if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = initial_error; + return length; } - } while (s==255); + do { + s = **ip; + (*ip)++; + length += s; + if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = loop_error; + return length; + } + } while (s==255); - return length; + return length; } /*! LZ4_decompress_generic() : @@ -1722,7 +1806,7 @@ LZ4_decompress_generic( /* decode literal length */ if (length == RUN_MASK) { variable_length_error error = ok; - length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error); if (error == initial_error) { goto _output_error; } if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ @@ -1746,12 +1830,12 @@ LZ4_decompress_generic( /* We don't need to check oend, since we check it once for each loop below */ if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ - memcpy(op, ip, 16); + LZ4_memcpy(op, ip, 16); } else { /* LZ4_decompress_fast() */ /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : * it doesn't know input length, and relies on end-of-block properties */ - memcpy(op, ip, 8); - if (length > 8) { memcpy(op+8, ip+8, 8); } + LZ4_memcpy(op, ip, 8); + if (length > 8) { LZ4_memcpy(op+8, ip+8, 8); } } ip += length; op = cpy; } @@ -1765,10 +1849,10 @@ LZ4_decompress_generic( length = token & ML_MASK; if (length == ML_MASK) { - variable_length_error error = ok; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ - length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); - if (error != ok) { goto _output_error; } + variable_length_error error = ok; + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error); + if (error != ok) { goto _output_error; } if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ length += MINMATCH; if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { @@ -1787,19 +1871,20 @@ LZ4_decompress_generic( assert(match <= op); assert(op + 18 <= oend); - memcpy(op, match, 8); - memcpy(op+8, match+8, 8); - memcpy(op+16, match+16, 2); + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op+8, match+8, 8); + LZ4_memcpy(op+16, match+16, 2); op += length; continue; } } } - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + if (checkOffset && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { if (partialDecoding) { - length = MIN(length, (size_t)(oend-op)); /* reach end of buffer */ + DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); + length = MIN(length, (size_t)(oend-op)); } else { goto _output_error; /* end-of-block condition violated */ } } @@ -1812,14 +1897,14 @@ LZ4_decompress_generic( /* match stretches into both external dictionary and current block */ size_t const copySize = (size_t)(lowPrefix - match); size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); + LZ4_memcpy(op, dictEnd - copySize, copySize); op += copySize; if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ BYTE* const endOfMatch = op + restSize; const BYTE* copyFrom = lowPrefix; while (op < endOfMatch) { *op++ = *copyFrom++; } } else { - memcpy(op, lowPrefix, restSize); + LZ4_memcpy(op, lowPrefix, restSize); op += restSize; } } continue; @@ -1860,7 +1945,7 @@ LZ4_decompress_generic( /* strictly "less than" on input, to re-enter the loop with at least one byte */ && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { /* Copy the literals */ - memcpy(op, ip, endOnInput ? 16 : 8); + LZ4_memcpy(op, ip, endOnInput ? 16 : 8); op += length; ip += length; /* The second stage: prepare for match copying, decode full info. @@ -1875,9 +1960,9 @@ LZ4_decompress_generic( && (offset >= 8) && (dict==withPrefix64k || match >= lowPrefix) ) { /* Copy the match. */ - memcpy(op + 0, match + 0, 8); - memcpy(op + 8, match + 8, 8); - memcpy(op +16, match +16, 2); + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op +16, match +16, 2); op += length + MINMATCH; /* Both stages worked, load the next token. */ continue; @@ -1891,7 +1976,7 @@ LZ4_decompress_generic( /* decode literal length */ if (length == RUN_MASK) { variable_length_error error = ok; - length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error); if (error == initial_error) { goto _output_error; } if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ @@ -1907,29 +1992,34 @@ LZ4_decompress_generic( || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) { /* We've either hit the input parsing restriction or the output parsing restriction. - * If we've hit the input parsing condition then this must be the last sequence. - * If we've hit the output parsing condition then we are either using partialDecoding - * or we've hit the output parsing condition. + * In the normal scenario, decoding a full block, it must be the last sequence, + * otherwise it's an error (invalid input or dimensions). + * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. */ if (partialDecoding) { /* Since we are partial decoding we may be in this block because of the output parsing * restriction, which is not valid since the output buffer is allowed to be undersized. */ assert(endOnInput); - /* If we're in this block because of the input parsing condition, then we must be on the - * last sequence (or invalid), so we must check that we exactly consume the input. + DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") + DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); + DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); + DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip)); + /* Finishing in the middle of a literals segment, + * due to lack of input. */ - if ((ip+length>iend-(2+1+LASTLITERALS)) && (ip+length != iend)) { goto _output_error; } - assert(ip+length <= iend); - /* We are finishing in the middle of a literals segment. - * Break after the copy. + if (ip+length > iend) { + length = (size_t)(iend-ip); + cpy = op + length; + } + /* Finishing in the middle of a literals segment, + * due to lack of output space. */ if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); } - assert(ip+length <= iend); } else { /* We must be on the last sequence because of the parsing limitations so check * that we exactly regenerate the original size (must be exact when !endOnInput). @@ -1938,16 +2028,22 @@ LZ4_decompress_generic( /* We must be on the last sequence (or invalid) because of the parsing limitations * so check that we exactly consume the input and don't overrun the output buffer. */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; } + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { + DEBUGLOG(6, "should have been last run of literals") + DEBUGLOG(6, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend); + DEBUGLOG(6, "or cpy(%p) > oend(%p)", cpy, oend); + goto _output_error; + } } - memmove(op, ip, length); /* supports overlapping memory regions, which only matters for in-place decompression scenarios */ + memmove(op, ip, length); /* supports overlapping memory regions; only matters for in-place decompression scenarios */ ip += length; op += length; - /* Necessarily EOF when !partialDecoding. When partialDecoding - * it is EOF if we've either filled the output buffer or hit - * the input parsing restriction. + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. */ - if (!partialDecoding || (cpy == oend) || (ip == iend)) { + if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) { break; } } else { @@ -1965,7 +2061,7 @@ LZ4_decompress_generic( _copy_match: if (length == ML_MASK) { variable_length_error error = ok; - length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); + length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error); if (error != ok) goto _output_error; if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ } @@ -1990,14 +2086,14 @@ LZ4_decompress_generic( /* match stretches into both external dictionary and current block */ size_t const copySize = (size_t)(lowPrefix - match); size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); + LZ4_memcpy(op, dictEnd - copySize, copySize); op += copySize; if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ BYTE* const endOfMatch = op + restSize; const BYTE* copyFrom = lowPrefix; while (op < endOfMatch) *op++ = *copyFrom++; } else { - memcpy(op, lowPrefix, restSize); + LZ4_memcpy(op, lowPrefix, restSize); op += restSize; } } continue; @@ -2016,7 +2112,7 @@ LZ4_decompress_generic( if (matchEnd > op) { /* overlap copy */ while (op < copyEnd) { *op++ = *match++; } } else { - memcpy(op, match, mlen); + LZ4_memcpy(op, match, mlen); } op = copyEnd; if (op == oend) { break; } @@ -2030,10 +2126,10 @@ LZ4_decompress_generic( op[2] = match[2]; op[3] = match[3]; match += inc32table[offset]; - memcpy(op+4, match, 4); + LZ4_memcpy(op+4, match, 4); match -= dec64table[offset]; } else { - memcpy(op, match, 8); + LZ4_memcpy(op, match, 8); match += 8; } op += 8; @@ -2048,7 +2144,7 @@ LZ4_decompress_generic( } while (op < cpy) { *op++ = *match++; } } else { - memcpy(op, match, 8); + LZ4_memcpy(op, match, 8); if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } } op = cpy; /* wildcopy correction */ @@ -2056,6 +2152,7 @@ LZ4_decompress_generic( /* end of decoding */ if (endOnInput) { + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ } else { return (int) (((const char*)ip)-src); /* Nb of input bytes read */ @@ -2070,7 +2167,7 @@ LZ4_decompress_generic( /*===== Instantiate the API decoding functions. =====*/ -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, @@ -2078,7 +2175,7 @@ int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int (BYTE*)dest, NULL, 0); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) { dstCapacity = MIN(targetOutputSize, dstCapacity); @@ -2087,7 +2184,7 @@ int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, noDict, (BYTE*)dst, NULL, 0); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, @@ -2097,7 +2194,7 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) /*===== Instantiate a few more decoding cases, used more than once. =====*/ -LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */ +LZ4_FORCE_O2 /* Exported, an obsolete API function. */ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, @@ -2113,7 +2210,7 @@ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int origin return LZ4_decompress_fast(source, dest, originalSize); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, size_t prefixSize) { @@ -2122,7 +2219,7 @@ static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, i (BYTE*)dest-prefixSize, NULL, 0); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const void* dictStart, size_t dictSize) @@ -2132,7 +2229,7 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, const void* dictStart, size_t dictSize) { @@ -2221,7 +2318,7 @@ int LZ4_decoderRingBufferSize(int maxBlockSize) If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setStreamDecode() */ -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; @@ -2261,7 +2358,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch return result; } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; @@ -2374,7 +2471,7 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, /* Obsolete Streaming functions */ -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } +int LZ4_sizeofStreamState(void) { return LZ4_STREAMSIZE; } int LZ4_resetStreamState(void* state, char* inputBuffer) { diff --git a/lz4libs/lz4.h b/lz4libs/lz4.h index 32108e23..7ab1e483 100644 --- a/lz4libs/lz4.h +++ b/lz4libs/lz4.h @@ -100,7 +100,7 @@ extern "C" { /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -186,7 +186,8 @@ LZ4LIB_API int LZ4_compressBound(int inputSize); The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). */ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -212,7 +213,18 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* d * New value is necessarily <= input value. * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) * or 0 if compression fails. -*/ + * + * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+): + * the produced compressed content could, in specific circumstances, + * require to be decompressed into a destination buffer larger + * by at least 1 byte than the content to decompress. + * If an application uses `LZ4_compress_destSize()`, + * it's highly recommended to update liblz4 to v1.9.2 or better. + * If this can't be done or ensured, + * the receiving decompression function should provide + * a dstCapacity which is > decompressedSize, by at least 1 byte. + * See https://github.com/lz4/lz4/issues/859 for details + */ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); @@ -220,25 +232,35 @@ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePt * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', * into destination buffer 'dst' of size 'dstCapacity'. * Up to 'targetOutputSize' bytes will be decoded. - * The function stops decoding on reaching this objective, - * which can boost performance when only the beginning of a block is required. + * The function stops decoding on reaching this objective. + * This can be useful to boost performance + * whenever only the beginning of a block is required. * - * @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity) + * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) * If source stream is detected malformed, function returns a negative result. * - * Note : @return can be < targetOutputSize, if compressed block contains less data. + * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : targetOutputSize must be <= dstCapacity * - * Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity, - * and expects targetOutputSize <= dstCapacity. - * It effectively stops decoding on reaching targetOutputSize, + * Note 3 : this function effectively stops decoding on reaching targetOutputSize, * so dstCapacity is kind of redundant. - * This is because in a previous version of this function, - * decoding operation would not "break" a sequence in the middle. - * As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize, + * This is because in older versions of this function, + * decoding operation would still write complete sequences. + * Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, * it could write more bytes, though only up to dstCapacity. * Some "margin" used to be required for this operation to work properly. - * This is no longer necessary. - * The function nonetheless keeps its signature, in an effort to not break API. + * Thankfully, this is no longer necessary. + * The function nonetheless keeps the same signature, in an effort to preserve API compatibility. + * + * Note 4 : If srcSize is the exact size of the block, + * then targetOutputSize can be any value, + * including larger than the block's decompressed size. + * The function will, at most, generate block's decompressed size. + * + * Note 5 : If srcSize is _larger_ than block's compressed size, + * then targetOutputSize **MUST** be <= block's decompressed size. + * Otherwise, *silent corruption will occur*. */ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); @@ -547,74 +569,64 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const #define LZ4_H_98237428734687 /*-************************************************************ - * PRIVATE DEFINITIONS + * Private Definitions ************************************************************** * Do not use these definitions directly. * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Accessing members will expose code to API and/or ABI break in future versions of the library. + * Accessing members will expose user code to API and/or ABI break in future versions of the library. **************************************************************/ #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; -struct LZ4_stream_t_internal { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint16_t dirty; - uint16_t tableType; - const uint8_t* dictionary; - const LZ4_stream_t_internal* dictCtx; - uint32_t dictSize; -}; - -typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - +# include + typedef int8_t LZ4_i8; + typedef uint8_t LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; #else + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef unsigned short LZ4_u16; + typedef unsigned int LZ4_u32; +#endif typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; struct LZ4_stream_t_internal { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned short dirty; - unsigned short tableType; - const unsigned char* dictionary; + LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; + LZ4_u32 currentOffset; + LZ4_u32 tableType; + const LZ4_byte* dictionary; const LZ4_stream_t_internal* dictCtx; - unsigned int dictSize; + LZ4_u32 dictSize; }; typedef struct { - const unsigned char* externalDict; - const unsigned char* prefixEnd; + const LZ4_byte* externalDict; size_t extDictSize; + const LZ4_byte* prefixEnd; size_t prefixSize; } LZ4_streamDecode_t_internal; -#endif /*! LZ4_stream_t : - * information structure to track an LZ4 stream. + * Do not use below internal definitions directly ! + * Declare or allocate an LZ4_stream_t instead. * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended. * The structure definition can be convenient for static allocation * (on stack, or as part of larger structure). * Init this structure with LZ4_initStream() before first use. * note : only use this definition in association with static linking ! - * this definition is not API/ABI safe, and may change in a future version. + * this definition is not API/ABI safe, and may change in future versions. */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ ) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +#define LZ4_STREAMSIZE 16416 /* static size, for inter-version compatibility */ +#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*)) union LZ4_stream_u { - unsigned long long table[LZ4_STREAMSIZE_U64]; + void* table[LZ4_STREAMSIZE_VOIDP]; LZ4_stream_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_stream_t */ +}; /* previously typedef'd to LZ4_stream_t */ + /*! LZ4_initStream() : v1.9.0+ * An LZ4_stream_t structure must be initialized at least once. @@ -667,22 +679,21 @@ union LZ4_streamDecode_u { #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) # else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) +# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") +# define LZ4_DEPRECATED(message) /* disabled */ # endif #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ -/* Obsolete compression functions */ +/*! Obsolete compression functions (since v1.7.3) */ LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); @@ -690,11 +701,12 @@ LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_co LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); -/* Obsolete decompression functions */ +/*! Obsolete decompression functions (since v1.8.0) */ LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); -/* Obsolete streaming functions; degraded functionality; do not use! +/* Obsolete streaming functions (since v1.7.0) + * degraded functionality; do not use! * * In order to perform streaming compression, these functions depended on data * that is no longer tracked in the state. They have been preserved as well as @@ -708,23 +720,22 @@ LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStre LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); -/* Obsolete streaming decoding functions */ +/*! Obsolete streaming decoding functions (since v1.7.0) */ LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); -/*! LZ4_decompress_fast() : **unsafe!** +/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : * These functions used to be faster than LZ4_decompress_safe(), - * but it has changed, and they are now slower than LZ4_decompress_safe(). + * but this is no longer the case. They are now slower. * This is because LZ4_decompress_fast() doesn't know the input size, - * and therefore must progress more cautiously in the input buffer to not read beyond the end of block. + * and therefore must progress more cautiously into the input buffer to not read beyond the end of block. * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. * * The last remaining LZ4_decompress_fast() specificity is that * it can decompress a block without knowing its compressed size. - * Such functionality could be achieved in a more secure manner, - * by also providing the maximum size of input buffer, - * but it would require new prototypes, and adaptation of the implementation to this new use case. + * Such functionality can be achieved in a more secure manner + * by employing LZ4_decompress_safe_partial(). * * Parameters: * originalSize : is the uncompressed size to regenerate. @@ -739,7 +750,6 @@ LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4 * But they may happen if input data is invalid (error or intentional tampering). * As a consequence, use these functions in trusted environments with trusted data **only**. */ - LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") diff --git a/lz4libs/lz4frame.c b/lz4libs/lz4frame.c index c9f630d6..ec02c92f 100644 --- a/lz4libs/lz4frame.c +++ b/lz4libs/lz4frame.c @@ -71,8 +71,8 @@ * towards another library or solution of their choice * by modifying below section. */ -#include /* malloc, calloc, free */ #ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# include /* malloc, calloc, free */ # define ALLOC(s) malloc(s) # define ALLOC_AND_ZERO(s) calloc(1,(s)) # define FREEMEM(p) free(p) @@ -533,7 +533,7 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict) * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. * Object can release its memory using LZ4F_freeCompressionContext(); */ -LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) +LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) { LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t)); if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed); @@ -541,20 +541,18 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c cctxPtr->version = version; cctxPtr->cStage = 0; /* Next stage : init stream */ - *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; + *LZ4F_compressionContextPtr = cctxPtr; return LZ4F_OK_NoError; } -LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr) { - LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; - if (cctxPtr != NULL) { /* support free on NULL */ - FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */ + FREEMEM(cctxPtr->lz4CtxPtr); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ FREEMEM(cctxPtr->tmpBuff); - FREEMEM(LZ4F_compressionContext); + FREEMEM(cctxPtr); } return LZ4F_OK_NoError; @@ -725,6 +723,9 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { + if (preferencesPtr && preferencesPtr->autoFlush) { + return LZ4F_compressBound_internal(srcSize, preferencesPtr, 0); + } return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1); } @@ -747,6 +748,7 @@ static size_t LZ4F_makeBlock(void* dst, (int)(srcSize), (int)(srcSize-1), level, cdict); if (cSize == 0) { /* compression failed */ + DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize); cSize = (U32)srcSize; LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+BHSize, src, srcSize); @@ -989,6 +991,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, BYTE* dstPtr = dstStart; size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); + DEBUGLOG(5,"LZ4F_compressEnd: dstCapacity=%u", (unsigned)dstCapacity); if (LZ4F_isError(flushSize)) return flushSize; dstPtr += flushSize; @@ -1002,6 +1005,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) { U32 const xxh = XXH32_digest(&(cctxPtr->xxh)); if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + DEBUGLOG(5,"Writing 32-bit content checksum"); LZ4F_writeLE32(dstPtr, xxh); dstPtr+=4; /* content Checksum */ } @@ -1112,6 +1116,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize size_t frameHeaderSize; const BYTE* srcPtr = (const BYTE*)src; + DEBUGLOG(5, "LZ4F_decodeHeader"); /* need to decode header to get frameInfo */ if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */ MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo)); @@ -1132,8 +1137,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize /* control magic number */ #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) { + DEBUGLOG(4, "frame header error : unknown magic number"); return err0r(LZ4F_ERROR_frameType_unknown); + } #endif dctx->frameInfo.frameType = LZ4F_frame; @@ -1282,15 +1289,20 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, /* LZ4F_updateDict() : - * only used for LZ4F_blockLinked mode */ + * only used for LZ4F_blockLinked mode + * Condition : dstPtr != NULL + */ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart, unsigned withinTmp) { - if (dctx->dictSize==0) - dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ + assert(dstPtr != NULL); + if (dctx->dictSize==0) { + dctx->dict = (const BYTE*)dstPtr; /* priority to prefix mode */ + } + assert(dctx->dict != NULL); - if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity, directly within dstBuffer */ + if (dctx->dict + dctx->dictSize == dstPtr) { /* prefix mode, everything within dstBuffer */ dctx->dictSize += dstSize; return; } @@ -1304,9 +1316,10 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, assert(dstSize < 64 KB); /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */ - /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOut */ + /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOutBuffer */ + assert(dctx->tmpOutBuffer != NULL); - if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */ + if (withinTmp && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */ /* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */ assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart); dctx->dictSize += dstSize; @@ -1378,17 +1391,21 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, const BYTE* const srcEnd = srcStart + *srcSizePtr; const BYTE* srcPtr = srcStart; BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* const dstEnd = dstStart + *dstSizePtr; + BYTE* const dstEnd = dstStart ? dstStart + *dstSizePtr : NULL; BYTE* dstPtr = dstStart; const BYTE* selectedIn = NULL; unsigned doAnotherStage = 1; size_t nextSrcSizeHint = 1; + DEBUGLOG(5, "LZ4F_decompress : %p,%u => %p,%u", + srcBuffer, (unsigned)*srcSizePtr, dstBuffer, (unsigned)*dstSizePtr); + if (dstBuffer == NULL) assert(*dstSizePtr == 0); MEM_INIT(&optionsNull, 0, sizeof(optionsNull)); if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; *srcSizePtr = 0; *dstSizePtr = 0; + assert(dctx != NULL); /* behaves as a state machine */ @@ -1398,6 +1415,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, { case dstage_getFrameHeader: + DEBUGLOG(6, "dstage_getFrameHeader"); if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */ if (LZ4F_isError(hSize)) return hSize; @@ -1411,6 +1429,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* fall-through */ case dstage_storeFrameHeader: + DEBUGLOG(6, "dstage_storeFrameHeader"); { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr)); memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); dctx->tmpInSize += sizeToCopy; @@ -1427,6 +1446,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; case dstage_init: + DEBUGLOG(6, "dstage_init"); if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0); /* internal buffers allocation */ { size_t const bufferNeeded = dctx->maxBlockSize @@ -1480,17 +1500,21 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } /* if (dctx->dStage == dstage_storeBlockHeader) */ /* decode block header */ - { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; + { U32 const blockHeader = LZ4F_readLE32(selectedIn); + size_t const nextCBlockSize = blockHeader & 0x7FFFFFFFU; size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize; - if (nextCBlockSize==0) { /* frameEnd signal, no more block */ + if (blockHeader==0) { /* frameEnd signal, no more block */ + DEBUGLOG(5, "end of frame"); dctx->dStage = dstage_getSuffix; break; } - if (nextCBlockSize > dctx->maxBlockSize) + if (nextCBlockSize > dctx->maxBlockSize) { return err0r(LZ4F_ERROR_maxBlockSize_invalid); - if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) { + } + if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) { /* next block is uncompressed */ dctx->tmpInTarget = nextCBlockSize; + DEBUGLOG(5, "next block is uncompressed (size %u)", (U32)nextCBlockSize); if (dctx->frameInfo.blockChecksumFlag) { (void)XXH32_reset(&dctx->blockChecksum, 0); } @@ -1508,20 +1532,26 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } case dstage_copyDirect: /* uncompressed block */ - { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr)); - size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); - memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctx->frameInfo.blockChecksumFlag) { - (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); - } - if (dctx->frameInfo.contentChecksumFlag) - (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); - if (dctx->frameInfo.contentSize) - dctx->frameRemainingSize -= sizeToCopy; + DEBUGLOG(6, "dstage_copyDirect"); + { size_t sizeToCopy; + if (dstPtr == NULL) { + sizeToCopy = 0; + } else { + size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr)); + sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); + memcpy(dstPtr, srcPtr, sizeToCopy); + if (dctx->frameInfo.blockChecksumFlag) { + (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + } + if (dctx->frameInfo.contentChecksumFlag) + (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= sizeToCopy; - /* history management (linked blocks only)*/ - if (dctx->frameInfo.blockMode == LZ4F_blockLinked) - LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0); + /* history management (linked blocks only)*/ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0); + } } srcPtr += sizeToCopy; dstPtr += sizeToCopy; @@ -1534,15 +1564,16 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; } dctx->tmpInTarget -= sizeToCopy; /* need to copy more */ - nextSrcSizeHint = dctx->tmpInTarget + - +(dctx->frameInfo.blockChecksumFlag ? BFSize : 0) - + BHSize /* next header size */; - doAnotherStage = 0; - break; } + nextSrcSizeHint = dctx->tmpInTarget + + +(dctx->frameInfo.blockChecksumFlag ? BFSize : 0) + + BHSize /* next header size */; + doAnotherStage = 0; + break; /* check block checksum for recently transferred uncompressed block */ case dstage_getBlockChecksum: + DEBUGLOG(6, "dstage_getBlockChecksum"); { const void* crcSrc; if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) { crcSrc = srcPtr; @@ -1562,8 +1593,12 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, { U32 const readCRC = LZ4F_readLE32(crcSrc); U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (readCRC != calcCRC) + DEBUGLOG(6, "compare block checksum"); + if (readCRC != calcCRC) { + DEBUGLOG(4, "incorrect block checksum: %08X != %08X", + readCRC, calcCRC); return err0r(LZ4F_ERROR_blockChecksum_invalid); + } #else (void)readCRC; (void)calcCRC; @@ -1573,6 +1608,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; case dstage_getCBlock: + DEBUGLOG(6, "dstage_getCBlock"); if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) { dctx->tmpInSize = 0; dctx->dStage = dstage_storeCBlock; @@ -1582,7 +1618,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, selectedIn = srcPtr; srcPtr += dctx->tmpInTarget; - if (0) /* jump over next block */ + if (0) /* always jump over next block */ case dstage_storeCBlock: { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize; size_t const inputLeft = (size_t)(srcEnd-srcPtr); @@ -1619,6 +1655,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, const char* dict = (const char*)dctx->dict; size_t dictSize = dctx->dictSize; int decodedSize; + assert(dstPtr != NULL); if (dict && dictSize > 1 GB) { /* the dictSize param is an int, avoid truncation / sign issues */ dict += dictSize - 64 KB; @@ -1636,8 +1673,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, dctx->frameRemainingSize -= (size_t)decodedSize; /* dictionary management */ - if (dctx->frameInfo.blockMode==LZ4F_blockLinked) + if (dctx->frameInfo.blockMode==LZ4F_blockLinked) { LZ4F_updateDict(dctx, dstPtr, (size_t)decodedSize, dstStart, 0); + } dstPtr += decodedSize; dctx->dStage = dstage_getBlockHeader; @@ -1684,7 +1722,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* fall-through */ case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ - { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr)); + DEBUGLOG(6, "dstage_flushOut"); + if (dstPtr != NULL) { + size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr)); memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy); /* dictionary management */ @@ -1693,16 +1733,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, dctx->tmpOutStart += sizeToCopy; dstPtr += sizeToCopy; - - if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */ - dctx->dStage = dstage_getBlockHeader; /* get next block */ - break; - } - /* could not flush everything : stop there, just request a block header */ - doAnotherStage = 0; - nextSrcSizeHint = BHSize; + } + if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */ + dctx->dStage = dstage_getBlockHeader; /* get next block */ break; } + /* could not flush everything : stop there, just request a block header */ + doAnotherStage = 0; + nextSrcSizeHint = BHSize; + break; case dstage_getSuffix: if (dctx->frameRemainingSize) @@ -1806,6 +1845,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2); if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */ && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */ + && (dctx->dict != NULL) /* dictionary exists */ && (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */ && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */ { @@ -1815,9 +1855,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; if (dctx->tmpOutSize > 64 KB) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; + assert(dctx->tmpOutBuffer != NULL); - if (copySize > 0) - memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); dctx->dict = dctx->tmpOutBuffer; dctx->dictSize = preserveSize + dctx->tmpOutStart; @@ -1825,8 +1865,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize; size_t const newDictSize = MIN(dctx->dictSize, 64 KB); - if (newDictSize > 0) - memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); dctx->dict = dctx->tmpOutBuffer; dctx->dictSize = newDictSize; diff --git a/lz4libs/lz4frame.h b/lz4libs/lz4frame.h index 391e4840..4573317e 100644 --- a/lz4libs/lz4frame.h +++ b/lz4libs/lz4frame.h @@ -66,17 +66,22 @@ extern "C" { *****************************************************************/ /* LZ4_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL - * LZ4FLIB_API : + * LZ4FLIB_VISIBILITY : * Control library symbols visibility. */ +#ifndef LZ4FLIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4FLIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4FLIB_VISIBILITY +# endif +#endif #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4FLIB_API __declspec(dllexport) +# define LZ4FLIB_API __declspec(dllexport) LZ4FLIB_VISIBILITY #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4FLIB_API __declspec(dllimport) -#elif defined(__GNUC__) && (__GNUC__ >= 4) -# define LZ4FLIB_API __attribute__ ((__visibility__ ("default"))) +# define LZ4FLIB_API __declspec(dllimport) LZ4FLIB_VISIBILITY #else -# define LZ4FLIB_API +# define LZ4FLIB_API LZ4FLIB_VISIBILITY #endif #ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS @@ -103,7 +108,7 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return /*-************************************ * Frame compression types - **************************************/ + ************************************* */ /* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */ #ifdef LZ4F_ENABLE_OBSOLETE_ENUMS # define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x @@ -113,7 +118,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return /* The larger the block size, the (slightly) better the compression ratio, * though there are diminishing returns. - * Larger blocks also increase memory usage on both compression and decompression sides. */ + * Larger blocks also increase memory usage on both compression and decompression sides. + */ typedef enum { LZ4F_default=0, LZ4F_max64KB=4, @@ -284,7 +290,7 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, * @return is always the same for a srcSize and prefsPtr. * prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. * tech details : - * @return includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. + * @return if automatic flushing is not enabled, includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. * It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd(). * @return doesn't include frame header, as it was already generated by LZ4F_compressBegin(). */ @@ -376,7 +382,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); * note : Frame header size is variable, but is guaranteed to be * >= LZ4F_HEADER_SIZE_MIN bytes, and <= LZ4F_HEADER_SIZE_MAX bytes. */ -size_t LZ4F_headerSize(const void* src, size_t srcSize); +LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize); /*! LZ4F_getFrameInfo() : * This function extracts frame parameters (max blockSize, dictID, etc.). @@ -426,8 +432,10 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, const void* srcBuffer, size_t* srcSizePtr); /*! LZ4F_decompress() : - * Call this function repetitively to regenerate compressed data from `srcBuffer`. - * The function will read up to *srcSizePtr bytes from srcBuffer, + * Call this function repetitively to regenerate data compressed in `srcBuffer`. + * + * The function requires a valid dctx state. + * It will read up to *srcSizePtr bytes from srcBuffer, * and decompress data into dstBuffer, of capacity *dstSizePtr. * * The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value). @@ -493,9 +501,9 @@ extern "C" { * Use at your own risk. */ #ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS -#define LZ4FLIB_STATIC_API LZ4FLIB_API +# define LZ4FLIB_STATIC_API LZ4FLIB_API #else -#define LZ4FLIB_STATIC_API +# define LZ4FLIB_STATIC_API #endif diff --git a/lz4libs/lz4hc.c b/lz4libs/lz4hc.c index 5922ed7b..77c9f430 100644 --- a/lz4libs/lz4hc.c +++ b/lz4libs/lz4hc.c @@ -53,7 +53,7 @@ #include "lz4hc.h" -/*=== Common LZ4 definitions ===*/ +/*=== Common definitions ===*/ #if defined(__GNUC__) # pragma GCC diagnostic ignored "-Wunused-function" #endif @@ -61,15 +61,16 @@ # pragma clang diagnostic ignored "-Wunused-function" #endif -/*=== Enums ===*/ -typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; - - #define LZ4_COMMONDEFS_ONLY #ifndef LZ4_SRC_INCLUDED #include "lz4.c" /* LZ4_count, constants, mem */ #endif + +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; + + /*=== Constants ===*/ #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) #define LZ4_OPT_NUM (1<<12) @@ -92,7 +93,7 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr) **************************************/ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) { - MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); + MEM_INIT(hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); } @@ -161,8 +162,7 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) { size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; - if (bitsToRotate == 0) - return pattern; + if (bitsToRotate == 0) return pattern; return LZ4HC_rotl32(pattern, (int)bitsToRotate); } @@ -172,7 +172,8 @@ static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) { const BYTE* const iStart = ip; - reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32; + reg_t const pattern = (sizeof(pattern)==8) ? + (reg_t)pattern32 + (((reg_t)pattern32) << (sizeof(pattern)*4)) : pattern32; while (likely(ip < iEnd-(sizeof(pattern)-1))) { reg_t const diff = LZ4_read_ARCH(ip) ^ pattern; @@ -270,7 +271,7 @@ LZ4HC_InsertAndGetWiderMatch ( DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)", matchIndex, lowestMatchIndex); - while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) { + while ((matchIndex>=lowestMatchIndex) && (nbAttempts>0)) { int matchLength=0; nbAttempts--; assert(matchIndex < ipIndex); @@ -389,8 +390,8 @@ LZ4HC_InsertAndGetWiderMatch ( if (lookBackLength==0) { /* no back possible */ size_t const maxML = MIN(currentSegmentLength, srcPatternLength); if ((size_t)longest < maxML) { - assert(base + matchIndex < ip); - if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break; + assert(base + matchIndex != ip); + if ((size_t)(ip - base) - matchIndex > LZ4_DISTANCE_MAX) break; assert(maxML < 2 GB); longest = (int)maxML; *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ @@ -410,7 +411,7 @@ LZ4HC_InsertAndGetWiderMatch ( } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */ if ( dict == usingDictCtxHc - && nbAttempts + && nbAttempts > 0 && ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) { size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->base); U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; @@ -460,74 +461,90 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index tabl * @return : 0 if ok, * 1 if buffer issue detected */ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( - const BYTE** ip, - BYTE** op, - const BYTE** anchor, + const BYTE** _ip, + BYTE** _op, + const BYTE** _anchor, int matchLength, const BYTE* const match, limitedOutput_directive limit, BYTE* oend) { +#define ip (*_ip) +#define op (*_op) +#define anchor (*_anchor) + size_t length; - BYTE* const token = (*op)++; + BYTE* const token = op++; #if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6) static const BYTE* start = NULL; static U32 totalCost = 0; - U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start); - U32 const ll = (U32)(*ip - *anchor); + U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start); + U32 const ll = (U32)(ip - anchor); U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0; U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; U32 const cost = 1 + llAdd + ll + 2 + mlAdd; - if (start==NULL) start = *anchor; /* only works for single segment */ + if (start==NULL) start = anchor; /* only works for single segment */ /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ - DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u", + DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5u, cost:%4u + %5u", pos, - (U32)(*ip - *anchor), matchLength, (U32)(*ip-match), + (U32)(ip - anchor), matchLength, (U32)(ip-match), cost, totalCost); totalCost += cost; #endif /* Encode Literal length */ - length = (size_t)(*ip - *anchor); - if ((limit) && ((*op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ + length = (size_t)(ip - anchor); + LZ4_STATIC_ASSERT(notLimited == 0); + /* Check output limit */ + if (limit && ((op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) { + DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)", + (int)length, (int)(oend - op)); + return 1; + } if (length >= RUN_MASK) { size_t len = length - RUN_MASK; *token = (RUN_MASK << ML_BITS); - for(; len >= 255 ; len -= 255) *(*op)++ = 255; - *(*op)++ = (BYTE)len; + for(; len >= 255 ; len -= 255) *op++ = 255; + *op++ = (BYTE)len; } else { *token = (BYTE)(length << ML_BITS); } /* Copy Literals */ - LZ4_wildCopy8(*op, *anchor, (*op) + length); - *op += length; + LZ4_wildCopy8(op, anchor, op + length); + op += length; /* Encode Offset */ - assert( (*ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */ - LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; + assert( (ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */ + LZ4_writeLE16(op, (U16)(ip - match)); op += 2; /* Encode MatchLength */ assert(matchLength >= MINMATCH); length = (size_t)matchLength - MINMATCH; - if ((limit) && (*op + (length / 255) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ + if (limit && (op + (length / 255) + (1 + LASTLITERALS) > oend)) { + DEBUGLOG(6, "Not enough room to write match length"); + return 1; /* Check output limit */ + } if (length >= ML_MASK) { *token += ML_MASK; length -= ML_MASK; - for(; length >= 510 ; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; } - if (length >= 255) { length -= 255; *(*op)++ = 255; } - *(*op)++ = (BYTE)length; + for(; length >= 510 ; length -= 510) { *op++ = 255; *op++ = 255; } + if (length >= 255) { length -= 255; *op++ = 255; } + *op++ = (BYTE)length; } else { *token += (BYTE)(length); } /* Prepare next loop */ - *ip += matchLength; - *anchor = *ip; + ip += matchLength; + anchor = ip; return 0; } +#undef ip +#undef op +#undef anchor LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( LZ4HC_CCtx_internal* const ctx, @@ -535,7 +552,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( char* const dest, int* srcSizePtr, int const maxOutputSize, - unsigned maxNbAttempts, + int maxNbAttempts, const limitedOutput_directive limit, const dictCtx_directive dict ) @@ -565,7 +582,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( /* init */ *srcSizePtr = 0; if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ - if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* Main Loop */ while (ip <= mflimit) { @@ -637,7 +654,11 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; ip = start2; optr = op; - if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) { + ml = ml2; + ref = ref2; + goto _dest_overflow; + } continue; } @@ -709,17 +730,18 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( _last_literals: /* Encode Last Literals */ { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ - size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; - size_t const totalSize = 1 + litLength + lastRunSize; + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ if (limit && (op + totalSize > oend)) { - if (limit == limitedOutput) return 0; /* Check output limit */ + if (limit == limitedOutput) return 0; /* adapt lastRunSize to fill 'dest' */ - lastRunSize = (size_t)(oend - op) - 1; - litLength = (lastRunSize + 255 - RUN_MASK) / 255; - lastRunSize -= litLength; + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; } - ip = anchor + lastRunSize; + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ if (lastRunSize >= RUN_MASK) { size_t accumulator = lastRunSize - RUN_MASK; @@ -739,9 +761,25 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( _dest_overflow: if (limit == fillOutput) { + /* Assumption : ip, anchor, ml and ref must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence overflowing"); op = optr; /* restore correct out pointer */ + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); assert(ml >= 0); + if ((size_t)ml > maxMlSize) ml = (int)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ml >= MFLIMIT) { + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, notLimited, oend); + } } goto _last_literals; } + /* compression failed */ return 0; } @@ -752,7 +790,7 @@ static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx, int const nbSearches, size_t sufficient_len, const limitedOutput_directive limit, int const fullUpdate, const dictCtx_directive dict, - HCfavor_e favorDecSpeed); + const HCfavor_e favorDecSpeed); LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( @@ -769,7 +807,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( typedef enum { lz4hc, lz4opt } lz4hc_strat_e; typedef struct { lz4hc_strat_e strat; - U32 nbSearches; + int nbSearches; U32 targetLength; } cParams_t; static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = { @@ -788,7 +826,8 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ }; - DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d)", ctx, src, *srcSizePtr); + DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)", + ctx, src, *srcSizePtr, limit); if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ @@ -808,7 +847,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( assert(cParam.strat == lz4opt); result = LZ4HC_compress_optimal(ctx, src, dst, srcSizePtr, dstCapacity, - (int)cParam.nbSearches, cParam.targetLength, limit, + cParam.nbSearches, cParam.targetLength, limit, cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ dict, favor); } @@ -881,27 +920,22 @@ LZ4HC_compress_generic ( int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); } -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - * it reports an aligment of 8-bytes, - * while actually aligning LZ4_streamHC_t on 4 bytes. */ static size_t LZ4_streamHC_t_alignment(void) { - struct { char c; LZ4_streamHC_t t; } t_a; - return sizeof(t_a) - sizeof(t_a.t); -} +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_streamHC_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_streamHC_t); +#else + return 1; /* effectively disabled */ #endif +} /* state is presumed correctly initialized, * in which case its size and alignment have already been validate */ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - * it reports an aligment of 8-bytes, - * while actually aligning LZ4_streamHC_t on 4 bytes. */ - assert(((size_t)state & (LZ4_streamHC_t_alignment() - 1)) == 0); /* check alignment */ -#endif - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + if (!LZ4_isAligned(state, LZ4_streamHC_t_alignment())) return 0; LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel); LZ4HC_init_internal (ctx, (const BYTE*)src); if (dstCapacity < LZ4_compressBound(srcSize)) @@ -950,10 +984,11 @@ int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* s /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { - LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); - if (LZ4_streamHCPtr==NULL) return NULL; - LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); /* full initialization, malloc'ed buffer can be full of garbage */ - return LZ4_streamHCPtr; + LZ4_streamHC_t* const state = + (LZ4_streamHC_t*)ALLOC_AND_ZERO(sizeof(LZ4_streamHC_t)); + if (state == NULL) return NULL; + LZ4_setCompressionLevel(state, LZ4HC_CLEVEL_DEFAULT); + return state; } int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) @@ -968,22 +1003,16 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) { LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer; - if (buffer == NULL) return NULL; - if (size < sizeof(LZ4_streamHC_t)) return NULL; -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - * it reports an aligment of 8-bytes, - * while actually aligning LZ4_streamHC_t on 4 bytes. */ - if (((size_t)buffer) & (LZ4_streamHC_t_alignment() - 1)) return NULL; /* alignment check */ -#endif /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE); - DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", LZ4_streamHCPtr, (unsigned)size); - /* end-base will trigger a clearTable on starting compression */ - LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; - LZ4_streamHCPtr->internal_donotuse.base = NULL; - LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; - LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0; - LZ4_streamHCPtr->internal_donotuse.dirty = 0; + DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", buffer, (unsigned)size); + /* check conditions */ + if (buffer == NULL) return NULL; + if (size < sizeof(LZ4_streamHC_t)) return NULL; + if (!LZ4_isAligned(buffer, LZ4_streamHC_t_alignment())) return NULL; + /* init */ + { LZ4HC_CCtx_internal* const hcstate = &(LZ4_streamHCPtr->internal_donotuse); + MEM_INIT(hcstate, 0, sizeof(*hcstate)); } LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); return LZ4_streamHCPtr; } @@ -1028,7 +1057,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize); + DEBUGLOG(4, "LZ4_loadDictHC(ctx:%p, dict:%p, dictSize:%d)", LZ4_streamHCPtr, dictionary, dictSize); assert(LZ4_streamHCPtr != NULL); if (dictSize > 64 KB) { dictionary += (size_t)dictSize - 64 KB; @@ -1069,14 +1098,15 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl ctxPtr->dictCtx = NULL; } -static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, - const char* src, char* dst, - int* srcSizePtr, int dstCapacity, - limitedOutput_directive limit) +static int +LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int dstCapacity, + limitedOutput_directive limit) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - DEBUGLOG(4, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d)", - LZ4_streamHCPtr, src, *srcSizePtr); + DEBUGLOG(5, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)", + LZ4_streamHCPtr, src, *srcSizePtr, limit); assert(ctxPtr != NULL); /* auto-init if forgotten */ if (ctxPtr->base == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src); @@ -1100,8 +1130,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, if (sourceEnd > dictEnd) sourceEnd = dictEnd; ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; - } - } + } } return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit); } @@ -1121,23 +1150,30 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch -/* dictionary saving */ - +/* LZ4_saveDictHC : + * save history content + * into a user-provided buffer + * which is then used to continue compression + */ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) { LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); - DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); + DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); + assert(prefixSize >= 0); if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; - memmove(safeBuffer, streamPtr->end - dictSize, dictSize); + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) + memmove(safeBuffer, streamPtr->end - dictSize, dictSize); { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base); streamPtr->end = (const BYTE*)safeBuffer + dictSize; streamPtr->base = streamPtr->end - endIndex; streamPtr->dictLimit = endIndex - (U32)dictSize; streamPtr->lowLimit = endIndex - (U32)dictSize; - if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; + if (streamPtr->nextToUpdate < streamPtr->dictLimit) + streamPtr->nextToUpdate = streamPtr->dictLimit; } return dictSize; } @@ -1287,8 +1323,13 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, const dictCtx_directive dict, const HCfavor_e favorDecSpeed) { + int retval = 0; #define TRAILING_LITERALS 3 +#ifdef LZ4HC_HEAPMODE + LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS)); +#else LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */ +#endif const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; @@ -1298,15 +1339,19 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, BYTE* op = (BYTE*) dst; BYTE* opSaved = (BYTE*) dst; BYTE* oend = op + dstCapacity; + int ovml = MINMATCH; /* overflow - last sequence */ + const BYTE* ovref = NULL; /* init */ +#ifdef LZ4HC_HEAPMODE + if (opt == NULL) goto _return_label; +#endif DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity); *srcSizePtr = 0; if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; /* Main Loop */ - assert(ip - anchor < LZ4_MAX_INPUT_SIZE); while (ip <= mflimit) { int const llen = (int)(ip - anchor); int best_mlen, best_off; @@ -1320,8 +1365,11 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, int const firstML = firstMatch.len; const BYTE* const matchPos = ip - firstMatch.off; opSaved = op; - if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) { /* updates ip, op and anchor */ + ovml = firstML; + ovref = matchPos; goto _dest_overflow; + } continue; } @@ -1463,7 +1511,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, best_off = opt[last_match_pos].off; cur = last_match_pos - best_mlen; - encode: /* cur, last_match_pos, best_mlen, best_off must be set */ +encode: /* cur, last_match_pos, best_mlen, best_off must be set */ assert(cur < LZ4_OPT_NUM); assert(last_match_pos >= 1); /* == 1 when only one candidate */ DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos); @@ -1493,25 +1541,31 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, assert(ml >= MINMATCH); assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX)); opSaved = op; - if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) { /* updates ip, op and anchor */ + ovml = ml; + ovref = ip - offset; goto _dest_overflow; - } } + } } } } /* while (ip <= mflimit) */ - _last_literals: +_last_literals: /* Encode Last Literals */ { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ - size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; - size_t const totalSize = 1 + litLength + lastRunSize; + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ if (limit && (op + totalSize > oend)) { - if (limit == limitedOutput) return 0; /* Check output limit */ + if (limit == limitedOutput) { /* Check output limit */ + retval = 0; + goto _return_label; + } /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (size_t)(oend - op) - 1; - litLength = (lastRunSize + 255 - RUN_MASK) / 255; - lastRunSize -= litLength; + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; } - ip = anchor + lastRunSize; + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ if (lastRunSize >= RUN_MASK) { size_t accumulator = lastRunSize - RUN_MASK; @@ -1527,12 +1581,35 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, /* End */ *srcSizePtr = (int) (((const char*)ip) - source); - return (int) ((char*)op-dst); + retval = (int) ((char*)op-dst); + goto _return_label; - _dest_overflow: - if (limit == fillOutput) { - op = opSaved; /* restore correct out pointer */ - goto _last_literals; - } - return 0; - } +_dest_overflow: +if (limit == fillOutput) { + /* Assumption : ip, anchor, ovml and ovref must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence overflowing (only %i bytes remaining)", (int)(oend-1-opSaved)); + op = opSaved; /* restore correct out pointer */ + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); assert(ovml >= 0); + if ((size_t)ovml > maxMlSize) ovml = (int)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ovml >= MFLIMIT) { + DEBUGLOG(6, "Space to end : %i + ml (%i)", (int)((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1), ovml); + DEBUGLOG(6, "Before : ip = %p, anchor = %p", ip, anchor); + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ovml, ovref, notLimited, oend); + DEBUGLOG(6, "After : ip = %p, anchor = %p", ip, anchor); + } } + goto _last_literals; +} +_return_label: +#ifdef LZ4HC_HEAPMODE + FREEMEM(opt); +#endif + return retval; +} diff --git a/lz4libs/lz4hc.h b/lz4libs/lz4hc.h index 44e35bbf..3d441fb6 100644 --- a/lz4libs/lz4hc.h +++ b/lz4libs/lz4hc.h @@ -198,57 +198,32 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; -struct LZ4HC_CCtx_internal -{ - uint32_t hashTable[LZ4HC_HASHTABLESIZE]; - uint16_t chainTable[LZ4HC_MAXD]; - const uint8_t* end; /* next block here to continue on current prefix */ - const uint8_t* base; /* All index relative to this position */ - const uint8_t* dictBase; /* alternate base for extDict */ - uint32_t dictLimit; /* below that point, need extDict */ - uint32_t lowLimit; /* below that point, no more dict */ - uint32_t nextToUpdate; /* index from which to continue dictionary update */ - short compressionLevel; - int8_t favorDecSpeed; /* favor decompression speed if this flag set, - otherwise, favor compression ratio */ - int8_t dirty; /* stream has to be fully reset if this flag is set */ - const LZ4HC_CCtx_internal* dictCtx; -}; - -#else - typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; struct LZ4HC_CCtx_internal { - unsigned int hashTable[LZ4HC_HASHTABLESIZE]; - unsigned short chainTable[LZ4HC_MAXD]; - const unsigned char* end; /* next block here to continue on current prefix */ - const unsigned char* base; /* All index relative to this position */ - const unsigned char* dictBase; /* alternate base for extDict */ - unsigned int dictLimit; /* below that point, need extDict */ - unsigned int lowLimit; /* below that point, no more dict */ - unsigned int nextToUpdate; /* index from which to continue dictionary update */ - short compressionLevel; - char favorDecSpeed; /* favor decompression speed if this flag set, - otherwise, favor compression ratio */ - char dirty; /* stream has to be fully reset if this flag is set */ + LZ4_u32 hashTable[LZ4HC_HASHTABLESIZE]; + LZ4_u16 chainTable[LZ4HC_MAXD]; + const LZ4_byte* end; /* next block here to continue on current prefix */ + const LZ4_byte* base; /* All index relative to this position */ + const LZ4_byte* dictBase; /* alternate base for extDict */ + LZ4_u32 dictLimit; /* below that point, need extDict */ + LZ4_u32 lowLimit; /* below that point, no more dict */ + LZ4_u32 nextToUpdate; /* index from which to continue dictionary update */ + short compressionLevel; + LZ4_i8 favorDecSpeed; /* favor decompression speed if this flag set, + otherwise, favor compression ratio */ + LZ4_i8 dirty; /* stream has to be fully reset if this flag is set */ const LZ4HC_CCtx_internal* dictCtx; }; -#endif - /* Do not use these definitions directly ! * Declare or allocate an LZ4_streamHC_t instead. */ -#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56 + ((sizeof(void*)==16) ? 56 : 0) /* AS400*/ ) /* 262200 or 262256*/ -#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) +#define LZ4_STREAMHCSIZE 262200 /* static size, for inter-version compatibility */ +#define LZ4_STREAMHCSIZE_VOIDP (LZ4_STREAMHCSIZE / sizeof(void*)) union LZ4_streamHC_u { - size_t table[LZ4_STREAMHCSIZE_SIZET]; + void* table[LZ4_STREAMHCSIZE_VOIDP]; LZ4HC_CCtx_internal internal_donotuse; }; /* previously typedef'd to LZ4_streamHC_t */ From 01ec59ec8194935627958faf6345678e727850e2 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 16 Jan 2021 13:57:30 +0000 Subject: [PATCH 149/280] Add Python 3.9 builds on Appveyor This required a shift to using the Visual Studio 2019 platform, as that's the only image that includes Python 3.9. This resulted in a significant refactor of the AppVeyor build. We now specify the compiler to be used for each Python version in order to follow: https://wiki.python.org/moin/WindowsCompilers. In addition, we no longer need to use the build.cmd script, and so remove that. --- appveyor.yml | 57 ++++++++++++++++++++++++++++++++-------------- appveyor/build.cmd | 21 ----------------- 2 files changed, 40 insertions(+), 38 deletions(-) delete mode 100644 appveyor/build.cmd diff --git a/appveyor.yml b/appveyor.yml index 45a3a029..c929fdbb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,14 +10,30 @@ environment: matrix: - - PYTHON: "C:\\Python35-x64" - PYTHON: "C:\\Python36-x64" - - PYTHON: "C:\\Python37-x64" - - PYTHON: "C:\\Python38-x64" - - PYTHON: "C:\\Python35" + TARGET: "x64" + VCVARS_VER: "14.2" - PYTHON: "C:\\Python36" + TARGET: "x86" + VCVARS_VER: "14.2" + - PYTHON: "C:\\Python37-x64" + TARGET: "x64" + VCVARS_VER: "14.2" - PYTHON: "C:\\Python37" + TARGET: "x86" + VCVARS_VER: "14.2" + - PYTHON: "C:\\Python38-x64" + TARGET: "x64" + VCVARS_VER: "14.2" - PYTHON: "C:\\Python38" + TARGET: "x86" + VCVARS_VER: "14.2" + - PYTHON: "C:\\Python39-x64" + TARGET: "x64" + VCVARS_VER: "14.2" + - PYTHON: "C:\\Python39" + TARGET: "x86" + VCVARS_VER: "14.2" matrix: # Immediately finish build once one of the jobs fails. @@ -25,6 +41,9 @@ matrix: platform: x64 +image: + - Visual Studio 2019 + install: # If there is a newer build queued for the same PR, cancel this one. # The AppVeyor 'rollout builds' option is supposed to serve the same @@ -39,39 +58,43 @@ install: - ps: "ls \"C:/\"" - ECHO "Installed SDKs:" - - ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\"" + - ps: "ls \"C:/Program Files (x86)/\"" + - ps: "ls \"C:/Program Files (x86)/Microsoft Visual Studio/\"" + + # Prepend relevant Python installation to the PATH. + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - # Prepend relevant Python installation to the PATH. Also prepend the location - # of the build.cmd file. - - "SET PATH=%CD%\\appveyor;%PYTHON%;%PYTHON%\\Scripts;%PATH%" + # Set up the build/compiler/linker environment + # See: https://github.com/upx/upx/blob/master/.appveyor.yml + - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" %TARGET% -vcvars_ver=%VCVARS_VER% # Check that we have the expected version and architecture for Python - - "build.cmd python --version" - - "build.cmd python -c \"import struct; print(struct.calcsize('P') * 8)\"" + - "python --version" + - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" # Upgrade to the latest version of pip to avoid it displaying warnings # about it being out of date. - - "build.cmd python -m pip install --disable-pip-version-check --user --upgrade pip" + - "python -m pip install --disable-pip-version-check --user --upgrade pip" # We need wheel installed to build wheels, and this isn't pulled in by setup.py - - "build.cmd python -m pip install wheel" + - "python -m pip install wheel" # Install tox for testing - - "build.cmd python -m pip install tox" + - "python -m pip install tox" # We need twine installed to push wheels to PyPI - - "build.cmd python -m pip install twine" + - "python -m pip install twine" build_script: # Build the compiled extension - - "build.cmd python setup.py build" + - "python setup.py build" test_script: - - "build.cmd tox -e py tests" + - "tox -e py tests" after_test: # If tests are successful, create binary packages for the project. - - "build.cmd python setup.py bdist_wheel" + - "python setup.py bdist_wheel" - ps: "ls dist" artifacts: diff --git a/appveyor/build.cmd b/appveyor/build.cmd deleted file mode 100644 index 243dc9a1..00000000 --- a/appveyor/build.cmd +++ /dev/null @@ -1,21 +0,0 @@ -@echo off -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows - -IF "%DISTUTILS_USE_SDK%"=="1" ( - ECHO Configuring environment to build with MSVC on a 64bit architecture - ECHO Using Windows SDK 7.1 - "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1 - CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release - SET MSSdk=1 - REM Need the following to allow tox to see the SDK compiler - SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB -) ELSE ( - ECHO Using default MSVC build environment -) - -CALL %* From 5af24afaeebc97cda333c951732ce0af0731b179 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 16 Jan 2021 13:56:31 +0000 Subject: [PATCH 150/280] Add Python 3.9 builds on Travis - Bump cibuildwheel to 1.7.4 - Use xcode-12.2 OSX image on Travis - Remove Python 3.5 builds - Upgrade to using Ubuntu Focal --- .travis.yml | 82 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2c06ddfe..0c11b2b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,5 @@ matrix: include: - - name: linux-py35 - language: python - python: 3.5 - sudo: required - services: - - docker - env: - - CIBW_BUILD=cp35-* - name: linux-py36 language: python python: 3.6 @@ -19,7 +11,7 @@ matrix: - name: linux-py37 language: python python: 3.7 - dist: xenial + dist: focal sudo: required services: - docker @@ -29,7 +21,7 @@ matrix: arch: arm64 language: python python: 3.7 - dist: xenial + dist: focal sudo: required services: - docker @@ -50,7 +42,7 @@ matrix: - name: linux-py38 language: python python: 3.8 - dist: xenial + dist: focal sudo: required services: - docker @@ -60,7 +52,7 @@ matrix: arch: arm64 language: python python: 3.8 - dist: xenial + dist: focal sudo: required services: - docker @@ -78,57 +70,93 @@ matrix: - git config --global user.name "junk" - git stash --all # Restore fresh checkout - cibuildwheel --output-dir dist - - name: osx-py35 - os: osx - language: generic + + - name: linux-py39 + language: python + python: 3.9 + dist: focal + sudo: required + services: + - docker env: - - CIBW_BUILD=cp35-* - - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" + - CIBW_BUILD=cp39-* + - name: linux-py39 + arch: arm64 + language: python + python: 3.9 + dist: focal + sudo: required + services: + - docker + env: + - PIP_NO_CACHE_DIR=off + - CIBW_BUILD=cp39-manylinux_aarch64 install: - - python -m pip install cibuildwheel==1.1.0 - - python -m pip install twine + - python -m pip install -U pip + - python -m pip install cibuildwheel==1.7.4 + - python -m pip install codecov tox twine script: + - tox tests + - codecov + - git config --global user.email "junk@junk.com" + - git config --global user.name "junk" + - git stash --all # Restore fresh checkout - cibuildwheel --output-dir dist - name: osx-py36 os: osx + osx_image: xcode12.2 language: generic env: - CIBW_BUILD=cp36-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - - python -m pip install cibuildwheel==1.1.0 - - python -m pip install twine + - python3 -m pip install cibuildwheel==1.7.4 + - python3 -m pip install twine script: - cibuildwheel --output-dir dist - name: osx-py37 os: osx + osx_image: xcode12.2 language: generic env: - CIBW_BUILD=cp37-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - - python -m pip install cibuildwheel==1.1.0 - - python -m pip install twine + - python3 -m pip install cibuildwheel==1.7.4 + - python3 -m pip install twine script: - cibuildwheel --output-dir dist - name: osx-py38 os: osx + osx_image: xcode12.2 language: generic env: - CIBW_BUILD=cp38-* - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" install: - - python -m pip install cibuildwheel==1.1.0 - - python -m pip install twine + - python3 -m pip install cibuildwheel==1.7.4 + - python3 -m pip install twine + script: + - cibuildwheel --output-dir dist + - name: osx-py39 + os: osx + osx_image: xcode12.2 + language: generic + env: + - CIBW_BUILD=cp39-* + - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" + - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" + install: + - python3 -m pip install cibuildwheel==1.7.4 + - python3 -m pip install twine script: - cibuildwheel --output-dir dist install: - python -m pip install -U pip - - python -m pip install cibuildwheel==1.1.0 + - python -m pip install cibuildwheel==1.7.4 - python -m pip install codecov tox twine script: - tox tests From 3e5a6fc62e82de6894652def98a5619b47bcfaeb Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Fri, 22 Jan 2021 23:33:09 +0000 Subject: [PATCH 151/280] Set dist to focal for Python 3.6 builds on Travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0c11b2b9..9a87bb33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ matrix: - name: linux-py36 language: python python: 3.6 + dist: focal sudo: required services: - docker From 815a2ac11e81b6485b12aa9f6b9e4fab8517b2b0 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 2 Jun 2021 21:26:37 +0300 Subject: [PATCH 152/280] Add readall to LZ4FrameFile and use it on python 3.10 --- lz4/frame/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 5fa03ce6..b77c425d 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -617,6 +617,17 @@ def peek(self, size=-1): # returns at least one byte (except at EOF) return self._buffer.peek(size) + def readall(self): + chunks = bytearray() + + while True: + data = self.read(io.DEFAULT_BUFFER_SIZE) + chunks += data + if not data: + break + + return bytes(chunks) + def read(self, size=-1): """Read up to ``size`` uncompressed bytes from the file. @@ -632,6 +643,9 @@ def read(self, size=-1): """ self._check_can_read() + + if size < 0 and sys.version_info >= (3, 10): + return self.readall() return self._buffer.read(size) def read1(self, size=-1): From f1b53964a55f172f0a5ead09159fe7f604acf566 Mon Sep 17 00:00:00 2001 From: smp30044 Date: Wed, 13 Oct 2021 09:08:23 -0700 Subject: [PATCH 153/280] Pass source_size kwarg through from frame.open to LZ4FrameFile instantiation --- lz4/frame/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 5fa03ce6..43bf772f 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -829,6 +829,7 @@ def open(filename, mode="rb", block_checksum=block_checksum, auto_flush=auto_flush, return_bytearray=return_bytearray, + source_size=source_size, ) if 't' in mode: From 226363d631f3927d56999cc8af17a33708f07a8b Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Tue, 19 Oct 2021 18:49:27 +0100 Subject: [PATCH 154/280] Add failing tests --- tests/frame/test_frame_9.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/frame/test_frame_9.py b/tests/frame/test_frame_9.py index 27d61607..51433934 100644 --- a/tests/frame/test_frame_9.py +++ b/tests/frame/test_frame_9.py @@ -1,5 +1,10 @@ +import array import os +import io +import pickle +import sys import lz4.frame +import pytest def test_issue_172_1(): @@ -42,3 +47,25 @@ def test_issue_172_3(): data = fp.read(16 * 1024 - 1) assert len(data) == 9 * 1024 assert data == input_data + + +def test_issue_227_1(): + q = array.array('Q', [1, 2, 3, 4, 5]) + LENGTH = len(q) * q.itemsize + + with lz4.frame.open(io.BytesIO(), 'w') as f: + assert f.write(q) == LENGTH + assert f.tell() == LENGTH + + +@pytest.mark.skipif( + sys.version_info < (3, 8), + reason="PickleBuffer only availiable in Python 3.8 or greater" +) +def test_issue_227_2(): + q = array.array('Q', [1, 2, 3, 4, 5]) + + c = lz4.frame.compress(q) + d = lz4.frame.LZ4FrameDecompressor().decompress(pickle.PickleBuffer(c)) + + assert memoryview(q).tobytes() == d From 2111b8aceb36f558218e6af61eb1fa6418872bdc Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Tue, 19 Oct 2021 18:49:55 +0100 Subject: [PATCH 155/280] Fix test failures --- lz4/frame/__init__.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 5fa03ce6..d3d3220f 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -387,6 +387,8 @@ def decompress(self, data, max_length=-1): # noqa: F811 bytes: Uncompressed data """ + if not isinstance(data, (bytes, bytearray)): + data = memoryview(data).tobytes() if self._unconsumed_data: data = self._unconsumed_data + data @@ -679,9 +681,10 @@ def readline(self, size=-1): def write(self, data): """Write a bytes object to the file. - Returns the number of uncompressed bytes written, which is always - ``len(data)``. Note that due to buffering, the file on disk may not - reflect the data written until close() is called. + Returns the number of uncompressed bytes written, which is + always the length of data in bytes. Note that due to buffering, + the file on disk may not reflect the data written until close() + is called. Args: data(bytes): uncompressed data to compress and write to the file @@ -690,11 +693,18 @@ def write(self, data): int: the number of uncompressed bytes written to the file """ + if isinstance(data, (bytes, bytearray)): + length = len(data) + else: + # accept any data that supports the buffer protocol + data = memoryview(data) + length = data.nbytes + self._check_can_write() compressed = self._compressor.compress(data) self._fp.write(compressed) - self._pos += len(data) - return len(data) + self._pos += length + return length def seek(self, offset, whence=io.SEEK_SET): """Change the file position. From f78505d88c3abb2ec0eac23ab1bd1dff2b254ba6 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Jun 2021 17:52:49 +0100 Subject: [PATCH 156/280] Remove appveyor.yml --- appveyor.yml | 125 --------------------------------------------------- 1 file changed, 125 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index c929fdbb..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,125 +0,0 @@ -# See: -# https://packaging.python.org/appveyor/ -# https://github.com/ogrisel/python-appveyor-demo -# https://www.appveyor.com/docs/appveyor-yml/ -# https://packaging.python.org/appveyor/ - -environment: - PYPI_PASSWORD: - secure: Sopzu1BZfXmABwhzM7mYSQ== - - matrix: - - - PYTHON: "C:\\Python36-x64" - TARGET: "x64" - VCVARS_VER: "14.2" - - PYTHON: "C:\\Python36" - TARGET: "x86" - VCVARS_VER: "14.2" - - PYTHON: "C:\\Python37-x64" - TARGET: "x64" - VCVARS_VER: "14.2" - - PYTHON: "C:\\Python37" - TARGET: "x86" - VCVARS_VER: "14.2" - - PYTHON: "C:\\Python38-x64" - TARGET: "x64" - VCVARS_VER: "14.2" - - PYTHON: "C:\\Python38" - TARGET: "x86" - VCVARS_VER: "14.2" - - PYTHON: "C:\\Python39-x64" - TARGET: "x64" - VCVARS_VER: "14.2" - - PYTHON: "C:\\Python39" - TARGET: "x86" - VCVARS_VER: "14.2" - -matrix: - # Immediately finish build once one of the jobs fails. - fast_finish: true - -platform: x64 - -image: - - Visual Studio 2019 - -install: - # If there is a newer build queued for the same PR, cancel this one. - # The AppVeyor 'rollout builds' option is supposed to serve the same - # purpose but it is problematic because it tends to cancel builds pushed - # directly to master instead of just PR builds (or the converse). - # credits: JuliaLang developers. - - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - throw "There are newer queued builds for this pull request, failing early." } - - ECHO "Filesystem root:" - - ps: "ls \"C:/\"" - - - ECHO "Installed SDKs:" - - ps: "ls \"C:/Program Files (x86)/\"" - - ps: "ls \"C:/Program Files (x86)/Microsoft Visual Studio/\"" - - # Prepend relevant Python installation to the PATH. - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - # Set up the build/compiler/linker environment - # See: https://github.com/upx/upx/blob/master/.appveyor.yml - - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" %TARGET% -vcvars_ver=%VCVARS_VER% - - # Check that we have the expected version and architecture for Python - - "python --version" - - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" - - # Upgrade to the latest version of pip to avoid it displaying warnings - # about it being out of date. - - "python -m pip install --disable-pip-version-check --user --upgrade pip" - - # We need wheel installed to build wheels, and this isn't pulled in by setup.py - - "python -m pip install wheel" - - # Install tox for testing - - "python -m pip install tox" - - # We need twine installed to push wheels to PyPI - - "python -m pip install twine" - -build_script: - # Build the compiled extension - - "python setup.py build" - -test_script: - - "tox -e py tests" - -after_test: - # If tests are successful, create binary packages for the project. - - "python setup.py bdist_wheel" - - ps: "ls dist" - -artifacts: - # Archive the generated packages in the ci.appveyor.com build report. - - path: dist\* - -deploy_script: - # Inspiration: https://github.com/AndrewAnnex/SpiceyPy/blob/master/appveyor.yml - - echo "Starting Artifact Deployment" - # populate pypirc file for twine - - echo [distutils] > %USERPROFILE%\\.pypirc - - echo index-servers = >> %USERPROFILE%\\.pypirc - - echo pypi >> %USERPROFILE%\\.pypirc - - echo [pypi] >> %USERPROFILE%\\.pypirc - - echo username=jgu >> %USERPROFILE%\\.pypirc - - echo password=%PYPI_PASSWORD% >> %USERPROFILE%\\.pypirc - # upload to pypi for windows - # set PATH=%BK_PATH% - # set HOME=%USERPROFILE% - - ps: >- - If ($env:APPVEYOR_REPO_TAG -eq "true") { - $env:pypi_upload = 'true' - } - Else { - write-output "Not on a tag, won't deploy to pypi" - } - - cmd: IF "%pypi_upload%"=="true" twine upload --skip-existing dist/* - - echo "Finished Artifact Deployment" From fffeff423ac85d0419a1fcd722b75e3109ca988d Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Jun 2021 17:53:07 +0100 Subject: [PATCH 157/280] Remove .travis.yml --- .travis.yml | 180 ---------------------------------------------------- 1 file changed, 180 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9a87bb33..00000000 --- a/.travis.yml +++ /dev/null @@ -1,180 +0,0 @@ -matrix: - include: - - name: linux-py36 - language: python - python: 3.6 - dist: focal - sudo: required - services: - - docker - env: - - CIBW_BUILD=cp36-* - - name: linux-py37 - language: python - python: 3.7 - dist: focal - sudo: required - services: - - docker - env: - - CIBW_BUILD=cp37-* - - name: linux-py37 - arch: arm64 - language: python - python: 3.7 - dist: focal - sudo: required - services: - - docker - env: - - PIP_NO_CACHE_DIR=off - - CIBW_BUILD=cp37-manylinux_aarch64 - install: - - python -m pip install -U pip - - python -m pip install cibuildwheel==1.7.4 - - python -m pip install codecov tox twine - script: - - tox tests - - codecov - - git config --global user.email "junk@junk.com" - - git config --global user.name "junk" - - git stash --all # Restore fresh checkout - - cibuildwheel --output-dir dist - - name: linux-py38 - language: python - python: 3.8 - dist: focal - sudo: required - services: - - docker - env: - - CIBW_BUILD=cp38-* - - name: linux-py38 - arch: arm64 - language: python - python: 3.8 - dist: focal - sudo: required - services: - - docker - env: - - PIP_NO_CACHE_DIR=off - - CIBW_BUILD=cp38-manylinux_aarch64 - install: - - python -m pip install -U pip - - python -m pip install cibuildwheel==1.7.4 - - python -m pip install codecov tox twine - script: - - tox tests - - codecov - - git config --global user.email "junk@junk.com" - - git config --global user.name "junk" - - git stash --all # Restore fresh checkout - - cibuildwheel --output-dir dist - - - name: linux-py39 - language: python - python: 3.9 - dist: focal - sudo: required - services: - - docker - env: - - CIBW_BUILD=cp39-* - - name: linux-py39 - arch: arm64 - language: python - python: 3.9 - dist: focal - sudo: required - services: - - docker - env: - - PIP_NO_CACHE_DIR=off - - CIBW_BUILD=cp39-manylinux_aarch64 - install: - - python -m pip install -U pip - - python -m pip install cibuildwheel==1.7.4 - - python -m pip install codecov tox twine - script: - - tox tests - - codecov - - git config --global user.email "junk@junk.com" - - git config --global user.name "junk" - - git stash --all # Restore fresh checkout - - cibuildwheel --output-dir dist - - name: osx-py36 - os: osx - osx_image: xcode12.2 - language: generic - env: - - CIBW_BUILD=cp36-* - - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" - install: - - python3 -m pip install cibuildwheel==1.7.4 - - python3 -m pip install twine - script: - - cibuildwheel --output-dir dist - - name: osx-py37 - os: osx - osx_image: xcode12.2 - language: generic - env: - - CIBW_BUILD=cp37-* - - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" - install: - - python3 -m pip install cibuildwheel==1.7.4 - - python3 -m pip install twine - script: - - cibuildwheel --output-dir dist - - name: osx-py38 - os: osx - osx_image: xcode12.2 - language: generic - env: - - CIBW_BUILD=cp38-* - - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" - install: - - python3 -m pip install cibuildwheel==1.7.4 - - python3 -m pip install twine - script: - - cibuildwheel --output-dir dist - - name: osx-py39 - os: osx - osx_image: xcode12.2 - language: generic - env: - - CIBW_BUILD=cp39-* - - CIBW_TEST_COMMAND="pushd {project}; tox tests ; popd" - - CIBW_BEFORE_BUILD="python -m pip install -U pip; python -m pip install tox twine" - install: - - python3 -m pip install cibuildwheel==1.7.4 - - python3 -m pip install twine - script: - - cibuildwheel --output-dir dist -install: - - python -m pip install -U pip - - python -m pip install cibuildwheel==1.7.4 - - python -m pip install codecov tox twine -script: - - tox tests - - codecov - - git config --global user.email "junk@junk.com" - - git config --global user.name "junk" - - git stash --all # Restore fresh checkout - - cibuildwheel --output-dir dist -deploy: - - provider: pypi - # server: https://test.pypi.org/legacy/ - user: jgu - password: - secure: "K1HppGu970i6WBKfnjQbCo0A0cYPih2rfM/rIDRzQXS8t+ZckyBywiKq/+4pkNJAfaHhQ8bHSz2J1GCq3Wz6DloGFHqEKAoHy5OgJxaK5nEp6avYvyHBSxrcZ6UfwFGMMMTAy4vfhnoJ3ARHaUFE4EIWYCVvP8/yCsE/6ir8AfozTssdYTUioJaYjSnowJ38nqa8H3hK3K5gke8aeclB+lnnWcyakgot65MfL2ov2mkkpWeiEn44ldV2iRKcl9a7Utgq3tJ1A634eUiNvIPpBYxnxPe++VRHA/mov9BWX0XfTmDuiVr+zDgMWdt3BK7f21r6P5DndpCZb8C2WzBFMKVz+rbuspbVJTol9Nt/9sVvSS6/fEBpXooRHRr9nP9EIt8Jlyv4CZ3iX/+T/912s84ZQCgqrhFNnRvECpmiVtHg49GecnMtJqC4jW57527c//oQPFyv/84Fd7fZ5jZg4LwhCl1C04Sqjmx8MFuMCR829CcZXJ2dAa5hQ/1v44SJAOKCQ0zDSdTqAy6x9RK/KNxnoQ9Nbn6ZFBUcjKx6rofyvXlwIcsQsjuzLw2mqmavZPduavVRi6cxK1WCEnaBVOXODWUSDatWchch1OcBm46X9UwwnHTsDHCVh6Bw+BDXhZuNw/72UA8z0sU/Val8iXBeThu33PMh3fo+uoqDf1g=" - distributions: "sdist" - skip_existing: true - skip_cleanup: true - on: - branch: master - tags: true From 424e648e857f70e78aefa44c966e75bff3e09418 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Jun 2021 23:25:30 +0100 Subject: [PATCH 158/280] Add github actions to build sdist and wheels --- .github/workflows/build_dist.yml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/build_dist.yml diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml new file mode 100644 index 00000000..c1ea7cb1 --- /dev/null +++ b/.github/workflows/build_dist.yml @@ -0,0 +1,53 @@ +name: Build + +on: [push, pull_request] + +# on: +# push: +# branches: +# - main +# tags: +# - '*' + + +jobs: + sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Install Python 3.x + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Build sdist + run: python setup.py sdist + - name: Save sdist + uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Install Python 3.x + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Build wheels + uses: joerick/cibuildwheel@v1.11.1 + env: + CIBW_ARCHS_LINUX: auto aarch64 + CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* + - name: Save wheels + uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl From f222eda214fe446f1bbefc0a4885b87cea0fd075 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 20 Jun 2021 08:58:25 +0100 Subject: [PATCH 159/280] Setup QEmu for github wheel workflow Building Aarch64 wheels on linux requires building in a virtual machine. --- .github/workflows/build_dist.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index c1ea7cb1..8053a51d 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -42,6 +42,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.x + - name: Set up QEMU # Needed to build aarch64 wheels + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v1 + with: + platforms: all - name: Build wheels uses: joerick/cibuildwheel@v1.11.1 env: From 71161851c3b04ee4056c60377003e2d5d67cfe6a Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 20 Jun 2021 10:19:46 +0100 Subject: [PATCH 160/280] Parallelise wheel building jobs --- .github/workflows/build_dist.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 8053a51d..a0a46b18 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -34,7 +34,8 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - + cibw_build: [cp36-*, cp37-*, cp38-*, cp39-*] + cibw_archs_linux: [auto, aarch64] steps: - name: Check out repository uses: actions/checkout@v2 @@ -43,15 +44,15 @@ jobs: with: python-version: 3.x - name: Set up QEMU # Needed to build aarch64 wheels - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64' uses: docker/setup-qemu-action@v1 with: platforms: all - name: Build wheels uses: joerick/cibuildwheel@v1.11.1 env: - CIBW_ARCHS_LINUX: auto aarch64 - CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* + CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} + CIBW_BUILD: ${{ matrix.cibw_build }} - name: Save wheels uses: actions/upload-artifact@v2 with: From 808bf5e5ac43319282d0ff8ca2efa247e3d77680 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 20 Jun 2021 10:23:29 +0100 Subject: [PATCH 161/280] Make github action build titles more descriptive --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index a0a46b18..dbee1d6b 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -29,7 +29,7 @@ jobs: path: dist/*.tar.gz wheels: - name: Build wheels on ${{ matrix.os }} + name: Build wheels on ${{ matrix.os }} CIBW_BUILD=${{ matrix.cibw_build }} CIBW_ARCHS_LINUX=${{ matrix.cibw_archs_linux }} runs-on: ${{ matrix.os }} strategy: matrix: From 7e14a4cec8bdcc87d225d91771121ba32e8e62e8 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 20 Jun 2021 11:56:11 +0100 Subject: [PATCH 162/280] Add upload to PyPi github action --- .github/workflows/build_dist.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index dbee1d6b..39288110 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -57,3 +57,17 @@ jobs: uses: actions/upload-artifact@v2 with: path: wheelhouse/*.whl + + upload_pypi: + name: Upload to PyPI + needs: [sdist, wheels] + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + - uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} From e13bbb89d3028fc1730e79a966edf2808df92f4c Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 20 Jun 2021 13:36:07 +0100 Subject: [PATCH 163/280] Add testing during wheel building for github CI --- .github/workflows/build_dist.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 39288110..2141b871 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -53,6 +53,8 @@ jobs: env: CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_TEST_COMMAND: "pushd {project} && tox tests && popd" + CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v2 with: From 0f69a4058d2b3f0bc9ec8ec0315b62153127e40e Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 20 Jun 2021 17:04:29 +0100 Subject: [PATCH 164/280] Disable Aarch64 builds on github CI --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 2141b871..a2390d7d 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -35,7 +35,7 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] cibw_build: [cp36-*, cp37-*, cp38-*, cp39-*] - cibw_archs_linux: [auto, aarch64] + cibw_archs_linux: [auto, ] # aarch64] steps: - name: Check out repository uses: actions/checkout@v2 From 4ccf1d273db8c6240a616fb1a45369fee791bd3e Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 20 Nov 2021 21:15:06 +0000 Subject: [PATCH 165/280] Disable stream tests with store_comp_size of 4 --- tests/stream/conftest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/stream/conftest.py b/tests/stream/conftest.py index c290ce0a..ea23e998 100644 --- a/tests/stream/conftest.py +++ b/tests/stream/conftest.py @@ -78,11 +78,11 @@ def buffer_size(request): 'store_comp_size': 2 } ), - ( - { - 'store_comp_size': 4 - } - ), + # ( + # { + # 'store_comp_size': 4 + # } + # ), ] ) def store_comp_size(request): From 3f1811f8b980aaed3fe547c9f25c4a6c6c1e17c1 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 20 Nov 2021 21:36:02 +0000 Subject: [PATCH 166/280] Use only 2 threads in test_block_0.py::test_2 --- tests/block/test_block_0.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index cca3e65b..d37ecf3b 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -84,7 +84,7 @@ def test_2(data, mode, store_size, dictionary): data_in = [data for i in range(32)] - pool = ThreadPool(8) + pool = ThreadPool(2) rt = partial(roundtrip, c_kwargs=c_kwargs, d_kwargs=d_kwargs, dictionary=dictionary) data_out = pool.map(rt, data_in) From 7e5bd66c117720b4514d92f70a834b2bb9de6b8a Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 20 Nov 2021 21:37:48 +0000 Subject: [PATCH 167/280] Remove pinning of dependencies in tox.ini --- tox.ini | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index c6de5abb..cd3223b2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,8 @@ [tox] envlist = py,flake8,docs minversion = 2.4.0 -# Pinning more-itertools and specifying virtualenv minimum version are -# workaround hacks for Python 2.7 and can be dropped when we drop support for -# Python 2.7. -requires = virtualenv >= 16.4.0 - more-itertools==4.2.0 +requires = virtualenv + more-itertools pathlib2 [testenv] From c59654cf62f7d2928b5f4fb6ebfb67e7f93c8f7b Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 21 Nov 2021 12:18:55 +0000 Subject: [PATCH 168/280] Disable windows builds --- .github/workflows/build_dist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index a2390d7d..58cab0ec 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -33,7 +33,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-latest, macOS-latest] # ,windows-latest] cibw_build: [cp36-*, cp37-*, cp38-*, cp39-*] cibw_archs_linux: [auto, ] # aarch64] steps: @@ -63,7 +63,7 @@ jobs: upload_pypi: name: Upload to PyPI needs: [sdist, wheels] - runs-on: ubuntu-latest + runs-on: [ubuntu-latest, macOS-latest] # ,windows-latest] if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - uses: actions/download-artifact@v2 From 541157437a7938ae91be303cb0eadd68ffdd1393 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 21 Nov 2021 13:17:20 +0000 Subject: [PATCH 169/280] Remove condition for push event to release to PyPi --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 58cab0ec..90423a9c 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -64,7 +64,7 @@ jobs: name: Upload to PyPI needs: [sdist, wheels] runs-on: [ubuntu-latest, macOS-latest] # ,windows-latest] - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + if: startsWith(github.ref, 'refs/tags/') steps: - uses: actions/download-artifact@v2 with: From eee9316f0833c9e521147280c49729d9c9c2ca53 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 21 Nov 2021 13:37:59 +0000 Subject: [PATCH 170/280] Trigger github actions on release --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 90423a9c..4002335f 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -1,6 +1,6 @@ name: Build -on: [push, pull_request] +on: [push, pull_request, release] # on: # push: From d1f4a2926b8f4ae8909324b507b6ee32e1623cdd Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 25 Nov 2021 19:44:57 +0000 Subject: [PATCH 171/280] Update github actions trigger --- .github/workflows/build_dist.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 4002335f..ad2cc457 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -1,14 +1,13 @@ name: Build -on: [push, pull_request, release] - -# on: -# push: -# branches: -# - main -# tags: -# - '*' +# on: [push, pull_request, release] +on: + push: + branches: + release: + types: [created] + pull_request: jobs: sdist: From 14f5fcf223d3744f451952a3d76c2ba44417c163 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 25 Nov 2021 20:01:38 +0000 Subject: [PATCH 172/280] Update github actions trigger to use published release --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index ad2cc457..1f0accee 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -6,7 +6,7 @@ on: push: branches: release: - types: [created] + types: [published] pull_request: jobs: From b6f1b07e5c3a3b6e32a471aa533ca5b32b070c0a Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 25 Nov 2021 20:40:01 +0000 Subject: [PATCH 173/280] Only run PyPi publish on ubuntu-latest --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 1f0accee..a63f8d99 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -62,7 +62,7 @@ jobs: upload_pypi: name: Upload to PyPI needs: [sdist, wheels] - runs-on: [ubuntu-latest, macOS-latest] # ,windows-latest] + runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: - uses: actions/download-artifact@v2 From 5b94dbdadb6d8a99b3df5723e84b5dd9616986ab Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 25 Nov 2021 21:07:26 +0000 Subject: [PATCH 174/280] Revert to using created release events --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index a63f8d99..7984b7fa 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -6,7 +6,7 @@ on: push: branches: release: - types: [published] + types: [created] pull_request: jobs: From 491050bec30f052205158996aa5f247ca106392f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 25 Nov 2021 22:37:28 +0000 Subject: [PATCH 175/280] Add skip existing when publishing to PyPi --- .github/workflows/build_dist.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 7984b7fa..5e79d3b4 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -72,3 +72,4 @@ jobs: - uses: pypa/gh-action-pypi-publish@master with: password: ${{ secrets.PYPI_API_TOKEN }} + skip_existing: true From 47f256270a82dd85be1ca61592f1b205e9961492 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 5 Dec 2021 13:50:25 +0000 Subject: [PATCH 176/280] Stop building python 3.6 wheels and start building python 3.10 wheels --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 5e79d3b4..92e78df4 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -33,7 +33,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macOS-latest] # ,windows-latest] - cibw_build: [cp36-*, cp37-*, cp38-*, cp39-*] + cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*] cibw_archs_linux: [auto, ] # aarch64] steps: - name: Check out repository From 449d2ce8071e6b629e392f79eb1b9164234629a2 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 5 Dec 2021 14:48:31 +0000 Subject: [PATCH 177/280] Use cibuildwheel 2.3.0 for wheel builds --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 92e78df4..f620cf80 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -48,7 +48,7 @@ jobs: with: platforms: all - name: Build wheels - uses: joerick/cibuildwheel@v1.11.1 + uses: joerick/cibuildwheel@v2.3.0 env: CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BUILD: ${{ matrix.cibw_build }} From 245f98f971e203233bff78e4b1bb19acef76481f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 12:08:23 +0000 Subject: [PATCH 178/280] Don't build the stream extension module by default This commit introduces the PYLZ4_EXPERIMENTAL environment variable to the setup.py file. If this is set to `1` or `TRUE` when setup.py is executed then the experimental modules are built. Otherwise they are not. This commit makes the stream module an experimental module. --- setup.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 0e1f8df5..ad014cc1 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import os from setuptools import setup, find_packages, Extension import sys from distutils import ccompiler @@ -33,6 +34,14 @@ def pkgconfig_installed_check(lib, required_version, default): return installed liblz4_found = pkgconfig_installed_check('liblz4', LZ4_REQUIRED_VERSION, default=False) +# Establish if we want to build experimental functionality or not. +experimental = os.environ.get("PYLZ4_EXPERIMENTAL", False) +if experimental is not False: + if experimental.upper() in ("1", "TRUE"): + experimental = True + else: + experimental = False + # Set up the extension modules. If a system wide lz4 library is found, and is # recent enough, we'll use that. Otherwise we'll build with the bundled one. If # we're building against the system lz4 library we don't set the compiler @@ -126,6 +135,11 @@ def pkgconfig_installed_check(lib, required_version, default): lz4stream_sources, **extension_kwargs) +ext_modules = [lz4version, lz4block, lz4frame] + +if experimental is True: + ext_modules.append(lz4stream) + install_requires = [] # On Python earlier than 3.0 the builtins package isn't included, but it is @@ -168,12 +182,7 @@ def pkgconfig_installed_check(lib, required_version, default): author_email='jonathan.underwood@gmail.com', url='https://github.com/python-lz4/python-lz4', packages=find_packages(), - ext_modules=[ - lz4version, - lz4block, - lz4frame, - lz4stream - ], + ext_modules=ext_modules, tests_require=tests_require, extras_require={ 'tests': tests_require, From 131c84772048e90dce1a0344ccc410039e69a946 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 12:29:41 +0000 Subject: [PATCH 179/280] Remove install_requires logic for older pythons --- setup.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/setup.py b/setup.py index ad014cc1..f80c41dc 100644 --- a/setup.py +++ b/setup.py @@ -140,14 +140,6 @@ def pkgconfig_installed_check(lib, required_version, default): if experimental is True: ext_modules.append(lz4stream) -install_requires = [] - -# On Python earlier than 3.0 the builtins package isn't included, but it is -# provided by the future package -if sys.version_info < (3, 0): - install_requires.append('future') - - # Dependencies for testing. We define a list here, so that we can # refer to it for the tests_require and the extras_require arguments # to setup below. The latter enables us to use pip install .[tests] to @@ -175,7 +167,6 @@ def pkgconfig_installed_check(lib, required_version, default): 'setuptools_scm', 'pkgconfig', ] + pytest_runner, - install_requires=install_requires, description="LZ4 Bindings for Python", long_description=open('README.rst', 'r').read(), author='Jonathan Underwood', From 9846115c0f6e2de63b2d24ff10039e07b84fc487 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 12:31:00 +0000 Subject: [PATCH 180/280] Only support python>=3.7 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f80c41dc..24ca6ca5 100644 --- a/setup.py +++ b/setup.py @@ -191,9 +191,9 @@ def pkgconfig_installed_check(lib, required_version, default): 'Intended Audience :: Developers', 'Programming Language :: C', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], ) From 02a8574b54ca4eacce3289d980dbbe662c2a9d8e Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 14:07:00 +0000 Subject: [PATCH 181/280] Update minimum python version to 3.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 24ca6ca5..9fddddc3 100644 --- a/setup.py +++ b/setup.py @@ -162,7 +162,7 @@ def pkgconfig_installed_check(lib, required_version, default): use_scm_version={ 'write_to': "lz4/version.py", }, - python_requires=">=3.5", + python_requires=">=3.7", setup_requires=[ 'setuptools_scm', 'pkgconfig', From 84487946a9c793ace6ed138705142e00b33ace79 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 14:47:26 +0000 Subject: [PATCH 182/280] Exclude the lz4.stream package from installation --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9fddddc3..6dd6124f 100644 --- a/setup.py +++ b/setup.py @@ -139,6 +139,9 @@ def pkgconfig_installed_check(lib, required_version, default): if experimental is True: ext_modules.append(lz4stream) + packages = find_packages() +else: + packages = find_packages(exclude=("lz4.stream",)) # Dependencies for testing. We define a list here, so that we can # refer to it for the tests_require and the extras_require arguments @@ -172,7 +175,7 @@ def pkgconfig_installed_check(lib, required_version, default): author='Jonathan Underwood', author_email='jonathan.underwood@gmail.com', url='https://github.com/python-lz4/python-lz4', - packages=find_packages(), + packages=packages, ext_modules=ext_modules, tests_require=tests_require, extras_require={ From 5311c9d2f724d8ff3fc4af2f5f137ff65ee05cc7 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 15:07:12 +0000 Subject: [PATCH 183/280] Add pyproject.toml --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..c5541b0a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2", "pkgconfig"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "lz4/version.py" \ No newline at end of file From 6b7bfa9626a430c1b3d4e61484611913ba170b89 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 15:13:57 +0000 Subject: [PATCH 184/280] Tidy up MANIFEST.in --- MANIFEST.in | 6 ------ 1 file changed, 6 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 092a3a47..9561fb10 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1 @@ include README.rst -include tests/*.py -exclude tests/*.pyc -exclude .gitignore -exclude .travis.yml -prune appveyor* -exclude appveyor.yml \ No newline at end of file From 4e0b924085d10508fe47385d5867c17254f0d001 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 15:15:20 +0000 Subject: [PATCH 185/280] Move testing of experimental features into own tox env --- tox.ini | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index cd3223b2..f57902da 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ passenv = * # PYTHONMALLOCSTATS = 'yes' usedevelop = True commands= - pytest --cov=lz4 --tb=long {posargs} + pytest --cov=lz4/block --cov=lz4/frame --tb=long {posargs} tests/block tests/frame [pytest] addopts = -x --tb=long --showlocals @@ -33,3 +33,14 @@ whitelist_externals=make extras = docs commands = make -C docs doctest html + +[testenv:experimental] +extras = tests +passenv = * +setenv = + PYLZ4_EXPERIMENTAL = "True" +# PYTHONMALLOC = pymalloc +# PYTHONMALLOCSTATS = 'yes' +usedevelop = True +commands= + pytest --cov=lz4/stream --tb=long {posargs} tests/stream From 211cfb40af0771a864d01745e8850b1604815a39 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 15:21:55 +0000 Subject: [PATCH 186/280] Remove tests arg from CIBW_TEST_COMMAND --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index f620cf80..28fbd1e6 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -52,7 +52,7 @@ jobs: env: CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BUILD: ${{ matrix.cibw_build }} - CIBW_TEST_COMMAND: "pushd {project} && tox tests && popd" + CIBW_TEST_COMMAND: "pushd {project} && tox && popd" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v2 From 44829a02751d95c31ae7a6075147bb041bbdf853 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 15:24:08 +0000 Subject: [PATCH 187/280] Re-enable windows builds --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 28fbd1e6..c1c1f5b9 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -32,7 +32,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macOS-latest] # ,windows-latest] + os: [ubuntu-latest, macOS-latest, windows-latest] cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*] cibw_archs_linux: [auto, ] # aarch64] steps: From 536c6c5abd0a89e0d51a4c613ddda5215937d963 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 15:34:01 +0000 Subject: [PATCH 188/280] Turn on experimental features when building docs --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index f57902da..d038f027 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,8 @@ commands = ignore = E501 [testenv:docs] +setenv = + PYLZ4_EXPERIMENTAL = "True" usedevelop = True whitelist_externals=make extras = docs From c070324010f44adc6be69892e781bfe2549a432f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 16:20:02 +0000 Subject: [PATCH 189/280] Pass env vars to tox docs environment --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index d038f027..afa5d3c0 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,7 @@ ignore = E501 [testenv:docs] setenv = PYLZ4_EXPERIMENTAL = "True" +passenv = * usedevelop = True whitelist_externals=make extras = docs From e8cf219fe953fa58f8aee3354a0fbebee8f2ec6b Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 16:38:22 +0000 Subject: [PATCH 190/280] Don't set usedevelop when building docs from tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index afa5d3c0..9647fdfe 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ ignore = E501 setenv = PYLZ4_EXPERIMENTAL = "True" passenv = * -usedevelop = True +# usedevelop = True whitelist_externals=make extras = docs commands = From 1e19bbdd4391b0f4b9f8a6ea832b985ad3a3e7d2 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 20:07:31 +0000 Subject: [PATCH 191/280] Fix setting of env vars in tox.ini This commit also cleans up whitespace in tox.ini --- tox.ini | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 9647fdfe..d84b02a1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py,flake8,docs +envlist = py, flake8, docs minversion = 2.4.0 requires = virtualenv more-itertools @@ -11,7 +11,7 @@ passenv = * # setenv = PYTHONMALLOC = pymalloc # PYTHONMALLOCSTATS = 'yes' usedevelop = True -commands= +commands = pytest --cov=lz4/block --cov=lz4/frame --tb=long {posargs} tests/block tests/frame [pytest] @@ -29,10 +29,10 @@ ignore = E501 [testenv:docs] setenv = - PYLZ4_EXPERIMENTAL = "True" + PYLZ4_EXPERIMENTAL = 1 passenv = * -# usedevelop = True -whitelist_externals=make +usedevelop = True +whitelist_externals = make extras = docs commands = make -C docs doctest html @@ -41,9 +41,9 @@ commands = extras = tests passenv = * setenv = - PYLZ4_EXPERIMENTAL = "True" + PYLZ4_EXPERIMENTAL = 1 # PYTHONMALLOC = pymalloc # PYTHONMALLOCSTATS = 'yes' usedevelop = True -commands= +commands = pytest --cov=lz4/stream --tb=long {posargs} tests/stream From 082da8654107386037bd7f4a3a55a030405183ed Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 20:19:16 +0000 Subject: [PATCH 192/280] Add a warning to the stream module docs This wwarning highlights that the stream module is considered experimental and unmaintained. --- docs/lz4.stream.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/lz4.stream.rst b/docs/lz4.stream.rst index d7b8c3af..6b9890c1 100644 --- a/docs/lz4.stream.rst +++ b/docs/lz4.stream.rst @@ -4,6 +4,20 @@ lz4.stream sub-package ====================== +.. warning:: + This module is unmaintained. + + This sub-package is considered experimental. It was submitted by a community + member who is not able to continue to maintain the module. + + This module is not built as part of the distributed wheels. If you wish to + build and use this module you will need to download and build from source + with the environment variable PYLZ4_EXPERIMENTAL set to TRUE. + + The module needs some re-write, and the tests need extensive work, for this + to become production ready. If you are interested in working on this, please + reach out to the package maintainers. + This sub-package provides the capability to compress and decompress data using the `stream specification `_, From 8f4336253cb0bdb1f9aacb979bebfcc7557e0658 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 20:35:07 +0000 Subject: [PATCH 193/280] Remove outdated requires from tox.ini --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index d84b02a1..b7539459 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,6 @@ [tox] envlist = py, flake8, docs minversion = 2.4.0 -requires = virtualenv - more-itertools - pathlib2 [testenv] extras = tests From 99a907b5b8f20a5af015b68501bcf4ad16dd04e3 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 21:23:32 +0000 Subject: [PATCH 194/280] Update cibuildwheel to 2.3.1 --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index c1c1f5b9..98fef9f9 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -48,7 +48,7 @@ jobs: with: platforms: all - name: Build wheels - uses: joerick/cibuildwheel@v2.3.0 + uses: joerick/cibuildwheel@v2.3.1 env: CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BUILD: ${{ matrix.cibw_build }} From 963130c331d54930e97db53ce50fcf0e364bd7ad Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 21:24:05 +0000 Subject: [PATCH 195/280] Stop using pushd/popd which are not available on dash --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 98fef9f9..07033e00 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -52,7 +52,7 @@ jobs: env: CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BUILD: ${{ matrix.cibw_build }} - CIBW_TEST_COMMAND: "pushd {project} && tox && popd" + CIBW_TEST_COMMAND: "cd {project} && tox" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v2 From 1f757a3791463f155fbf1861081fbe59bc02b69e Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 22:42:06 +0000 Subject: [PATCH 196/280] Use tox -c parameter to locate tox.ini file --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 07033e00..027547cf 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -52,7 +52,7 @@ jobs: env: CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BUILD: ${{ matrix.cibw_build }} - CIBW_TEST_COMMAND: "cd {project} && tox" + CIBW_TEST_COMMAND: "tox -c {project}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v2 From f41d7f7e687e1ade10f32aef0fa638452968608c Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 22:54:36 +0000 Subject: [PATCH 197/280] Specify full repo depth when checking out repos This is necessary to ensure tags are retrieved to enable setuptools_scm to work correctly. --- .github/workflows/build_dist.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 027547cf..3a2020fd 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -16,6 +16,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v2 + with: + fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work - name: Install Python 3.x uses: actions/setup-python@v2 with: @@ -38,6 +40,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v2 + with: + fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work - name: Install Python 3.x uses: actions/setup-python@v2 with: From 86e5c75915d55f9979fc03efd3d1cc6bb2628366 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 13 Feb 2022 23:02:06 +0000 Subject: [PATCH 198/280] Disable musllinux wheel builds --- .github/workflows/build_dist.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 3a2020fd..59216262 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -37,6 +37,7 @@ jobs: os: [ubuntu-latest, macOS-latest, windows-latest] cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*] cibw_archs_linux: [auto, ] # aarch64] + cibw_skip: ["cp*-musllinux*", ] steps: - name: Check out repository uses: actions/checkout@v2 @@ -56,6 +57,7 @@ jobs: env: CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_SKIP: ${{ matrix.cibw_skip }} CIBW_TEST_COMMAND: "tox -c {project}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels From 54932796c78406ba24c8cfd0119ce56772fa0fb8 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 17 Feb 2022 08:42:22 +0000 Subject: [PATCH 199/280] Update README.rst to reflect the experimental nature of the streaming bindings --- README.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 3050f485..15ba4445 100644 --- a/README.rst +++ b/README.rst @@ -27,15 +27,17 @@ Introduction This package provides python bindings for the `LZ4 compression library `_. -The bindings provided in this package cover the `frame format -`_, the `block -format `_, and the -`streaming format -`_ +The production ready bindings provided in this package cover the `frame format +`_, and the +`block format `_ specifications. The frame format bindings are the recommended ones to use, as this guarantees interoperability with other implementations and language bindings. +Experimental bindings for the the `streaming format +`_ +specification are also included, but further work on those is required. + The API provided by the frame format bindings follows that of the LZMA, zlib, gzip and bzip2 compression libraries which are provided with the Python standard library. As such, these LZ4 bindings should provide a drop-in alternative to the From 8825372cf94e81cb675857c6219a395b1352a9cc Mon Sep 17 00:00:00 2001 From: Pedro Batista Date: Sat, 26 Mar 2022 13:43:08 -0300 Subject: [PATCH 200/280] WIP - Add flush method to LZ4FrameCompressor --- lz4/frame/__init__.py | 28 ++++++++++++++++++++++------ tests/frame/test_frame_10.py | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 tests/frame/test_frame_10.py diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 789d7ad9..8d287290 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -219,13 +219,13 @@ def begin(self, source_size=0): block_checksum=self.block_checksum, auto_flush=self.auto_flush, return_bytearray=self.return_bytearray, - source_size=source_size + source_size=source_size, ) self._started = True return result else: raise RuntimeError( - 'LZ4FrameCompressor.begin() called after already initialized' + "LZ4FrameCompressor.begin() called after already initialized" ) def compress(self, data): # noqa: F811 @@ -294,6 +294,10 @@ def reset(self): self._context = None self._started = False + def has_context(self): + + return self._context is not None + class LZ4FrameDecompressor(object): """Create a LZ4 frame decompressor object. @@ -525,9 +529,8 @@ def __init__(self, filename=None, mode='r', self._buffer = io.BufferedReader(raw) if self._mode == _MODE_WRITE: - self._fp.write( - self._compressor.begin(source_size=source_size) - ) + self._source_size = source_size + self._fp.write(self._compressor.begin(source_size=source_size)) def close(self): """Flush and close the file. @@ -542,7 +545,8 @@ def close(self): self._buffer.close() self._buffer = None elif self._mode == _MODE_WRITE: - self._fp.write(self._compressor.flush()) + if self._compressor.has_context(): + self._fp.write(self._compressor.flush()) self._compressor = None finally: try: @@ -720,6 +724,18 @@ def write(self, data): self._pos += length return length + def flush(self): + """Flush the file, keeping it open. + + May be called more than once without error. The file may continue + to be used normally after flushing. + """ + if self.writable() and self._compressor.has_context(): + self._fp.write(self._compressor.flush()) + header = self._compressor.begin(source_size=self._source_size) + self._fp.write(header) + self._fp.flush() + def seek(self, offset, whence=io.SEEK_SET): """Change the file position. diff --git a/tests/frame/test_frame_10.py b/tests/frame/test_frame_10.py new file mode 100644 index 00000000..a4da2327 --- /dev/null +++ b/tests/frame/test_frame_10.py @@ -0,0 +1,22 @@ +import lz4.frame as lz4frame + + +def test_lz4frame_open_write_read_text_iter(): + data_1 = u"This is a..." + data_2 = u"...test string!" + + fp = lz4frame.open("testfile", mode="wt") + fp.write(data_1) + fp.flush() + # fp.flush() + + # fp.write(data_2) + + with lz4frame.open("testfile", mode="rt") as fp_read: + print(fp_read.read()) + assert fp_read.read() == data_1 + + fp.flush() + + with lz4frame.open("testfile", mode="rt") as fp_read: + assert fp_read.read() == data_1 + data_2 From d4bdd1fdc200a8dfcc07ed66ab1e9140a617bf0d Mon Sep 17 00:00:00 2001 From: Pedro Batista Date: Sat, 26 Mar 2022 15:15:22 -0300 Subject: [PATCH 201/280] Add flush method to LZ4FrameCompressor --- lz4/frame/__init__.py | 11 ++++++++--- tests/frame/test_frame_10.py | 24 +++++++++++------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 8d287290..791fb0d9 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -295,9 +295,11 @@ def reset(self): self._started = False def has_context(self): - return self._context is not None + def started(self): + return self._started + class LZ4FrameDecompressor(object): """Create a LZ4 frame decompressor object. @@ -719,6 +721,11 @@ def write(self, data): length = data.nbytes self._check_can_write() + + if not self._compressor.started(): + header = self._compressor.begin(source_size=self._source_size) + self._fp.write(header) + compressed = self._compressor.compress(data) self._fp.write(compressed) self._pos += length @@ -732,8 +739,6 @@ def flush(self): """ if self.writable() and self._compressor.has_context(): self._fp.write(self._compressor.flush()) - header = self._compressor.begin(source_size=self._source_size) - self._fp.write(header) self._fp.flush() def seek(self, offset, whence=io.SEEK_SET): diff --git a/tests/frame/test_frame_10.py b/tests/frame/test_frame_10.py index a4da2327..d8789a3f 100644 --- a/tests/frame/test_frame_10.py +++ b/tests/frame/test_frame_10.py @@ -2,21 +2,19 @@ def test_lz4frame_open_write_read_text_iter(): - data_1 = u"This is a..." - data_2 = u"...test string!" + data_1 = b"This is a..." + data_2 = b" test string!" - fp = lz4frame.open("testfile", mode="wt") - fp.write(data_1) - fp.flush() - # fp.flush() + with lz4frame.open("testfile", mode="w") as fp_write: + fp_write.write(data_1) + fp_write.flush() - # fp.write(data_2) + fp_write.write(data_2) - with lz4frame.open("testfile", mode="rt") as fp_read: - print(fp_read.read()) - assert fp_read.read() == data_1 + with lz4frame.open("testfile", mode="r") as fp_read: + assert fp_read.read() == data_1 - fp.flush() + fp_write.flush() - with lz4frame.open("testfile", mode="rt") as fp_read: - assert fp_read.read() == data_1 + data_2 + with lz4frame.open("testfile", mode="r") as fp_read: + assert fp_read.read() == data_1 + data_2 From 35ff9667773d5c7e2113cbd2aa80f7677bc764d5 Mon Sep 17 00:00:00 2001 From: Pedro Batista Date: Tue, 29 Mar 2022 13:09:26 -0300 Subject: [PATCH 202/280] Move flush test to existing file --- tests/frame/test_frame_10.py | 20 -------------------- tests/frame/test_frame_6.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 20 deletions(-) delete mode 100644 tests/frame/test_frame_10.py diff --git a/tests/frame/test_frame_10.py b/tests/frame/test_frame_10.py deleted file mode 100644 index d8789a3f..00000000 --- a/tests/frame/test_frame_10.py +++ /dev/null @@ -1,20 +0,0 @@ -import lz4.frame as lz4frame - - -def test_lz4frame_open_write_read_text_iter(): - data_1 = b"This is a..." - data_2 = b" test string!" - - with lz4frame.open("testfile", mode="w") as fp_write: - fp_write.write(data_1) - fp_write.flush() - - fp_write.write(data_2) - - with lz4frame.open("testfile", mode="r") as fp_read: - assert fp_read.read() == data_1 - - fp_write.flush() - - with lz4frame.open("testfile", mode="r") as fp_read: - assert fp_read.read() == data_1 + data_2 diff --git a/tests/frame/test_frame_6.py b/tests/frame/test_frame_6.py index 335d09e4..c20a4f31 100644 --- a/tests/frame/test_frame_6.py +++ b/tests/frame/test_frame_6.py @@ -98,3 +98,22 @@ def test_lz4frame_open_write_read( data_out = fp.read() assert data_out == data + + +def test_lz4frame_flush(): + data_1 = b"This is a..." + data_2 = b" test string!" + + with lz4frame.open("testfile", mode="w") as fp_write: + fp_write.write(data_1) + fp_write.flush() + + fp_write.write(data_2) + + with lz4frame.open("testfile", mode="r") as fp_read: + assert fp_read.read() == data_1 + + fp_write.flush() + + with lz4frame.open("testfile", mode="r") as fp_read: + assert fp_read.read() == data_1 + data_2 From 287b77e9a45c3bf14b46e27c7b02c8d56af8ef32 Mon Sep 17 00:00:00 2001 From: Pedro Batista Date: Thu, 31 Mar 2022 11:54:04 -0300 Subject: [PATCH 203/280] Add docstring to ``started`` and ``has_context`` --- lz4/frame/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 791fb0d9..b2d9fc8c 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -295,9 +295,21 @@ def reset(self): self._started = False def has_context(self): + """Return whether the compression context exists. + + Returns: + bool: ``True`` if the compression context exists, ``False`` + otherwise. + """ return self._context is not None def started(self): + """Return whether the compression frame has been started. + + Returns: + bool: ``True`` if the compression frame has been started, ``False`` + otherwise. + """ return self._started From e7c510918d13e253e8d5595217a2588f4fb1d778 Mon Sep 17 00:00:00 2001 From: Pedro Batista Date: Thu, 31 Mar 2022 11:59:39 -0300 Subject: [PATCH 204/280] Simplify flushing on file close --- lz4/frame/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index b2d9fc8c..00f3e64a 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -559,8 +559,7 @@ def close(self): self._buffer.close() self._buffer = None elif self._mode == _MODE_WRITE: - if self._compressor.has_context(): - self._fp.write(self._compressor.flush()) + self.flush() self._compressor = None finally: try: From c54e91d5970b7299a8e1b4f7f6e879a1a1347f0c Mon Sep 17 00:00:00 2001 From: squaresmile Date: Mon, 4 Apr 2022 14:06:38 +0700 Subject: [PATCH 205/280] Added aarch64 wheels --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 59216262..ae6a98fa 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -36,7 +36,7 @@ jobs: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*] - cibw_archs_linux: [auto, ] # aarch64] + cibw_archs_linux: [auto, aarch64] cibw_skip: ["cp*-musllinux*", ] steps: - name: Check out repository From 499ee4af50885e880eff4fb7d41b192c5e984e6a Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Mon, 2 May 2022 22:28:03 +0100 Subject: [PATCH 206/280] Fix occasional failure of test_block_decompress_mem_usage() (#253) Increase the memory usage increase limit and also invoke the garbage collector ahead of testing memory usage increase. --- tests/block/test_block_3.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py index 0c3fb082..3fcb175b 100644 --- a/tests/block/test_block_3.py +++ b/tests/block/test_block_3.py @@ -1,3 +1,4 @@ +import gc import lz4.block import pytest @@ -29,10 +30,11 @@ def test_block_decompress_mem_usage(data): decompressed = lz4.block.decompress(compressed) # noqa: F841 if i % 100 == 0: + gc.collect() snapshot = tracemalloc.take_snapshot() if prev_snapshot: stats = snapshot.compare_to(prev_snapshot, 'lineno') - assert stats[0].size_diff < (1024 * 4) + assert stats[0].size_diff < (1024 * 8) prev_snapshot = snapshot From a19f75d0f9666a5eed3cc561140081b72169de74 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 22 May 2022 13:21:13 +0100 Subject: [PATCH 207/280] Revert "Added aarch64 wheels" This reverts commit c54e91d5970b7299a8e1b4f7f6e879a1a1347f0c. --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index ae6a98fa..59216262 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -36,7 +36,7 @@ jobs: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*] - cibw_archs_linux: [auto, aarch64] + cibw_archs_linux: [auto, ] # aarch64] cibw_skip: ["cp*-musllinux*", ] steps: - name: Check out repository From 43fe65d0665adb5183103d1b6b921c8853df0afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Duarte?= Date: Mon, 4 Jul 2022 20:43:02 +0100 Subject: [PATCH 208/280] Fix memory leak in lz4.frame.decompress --- lz4/frame/_frame.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index ebf84a0f..34606653 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1068,6 +1068,7 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size, PyErr_Format (PyExc_RuntimeError, "LZ4F_decompress failed with code: %s", LZ4F_getErrorName (result)); + PyMem_Free (destination); return NULL; } From 535474a464ca2065f5a40a05b1fe9fcec14c09f5 Mon Sep 17 00:00:00 2001 From: David Black Date: Tue, 29 Nov 2022 11:20:11 +1100 Subject: [PATCH 209/280] Build wheels for python 3.11 --- .github/workflows/build_dist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 59216262..6adb1928 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -35,7 +35,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] - cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*] + cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*] cibw_archs_linux: [auto, ] # aarch64] cibw_skip: ["cp*-musllinux*", ] steps: @@ -53,7 +53,7 @@ jobs: with: platforms: all - name: Build wheels - uses: joerick/cibuildwheel@v2.3.1 + uses: pypa/cibuildwheel@v2.11.2 env: CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BUILD: ${{ matrix.cibw_build }} From 9a9d1ba9cdefe4a60e04232173e62cdf355cfc52 Mon Sep 17 00:00:00 2001 From: David Black Date: Tue, 29 Nov 2022 11:22:22 +1100 Subject: [PATCH 210/280] Add python 3.11 as a supported version. Signed-off-by: David Black --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6dd6124f..2f2704fb 100644 --- a/setup.py +++ b/setup.py @@ -198,5 +198,6 @@ def pkgconfig_installed_check(lib, required_version, default): 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ], ) From fa54bdda1ed3fa6743b980e336a986c24a7b4349 Mon Sep 17 00:00:00 2001 From: David Black Date: Wed, 14 Dec 2022 09:20:35 +1100 Subject: [PATCH 211/280] Fix some flake8 issues. Signed-off-by: David Black --- tests/block/test_block_0.py | 4 ++-- tests/stream/test_stream_0.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index d37ecf3b..8fc0f488 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -55,12 +55,12 @@ def setup_kwargs(mode, store_size, c_return_bytearray=None, d_return_bytearray=N c_kwargs.update(store_size) - if(c_return_bytearray): + if c_return_bytearray: c_kwargs.update(c_return_bytearray) d_kwargs = {} - if(d_return_bytearray): + if d_return_bytearray: d_kwargs.update(d_return_bytearray) return (c_kwargs, d_kwargs) diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index fd2354ef..03b19f3f 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -80,12 +80,12 @@ def setup_kwargs(strategy, mode, buffer_size, store_comp_size, c_kwargs['buffer_size'] = buffer_size c_kwargs.update(store_comp_size) - if(c_return_bytearray): + if c_return_bytearray: c_kwargs.update(c_return_bytearray) d_kwargs = {} - if(d_return_bytearray): + if d_return_bytearray: d_kwargs.update(d_return_bytearray) d_kwargs['strategy'] = strategy From 1137d3d53950369cbe0ae75e0d51efd5f7f447c8 Mon Sep 17 00:00:00 2001 From: David Black Date: Wed, 14 Dec 2022 10:18:32 +1100 Subject: [PATCH 212/280] Change whitelist_externals to be allowlist_externals. Signed-off-by: David Black --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b7539459..86f8ebf2 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ setenv = PYLZ4_EXPERIMENTAL = 1 passenv = * usedevelop = True -whitelist_externals = make +allowlist_externals = make extras = docs commands = make -C docs doctest html From 33cab7f75d507037e2bcf9f1ae27c632684c7c4e Mon Sep 17 00:00:00 2001 From: David Black Date: Wed, 14 Dec 2022 11:36:45 +1100 Subject: [PATCH 213/280] Enable experimental by default. Signed-off-by: David Black --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2f2704fb..5f06f9d9 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,8 @@ def pkgconfig_installed_check(lib, required_version, default): liblz4_found = pkgconfig_installed_check('liblz4', LZ4_REQUIRED_VERSION, default=False) # Establish if we want to build experimental functionality or not. -experimental = os.environ.get("PYLZ4_EXPERIMENTAL", False) -if experimental is not False: +experimental = os.environ.get("PYLZ4_EXPERIMENTAL", True) +if experimental is not True: if experimental.upper() in ("1", "TRUE"): experimental = True else: From 62bbd9813e5bc711f57387cd8a63fe9e4a233352 Mon Sep 17 00:00:00 2001 From: David Black Date: Wed, 21 Dec 2022 14:24:08 +1100 Subject: [PATCH 214/280] Revert "Enable experimental by default." This reverts commit 33cab7f75d507037e2bcf9f1ae27c632684c7c4e. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5f06f9d9..2f2704fb 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,8 @@ def pkgconfig_installed_check(lib, required_version, default): liblz4_found = pkgconfig_installed_check('liblz4', LZ4_REQUIRED_VERSION, default=False) # Establish if we want to build experimental functionality or not. -experimental = os.environ.get("PYLZ4_EXPERIMENTAL", True) -if experimental is not True: +experimental = os.environ.get("PYLZ4_EXPERIMENTAL", False) +if experimental is not False: if experimental.upper() in ("1", "TRUE"): experimental = True else: From 6e4c563d4f186151905ea815b636df58b5d25d3d Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 28 Dec 2022 22:10:49 +0000 Subject: [PATCH 215/280] Replace deprecated whitelist with allowlist in tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b7539459..86f8ebf2 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ setenv = PYLZ4_EXPERIMENTAL = 1 passenv = * usedevelop = True -whitelist_externals = make +allowlist_externals = make extras = docs commands = make -C docs doctest html From f21d3e4ede3b6510a232f1bd12a5fcc6ee4ac626 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 28 Dec 2022 22:58:58 +0000 Subject: [PATCH 216/280] Set PYLZ4_EXPERIMENTAL for the tox package build Starting with Tox 4.0 environment variables for test environments are not passed to the tox package building environment. To work around this issue we leverage the .pkg environment, which is used for package builds. https://github.com/tox-dev/tox/issues/2784 --- tox.ini | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tox.ini b/tox.ini index 86f8ebf2..5a30280b 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,15 @@ extras = docs commands = make -C docs doctest html +[testenv:.pkg] +# This section is needed to pass PYLZ4_EXPERIMENTAL into the +# environment used to build the lz4 package ahead of creating the tox +# virtual environment. Ideally we would like to be able to set this +# only for docs building, but at the moment (tox 4.0.18) tox does not +# allow that. See: https://github.com/tox-dev/tox/issues/2784 +setenv = + PYLZ4_EXPERIMENTAL = 1 + [testenv:experimental] extras = tests passenv = * From de1f9ab80387f5f815d250e9d616a7e985409427 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 29 Dec 2022 00:31:12 +0000 Subject: [PATCH 217/280] Update bundled libraries to lz4 1.9.4 --- lz4libs/lz4.c | 617 ++++++++++++++++++++++++----------- lz4libs/lz4.h | 160 ++++++--- lz4libs/lz4frame.c | 663 ++++++++++++++++++++++++-------------- lz4libs/lz4frame.h | 175 +++++++--- lz4libs/lz4frame_static.h | 2 +- lz4libs/lz4hc.c | 238 +++++++------- lz4libs/lz4hc.h | 28 +- 7 files changed, 1221 insertions(+), 662 deletions(-) diff --git a/lz4libs/lz4.c b/lz4libs/lz4.c index 9f5e9bfa..654bfdf3 100644 --- a/lz4libs/lz4.c +++ b/lz4libs/lz4.c @@ -1,6 +1,6 @@ /* LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-present, Yann Collet. + Copyright (C) 2011-2020, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -124,6 +124,7 @@ #if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ # include /* only present in VS2005+ */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 6237) /* disable: C6237: conditional expression is always 0 */ #endif /* _MSC_VER */ #ifndef LZ4_FORCE_INLINE @@ -187,7 +188,27 @@ /*-************************************ * Memory routines **************************************/ -#ifdef LZ4_USER_MEMORY_FUNCTIONS + +/*! LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION : + * Disable relatively high-level LZ4/HC functions that use dynamic memory + * allocation functions (malloc(), calloc(), free()). + * + * Note that this is a compile-time switch. And since it disables + * public/stable LZ4 v1 API functions, we don't recommend using this + * symbol to generate a library for distribution. + * + * The following public functions are removed when this symbol is defined. + * - lz4 : LZ4_createStream, LZ4_freeStream, + * LZ4_createStreamDecode, LZ4_freeStreamDecode, LZ4_create (deprecated) + * - lz4hc : LZ4_createStreamHC, LZ4_freeStreamHC, + * LZ4_createHC (deprecated), LZ4_freeHC (deprecated) + * - lz4frame, lz4file : All LZ4F_* functions + */ +#if defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +# define ALLOC(s) lz4_error_memory_allocation_is_disabled +# define ALLOC_AND_ZERO(s) lz4_error_memory_allocation_is_disabled +# define FREEMEM(p) lz4_error_memory_allocation_is_disabled +#elif defined(LZ4_USER_MEMORY_FUNCTIONS) /* memory management functions can be customized by user project. * Below functions must exist somewhere in the Project * and be available at link time */ @@ -204,8 +225,13 @@ void LZ4_free(void* p); # define FREEMEM(p) free(p) #endif -#include /* memset, memcpy */ -#define MEM_INIT(p,v,s) memset((p),(v),(s)) +#if ! LZ4_FREESTANDING +# include /* memset, memcpy */ +#endif +#if !defined(LZ4_memset) +# define LZ4_memset(p,v,s) memset((p),(v),(s)) +#endif +#define MEM_INIT(p,v,s) LZ4_memset((p),(v),(s)) /*-************************************ @@ -316,10 +342,20 @@ typedef enum { * memcpy() as if it were standard compliant, so it can inline it in freestanding * environments. This is needed when decompressing the Linux Kernel, for example. */ -#if defined(__GNUC__) && (__GNUC__ >= 4) -#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) -#else -#define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +#if !defined(LZ4_memcpy) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +# else +# define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +# endif +#endif + +#if !defined(LZ4_memmove) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memmove __builtin_memmove +# else +# define LZ4_memmove memmove +# endif #endif static unsigned LZ4_isLittleEndian(void) @@ -343,14 +379,14 @@ static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ /* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; +typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) LZ4_unalign; -static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } +static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalign*)ptr)->uArch; } -static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign*)memPtr)->u32 = value; } #else /* safe and portable access using memcpy() */ @@ -421,10 +457,12 @@ static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; #ifndef LZ4_FAST_DEC_LOOP # if defined __i386__ || defined _M_IX86 || defined __x86_64__ || defined _M_X64 # define LZ4_FAST_DEC_LOOP 1 +# elif defined(__aarch64__) && defined(__APPLE__) +# define LZ4_FAST_DEC_LOOP 1 # elif defined(__aarch64__) && !defined(__clang__) - /* On aarch64, we disable this optimization for clang because on certain - * mobile chipsets, performance is reduced with clang. For information - * refer to https://github.com/lz4/lz4/pull/707 */ + /* On non-Apple aarch64, we disable this optimization for clang because + * on certain mobile chipsets, performance is reduced with clang. For + * more information refer to https://github.com/lz4/lz4/pull/707 */ # define LZ4_FAST_DEC_LOOP 1 # else # define LZ4_FAST_DEC_LOOP 0 @@ -486,7 +524,14 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si case 2: LZ4_memcpy(v, srcPtr, 2); LZ4_memcpy(&v[2], srcPtr, 2); +#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */ +# pragma warning(push) +# pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */ +#endif LZ4_memcpy(&v[4], v, 4); +#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */ +# pragma warning(pop) +#endif break; case 4: LZ4_memcpy(v, srcPtr, 4); @@ -515,9 +560,20 @@ static unsigned LZ4_NbCommonBytes (reg_t val) assert(val != 0); if (LZ4_isLittleEndian()) { if (sizeof(val) == 8) { -# if defined(_MSC_VER) && (_MSC_VER >= 1800) && defined(_M_AMD64) && !defined(LZ4_FORCE_SW_BITCOUNT) +# if defined(_MSC_VER) && (_MSC_VER >= 1800) && (defined(_M_AMD64) && !defined(_M_ARM64EC)) && !defined(LZ4_FORCE_SW_BITCOUNT) +/*-************************************************************************************************* +* ARM64EC is a Microsoft-designed ARM64 ABI compatible with AMD64 applications on ARM64 Windows 11. +* The ARM64EC ABI does not support AVX/AVX2/AVX512 instructions, nor their relevant intrinsics +* including _tzcnt_u64. Therefore, we need to neuter the _tzcnt_u64 code path for ARM64EC. +****************************************************************************************************/ +# if defined(__clang__) && (__clang_major__ < 10) + /* Avoid undefined clang-cl intrinsics issue. + * See https://github.com/lz4/lz4/pull/1017 for details. */ + return (unsigned)__builtin_ia32_tzcnt_u64(val) >> 3; +# else /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ return (unsigned)_tzcnt_u64(val) >> 3; +# endif # elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; _BitScanForward64(&r, (U64)val); @@ -652,10 +708,10 @@ typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere * else in memory, starting at ctx->dictionary with length * ctx->dictSize. - * - usingDictCtx : Like usingExtDict, but everything concerning the preceding - * content is in a separate context, pointed to by - * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table - * entries in the current context that refer to positions + * - usingDictCtx : Everything concerning the preceding content is + * in a separate context, pointed to by ctx->dictCtx. + * ctx->dictionary, ctx->dictSize, and table entries + * in the current context that refer to positions * preceding the beginning of the current compression are * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx * ->dictSize describe the location and size of the preceding @@ -672,12 +728,12 @@ typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } -int LZ4_sizeofState(void) { return LZ4_STREAMSIZE; } +int LZ4_sizeofState(void) { return sizeof(LZ4_stream_t); } -/*-************************************ -* Internal Definitions used in Tests -**************************************/ +/*-**************************************** +* Internal Definitions, used only in Tests +*******************************************/ #if defined (__cplusplus) extern "C" { #endif @@ -687,7 +743,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const void* dictStart, size_t dictSize); - +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize); #if defined (__cplusplus) } #endif @@ -827,9 +885,10 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx, } } - /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster - * than compressing without a gap. However, compressing with - * currentOffset == 0 is faster still, so we preserve that case. + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, + * is faster than compressing without a gap. + * However, compressing with currentOffset == 0 is faster still, + * so we preserve that case. */ if (cctx->currentOffset != 0 && tableType == byU32) { DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); @@ -853,7 +912,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( const char* const source, char* const dest, const int inputSize, - int *inputConsumed, /* only written when outputDirective == fillOutput */ + int* inputConsumed, /* only written when outputDirective == fillOutput */ const int maxOutputSize, const limitedOutput_directive outputDirective, const tableType_t tableType, @@ -885,7 +944,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( /* the dictCtx currentOffset is indexed on the start of the dictionary, * while a dictionary in the current context precedes the currentOffset */ - const BYTE* dictBase = !dictionary ? NULL : (dictDirective == usingDictCtx) ? + const BYTE* dictBase = (dictionary == NULL) ? NULL : + (dictDirective == usingDictCtx) ? dictionary + dictSize - dictCtx->currentOffset : dictionary + dictSize - startIndex; @@ -981,10 +1041,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( match = base + matchIndex; lowLimit = (const BYTE*)source; } - } else if (dictDirective==usingExtDict) { + } else if (dictDirective == usingExtDict) { if (matchIndex < startIndex) { DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); assert(startIndex - matchIndex >= MINMATCH); + assert(dictBase); match = dictBase + matchIndex; lowLimit = dictionary; } else { @@ -1048,7 +1109,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( _next_match: /* at this stage, the following variables must be correctly set : * - ip : at start of LZ operation - * - match : at start of previous pattern occurence; can be within current prefix, or within extDict + * - match : at start of previous pattern occurrence; can be within current prefix, or within extDict * - offset : if maybe_ext_memSegment==1 (constant) * - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written @@ -1173,6 +1234,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( } } else if (dictDirective==usingExtDict) { if (matchIndex < startIndex) { + assert(dictBase); match = dictBase + matchIndex; lowLimit = dictionary; /* required for match length counter */ } else { @@ -1355,7 +1417,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp { int result; #if (LZ4_HEAPMODE) - LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ if (ctxPtr == NULL) return 0; #else LZ4_stream_t ctx; @@ -1420,15 +1482,17 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe * Streaming functions ********************************/ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) LZ4_stream_t* LZ4_createStream(void) { LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= sizeof(LZ4_stream_t_internal)); DEBUGLOG(4, "LZ4_createStream %p", lz4s); if (lz4s == NULL) return NULL; LZ4_initStream(lz4s, sizeof(*lz4s)); return lz4s; } +#endif static size_t LZ4_stream_t_alignment(void) { @@ -1462,6 +1526,7 @@ void LZ4_resetStream_fast(LZ4_stream_t* ctx) { LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); } +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { if (!LZ4_stream) return 0; /* support free on NULL */ @@ -1469,6 +1534,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) FREEMEM(LZ4_stream); return (0); } +#endif #define HASH_UNIT sizeof(reg_t) @@ -1514,8 +1580,9 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) return (int)dict->dictSize; } -void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) { - const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL : +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) +{ + const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL : &(dictionaryStream->internal_donotuse); DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", @@ -1568,36 +1635,40 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, int acceleration) { const tableType_t tableType = byU32; - LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize; + LZ4_stream_t_internal* const streamPtr = &LZ4_stream->internal_donotuse; + const char* dictEnd = streamPtr->dictSize ? (const char*)streamPtr->dictionary + streamPtr->dictSize : NULL; - DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize); - LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ + LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; /* invalidate tiny dictionaries */ - if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */ - && (dictEnd != (const BYTE*)source) ) { + if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */ + && (dictEnd != source) /* prefix mode */ + && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ + && (streamPtr->dictCtx == NULL) /* usingDictCtx */ + ) { DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + /* remove dictionary existence from history, to employ faster prefix mode */ streamPtr->dictSize = 0; streamPtr->dictionary = (const BYTE*)source; - dictEnd = (const BYTE*)source; + dictEnd = source; } /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { + { const char* const sourceEnd = source + inputSize; + if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) { streamPtr->dictSize = (U32)(dictEnd - sourceEnd); if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; + streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize; } } /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) { + if (dictEnd == source) { if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); else @@ -1623,7 +1694,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, } else { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); } - } else { + } else { /* small data <= 4 KB */ if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); } else { @@ -1661,21 +1732,25 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* /*! LZ4_saveDict() : * If previously compressed data block is not guaranteed to remain available at its memory location, * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable, + * one can therefore call LZ4_compress_fast_continue() right after. + * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. */ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + + DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer); if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } if (safeBuffer == NULL) assert(dictSize == 0); - if (dictSize > 0) - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + if (dictSize > 0) { + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + assert(dict->dictionary); + LZ4_memmove(safeBuffer, previousDictEnd - dictSize, (size_t)dictSize); + } dict->dictionary = (const BYTE*)safeBuffer; dict->dictSize = (U32)dictSize; @@ -1689,39 +1764,163 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) * Decompression functions ********************************/ -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; #undef MIN #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +/* variant for decompress_unsafe() + * does not know end of input + * presumes input is well formed + * note : will consume at least one byte */ +size_t read_long_length_no_check(const BYTE** pp) +{ + size_t b, l = 0; + do { b = **pp; (*pp)++; l += b; } while (b==255); + DEBUGLOG(6, "read_long_length_no_check: +length=%zu using %zu input bytes", l, l/255 + 1) + return l; +} + +/* core decoder variant for LZ4_decompress_fast*() + * for legacy support only : these entry points are deprecated. + * - Presumes input is correctly formed (no defense vs malformed inputs) + * - Does not know input size (presume input buffer is "large enough") + * - Decompress a full block (only) + * @return : nb of bytes read from input. + * Note : this variant is not optimized for speed, just for maintenance. + * the goal is to remove support of decompress_fast*() variants by v2.0 +**/ +LZ4_FORCE_INLINE int +LZ4_decompress_unsafe_generic( + const BYTE* const istart, + BYTE* const ostart, + int decompressedSize, + + size_t prefixSize, + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note: =0 if dictStart==NULL */ + ) +{ + const BYTE* ip = istart; + BYTE* op = (BYTE*)ostart; + BYTE* const oend = ostart + decompressedSize; + const BYTE* const prefixStart = ostart - prefixSize; + + DEBUGLOG(5, "LZ4_decompress_unsafe_generic"); + if (dictStart == NULL) assert(dictSize == 0); + + while (1) { + /* start new sequence */ + unsigned token = *ip++; + + /* literals */ + { size_t ll = token >> ML_BITS; + if (ll==15) { + /* long literal length */ + ll += read_long_length_no_check(&ip); + } + if ((size_t)(oend-op) < ll) return -1; /* output buffer overflow */ + LZ4_memmove(op, ip, ll); /* support in-place decompression */ + op += ll; + ip += ll; + if ((size_t)(oend-op) < MFLIMIT) { + if (op==oend) break; /* end of block */ + DEBUGLOG(5, "invalid: literals end at distance %zi from end of block", oend-op); + /* incorrect end of block : + * last match must start at least MFLIMIT==12 bytes before end of output block */ + return -1; + } } + + /* match */ + { size_t ml = token & 15; + size_t const offset = LZ4_readLE16(ip); + ip+=2; + + if (ml==15) { + /* long literal length */ + ml += read_long_length_no_check(&ip); + } + ml += MINMATCH; + + if ((size_t)(oend-op) < ml) return -1; /* output buffer overflow */ + + { const BYTE* match = op - offset; + + /* out of range */ + if (offset > (size_t)(op - prefixStart) + dictSize) { + DEBUGLOG(6, "offset out of range"); + return -1; + } + + /* check special case : extDict */ + if (offset > (size_t)(op - prefixStart)) { + /* extDict scenario */ + const BYTE* const dictEnd = dictStart + dictSize; + const BYTE* extMatch = dictEnd - (offset - (size_t)(op-prefixStart)); + size_t const extml = (size_t)(dictEnd - extMatch); + if (extml > ml) { + /* match entirely within extDict */ + LZ4_memmove(op, extMatch, ml); + op += ml; + ml = 0; + } else { + /* match split between extDict & prefix */ + LZ4_memmove(op, extMatch, extml); + op += extml; + ml -= extml; + } + match = prefixStart; + } + + /* match copy - slow variant, supporting overlap copy */ + { size_t u; + for (u=0; u= lencheck. - * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so. - * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so. - * error (output) - error code. Should be set to 0 before call. - */ -typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error; -LZ4_FORCE_INLINE unsigned -read_variable_length(const BYTE**ip, const BYTE* lencheck, - int loop_check, int initial_check, - variable_length_error* error) -{ - U32 length = 0; - U32 s; - if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ - *error = initial_error; - return length; + * @ip : input pointer + * @ilimit : position after which if length is not decoded, the input is necessarily corrupted. + * @initial_check - check ip >= ipmax before start of loop. Returns initial_error if so. + * @error (output) - error code. Must be set to 0 before call. +**/ +typedef size_t Rvl_t; +static const Rvl_t rvl_error = (Rvl_t)(-1); +LZ4_FORCE_INLINE Rvl_t +read_variable_length(const BYTE** ip, const BYTE* ilimit, + int initial_check) +{ + Rvl_t s, length = 0; + assert(ip != NULL); + assert(*ip != NULL); + assert(ilimit != NULL); + if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */ + return rvl_error; } do { s = **ip; (*ip)++; length += s; - if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ - *error = loop_error; - return length; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length)<8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; } } while (s==255); @@ -1741,7 +1940,6 @@ LZ4_decompress_generic( int srcSize, int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ - endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */ earlyEnd_directive partialDecoding, /* full, partial */ dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ @@ -1749,7 +1947,7 @@ LZ4_decompress_generic( const size_t dictSize /* note : = 0 if noDict */ ) { - if (src == NULL) { return -1; } + if ((src == NULL) || (outputSize < 0)) { return -1; } { const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; @@ -1760,13 +1958,12 @@ LZ4_decompress_generic( const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + const int checkOffset = (dictSize < (int)(64 KB)); /* Set up the "end" pointers for the shortcut. */ - const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; - const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; + const BYTE* const shortiend = iend - 14 /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - 14 /*maxLL*/ - 18 /*maxML*/; const BYTE* match; size_t offset; @@ -1778,83 +1975,70 @@ LZ4_decompress_generic( /* Special cases */ assert(lowPrefix <= op); - if ((endOnInput) && (unlikely(outputSize==0))) { + if (unlikely(outputSize==0)) { /* Empty output buffer */ if (partialDecoding) return 0; return ((srcSize==1) && (*ip==0)) ? 0 : -1; } - if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); } - if ((endOnInput) && unlikely(srcSize==0)) { return -1; } + if (unlikely(srcSize==0)) { return -1; } - /* Currently the fast loop shows a regression on qualcomm arm chips. */ + /* LZ4_FAST_DEC_LOOP: + * designed for modern OoO performance cpus, + * where copying reliably 32-bytes is preferable to an unpredictable branch. + * note : fast loop may show a regression for some client arm chips. */ #if LZ4_FAST_DEC_LOOP if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { DEBUGLOG(6, "skip fast decode loop"); goto safe_decode; } - /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */ + /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */ while (1) { /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ assert(oend - op >= FASTLOOP_SAFE_DISTANCE); - if (endOnInput) { assert(ip < iend); } + assert(ip < iend); token = *ip++; length = token >> ML_BITS; /* literal length */ - assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ - /* decode literal length */ if (length == RUN_MASK) { - variable_length_error error = ok; - length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error); - if (error == initial_error) { goto _output_error; } - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ /* copy literals */ cpy = op+length; LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); - if (endOnInput) { /* LZ4_decompress_safe() */ - if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } - LZ4_wildCopy32(op, ip, cpy); - } else { /* LZ4_decompress_fast() */ - if (cpy>oend-8) { goto safe_literal_copy; } - LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : - * it doesn't know input length, and only relies on end-of-block properties */ - } + if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } + LZ4_wildCopy32(op, ip, cpy); ip += length; op = cpy; } else { cpy = op+length; - if (endOnInput) { /* LZ4_decompress_safe() */ - DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); - /* We don't need to check oend, since we check it once for each loop below */ - if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } - /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ - LZ4_memcpy(op, ip, 16); - } else { /* LZ4_decompress_fast() */ - /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : - * it doesn't know input length, and relies on end-of-block properties */ - LZ4_memcpy(op, ip, 8); - if (length > 8) { LZ4_memcpy(op+8, ip+8, 8); } - } + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* We don't need to check oend, since we check it once for each loop below */ + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } + /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */ + LZ4_memcpy(op, ip, 16); ip += length; op = cpy; } /* get offset */ offset = LZ4_readLE16(ip); ip+=2; match = op - offset; - assert(match <= op); + assert(match <= op); /* overflow check */ /* get matchlength */ length = token & ML_MASK; if (length == ML_MASK) { - variable_length_error error = ok; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ - length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error); - if (error != ok) { goto _output_error; } - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { goto _output_error; } + length += addl; length += MINMATCH; + if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { goto safe_match_copy; } @@ -1864,7 +2048,7 @@ LZ4_decompress_generic( goto safe_match_copy; } - /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ + /* Fastpath check: skip LZ4_wildCopy32 when true */ if ((dict == withPrefix64k) || (match >= lowPrefix)) { if (offset >= 8) { assert(match >= lowPrefix); @@ -1881,6 +2065,7 @@ LZ4_decompress_generic( if (checkOffset && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); if (unlikely(op+length > oend-LASTLITERALS)) { if (partialDecoding) { DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); @@ -1891,7 +2076,7 @@ LZ4_decompress_generic( if (length <= (size_t)(lowPrefix-match)) { /* match fits entirely within external dictionary : just copy */ - memmove(op, dictEnd - (lowPrefix-match), length); + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); op += length; } else { /* match stretches into both external dictionary and current block */ @@ -1927,11 +2112,10 @@ LZ4_decompress_generic( /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ while (1) { + assert(ip < iend); token = *ip++; length = token >> ML_BITS; /* literal length */ - assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ - /* A two-stage shortcut for the most common case: * 1) If the literal length is 0..14, and there is enough space, * enter the shortcut and copy 16 bytes on behalf of the literals @@ -1941,11 +2125,11 @@ LZ4_decompress_generic( * those 18 bytes earlier, upon entering the shortcut (in other words, * there is a combined check for both stages). */ - if ( (endOnInput ? length != RUN_MASK : length <= 8) + if ( (length != RUN_MASK) /* strictly "less than" on input, to re-enter the loop with at least one byte */ - && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { + && likely((ip < shortiend) & (op <= shortoend)) ) { /* Copy the literals */ - LZ4_memcpy(op, ip, endOnInput ? 16 : 8); + LZ4_memcpy(op, ip, 16); op += length; ip += length; /* The second stage: prepare for match copying, decode full info. @@ -1975,11 +2159,11 @@ LZ4_decompress_generic( /* decode literal length */ if (length == RUN_MASK) { - variable_length_error error = ok; - length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error); - if (error == initial_error) { goto _output_error; } - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ } /* copy literals */ @@ -1988,9 +2172,7 @@ LZ4_decompress_generic( safe_literal_copy: #endif LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); - if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { + if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) { /* We've either hit the input parsing restriction or the output parsing restriction. * In the normal scenario, decoding a full block, it must be the last sequence, * otherwise it's an error (invalid input or dimensions). @@ -2000,7 +2182,6 @@ LZ4_decompress_generic( /* Since we are partial decoding we may be in this block because of the output parsing * restriction, which is not valid since the output buffer is allowed to be undersized. */ - assert(endOnInput); DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); @@ -2021,21 +2202,17 @@ LZ4_decompress_generic( length = (size_t)(oend-op); } } else { - /* We must be on the last sequence because of the parsing limitations so check - * that we exactly regenerate the original size (must be exact when !endOnInput). - */ - if ((!endOnInput) && (cpy != oend)) { goto _output_error; } /* We must be on the last sequence (or invalid) because of the parsing limitations * so check that we exactly consume the input and don't overrun the output buffer. */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { + if ((ip+length != iend) || (cpy > oend)) { DEBUGLOG(6, "should have been last run of literals") DEBUGLOG(6, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend); DEBUGLOG(6, "or cpy(%p) > oend(%p)", cpy, oend); goto _output_error; } } - memmove(op, ip, length); /* supports overlapping memory regions; only matters for in-place decompression scenarios */ + LZ4_memmove(op, ip, length); /* supports overlapping memory regions, for in-place decompression scenarios */ ip += length; op += length; /* Necessarily EOF when !partialDecoding. @@ -2047,7 +2224,7 @@ LZ4_decompress_generic( break; } } else { - LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ + LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */ ip += length; op = cpy; } @@ -2060,10 +2237,10 @@ LZ4_decompress_generic( _copy_match: if (length == ML_MASK) { - variable_length_error error = ok; - length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error); - if (error != ok) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ } length += MINMATCH; @@ -2073,6 +2250,7 @@ LZ4_decompress_generic( if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); if (unlikely(op+length > oend-LASTLITERALS)) { if (partialDecoding) length = MIN(length, (size_t)(oend-op)); else goto _output_error; /* doesn't respect parsing restriction */ @@ -2080,7 +2258,7 @@ LZ4_decompress_generic( if (length <= (size_t)(lowPrefix-match)) { /* match fits entirely within external dictionary : just copy */ - memmove(op, dictEnd - (lowPrefix-match), length); + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); op += length; } else { /* match stretches into both external dictionary and current block */ @@ -2151,12 +2329,8 @@ LZ4_decompress_generic( } /* end of decoding */ - if (endOnInput) { - DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); - return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - } else { - return (int) (((const char*)ip)-src); /* Nb of input bytes read */ - } + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ /* Overflow error detected */ _output_error: @@ -2171,7 +2345,7 @@ LZ4_FORCE_O2 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, - endOnInputSize, decode_full_block, noDict, + decode_full_block, noDict, (BYTE*)dest, NULL, 0); } @@ -2180,16 +2354,17 @@ int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, { dstCapacity = MIN(targetOutputSize, dstCapacity); return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, - endOnInputSize, partial_decode, + partial_decode, noDict, (BYTE*)dst, NULL, 0); } LZ4_FORCE_O2 int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, decode_full_block, withPrefix64k, - (BYTE*)dest - 64 KB, NULL, 0); + DEBUGLOG(5, "LZ4_decompress_fast"); + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, NULL, 0); } /*===== Instantiate a few more decoding cases, used more than once. =====*/ @@ -2198,16 +2373,25 @@ LZ4_FORCE_O2 /* Exported, an obsolete API function. */ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, decode_full_block, withPrefix64k, + decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withPrefix64k(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 0); } /* Another obsolete API function, paired with the previous one. */ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) { - /* LZ4_decompress_fast doesn't validate match offsets, - * and thus serves well with any prefixed dictionary. */ - return LZ4_decompress_fast(source, dest, originalSize); + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 64 KB, NULL, 0); } LZ4_FORCE_O2 @@ -2215,7 +2399,17 @@ static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, i size_t prefixSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, decode_full_block, noDict, + decode_full_block, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withSmallPrefix(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, + size_t prefixSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, noDict, (BYTE*)dest-prefixSize, NULL, 0); } @@ -2225,7 +2419,18 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, decode_full_block, usingExtDict, + decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } @@ -2233,9 +2438,9 @@ LZ4_FORCE_O2 static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, const void* dictStart, size_t dictSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, decode_full_block, usingExtDict, - (BYTE*)dest, (const BYTE*)dictStart, dictSize); + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, (const BYTE*)dictStart, dictSize); } /* The "double dictionary" mode, for use with e.g. ring buffers: the first part @@ -2247,26 +2452,17 @@ int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compresse size_t prefixSize, const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, decode_full_block, usingExtDict, - (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); -} - -LZ4_FORCE_INLINE -int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize, - size_t prefixSize, const void* dictStart, size_t dictSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, decode_full_block, usingExtDict, + decode_full_block, usingExtDict, (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); } /*===== streaming decompression functions =====*/ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); - LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */ - return lz4s; + LZ4_STATIC_ASSERT(sizeof(LZ4_streamDecode_t) >= sizeof(LZ4_streamDecode_t_internal)); + return (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); } int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) @@ -2275,6 +2471,7 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) FREEMEM(LZ4_stream); return 0; } +#endif /*! LZ4_setStreamDecode() : * Use this function to instruct where to find the dictionary. @@ -2285,8 +2482,13 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz4sd->prefixSize = (size_t)dictSize; + if (dictSize) { + assert(dictionary != NULL); + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + } else { + lz4sd->prefixEnd = (const BYTE*) dictionary; + } lz4sd->externalDict = NULL; lz4sd->extDictSize = 0; return 1; @@ -2358,29 +2560,35 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch return result; } -LZ4_FORCE_O2 -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) +LZ4_FORCE_O2 int +LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* source, char* dest, int originalSize) { - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + LZ4_streamDecode_t_internal* const lz4sd = + (assert(LZ4_streamDecode!=NULL), &LZ4_streamDecode->internal_donotuse); int result; + + DEBUGLOG(5, "LZ4_decompress_fast_continue (toDecodeSize=%i)", originalSize); assert(originalSize >= 0); if (lz4sd->prefixSize == 0) { + DEBUGLOG(5, "first invocation : no prefix nor extDict"); assert(lz4sd->extDictSize == 0); result = LZ4_decompress_fast(source, dest, originalSize); if (result <= 0) return result; lz4sd->prefixSize = (size_t)originalSize; lz4sd->prefixEnd = (BYTE*)dest + originalSize; } else if (lz4sd->prefixEnd == (BYTE*)dest) { - if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0) - result = LZ4_decompress_fast(source, dest, originalSize); - else - result = LZ4_decompress_fast_doubleDict(source, dest, originalSize, - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + DEBUGLOG(5, "continue using existing prefix"); + result = LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + lz4sd->prefixSize, + lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize += (size_t)originalSize; lz4sd->prefixEnd += originalSize; } else { + DEBUGLOG(5, "prefix becomes extDict"); lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; result = LZ4_decompress_fast_extDict(source, dest, originalSize, @@ -2416,10 +2624,27 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); } +int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_partial_withPrefix64k(source, dest, compressedSize, targetOutputSize, dstCapacity); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_withSmallPrefix(source, dest, compressedSize, targetOutputSize, dstCapacity, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_forceExtDict(source, dest, compressedSize, targetOutputSize, dstCapacity, dictStart, (size_t)dictSize); +} + int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { if (dictSize==0 || dictStart+dictSize == dest) - return LZ4_decompress_fast(source, dest, originalSize); + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + (size_t)dictSize, NULL, 0); assert(dictSize >= 0); return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); } @@ -2471,7 +2696,7 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, /* Obsolete Streaming functions */ -int LZ4_sizeofStreamState(void) { return LZ4_STREAMSIZE; } +int LZ4_sizeofStreamState(void) { return sizeof(LZ4_stream_t); } int LZ4_resetStreamState(void* state, char* inputBuffer) { @@ -2480,11 +2705,13 @@ int LZ4_resetStreamState(void* state, char* inputBuffer) return 0; } +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) void* LZ4_create (char* inputBuffer) { (void)inputBuffer; return LZ4_createStream(); } +#endif char* LZ4_slideInputBuffer (void* state) { diff --git a/lz4libs/lz4.h b/lz4libs/lz4.h index 7ab1e483..491c6087 100644 --- a/lz4libs/lz4.h +++ b/lz4libs/lz4.h @@ -1,7 +1,7 @@ /* * LZ4 - Fast LZ compression algorithm * Header File - * Copyright (C) 2011-present, Yann Collet. + * Copyright (C) 2011-2020, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -97,36 +97,77 @@ extern "C" { # define LZ4LIB_API LZ4LIB_VISIBILITY #endif +/*! LZ4_FREESTANDING : + * When this macro is set to 1, it enables "freestanding mode" that is + * suitable for typical freestanding environment which doesn't support + * standard C library. + * + * - LZ4_FREESTANDING is a compile-time switch. + * - It requires the following macros to be defined: + * LZ4_memcpy, LZ4_memmove, LZ4_memset. + * - It only enables LZ4/HC functions which don't use heap. + * All LZ4F_* functions are not supported. + * - See tests/freestanding.c to check its basic setup. + */ +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) +# define LZ4_HEAPMODE 0 +# define LZ4HC_HEAPMODE 0 +# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1 +# if !defined(LZ4_memcpy) +# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'." +# endif +# if !defined(LZ4_memset) +# error "LZ4_FREESTANDING requires macro 'LZ4_memset'." +# endif +# if !defined(LZ4_memmove) +# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'." +# endif +#elif ! defined(LZ4_FREESTANDING) +# define LZ4_FREESTANDING 0 +#endif + + /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) #define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE #define LZ4_QUOTE(str) #str #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) -#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) +#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */ -LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */ -LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */ +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */ /*-************************************ * Tuning parameter **************************************/ +#define LZ4_MEMORY_USAGE_MIN 10 +#define LZ4_MEMORY_USAGE_DEFAULT 14 +#define LZ4_MEMORY_USAGE_MAX 20 + /*! * LZ4_MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio. - * Reduced memory usage may improve speed, thanks to better cache locality. + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; ) + * Increasing memory usage improves compression ratio, at the cost of speed. + * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality. * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #ifndef LZ4_MEMORY_USAGE -# define LZ4_MEMORY_USAGE 14 +# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT #endif +#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN) +# error "LZ4_MEMORY_USAGE is too small !" +#endif + +#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX) +# error "LZ4_MEMORY_USAGE is too large !" +#endif /*-************************************ * Simple Functions @@ -270,8 +311,25 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcS ***********************************************/ typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ +/** + Note about RC_INVOKED + + - RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio). + https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros + + - Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars) + and reports warning "RC4011: identifier truncated". + + - To eliminate the warning, we surround long preprocessor symbol with + "#if !defined(RC_INVOKED) ... #endif" block that means + "skip this block when rc.exe is trying to read it". +*/ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif /*! LZ4_resetStream_fast() : v1.9.0+ * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks @@ -355,8 +413,12 @@ typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ * creation / destruction of streaming decompression tracking context. * A tracking context can be re-used multiple times. */ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif /*! LZ4_setStreamDecode() : * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. @@ -406,7 +468,10 @@ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. */ -LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); +LZ4LIB_API int +LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* src, char* dst, + int srcSize, int dstCapacity); /*! LZ4_decompress_*_usingDict() : @@ -417,7 +482,16 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod * Performance tip : Decompression speed can be substantially increased * when dst == dictStart + dictSize. */ -LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); +LZ4LIB_API int +LZ4_decompress_safe_usingDict(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize); + +LZ4LIB_API int +LZ4_decompress_safe_partial_usingDict(const char* src, char* dst, + int compressedSize, + int targetOutputSize, int maxOutputSize, + const char* dictStart, int dictSize); #endif /* LZ4_H_2983827168210 */ @@ -496,13 +570,15 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c * stream (and source buffer) must remain in-place / accessible / unchanged * through the completion of the first compression call on the stream. */ -LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); +LZ4LIB_STATIC_API void +LZ4_attach_dictionary(LZ4_stream_t* workingStream, + const LZ4_stream_t* dictionaryStream); /*! In-place compression and decompression * * It's possible to have input and output sharing the same buffer, - * for highly contrained memory environments. + * for highly constrained memory environments. * In both cases, it requires input to lay at the end of the buffer, * and decompression to start at beginning of the buffer. * Buffer size must feature some margin, hence be larger than final size. @@ -592,38 +668,26 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const typedef unsigned int LZ4_u32; #endif +/*! LZ4_stream_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_stream_t object. +**/ + typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; struct LZ4_stream_t_internal { LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; - LZ4_u32 currentOffset; - LZ4_u32 tableType; const LZ4_byte* dictionary; const LZ4_stream_t_internal* dictCtx; + LZ4_u32 currentOffset; + LZ4_u32 tableType; LZ4_u32 dictSize; + /* Implicit padding to ensure structure is aligned */ }; -typedef struct { - const LZ4_byte* externalDict; - size_t extDictSize; - const LZ4_byte* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - - -/*! LZ4_stream_t : - * Do not use below internal definitions directly ! - * Declare or allocate an LZ4_stream_t instead. - * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended. - * The structure definition can be convenient for static allocation - * (on stack, or as part of larger structure). - * Init this structure with LZ4_initStream() before first use. - * note : only use this definition in association with static linking ! - * this definition is not API/ABI safe, and may change in future versions. - */ -#define LZ4_STREAMSIZE 16416 /* static size, for inter-version compatibility */ -#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*)) +#define LZ4_STREAM_MINSIZE ((1UL << LZ4_MEMORY_USAGE) + 32) /* static size, for inter-version compatibility */ union LZ4_stream_u { - void* table[LZ4_STREAMSIZE_VOIDP]; + char minStateSize[LZ4_STREAM_MINSIZE]; LZ4_stream_t_internal internal_donotuse; }; /* previously typedef'd to LZ4_stream_t */ @@ -641,21 +705,25 @@ union LZ4_stream_u { * In which case, the function will @return NULL. * Note2: An LZ4_stream_t structure guarantees correct alignment and size. * Note3: Before v1.9.0, use LZ4_resetStream() instead - */ +**/ LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size); /*! LZ4_streamDecode_t : - * information structure to track an LZ4 stream during decompression. - * init this structure using LZ4_setStreamDecode() before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! - */ -#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ ) -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_streamDecode_t object. +**/ +typedef struct { + const LZ4_byte* externalDict; + const LZ4_byte* prefixEnd; + size_t extDictSize; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#define LZ4_STREAMDECODE_MINSIZE 32 union LZ4_streamDecode_u { - unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + char minStateSize[LZ4_STREAMDECODE_MINSIZE]; LZ4_streamDecode_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_streamDecode_t */ diff --git a/lz4libs/lz4frame.c b/lz4libs/lz4frame.c index ec02c92f..174f9ae4 100644 --- a/lz4libs/lz4frame.c +++ b/lz4libs/lz4frame.c @@ -45,7 +45,7 @@ * Compiler Options **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif @@ -62,6 +62,19 @@ #endif +/*-************************************ +* Library declarations +**************************************/ +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" +#define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + /*-************************************ * Memory routines **************************************/ @@ -70,7 +83,13 @@ * malloc(), calloc() and free() * towards another library or solution of their choice * by modifying below section. - */ +**/ + +#include /* memset, memcpy, memmove */ +#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# define MEM_INIT(p,v,s) memset((p),(v),(s)) +#endif + #ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ # include /* malloc, calloc, free */ # define ALLOC(s) malloc(s) @@ -78,23 +97,42 @@ # define FREEMEM(p) free(p) #endif -#include /* memset, memcpy, memmove */ -#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ -# define MEM_INIT(p,v,s) memset((p),(v),(s)) -#endif +static void* LZ4F_calloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom calloc defined : use it */ + if (cmem.customCalloc != NULL) { + return cmem.customCalloc(cmem.opaqueState, s); + } + /* nothing defined : use default 's calloc() */ + if (cmem.customAlloc == NULL) { + return ALLOC_AND_ZERO(s); + } + /* only custom alloc defined : use it, and combine it with memset() */ + { void* const p = cmem.customAlloc(cmem.opaqueState, s); + if (p != NULL) MEM_INIT(p, 0, s); + return p; +} } +static void* LZ4F_malloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom malloc defined : use it */ + if (cmem.customAlloc != NULL) { + return cmem.customAlloc(cmem.opaqueState, s); + } + /* nothing defined : use default 's malloc() */ + return ALLOC(s); +} -/*-************************************ -* Library declarations -**************************************/ -#define LZ4F_STATIC_LINKING_ONLY -#include "lz4frame.h" -#define LZ4_STATIC_LINKING_ONLY -#include "lz4.h" -#define LZ4_HC_STATIC_LINKING_ONLY -#include "lz4hc.h" -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" +static void LZ4F_free(void* p, LZ4F_CustomMem cmem) +{ + /* custom malloc defined : use it */ + if (cmem.customFree != NULL) { + cmem.customFree(cmem.opaqueState, p); + return; + } + /* nothing defined : use default 's free() */ + FREEMEM(p); +} /*-************************************ @@ -143,7 +181,7 @@ static int g_debuglog_enable = 1; #endif -/* unoptimized version; solves endianess & alignment issues */ +/* unoptimized version; solves endianness & alignment issues */ static U32 LZ4F_readLE32 (const void* src) { const BYTE* const srcPtr = (const BYTE*)src; @@ -206,8 +244,6 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define _4BITS 0x0F #define _8BITS 0xFF -#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U -#define LZ4F_MAGICNUMBER 0x184D2204U #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB @@ -220,22 +256,27 @@ static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checks /*-************************************ * Structures and local types **************************************/ + +typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t; + typedef struct LZ4F_cctx_s { + LZ4F_CustomMem cmem; LZ4F_preferences_t prefs; U32 version; U32 cStage; const LZ4F_CDict* cdict; size_t maxBlockSize; size_t maxBufferSize; - BYTE* tmpBuff; - BYTE* tmpIn; - size_t tmpInSize; + BYTE* tmpBuff; /* internal buffer, for streaming */ + BYTE* tmpIn; /* starting position of data compress within internal buffer (>= tmpBuff) */ + size_t tmpInSize; /* amount of data to compress after tmpIn */ U64 totalInSize; XXH32_state_t xxh; void* lz4CtxPtr; U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ + LZ4F_blockCompression_t blockCompression; } LZ4F_cctx_t; @@ -264,27 +305,33 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult) return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult); } -static LZ4F_errorCode_t err0r(LZ4F_errorCodes code) +static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code) { /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); return (LZ4F_errorCode_t)-(ptrdiff_t)code; } +#define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e) + +#define RETURN_ERROR_IF(c,e) if (c) RETURN_ERROR(e) + +#define FORWARD_IF_ERROR(r) if (LZ4F_isError(r)) return (r) + unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } -size_t LZ4F_getBlockSize(unsigned blockSizeID) +size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID) { static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB) - return err0r(LZ4F_ERROR_maxBlockSize_invalid); - blockSizeID -= LZ4F_max64KB; - return blockSizes[blockSizeID]; -} + RETURN_ERROR(maxBlockSize_invalid); + { int const blockSizeIdx = (int)blockSizeID - (int)LZ4F_max64KB; + return blockSizes[blockSizeIdx]; +} } /*-************************************ * Private functions @@ -397,21 +444,20 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, MEM_INIT(&options, 0, sizeof(options)); options.stableSrc = 1; - if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */ - return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + RETURN_ERROR_IF(dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs), dstMaxSize_tooSmall); { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */ - if (LZ4F_isError(headerSize)) return headerSize; + FORWARD_IF_ERROR(headerSize); dstPtr += headerSize; /* header size */ } assert(dstEnd >= dstPtr); { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options); - if (LZ4F_isError(cSize)) return cSize; + FORWARD_IF_ERROR(cSize); dstPtr += cSize; } assert(dstEnd >= dstPtr); { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */ - if (LZ4F_isError(tailSize)) return tailSize; + FORWARD_IF_ERROR(tailSize); dstPtr += tailSize; } assert(dstEnd >= dstStart); @@ -432,27 +478,26 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, { size_t result; #if (LZ4F_HEAPMODE) - LZ4F_cctx_t *cctxPtr; + LZ4F_cctx_t* cctxPtr; result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION); - if (LZ4F_isError(result)) return result; + FORWARD_IF_ERROR(result); #else LZ4F_cctx_t cctx; LZ4_stream_t lz4ctx; - LZ4F_cctx_t *cctxPtr = &cctx; + LZ4F_cctx_t* const cctxPtr = &cctx; - DEBUGLOG(4, "LZ4F_compressFrame"); MEM_INIT(&cctx, 0, sizeof(cctx)); cctx.version = LZ4F_VERSION; cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ - if (preferencesPtr == NULL || - preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN) - { + if ( preferencesPtr == NULL + || preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN ) { LZ4_initStream(&lz4ctx, sizeof(lz4ctx)); cctxPtr->lz4CtxPtr = &lz4ctx; cctxPtr->lz4CtxAlloc = 1; cctxPtr->lz4CtxState = 1; } #endif + DEBUGLOG(4, "LZ4F_compressFrame"); result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity, srcBuffer, srcSize, @@ -461,10 +506,9 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, #if (LZ4F_HEAPMODE) LZ4F_freeCompressionContext(cctxPtr); #else - if (preferencesPtr != NULL && - preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) - { - FREEMEM(cctxPtr->lz4CtxPtr); + if ( preferencesPtr != NULL + && preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN ) { + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); } #endif return result; @@ -476,30 +520,31 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, *****************************************************/ struct LZ4F_CDict_s { + LZ4F_CustomMem cmem; void* dictContent; LZ4_stream_t* fastCtx; LZ4_streamHC_t* HCCtx; }; /* typedef'd to LZ4F_CDict within lz4frame_static.h */ -/*! LZ4F_createCDict() : - * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. - * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. - * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict - * @return : digested dictionary for compression, or NULL if failed */ -LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) +LZ4F_CDict* +LZ4F_createCDict_advanced(LZ4F_CustomMem cmem, const void* dictBuffer, size_t dictSize) { const char* dictStart = (const char*)dictBuffer; - LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict)); - DEBUGLOG(4, "LZ4F_createCDict"); + LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), cmem); + DEBUGLOG(4, "LZ4F_createCDict_advanced"); if (!cdict) return NULL; + cdict->cmem = cmem; if (dictSize > 64 KB) { dictStart += dictSize - 64 KB; dictSize = 64 KB; } - cdict->dictContent = ALLOC(dictSize); - cdict->fastCtx = LZ4_createStream(); - cdict->HCCtx = LZ4_createStreamHC(); + cdict->dictContent = LZ4F_malloc(dictSize, cmem); + cdict->fastCtx = (LZ4_stream_t*)LZ4F_malloc(sizeof(LZ4_stream_t), cmem); + if (cdict->fastCtx) + LZ4_initStream(cdict->fastCtx, sizeof(LZ4_stream_t)); + cdict->HCCtx = (LZ4_streamHC_t*)LZ4F_malloc(sizeof(LZ4_streamHC_t), cmem); + if (cdict->HCCtx) + LZ4_initStream(cdict->HCCtx, sizeof(LZ4_streamHC_t)); if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { LZ4F_freeCDict(cdict); return NULL; @@ -511,13 +556,25 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) return cdict; } +/*! LZ4F_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict + * @return : digested dictionary for compression, or NULL if failed */ +LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) +{ + DEBUGLOG(4, "LZ4F_createCDict"); + return LZ4F_createCDict_advanced(LZ4F_defaultCMem, dictBuffer, dictSize); +} + void LZ4F_freeCDict(LZ4F_CDict* cdict) { if (cdict==NULL) return; /* support free on NULL */ - FREEMEM(cdict->dictContent); - LZ4_freeStream(cdict->fastCtx); - LZ4_freeStreamHC(cdict->HCCtx); - FREEMEM(cdict); + LZ4F_free(cdict->dictContent, cdict->cmem); + LZ4F_free(cdict->fastCtx, cdict->cmem); + LZ4F_free(cdict->HCCtx, cdict->cmem); + LZ4F_free(cdict, cdict->cmem); } @@ -525,6 +582,20 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict) * Advanced compression functions ***********************************/ +LZ4F_cctx* +LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version) +{ + LZ4F_cctx* const cctxPtr = + (LZ4F_cctx*)LZ4F_calloc(sizeof(LZ4F_cctx), customMem); + if (cctxPtr==NULL) return NULL; + + cctxPtr->cmem = customMem; + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */ + + return cctxPtr; +} + /*! LZ4F_createCompressionContext() : * The first thing to do is to create a compressionContext object, which will be used in all compression operations. * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. @@ -532,17 +603,16 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict) * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. * Object can release its memory using LZ4F_freeCompressionContext(); - */ -LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) +**/ +LZ4F_errorCode_t +LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) { - LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t)); - if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed); - - cctxPtr->version = version; - cctxPtr->cStage = 0; /* Next stage : init stream */ - - *LZ4F_compressionContextPtr = cctxPtr; + assert(LZ4F_compressionContextPtr != NULL); /* considered a violation of narrow contract */ + /* in case it nonetheless happen in production */ + RETURN_ERROR_IF(LZ4F_compressionContextPtr == NULL, parameter_null); + *LZ4F_compressionContextPtr = LZ4F_createCompressionContext_advanced(LZ4F_defaultCMem, version); + RETURN_ERROR_IF(*LZ4F_compressionContextPtr==NULL, allocation_failed); return LZ4F_OK_NoError; } @@ -550,11 +620,10 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionConte LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr) { if (cctxPtr != NULL) { /* support free on NULL */ - FREEMEM(cctxPtr->lz4CtxPtr); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ - FREEMEM(cctxPtr->tmpBuff); - FREEMEM(cctxPtr); + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ + LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem); + LZ4F_free(cctxPtr, cctxPtr->cmem); } - return LZ4F_OK_NoError; } @@ -588,11 +657,21 @@ static void LZ4F_initStream(void* ctx, } } +static int ctxTypeID_to_size(int ctxTypeID) { + switch(ctxTypeID) { + case 1: + return LZ4_sizeofState(); + case 2: + return LZ4_sizeofStateHC(); + default: + return 0; + } +} /*! LZ4F_compressBegin_usingCDict() : - * init streaming compression and writes frame header into dstBuffer. - * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. - * @return : number of bytes written into dstBuffer for the header + * init streaming compression AND writes frame header into @dstBuffer. + * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * @return : number of bytes written into @dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, @@ -600,41 +679,46 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, const LZ4F_CDict* cdict, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefNull; + LZ4F_preferences_t const prefNull = LZ4F_INIT_PREFERENCES; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; - BYTE* headerStart; - if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - MEM_INIT(&prefNull, 0, sizeof(prefNull)); + RETURN_ERROR_IF(dstCapacity < maxFHSize, dstMaxSize_tooSmall); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; - /* Ctx Management */ + /* cctx Management */ { U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; - if (cctxPtr->lz4CtxAlloc < ctxTypeID) { - FREEMEM(cctxPtr->lz4CtxPtr); + int requiredSize = ctxTypeID_to_size(ctxTypeID); + int allocatedSize = ctxTypeID_to_size(cctxPtr->lz4CtxAlloc); + if (allocatedSize < requiredSize) { + /* not enough space allocated */ + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - cctxPtr->lz4CtxPtr = LZ4_createStream(); + /* must take ownership of memory allocation, + * in order to respect custom allocator contract */ + cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_stream_t), cctxPtr->cmem); + if (cctxPtr->lz4CtxPtr) + LZ4_initStream(cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t)); } else { - cctxPtr->lz4CtxPtr = LZ4_createStreamHC(); + cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_streamHC_t), cctxPtr->cmem); + if (cctxPtr->lz4CtxPtr) + LZ4_initStreamHC(cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t)); } - if (cctxPtr->lz4CtxPtr == NULL) - return err0r(LZ4F_ERROR_allocation_failed); + RETURN_ERROR_IF(cctxPtr->lz4CtxPtr == NULL, allocation_failed); cctxPtr->lz4CtxAlloc = ctxTypeID; cctxPtr->lz4CtxState = ctxTypeID; } else if (cctxPtr->lz4CtxState != ctxTypeID) { - /* otherwise, a sufficient buffer is allocated, but we need to - * reset it to the correct context type */ + /* otherwise, a sufficient buffer is already allocated, + * but we need to reset it to the correct context type */ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - LZ4_initStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr, sizeof (LZ4_stream_t)); + LZ4_initStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t)); } else { - LZ4_initStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t)); - LZ4_setCompressionLevel((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + LZ4_initStreamHC((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); } cctxPtr->lz4CtxState = ctxTypeID; - } - } + } } /* Buffer Management */ if (cctxPtr->prefs.frameInfo.blockSizeID == 0) @@ -647,9 +731,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, if (cctxPtr->maxBufferSize < requiredBuffSize) { cctxPtr->maxBufferSize = 0; - FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize); - if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed); + LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem); + cctxPtr->tmpBuff = (BYTE*)LZ4F_calloc(requiredBuffSize, cctxPtr->cmem); + RETURN_ERROR_IF(cctxPtr->tmpBuff == NULL, allocation_failed); cctxPtr->maxBufferSize = requiredBuffSize; } } cctxPtr->tmpIn = cctxPtr->tmpBuff; @@ -669,31 +753,32 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, /* Magic Number */ LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); dstPtr += 4; - headerStart = dstPtr; - - /* FLG Byte */ - *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ - + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) - + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) - + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3) - + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) - + (cctxPtr->prefs.frameInfo.dictID > 0) ); - /* BD Byte */ - *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); - /* Optional Frame content size field */ - if (cctxPtr->prefs.frameInfo.contentSize) { - LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); - dstPtr += 8; - cctxPtr->totalInSize = 0; - } - /* Optional dictionary ID field */ - if (cctxPtr->prefs.frameInfo.dictID) { - LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID); - dstPtr += 4; + { BYTE* const headerStart = dstPtr; + + /* FLG Byte */ + *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) + + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) + + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3) + + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) + + (cctxPtr->prefs.frameInfo.dictID > 0) ); + /* BD Byte */ + *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* Optional Frame content size field */ + if (cctxPtr->prefs.frameInfo.contentSize) { + LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); + dstPtr += 8; + cctxPtr->totalInSize = 0; + } + /* Optional dictionary ID field */ + if (cctxPtr->prefs.frameInfo.dictID) { + LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID); + dstPtr += 4; + } + /* Header CRC Byte */ + *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart)); + dstPtr++; } - /* Header CRC Byte */ - *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart)); - dstPtr++; cctxPtr->cStage = 1; /* header written, now request input data block */ return (size_t)(dstPtr - dstStart); @@ -701,9 +786,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, /*! LZ4F_compressBegin() : - * init streaming compression and writes frame header into dstBuffer. - * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. - * preferencesPtr can be NULL, in which case default parameters are selected. + * init streaming compression AND writes frame header into @dstBuffer. + * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * @preferencesPtr can be NULL, in which case default parameters are selected. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ @@ -744,11 +829,13 @@ static size_t LZ4F_makeBlock(void* dst, LZ4F_blockChecksum_t crcFlag) { BYTE* const cSizePtr = (BYTE*)dst; - U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), - (int)(srcSize), (int)(srcSize-1), - level, cdict); - if (cSize == 0) { /* compression failed */ - DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize); + U32 cSize; + assert(compress != NULL); + cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), + (int)(srcSize), (int)(srcSize-1), + level, cdict); + + if (cSize == 0 || cSize >= srcSize) { cSize = (U32)srcSize; LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+BHSize, src, srcSize); @@ -766,6 +853,7 @@ static size_t LZ4F_makeBlock(void* dst, static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { int const acceleration = (level < 0) ? -level + 1 : 1; + DEBUGLOG(5, "LZ4F_compressBlock (srcSize=%i)", srcSize); LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent); if (cdict) { return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); @@ -778,6 +866,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in { int const acceleration = (level < 0) ? -level + 1 : 1; (void)cdict; /* init once at beginning of frame */ + DEBUGLOG(5, "LZ4F_compressBlock_continue (srcSize=%i)", srcSize); return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); } @@ -796,8 +885,15 @@ static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); } -static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) +static int LZ4F_doNotCompressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + (void)ctx; (void)src; (void)dst; (void)srcSize; (void)dstCapacity; (void)level; (void)cdict; + return 0; +} + +static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level, LZ4F_blockCompression_t compressMode) { + if (compressMode == LZ4B_UNCOMPRESSED) return LZ4F_doNotCompressBlock; if (level < LZ4HC_CLEVEL_MIN) { if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock; return LZ4F_compressBlock_continue; @@ -806,6 +902,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev return LZ4F_compressBlockHC_continue; } +/* Save history (up to 64KB) into @tmpBuff */ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) @@ -815,38 +912,57 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; -/*! LZ4F_compressUpdate() : +static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } }; + + + /*! LZ4F_compressUpdateImpl() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). - * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If the block compression does not match the compression of the previous block, the old data is flushed + * and operations continue with the new compression mode. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on. + * @compressOptionsPtr is optional : provide NULL to mean "default". * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. */ -size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, - void* dstBuffer, size_t dstCapacity, +static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, - const LZ4F_compressOptions_t* compressOptionsPtr) -{ - LZ4F_compressOptions_t cOptionsNull; + const LZ4F_compressOptions_t* compressOptionsPtr, + LZ4F_blockCompression_t blockCompression) + { size_t const blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcPtr + srcSize; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; LZ4F_lastBlockStatus lastBlockCompressed = notDone; - compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - + compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, blockCompression); + size_t bytesWritten; DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize); - if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); + RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */ if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) - return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull)); - if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + RETURN_ERROR(dstMaxSize_tooSmall); + + if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize) + RETURN_ERROR(dstMaxSize_tooSmall); + + /* flush currently written block, to continue with new block compression */ + if (cctxPtr->blockCompression != blockCompression) { + bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); + dstPtr += bytesWritten; + cctxPtr->blockCompression = blockCompression; + } + + if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull; /* complete tmp buffer */ if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; + assert(blockSize > cctxPtr->tmpInSize); if (sizeToCopy > srcSize) { /* add src to tmpIn buffer */ memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); @@ -864,11 +980,9 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); - if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; - } - } + } } while ((size_t)(srcEnd - srcPtr) >= blockSize) { /* compress full blocks */ @@ -882,33 +996,38 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { - /* compress remaining input < blockSize */ + /* autoFlush : remaining input (< blockSize) is compressed */ lastBlockCompressed = fromSrcBuffer; dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, (size_t)(srcEnd - srcPtr), compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); - srcPtr = srcEnd; + srcPtr = srcEnd; } - /* preserve dictionary if necessary */ + /* preserve dictionary within @tmpBuff whenever necessary */ if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { + /* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */ + assert(blockCompression == LZ4B_COMPRESSED); if (compressOptionsPtr->stableSrc) { - cctxPtr->tmpIn = cctxPtr->tmpBuff; + cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */ } else { int const realDictSize = LZ4F_localSaveDict(cctxPtr); - if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC); + assert(0 <= realDictSize && realDictSize <= 64 KB); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } } /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */ - && !(cctxPtr->prefs.autoFlush)) + if (!(cctxPtr->prefs.autoFlush) /* no autoflush : there may be some data left within internal buffer */ + && (cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) ) /* not enough room to store next block */ { + /* only preserve 64KB within internal buffer. Ensures there is enough room for next block. + * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */ int const realDictSize = LZ4F_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)); } /* some input data left, necessarily < blockSize */ @@ -926,6 +1045,53 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, return (size_t)(dstPtr - dstStart); } +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_COMPRESSED); +} + +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * This is only supported when LZ4F_blockIndependent is used + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) { + RETURN_ERROR_IF(cctxPtr->prefs.frameInfo.blockMode != LZ4F_blockIndependent, blockMode_invalid); + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_UNCOMPRESSED); +} + /*! LZ4F_flush() : * When compressed data must be sent immediately, without waiting for a block to be filled, @@ -944,13 +1110,12 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, compressFunc_t compress; if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ - if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize)) - return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - (void)compressOptionsPtr; /* not yet useful */ + RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); + RETURN_ERROR_IF(dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize), dstMaxSize_tooSmall); + (void)compressOptionsPtr; /* not useful (yet) */ /* select compression function */ - compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); + compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, cctxPtr->blockCompression); /* compress tmp buffer */ dstPtr += LZ4F_makeBlock(dstPtr, @@ -992,19 +1157,19 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); DEBUGLOG(5,"LZ4F_compressEnd: dstCapacity=%u", (unsigned)dstCapacity); - if (LZ4F_isError(flushSize)) return flushSize; + FORWARD_IF_ERROR(flushSize); dstPtr += flushSize; assert(flushSize <= dstCapacity); dstCapacity -= flushSize; - if (dstCapacity < 4) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + RETURN_ERROR_IF(dstCapacity < 4, dstMaxSize_tooSmall); LZ4F_writeLE32(dstPtr, 0); dstPtr += 4; /* endMark */ if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) { U32 const xxh = XXH32_digest(&(cctxPtr->xxh)); - if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + RETURN_ERROR_IF(dstCapacity < 8, dstMaxSize_tooSmall); DEBUGLOG(5,"Writing 32-bit content checksum"); LZ4F_writeLE32(dstPtr, xxh); dstPtr+=4; /* content Checksum */ @@ -1015,7 +1180,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, if (cctxPtr->prefs.frameInfo.contentSize) { if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) - return err0r(LZ4F_ERROR_frameSize_wrong); + RETURN_ERROR(frameSize_wrong); } return (size_t)(dstPtr - dstStart); @@ -1039,6 +1204,7 @@ typedef enum { } dStage_t; struct LZ4F_dctx_s { + LZ4F_CustomMem cmem; LZ4F_frameInfo_t frameInfo; U32 version; dStage_t dStage; @@ -1056,26 +1222,37 @@ struct LZ4F_dctx_s { size_t tmpOutStart; XXH32_state_t xxh; XXH32_state_t blockChecksum; + int skipChecksum; BYTE header[LZ4F_HEADER_SIZE_MAX]; }; /* typedef'd to LZ4F_dctx in lz4frame.h */ +LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version) +{ + LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), customMem); + if (dctx == NULL) return NULL; + + dctx->cmem = customMem; + dctx->version = version; + return dctx; +} + /*! LZ4F_createDecompressionContext() : * Create a decompressionContext object, which will track all decompression operations. * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. * Object can later be released using LZ4F_freeDecompressionContext(). * @return : if != 0, there was an error during context creation. */ -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) +LZ4F_errorCode_t +LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { - LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx)); - if (dctx == NULL) { /* failed allocation */ - *LZ4F_decompressionContextPtr = NULL; - return err0r(LZ4F_ERROR_allocation_failed); - } + assert(LZ4F_decompressionContextPtr != NULL); /* violation of narrow contract */ + RETURN_ERROR_IF(LZ4F_decompressionContextPtr == NULL, parameter_null); /* in case it nonetheless happen in production */ - dctx->version = versionNumber; - *LZ4F_decompressionContextPtr = dctx; + *LZ4F_decompressionContextPtr = LZ4F_createDecompressionContext_advanced(LZ4F_defaultCMem, versionNumber); + if (*LZ4F_decompressionContextPtr == NULL) { /* failed allocation */ + RETURN_ERROR(allocation_failed); + } return LZ4F_OK_NoError; } @@ -1084,9 +1261,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx) LZ4F_errorCode_t result = LZ4F_OK_NoError; if (dctx != NULL) { /* can accept NULL input, like free() */ result = (LZ4F_errorCode_t)dctx->dStage; - FREEMEM(dctx->tmpIn); - FREEMEM(dctx->tmpOutBuffer); - FREEMEM(dctx); + LZ4F_free(dctx->tmpIn, dctx->cmem); + LZ4F_free(dctx->tmpOutBuffer, dctx->cmem); + LZ4F_free(dctx, dctx->cmem); } return result; } @@ -1099,6 +1276,7 @@ void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) dctx->dStage = dstage_getFrameHeader; dctx->dict = NULL; dctx->dictSize = 0; + dctx->skipChecksum = 0; } @@ -1118,7 +1296,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize DEBUGLOG(5, "LZ4F_decodeHeader"); /* need to decode header to get frameInfo */ - if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */ + RETURN_ERROR_IF(srcSize < minFHSize, frameHeader_incomplete); /* minimal frame header size */ MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo)); /* special case : skippable frames */ @@ -1132,14 +1310,13 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize } else { dctx->dStage = dstage_getSFrameSize; return 4; - } - } + } } /* control magic number */ #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) { DEBUGLOG(4, "frame header error : unknown magic number"); - return err0r(LZ4F_ERROR_frameType_unknown); + RETURN_ERROR(frameType_unknown); } #endif dctx->frameInfo.frameType = LZ4F_frame; @@ -1153,8 +1330,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize contentChecksumFlag = (FLG>>2) & _1BIT; dictIDFlag = FLG & _1BIT; /* validate */ - if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */ - if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */ + if (((FLG>>1)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */ + if (version != 1) RETURN_ERROR(headerVersion_wrong); /* Version Number, only supported value */ } /* Frame Header Size */ @@ -1173,17 +1350,16 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize { U32 const BD = srcPtr[5]; blockSizeID = (BD>>4) & _3BITS; /* validate */ - if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */ - if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */ + if (((BD>>7)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */ + if (blockSizeID < 4) RETURN_ERROR(maxBlockSize_invalid); /* 4-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bits */ } /* check header */ assert(frameHeaderSize > 5); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); - if (HC != srcPtr[frameHeaderSize-1]) - return err0r(LZ4F_ERROR_headerChecksum_invalid); + RETURN_ERROR_IF(HC != srcPtr[frameHeaderSize-1], headerChecksum_invalid); } #endif @@ -1192,10 +1368,9 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag; dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; - dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID); + dctx->maxBlockSize = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID); if (contentSizeFlag) - dctx->frameRemainingSize = - dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + dctx->frameRemainingSize = dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); if (dictIDFlag) dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5); @@ -1211,11 +1386,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize */ size_t LZ4F_headerSize(const void* src, size_t srcSize) { - if (src == NULL) return err0r(LZ4F_ERROR_srcPtr_wrong); + RETURN_ERROR_IF(src == NULL, srcPtr_wrong); /* minimal srcSize to determine header size */ if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH) - return err0r(LZ4F_ERROR_frameHeader_incomplete); + RETURN_ERROR(frameHeader_incomplete); /* special case : skippable frames */ if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) @@ -1224,7 +1399,7 @@ size_t LZ4F_headerSize(const void* src, size_t srcSize) /* control magic number */ #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) - return err0r(LZ4F_ERROR_frameType_unknown); + RETURN_ERROR(frameType_unknown); #endif /* Frame Header Size */ @@ -1266,13 +1441,13 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, if (dctx->dStage == dstage_storeFrameHeader) { /* frame decoding already started, in the middle of header => automatic fail */ *srcSizePtr = 0; - return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted); + RETURN_ERROR(frameDecoding_alreadyStarted); } else { size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; } if (*srcSizePtr < hSize) { *srcSizePtr=0; - return err0r(LZ4F_ERROR_frameHeader_incomplete); + RETURN_ERROR(frameHeader_incomplete); } { size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); @@ -1290,16 +1465,14 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, /* LZ4F_updateDict() : * only used for LZ4F_blockLinked mode - * Condition : dstPtr != NULL + * Condition : @dstPtr != NULL */ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart, unsigned withinTmp) { assert(dstPtr != NULL); - if (dctx->dictSize==0) { - dctx->dict = (const BYTE*)dstPtr; /* priority to prefix mode */ - } + if (dctx->dictSize==0) dctx->dict = (const BYTE*)dstPtr; /* will lead to prefix mode */ assert(dctx->dict != NULL); if (dctx->dict + dctx->dictSize == dstPtr) { /* prefix mode, everything within dstBuffer */ @@ -1362,7 +1535,6 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, } - /*! LZ4F_decompress() : * Call this function repetitively to regenerate compressed data in srcBuffer. * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer @@ -1406,6 +1578,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, *srcSizePtr = 0; *dstSizePtr = 0; assert(dctx != NULL); + dctx->skipChecksum |= (decompressOptionsPtr->skipChecksums != 0); /* once set, disable for the remainder of the frame */ /* behaves as a state machine */ @@ -1418,7 +1591,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, DEBUGLOG(6, "dstage_getFrameHeader"); if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */ - if (LZ4F_isError(hSize)) return hSize; + FORWARD_IF_ERROR(hSize); srcPtr += hSize; break; } @@ -1440,9 +1613,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, doAnotherStage = 0; /* not enough src data, ask for some more */ break; } - { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */ - if (LZ4F_isError(hSize)) return hSize; - } + FORWARD_IF_ERROR( LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget) ); /* will update dStage appropriately */ break; case dstage_init: @@ -1453,14 +1624,12 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0); if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ - FREEMEM(dctx->tmpIn); - dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + BFSize /* block checksum */); - if (dctx->tmpIn == NULL) - return err0r(LZ4F_ERROR_allocation_failed); - FREEMEM(dctx->tmpOutBuffer); - dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded); - if (dctx->tmpOutBuffer== NULL) - return err0r(LZ4F_ERROR_allocation_failed); + LZ4F_free(dctx->tmpIn, dctx->cmem); + dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, dctx->cmem); + RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed); + LZ4F_free(dctx->tmpOutBuffer, dctx->cmem); + dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, dctx->cmem); + RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed); dctx->maxBufferSize = bufferNeeded; } } dctx->tmpInSize = 0; @@ -1509,7 +1678,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; } if (nextCBlockSize > dctx->maxBlockSize) { - return err0r(LZ4F_ERROR_maxBlockSize_invalid); + RETURN_ERROR(maxBlockSize_invalid); } if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) { /* next block is uncompressed */ @@ -1540,11 +1709,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr)); sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctx->frameInfo.blockChecksumFlag) { - (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + if (!dctx->skipChecksum) { + if (dctx->frameInfo.blockChecksumFlag) { + (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + } + if (dctx->frameInfo.contentChecksumFlag) + (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); } - if (dctx->frameInfo.contentChecksumFlag) - (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); if (dctx->frameInfo.contentSize) dctx->frameRemainingSize -= sizeToCopy; @@ -1590,14 +1761,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } crcSrc = dctx->header; } - { U32 const readCRC = LZ4F_readLE32(crcSrc); + if (!dctx->skipChecksum) { + U32 const readCRC = LZ4F_readLE32(crcSrc); U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION DEBUGLOG(6, "compare block checksum"); if (readCRC != calcCRC) { DEBUGLOG(4, "incorrect block checksum: %08X != %08X", readCRC, calcCRC); - return err0r(LZ4F_ERROR_blockChecksum_invalid); + RETURN_ERROR(blockChecksum_invalid); } #else (void)readCRC; @@ -1637,37 +1809,44 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } /* At this stage, input is large enough to decode a block */ + + /* First, decode and control block checksum if it exists */ if (dctx->frameInfo.blockChecksumFlag) { + assert(dctx->tmpInTarget >= 4); dctx->tmpInTarget -= 4; assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */ { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (readBlockCrc != calcBlockCrc) - return err0r(LZ4F_ERROR_blockChecksum_invalid); + RETURN_ERROR_IF(readBlockCrc != calcBlockCrc, blockChecksum_invalid); #else (void)readBlockCrc; (void)calcBlockCrc; #endif } } - if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) { + /* decode directly into destination buffer if there is enough room */ + if ( ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) + /* unless the dictionary is stored in tmpOut: + * in which case it's faster to decode within tmpOut + * to benefit from prefix speedup */ + && !(dctx->dict!= NULL && (const BYTE*)dctx->dict + dctx->dictSize == dctx->tmpOut) ) + { const char* dict = (const char*)dctx->dict; size_t dictSize = dctx->dictSize; int decodedSize; assert(dstPtr != NULL); if (dict && dictSize > 1 GB) { - /* the dictSize param is an int, avoid truncation / sign issues */ + /* overflow control : dctx->dictSize is an int, avoid truncation / sign issues */ dict += dictSize - 64 KB; dictSize = 64 KB; } - /* enough capacity in `dst` to decompress directly there */ decodedSize = LZ4_decompress_safe_usingDict( (const char*)selectedIn, (char*)dstPtr, (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, dict, (int)dictSize); - if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */ - if (dctx->frameInfo.contentChecksumFlag) + RETURN_ERROR_IF(decodedSize < 0, decompressionFailed); + if ((dctx->frameInfo.contentChecksumFlag) && (!dctx->skipChecksum)) XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize); if (dctx->frameInfo.contentSize) dctx->frameRemainingSize -= (size_t)decodedSize; @@ -1678,25 +1857,27 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } dstPtr += decodedSize; - dctx->dStage = dstage_getBlockHeader; + dctx->dStage = dstage_getBlockHeader; /* end of block, let's get another one */ break; } /* not enough place into dst : decode into tmpOut */ - /* ensure enough place for tmpOut */ + + /* manage dictionary */ if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { if (dctx->dict == dctx->tmpOutBuffer) { + /* truncate dictionary to 64 KB if too big */ if (dctx->dictSize > 128 KB) { memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB); dctx->dictSize = 64 KB; } dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize; - } else { /* dict not within tmp */ + } else { /* dict not within tmpOut */ size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB); dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace; } } - /* Decode block */ + /* Decode block into tmpOut */ { const char* dict = (const char*)dctx->dict; size_t dictSize = dctx->dictSize; int decodedSize; @@ -1709,9 +1890,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, (const char*)selectedIn, (char*)dctx->tmpOut, (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, dict, (int)dictSize); - if (decodedSize < 0) /* decompression failed */ - return err0r(LZ4F_ERROR_decompressionFailed); - if (dctx->frameInfo.contentChecksumFlag) + RETURN_ERROR_IF(decodedSize < 0, decompressionFailed); + if (dctx->frameInfo.contentChecksumFlag && !dctx->skipChecksum) XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize); if (dctx->frameInfo.contentSize) dctx->frameRemainingSize -= (size_t)decodedSize; @@ -1744,8 +1924,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; case dstage_getSuffix: - if (dctx->frameRemainingSize) - return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ + RETURN_ERROR_IF(dctx->frameRemainingSize, frameSize_wrong); /* incorrect frame size decoded */ if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */ nextSrcSizeHint = 0; LZ4F_resetDecompressionContext(dctx); @@ -1777,20 +1956,20 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } /* if (dctx->dStage == dstage_storeSuffix) */ /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */ - { U32 const readCRC = LZ4F_readLE32(selectedIn); + if (!dctx->skipChecksum) { + U32 const readCRC = LZ4F_readLE32(selectedIn); U32 const resultCRC = XXH32_digest(&(dctx->xxh)); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (readCRC != resultCRC) - return err0r(LZ4F_ERROR_contentChecksum_invalid); + RETURN_ERROR_IF(readCRC != resultCRC, contentChecksum_invalid); #else (void)readCRC; (void)resultCRC; #endif - nextSrcSizeHint = 0; - LZ4F_resetDecompressionContext(dctx); - doAnotherStage = 0; - break; } + nextSrcSizeHint = 0; + LZ4F_resetDecompressionContext(dctx); + doAnotherStage = 0; + break; case dstage_getSFrameSize: if ((srcEnd - srcPtr) >= 4) { @@ -1841,7 +2020,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } /* switch (dctx->dStage) */ } /* while (doAnotherStage) */ - /* preserve history within tmp whenever necessary */ + /* preserve history within tmpOut whenever necessary */ LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2); if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */ && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */ diff --git a/lz4libs/lz4frame.h b/lz4libs/lz4frame.h index 4573317e..1bdf6c4f 100644 --- a/lz4libs/lz4frame.h +++ b/lz4libs/lz4frame.h @@ -1,7 +1,7 @@ /* - LZ4 auto-framing library + LZ4F - LZ4-Frame library Header File - Copyright (C) 2011-2017, Yann Collet. + Copyright (C) 2011-2020, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -39,7 +39,7 @@ * LZ4F also offers streaming capabilities. * * lz4.h is not required when using lz4frame.h, - * except to extract common constant such as LZ4_VERSION_NUMBER. + * except to extract common constants such as LZ4_VERSION_NUMBER. * */ #ifndef LZ4F_H_09782039843 @@ -54,12 +54,12 @@ extern "C" { /** - Introduction - - lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md). - lz4frame.h provides frame compression functions that take care - of encoding standard metadata alongside LZ4-compressed blocks. -*/ + * Introduction + * + * lz4frame.h implements LZ4 frame specification: see doc/lz4_Frame_format.md . + * LZ4 Frames are compatible with `lz4` CLI, + * and designed to be interoperable with any system. +**/ /*-*************************************************************** * Compiler specifics @@ -210,7 +210,7 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void); /* v1.8.0+ */ * Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences. * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences. * Note : this result is only usable with LZ4F_compressFrame(). - * It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed. + * It may also be relevant to LZ4F_compressUpdate() _only if_ no flush() operation is ever performed. */ LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); @@ -230,7 +230,7 @@ LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, * Advanced compression functions *************************************/ typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */ -typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with previous API version */ +typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with older APIs, prefer using LZ4F_cctx */ typedef struct { unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */ @@ -243,20 +243,27 @@ typedef struct { LZ4FLIB_API unsigned LZ4F_getVersion(void); /*! LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. - * The function will provide a pointer to a fully allocated LZ4F_cctx object. - * If @return != zero, there was an error during context creation. - * Object can release its memory using LZ4F_freeCompressionContext(); - */ + * The first thing to do is to create a compressionContext object, + * which will keep track of operation state during streaming compression. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version, + * and a pointer to LZ4F_cctx*, to write the resulting pointer into. + * @version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. + * The function provides a pointer to a fully allocated LZ4F_cctx object. + * @cctxPtr MUST be != NULL. + * If @return != zero, context creation failed. + * A created compression context can be employed multiple times for consecutive streaming operations. + * Once all streaming compression jobs are completed, + * the state object can be released using LZ4F_freeCompressionContext(). + * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored. + * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing). +**/ LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /*---- Compression ----*/ -#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */ +#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected parameters */ #define LZ4F_HEADER_SIZE_MAX 19 /* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */ @@ -301,8 +308,9 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* * Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations. * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). - * LZ4F_compressUpdate() doesn't guarantee error recovery. - * When an error occurs, compression context must be freed or resized. + * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) @@ -347,8 +355,12 @@ typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */ typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */ typedef struct { - unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */ - unsigned reserved[3]; /* must be set to zero for forward compatibility */ + unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified between invocations. + * This optimization skips storage operations in tmp buffers. */ + unsigned skipChecksums; /* disable checksum calculation and verification, even when one is present in frame, to save CPU time. + * Setting this option to 1 once disables all checksums for the rest of the frame. */ + unsigned reserved1; /* must be set to zero for forward compatibility */ + unsigned reserved0; /* idem */ } LZ4F_decompressOptions_t; @@ -356,9 +368,10 @@ typedef struct { /*! LZ4F_createDecompressionContext() : * Create an LZ4F_dctx object, to track all decompression operations. - * The version provided MUST be LZ4F_VERSION. - * The function provides a pointer to an allocated and initialized LZ4F_dctx object. - * The result is an errorCode, which can be tested using LZ4F_isError(). + * @version provided MUST be LZ4F_VERSION. + * @dctxPtr MUST be valid. + * The function fills @dctxPtr with the value of a pointer to an allocated and initialized LZ4F_dctx object. + * The @return is an errorCode, which can be tested using LZ4F_isError(). * dctx memory can be released using LZ4F_freeDecompressionContext(); * Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released. * That is, it should be == 0 if decompression has been completed fully and correctly. @@ -371,6 +384,8 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); * Streaming decompression functions *************************************/ +#define LZ4F_MAGICNUMBER 0x184D2204U +#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U #define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5 /*! LZ4F_headerSize() : v1.9.0+ @@ -386,7 +401,7 @@ LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize); /*! LZ4F_getFrameInfo() : * This function extracts frame parameters (max blockSize, dictID, etc.). - * Its usage is optional: user can call LZ4F_decompress() directly. + * Its usage is optional: user can also invoke LZ4F_decompress() directly. * * Extracted information will fill an existing LZ4F_frameInfo_t structure. * This can be useful for allocation and dictionary identification purposes. @@ -427,9 +442,10 @@ LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize); * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. */ -LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, - LZ4F_frameInfo_t* frameInfoPtr, - const void* srcBuffer, size_t* srcSizePtr); +LZ4FLIB_API size_t +LZ4F_getFrameInfo(LZ4F_dctx* dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); /*! LZ4F_decompress() : * Call this function repetitively to regenerate data compressed in `srcBuffer`. @@ -462,10 +478,11 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, * * After a frame is fully decoded, dctx can be used again to decompress another frame. */ -LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const LZ4F_decompressOptions_t* dOptPtr); +LZ4FLIB_API size_t +LZ4F_decompress(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* dOptPtr); /*! LZ4F_resetDecompressionContext() : added in v1.8.0 @@ -529,6 +546,8 @@ extern "C" { ITEM(ERROR_headerChecksum_invalid) \ ITEM(ERROR_contentChecksum_invalid) \ ITEM(ERROR_frameDecoding_alreadyStarted) \ + ITEM(ERROR_compressionState_uninitialized) \ + ITEM(ERROR_parameter_null) \ ITEM(ERROR_maxCode) #define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, @@ -539,7 +558,31 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); -LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned); + +/*! LZ4F_getBlockSize() : + * Return, in scalar format (size_t), + * the maximum block size associated with blockSizeID. +**/ +LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID); + +/*! LZ4F_uncompressedUpdate() : + * LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary. + * Important rule: dstCapacity MUST be large enough to store the entire source buffer as + * no compression is done for this operation + * If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode). + * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously a compressed block was written, buffered data is flushed + * before appending uncompressed data is continued. + * This is only supported when LZ4F_blockIndependent is used + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. + * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ +LZ4FLIB_STATIC_API size_t +LZ4F_uncompressedUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); /********************************** * Bulk processing dictionary API @@ -583,12 +626,12 @@ LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict); * but it's not recommended, as it's the only way to provide dictID in the frame header. * @return : number of bytes written into dstBuffer. * or an error code if it fails (can be tested using LZ4F_isError()) */ -LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict( - LZ4F_cctx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* preferencesPtr); +LZ4FLIB_STATIC_API size_t +LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr); /*! LZ4F_compressBegin_usingCDict() : @@ -598,23 +641,49 @@ LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict( * however, it's the only way to provide dictID in the frame header. * @return : number of bytes written into dstBuffer for the header, * or an error code (which can be tested using LZ4F_isError()) */ -LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict( - LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* prefsPtr); +LZ4FLIB_STATIC_API size_t +LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* prefsPtr); /*! LZ4F_decompress_usingDict() : * Same as LZ4F_decompress(), using a predefined dictionary. * Dictionary is used "in place", without any preprocessing. - * It must remain accessible throughout the entire frame decoding. */ -LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict( - LZ4F_dctx* dctxPtr, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const void* dict, size_t dictSize, - const LZ4F_decompressOptions_t* decompressOptionsPtr); +** It must remain accessible throughout the entire frame decoding. */ +LZ4FLIB_STATIC_API size_t +LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass custom allocation/free functions. + * LZ4F_customMem is provided at state creation time, using LZ4F_create*_advanced() listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size); +typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size); +typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address); +typedef struct { + LZ4F_AllocFunction customAlloc; + LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */ + LZ4F_FreeFunction customFree; + void* opaqueState; +} LZ4F_CustomMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); +LZ4FLIB_STATIC_API LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); +LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict_advanced(LZ4F_CustomMem customMem, const void* dictBuffer, size_t dictSize); + #if defined (__cplusplus) } diff --git a/lz4libs/lz4frame_static.h b/lz4libs/lz4frame_static.h index 925a2c5c..2b44a631 100644 --- a/lz4libs/lz4frame_static.h +++ b/lz4libs/lz4frame_static.h @@ -1,7 +1,7 @@ /* LZ4 auto-framing library Header File for static linking only - Copyright (C) 2011-2016, Yann Collet. + Copyright (C) 2011-2020, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) diff --git a/lz4libs/lz4hc.c b/lz4libs/lz4hc.c index 77c9f430..b21ad6bb 100644 --- a/lz4libs/lz4hc.c +++ b/lz4libs/lz4hc.c @@ -1,6 +1,6 @@ /* LZ4 HC - High Compression Mode of LZ4 - Copyright (C) 2011-2017, Yann Collet. + Copyright (C) 2011-2020, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -42,7 +42,7 @@ * Select how default compression function will allocate workplace memory, * in stack (0:fastest), or in heap (1:requires malloc()). * Since workplace is rather large, heap mode is recommended. - */ +**/ #ifndef LZ4HC_HEAPMODE # define LZ4HC_HEAPMODE 1 #endif @@ -99,18 +99,20 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start) { - uptrval startingOffset = (uptrval)(hc4->end - hc4->base); - if (startingOffset > 1 GB) { + size_t const bufferSize = (size_t)(hc4->end - hc4->prefixStart); + size_t newStartingOffset = bufferSize + hc4->dictLimit; + assert(newStartingOffset >= bufferSize); /* check overflow */ + if (newStartingOffset > 1 GB) { LZ4HC_clearTables(hc4); - startingOffset = 0; + newStartingOffset = 0; } - startingOffset += 64 KB; - hc4->nextToUpdate = (U32) startingOffset; - hc4->base = start - startingOffset; + newStartingOffset += 64 KB; + hc4->nextToUpdate = (U32)newStartingOffset; + hc4->prefixStart = start; hc4->end = start; - hc4->dictBase = start - startingOffset; - hc4->dictLimit = (U32) startingOffset; - hc4->lowLimit = (U32) startingOffset; + hc4->dictStart = start; + hc4->dictLimit = (U32)newStartingOffset; + hc4->lowLimit = (U32)newStartingOffset; } @@ -119,12 +121,15 @@ LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) { U16* const chainTable = hc4->chainTable; U32* const hashTable = hc4->hashTable; - const BYTE* const base = hc4->base; - U32 const target = (U32)(ip - base); + const BYTE* const prefixPtr = hc4->prefixStart; + U32 const prefixIdx = hc4->dictLimit; + U32 const target = (U32)(ip - prefixPtr) + prefixIdx; U32 idx = hc4->nextToUpdate; + assert(ip >= prefixPtr); + assert(target >= prefixIdx); while (idx < target) { - U32 const h = LZ4HC_hashPtr(base+idx); + U32 const h = LZ4HC_hashPtr(prefixPtr+idx-prefixIdx); size_t delta = idx - hashTable[h]; if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX; DELTANEXTU16(chainTable, idx) = (U16)delta; @@ -193,15 +198,14 @@ LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) BYTE const byte = (BYTE)(pattern >> bitOffset); if (*ip != byte) break; ip ++; bitOffset -= 8; - } - } + } } return (unsigned)(ip - iStart); } /* LZ4HC_reverseCountPattern() : * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) - * read using natural platform endianess */ + * read using natural platform endianness */ static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) { @@ -211,7 +215,7 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) if (LZ4_read32(ip-4) != pattern) break; ip -= 4; } - { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */ + { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianness */ while (likely(ip>iLow)) { if (ip[-1] != *bytePtr) break; ip--; bytePtr--; @@ -234,28 +238,28 @@ typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( - LZ4HC_CCtx_internal* hc4, - const BYTE* const ip, - const BYTE* const iLowLimit, - const BYTE* const iHighLimit, - int longest, - const BYTE** matchpos, - const BYTE** startpos, - const int maxNbAttempts, - const int patternAnalysis, - const int chainSwap, - const dictCtx_directive dict, - const HCfavor_e favorDecSpeed) + LZ4HC_CCtx_internal* const hc4, + const BYTE* const ip, + const BYTE* const iLowLimit, const BYTE* const iHighLimit, + int longest, + const BYTE** matchpos, + const BYTE** startpos, + const int maxNbAttempts, + const int patternAnalysis, const int chainSwap, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx; - const BYTE* const base = hc4->base; - const U32 dictLimit = hc4->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 ipIndex = (U32)(ip - base); - const U32 lowestMatchIndex = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; - const BYTE* const dictBase = hc4->dictBase; + const BYTE* const prefixPtr = hc4->prefixStart; + const U32 prefixIdx = hc4->dictLimit; + const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx; + const int withinStartDistance = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex); + const U32 lowestMatchIndex = (withinStartDistance) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; + const BYTE* const dictStart = hc4->dictStart; + const U32 dictIdx = hc4->lowLimit; + const BYTE* const dictEnd = dictStart + prefixIdx - dictIdx; int const lookBackLength = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; U32 matchChainPos = 0; @@ -277,14 +281,13 @@ LZ4HC_InsertAndGetWiderMatch ( assert(matchIndex < ipIndex); if (favorDecSpeed && (ipIndex - matchIndex < 8)) { /* do nothing */ - } else if (matchIndex >= dictLimit) { /* within current Prefix */ - const BYTE* const matchPtr = base + matchIndex; - assert(matchPtr >= lowPrefixPtr); + } else if (matchIndex >= prefixIdx) { /* within current Prefix */ + const BYTE* const matchPtr = prefixPtr + matchIndex - prefixIdx; assert(matchPtr < ip); assert(longest >= 1); if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) { if (LZ4_read32(matchPtr) == pattern) { - int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0; + int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, prefixPtr) : 0; matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); matchLength -= back; if (matchLength > longest) { @@ -293,24 +296,25 @@ LZ4HC_InsertAndGetWiderMatch ( *startpos = ip + back; } } } } else { /* lowestMatchIndex <= matchIndex < dictLimit */ - const BYTE* const matchPtr = dictBase + matchIndex; - if (LZ4_read32(matchPtr) == pattern) { - const BYTE* const dictStart = dictBase + hc4->lowLimit; + const BYTE* const matchPtr = dictStart + (matchIndex - dictIdx); + assert(matchIndex >= dictIdx); + if ( likely(matchIndex <= prefixIdx - 4) + && (LZ4_read32(matchPtr) == pattern) ) { int back = 0; - const BYTE* vLimit = ip + (dictLimit - matchIndex); + const BYTE* vLimit = ip + (prefixIdx - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) - matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit); + matchLength += LZ4_count(ip+matchLength, prefixPtr, iHighLimit); back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0; matchLength -= back; if (matchLength > longest) { longest = matchLength; - *matchpos = base + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */ + *matchpos = prefixPtr - prefixIdx + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */ *startpos = ip + back; } } } - if (chainSwap && matchLength==longest) { /* better match => select a better chain */ + if (chainSwap && matchLength==longest) { /* better match => select a better chain */ assert(lookBackLength==0); /* search forward only */ if (matchIndex + (U32)longest <= ipIndex) { int const kTrigger = 4; @@ -326,8 +330,7 @@ LZ4HC_InsertAndGetWiderMatch ( distanceToNextMatch = candidateDist; matchChainPos = (U32)pos; accel = 1 << kTrigger; - } - } + } } if (distanceToNextMatch > 1) { if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ matchIndex -= distanceToNextMatch; @@ -347,23 +350,24 @@ LZ4HC_InsertAndGetWiderMatch ( repeat = rep_not; } } if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) - && LZ4HC_protectDictEnd(dictLimit, matchCandidateIdx) ) { - const int extDict = matchCandidateIdx < dictLimit; - const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx; + && LZ4HC_protectDictEnd(prefixIdx, matchCandidateIdx) ) { + const int extDict = matchCandidateIdx < prefixIdx; + const BYTE* const matchPtr = (extDict ? dictStart - dictIdx : prefixPtr - prefixIdx) + matchCandidateIdx; if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ - const BYTE* const dictStart = dictBase + hc4->lowLimit; - const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit; + const BYTE* const iLimit = extDict ? dictEnd : iHighLimit; size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); if (extDict && matchPtr + forwardPatternLength == iLimit) { U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern); - forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern); + forwardPatternLength += LZ4HC_countPattern(prefixPtr, iHighLimit, rotatedPattern); } - { const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr; + { const BYTE* const lowestMatchPtr = extDict ? dictStart : prefixPtr; size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); size_t currentSegmentLength; - if (!extDict && matchPtr - backLength == lowPrefixPtr && hc4->lowLimit < dictLimit) { + if (!extDict + && matchPtr - backLength == prefixPtr + && dictIdx < prefixIdx) { U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern); - backLength += LZ4HC_reverseCountPattern(dictBase + dictLimit, dictStart, rotatedPattern); + backLength += LZ4HC_reverseCountPattern(dictEnd, dictStart, rotatedPattern); } /* Limit backLength not go further than lowestMatchIndex */ backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex); @@ -373,28 +377,28 @@ LZ4HC_InsertAndGetWiderMatch ( if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ - if (LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) + if (LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) matchIndex = newMatchIndex; else { /* Can only happen if started in the prefix */ - assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict); - matchIndex = dictLimit; + assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); + matchIndex = prefixIdx; } } else { U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ - if (!LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) { - assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict); - matchIndex = dictLimit; + if (!LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) { + assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); + matchIndex = prefixIdx; } else { matchIndex = newMatchIndex; if (lookBackLength==0) { /* no back possible */ size_t const maxML = MIN(currentSegmentLength, srcPatternLength); if ((size_t)longest < maxML) { - assert(base + matchIndex != ip); - if ((size_t)(ip - base) - matchIndex > LZ4_DISTANCE_MAX) break; + assert(prefixPtr - prefixIdx + matchIndex != ip); + if ((size_t)(ip - prefixPtr) + prefixIdx - matchIndex > LZ4_DISTANCE_MAX) break; assert(maxML < 2 GB); longest = (int)maxML; - *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ + *matchpos = prefixPtr - prefixIdx + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ *startpos = ip; } { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); @@ -413,12 +417,12 @@ LZ4HC_InsertAndGetWiderMatch ( if ( dict == usingDictCtxHc && nbAttempts > 0 && ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) { - size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->base); + size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit; U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; assert(dictEndOffset <= 1 GB); matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { - const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; + const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + dictMatchIndex; if (LZ4_read32(matchPtr) == pattern) { int mlt; @@ -426,11 +430,11 @@ LZ4HC_InsertAndGetWiderMatch ( const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; - back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0; + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->prefixStart) : 0; mlt -= back; if (mlt > longest) { longest = mlt; - *matchpos = base + matchIndex + back; + *matchpos = prefixPtr - prefixIdx + matchIndex + back; *startpos = ip + back; } } @@ -442,13 +446,13 @@ LZ4HC_InsertAndGetWiderMatch ( return longest; } -LZ4_FORCE_INLINE -int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ - const BYTE* const ip, const BYTE* const iLimit, - const BYTE** matchpos, - const int maxNbAttempts, - const int patternAnalysis, - const dictCtx_directive dict) +LZ4_FORCE_INLINE int +LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + const BYTE** matchpos, + const int maxNbAttempts, + const int patternAnalysis, + const dictCtx_directive dict) { const BYTE* uselessPtr = ip; /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), @@ -751,7 +755,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( } else { *op++ = (BYTE)(lastRunSize << ML_BITS); } - memcpy(op, anchor, lastRunSize); + LZ4_memcpy(op, anchor, lastRunSize); op += lastRunSize; } @@ -884,13 +888,13 @@ LZ4HC_compress_generic_dictCtx ( limitedOutput_directive limit ) { - const size_t position = (size_t)(ctx->end - ctx->base) - ctx->lowLimit; + const size_t position = (size_t)(ctx->end - ctx->prefixStart) + (ctx->dictLimit - ctx->lowLimit); assert(ctx->dictCtx != NULL); if (position >= 64 KB) { ctx->dictCtx = NULL; return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); } else if (position == 0 && *srcSizePtr > 4 KB) { - memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); + LZ4_memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); LZ4HC_setExternalDict(ctx, (const BYTE *)src); ctx->compressionLevel = (short)cLevel; return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); @@ -953,13 +957,15 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { + int cSize; #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); + if (statePtr==NULL) return 0; #else LZ4_streamHC_t state; LZ4_streamHC_t* const statePtr = &state; #endif - int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); + cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 FREEMEM(statePtr); #endif @@ -982,6 +988,7 @@ int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* s * Streaming Functions **************************************/ /* allocation */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) LZ4_streamHC_t* LZ4_createStreamHC(void) { LZ4_streamHC_t* const state = @@ -998,13 +1005,12 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) FREEMEM(LZ4_streamHCPtr); return 0; } +#endif LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) { LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer; - /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ - LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE); DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", buffer, (unsigned)size); /* check conditions */ if (buffer == NULL) return NULL; @@ -1030,9 +1036,13 @@ void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev if (LZ4_streamHCPtr->internal_donotuse.dirty) { LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); } else { - /* preserve end - base : can trigger clearTable's threshold */ - LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; - LZ4_streamHCPtr->internal_donotuse.base = NULL; + /* preserve end - prefixStart : can trigger clearTable's threshold */ + if (LZ4_streamHCPtr->internal_donotuse.end != NULL) { + LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.prefixStart; + } else { + assert(LZ4_streamHCPtr->internal_donotuse.prefixStart == NULL); + } + LZ4_streamHCPtr->internal_donotuse.prefixStart = NULL; LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; } LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); @@ -1083,14 +1093,14 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); - if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4) + if (ctxPtr->end >= ctxPtr->prefixStart + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->dictStart = ctxPtr->prefixStart; + ctxPtr->dictLimit += (U32)(ctxPtr->end - ctxPtr->prefixStart); + ctxPtr->prefixStart = newBlock; ctxPtr->end = newBlock; ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ @@ -1109,11 +1119,11 @@ LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, LZ4_streamHCPtr, src, *srcSizePtr, limit); assert(ctxPtr != NULL); /* auto-init if forgotten */ - if (ctxPtr->base == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src); + if (ctxPtr->prefixStart == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src); /* Check overflow */ - if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { - size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; + if ((size_t)(ctxPtr->end - ctxPtr->prefixStart) + ctxPtr->dictLimit > 2 GB) { + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->prefixStart); if (dictSize > 64 KB) dictSize = 64 KB; LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); } @@ -1124,13 +1134,16 @@ LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr; - const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; - const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; + const BYTE* const dictBegin = ctxPtr->dictStart; + const BYTE* const dictEnd = ctxPtr->dictStart + (ctxPtr->dictLimit - ctxPtr->lowLimit); if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) { if (sourceEnd > dictEnd) sourceEnd = dictEnd; - ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); - if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; - } } + ctxPtr->lowLimit += (U32)(sourceEnd - ctxPtr->dictStart); + ctxPtr->dictStart += (U32)(sourceEnd - ctxPtr->dictStart); + if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) { + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictStart = ctxPtr->prefixStart; + } } } return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit); } @@ -1158,7 +1171,7 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) { LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; - int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); + int const prefixSize = (int)(streamPtr->end - streamPtr->prefixStart); DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); assert(prefixSize >= 0); if (dictSize > 64 KB) dictSize = 64 KB; @@ -1166,12 +1179,13 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS if (dictSize > prefixSize) dictSize = prefixSize; if (safeBuffer == NULL) assert(dictSize == 0); if (dictSize > 0) - memmove(safeBuffer, streamPtr->end - dictSize, dictSize); - { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base); + LZ4_memmove(safeBuffer, streamPtr->end - dictSize, dictSize); + { U32 const endIndex = (U32)(streamPtr->end - streamPtr->prefixStart) + streamPtr->dictLimit; streamPtr->end = (const BYTE*)safeBuffer + dictSize; - streamPtr->base = streamPtr->end - endIndex; + streamPtr->prefixStart = streamPtr->end - dictSize; streamPtr->dictLimit = endIndex - (U32)dictSize; streamPtr->lowLimit = endIndex - (U32)dictSize; + streamPtr->dictStart = streamPtr->prefixStart; if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; } @@ -1199,7 +1213,7 @@ int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, /* Deprecated streaming functions */ -int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } +int LZ4_sizeofStreamStateHC(void) { return sizeof(LZ4_streamHC_t); } /* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t) * @return : 0 on success, !=0 if error */ @@ -1211,6 +1225,7 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer) return 0; } +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) void* LZ4_createHC (const char* inputBuffer) { LZ4_streamHC_t* const hc4 = LZ4_createStreamHC(); @@ -1225,6 +1240,7 @@ int LZ4_freeHC (void* LZ4HC_Data) FREEMEM(LZ4HC_Data); return 0; } +#endif int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) { @@ -1238,11 +1254,11 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, c char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { - LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data; - const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit; + LZ4_streamHC_t* const ctx = (LZ4_streamHC_t*)LZ4HC_Data; + const BYTE* bufferStart = ctx->internal_donotuse.prefixStart - ctx->internal_donotuse.dictLimit + ctx->internal_donotuse.lowLimit; LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel); /* avoid const char * -> char * conversion warning :( */ - return (char *)(uptrval)bufferStart; + return (char*)(uptrval)bufferStart; } @@ -1325,7 +1341,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, { int retval = 0; #define TRAILING_LITERALS 3 -#ifdef LZ4HC_HEAPMODE +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS)); #else LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */ @@ -1343,7 +1359,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, const BYTE* ovref = NULL; /* init */ -#ifdef LZ4HC_HEAPMODE +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 if (opt == NULL) goto _return_label; #endif DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity); @@ -1575,7 +1591,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, } else { *op++ = (BYTE)(lastRunSize << ML_BITS); } - memcpy(op, anchor, lastRunSize); + LZ4_memcpy(op, anchor, lastRunSize); op += lastRunSize; } @@ -1608,7 +1624,7 @@ if (limit == fillOutput) { goto _last_literals; } _return_label: -#ifdef LZ4HC_HEAPMODE +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 FREEMEM(opt); #endif return retval; diff --git a/lz4libs/lz4hc.h b/lz4libs/lz4hc.h index 3d441fb6..e937acfe 100644 --- a/lz4libs/lz4hc.h +++ b/lz4libs/lz4hc.h @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2017, Yann Collet. + Copyright (C) 2011-2020, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -198,14 +198,17 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) +/* Never ever use these definitions directly ! + * Declare or allocate an LZ4_streamHC_t instead. +**/ typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; struct LZ4HC_CCtx_internal { LZ4_u32 hashTable[LZ4HC_HASHTABLESIZE]; LZ4_u16 chainTable[LZ4HC_MAXD]; const LZ4_byte* end; /* next block here to continue on current prefix */ - const LZ4_byte* base; /* All index relative to this position */ - const LZ4_byte* dictBase; /* alternate base for extDict */ + const LZ4_byte* prefixStart; /* Indexes relative to this position */ + const LZ4_byte* dictStart; /* alternate reference for extDict */ LZ4_u32 dictLimit; /* below that point, need extDict */ LZ4_u32 lowLimit; /* below that point, no more dict */ LZ4_u32 nextToUpdate; /* index from which to continue dictionary update */ @@ -216,20 +219,15 @@ struct LZ4HC_CCtx_internal const LZ4HC_CCtx_internal* dictCtx; }; - -/* Do not use these definitions directly ! - * Declare or allocate an LZ4_streamHC_t instead. - */ -#define LZ4_STREAMHCSIZE 262200 /* static size, for inter-version compatibility */ -#define LZ4_STREAMHCSIZE_VOIDP (LZ4_STREAMHCSIZE / sizeof(void*)) +#define LZ4_STREAMHC_MINSIZE 262200 /* static size, for inter-version compatibility */ union LZ4_streamHC_u { - void* table[LZ4_STREAMHCSIZE_VOIDP]; + char minStateSize[LZ4_STREAMHC_MINSIZE]; LZ4HC_CCtx_internal internal_donotuse; }; /* previously typedef'd to LZ4_streamHC_t */ /* LZ4_streamHC_t : * This structure allows static allocation of LZ4 HC streaming state. - * This can be used to allocate statically, on state, or as part of a larger structure. + * This can be used to allocate statically on stack, or as part of a larger structure. * * Such state **must** be initialized using LZ4_initStreamHC() before first use. * @@ -244,7 +242,7 @@ union LZ4_streamHC_u { * Required before first use of a statically allocated LZ4_streamHC_t. * Before v1.9.0 : use LZ4_resetStreamHC() instead */ -LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size); +LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC(void* buffer, size_t size); /*-************************************ @@ -272,9 +270,11 @@ LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_comp * LZ4_slideInputBufferHC() will truncate the history of the stream, rather * than preserve a window-sized chunk of history. */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer); -LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data); +#endif +LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int LZ4_sizeofStreamStateHC(void); @@ -305,7 +305,7 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL * They should not be linked from DLL, * as there is no guarantee of API stability yet. * Prototypes will be promoted to "stable" status - * after successfull usage in real-life scenarios. + * after successful usage in real-life scenarios. ***************************************************/ #ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */ #ifndef LZ4_HC_SLO_098092834 From 06b6a58a748b9d73f891c344b97589a18dd67c3e Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 29 Dec 2022 09:30:49 +0000 Subject: [PATCH 218/280] Fix the use of package envs in tox --- tox.ini | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tox.ini b/tox.ini index 5a30280b..6c52f84a 100644 --- a/tox.ini +++ b/tox.ini @@ -25,8 +25,7 @@ commands = ignore = E501 [testenv:docs] -setenv = - PYLZ4_EXPERIMENTAL = 1 +package_env = .pkg-experimental passenv = * usedevelop = True allowlist_externals = make @@ -34,20 +33,18 @@ extras = docs commands = make -C docs doctest html -[testenv:.pkg] -# This section is needed to pass PYLZ4_EXPERIMENTAL into the -# environment used to build the lz4 package ahead of creating the tox -# virtual environment. Ideally we would like to be able to set this -# only for docs building, but at the moment (tox 4.0.18) tox does not -# allow that. See: https://github.com/tox-dev/tox/issues/2784 +[testenv:.pkg-experimental] +# This environment is used when building the lz4 package (wheel) that +# is subsequently installed into the tox environment for any +# environment referencing this one via package_env. +# https://tox.wiki/en/latest/config.html#package_env-env setenv = PYLZ4_EXPERIMENTAL = 1 [testenv:experimental] extras = tests passenv = * -setenv = - PYLZ4_EXPERIMENTAL = 1 +package_env = .pkg-experimental # PYTHONMALLOC = pymalloc # PYTHONMALLOCSTATS = 'yes' usedevelop = True From 305461ee816a6ebe39cfcda9121a62c2ffd098b2 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 29 Dec 2022 11:24:27 +0000 Subject: [PATCH 219/280] Fix test_block_checksum_failure() (#254) --- tests/frame/test_frame_3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/frame/test_frame_3.py b/tests/frame/test_frame_3.py index a7835a46..c0b0028e 100644 --- a/tests/frame/test_frame_3.py +++ b/tests/frame/test_frame_3.py @@ -53,5 +53,5 @@ def test_block_checksum_failure(data): message = r'^LZ4F_decompress failed with code: ERROR_blockChecksum_invalid$' if len(compressed) > 32: with pytest.raises(RuntimeError, match=message): - compressed[22] = compressed[18] ^ 0x42 + compressed[22] = compressed[22] ^ 0x42 lz4frame.decompress(compressed) From cae2a1585b3ac4e75088f8cde54a6f5c051e9996 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 29 Dec 2022 19:34:34 +0000 Subject: [PATCH 220/280] Use cibuildwheel to generate arm64 and aarch64 wheels --- .github/workflows/build_dist.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 6adb1928..5a06b300 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -36,8 +36,6 @@ jobs: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*] - cibw_archs_linux: [auto, ] # aarch64] - cibw_skip: ["cp*-musllinux*", ] steps: - name: Check out repository uses: actions/checkout@v2 @@ -53,12 +51,15 @@ jobs: with: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.11.2 + uses: pypa/cibuildwheel@v2.11.4 env: - CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} + CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" + CIBW_ARCHS_MACOS: "x86_64 arm64 universal2" + CIBW_ARCHS_WINDOWS: "AMD64 x86" CIBW_BUILD: ${{ matrix.cibw_build }} - CIBW_SKIP: ${{ matrix.cibw_skip }} + CIBW_SKIP: "cp*-musllinux*" CIBW_TEST_COMMAND: "tox -c {project}" + CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{aarch64,ppc64le,s390x}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v2 From c8b526ba4d2ecbaa99e99eea11ad151cebe9210c Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 29 Dec 2022 19:40:16 +0000 Subject: [PATCH 221/280] Clean up github actions job name --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 5a06b300..95d8173e 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -30,7 +30,7 @@ jobs: path: dist/*.tar.gz wheels: - name: Build wheels on ${{ matrix.os }} CIBW_BUILD=${{ matrix.cibw_build }} CIBW_ARCHS_LINUX=${{ matrix.cibw_archs_linux }} + name: Build wheels on ${{ matrix.os }} CIBW_BUILD=${{ matrix.cibw_build }} runs-on: ${{ matrix.os }} strategy: matrix: From ecc325562e2fa3f25174602bc9561008207c5748 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 29 Dec 2022 19:53:39 +0000 Subject: [PATCH 222/280] Disable universal2 wheel building on MacOS This fails at present. See: https://github.com/pypa/cibuildwheel/issues/1385 --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 95d8173e..31f6c3db 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -54,7 +54,7 @@ jobs: uses: pypa/cibuildwheel@v2.11.4 env: CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" - CIBW_ARCHS_MACOS: "x86_64 arm64 universal2" + CIBW_ARCHS_MACOS: "x86_64 arm64" # universal2" CIBW_ARCHS_WINDOWS: "AMD64 x86" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" From 0906b2058238b9c3909b34861ed661595a5f58b2 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 29 Dec 2022 20:32:52 +0000 Subject: [PATCH 223/280] Fix QEMU installation on github actions --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 31f6c3db..b8b349e6 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -46,7 +46,7 @@ jobs: with: python-version: 3.x - name: Set up QEMU # Needed to build aarch64 wheels - if: runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64' + if: runner.os == 'Linux' uses: docker/setup-qemu-action@v1 with: platforms: all From db26ca10d5e173bb96d512705631f328abe749e1 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 29 Dec 2022 20:54:28 +0000 Subject: [PATCH 224/280] Update versions of github actions to most recent stable --- .github/workflows/build_dist.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index b8b349e6..6f8838a5 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -38,16 +38,16 @@ jobs: cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*] steps: - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work - name: Install Python 3.x - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.x - name: Set up QEMU # Needed to build aarch64 wheels if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 with: platforms: all - name: Build wheels @@ -72,11 +72,11 @@ jobs: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: artifact path: dist - - uses: pypa/gh-action-pypi-publish@master + - uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} skip_existing: true From c7c2708c31160d86da344464c353ab56bb56a639 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 29 Dec 2022 21:18:47 +0000 Subject: [PATCH 225/280] Update some more github action versions --- .github/workflows/build_dist.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 6f8838a5..2c93cdda 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -15,17 +15,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work - name: Install Python 3.x - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.x - name: Build sdist run: python setup.py sdist - name: Save sdist - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: path: dist/*.tar.gz From 076150fb6caf9fa361a8b63c2239d153f48b8d1c Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Fri, 30 Dec 2022 00:14:24 +0000 Subject: [PATCH 226/280] Add github actions status badge to README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 15ba4445..b9dcfa79 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,8 @@ python-lz4 Status ====== -.. image:: https://travis-ci.org/python-lz4/python-lz4.svg?branch=master - :target: https://travis-ci.org/python-lz4/python-lz4 +.. image:: https://github.com/python-lz4/python-lz4/actions/workflows/build_dist.yml/badge.svg + :target: https://github.com/python-lz4/python-lz4/actions/workflows/build_dist.yml :alt: Build Status .. image:: https://ci.appveyor.com/api/projects/status/r2qvw9mlfo63lklo/branch/master?svg=true From 8a4aa23aca1255de5e29fc09994b0a8cd7c404a5 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Fri, 30 Dec 2022 00:29:53 +0000 Subject: [PATCH 227/280] Remove AppVeyor badge from README --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index b9dcfa79..a75bc4d4 100644 --- a/README.rst +++ b/README.rst @@ -9,10 +9,6 @@ Status :target: https://github.com/python-lz4/python-lz4/actions/workflows/build_dist.yml :alt: Build Status -.. image:: https://ci.appveyor.com/api/projects/status/r2qvw9mlfo63lklo/branch/master?svg=true - :target: https://ci.appveyor.com/project/jonathanunderwood/python-lz4 - :alt: Build Status Windows - .. image:: https://readthedocs.org/projects/python-lz4/badge/?version=stable :target: https://readthedocs.org/projects/python-lz4/ :alt: Documentation From d47235998c3f54d355646b61e7ff19a182f01ac5 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Fri, 30 Dec 2022 20:40:20 +0000 Subject: [PATCH 228/280] Remove dependency on distutils (#236) --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2f2704fb..b499e01a 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ #!/usr/bin/env python import os from setuptools import setup, find_packages, Extension +from setuptools.command.build_ext import new_compiler import sys -from distutils import ccompiler + # Note: if updating LZ4_REQUIRED_VERSION you need to update docs/install.rst as # well. @@ -97,7 +98,7 @@ def pkgconfig_installed_check(lib, required_version, default): ] ) -compiler = ccompiler.get_default_compiler() +compiler = new_compiler().compiler_type if compiler == 'msvc': extension_kwargs['extra_compile_args'] = [ From acd7557dd837966961e02c3a958d04f700e3d8b8 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Fri, 30 Dec 2022 21:39:49 +0000 Subject: [PATCH 229/280] Add env var to control building against system LZ4 --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b499e01a..32f3434e 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,13 @@ def pkgconfig_installed_check(lib, required_version, default): 'lz4/stream/_stream.c' ] -if liblz4_found is True: +use_system_liblz4_env = os.environ.get("PYLZ4_USE_SYSTEM_LZ4", "True") +if use_system_liblz4_env.upper() in ("1", "TRUE"): + use_system_liblz4 = True +else: + use_system_liblz4 = False + +if liblz4_found is True and use_system_liblz4 is True: extension_kwargs['libraries'] = ['lz4'] else: extension_kwargs['include_dirs'] = ['lz4libs'] From 9a1d7c2ef09b01c00db8da31fc36c0ca73e59a14 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Fri, 30 Dec 2022 21:40:57 +0000 Subject: [PATCH 230/280] Force using bundled LZ4 when building wheels --- .github/workflows/build_dist.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 2c93cdda..c02e25e8 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -53,6 +53,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.11.4 env: + PYLZ4_USE_SYSTEM_LZ4: "False" CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_MACOS: "x86_64 arm64" # universal2" CIBW_ARCHS_WINDOWS: "AMD64 x86" From 783967171cbfa6c29807cf21b9d64fbb9e86cbab Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Fri, 30 Dec 2022 21:42:27 +0000 Subject: [PATCH 231/280] Clean up handling of PYLZ4_EXPERIMENTAL --- setup.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 32f3434e..f793adf3 100644 --- a/setup.py +++ b/setup.py @@ -36,12 +36,11 @@ def pkgconfig_installed_check(lib, required_version, default): liblz4_found = pkgconfig_installed_check('liblz4', LZ4_REQUIRED_VERSION, default=False) # Establish if we want to build experimental functionality or not. -experimental = os.environ.get("PYLZ4_EXPERIMENTAL", False) -if experimental is not False: - if experimental.upper() in ("1", "TRUE"): - experimental = True - else: - experimental = False +experimental_env = os.environ.get("PYLZ4_EXPERIMENTAL", "False") +if experimental_env.upper() in ("1", "TRUE"): + experimental = True +else: + experimental = False # Set up the extension modules. If a system wide lz4 library is found, and is # recent enough, we'll use that. Otherwise we'll build with the bundled one. If From d1a071add32e9140324f2729c783211471ecdfd9 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Fri, 30 Dec 2022 21:59:46 +0000 Subject: [PATCH 232/280] Ensure PYLZ4_USE_SYSTEM_LZ4 env var passed to cibuildwheel --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index c02e25e8..6fd9b2f2 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -53,7 +53,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.11.4 env: - PYLZ4_USE_SYSTEM_LZ4: "False" + CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_MACOS: "x86_64 arm64" # universal2" CIBW_ARCHS_WINDOWS: "AMD64 x86" From b1b3afd4dc37816c1a93a471e89761f5256d45db Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Fri, 30 Dec 2022 22:16:46 +0000 Subject: [PATCH 233/280] Don't link against system lz4 if PYLZ4_USE_SYSTEM_LZ4 is not True --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f793adf3..387f5c8a 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ def pkgconfig_installed_check(lib, required_version, default): '/wd4820', ] elif compiler in ('unix', 'mingw32'): - if liblz4_found: + if liblz4_found is True and use_system_liblz4 is True: extension_kwargs = pkgconfig_parse('liblz4') else: extension_kwargs['extra_compile_args'] = [ From 231b29c2776128c1a8c8ab5d5cec765c00615966 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 31 Dec 2022 00:26:27 +0000 Subject: [PATCH 234/280] Add .readthedocs.yml --- .readthedocs.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..1ad8c58f --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,25 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-20.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# If using Sphinx, optionally build your docs in additional formats such as PDF +formats: + - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt From 837d0d68d1b97a91a432aecf0e983f8fe8d60ba3 Mon Sep 17 00:00:00 2001 From: David Evans Date: Thu, 21 Dec 2023 08:30:22 +0000 Subject: [PATCH 235/280] Use `importlib.metadata` over `pkg_resources` This is "The Modern Way" and is requoired for Python 3.12. https://docs.python.org/3.12/library/importlib.metadata.html As `importlib.metadata` was added in Python 3.8, and as we still support Python 3.7, we add a fallback import for older versions. --- docs/conf.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 51b6576f..aefff88e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,8 +55,13 @@ # |version| and |release|, also used in various other places throughout the # built documents. # -from pkg_resources import get_distribution -release = get_distribution('lz4').version +try: + import importlib.metadata +except ImportError: + from pkg_resources import get_distribution + release = get_distribution('lz4').version +else: + release = importlib.metadata.version('lz4') version = release # The language for content autogenerated by Sphinx. Refer to documentation From d41a6760eeed3800edf02ad862c976b5daedfd70 Mon Sep 17 00:00:00 2001 From: David Evans Date: Thu, 21 Dec 2023 08:09:25 +0000 Subject: [PATCH 236/280] Add Python 3.12 to the build matrix This required updating to the latest version of `pypa/cibuildwheel`. --- .github/workflows/build_dist.yml | 4 ++-- setup.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 6fd9b2f2..19999af7 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -35,7 +35,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] - cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*] + cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*, cp312-*] steps: - name: Check out repository uses: actions/checkout@v3 @@ -51,7 +51,7 @@ jobs: with: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.11.4 + uses: pypa/cibuildwheel@v2.16.2 env: CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" diff --git a/setup.py b/setup.py index 387f5c8a..2fbd2cac 100644 --- a/setup.py +++ b/setup.py @@ -205,5 +205,6 @@ def pkgconfig_installed_check(lib, required_version, default): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], ) From b8844b1443b449302dc973e49e895c0fe0d84374 Mon Sep 17 00:00:00 2001 From: David Evans Date: Thu, 21 Dec 2023 13:53:16 +0000 Subject: [PATCH 237/280] Add explicit dependency on `setuptools` --- .github/workflows/build_dist.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 19999af7..682b69d3 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -22,6 +22,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.x + - name: Install setuptools + run: pip install setuptools - name: Build sdist run: python setup.py sdist - name: Save sdist From 7182e7d0bd7d70db65cc4f32cf0462c7fb71f5cb Mon Sep 17 00:00:00 2001 From: David Evans Date: Thu, 21 Dec 2023 14:11:30 +0000 Subject: [PATCH 238/280] Drop support for Python 3.7 This reached end-of-life in June 2023: https://devguide.python.org/versions/#unsupported-versions And attempting to build the wheel fails with an error: ``` docs: install_package_deps /project> python -I -m pip install sphinx-bootstrap-theme 'sphinx>=1.6.0' docs: install_package /project> python -I -m pip install --force-reinstall --no-deps /project/.tox/.tmp/package/3/lz4-0.1.dev1180+g91fe6ef-0.editable-cp37-cp37m-linux_x86_64.whl docs: commands[0] /project> make -C docs doctest html make: Entering directory `/project/docs' sphinx-build -b doctest -d _build/doctrees . _build/doctest Running Sphinx v5.3.0 Extension error: Could not import extension sphinx.builders.linkcheck (exception: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'OpenSSL 1.0.2k-fips 26 Jan 2017'. See: https://github.com/urllib3/urllib3/issues/2168) ``` --- .github/workflows/build_dist.yml | 2 +- docs/install.rst | 2 +- setup.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 682b69d3..fe70b794 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -37,7 +37,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] - cibw_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*, cp312-*] + cibw_build: [cp38-*, cp39-*, cp310-*, cp311-*, cp312-*] steps: - name: Check out repository uses: actions/checkout@v3 diff --git a/docs/install.rst b/docs/install.rst index 3d531f7b..664ffd3b 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,7 +22,7 @@ The LZ4 bindings require linking to the LZ4 library, and so if there is not a pre-compiled wheel available for your platform you will need to have a suitable C compiler available, as well as the Python development header files. On Debian/Ubuntu based systems the header files for Python are found in the -distribution package ``pythonX.Y-dev`` e.g. ``python3.7-dev``. On Fedora/Red Hat +distribution package ``pythonX.Y-dev`` e.g. ``python3.8-dev``. On Fedora/Red Hat based systems, the Python header files are found in the distribution package ``python-devel``. diff --git a/setup.py b/setup.py index 2fbd2cac..f3a226d2 100644 --- a/setup.py +++ b/setup.py @@ -171,7 +171,7 @@ def pkgconfig_installed_check(lib, required_version, default): use_scm_version={ 'write_to': "lz4/version.py", }, - python_requires=">=3.7", + python_requires=">=3.8", setup_requires=[ 'setuptools_scm', 'pkgconfig', @@ -200,7 +200,6 @@ def pkgconfig_installed_check(lib, required_version, default): 'Intended Audience :: Developers', 'Programming Language :: C', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', From a71746a0f210dd54e9b6ef94f558e896e218616a Mon Sep 17 00:00:00 2001 From: Peter Feerick Date: Sun, 7 Jan 2024 13:09:48 +1000 Subject: [PATCH 239/280] fix(doc): Install command missing a parameter The `--no-binary` option needs a parameter, as indicated in the pip documentation here https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-no-binary --- docs/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.rst b/docs/install.rst index 664ffd3b..f6eeafed 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -40,7 +40,7 @@ this command will result in the extension modules being compiled from source:: On systems for which pre-built wheels are available, the following command will force a local compilation of the extension modules from source:: - $ pip install --no-binary --no-cache-dir lz4 + $ pip install --no-binary :all: --no-cache-dir lz4 The package can also be installed manually from a checkout of the source code git repository:: From 4929607b62e9275368677b73f4372d0101f400b7 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Thu, 7 Mar 2024 09:34:04 +0100 Subject: [PATCH 240/280] docs/lz4.block.rst: fix typo --- docs/lz4.block.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lz4.block.rst b/docs/lz4.block.rst index 0b2ef103..5c7f0c78 100644 --- a/docs/lz4.block.rst +++ b/docs/lz4.block.rst @@ -84,7 +84,7 @@ can be used in this case. True In this example we are catching the `lz4.block.LZ4BlockError` -exception. This exception is raisedd if the LZ4 library call fails, +exception. This exception is raised if the LZ4 library call fails, which can be caused by either the buffer used to store the uncompressed data (as set by `usize`) being too small, or the input compressed data being invalid - it is not possible to distinguish the From d2248984f30156726423ab1b31790fc16e031c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Duarte?= Date: Wed, 18 Sep 2024 17:10:31 +0000 Subject: [PATCH 241/280] Update CI - Updates all GH Actions to their latest versions - Removes Python 3.8 (EOL) - Adds Python 3.13 - Stops fail-fast to avoid wasting builds for a single failure --- .github/workflows/build_dist.yml | 70 +++++++++++++++++++------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index fe70b794..b38968dd 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -9,55 +9,68 @@ on: types: [created] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: sdist: name: Build source distribution runs-on: ubuntu-latest steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work - - name: Install Python 3.x - uses: actions/setup-python@v4 - with: - python-version: 3.x - - name: Install setuptools - run: pip install setuptools - - name: Build sdist - run: python setup.py sdist - - name: Save sdist - uses: actions/upload-artifact@v3 - with: - path: dist/*.tar.gz + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + - name: Install Python 3.x + uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Install setuptools + run: pip install setuptools + - name: Build sdist + run: python setup.py sdist + - name: Save sdist + uses: actions/upload-artifact@v4 + with: + name: cibw-sdist + path: dist/*.tar.gz wheels: name: Build wheels on ${{ matrix.os }} CIBW_BUILD=${{ matrix.cibw_build }} runs-on: ${{ matrix.os }} strategy: + # since multiple builds run at the same time, cancelling them all when one + # fails is wasteful and forces handling build problems one by one instead + # of showing a "full picture" + fail-fast: false matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] - cibw_build: [cp38-*, cp39-*, cp310-*, cp311-*, cp312-*] + os: + - ubuntu-latest + - macos-13 # x86 + - macos-latest # arm + - windows-latest + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] steps: - name: Check out repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work - name: Install Python 3.x - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.x - - name: Set up QEMU # Needed to build aarch64 wheels + - name: Set up QEMU # Needed to build aarch64 wheels if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.16.2 + uses: pypa/cibuildwheel@v2.21 env: CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" - CIBW_ARCHS_MACOS: "x86_64 arm64" # universal2" + CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches CIBW_ARCHS_WINDOWS: "AMD64 x86" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" @@ -65,8 +78,9 @@ jobs: CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{aarch64,ppc64le,s390x}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: + name: cibw-wheels-${{ strategy.job-index }} path: wheelhouse/*.whl upload_pypi: @@ -75,9 +89,9 @@ jobs: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: - name: artifact + pattern: cibw-* path: dist - uses: pypa/gh-action-pypi-publish@release/v1 with: From feb19af092628ed0804814dd161084700a33bcab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Duarte?= Date: Thu, 17 Oct 2024 08:58:50 +0000 Subject: [PATCH 242/280] Update setup.py Python versions --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f3a226d2..9bc91f01 100644 --- a/setup.py +++ b/setup.py @@ -171,7 +171,7 @@ def pkgconfig_installed_check(lib, required_version, default): use_scm_version={ 'write_to': "lz4/version.py", }, - python_requires=">=3.8", + python_requires=">=3.9", setup_requires=[ 'setuptools_scm', 'pkgconfig', @@ -200,10 +200,10 @@ def pkgconfig_installed_check(lib, required_version, default): 'Intended Audience :: Developers', 'Programming Language :: C', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', ], ) From 4d9bced9173c6deb9c19e46d07b16861df8fc539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Duarte?= Date: Thu, 14 Nov 2024 11:25:02 +0000 Subject: [PATCH 243/280] revert artifact renaming --- .github/workflows/build_dist.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index b38968dd..ad4763b5 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -33,7 +33,6 @@ jobs: - name: Save sdist uses: actions/upload-artifact@v4 with: - name: cibw-sdist path: dist/*.tar.gz wheels: @@ -80,7 +79,6 @@ jobs: - name: Save wheels uses: actions/upload-artifact@v4 with: - name: cibw-wheels-${{ strategy.job-index }} path: wheelhouse/*.whl upload_pypi: @@ -91,9 +89,9 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - pattern: cibw-* + pattern: artifact path: dist - uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} - skip_existing: true + skip-existing: true From a46e255cb7b64ece87ec63b3bdd2a1d87fe3cacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Duarte?= Date: Thu, 14 Nov 2024 17:14:08 +0000 Subject: [PATCH 244/280] Add file extensions to uploaded archives --- .github/workflows/build_dist.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index ad4763b5..1e815c18 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -33,6 +33,7 @@ jobs: - name: Save sdist uses: actions/upload-artifact@v4 with: + name: cibw-sdist.tar.gz path: dist/*.tar.gz wheels: @@ -79,6 +80,7 @@ jobs: - name: Save wheels uses: actions/upload-artifact@v4 with: + name: cibw-wheels-${{ strategy.job-index }}.whl path: wheelhouse/*.whl upload_pypi: @@ -89,7 +91,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - pattern: artifact + pattern: cibw-* path: dist - uses: pypa/gh-action-pypi-publish@release/v1 with: From be4b2a92feb2a2db91bd627db2ed49cdc3ad6521 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 26 Jan 2025 18:28:09 +0000 Subject: [PATCH 245/280] Fix publishing to pypi --- .github/workflows/build_dist.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 1e815c18..3189d4a2 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -80,8 +80,8 @@ jobs: - name: Save wheels uses: actions/upload-artifact@v4 with: - name: cibw-wheels-${{ strategy.job-index }}.whl - path: wheelhouse/*.whl + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl upload_pypi: name: Upload to PyPI @@ -93,6 +93,7 @@ jobs: with: pattern: cibw-* path: dist + merge-multiple: true - uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} From b5e5217b1a783da004efef0b0d2ea0ce64241b84 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 26 Jan 2025 19:42:04 +0000 Subject: [PATCH 246/280] Disable aarch64 builds due to compiler failure --- .github/workflows/build_dist.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 3189d4a2..879db7d5 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,7 +69,8 @@ jobs: uses: pypa/cibuildwheel@v2.21 env: CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" - CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" + # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" + CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches CIBW_ARCHS_WINDOWS: "AMD64 x86" CIBW_BUILD: ${{ matrix.cibw_build }} From b20fdf35e659d697bcc82a81ac0aec6bf95bdbc3 Mon Sep 17 00:00:00 2001 From: Rudolf Kolbe Date: Wed, 2 Apr 2025 00:03:43 +0200 Subject: [PATCH 247/280] build_dist workflow - (re)add arm support for Linux and Windows (#300) This enables Arm support for Linux and Windows. --- .github/workflows/build_dist.yml | 46 +++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 879db7d5..ac70871a 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -26,10 +26,8 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.x - - name: Install setuptools - run: pip install setuptools - name: Build sdist - run: python setup.py sdist + run: pipx run build --sdist - name: Save sdist uses: actions/upload-artifact@v4 with: @@ -72,11 +70,47 @@ jobs: # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches - CIBW_ARCHS_WINDOWS: "AMD64 x86" + CIBW_ARCHS_WINDOWS: "AMD64 x86 ARM64" + CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_SKIP: "cp*-musllinux*" + CIBW_TEST_COMMAND: "tox -c {project}" + CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{ppc64le,s390x} *-win_arm64" + CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" + - name: Save wheels + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + wheels_linux_arm: + name: Build wheels on ${{ matrix.os }} CIBW_BUILD=${{ matrix.cibw_build }} + runs-on: ${{ matrix.os }} + strategy: + # since multiple builds run at the same time, cancelling them all when one + # fails is wasteful and forces handling build problems one by one instead + # of showing a "full picture" + fail-fast: false + matrix: + os: + - ubuntu-24.04-arm + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + - name: Install Python 3.x + uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: pypa/cibuildwheel@v2.21 + env: + CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" + CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" CIBW_TEST_COMMAND: "tox -c {project}" - CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{aarch64,ppc64le,s390x}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v4 @@ -86,7 +120,7 @@ jobs: upload_pypi: name: Upload to PyPI - needs: [sdist, wheels] + needs: [sdist, wheels, wheels_linux_arm] runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: From 7633d8d48f4b8ff6da495e0c426ee292d00b97e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 17 Mar 2025 16:09:57 -0500 Subject: [PATCH 248/280] Enable CPython free-threaded wheel builds --- .github/workflows/build_dist.yml | 11 ++++++++- lz4/_version.c | 4 +++ lz4/block/_block.c | 4 +++ lz4/frame/_frame.c | 4 +++ lz4/stream/_stream.c | 4 +++ tests/block/conftest.py | 18 ++++++++++++++ tests/block/test_block_0.py | 14 ++++++++++- tests/block/test_block_3.py | 1 + tests/frame/test_frame_2.py | 7 ++++++ tests/frame/test_frame_5.py | 8 +++--- tests/frame/test_frame_6.py | 42 +++++++++++++++++++------------- tests/frame/test_frame_8.py | 8 +++--- tests/frame/test_frame_9.py | 25 +++++++++++-------- tests/stream/test_stream_0.py | 1 + tests/stream/test_stream_3.py | 1 + 15 files changed, 117 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index ac70871a..b1e325c3 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -48,7 +48,7 @@ jobs: - macos-13 # x86 - macos-latest # arm - windows-latest - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] steps: - name: Check out repository uses: actions/checkout@v4 @@ -63,6 +63,14 @@ jobs: uses: docker/setup-qemu-action@v3 with: platforms: all + + - name: Setup free-threading variables + if: ${{ endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" + echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV" + echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.21 env: @@ -110,6 +118,7 @@ jobs: CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" + CIBW_ENABLE: cpython-freethreading CIBW_TEST_COMMAND: "tox -c {project}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels diff --git a/lz4/_version.c b/lz4/_version.c index c611f0b3..77952415 100644 --- a/lz4/_version.c +++ b/lz4/_version.c @@ -113,5 +113,9 @@ PyInit__version(void) if (module == NULL) return NULL; + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 3e904a03..daa9fd5b 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -518,5 +518,9 @@ PyInit__block(void) Py_INCREF(LZ4BlockError); PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError); + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 34606653..d081cce9 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1677,5 +1677,9 @@ PyInit__frame(void) PyModule_AddIntConstant (module, "BLOCKSIZE_MAX1MB", LZ4F_max1MB); PyModule_AddIntConstant (module, "BLOCKSIZE_MAX4MB", LZ4F_max4MB); + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 522fdeda..6351d969 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -1649,5 +1649,9 @@ PyInit__stream(void) Py_INCREF (LZ4StreamError); PyModule_AddObject (module, "LZ4StreamError", LZ4StreamError); + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/tests/block/conftest.py b/tests/block/conftest.py index 0b3578ea..b25c239e 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -3,6 +3,24 @@ import sys +class EmptyMemoryView(): + def __init__(self): + self.data = b'' + self.view = None + + def __buffer__(self, flags: int, /) -> memoryview: + if self.view is None: + self.view = memoryview(self.data) + return self.view + + def __release_buffer__(self, buffer: memoryview, /): + breakpoint() + buffer.release() + + def __len__(self): + return 0 + + test_data = [ (b''), (os.urandom(8 * 1024)), diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 8fc0f488..31513b47 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -1,6 +1,9 @@ import lz4.block from multiprocessing.pool import ThreadPool import sys +import copy +import inspect +import pytest from functools import partial if sys.version_info <= (3, 2): import struct @@ -79,10 +82,19 @@ def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dicti # Test multi threaded usage with all valid variations of input +@pytest.mark.thread_unsafe def test_2(data, mode, store_size, dictionary): (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size) - data_in = [data for i in range(32)] + def copy_buf(data): + data_x = data + if isinstance(data, memoryview): + data_x = memoryview(copy.deepcopy(data.obj)) + elif isinstance(data, bytearray): + data_x = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + return data_x + + data_in = [copy_buf(data) for i in range(32)] pool = ThreadPool(2) rt = partial(roundtrip, c_kwargs=c_kwargs, diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py index 3fcb175b..88461b7a 100644 --- a/tests/block/test_block_3.py +++ b/tests/block/test_block_3.py @@ -18,6 +18,7 @@ def data(request): return request.param +@pytest.mark.thread_unsafe def test_block_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index 80b44b87..14a3f6b0 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -1,6 +1,8 @@ import lz4.frame as lz4frame import pytest import os +import copy +import inspect import sys from . helpers import ( get_chunked, @@ -41,6 +43,11 @@ def test_roundtrip_chunked(data, block_size, block_linked, data, c_chunks, d_chunks = data + if isinstance(data, memoryview): + data = memoryview(copy.deepcopy(data.obj)) + elif isinstance(data, bytearray): + data = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + c_context = lz4frame.create_compression_context() kwargs = {} diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 05daf283..40836462 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -8,6 +8,8 @@ (b'a' * 1024 * 1024), ] +pytestmark = pytest.mark.thread_unsafe + @pytest.fixture( params=test_data, @@ -66,17 +68,17 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot -def test_frame_open_decompress_mem_usage(data): +def test_frame_open_decompress_mem_usage(tmp_path, data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() - with lz4.frame.open('test.lz4', 'w') as f: + with lz4.frame.open(tmp_path / 'test.lz4', 'w') as f: f.write(data) prev_snapshot = None for i in range(1000): - with lz4.frame.open('test.lz4', 'r') as f: + with lz4.frame.open(tmp_path / 'test.lz4', 'r') as f: decompressed = f.read() # noqa: F841 if i % 100 == 0: diff --git a/tests/frame/test_frame_6.py b/tests/frame/test_frame_6.py index c20a4f31..4f4185ee 100644 --- a/tests/frame/test_frame_6.py +++ b/tests/frame/test_frame_6.py @@ -1,5 +1,6 @@ import os import pytest +import threading import lz4.frame as lz4frame test_data = [ @@ -33,40 +34,45 @@ def compression_level(request): return request.param -def test_lz4frame_open_write(data): - with lz4frame.open('testfile', mode='wb') as fp: +def test_lz4frame_open_write(tmp_path, data): + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp: fp.write(data) -def test_lz4frame_open_write_read_defaults(data): - with lz4frame.open('testfile', mode='wb') as fp: +def test_lz4frame_open_write_read_defaults(tmp_path, data): + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp: fp.write(data) - with lz4frame.open('testfile', mode='r') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp: data_out = fp.read() assert data_out == data -def test_lz4frame_open_write_read_text(): +def test_lz4frame_open_write_read_text(tmp_path): data = u'This is a test string' - with lz4frame.open('testfile', mode='wt') as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp: fp.write(data) - with lz4frame.open('testfile', mode='rt') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp: data_out = fp.read() assert data_out == data -def test_lz4frame_open_write_read_text_iter(): +def test_lz4frame_open_write_read_text_iter(tmp_path): data = u'This is a test string' - with lz4frame.open('testfile', mode='wt') as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp: fp.write(data) data_out = '' - with lz4frame.open('testfile', mode='rt') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp: for line in fp: data_out += line assert data_out == data def test_lz4frame_open_write_read( + tmp_path, data, compression_level, block_linked, @@ -91,29 +97,31 @@ def test_lz4frame_open_write_read( kwargs['return_bytearray'] = return_bytearray kwargs['mode'] = 'wb' - with lz4frame.open('testfile', **kwargs) as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', **kwargs) as fp: fp.write(data) - with lz4frame.open('testfile', mode='r') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp: data_out = fp.read() assert data_out == data -def test_lz4frame_flush(): +def test_lz4frame_flush(tmp_path): data_1 = b"This is a..." data_2 = b" test string!" + thread_id = threading.get_native_id() - with lz4frame.open("testfile", mode="w") as fp_write: + with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="w") as fp_write: fp_write.write(data_1) fp_write.flush() fp_write.write(data_2) - with lz4frame.open("testfile", mode="r") as fp_read: + with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read: assert fp_read.read() == data_1 fp_write.flush() - with lz4frame.open("testfile", mode="r") as fp_read: + with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read: assert fp_read.read() == data_1 + data_2 diff --git a/tests/frame/test_frame_8.py b/tests/frame/test_frame_8.py index 159534ae..cfaeaace 100644 --- a/tests/frame/test_frame_8.py +++ b/tests/frame/test_frame_8.py @@ -1,12 +1,14 @@ +import threading import lz4.frame as lz4frame -def test_lz4frame_open_write_read_text_iter(): +def test_lz4frame_open_write_read_text_iter(tmp_path): data = u'This is a test string' - with lz4frame.open('testfile', mode='wt') as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp: fp.write(data) data_out = '' - with lz4frame.open('testfile', mode='rt') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp: for line in fp: data_out += line assert data_out == data diff --git a/tests/frame/test_frame_9.py b/tests/frame/test_frame_9.py index 51433934..c5335aed 100644 --- a/tests/frame/test_frame_9.py +++ b/tests/frame/test_frame_9.py @@ -3,11 +3,12 @@ import io import pickle import sys +import threading import lz4.frame import pytest -def test_issue_172_1(): +def test_issue_172_1(tmp_path): """Test reproducer for issue 172 Issue 172 is a reported failure occurring on Windows 10 only. This bug was @@ -16,34 +17,38 @@ def test_issue_172_1(): """ input_data = 8 * os.urandom(1024) - with lz4.frame.open('testfile_small', 'wb') as fp: + thread_id = threading.get_native_id() + + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp: bytes_written = fp.write(input_data) # noqa: F841 - with lz4.frame.open('testfile_small', 'rb') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp: data = fp.read(10) assert len(data) == 10 -def test_issue_172_2(): +def test_issue_172_2(tmp_path): input_data = 9 * os.urandom(1024) - with lz4.frame.open('testfile_small', 'w') as fp: + thread_id = threading.get_native_id() + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'w') as fp: bytes_written = fp.write(input_data) # noqa: F841 - with lz4.frame.open('testfile_small', 'r') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'r') as fp: data = fp.read(10) assert len(data) == 10 -def test_issue_172_3(): +def test_issue_172_3(tmp_path): input_data = 9 * os.urandom(1024) - with lz4.frame.open('testfile_small', 'wb') as fp: + thread_id = threading.get_native_id() + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp: bytes_written = fp.write(input_data) # noqa: F841 - with lz4.frame.open('testfile_small', 'rb') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp: data = fp.read(10) assert len(data) == 10 - with lz4.frame.open('testfile_small', 'rb') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp: data = fp.read(16 * 1024 - 1) assert len(data) == 9 * 1024 assert data == input_data diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index 03b19f3f..cac07bdd 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -96,6 +96,7 @@ def setup_kwargs(strategy, mode, buffer_size, store_comp_size, # Test single threaded usage with all valid variations of input +@pytest.mark.thread_unsafe def test_1(data, strategy, mode, buffer_size, store_comp_size, c_return_bytearray, d_return_bytearray, dictionary): if buffer_size >= (1 << (8 * store_comp_size['store_comp_size'])): diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index 2b52d6b5..fed93d2c 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -71,6 +71,7 @@ def data(request): return request.param +@pytest.mark.thread_unsafe def test_block_decompress_mem_usage(data, buffer_size): kwargs = { 'strategy': "double_buffer", From 0f5c05e7ccfc35f5f1d2d61cebb9f40480116444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 11:42:55 -0500 Subject: [PATCH 249/280] Update cibuildwheel version --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index b1e325c3..38f40dda 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -72,7 +72,7 @@ jobs: echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV" echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.21 + uses: pypa/cibuildwheel@v2.23.2 env: CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" From c0b3bb5c236cec8e1563411228d5133893c1a7f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 12:32:44 -0500 Subject: [PATCH 250/280] Variable misspell fix --- lz4/_version.c | 2 +- lz4/block/_block.c | 2 +- lz4/frame/_frame.c | 2 +- lz4/stream/_stream.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lz4/_version.c b/lz4/_version.c index 77952415..af606abe 100644 --- a/lz4/_version.c +++ b/lz4/_version.c @@ -114,7 +114,7 @@ PyInit__version(void) return NULL; #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; diff --git a/lz4/block/_block.c b/lz4/block/_block.c index daa9fd5b..993cc44c 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -519,7 +519,7 @@ PyInit__block(void) PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError); #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index d081cce9..e62c72c6 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1678,7 +1678,7 @@ PyInit__frame(void) PyModule_AddIntConstant (module, "BLOCKSIZE_MAX4MB", LZ4F_max4MB); #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 6351d969..f0dfad57 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -1650,7 +1650,7 @@ PyInit__stream(void) PyModule_AddObject (module, "LZ4StreamError", LZ4StreamError); #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; From 99321aa0166e6398276988a4d8c800568239177b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 14:38:33 -0500 Subject: [PATCH 251/280] Update environment variables --- .github/workflows/build_dist.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 38f40dda..77862f1a 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,12 +69,15 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV" - echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" + - name: Setup environment + if: ${{ !endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4="False" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: - CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches From 2258e28190e3f037617ae48fe329035b60c28ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 14:44:07 -0500 Subject: [PATCH 252/280] Ensure CIBW_ENVIRONMENT is set --- .github/workflows/build_dist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 77862f1a..0c659e3f 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,12 +69,12 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4="False" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: From a20d167678f4fec73d41e811f809ec2f1799bab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 15:28:12 -0500 Subject: [PATCH 253/280] Pass tox overrides through CLI --- .github/workflows/build_dist.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 0c659e3f..62d84da4 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,12 +69,14 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: From cc5c9b18002863ebb4fcff7ff58db05cdd3e0db3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 15:39:14 -0500 Subject: [PATCH 254/280] Ensure test command is applied --- .github/workflows/build_dist.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 62d84da4..e6541dda 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -84,9 +84,9 @@ jobs: CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches CIBW_ARCHS_WINDOWS: "AMD64 x86 ARM64" + CIBW_ENABLE: cpython-freethreading CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" - CIBW_TEST_COMMAND: "tox -c {project}" CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{ppc64le,s390x} *-win_arm64" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels @@ -106,7 +106,7 @@ jobs: matrix: os: - ubuntu-24.04-arm - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] steps: - name: Check out repository uses: actions/checkout@v4 @@ -116,15 +116,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.x + - name: Setup free-threading variables + if: ${{ endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" + - name: Setup environment + if: ${{ !endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.21 env: - CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" CIBW_ENABLE: cpython-freethreading - CIBW_TEST_COMMAND: "tox -c {project}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v4 From 40bcbd8c88c27416b9dc430e65e7f07741c7dfa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 15:57:10 -0500 Subject: [PATCH 255/280] Check why using temp paths increase memory usage --- tests/frame/test_frame_5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 40836462..dcbe4aea 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -68,17 +68,17 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot -def test_frame_open_decompress_mem_usage(tmp_path, data): +def test_frame_open_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() - with lz4.frame.open(tmp_path / 'test.lz4', 'w') as f: + with lz4.frame.open('test.lz4', 'w') as f: f.write(data) prev_snapshot = None for i in range(1000): - with lz4.frame.open(tmp_path / 'test.lz4', 'r') as f: + with lz4.frame.open('test.lz4', 'r') as f: decompressed = f.read() # noqa: F841 if i % 100 == 0: From ad3ec0d5810615afc9dff3d2b18f6d29d2066223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 2 Apr 2025 17:13:03 -0500 Subject: [PATCH 256/280] Copy bytearrays directly --- tests/block/conftest.py | 18 ------------------ tests/block/test_block_0.py | 3 ++- tests/frame/test_frame_2.py | 4 +++- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/block/conftest.py b/tests/block/conftest.py index b25c239e..0b3578ea 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -3,24 +3,6 @@ import sys -class EmptyMemoryView(): - def __init__(self): - self.data = b'' - self.view = None - - def __buffer__(self, flags: int, /) -> memoryview: - if self.view is None: - self.view = memoryview(self.data) - return self.view - - def __release_buffer__(self, buffer: memoryview, /): - breakpoint() - buffer.release() - - def __len__(self): - return 0 - - test_data = [ (b''), (os.urandom(8 * 1024)), diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 31513b47..8a22e57f 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -91,7 +91,8 @@ def copy_buf(data): if isinstance(data, memoryview): data_x = memoryview(copy.deepcopy(data.obj)) elif isinstance(data, bytearray): - data_x = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + data_x = bytearray() + data_x[:] = data return data_x data_in = [copy_buf(data) for i in range(32)] diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index 14a3f6b0..f7e6fa3f 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -46,7 +46,9 @@ def test_roundtrip_chunked(data, block_size, block_linked, if isinstance(data, memoryview): data = memoryview(copy.deepcopy(data.obj)) elif isinstance(data, bytearray): - data = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + data_2 = bytearray() + data_2[:] = data + data = data_2 c_context = lz4frame.create_compression_context() From 7a88fb9351ddfd65936dab78b79e41e910add51c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 2 Apr 2025 17:22:17 -0500 Subject: [PATCH 257/280] Remove unused import --- tests/block/test_block_0.py | 1 - tests/frame/test_frame_2.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 8a22e57f..f2c3cd18 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -2,7 +2,6 @@ from multiprocessing.pool import ThreadPool import sys import copy -import inspect import pytest from functools import partial if sys.version_info <= (3, 2): diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index f7e6fa3f..230867e6 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -2,7 +2,6 @@ import pytest import os import copy -import inspect import sys from . helpers import ( get_chunked, From 737bac1a663c4d15347e98359b7a11c7ddc96f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 2 Apr 2025 17:41:35 -0500 Subject: [PATCH 258/280] Copy memoryview on test_1 --- tests/block/test_block_0.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index f2c3cd18..561b1858 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -70,6 +70,13 @@ def setup_kwargs(mode, store_size, c_return_bytearray=None, d_return_bytearray=N # Test single threaded usage with all valid variations of input def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dictionary): + if isinstance(data, memoryview): + data = memoryview(copy.deepcopy(data.obj)) + elif isinstance(data, bytearray): + data_x = bytearray() + data_x[:] = data + data = data_x + (c_kwargs, d_kwargs) = setup_kwargs( mode, store_size, c_return_bytearray, d_return_bytearray) From b7c0b665eb70706a1c6e334e54a1b9fa18c96cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 12:26:53 -0500 Subject: [PATCH 259/280] Upgrade pypa/cibuildwheel action in arm64 job --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index e6541dda..4af62b60 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -130,7 +130,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.21 + uses: pypa/cibuildwheel@v2.23.2 env: CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} From 61957cb0dacf51b4cf0f351b740119bb576428b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 14:45:57 -0500 Subject: [PATCH 260/280] CI: set parallelism to 2 in aarch64 --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 4af62b60..e3276807 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -121,7 +121,7 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=2" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} From 689440cdabe92a8a407cf96e43847eec1b956ec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 15:11:26 -0500 Subject: [PATCH 261/280] CI: disable pytest-run-parallel altogeogether in aarch64 --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index e3276807..6765e991 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -121,7 +121,7 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=2" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=1" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} From e54341fac7984222faec6e789ddc97486e6e88d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 15:24:42 -0500 Subject: [PATCH 262/280] CI: disable armv7l --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 6765e991..9b9964b9 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -132,7 +132,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: - CIBW_ARCHS_LINUX: "aarch64 armv7l" + CIBW_ARCHS_LINUX: "aarch64" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" CIBW_ENABLE: cpython-freethreading From 022d53815abf97e6eb49d0be3a27c74541c782ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 9 Apr 2025 16:43:32 -0500 Subject: [PATCH 263/280] Address review comments --- .github/workflows/build_dist.yml | 2 ++ tests/block/test_block_0.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 9b9964b9..b907acc0 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -120,6 +120,8 @@ jobs: if: ${{ endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | + # Variables are set in order to be passed down to both cibuildwheel and the + # Docker image spawned by that action echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=1" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 561b1858..a7731c3a 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -93,12 +93,13 @@ def test_2(data, mode, store_size, dictionary): (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size) def copy_buf(data): - data_x = data if isinstance(data, memoryview): data_x = memoryview(copy.deepcopy(data.obj)) elif isinstance(data, bytearray): data_x = bytearray() data_x[:] = data + else: + data_x = data return data_x data_in = [copy_buf(data) for i in range(32)] From 708e6d49edf09079936c444c0516236dbe406426 Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Thu, 29 May 2025 12:06:30 +0200 Subject: [PATCH 264/280] Correct the import of _compression for Python 3.14 This is backwards compatible with all supported versions of Python. --- lz4/frame/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 00f3e64a..60bf7dce 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -25,9 +25,9 @@ __doc__ = _doc try: - import _compression # Python 3.6 and later + import compression._common._streams as _compression # Python 3.14 except ImportError: - from . import _compression + import _compression # Python 3.6 - 3.13 BLOCKSIZE_DEFAULT = _BLOCKSIZE_DEFAULT From d8387509dd17dcefec6041bf242598ada30a505f Mon Sep 17 00:00:00 2001 From: Karolina Surma <33810531+befeleme@users.noreply.github.com> Date: Tue, 7 Oct 2025 09:19:38 +0200 Subject: [PATCH 265/280] Update lz4/frame/__init__.py Co-authored-by: Christian Clauss --- lz4/frame/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 60bf7dce..91ad50bf 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -27,7 +27,7 @@ try: import compression._common._streams as _compression # Python 3.14 except ImportError: - import _compression # Python 3.6 - 3.13 + import _compression # Python 3.9 - 3.13 BLOCKSIZE_DEFAULT = _BLOCKSIZE_DEFAULT From 5c9f5561e4353978746b2af6fac1338668e5473b Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 24 Sep 2025 13:13:08 +0200 Subject: [PATCH 266/280] Keep GitHub Actions up to date with GitHub's Dependabot * [Keeping your software supply chain secure with Dependabot](https://docs.github.com/en/code-security/dependabot) * [Keeping your actions up to date with Dependabot](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot) * [Configuration options for the `dependabot.yml` file - package-ecosystem](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem) --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..be006de9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Keep GitHub Actions up to date with GitHub's Dependabot... +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + groups: + github-actions: + patterns: + - "*" # Group all Actions updates into a single larger pull request + schedule: + interval: weekly From 8c82a0df102fed1221bb5edf8dfa3f72017160a6 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 24 Sep 2025 12:46:16 +0200 Subject: [PATCH 267/280] Build wheels for Python 3.14 Caution: The new `compression` module in Python >= 3.14 Standard Library causes compatibility problems. * https://docs.python.org/3.14/library/compression.html * #302 * #303 * #308 --- Python v3.14 -- October 7th * https://www.python.org/download/pre-releases * https://www.python.org/downloads/release/python-3140rc3 * https://docs.python.org/3.14/whatsnew/3.14.html What's new in Python 3.14: ___PEP 784: Adding Zstandard to the standard library___ * https://docs.python.org/3.14/whatsnew/3.14.html#whatsnew314-pep784 --- .github/workflows/build_dist.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index b907acc0..836e216b 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -19,11 +19,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.x - name: Build sdist @@ -45,17 +45,17 @@ jobs: matrix: os: - ubuntu-latest - - macos-13 # x86 + - macos-15-intel # x86 - macos-latest # arm - windows-latest - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.x - name: Set up QEMU # Needed to build aarch64 wheels @@ -78,7 +78,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.23.2 + uses: pypa/cibuildwheel@v3.2.0 env: # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" @@ -106,14 +106,14 @@ jobs: matrix: os: - ubuntu-24.04-arm - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.x - name: Setup free-threading variables @@ -132,7 +132,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.23.2 + uses: pypa/cibuildwheel@v3.2.0 env: CIBW_ARCHS_LINUX: "aarch64" CIBW_BUILD: ${{ matrix.cibw_build }} @@ -151,7 +151,7 @@ jobs: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v5 with: pattern: cibw-* path: dist From 438df39de27dc1d44fdc45c6fd3256611758d099 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Oct 2025 17:21:16 +0200 Subject: [PATCH 268/280] Remove cp314t from cibw_build matrix --- .github/workflows/build_dist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 836e216b..1556839d 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -48,7 +48,7 @@ jobs: - macos-15-intel # x86 - macos-latest # arm - windows-latest - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*] steps: - name: Check out repository uses: actions/checkout@v5 @@ -106,7 +106,7 @@ jobs: matrix: os: - ubuntu-24.04-arm - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*] steps: - name: Check out repository uses: actions/checkout@v5 From 849e1545f36ec5036052dad80e402c2e4d0b2ed2 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 24 Sep 2025 14:32:23 +0200 Subject: [PATCH 269/280] Fix typos discovered by codespell --- lz4/frame/__init__.py | 6 +++--- lz4/frame/_frame.c | 4 ++-- lz4/stream/__init__.py | 10 +++++----- lz4/stream/_stream.c | 2 +- tests/frame/test_frame_9.py | 2 +- tests/stream/test_stream_0.py | 2 +- tests/stream/test_stream_1.py | 10 +++++----- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 91ad50bf..2a82cd08 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -268,7 +268,7 @@ def flush(self): This returns a ``bytes`` or ``bytearray`` object containing any data stored in the compressor's internal buffers and a frame footer. - The LZ4FrameCompressor instance may be re-used after this method has + The LZ4FrameCompressor instance may be reused after this method has been called to create a new frame of compressed data. Returns: @@ -287,7 +287,7 @@ def flush(self): def reset(self): """Reset the `LZ4FrameCompressor` instance. - This allows the `LZ4FrameCompression` instance to be re-used after an + This allows the `LZ4FrameCompression` instance to be reused after an error. """ @@ -360,7 +360,7 @@ def __exit__(self, exception_type, exception, traceback): def reset(self): """Reset the decompressor state. - This is useful after an error occurs, allowing re-use of the instance. + This is useful after an error occurs, allowing reuse of the instance. """ reset_decompression_context(self._context) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index e62c72c6..440b0b50 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1330,7 +1330,7 @@ PyDoc_STRVAR( ); #define COMPRESS_KWARGS_DOCSTRING \ - " block_size (int): Sepcifies the maximum blocksize to use.\n" \ + " block_size (int): Specifies the maximum blocksize to use.\n" \ " Options:\n\n" \ " - `lz4.frame.BLOCKSIZE_DEFAULT`: the lz4 library default\n" \ " - `lz4.frame.BLOCKSIZE_MAX64KB`: 64 kB\n" \ @@ -1466,7 +1466,7 @@ PyDoc_STRVAR "data will also be included in the returned data.\n" \ "\n" \ "If the ``end_frame`` argument is ``True``, the compression context will be\n" \ - "reset and can be re-used.\n" \ + "reset and can be reused.\n" \ "\n" \ "Args:\n" \ " context (cCtx): Compression context\n" \ diff --git a/lz4/stream/__init__.py b/lz4/stream/__init__.py index 4b007464..58ce298a 100644 --- a/lz4/stream/__init__.py +++ b/lz4/stream/__init__.py @@ -32,7 +32,7 @@ def __init__(self, strategy, buffer_size, return_bytearray=False, store_comp_siz perform decompression using this initial dictionary. Raises: - Exceptions occuring during the context initialization. + Exceptions occurring during the context initialization. OverflowError: raised if the ``dictionary`` parameter is too large for the LZ4 context. @@ -73,7 +73,7 @@ def decompress(self, chunk): bytes or bytearray: Decompressed data. Raises: - Exceptions occuring during decompression. + Exceptions occurring during decompression. ValueError: raised if the source is inconsistent with a finite LZ4 stream block chain. @@ -96,7 +96,7 @@ def get_block(self, stream): bytes or bytearray: LZ4 compressed data block. Raises: - Exceptions occuring while getting the first block from ``stream``. + Exceptions occurring while getting the first block from ``stream``. BufferError: raised if the function cannot return a complete LZ4 compressed block from the stream (i.e. the stream does not hold @@ -150,7 +150,7 @@ def __init__(self, strategy, buffer_size, mode="default", acceleration=True, com perform compression using this initial dictionary. Raises: - Exceptions occuring during the context initialization. + Exceptions occurring during the context initialization. OverflowError: raised if the ``dictionary`` parameter is too large for the LZ4 context. @@ -194,7 +194,7 @@ def compress(self, chunk): bytes or bytearray: Compressed data. Raises: - Exceptions occuring during compression. + Exceptions occurring during compression. OverflowError: raised if the source is too large for being compressed in the given context. diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index f0dfad57..4c51d89d 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -931,7 +931,7 @@ _create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) context->output.len = buffer_size; total_size = context->output.len; - /* Here we cannot assert the maximal theorical decompressed chunk length + /* Here we cannot assert the maximal theoretical decompressed chunk length * will fit in one page of the double_buffer, i.e.: * assert( !(double_buffer.page_size < _LZ4_inputBound(store_max_size)) ) * diff --git a/tests/frame/test_frame_9.py b/tests/frame/test_frame_9.py index c5335aed..6f7fc0db 100644 --- a/tests/frame/test_frame_9.py +++ b/tests/frame/test_frame_9.py @@ -65,7 +65,7 @@ def test_issue_227_1(): @pytest.mark.skipif( sys.version_info < (3, 8), - reason="PickleBuffer only availiable in Python 3.8 or greater" + reason="PickleBuffer only available in Python 3.8 or greater" ) def test_issue_227_2(): q = array.array('Q', [1, 2, 3, 4, 5]) diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index cac07bdd..3cd05d6f 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -112,6 +112,6 @@ def test_1(data, strategy, mode, buffer_size, store_comp_size, # Test multi threaded: # Not relevant in the lz4.stream case (the process is highly sequential, -# and re-use/share the same context from one input chunk to the next one). +# and reuse/share the same context from one input chunk to the next one). def test_2(data, strategy, mode, buffer_size, store_comp_size, dictionary): # noqa pass diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py index 6b49267e..481de8aa 100644 --- a/tests/stream/test_stream_1.py +++ b/tests/stream/test_stream_1.py @@ -136,7 +136,7 @@ def test_invalid_config_c_4(store_comp_size): c_kwargs.update(store_comp_size) if store_comp_size['store_comp_size'] >= 4: - # No need for skiping this test case, since arguments check is + # No need for skipping this test case, since arguments check is # expecting to raise an error. # Make sure the page size is larger than what the input bound will be, @@ -169,7 +169,7 @@ def test_invalid_config_d_4(store_comp_size): # but still fit in 4 bytes d_kwargs['buffer_size'] -= 1 - # No failure expected during instanciation/initialization + # No failure expected during instantiation/initialization lz4.stream.LZ4StreamDecompressor(**d_kwargs) @@ -199,7 +199,7 @@ def test_invalid_config_d_5(): d_kwargs = {} d_kwargs['strategy'] = "double_buffer" - # No failure expected during instanciation/initialization + # No failure expected during instantiation/initialization d_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE if sys.maxsize < 0xffffffff: @@ -207,7 +207,7 @@ def test_invalid_config_d_5(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) - # No failure expected during instanciation/initialization + # No failure expected during instantiation/initialization d_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE + 1 if sys.maxsize < 0xffffffff: @@ -215,7 +215,7 @@ def test_invalid_config_d_5(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) - # No failure expected during instanciation/initialization + # No failure expected during instantiation/initialization d_kwargs['buffer_size'] = _4GB - 1 # 4GB - 1 (to fit in 4 bytes) if sys.maxsize < 0xffffffff: From 59b2d8176072bdee50d38cc68ec65c33b928a980 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Oct 2025 17:15:14 +0000 Subject: [PATCH 270/280] Bump pypa/cibuildwheel in the github-actions group across 1 directory Bumps the github-actions group with 1 update in the / directory: [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel). Updates `pypa/cibuildwheel` from 3.2.0 to 3.2.1 - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v3.2.0...v3.2.1) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-version: 3.2.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/build_dist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 1556839d..1af30208 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -78,7 +78,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v3.2.0 + uses: pypa/cibuildwheel@v3.2.1 env: # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" @@ -132,7 +132,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v3.2.0 + uses: pypa/cibuildwheel@v3.2.1 env: CIBW_ARCHS_LINUX: "aarch64" CIBW_BUILD: ${{ matrix.cibw_build }} From d5daffd8356e71cb075fef5071bbbe75b274d109 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 23 Oct 2025 14:26:19 +0200 Subject: [PATCH 271/280] setup.py: Add support for Python 3.14 and Free Threading https://pypi.org/classifiers --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 9bc91f01..d9d8191e 100644 --- a/setup.py +++ b/setup.py @@ -205,5 +205,7 @@ def pkgconfig_installed_check(lib, required_version, default): 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', + 'Programming Language :: Python :: Free Threading', ], ) From 00e0ae93e7e7f81fcc9e9d9768a9b95c01360adb Mon Sep 17 00:00:00 2001 From: Hannes Braun Date: Tue, 3 Feb 2026 15:06:43 +0100 Subject: [PATCH 272/280] Switch to SPDX license expression License classifiers are deprecated. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d9d8191e..350b551d 100644 --- a/setup.py +++ b/setup.py @@ -180,6 +180,7 @@ def pkgconfig_installed_check(lib, required_version, default): long_description=open('README.rst', 'r').read(), author='Jonathan Underwood', author_email='jonathan.underwood@gmail.com', + license='BSD-3-Clause', url='https://github.com/python-lz4/python-lz4', packages=packages, ext_modules=ext_modules, @@ -196,7 +197,6 @@ def pkgconfig_installed_check(lib, required_version, default): }, classifiers=[ 'Development Status :: 5 - Production/Stable', - 'License :: OSI Approved :: BSD License', 'Intended Audience :: Developers', 'Programming Language :: C', 'Programming Language :: Python', From e000ef843a760998462f9a7b10b52d6bddbb8532 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 08:31:48 -0700 Subject: [PATCH 273/280] Enable cp314t wheel builds --- .github/workflows/build_dist.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 1af30208..e2fd4fc0 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -48,7 +48,7 @@ jobs: - macos-15-intel # x86 - macos-latest # arm - windows-latest - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository uses: actions/checkout@v5 @@ -106,7 +106,7 @@ jobs: matrix: os: - ubuntu-24.04-arm - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository uses: actions/checkout@v5 @@ -116,17 +116,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: 3.x - - name: Setup free-threading variables - if: ${{ endsWith(matrix.cibw_build, 't-*') }} - shell: bash -l {0} - run: | - # Variables are set in order to be passed down to both cibuildwheel and the - # Docker image spawned by that action - echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=1" >> "$GITHUB_ENV" - echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment - if: ${{ !endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" From 8427b5691dea5909768c805bc4fc3b1935549fa1 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 09:20:20 -0700 Subject: [PATCH 274/280] mark tests that use tracemalloc as thread-unsafe --- tests/block/test_block_2.py | 4 ++++ tests/block/test_block_3.py | 4 +++- tests/frame/test_frame_5.py | 9 +++++++++ tests/stream/test_stream_3.py | 4 +++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/block/test_block_2.py b/tests/block/test_block_2.py index a2aea4da..a75e92f0 100644 --- a/tests/block/test_block_2.py +++ b/tests/block/test_block_2.py @@ -19,6 +19,10 @@ # fragile. +@pytest.mark.thread_unsafe( + reason=("Large multithreaded allocations will likely exhaust " + "system memory.") +) @pytest.mark.skipif( os.environ.get('TRAVIS') is not None, reason='Skipping test on Travis due to insufficient memory' diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py index 88461b7a..07f8db24 100644 --- a/tests/block/test_block_3.py +++ b/tests/block/test_block_3.py @@ -18,7 +18,9 @@ def data(request): return request.param -@pytest.mark.thread_unsafe +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_block_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index dcbe4aea..85e6fa0f 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -21,6 +21,9 @@ def data(request): return request.param +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_frame_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') @@ -43,6 +46,9 @@ def test_frame_decompress_mem_usage(data): prev_snapshot = snapshot +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_frame_decompress_chunk_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() @@ -68,6 +74,9 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_frame_open_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index fed93d2c..58094e13 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -71,7 +71,9 @@ def data(request): return request.param -@pytest.mark.thread_unsafe +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_block_decompress_mem_usage(data, buffer_size): kwargs = { 'strategy': "double_buffer", From 30a7a7eac52b222f2842710727eba40222cef22e Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 09:38:16 -0700 Subject: [PATCH 275/280] delete unnecessary module-scope mark --- tests/frame/test_frame_5.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 85e6fa0f..497c8a1c 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -8,9 +8,6 @@ (b'a' * 1024 * 1024), ] -pytestmark = pytest.mark.thread_unsafe - - @pytest.fixture( params=test_data, ids=[ From c5c5130b2ed1cf276b9f01d7a4e6fe8c6cff9a36 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 10:30:37 -0700 Subject: [PATCH 276/280] add missing thread-unsafe reasons --- tests/block/test_block_0.py | 5 ++++- tests/frame/test_frame_5.py | 1 + tests/stream/test_stream_0.py | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index a7731c3a..f39da18a 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -88,7 +88,10 @@ def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dicti # Test multi threaded usage with all valid variations of input -@pytest.mark.thread_unsafe +@pytest.mark.thread_unsafe( + reason=("Test is already explicitly multithreaded, should " + "not be run in a thread pool.") +) def test_2(data, mode, store_size, dictionary): (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 497c8a1c..9e4b83ac 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -8,6 +8,7 @@ (b'a' * 1024 * 1024), ] + @pytest.fixture( params=test_data, ids=[ diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index 3cd05d6f..c7db76ee 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -96,7 +96,10 @@ def setup_kwargs(strategy, mode, buffer_size, store_comp_size, # Test single threaded usage with all valid variations of input -@pytest.mark.thread_unsafe +@pytest.mark.thread_unsafe( + reason=("test modifies fixtures that would be shared, see " + "https://github.com/Quansight-Labs/pytest-run-parallel/issues/14") +) def test_1(data, strategy, mode, buffer_size, store_comp_size, c_return_bytearray, d_return_bytearray, dictionary): if buffer_size >= (1 << (8 * store_comp_size['store_comp_size'])): From ebb04a3525e7d146471bd2c394ba4aa63053db41 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 10:30:55 -0700 Subject: [PATCH 277/280] suppress warnings when pytest-run-parallel isn't available --- tests/conftest.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..0fefd8e0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,13 @@ +try: + import pytest_run_parallel # noqa: F401 + PARALLEL_RUN_AVALIABLE = True +except ModuleNotFoundError: + PARALLEL_RUN_AVALIABLE = False + + +def pytest_configure(config): + if not PARALLEL_RUN_AVALIABLE: + config.addinivalue_line( + "markers", + "thread_unsafe: mark the test function as single-threaded", + ) From 43b672d78f728cc2ea33cd5f0a5949cfcf88572f Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 13:09:23 -0700 Subject: [PATCH 278/280] disable coverage for tests that measure memory usage --- tests/block/test_block_3.py | 2 ++ tests/frame/test_frame_5.py | 6 ++++++ tests/stream/test_stream_3.py | 2 ++ 3 files changed, 10 insertions(+) diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py index 07f8db24..86bd9933 100644 --- a/tests/block/test_block_3.py +++ b/tests/block/test_block_3.py @@ -18,6 +18,8 @@ def data(request): return request.param +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 9e4b83ac..29e7765a 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -19,6 +19,8 @@ def data(request): return request.param +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) @@ -44,6 +46,8 @@ def test_frame_decompress_mem_usage(data): prev_snapshot = snapshot +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) @@ -72,6 +76,8 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index 58094e13..a18ed561 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -71,6 +71,8 @@ def data(request): return request.param +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) From 33612bddc2fa9df7471e01e07bf597159aba2e07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 17:37:01 +0000 Subject: [PATCH 279/280] Bump the github-actions group across 1 directory with 4 updates Bumps the github-actions group with 4 updates in the / directory: [actions/checkout](https://github.com/actions/checkout), [actions/upload-artifact](https://github.com/actions/upload-artifact), [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) and [actions/download-artifact](https://github.com/actions/download-artifact). Updates `actions/checkout` from 5 to 6 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) Updates `actions/upload-artifact` from 4 to 5 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v5) Updates `pypa/cibuildwheel` from 3.2.1 to 3.3.0 - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v3.2.1...v3.3.0) Updates `actions/download-artifact` from 5 to 6 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/upload-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: pypa/cibuildwheel dependency-version: 3.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: actions/download-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/build_dist.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index e2fd4fc0..d33b3520 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x @@ -29,7 +29,7 @@ jobs: - name: Build sdist run: pipx run build --sdist - name: Save sdist - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: cibw-sdist.tar.gz path: dist/*.tar.gz @@ -51,7 +51,7 @@ jobs: cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x @@ -78,7 +78,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v3.2.1 + uses: pypa/cibuildwheel@v3.3.0 env: # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" @@ -90,7 +90,7 @@ jobs: CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{ppc64le,s390x} *-win_arm64" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl @@ -109,7 +109,7 @@ jobs: cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x @@ -122,7 +122,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v3.2.1 + uses: pypa/cibuildwheel@v3.3.0 env: CIBW_ARCHS_LINUX: "aarch64" CIBW_BUILD: ${{ matrix.cibw_build }} @@ -130,7 +130,7 @@ jobs: CIBW_ENABLE: cpython-freethreading CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl @@ -141,7 +141,7 @@ jobs: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/download-artifact@v5 + - uses: actions/download-artifact@v6 with: pattern: cibw-* path: dist From 029f36597b8d182484f3c38c6be6a71ad07ddc96 Mon Sep 17 00:00:00 2001 From: Johannes Jordan Date: Mon, 12 Jan 2026 13:29:35 +0100 Subject: [PATCH 280/280] Strip debug symbols from lz4 build When a wheel is built using lz4 library source files bundled with the package, we advise the linker to strip debug symbols from the binary output. This leads to a significant size reduction of the wheel. Fixes #305 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 350b551d..eb9030fa 100644 --- a/setup.py +++ b/setup.py @@ -121,6 +121,8 @@ def pkgconfig_installed_check(lib, required_version, default): '-Wall', '-Wundef' ] + # strip debug symbols from libraries to reduce wheel size + extension_kwargs['extra_link_args'] = ['-s'] else: print('Unrecognized compiler: {0}'.format(compiler)) sys.exit(1)