From 58c4065a73f85f565da8db5a3f9614117dae6c49 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 7 Feb 2018 22:34:07 +0000 Subject: [PATCH 01/10] Add homepage to README.rst --- README.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e8547a29..ddf6863a 100644 --- a/README.rst +++ b/README.rst @@ -43,7 +43,14 @@ generated using Sphinx. Documentation is also hosted on readthedocs. :master: http://python-lz4.readthedocs.io/en/stable/ :development: http://python-lz4.readthedocs.io/en/latest/ - + +Homepage +======== + +The `project homepage `_ is hosted +on Github. Please report any issues you find using the `issue tracker +`_. + Licensing ========= Code specific to this project is covered by the `BSD 3-Clause License From 37d713c87b2513f500505af23ba6a2ce6ca81271 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 7 Feb 2018 22:38:24 +0000 Subject: [PATCH 02/10] Add more infor to README.rst --- README.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ddf6863a..e12c8ebd 100644 --- a/README.rst +++ b/README.rst @@ -28,8 +28,11 @@ The bindings provided in this package cover the `frame format format bindings are the recommended ones to use, as this guarantees interoperability with other implementations and language bindings. -A future release may implement support for the LZ4 stream format. Patches and -help are welcome. +The API provided by the frame format bindings follows that of the LZMA, zlib, +gzip and bzip2 compression libraries which are provided with the Python standard +library. As such, these LZ4 bindings should provide a drop-in alternative to the +compression libraries shipped with Python. The package provides context managers +and file handlers support. Documenation ============ From 2735b7bcaf79e5ae4c4f3f9f63d317271ed9629d Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 7 Feb 2018 22:44:08 +0000 Subject: [PATCH 03/10] Improve docstrings --- lz4/frame/__init__.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 6c90dda2..9124446e 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -447,7 +447,10 @@ def close(self): @property def closed(self): - """True if this file is closed. + """Returns ``True`` if this file is closed. + + Returns: + bool: ``True`` if the file is closed, ``False`` otherwise. """ return self._mode == _MODE_CLOSED @@ -455,6 +458,9 @@ def closed(self): def fileno(self): """Return the file descriptor for the underlying file. + Returns: + file object: file descriptor for file. + """ self._check_not_closed() return self._fp.fileno() @@ -462,12 +468,19 @@ def fileno(self): def seekable(self): """Return whether the file supports seeking. + Returns: + bool: ``True`` if the file supports seeking, ``False`` otherwise. + """ return self.readable() and self._buffer.seekable() def readable(self): """Return whether the file was opened for reading. + Returns: + bool: ``True`` if the file was opened for reading, ``False`` + otherwise. + """ self._check_not_closed() return self._mode == _MODE_READ @@ -475,6 +488,10 @@ def readable(self): def writable(self): """Return whether the file was opened for writing. + Returns: + bool: ``True`` if the file was opened for writing, ``False`` + otherwise. + """ self._check_not_closed() return self._mode == _MODE_WRITE @@ -485,6 +502,9 @@ def peek(self, size=-1): Always returns at least one byte of data, unless at EOF. The exact number of bytes returned is unspecified. + Returns: + bytes: uncompressed data + """ self._check_can_read() # Relies on the undocumented fact that BufferedReader.peek() always @@ -572,9 +592,9 @@ def seek(self, offset, whence=io.SEEK_SET): The new position is specified by ``offset``, relative to the position indicated by ``whence``. Possible values for ``whence`` are: - - io.SEEK_SET or 0: start of stream (default): offset must not be negative - - io.SEEK_CUR or 1: current stream position - - io.SEEK_END or 2: end of stream; offset must not be positive + - ``io.SEEK_SET`` or 0: start of stream (default): offset must not be negative + - ``io.SEEK_CUR`` or 1: current stream position + - ``io.SEEK_END`` or 2: end of stream; offset must not be positive Returns the new file position. From da212ffc411c629665270e075d9f9ab9875e40ef Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 7 Feb 2018 23:59:03 +0000 Subject: [PATCH 04/10] Add documentation for frame module attributes --- docs/lz4.frame.rst | 29 +++++++++++++++++++++++++++++ docs/quickstart.rst | 1 + 2 files changed, 30 insertions(+) diff --git a/docs/lz4.frame.rst b/docs/lz4.frame.rst index 040f451b..5a918e34 100644 --- a/docs/lz4.frame.rst +++ b/docs/lz4.frame.rst @@ -81,3 +81,32 @@ equivalent functionalities in the Python standard library. .. autofunction:: lz4.frame.open .. autoclass:: lz4.frame.LZ4FrameFile :members: + +Module attributes +----------------- + +A number of module attributes are defined for convenience. These are detailed below. + +Compression level +~~~~~~~~~~~~~~~~~ + +The following module attributes can be used when setting the +``compression_level`` argument. + +.. autoattribute:: lz4.frame.COMPRESSIONLEVEL_MIN + +.. autoattribute:: lz4.frame.COMPRESSIONLEVEL_MINHC + +.. autoattribute:: lz4.frame.COMPRESSIONLEVEL_MAX + + +Block size +~~~~~~~~~~ + +The following attributes can be used when setting the ``block_size`` argument. + +.. autoattribute:: lz4.frame.BLOCKSIZE_DEFAULT +.. autoattribute:: lz4.frame.BLOCKSIZE_MAX64KB +.. autoattribute:: lz4.frame.BLOCKSIZE_MAX256KB +.. autoattribute:: lz4.frame.BLOCKSIZE_MAX1MB +.. autoattribute:: lz4.frame.BLOCKSIZE_MAX4MB diff --git a/docs/quickstart.rst b/docs/quickstart.rst index fdb14c1b..fdccbf9f 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,4 +1,5 @@ .. py:currentmodule:: lz4.frame +.. default-role:: obj Quickstart ========== From afc1ac37465b096bde1ee08cbb323fd48c477d81 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 8 Feb 2018 00:03:23 +0000 Subject: [PATCH 05/10] Improve docstrings in _frame.c --- lz4/frame/_frame.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index fd2f63ad..58359e27 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1400,11 +1400,11 @@ PyDoc_STRVAR( #define COMPRESS_KWARGS_DOCSTRING \ " block_size (int): Sepcifies the maximum blocksize to use.\n" \ " Options:\n\n" \ - " - ``lz4.frame.BLOCKSIZE_DEFAULT`` or 0: the lz4 library default\n" \ - " - ``lz4.frame.BLOCKSIZE_MAX64KB`` or 4: 64 kB\n" \ - " - ``lz4.frame.BLOCKSIZE_MAX256KB`` or 5: 256 kB\n" \ - " - ``lz4.frame.BLOCKSIZE_MAX1MB`` or 6: 1 MB\n" \ - " - ``lz4.frame.BLOCKSIZE_MAX4MB`` or 7: 4 MB\n\n" \ + " - `lz4.frame.BLOCKSIZE_DEFAULT` or 0: the lz4 library default\n" \ + " - `lz4.frame.BLOCKSIZE_MAX64KB` or 4: 64 kB\n" \ + " - `lz4.frame.BLOCKSIZE_MAX256KB` or 5: 256 kB\n" \ + " - `lz4.frame.BLOCKSIZE_MAX1MB` or 6: 1 MB\n" \ + " - `lz4.frame.BLOCKSIZE_MAX4MB` or 7: 4 MB\n\n" \ " If unspecified, will default to ``lz4.frame.BLOCKSIZE_DEFAULT``\n" \ " which is currently equal to ``lz4.frame.BLOCKSIZE_MAX64KB``.\n" \ " block_linked (bool): Specifies whether to use block-linked\n" \ @@ -1416,11 +1416,11 @@ PyDoc_STRVAR( " Values below 0 will enable \"fast acceleration\", proportional\n" \ " to the value. Values above 16 will be treated as 16.\n" \ " The following module constants are provided as a convenience:\n\n" \ - " - lz4.frame.COMPRESSIONLEVEL_MIN: Minimum compression (0, the\n" \ + " - `lz4.frame.COMPRESSIONLEVEL_MIN`: Minimum compression (0, the\n" \ " default)\n" \ - " - lz4.frame.COMPRESSIONLEVEL_MINHC: Minimum high-compression\n" \ + " - `lz4.frame.COMPRESSIONLEVEL_MINHC`: Minimum high-compression\n" \ " mode (3)\n" \ - " - lz4.frame.COMPRESSIONLEVEL_MAX: Maximum compression (16)\n\n" \ + " - `lz4.frame.COMPRESSIONLEVEL_MAX`: Maximum compression (16)\n\n" \ " content_checksum (bool): Specifies whether to enable checksumming\n" \ " of the uncompressed content. If True, a checksum is stored at the\n" \ " end of the frame, and checked during decompression. Default is\n" \ From 4cd9e10711e5d629045b530902c240278f775ab6 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 8 Feb 2018 00:17:15 +0000 Subject: [PATCH 06/10] Del _context objects in __exit__ (#112) --- lz4/frame/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 9124446e..34c84a35 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -93,8 +93,8 @@ def __enter__(self): def __exit__(self, exception_type, exception, traceback): # The compression context is created with an appropriate destructor, so - # no need to del it here - pass + # no del it here and let GC collect it + del self._context def begin(self, source_size=0): """Begin a compression frame. The returned data contains frame header @@ -236,8 +236,8 @@ def __enter__(self): def __exit__(self, exception_type, exception, traceback): # The decompression context is created with an appropriate destructor, - # so no need to del it here - pass + # so del it here and let GC collect it + del self._context def reset(self): """Reset the decompressor state. This is useful after an error occurs, allowing From f3151ec1631984b7a9ddce9066a85921d8f77454 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 10 Feb 2018 08:43:29 +0000 Subject: [PATCH 07/10] Ensure attributes are set to None in __exit__ methods --- lz4/frame/__init__.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 34c84a35..e44e62ca 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -92,9 +92,16 @@ def __enter__(self): return self def __exit__(self, exception_type, exception, traceback): - # The compression context is created with an appropriate destructor, so - # no del it here and let GC collect it - del self._context + self.block_size = None + self.block_linked = None + self.compression_level = None + self.content_checksum = None + self.block_checksum = None + self.auto_flush = None + self.return_bytearray = None + self._context = None + self._started = False + def begin(self, source_size=0): """Begin a compression frame. The returned data contains frame header @@ -235,9 +242,12 @@ def __enter__(self): return self def __exit__(self, exception_type, exception, traceback): - # The decompression context is created with an appropriate destructor, - # so del it here and let GC collect it - del self._context + self._context = None + self.eof = None + self.needs_input = None + self.unused_data = None + self._unconsumed_data = None + self._return_bytearray = None def reset(self): """Reset the decompressor state. This is useful after an error occurs, allowing From 82dd41318df50d645f257ff89eed1878cff11c89 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 10 Feb 2018 09:31:37 +0000 Subject: [PATCH 08/10] Move attribute definition to __init__.py and add docstrings --- lz4/frame/__init__.py | 76 +++++++++++++++++++++++++++++++++++++++++-- lz4/frame/_frame.c | 3 -- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index e44e62ca..e127a845 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -3,8 +3,25 @@ import os import builtins import sys -from ._frame import * -from ._frame import __doc__ as _doc +from ._frame import ( + compress, + decompress, + create_compression_context, + compress_begin, + compress_chunk, + compress_flush, + create_decompression_context, + reset_decompression_context, + decompress_chunk, + get_frame_info, + BLOCKSIZE_DEFAULT as _BLOCKSIZE_DEFAULT, + BLOCKSIZE_MAX64KB as _BLOCKSIZE_MAX64KB, + BLOCKSIZE_MAX256KB as _BLOCKSIZE_MAX256KB, + BLOCKSIZE_MAX1MB as _BLOCKSIZE_MAX1MB, + BLOCKSIZE_MAX4MB as _BLOCKSIZE_MAX4MB, + __doc__ as _doc +) + __doc__ = _doc try: @@ -13,6 +30,61 @@ from . import _compression +BLOCKSIZE_DEFAULT = _BLOCKSIZE_DEFAULT +"""Specifying ``block_size=lz4.frame.BLOCKSIZE_DEFAULT`` will instruct the LZ4 +library to use the default maximum blocksize. + +""" + +BLOCKSIZE_MAX64KB = _BLOCKSIZE_MAX64KB +"""Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX64KB`` will instruct the LZ4 +library to create blocks containing a maximum of 64 kB of uncompressed data. + +""" + +BLOCKSIZE_MAX256KB = _BLOCKSIZE_MAX256KB +"""Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX256KB`` will instruct the LZ4 +library to create blocks containing a maximum of 256 kB of uncompressed data. + +""" +"""Specifying ``block_size=lz4.frame.BLOCKSIZE_DEFAULT`` will instruct the LZ4 +library to use the default maximum blocksize. + +""" + +BLOCKSIZE_MAX1MB = _BLOCKSIZE_MAX1MB +"""Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX1MB`` will instruct the LZ4 +library to create blocks containing a maximum of 1 MB of uncompressed data. + +""" + +BLOCKSIZE_MAX4MB = _BLOCKSIZE_MAX4MB +"""Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX4MB`` will instruct the LZ4 +library to create blocks containing a maximum of 4 MB of uncompressed data. + +""" + +COMPRESSIONLEVEL_MIN = 0 +"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MIN`` will instruct +the LZ4 library to use a compression level of 0 + +""" + +COMPRESSIONLEVEL_MINHC = 3 +"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MINHC`` will +instruct the LZ4 library to use a compression level of 3, the minimum for the +high compression mode. + +""" + +COMPRESSIONLEVEL_MAX = 16 +"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MAX`` will instruct +the LZ4 library to use a compression level of 16, the highest compression level +available. + +""" + + class LZ4FrameCompressor(object): """Create a LZ4 compressor object, which can be used to compress data incrementally. diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 58359e27..1d6f08c6 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1706,9 +1706,6 @@ MODULE_INIT_FUNC (_frame) PyModule_AddIntConstant (module, "BLOCKSIZE_MAX256KB", LZ4F_max256KB); PyModule_AddIntConstant (module, "BLOCKSIZE_MAX1MB", LZ4F_max1MB); PyModule_AddIntConstant (module, "BLOCKSIZE_MAX4MB", LZ4F_max4MB); - PyModule_AddIntConstant (module, "COMPRESSIONLEVEL_MIN", 0); - PyModule_AddIntConstant (module, "COMPRESSIONLEVEL_MINHC", 3); - PyModule_AddIntConstant (module, "COMPRESSIONLEVEL_MAX", 16); return module; } From cd8e2ddfa8cde77b1f639c9197b2b0e3d642374f Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 10 Feb 2018 09:59:15 +0000 Subject: [PATCH 09/10] Use autodoc to add docstrings for frame module attributes --- docs/lz4.frame.rst | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/lz4.frame.rst b/docs/lz4.frame.rst index 5a918e34..fe118256 100644 --- a/docs/lz4.frame.rst +++ b/docs/lz4.frame.rst @@ -93,20 +93,31 @@ Compression level The following module attributes can be used when setting the ``compression_level`` argument. -.. autoattribute:: lz4.frame.COMPRESSIONLEVEL_MIN +.. autodata:: lz4.frame.COMPRESSIONLEVEL_MIN + :annotation: -.. autoattribute:: lz4.frame.COMPRESSIONLEVEL_MINHC - -.. autoattribute:: lz4.frame.COMPRESSIONLEVEL_MAX +.. autodata:: lz4.frame.COMPRESSIONLEVEL_MINHC + :annotation: +.. autodata:: lz4.frame.COMPRESSIONLEVEL_MAX + :annotation: Block size ~~~~~~~~~~ The following attributes can be used when setting the ``block_size`` argument. -.. autoattribute:: lz4.frame.BLOCKSIZE_DEFAULT -.. autoattribute:: lz4.frame.BLOCKSIZE_MAX64KB -.. autoattribute:: lz4.frame.BLOCKSIZE_MAX256KB -.. autoattribute:: lz4.frame.BLOCKSIZE_MAX1MB -.. autoattribute:: lz4.frame.BLOCKSIZE_MAX4MB +.. autodata:: lz4.frame.BLOCKSIZE_DEFAULT + :annotation: + +.. autodata:: lz4.frame.BLOCKSIZE_MAX64KB + :annotation: + +.. autodata:: lz4.frame.BLOCKSIZE_MAX256KB + :annotation: + +.. autodata:: lz4.frame.BLOCKSIZE_MAX1MB + :annotation: + +.. autodata:: lz4.frame.BLOCKSIZE_MAX4MB + :annotation: From c343400d9807e8969a279170f33e65b424bc0795 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 10 Feb 2018 21:36:14 +0000 Subject: [PATCH 10/10] Update bundled py3c headers to version 1.0 files --- py3c/py3c/comparison.h | 20 ++++++++++++++++++++ py3c/py3c/compat.h | 6 +++--- py3c/py3c/py3shims.h | 17 ++++++++++++++++- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/py3c/py3c/comparison.h b/py3c/py3c/comparison.h index 61c6239e..9d3765a4 100644 --- a/py3c/py3c/comparison.h +++ b/py3c/py3c/comparison.h @@ -13,6 +13,26 @@ return Py_INCREF(Py_NotImplemented), Py_NotImplemented #endif +#ifndef Py_UNREACHABLE +#define Py_UNREACHABLE() abort() +#endif + +#ifndef Py_RETURN_RICHCOMPARE +#define Py_RETURN_RICHCOMPARE(val1, val2, op) \ + do { \ + switch (op) { \ + case Py_EQ: if ((val1) == (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ + case Py_NE: if ((val1) != (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ + case Py_LT: if ((val1) < (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ + case Py_GT: if ((val1) > (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ + case Py_LE: if ((val1) <= (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ + case Py_GE: if ((val1) >= (val2)) Py_RETURN_TRUE; Py_RETURN_FALSE; \ + default: \ + Py_UNREACHABLE(); \ + } \ + } while (0) +#endif + #define PY3C_RICHCMP(val1, val2, op) \ ((op) == Py_EQ) ? PyBool_FromLong((val1) == (val2)) : \ ((op) == Py_NE) ? PyBool_FromLong((val1) != (val2)) : \ diff --git a/py3c/py3c/compat.h b/py3c/py3c/compat.h index b80c9735..15d32def 100644 --- a/py3c/py3c/compat.h +++ b/py3c/py3c/compat.h @@ -28,7 +28,7 @@ #define PyStr_InternFromString PyUnicode_InternFromString #define PyStr_Decode PyUnicode_Decode -#define PyStr_AsUTF8String PyUnicode_AsUTF8String // returns PyBytes +#define PyStr_AsUTF8String PyUnicode_AsUTF8String /* returns PyBytes */ #define PyStr_AsUTF8 PyUnicode_AsUTF8 #define PyStr_AsUTF8AndSize PyUnicode_AsUTF8AndSize @@ -74,11 +74,11 @@ #define PyStr_Decode PyString_Decode #ifdef __GNUC__ -static PyObject * PyStr_Concat(PyObject *left, PyObject *right) __attribute__ ((unused)); +static PyObject *PyStr_Concat(PyObject *left, PyObject *right) __attribute__ ((unused)); #endif static PyObject *PyStr_Concat(PyObject *left, PyObject *right) { PyObject *str = left; - Py_INCREF(left); // reference to old left will be stolen + Py_INCREF(left); /* reference to old left will be stolen */ PyString_Concat(&str, right); if (str) { return str; diff --git a/py3c/py3c/py3shims.h b/py3c/py3c/py3shims.h index b1375a4b..947595e9 100644 --- a/py3c/py3c/py3shims.h +++ b/py3c/py3c/py3shims.h @@ -3,7 +3,7 @@ */ /* - * Shims for the PyMem_Raw* functions added inPython 3.3 + * Shims for new functionality from in Python 3.3+ * * See https://docs.python.org/3/c-api/memory.html#raw-memory-interface */ @@ -14,6 +14,19 @@ #include +/* Py_UNUSED - added in Python 3.4, documneted in 3.7 */ + +#ifndef Py_UNUSED +#ifdef __GNUC__ +#define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) +#else +#define Py_UNUSED(name) _unused_ ## name +#endif +#endif + + +/* PyMem_Raw{Malloc,Realloc,Free} - added in Python 3.4 */ + #if PY_MAJOR_VERSION < 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 4) #define PyMem_RawMalloc(n) malloc((n) || 1) #define PyMem_RawRealloc(p, n) realloc(p, (n) || 1) @@ -21,6 +34,8 @@ #endif /* version < 3.4 */ +/* PyMem_RawCalloc - added in Python 3.5 */ + #if PY_MAJOR_VERSION < 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 5) #define PyMem_RawCalloc(n, s) calloc((n) || 1, (s) || 1) #endif /* version < 3.5 */