From 3918e285b36154c69138587ef88c23884fb904ca Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Wed, 14 Feb 2018 23:41:48 +0000 Subject: [PATCH 1/6] Fix some Flake8 errors --- lz4/frame/__init__.py | 184 +++++++++++++++++++++++------------------- 1 file changed, 99 insertions(+), 85 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index a75cc35c..f7513532 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -4,7 +4,7 @@ import os import builtins import sys -from ._frame import ( +from ._frame import ( # noqa: F401 compress, decompress, create_compression_context, @@ -26,8 +26,8 @@ __doc__ = _doc try: - import _compression # Python 3.6 and later -except: + import _compression # Python 3.6 and later +except ImportError: from . import _compression @@ -50,7 +50,7 @@ """ """Specifying ``block_size=lz4.frame.BLOCKSIZE_DEFAULT`` will instruct the LZ4 -library to use the default maximum blocksize. +library to use the default maximum block size. """ @@ -67,8 +67,8 @@ """ COMPRESSIONLEVEL_MIN = 0 -"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MIN`` will instruct -the LZ4 library to use a compression level of 0 +"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MIN`` will +instruct the LZ4 library to use a compression level of 0 """ @@ -80,9 +80,9 @@ """ COMPRESSIONLEVEL_MAX = 16 -"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MAX`` will instruct -the LZ4 library to use a compression level of 16, the highest compression level -available. +"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MAX`` will +instruct the LZ4 library to use a compression level of 16, the highest +compression level available. """ @@ -104,9 +104,9 @@ class LZ4FrameCompressor(object): If unspecified, will default to `lz4.frame.BLOCKSIZE_DEFAULT` which is equal to `lz4.frame.BLOCKSIZE_MAX64KB`. block_linked (bool): Specifies whether to use block-linked - compression. If ``True``, the compression ratio is improved, especially - for small block sizes. If ``False`` the blocks are compressed independently. - The default is ``True``. + compression. If ``True``, the compression ratio is improved, + especially for small block sizes. If ``False`` the blocks are + compressed independently. The default is ``True``. compression_level (int): Specifies the level of compression used. Values between 0-16 are valid, with 0 (default) being the lowest compression (0-2 are the same value), and 16 the highest. @@ -119,24 +119,24 @@ class LZ4FrameCompressor(object): - `lz4.frame.COMPRESSIONLEVEL_MAX`: Maximum compression (16) content_checksum (bool): Specifies whether to enable checksumming of - the payload content. If ``True`` a checksum of the uncompressed data - is stored at the end of the compressed frame which is checked during - decompression. The default is ``False``. + the payload content. If ``True``, a checksum of the uncompressed + data is stored at the end of the compressed frame which is checked + during decompression. The default is ``False``. block_checksum (bool): Specifies whether to enable checksumming of - the content of each block. If ``True`` a checksum of the uncompressed - data in each block in the frame is stored at the end of each block. - If present, these checksums will be used to validate the data during - decompression. The default is ``False``, meaning block checksums are not - calculated and stored. This functionality is only supported if the - underlying LZ4 library has version >= 1.8.0. Attempting to set this - value to ``True`` with a version of LZ4 < 1.8.0 will cause a ``RuntimeError`` - to be raised. - auto_flush (bool): When ``False``, the LZ4 library may buffer data until a - block is full. When ``True`` no buffering occurs, and partially full - blocks may be returned. The default is ``False``. - return_bytearray (bool): When ``False`` a ``bytes`` object is returned from the - calls to methods of this class. When ``True`` a ``bytearray`` object will be - returned. The default is ``False``. + the content of each block. If ``True`` a checksum of the + uncompressed data in each block in the frame is stored at the end + of each block. If present, these checksums will be used to + validate the data during decompression. The default is ``False``, + meaning block checksums are not calculated and stored. This + functionality is only supported if the underlying LZ4 library has + version >= 1.8.0. Attempting to set this value to ``True`` with a + version of LZ4 < 1.8.0 will cause a ``RuntimeError`` to be raised. + auto_flush (bool): When ``False``, the LZ4 library may buffer data + until a block is full. When ``True`` no buffering occurs, and + partially full blocks may be returned. The default is ``False``. + return_bytearray (bool): When ``False`` a ``bytes`` object is returned + from the calls to methods of this class. When ``True`` a + ``bytearray`` object will be returned. The default is ``False``. """ def __init__(self, @@ -153,7 +153,8 @@ def __init__(self, self.content_checksum = content_checksum if block_checksum and lz4.library_version_number() < 10800: raise RuntimeError( - 'Attempt to set block_checksum to True with LZ4 library version < 10800' + 'Attempt to set block_checksum to True with LZ4 library' + 'version < 10800' ) self.block_checksum = block_checksum self.auto_flush = auto_flush @@ -176,7 +177,6 @@ def __exit__(self, exception_type, exception, traceback): self._context = None self._started = False - def begin(self, source_size=0): """Begin a compression frame. The returned data contains frame header information. The data returned from subsequent calls to ``compress()`` @@ -212,7 +212,7 @@ def begin(self, source_size=0): 'LZ4FrameCompressor.begin() called after already initialized' ) - def compress(self, data): + def compress(self, data): # noqa: F811 """Compress ``data`` (a ``bytes`` object), returning a bytes object containing compressed data the input. @@ -245,14 +245,16 @@ def compress(self, data): return result def flush(self): - """Finish the compression process, returning a bytes object containing any data - stored in the compressor's internal buffers and a frame footer. + """Finish the compression process, returning a bytes object containing + any data stored in the compressor's internal buffers and a frame + footer. The LZ4FrameCompressor instance may be re-used after this method has been called to create a new frame of compressed data. Returns: - bytes or bytearray: any remaining buffered compressed data and frame footer. + bytes or bytearray: any remaining buffered compressed data and + frame footer. """ result = compress_flush( @@ -264,9 +266,10 @@ def flush(self): self._started = False return result - @deprecation.deprecated(deprecated_in="0.23.1", removed_in="1.0", + @deprecation.deprecated(deprecated_in='0.23.1', removed_in='1.0', current_version=lz4.__version__, - details="Use the LZ4FrameCompressor.flush() method instead") + details='Use the LZ4FrameCompressor.flush() method' + 'instead') def finalize(self): """This function is identical to `LZ4FrameCompressor.flush()` and is provided for backwards compatibility only. You should migrate your code to use @@ -277,7 +280,8 @@ def finalize(self): return result def reset(self): - """Reset the LZ4FrameCompressor instance allowing it to be re-used after an error. + """Reset the LZ4FrameCompressor instance allowing it to be re-used + after an error. """ self._context = None @@ -285,25 +289,25 @@ def reset(self): class LZ4FrameDecompressor(object): - """Create a LZ4 frame decompressor object, which can be used to decompress data - incrementally. + """Create a LZ4 frame decompressor object, which can be used to decompress + data incrementally. For a more convenient way of decompressing an entire compressed frame at once, see `lz4.frame.decompress()`. Args: - return_bytearray (bool): When ``False`` a bytes object is returned from the - calls to methods of this class. When ``True`` a bytearray object will be - returned. The default is ``False``. + return_bytearray (bool): When ``False`` a bytes object is returned from + the calls to methods of this class. When ``True`` a bytearray + object will be returned. The default is ``False``. Attributes: - eof (bool): ``True`` if the end-of-stream marker has been reached. ``False`` - otherwise. + eof (bool): ``True`` if the end-of-stream marker has been reached. + ``False`` otherwise. unused_data (bytes): Data found after the end of the compressed stream. Before the end of the frame is reached, this will be ``b''``. - needs_input (bool): ``False`` if the ``decompress()`` method can provide more - decompressed data before requiring new uncompressed input. ``True`` - otherwise. + needs_input (bool): ``False`` if the ``decompress()`` method can + provide more decompressed data before requiring new uncompressed + input. ``True`` otherwise. """ @@ -338,8 +342,7 @@ def reset(self): self.unused_data = None self._unconsumed_data = b'' - - def decompress(self, data, max_length=-1): + def decompress(self, data, max_length=-1): # noqa: F811 """Decompresses part or all of an LZ4 frame of compressed data. The returned data should be concatenated with the output of any previous calls to `decompress()`. @@ -401,10 +404,10 @@ def decompress(self, data, max_length=-1): return decompressed -_MODE_CLOSED = 0 -_MODE_READ = 1 +_MODE_CLOSED = 0 +_MODE_READ = 1 # Value 2 no longer used -_MODE_WRITE = 3 +_MODE_WRITE = 3 class LZ4FrameFile(_compression.BaseStream): @@ -417,8 +420,9 @@ class LZ4FrameFile(_compression.BaseStream): returned as bytes, and data to be written must be given as bytes. When opening a file for writing, the settings used by the compressor can be - specified. The underlying compressor object is `lz4.frame.LZ4FrameCompressor`. - See the docstrings for that class for details on compression options. + specified. The underlying compressor object is + `lz4.frame.LZ4FrameCompressor`. See the docstrings for that class for + details on compression options. Args: filename(str, bytes, PathLike, file object): can be either an actual @@ -429,16 +433,18 @@ class LZ4FrameFile(_compression.BaseStream): Keyword Args: mode(str): mode can be ``'r'`` for reading (default), ``'w'`` for (over)writing, ``'x'`` for creating exclusively, or ``'a'`` - for appending. These can equivalently be given as ``'rb'``, ``'wb'``, - ``'xb'`` and ``'ab'`` respectively. + for appending. These can equivalently be given as ``'rb'``, + ``'wb'``, ``'xb'`` and ``'ab'`` respectively. return_bytearray (bool): When ``False`` a bytes object is returned from the calls to methods of this class. When ``True`` a ``bytearray`` object will be returned. The default is ``False``. - source_size (int): Optionally specify the total size of the uncompressed - data. If specified, will be stored in the compressed frame header as - an 8-byte field for later use during decompression. Default is ``0`` - (no size stored). Only used for writing compressed files. - block_size (int): Compressor setting. See `lz4.frame.LZ4FrameCompressor`. + source_size (int): Optionally specify the total size of the + uncompressed data. If specified, will be stored in the compressed + frame header as an 8-byte field for later use during decompression. + Default is ``0`` (no size stored). Only used for writing + compressed files. + block_size (int): Compressor setting. See + `lz4.frame.LZ4FrameCompressor`. block_linked (bool): Compressor setting. See `lz4.frame.LZ4FrameCompressor`. compression_level (int): Compressor setting. See @@ -498,7 +504,9 @@ def __init__(self, filename=None, mode='r', self._fp = filename self._mode = mode_code else: - raise TypeError('filename must be a str, bytes, file or PathLike object') + raise TypeError( + 'filename must be a str, bytes, file or PathLike object' + ) if self._mode == _MODE_READ: raw = _compression.DecompressReader(self._fp, LZ4FrameDecompressor) @@ -617,9 +625,9 @@ def read(self, size=-1): return self._buffer.read(size) def read1(self, size=-1): - """Read up to ``size`` uncompressed bytes, while trying to avoid making multiple - reads from the underlying stream. Reads up to a buffer's worth of data - if ``size`` is negative. + """Read up to ``size`` uncompressed bytes, while trying to avoid making + multiple reads from the underlying stream. Reads up to a buffer's worth + of data if ``size`` is negative. Returns ``b''`` if the file is at EOF. @@ -680,7 +688,8 @@ def seek(self, offset, whence=io.SEEK_SET): The new position is specified by ``offset``, relative to the position indicated by ``whence``. Possible values for ``whence`` are: - - ``io.SEEK_SET`` or 0: start of stream (default): offset must not be negative + - ``io.SEEK_SET`` or 0: start of stream (default): offset must not be + negative - ``io.SEEK_CUR`` or 1: current stream position - ``io.SEEK_END`` or 2: end of stream; offset must not be positive @@ -735,9 +744,9 @@ def open(filename, mode="rb", PathLike object), in which case the named file is opened, or it can be an existing file object to read from or write to. - The ``mode`` argument can be ``'r'``, ``'rb'`` (default), ``'w'``, ``'wb'``, - ``'x'``, ``'xb'``, ``'a'``, or ``'ab'`` for binary mode, or ``'rt'``, - ``'wt'``, ``'xt'``, or ``'at'`` for text mode. + The ``mode`` argument can be ``'r'``, ``'rb'`` (default), ``'w'``, + ``'wb'``, ``'x'``, ``'xb'``, ``'a'``, or ``'ab'`` for binary mode, or + ``'rt'``, ``'wt'``, ``'xt'``, or ``'at'`` for text mode. For binary mode, this function is equivalent to the `LZ4FrameFile` constructor: `LZ4FrameFile(filename, mode, ...)`. @@ -751,21 +760,24 @@ def open(filename, mode="rb", Keyword Args: mode (str): mode for opening the file - encoding (str): the name of the encoding that the stream will be decoded - or encoded with. It defaults to ``locale.getpreferredencoding(False)``. - See ``io.TextIOWrapper`` for further details. + encoding (str): the name of the encoding that will be used for + encoding/deconging the stream. It defaults to + ``locale.getpreferredencoding(False)``. See ``io.TextIOWrapper`` + for further details. errors (str): specifies how encoding and decoding errors are to be handled. See ``io.TextIOWrapper`` for further details. newline (str): controls how line endings are handled. See ``io.TextIOWrapper`` for further details. - return_bytearray (bool): When ``False`` a bytes object is returned from the - calls to methods of this class. When ``True`` a bytearray object will be - returned. The default is ``False``. - source_size (int): Optionally specify the total size of the uncompressed - data. If specified, will be stored in the compressed frame header as - an 8-byte field for later use during decompression. Default is 0 - (no size stored). Only used for writing compressed files. - block_size (int): Compressor setting. See `lz4.frame.LZ4FrameCompressor`. + return_bytearray (bool): When ``False`` a bytes object is returned + from the calls to methods of this class. When ``True`` a bytearray + object will be returned. The default is ``False``. + source_size (int): Optionally specify the total size of the + uncompressed data. If specified, will be stored in the compressed + frame header as an 8-byte field for later use during decompression. + Default is 0 (no size stored). Only used for writing compressed + files. + block_size (int): Compressor setting. See + `lz4.frame.LZ4FrameCompressor`. block_linked (bool): Compressor setting. See `lz4.frame.LZ4FrameCompressor`. compression_level (int): Compressor setting. See @@ -778,12 +790,14 @@ def open(filename, mode="rb", `lz4.frame.LZ4FrameCompressor`. """ - if "t" in mode: - if "b" in mode: - raise ValueError("Invalid mode: %r" % (mode,)) + if 't' in mode: + if 'b' in mode: + raise ValueError('Invalid mode: %r' % (mode,)) else: if encoding is not None: - raise ValueError("Argument 'encoding' not supported in binary mode") + raise ValueError( + "Argument 'encoding' not supported in binary mode" + ) if errors is not None: raise ValueError("Argument 'errors' not supported in binary mode") if newline is not None: From ca26fe73115aebfd729c98121d07ade13b72a689 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Thu, 15 Feb 2018 00:08:35 +0000 Subject: [PATCH 2/6] Cleanup some docstrings --- lz4/frame/_frame.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 5b7eab91..c14d03f1 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1506,7 +1506,7 @@ PyDoc_STRVAR " default is False.\n" \ "\n" \ "Returns:\n" \ - " bytes or bytearray: Compressed data\n\n" \ + " bytes or bytearray: Compressed data.\n\n" \ "Notes:\n" \ " If auto flush is disabled (``auto_flush=False`` when calling\n" \ " `lz4.frame.compress_begin`) this function may buffer and retain\n" \ @@ -1539,8 +1539,8 @@ PyDoc_STRVAR " The default is ``False``.\n" \ "\n" \ "Returns:\n" \ - " bytes or bytearray: Remaining (buffered) compressed data, and\n" \ - " optionally an end frame marker and frame content checksum.\n" \ + " bytes or bytearray: Any buffered compressed data, and optionally\n" \ + " an end of frame marker and frame content checksum.\n" \ "\n" \ "Notes:\n" \ " If ``end_frame`` is ``False`` but the underlying LZ4 library doesn't" \ From 7b702507073da7dce79ef4a5b5096c77b3aa0234 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 17 Feb 2018 10:01:36 +0000 Subject: [PATCH 3/6] Improve some docstrings --- lz4/frame/__init__.py | 107 ++++++++++++++++++++++++--------------- lz4/frame/_frame.c | 113 ++++++++++++++++++++++++------------------ 2 files changed, 132 insertions(+), 88 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index f7513532..4c0abb69 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -32,55 +32,67 @@ BLOCKSIZE_DEFAULT = _BLOCKSIZE_DEFAULT -"""Specifying ``block_size=lz4.frame.BLOCKSIZE_DEFAULT`` will instruct the LZ4 +"""Specifier for the default block size. + +Specifying ``block_size=lz4.frame.BLOCKSIZE_DEFAULT`` will instruct the LZ4 library to use the default maximum blocksize. This is currently equivalent to `lz4.frame.BLOCKSIZE_MAX64KB` """ BLOCKSIZE_MAX64KB = _BLOCKSIZE_MAX64KB -"""Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX64KB`` will instruct the LZ4 +"""Specifier for a maximum block size of 64 kB. + +Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX64KB`` will instruct the LZ4 library to create blocks containing a maximum of 64 kB of uncompressed data. """ BLOCKSIZE_MAX256KB = _BLOCKSIZE_MAX256KB -"""Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX256KB`` will instruct the LZ4 -library to create blocks containing a maximum of 256 kB of uncompressed data. +"""Specifier for a maximum block size of 256 kB. -""" -"""Specifying ``block_size=lz4.frame.BLOCKSIZE_DEFAULT`` will instruct the LZ4 -library to use the default maximum block size. +Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX256KB`` will instruct the LZ4 +library to create blocks containing a maximum of 256 kB of uncompressed data. """ BLOCKSIZE_MAX1MB = _BLOCKSIZE_MAX1MB -"""Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX1MB`` will instruct the LZ4 +"""Specifier for a maximum block size of 1 MB. + +Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX1MB`` will instruct the LZ4 library to create blocks containing a maximum of 1 MB of uncompressed data. """ BLOCKSIZE_MAX4MB = _BLOCKSIZE_MAX4MB -"""Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX4MB`` will instruct the LZ4 +"""Specifier for a maximum block size of 4 MB. + +Specifying ``block_size=lz4.frame.BLOCKSIZE_MAX4MB`` will instruct the LZ4 library to create blocks containing a maximum of 4 MB of uncompressed data. """ COMPRESSIONLEVEL_MIN = 0 -"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MIN`` will +"""Specifier for the minimum compression level. + +Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MIN`` will instruct the LZ4 library to use a compression level of 0 """ COMPRESSIONLEVEL_MINHC = 3 -"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MINHC`` will +"""Specifier for the minimum compression level for high compression mode. + +Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MINHC`` will instruct the LZ4 library to use a compression level of 3, the minimum for the high compression mode. """ COMPRESSIONLEVEL_MAX = 16 -"""Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MAX`` will +"""Specifier for the maximum compression level. + +Specifying ``compression_level=lz4.frame.COMPRESSIONLEVEL_MAX`` will instruct the LZ4 library to use a compression level of 16, the highest compression level available. @@ -88,8 +100,9 @@ class LZ4FrameCompressor(object): - """Create a LZ4 compressor object, which can be used to compress data - incrementally. + """Create a LZ4 frame compressor object. + + This object can be used to compress data incrementally. Args: block_size (int): Specifies the maximum blocksize to use. @@ -137,6 +150,7 @@ class LZ4FrameCompressor(object): return_bytearray (bool): When ``False`` a ``bytes`` object is returned from the calls to methods of this class. When ``True`` a ``bytearray`` object will be returned. The default is ``False``. + """ def __init__(self, @@ -178,9 +192,11 @@ def __exit__(self, exception_type, exception, traceback): self._started = False def begin(self, source_size=0): - """Begin a compression frame. The returned data contains frame header - information. The data returned from subsequent calls to ``compress()`` - should be concatenated with this header. + """Begin a compression frame. + + The returned data contains frame header information. The data returned + from subsequent calls to ``compress()`` should be concatenated with + this header. Keyword Args: source_size (int): Optionally specify the total size of the @@ -190,6 +206,7 @@ def begin(self, source_size=0): Returns: bytes or bytearray: frame header data + """ if self._started is False: @@ -213,8 +230,10 @@ def begin(self, source_size=0): ) def compress(self, data): # noqa: F811 - """Compress ``data`` (a ``bytes`` object), returning a bytes object - containing compressed data the input. + """Compresses data and returns it. + + This compresses ``data`` (a ``bytes`` object), returning a bytes or + bytearray object containing compressed data the input. If ``auto_flush`` has been set to ``False``, some of ``data`` may be buffered internally, for use in later calls to @@ -245,16 +264,16 @@ def compress(self, data): # noqa: F811 return result def flush(self): - """Finish the compression process, returning a bytes object containing - any data stored in the compressor's internal buffers and a frame - footer. + """Finish the compression process. + + This returns a bytes or bytearray object containing any data stored in + the compressor's internal buffers and a frame footer. The LZ4FrameCompressor instance may be re-used after this method has been called to create a new frame of compressed data. Returns: - bytes or bytearray: any remaining buffered compressed data and - frame footer. + bytes or bytearray: compressed data and frame footer. """ result = compress_flush( @@ -271,17 +290,20 @@ def flush(self): details='Use the LZ4FrameCompressor.flush() method' 'instead') def finalize(self): - """This function is identical to `LZ4FrameCompressor.flush()` and is provided - for backwards compatibility only. You should migrate your code to use - `LZ4FrameCompressor.flush()`. + """This function is identical to `LZ4FrameCompressor.flush()`. + + This is provided for backwards compatibility only. You should migrate + your code to use `LZ4FrameCompressor.flush()`. """ result = self.flush() return result def reset(self): - """Reset the LZ4FrameCompressor instance allowing it to be re-used - after an error. + """Reset the `LZ4FrameCompressor` instance. + + This allows the `LZ4FrameCompression` instance to be re-used after an + error. """ self._context = None @@ -289,8 +311,9 @@ def reset(self): class LZ4FrameDecompressor(object): - """Create a LZ4 frame decompressor object, which can be used to decompress - data incrementally. + """Create a LZ4 frame decompressor object. + + This can be used to decompress data incrementally. For a more convenient way of decompressing an entire compressed frame at once, see `lz4.frame.decompress()`. @@ -332,8 +355,9 @@ def __exit__(self, exception_type, exception, traceback): self._return_bytearray = None def reset(self): - """Reset the decompressor state. This is useful after an error occurs, allowing - re-use of the instance. + """Reset the decompressor state. + + This is useful after an error occurs, allowing re-use of the instance. """ reset_decompression_context(self._context) @@ -343,9 +367,10 @@ def reset(self): self._unconsumed_data = b'' def decompress(self, data, max_length=-1): # noqa: F811 - """Decompresses part or all of an LZ4 frame of compressed data. The returned - data should be concatenated with the output of any previous calls to - `decompress()`. + """Decompresses part or all of an LZ4 frame of compressed data. + + The returned data should be concatenated with the output of any + previous calls to `decompress()`. If ``max_length`` is non-negative, returns at most ``max_length`` bytes of decompressed data. If this limit is reached and further output can @@ -625,9 +650,13 @@ def read(self, size=-1): return self._buffer.read(size) def read1(self, size=-1): - """Read up to ``size`` uncompressed bytes, while trying to avoid making - multiple reads from the underlying stream. Reads up to a buffer's worth - of data if ``size`` is negative. + """Read up to ``size`` uncompressed bytes. + + This method tries to avoid making multiple reads from the underlying + stream. + + This method reads up to a buffer's worth of data if ``size`` is + negative. Returns ``b''`` if the file is at EOF. diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index c14d03f1..cfd89bc8 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1392,14 +1392,15 @@ PyDoc_STRVAR( create_compression_context__doc, "create_compression_context()\n" \ "\n" \ - "Creates a Compression Context object, which will be used in all\n" \ - "compression operations.\n" \ + "Creates a compression context object.\n" \ + "\n" \ + "The compression object is required for compression operations.\n" \ "\n" \ "Returns:\n" \ " cCtx: A compression context\n" ); -#define COMPRESS_KWARGS_DOCSTRING \ +#define COMPRESS_KWARGS_DOCSTRING \ " block_size (int): Sepcifies the maximum blocksize to use.\n" \ " Options:\n\n" \ " - `lz4.frame.BLOCKSIZE_DEFAULT`: the lz4 library default\n" \ @@ -1435,26 +1436,31 @@ PyDoc_STRVAR( " ``False`` meaning block checksums are not calculated and stored.\n" \ " This functionality is only supported if the underlying LZ4\n" \ " library has version >= 1.8.0. Attempting to set this value\n" \ - " to ``True`` with a version of LZ4 < 1.8.0 will cause a ``RuntimeError``\n" \ - " to be raised.\n" \ - " return_bytearray (bool): If ``True`` a ``bytearray`` object will be returned.\n" \ - " If ``False``, a string of bytes is returned. The default is ``False``.\n" \ + " to ``True`` with a version of LZ4 < 1.8.0 will cause a\n" \ + " ``RuntimeError`` to be raised.\n" \ + " return_bytearray (bool): If ``True`` a ``bytearray`` object will be\n" \ + " returned. If ``False``, a string of bytes is returned. The default\n" \ + " is ``False``.\n" \ PyDoc_STRVAR( compress__doc, "compress(data, compression_level=0, block_size=0, content_checksum=0,\n" \ "block_linked=True, store_size=True, return_bytearray=False)\n" \ "\n" \ - "Compresses ``data`` returning the compressed data as a complete frame.\n\n" \ + "Compresses ``data`` returning the compressed data as a complete frame.\n" \ + "\n" \ "The returned data includes a header and endmark and so is suitable\n" \ - "for writing to a file.\n\n" \ + "for writing to a file.\n" \ + "\n" \ "Args:\n" \ - " data (str, bytes or buffer-compatible object): data to compress\n\n" \ + " data (str, bytes or buffer-compatible object): data to compress\n" \ + "\n" \ "Keyword Args:\n" \ COMPRESS_KWARGS_DOCSTRING \ " store_size (bool): If ``True`` then the frame will include an 8-byte\n" \ " header field that is the uncompressed size of data included\n" \ - " within the frame. Default is ``True``.\n\n" \ + " within the frame. Default is ``True``.\n" \ + "\n" \ "Returns:\n" \ " bytes or bytearray: Compressed data\n" ); @@ -1470,13 +1476,12 @@ PyDoc_STRVAR " context (cCtx): A compression context.\n\n" \ "Keyword Args:\n" \ COMPRESS_KWARGS_DOCSTRING \ - " auto_flush (bool): Enable or disable autoFlush. When autoFlush is\n" \ - " disabled, the LZ4 library may buffer data internally until a\n" \ - " block is full. Default is ``False`` (autoFlush disabled).\n\n" \ + " auto_flush (bool): Enable or disable autoFlush. When autoFlush is disabled\n" \ + " the LZ4 library may buffer data internally until a block is full.\n" \ + " Default is ``False`` (autoFlush disabled).\n\n" \ " source_size (int): This optionally specifies the uncompressed size\n" \ - " of the data to be compressed. If specified, the size will be\n" \ - " stored in the frame header for use during decompression.\n" \ - " Default is ``True``.\n" \ + " of the data to be compressed. If specified, the size will be stored\n" \ + " in the frame header for use during decompression. Default is ``True``\n" \ " return_bytearray (bool): If ``True`` a bytearray object will be returned.\n" \ " If ``False``, a string of bytes is returned. Default is ``False``.\n\n" \ "Returns:\n" \ @@ -1493,7 +1498,7 @@ PyDoc_STRVAR "Compresses blocks of data and returns the compressed data.\n" \ "\n" \ "The returned data should be concatenated with the data returned from\n" \ - "`lz4.frame.compress_begin` and any subsequent calls to\n" \ + "`lz4.frame.compress_begin` and any subsequent calls to\n" \ "`lz4.frame.compress_chunk`.\n" \ "\n" \ "Args:\n" \ @@ -1519,33 +1524,38 @@ PyDoc_STRVAR compress_flush__doc, "compress_flush(context, end_frame=True, return_bytearray=False)\n" \ "\n" \ - "Flushes a compression context returning any data buffed in the context\n" \ - "as compressed data. The returned data should be appended to the\n" \ - "output of previous calls to ``lz4.frame.compress_chunk``. The\n" \ - "``end_frame`` argument specifies whether or not the frame should be\n" \ - "ended.\n" \ + "Flushes any buffered data held in the compression context.\n" \ + "\n" \ + "This flushes any data buffed in the compression context, returning it as\n" \ + "compressed data. The returned data should be appended to the output of\n" \ + "previous calls to ``lz4.frame.compress_chunk``.\n" \ + "\n" \ + "The ``end_frame`` argument specifies whether or not the frame should be\n" \ + "ended. If this is ``True`` and end of frame marker will be appended to\n" \ + "the returned data. In this case, if ``content_checksum`` was ``True``\n" \ + "when calling `lz4.frame.compress_begin`, then a checksum of the uncompressed\n" \ + "data will also be included in the returned data.\n" \ + "\n" \ + "If the ``end_frame`` argument is ``True``, the compression context will be\n" \ + "reset and can be re-used.\n" \ "\n" \ "Args:\n" \ " context (cCtx): Compression context\n" \ - " end_frame (bool): If ``True``, in addition to flushing any buffered\n" \ - " data, an end frame marker (and possibly a checksum) will be\n" \ - " appended to the data returned. In this case the compression\n" \ - " context will be reset and can be used for creating a new frame.\n" \ - " Default is ``True``.\n" \ "\n" \ "Keyword Args:\n" \ + " end_frame (bool): If ``True`` the frame will be ended. Default is\n" \ + " ``True``.\n" \ " return_bytearray (bool): If ``True`` a ``bytearray`` object will\n" \ " be returned. If ``False``, a ``bytes`` object is returned.\n" \ " The default is ``False``.\n" \ "\n" \ "Returns:\n" \ - " bytes or bytearray: Any buffered compressed data, and optionally\n" \ - " an end of frame marker and frame content checksum.\n" \ + " bytes or bytearray: compressed data.\n" \ "\n" \ "Notes:\n" \ - " If ``end_frame`` is ``False`` but the underlying LZ4 library doesn't" \ - " support flushing without ending the frame, a ``RuntimeError``\n" \ - " will be raised.\n" + " If ``end_frame`` is ``False`` but the underlying LZ4 library does not" \ + " support flushing without ending the frame, a ``RuntimeError`` will be\n" \ + " raised.\n" ); PyDoc_STRVAR @@ -1567,12 +1577,12 @@ PyDoc_STRVAR " - ``content_size`` (int): uncompressed size in bytes of\n" \ " frame content\n" \ " - ``block_linked`` (bool): specifies whether the frame contains\n" \ - " blocks which are independently compressed (``False``) or\n" \ + " blocks which are independently compressed (``False``) or linked\n" \ " linked (``True``)\n" \ - " - ``block_checksum`` (bool): specifies whether each block\n" \ - " contains a checksum of its contents\n" \ - " - ``skippable`` (bool): whether the block is skippable (``True``)\n" \ - " or not (``False``)\n" + " - ``block_checksum`` (bool): specifies whether each block contains a\n" \ + " checksum of its contents\n" \ + " - ``skippable`` (bool): whether the block is skippable (``True``) or\n" \ + " not (``False``)\n" ); PyDoc_STRVAR @@ -1580,8 +1590,9 @@ PyDoc_STRVAR create_decompression_context__doc, "create_decompression_context()\n" \ "\n" \ - "Creates a decompression context object, which will be used for\n" \ - "decompression operations.\n" \ + "Creates a decompression context object.\n" \ + "\n" \ + "A decompression context is needed for decompression operations.\n" \ "\n" \ "Returns:\n" \ " dCtx: A decompression context\n" @@ -1592,11 +1603,12 @@ PyDoc_STRVAR reset_decompression_context__doc, "reset_decompression_context(context)\n" \ "\n" \ - "Resets a decompression context object. This is useful for recovering\n" \ - "from an error or for stopping an unfinished decompression and starting\n" \ - "a new one with the same context\n" \ - "\n" \ - "Args:\n" \ + "Resets a decompression context object.\n" \ + "\n" \ + "This is useful for recovering from an error or for stopping an unfinished\n" \ + "decompression and starting a new one with the same context\n" \ + "\n" \ + "Args:\n" \ " context (dCtx): A decompression context\n" ); @@ -1634,13 +1646,16 @@ PyDoc_STRVAR decompress_chunk__doc, "decompress(context, data)\n" \ "\n" \ - "Decompresses part of a frame of data. The returned uncompressed data\n" \ - "should be catenated with the data returned from previous calls to\n" \ - "`decompress_chunk`\n\n" \ + "Decompresses part of a frame of compressed data.\n" \ + "\n" \ + "The returned uncompressed data should be concatenated with the data returned\n" \ + "from previous calls to `lz4.frame.decompress_chunk`\n" \ + "\n" \ "Args:\n" \ " context (dCtx): decompression context\n" \ " data (str, bytes or buffer-compatible object): part of a LZ4\n" \ - " frame of compressed data\n\n" \ + " frame of compressed data\n" \ + "\n" \ "Keyword Args:\n" \ " max_length (int): if non-negative this specifies the maximum number" \ " of bytes of uncompressed data to return. Default is ``-1``.\n" \ From f7698d8e250f04b8028b4c956bab2ee55853c0f5 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 17 Feb 2018 10:06:27 +0000 Subject: [PATCH 4/6] Improve some docstrings --- lz4/frame/_frame.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index cfd89bc8..5e8dbcac 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1664,14 +1664,17 @@ PyDoc_STRVAR " default is ``False``.\n" \ "\n" \ "Returns:\n" \ - " tuple: (uncompressed data, bytes read, end of frame indicator)\n" \ + " tuple: uncompressed data, bytes read, end of frame indicator\n" \ + "\n" \ " This function returns a tuple consisting of:\n" \ "\n" \ - " - bytes or bytearray: Uncompressed data\n" \ - " - int: Number of bytes consumed from input ``data``\n" \ - " - bool: ``True`` if the end of the compressed frame has been\n" \ - " reached. ``False`` otherwise.\n" - ); + " - The uncompressed data as a ``bytes`` or ``bytearray`` object\n" \ + " - The number of bytes consumed from input ``data`` as an ``int``\n" \ + " - The end of frame indicator as a ``bool``.\n" \ + "\n" + "The end of frame indicator is ``True`` if the end of the compressed frame\n" \ + "has been reached, or ``False`` otherwise\n" + ); static PyMethodDef module_methods[] = { From 342fba4a655f6b5461d766504639df971d57e3a7 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 17 Feb 2018 10:12:25 +0000 Subject: [PATCH 5/6] Further improve docstrings --- lz4/frame/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 4c0abb69..e4d73924 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -266,8 +266,8 @@ def compress(self, data): # noqa: F811 def flush(self): """Finish the compression process. - This returns a bytes or bytearray object containing any data stored in - the compressor's internal buffers and a frame footer. + This returns a ``bytes`` or ``bytearray`` object containing any data + stored in the compressor's internal buffers and a frame footer. The LZ4FrameCompressor instance may be re-used after this method has been called to create a new frame of compressed data. From 4dddd9f34411c12dea7197256ce6a6e38bbfcdab Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sat, 17 Feb 2018 10:54:07 +0000 Subject: [PATCH 6/6] Don't show the doctest directives in quickstart.rst output --- docs/quickstart.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 9d0cfe1c..8c4e3a52 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -11,7 +11,7 @@ The recommended binding to use is the LZ4 frame format binding, since this provides interoperability with other implementations and language bindings. The simplest way to use the frame bindings is via the :py:func:`compress` and -:py:func:`decompress` functions:: +:py:func:`decompress` functions: .. doctest:: @@ -34,7 +34,7 @@ Working with data in chunks --------------------------- It's often inconvenient to hold the full data in memory, and so functions are -also provided to compress and decompress data in chunks:: +also provided to compress and decompress data in chunks: .. doctest:: @@ -61,7 +61,7 @@ can be disabled by specifying ``auto_flush=True`` when calling time without ending the frame by calling :py:func:`compress_flush` with ``end_frame=False``. -Decompressing data can also be done in a chunked fashion:: +Decompressing data can also be done in a chunked fashion: .. doctest:: @@ -82,7 +82,7 @@ marker. Rather than managing compression and decompression context objects manually, it is more convenient to use the :py:class:`LZ4FrameCompressor` and :py:class:`LZ4FrameDecompressor` classes which provide context manager -functionality:: +functionality: .. doctest:: @@ -108,7 +108,7 @@ The frame bindings provide capability for working with files containing LZ4 frame compressed data. This functionality is intended to be a drop in replacement for that offered in the Python standard library for bz2, gzip and LZMA compressed files. The :py:func:`lz4.frame.open()` function is the most -convenient way to work with compressed data files:: +convenient way to work with compressed data files: .. doctest::