Skip to content

gh-89550: Buffer GzipFile.write to reduce execution time by ~15% #101251

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gpshead merged 17 commits into from
May 8, 2023
Merged

gh-89550: Buffer GzipFile.write to reduce execution time by ~15% #101251

Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
943ca9c
initial testing to see if it's faster
CCLDArjun Jan 22, 2023
6f4701b
Merge branch 'python:main' into buffered-gzip-writes
CCLDArjun Jan 22, 2023
8653faf
use io.BufferedWriter to buffer gzip writes
CCLDArjun Jan 22, 2023
1c36625
:Merge branch 'buffered-gzip-writes' of https://github.com/ccldarjun/…
CCLDArjun Jan 22, 2023
bc4d929
news blurb
CCLDArjun Jan 22, 2023
ef9dd50
Merge branch 'main' into buffered-gzip-writes
CCLDArjun Jan 26, 2023
b3aa1f7
added buffer_size for file writes
CCLDArjun Feb 18, 2023
ad2dc9b
:Merge branch 'buffered-gzip-writes' of https://github.com/ccldarjun/…
CCLDArjun Feb 18, 2023
a929ac8
comment for _write_raw
CCLDArjun Feb 18, 2023
d1618f2
common DEFAULT_WRITE_BUFFER_SIZE
CCLDArjun Feb 18, 2023
a7efd1b
Merge branch 'main' into buffered-gzip-writes
CCLDArjun Feb 18, 2023
d0da047
Merge branch 'main' into buffered-gzip-writes
CCLDArjun Feb 22, 2023
fe31588
use constant for write buffer size
CCLDArjun Feb 23, 2023
875ebd8
Merge branch 'main' into buffered-gzip-writes
CCLDArjun Apr 24, 2023
e33fc19
Merge branch 'main' into buffered-gzip-writes
AlexWaygood May 8, 2023
5ef4674
improve news wording and ReSTify
gpshead May 8, 2023
c0f3d2a
Merge branch 'main' into buffered-gzip-writes
gpshead May 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
use constant for write buffer size
  • Loading branch information
@CCLDArjun
CCLDArjun committed Feb 23, 2023
commit fe31588314f11c40d5a9de54bd15ec746898bf6e
22 changes: 10 additions & 12 deletions Lib/gzip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,11 @@
_COMPRESS_LEVEL_BEST = 9

READ_BUFFER_SIZE = 128 * 1024
DEFAULT_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE
_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE


def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST,
encoding=None, errors=None, newline=None,
buffer_size=DEFAULT_WRITE_BUFFER_SIZE):
encoding=None, errors=None, newline=None):
"""Open a gzip-compressed file in binary or text mode.

The filename argument can be an actual filename (a str or bytes object), or
Expand Down Expand Up @@ -59,9 +58,9 @@ def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST,

gz_mode = mode.replace("t", "")
if isinstance(filename, (str, bytes, os.PathLike)):
binary_file = GzipFile(filename, gz_mode, compresslevel, buffer_size=buffer_size)
binary_file = GzipFile(filename, gz_mode, compresslevel)
elif hasattr(filename, "read") or hasattr(filename, "write"):
binary_file = GzipFile(None, gz_mode, compresslevel, filename, buffer_size=buffer_size)
binary_file = GzipFile(None, gz_mode, compresslevel, filename)
else:
raise TypeError("filename must be a str or bytes object, or a file")

Expand Down Expand Up @@ -151,8 +150,7 @@ class GzipFile(_compression.BaseStream):
myfileobj = None

def __init__(self, filename=None, mode=None,
compresslevel=_COMPRESS_LEVEL_BEST, fileobj=None, mtime=None,
buffer_size=DEFAULT_WRITE_BUFFER_SIZE):
compresslevel=_COMPRESS_LEVEL_BEST, fileobj=None, mtime=None):
"""Constructor for the GzipFile class.

At least one of fileobj and filename must be given a
Expand Down Expand Up @@ -202,7 +200,6 @@ def __init__(self, filename=None, mode=None,
if mode is None:
mode = getattr(fileobj, 'mode', 'rb')

self.buffer_size = buffer_size

if mode.startswith('r'):
self.mode = READ
Expand All @@ -226,8 +223,9 @@ def __init__(self, filename=None, mode=None,
zlib.DEF_MEM_LEVEL,
0)
self._write_mtime = mtime
self._buffer_size = _WRITE_BUFFER_SIZE
self._buffer = io.BufferedWriter(_WriteBufferStream(self),
buffer_size=buffer_size)
buffer_size=self._buffer_size)
else:
raise ValueError("Invalid mode: {!r}".format(mode))

Expand Down Expand Up @@ -410,10 +408,10 @@ def seek(self, offset, whence=io.SEEK_SET):
if offset < self.offset:
raise OSError('Negative seek in write mode')
count = offset - self.offset
chunk = b'\0' * self.buffer_size
for i in range(count // self.buffer_size):
chunk = b'\0' * self._buffer_size
for i in range(count // self._buffer_size):
self.write(chunk)
self.write(b'\0' * (count % self.buffer_size))
self.write(b'\0' * (count % self._buffer_size))
elif self.mode == READ:
self._check_not_closed()
return self._buffer.seek(offset, whence)
Expand Down