Skip to content

gh-95534: Improve gzip reading speed by 10% #97664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gpshead merged 33 commits into from
Oct 17, 2022
Merged

gh-95534: Improve gzip reading speed by 10% #97664

Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
1e13a89
Add code from python-isal project
rhpvorderman Sep 28, 2022
6a5cdfd
Reorder code
rhpvorderman Sep 28, 2022
809ad5f
Add ZlibDecompressor
rhpvorderman Sep 28, 2022
03254b8
Add zlibdecompressor object
rhpvorderman Sep 28, 2022
669848a
Fix compile warnings
rhpvorderman Sep 28, 2022
69ff613
Do not use class input
rhpvorderman Sep 28, 2022
6fa43ae
Fix lock stuff
rhpvorderman Sep 28, 2022
cdc5972
Fix incorrect error handling
rhpvorderman Sep 28, 2022
7820627
Rework _GzipReader to be more efficient
rhpvorderman Sep 28, 2022
6f8b64a
Properly initialize zstate
rhpvorderman Sep 30, 2022
3e2a4f5
Add blurb for increased gzip read speed
rhpvorderman Sep 30, 2022
070df1c
Make sure self->initialised is set to 0. Reword some comments.
rhpvorderman Sep 30, 2022
70b7d4d
Add appropriate doctype in blurb
rhpvorderman Sep 30, 2022
22d3893
Merge branch 'main' into gh-95534
rhpvorderman Sep 30, 2022
18a7692
Add missing NULL member to ZlibDecompressor_Members
rhpvorderman Sep 30, 2022
d54c8b5
Merge branch 'gh-95534' of github.com:rhpvorderman/cpython into gh-95534
rhpvorderman Sep 30, 2022
c90096f
Remove double comment
rhpvorderman Sep 30, 2022
1c15839
Use READ_BUFFER_SIZE in python -m gzip command line application
rhpvorderman Sep 30, 2022
d0ff4f0
Fix error in news entry
rhpvorderman Sep 30, 2022
afd92ab
minor edit, use +=
gpshead Sep 30, 2022
922ac5c
Throw compile warning on zlib versions that are too old
rhpvorderman Oct 2, 2022
dc7de61
Use bool instead of int
rhpvorderman Oct 2, 2022
ca12c1f
Correct spelling of insufficient
rhpvorderman Oct 2, 2022
1ce342b
Put brackets around if statement
rhpvorderman Oct 2, 2022
0b7735e
Remove strange default case
rhpvorderman Oct 2, 2022
043a376
Remove unnecessary zero op
rhpvorderman Oct 2, 2022
2a653a9
Change RetVal to return_value
rhpvorderman Oct 2, 2022
475aef6
Change char to bool
rhpvorderman Oct 2, 2022
41ba076
Properly bracketify if-else clause
rhpvorderman Oct 2, 2022
5f1901d
Prefix underscore to _ZlibDecompressor name
rhpvorderman Oct 2, 2022
c5d6888
Copy explanation about zdict from python docs into function docstring
rhpvorderman Oct 2, 2022
9d60339
Merge branch 'gh-95534' of github.com:rhpvorderman/cpython into gh-95534
rhpvorderman Oct 2, 2022
e3da415
Add tests for _ZlibDecompressor
rhpvorderman Oct 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Reorder code
  • Loading branch information
@rhpvorderman
rhpvorderman committed Sep 28, 2022
commit 6a5cdfd0c36ec5969fd6e60f5bfabf59bf0638a6
148 changes: 79 additions & 69 deletions Modules/zlibmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1351,36 +1351,6 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls,
return RetVal;
}

#include "clinic/zlibmodule.c.h"

static PyMethodDef comp_methods[] =
{
ZLIB_COMPRESS_COMPRESS_METHODDEF
ZLIB_COMPRESS_FLUSH_METHODDEF
ZLIB_COMPRESS_COPY_METHODDEF
ZLIB_COMPRESS___COPY___METHODDEF
ZLIB_COMPRESS___DEEPCOPY___METHODDEF
{NULL, NULL}
};

static PyMethodDef Decomp_methods[] =
{
ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF
ZLIB_DECOMPRESS_FLUSH_METHODDEF
ZLIB_DECOMPRESS_COPY_METHODDEF
ZLIB_DECOMPRESS___COPY___METHODDEF
ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF
{NULL, NULL}
};

#define COMP_OFF(x) offsetof(compobject, x)
static PyMemberDef Decomp_members[] = {
{"unused_data", T_OBJECT, COMP_OFF(unused_data), READONLY},
{"unconsumed_tail", T_OBJECT, COMP_OFF(unconsumed_tail), READONLY},
{"eof", T_BOOL, COMP_OFF(eof), READONLY},
{NULL},
};


typedef struct {
PyObject_HEAD
Expand Down Expand Up @@ -1484,6 +1454,7 @@ decompress_buf(ZlibDecompressor *self, Py_ssize_t max_length)
PyObject *RetVal = NULL;
Comment thread
rhpvorderman marked this conversation as resolved.
Outdated
Py_ssize_t hard_limit;
Py_ssize_t obuflen;
zlibstate *state = PyType_GetModuleState(Py_TYPE(self));

int err;

Expand Down Expand Up @@ -1680,77 +1651,116 @@ decompress(ZlibDecompressor *self, PyTypeObject *cls, uint8_t *data,
return NULL;
}

PyDoc_STRVAR(igzip_lib_IgzipDecompressor___init____doc__,
"IgzipDecompressor(flag=0, hist_bits=15, zdict=b\'\')\n"
PyDoc_STRVAR(ZlibDecompressor__new____doc__,
"_ZlibDecompressor(wbits=15, zdict=b\'\')\n"
"--\n"
"\n"
"Create a decompressor object for decompressing data incrementally.\n"
"\n"
" wbits\n"
" The lookback distance is 2 ^ hist_bits.\n"
" wbits = 15\n"
" zdict\n"
" Dictionary used for decompressing the data\n"
"\n"
"For one-shot decompression, use the decompress() function instead.");
" The predefined compression dictionary. This must be the same\n"
Copy link
Copy Markdown
Member

@gpshead gpshead Sep 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Describe what type this must be (usually bytes?) so that nobody is confused thinking it is a Python dict.

Consider using O! in your format below to have the API do a type check for you rather than failing later on when its use is attempted via the buffer API.

Copy link
Copy Markdown
Contributor Author

@rhpvorderman rhpvorderman Oct 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The problem with O! is that it cannot correctly accept bytes and bytearray simultaneously. Or is there a better solution for this?

" dictionary as used by the compressor that produced the input data.\n"
"\n");

static PyObject *
igzip_lib_IgzipDecompressor__new__(PyTypeObject *type,
PyObject *args,
PyObject *kwargs)
ZlibDecompressor__new__(PyTypeObject *cls,
PyObject *args,
PyObject *kwargs)
{
static char *keywords[] = {"flag", "hist_bits", "zdict", NULL};
static char *format = "|iiO:IgzipDecompressor";
int flag = ISAL_DEFLATE;
int hist_bits = ISAL_DEF_MAX_HIST_BITS;
static char *keywords[] = {"wbits", "zdict", NULL};
static char *format = "|iO:IgzipDecompressor";
int wbits = MAX_WBITS;
PyObject *zdict = NULL;
zlibstate *state = PyType_GetModuleState(cls);

if (!PyArg_ParseTupleAndKeywords(
args, kwargs, format, keywords, &flag, &hist_bits, &zdict)) {
args, kwargs, format, keywords, &wbits, &zdict)) {
return NULL;
}
IgzipDecompressor *self = PyObject_New(IgzipDecompressor, type);
ZlibDecompressor *self = PyObject_New(ZlibDecompressor, cls);
int err;
self->eof = 0;
self->needs_input = 1;
self->avail_in_real = 0;
self->input_buffer = NULL;
self->input_buffer_size = 0;
self->zdict = zdict;
self->zst.opaque = NULL;
self->zst.zalloc = PyZlib_Malloc;
self->zst.zfree = PyZlib_Free;
self->unused_data = PyBytes_FromStringAndSize(NULL, 0);
if (self->unused_data == NULL) {
Py_CLEAR(self);
return NULL;
}
isal_inflate_init(&(self->state));
self->state.hist_bits = hist_bits;
self->state.crc_flag = flag;
if (self->zdict != NULL){
Py_buffer zdict_buf;
if (PyObject_GetBuffer(self->zdict, &zdict_buf, PyBUF_SIMPLE) == -1) {
Py_CLEAR(self);
int err = inflateInit2(&(self->zst), wbits);
switch (err) {
case Z_OK:
self->is_initialised = 1;
if (self->zdict != NULL && wbits < 0) {
#ifdef AT_LEAST_ZLIB_1_2_2_1
Comment thread
rhpvorderman marked this conversation as resolved.
Outdated
if (set_inflate_zdict(state, self) < 0) {
Py_DECREF(self);
return NULL;
}
if ((size_t)zdict_buf.len > UINT32_MAX) {
PyErr_SetString(PyExc_OverflowError,
"zdict length does not fit in an unsigned 32-bits int");
PyBuffer_Release(&zdict_buf);
Py_CLEAR(self);
}
#else
PyErr_Format(state->ZlibError,
"zlib version %s does not allow raw inflate with dictionary",
ZLIB_VERSION);
Py_DECREF(self);
return NULL;
#endif
}
err = isal_inflate_set_dict(&(self->state), zdict_buf.buf,
(uint32_t)zdict_buf.len);
PyBuffer_Release(&zdict_buf);
if (err != ISAL_DECOMP_OK) {
isal_inflate_error(err);
Py_CLEAR(self);
return NULL;
}
return (PyObject *)self;
case Z_STREAM_ERROR:
Py_DECREF(self);
PyErr_SetString(PyExc_ValueError, "Invalid initialization option");
return NULL;
case Z_MEM_ERROR:
Py_DECREF(self);
PyErr_SetString(PyExc_MemoryError,
"Can't allocate memory for decompression object");
return NULL;
default:
zlib_error(state, self->zst, err, "while creating decompression object");
Py_DECREF(self);
return NULL;
}
return (PyObject *)self;
}

#include "clinic/zlibmodule.c.h"

static PyMethodDef comp_methods[] =
{
ZLIB_COMPRESS_COMPRESS_METHODDEF
ZLIB_COMPRESS_FLUSH_METHODDEF
ZLIB_COMPRESS_COPY_METHODDEF
ZLIB_COMPRESS___COPY___METHODDEF
ZLIB_COMPRESS___DEEPCOPY___METHODDEF
{NULL, NULL}
};

static PyMethodDef Decomp_methods[] =
{
ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF
ZLIB_DECOMPRESS_FLUSH_METHODDEF
ZLIB_DECOMPRESS_COPY_METHODDEF
ZLIB_DECOMPRESS___COPY___METHODDEF
ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF
{NULL, NULL}
};

#define COMP_OFF(x) offsetof(compobject, x)
static PyMemberDef Decomp_members[] = {
{"unused_data", T_OBJECT, COMP_OFF(unused_data), READONLY},
{"unconsumed_tail", T_OBJECT, COMP_OFF(unconsumed_tail), READONLY},
{"eof", T_BOOL, COMP_OFF(eof), READONLY},
{NULL},
};

static PyMethodDef IgzipDecompressor_methods[] = {
IGZIP_LIB_IGZIPDECOMPRESSOR_DECOMPRESS_METHODDEF,
ZLIBCOMPRESSOR_DECOMPRESS_METHODDEF,
{NULL}
};

Expand Down