/* _lzma - Low-level Python interface to liblzma.

Initial implementation by Per Øyvind Karlsen.

Rewritten by Nadeem Vawda.

*/

#define PY_SSIZE_T_CLEAN

#include "Python.h"

#include "structmember.h" // PyMemberDef

#include <stdarg.h>

#include <string.h>

#include <lzma.h>

#define ACQUIRE_LOCK(obj) do { \

if (!PyThread_acquire_lock((obj)->lock, 0)) { \

Py_BEGIN_ALLOW_THREADS \

PyThread_acquire_lock((obj)->lock, 1); \

Py_END_ALLOW_THREADS \

} } while (0)

#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock)

typedef struct {

PyTypeObject *lzma_compressor_type;

PyTypeObject *lzma_decompressor_type;

PyObject *error;

PyObject *empty_tuple;

} _lzma_state;

static inline _lzma_state*

get_lzma_state(PyObject *module)

{

void *state = PyModule_GetState(module);

assert(state != NULL);

return (_lzma_state *)state;

}

/* Container formats: */

enum {

FORMAT_AUTO,

FORMAT_XZ,

FORMAT_ALONE,

FORMAT_RAW,

};

#define LZMA_CHECK_UNKNOWN (LZMA_CHECK_ID_MAX + 1)

typedef struct {

PyObject_HEAD

lzma_allocator alloc;

lzma_stream lzs;

int flushed;

PyThread_type_lock lock;

} Compressor;

typedef struct {

PyObject_HEAD

lzma_allocator alloc;

lzma_stream lzs;

int check;

char eof;

PyObject *unused_data;

char needs_input;

uint8_t *input_buffer;

size_t input_buffer_size;

PyThread_type_lock lock;

} Decompressor;

/* Helper functions. */

static int

catch_lzma_error(_lzma_state *state, lzma_ret lzret)

{

switch (lzret) {

case LZMA_OK:

case LZMA_GET_CHECK:

case LZMA_NO_CHECK:

case LZMA_STREAM_END:

return 0;

case LZMA_UNSUPPORTED_CHECK:

PyErr_SetString(state->error, "Unsupported integrity check");

return 1;

case LZMA_MEM_ERROR:

PyErr_NoMemory();

return 1;

case LZMA_MEMLIMIT_ERROR:

PyErr_SetString(state->error, "Memory usage limit exceeded");

return 1;

case LZMA_FORMAT_ERROR:

PyErr_SetString(state->error, "Input format not supported by decoder");

return 1;

case LZMA_OPTIONS_ERROR:

PyErr_SetString(state->error, "Invalid or unsupported options");

return 1;

case LZMA_DATA_ERROR:

PyErr_SetString(state->error, "Corrupt input data");

return 1;

case LZMA_BUF_ERROR:

PyErr_SetString(state->error, "Insufficient buffer space");

return 1;

case LZMA_PROG_ERROR:

PyErr_SetString(state->error, "Internal error");

return 1;

default:

PyErr_Format(state->error, "Unrecognized error from liblzma: %d", lzret);

return 1;

}

}

static void*

PyLzma_Malloc(void *opaque, size_t items, size_t size)

{

if (size != 0 && items > (size_t)PY_SSIZE_T_MAX / size) {

return NULL;

}

/* PyMem_Malloc() cannot be used:

the GIL is not held when lzma_code() is called */

return PyMem_RawMalloc(items * size);

}

static void

PyLzma_Free(void *opaque, void *ptr)

{

PyMem_RawFree(ptr);

}

#if BUFSIZ < 8192

#define INITIAL_BUFFER_SIZE 8192

#else

#define INITIAL_BUFFER_SIZE BUFSIZ

#endif

static int

grow_buffer(PyObject **buf, Py_ssize_t max_length)

{

Py_ssize_t size = PyBytes_GET_SIZE(*buf);

Py_ssize_t newsize = size + (size >> 3) + 6;

if (max_length > 0 && newsize > max_length) {

newsize = max_length;

}

return _PyBytes_Resize(buf, newsize);

}

/* Some custom type conversions for PyArg_ParseTupleAndKeywords(),

since the predefined conversion specifiers do not suit our needs:

uint32_t - the "I" (unsigned int) specifier is the right size, but

silently ignores overflows on conversion.

lzma_vli - the "K" (unsigned long long) specifier is the right

size, but like "I" it silently ignores overflows on conversion.

lzma_mode and lzma_match_finder - these are enumeration types, and

so the size of each is implementation-defined. Worse, different

enum types can be of different sizes within the same program, so

to be strictly correct, we need to define two separate converters.

*/

#define INT_TYPE_CONVERTER_FUNC(TYPE, FUNCNAME) \

static int \

FUNCNAME(PyObject *obj, void *ptr) \

{ \

unsigned long long val; \

\

val = PyLong_AsUnsignedLongLong(obj); \

if (PyErr_Occurred()) \

return 0; \

if ((unsigned long long)(TYPE)val != val) { \

PyErr_SetString(PyExc_OverflowError, \

"Value too large for " #TYPE " type"); \

return 0; \

} \

*(TYPE *)ptr = (TYPE)val; \

return 1; \

}

INT_TYPE_CONVERTER_FUNC(uint32_t, uint32_converter)

INT_TYPE_CONVERTER_FUNC(lzma_vli, lzma_vli_converter)

INT_TYPE_CONVERTER_FUNC(lzma_mode, lzma_mode_converter)

INT_TYPE_CONVERTER_FUNC(lzma_match_finder, lzma_mf_converter)

#undef INT_TYPE_CONVERTER_FUNC

/* Filter specifier parsing.

This code handles converting filter specifiers (Python dicts) into

the C lzma_filter structs expected by liblzma. */

static void *

parse_filter_spec_lzma(_lzma_state *state, PyObject *spec)

{

static char *optnames[] = {"id", "preset", "dict_size", "lc", "lp",

"pb", "mode", "nice_len", "mf", "depth", NULL};

PyObject *id;

PyObject *preset_obj;

uint32_t preset = LZMA_PRESET_DEFAULT;

lzma_options_lzma *options;

/* First, fill in default values for all the options using a preset.

Then, override the defaults with any values given by the caller. */

preset_obj = PyMapping_GetItemString(spec, "preset");

if (preset_obj == NULL) {

if (PyErr_ExceptionMatches(PyExc_KeyError)) {

PyErr_Clear();

}

else {

return NULL;

}

} else {

int ok = uint32_converter(preset_obj, &preset);

Py_DECREF(preset_obj);

if (!ok) {

return NULL;

}

}

options = (lzma_options_lzma *)PyMem_Calloc(1, sizeof *options);

if (options == NULL) {

return PyErr_NoMemory();

}

if (lzma_lzma_preset(options, preset)) {

PyMem_Free(options);

PyErr_Format(state->error, "Invalid compression preset: %u", preset);

return NULL;

}

if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec,

"|OOO&O&O&O&O&O&O&O&", optnames,

&id, &preset_obj,

uint32_converter, &options->dict_size,

uint32_converter, &options->lc,

uint32_converter, &options->lp,

uint32_converter, &options->pb,

lzma_mode_converter, &options->mode,

uint32_converter, &options->nice_len,

lzma_mf_converter, &options->mf,

uint32_converter, &options->depth)) {

PyErr_SetString(PyExc_ValueError,

"Invalid filter specifier for LZMA filter");

PyMem_Free(options);

return NULL;

}

return options;

}

static void *

parse_filter_spec_delta(_lzma_state *state, PyObject *spec)

{

static char *optnames[] = {"id", "dist", NULL};

PyObject *id;

uint32_t dist = 1;

lzma_options_delta *options;

if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames,

&id, uint32_converter, &dist)) {

PyErr_SetString(PyExc_ValueError,

"Invalid filter specifier for delta filter");

return NULL;

}

options = (lzma_options_delta *)PyMem_Calloc(1, sizeof *options);

if (options == NULL) {

return PyErr_NoMemory();

}

options->type = LZMA_DELTA_TYPE_BYTE;

options->dist = dist;

return options;

}

static void *

parse_filter_spec_bcj(_lzma_state *state, PyObject *spec)

{

static char *optnames[] = {"id", "start_offset", NULL};

PyObject *id;

uint32_t start_offset = 0;

lzma_options_bcj *options;

if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames,

&id, uint32_converter, &start_offset)) {

PyErr_SetString(PyExc_ValueError,

"Invalid filter specifier for BCJ filter");

return NULL;

}

options = (lzma_options_bcj *)PyMem_Calloc(1, sizeof *options);

if (options == NULL) {

return PyErr_NoMemory();

}

options->start_offset = start_offset;

return options;

}

static int

lzma_filter_converter(_lzma_state *state, PyObject *spec, void *ptr)

{

lzma_filter *f = (lzma_filter *)ptr;

PyObject *id_obj;

if (!PyMapping_Check(spec)) {

PyErr_SetString(PyExc_TypeError,

"Filter specifier must be a dict or dict-like object");

return 0;

}

id_obj = PyMapping_GetItemString(spec, "id");

if (id_obj == NULL) {

if (PyErr_ExceptionMatches(PyExc_KeyError))

PyErr_SetString(PyExc_ValueError,

"Filter specifier must have an \"id\" entry");

return 0;

}

f->id = PyLong_AsUnsignedLongLong(id_obj);

Py_DECREF(id_obj);

if (PyErr_Occurred()) {

return 0;

}

switch (f->id) {

case LZMA_FILTER_LZMA1:

case LZMA_FILTER_LZMA2:

f->options = parse_filter_spec_lzma(state, spec);

return f->options != NULL;

case LZMA_FILTER_DELTA:

f->options = parse_filter_spec_delta(state, spec);

return f->options != NULL;

case LZMA_FILTER_X86:

case LZMA_FILTER_POWERPC:

case LZMA_FILTER_IA64:

case LZMA_FILTER_ARM:

case LZMA_FILTER_ARMTHUMB:

case LZMA_FILTER_SPARC:

f->options = parse_filter_spec_bcj(state, spec);

return f->options != NULL;

default:

PyErr_Format(PyExc_ValueError, "Invalid filter ID: %llu", f->id);

return 0;

}

}

static void

free_filter_chain(lzma_filter filters[])

{

for (int i = 0; filters[i].id != LZMA_VLI_UNKNOWN; i++) {

PyMem_Free(filters[i].options);

}

}

static int

parse_filter_chain_spec(_lzma_state *state, lzma_filter filters[], PyObject *filterspecs)

{

Py_ssize_t i, num_filters;

num_filters = PySequence_Length(filterspecs);

if (num_filters == -1) {

return -1;

}

if (num_filters > LZMA_FILTERS_MAX) {

PyErr_Format(PyExc_ValueError,

"Too many filters - liblzma supports a maximum of %d",

LZMA_FILTERS_MAX);

return -1;

}

for (i = 0; i < num_filters; i++) {

int ok = 1;

PyObject *spec = PySequence_GetItem(filterspecs, i);

if (spec == NULL || !lzma_filter_converter(state, spec, &filters[i])) {

ok = 0;

}

Py_XDECREF(spec);

if (!ok) {

filters[i].id = LZMA_VLI_UNKNOWN;

free_filter_chain(filters);

return -1;

}

}

filters[num_filters].id = LZMA_VLI_UNKNOWN;

return 0;

}

/* Filter specifier construction.

This code handles converting C lzma_filter structs into

Python-level filter specifiers (represented as dicts). */

static int

spec_add_field(PyObject *spec, _Py_Identifier *key, unsigned long long value)

{

int status;

PyObject *value_object;

value_object = PyLong_FromUnsignedLongLong(value);

if (value_object == NULL) {

return -1;

}

status = _PyDict_SetItemId(spec, key, value_object);

Py_DECREF(value_object);

return status;

}

static PyObject *

build_filter_spec(const lzma_filter *f)

{

PyObject *spec;

spec = PyDict_New();

if (spec == NULL) {

return NULL;

}

#define ADD_FIELD(SOURCE, FIELD) \

do { \

_Py_IDENTIFIER(FIELD); \

if (spec_add_field(spec, &PyId_##FIELD, SOURCE->FIELD) == -1) \

goto error;\

} while (0)

ADD_FIELD(f, id);

switch (f->id) {

/* For LZMA1 filters, lzma_properties_{encode,decode}() only look at the

lc, lp, pb, and dict_size fields. For LZMA2 filters, only the

dict_size field is used. */

case LZMA_FILTER_LZMA1: {

lzma_options_lzma *options = f->options;

ADD_FIELD(options, lc);

ADD_FIELD(options, lp);

ADD_FIELD(options, pb);

ADD_FIELD(options, dict_size);

break;

}

case LZMA_FILTER_LZMA2: {

lzma_options_lzma *options = f->options;

ADD_FIELD(options, dict_size);

break;

}

case LZMA_FILTER_DELTA: {

lzma_options_delta *options = f->options;

ADD_FIELD(options, dist);

break;

}

case LZMA_FILTER_X86:

case LZMA_FILTER_POWERPC:

case LZMA_FILTER_IA64:

case LZMA_FILTER_ARM:

case LZMA_FILTER_ARMTHUMB:

case LZMA_FILTER_SPARC: {

lzma_options_bcj *options = f->options;

ADD_FIELD(options, start_offset);

break;

}

default:

PyErr_Format(PyExc_ValueError, "Invalid filter ID: %llu", f->id);

goto error;

}

#undef ADD_FIELD

return spec;

error:

Py_DECREF(spec);

return NULL;

}

/*[clinic input]

module _lzma

class _lzma.LZMACompressor "Compressor *" "&Compressor_type"

class _lzma.LZMADecompressor "Decompressor *" "&Decompressor_type"

[clinic start generated code]*/

/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2c14bbe05ff0c147]*/

#include "clinic/_lzmamodule.c.h"

/*[python input]

class lzma_vli_converter(CConverter):

type = 'lzma_vli'

converter = 'lzma_vli_converter'

class lzma_filter_converter(CConverter):

type = 'lzma_filter'

converter = 'lzma_filter_converter'

c_default = c_ignored_default = "{LZMA_VLI_UNKNOWN, NULL}"

def cleanup(self):

name = ensure_legal_c_identifier(self.name)

return ('if (%(name)s.id != LZMA_VLI_UNKNOWN)\n'

' PyMem_Free(%(name)s.options);\n') % {'name': name}

[python start generated code]*/

/*[python end generated code: output=da39a3ee5e6b4b0d input=74fe7631ce377a94]*/

/* LZMACompressor class. */

static PyObject *

compress(Compressor *c, uint8_t *data, size_t len, lzma_action action)

{

Py_ssize_t data_size = 0;

PyObject *result;

_lzma_state *state = PyType_GetModuleState(Py_TYPE(c));

assert(state != NULL);

result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE);

if (result == NULL) {

return NULL;

}

c->lzs.next_in = data;

c->lzs.avail_in = len;

c->lzs.next_out = (uint8_t *)PyBytes_AS_STRING(result);

c->lzs.avail_out = PyBytes_GET_SIZE(result);

for (;;) {

lzma_ret lzret;

Py_BEGIN_ALLOW_THREADS

lzret = lzma_code(&c->lzs, action);

data_size = (char *)c->lzs.next_out - PyBytes_AS_STRING(result);

if (lzret == LZMA_BUF_ERROR && len == 0 && c->lzs.avail_out > 0) {

lzret = LZMA_OK; /* That wasn't a real error */

}

Py_END_ALLOW_THREADS

if (catch_lzma_error(state, lzret)) {

goto error;

}

if ((action == LZMA_RUN && c->lzs.avail_in == 0) ||

(action == LZMA_FINISH && lzret == LZMA_STREAM_END)) {

break;

} else if (c->lzs.avail_out == 0) {

if (grow_buffer(&result, -1) == -1)

goto error;

c->lzs.next_out = (uint8_t *)PyBytes_AS_STRING(result) + data_size;

c->lzs.avail_out = PyBytes_GET_SIZE(result) - data_size;

}

}

if (data_size != PyBytes_GET_SIZE(result))

if (_PyBytes_Resize(&result, data_size) == -1) {

goto error;

}

return result;

error:

Py_XDECREF(result);

return NULL;

}

/*[clinic input]

_lzma.LZMACompressor.compress

data: Py_buffer

/

Provide data to the compressor object.

Returns a chunk of compressed data if possible, or b'' otherwise.

When you have finished providing data to the compressor, call the

flush() method to finish the compression process.

[clinic start generated code]*/

static PyObject *

_lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data)

/*[clinic end generated code: output=31f615136963e00f input=64019eac7f2cc8d0]*/

{

PyObject *result = NULL;

ACQUIRE_LOCK(self);

if (self->flushed) {

PyErr_SetString(PyExc_ValueError, "Compressor has been flushed");

}

else {

result = compress(self, data->buf, data->len, LZMA_RUN);

}

RELEASE_LOCK(self);

return result;

}

/*[clinic input]

_lzma.LZMACompressor.flush

Finish the compression process.

Returns the compressed data left in internal buffers.

The compressor object may not be used after this method is called.

[clinic start generated code]*/

static PyObject *

_lzma_LZMACompressor_flush_impl(Compressor *self)

/*[clinic end generated code: output=fec21f3e22504f50 input=6b369303f67ad0a8]*/

{

PyObject *result = NULL;

ACQUIRE_LOCK(self);

if (self->flushed) {

PyErr_SetString(PyExc_ValueError, "Repeated call to flush()");

} else {

self->flushed = 1;

result = compress(self, NULL, 0, LZMA_FINISH);

}

RELEASE_LOCK(self);

return result;

}

static int

Compressor_init_xz(_lzma_state *state, lzma_stream *lzs,

int check, uint32_t preset, PyObject *filterspecs)

{

lzma_ret lzret;

if (filterspecs == Py_None) {

lzret = lzma_easy_encoder(lzs, preset, check);

} else {

lzma_filter filters[LZMA_FILTERS_MAX + 1];

if (parse_filter_chain_spec(state, filters, filterspecs) == -1)

return -1;

lzret = lzma_stream_encoder(lzs, filters, check);

free_filter_chain(filters);

}

if (catch_lzma_error(state, lzret)) {

return -1;

}

else {

return 0;

}

}

static int

Compressor_init_alone(_lzma_state *state, lzma_stream *lzs, uint32_t preset, PyObject *filterspecs)

{

lzma_ret lzret;

if (filterspecs == Py_None) {

lzma_options_lzma options;

if (lzma_lzma_preset(&options, preset)) {

PyErr_Format(state->error, "Invalid compression preset: %u", preset);

return -1;

}

lzret = lzma_alone_encoder(lzs, &options);

} else {

lzma_filter filters[LZMA_FILTERS_MAX + 1];

if (parse_filter_chain_spec(state, filters, filterspecs) == -1)

return -1;

if (filters[0].id == LZMA_FILTER_LZMA1 &&

filters[1].id == LZMA_VLI_UNKNOWN) {

lzret = lzma_alone_encoder(lzs, filters[0].options);

} else {

PyErr_SetString(PyExc_ValueError,

"Invalid filter chain for FORMAT_ALONE - "

"must be a single LZMA1 filter");

lzret = LZMA_PROG_ERROR;

}

free_filter_chain(filters);

}

if (PyErr_Occurred() || catch_lzma_error(state, lzret)) {

return -1;

}

else {

return 0;

}

}

static int

Compressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspecs)

{

lzma_filter filters[LZMA_FILTERS_MAX + 1];

lzma_ret lzret;

if (filterspecs == Py_None) {

PyErr_SetString(PyExc_ValueError,

"Must specify filters for FORMAT_RAW");

return -1;

}

if (parse_filter_chain_spec(state, filters, filterspecs) == -1) {

return -1;

}

lzret = lzma_raw_encoder(lzs, filters);

free_filter_chain(filters);

if (catch_lzma_error(state, lzret)) {

return -1;

}

else {

return 0;

}

}

/*[-clinic input]

_lzma.LZMACompressor.__init__

format: int(c_default="FORMAT_XZ") = FORMAT_XZ

The container format to use for the output. This can

be FORMAT_XZ (default), FORMAT_ALONE, or FORMAT_RAW.

check: int(c_default="-1") = unspecified

The integrity check to use. For FORMAT_XZ, the default

is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not support integrity

checks; for these formats, check must be omitted, or be CHECK_NONE.

preset: object = None

If provided should be an integer in the range 0-9, optionally

OR-ed with the constant PRESET_EXTREME.

filters: object = None

If provided should be a sequence of dicts. Each dict should

have an entry for "id" indicating the ID of the filter, plus

additional entries for options to the filter.

Create a compressor object for compressing data incrementally.

The settings used by the compressor can be specified either as a

preset compression level (with the 'preset' argument), or in detail

as a custom filter chain (with the 'filters' argument). For FORMAT_XZ

and FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset

level. For FORMAT_RAW, the caller must always specify a filter chain;

the raw compressor does not support preset compression levels.

For one-shot compression, use the compress() function instead.

[-clinic start generated code]*/

static int

Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs)

{

static char *arg_names[] = {"format", "check", "preset", "filters", NULL};

int format = FORMAT_XZ;

int check = -1;

uint32_t preset = LZMA_PRESET_DEFAULT;

PyObject *preset_obj = Py_None;

PyObject *filterspecs = Py_None;

_lzma_state *state = PyType_GetModuleState(Py_TYPE(self));

assert(state != NULL);

if (!PyArg_ParseTupleAndKeywords(args, kwargs,

"|iiOO:LZMACompressor", arg_names,

&format, &check, &preset_obj,

&filterspecs)) {

return -1;

}

if (format != FORMAT_XZ && check != -1 && check != LZMA_CHECK_NONE) {

PyErr_SetString(PyExc_ValueError,

"Integrity checks are only supported by FORMAT_XZ");

return -1;

}

if (preset_obj != Py_None && filterspecs != Py_None) {

PyErr_SetString(PyExc_ValueError,

"Cannot specify both preset and filter chain");

return -1;

}

if (preset_obj != Py_None) {

if (!uint32_converter(preset_obj, &preset)) {

return -1;

}

}

self->alloc.opaque = NULL;

self->alloc.alloc = PyLzma_Malloc;

self->alloc.free = PyLzma_Free;

self->lzs.allocator = &self->alloc;

self->lock = PyThread_allocate_lock();

if (self->lock == NULL) {

PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock");

return -1;

}

self->flushed = 0;

switch (format) {

case FORMAT_XZ:

if (check == -1) {

check = LZMA_CHECK_CRC64;

}

if (Compressor_init_xz(state, &self->lzs, check, preset, filterspecs) != 0) {

break;

}

return 0;

case FORMAT_ALONE:

if (Compressor_init_alone(state, &self->lzs, preset, filterspecs) != 0) {

break;

}

return 0;

case FORMAT_RAW:

if (Compressor_init_raw(state, &self->lzs, filterspecs) != 0) {

break;

}

return 0;

default:

PyErr_Format(PyExc_ValueError,

"Invalid container format: %d", format);

break;

}

PyThread_free_lock(self->lock);

self->lock = NULL;

return -1;

}

static void

Compressor_dealloc(Compressor *self)

{

lzma_end(&self->lzs);

if (self->lock != NULL) {

PyThread_free_lock(self->lock);

}

PyTypeObject *tp = Py_TYPE(self);

tp->tp_free((PyObject *)self);

Py_DECREF(tp);

}

static PyMethodDef Compressor_methods[] = {

_LZMA_LZMACOMPRESSOR_COMPRESS_METHODDEF

_LZMA_LZMACOMPRESSOR_FLUSH_METHODDEF

{NULL}

};

static int

Compressor_traverse(Compressor *self, visitproc visit, void *arg)

{

Py_VISIT(Py_TYPE(self));

return 0;

}

PyDoc_STRVAR(Compressor_doc,

"LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None)\n"

"\n"

"Create a compressor object for compressing data incrementally.\n"

"\n"

"format specifies the container format to use for the output. This can\n"

"be FORMAT_XZ (default), FORMAT_ALONE, or FORMAT_RAW.\n"

"\n"

"check specifies the integrity check to use. For FORMAT_XZ, the default\n"

"is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not support integrity\n"

"checks; for these formats, check must be omitted, or be CHECK_NONE.\n"

"\n"

"The settings used by the compressor can be specified either as a\n"

"preset compression level (with the 'preset' argument), or in detail\n"

"as a custom filter chain (with the 'filters' argument). For FORMAT_XZ\n"

"and FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset\n"

"level. For FORMAT_RAW, the caller must always specify a filter chain;\n"

"the raw compressor does not support preset compression levels.\n"

"\n"

"preset (if provided) should be an integer in the range 0-9, optionally\n"

"OR-ed with the constant PRESET_EXTREME.\n"

"\n"

"filters (if provided) should be a sequence of dicts. Each dict should\n"

"have an entry for \"id\" indicating the ID of the filter, plus\n"

"additional entries for options to the filter.\n"

"\n"

"For one-shot compression, use the compress() function instead.\n");

static PyType_Slot lzma_compressor_type_slots[] = {

{Py_tp_dealloc, Compressor_dealloc},

{Py_tp_methods, Compressor_methods},

{Py_tp_init, Compressor_init},

{Py_tp_new, PyType_GenericNew},

{Py_tp_doc, (char *)Compressor_doc},

{Py_tp_traverse, Compressor_traverse},

{0, 0}

};

static PyType_Spec lzma_compressor_type_spec = {

.name = "_lzma.LZMACompressor",

.basicsize = sizeof(Compressor),

// Calling PyType_GetModuleState() on a subclass is not safe.

// lzma_compressor_type_spec does not have Py_TPFLAGS_BASETYPE flag

// which prevents to create a subclass.

// So calling PyType_GetModuleState() in this file is always safe.

.flags = Py_TPFLAGS_DEFAULT,

.slots = lzma_compressor_type_slots,

};

/* LZMADecompressor class. */

/* Decompress data of length d->lzs.avail_in in d->lzs.next_in. The output

buffer is allocated dynamically and returned. At most max_length bytes are

returned, so some of the input may not be consumed. d->lzs.next_in and

d->lzs.avail_in are updated to reflect the consumed input. */

static PyObject*

decompress_buf(Decompressor *d, Py_ssize_t max_length)

{

Py_ssize_t data_size = 0;

PyObject *result;

lzma_stream *lzs = &d->lzs;

_lzma_state *state = PyType_GetModuleState(Py_TYPE(d));

assert(state != NULL);

if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE) {

result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE);

}

else {

result = PyBytes_FromStringAndSize(NULL, max_length);

}

if (result == NULL) {

return NULL;

}

lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result);

lzs->avail_out = PyBytes_GET_SIZE(result);

for (;;) {

lzma_ret lzret;

Py_BEGIN_ALLOW_THREADS

lzret = lzma_code(lzs, LZMA_RUN);

data_size = (char *)lzs->next_out - PyBytes_AS_STRING(result);

if (lzret == LZMA_BUF_ERROR && lzs->avail_in == 0 && lzs->avail_out > 0) {

lzret = LZMA_OK; /* That wasn't a real error */

}

Py_END_ALLOW_THREADS

if (catch_lzma_error(state, lzret)) {

goto error;

}

if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK) {

d->check = lzma_get_check(&d->lzs);

}

if (lzret == LZMA_STREAM_END) {

d->eof = 1;

break;

} else if (lzs->avail_out == 0) {

/* Need to check lzs->avail_out before lzs->avail_in.

Maybe lzs's internal state still have a few bytes

can be output, grow the output buffer and continue

if max_lengh < 0. */

if (data_size == max_length) {

break;

}

if (grow_buffer(&result, max_length) == -1) {

goto error;

}

lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result) + data_size;

lzs->avail_out = PyBytes_GET_SIZE(result) - data_size;

} else if (lzs->avail_in == 0) {

break;

}

}

if (data_size != PyBytes_GET_SIZE(result)) {

if (_PyBytes_Resize(&result, data_size) == -1) {

goto error;

}

}

return result;

error:

Py_XDECREF(result);

return NULL;

}

static PyObject *

decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length)

{

char input_buffer_in_use;

PyObject *result;

lzma_stream *lzs = &d->lzs;

/* Prepend unconsumed input if necessary */

if (lzs->next_in != NULL) {

size_t avail_now, avail_total;

/* Number of bytes we can append to input buffer */

avail_now = (d->input_buffer + d->input_buffer_size)

- (lzs->next_in + lzs->avail_in);

/* Number of bytes we can append if we move existing

contents to beginning of buffer (overwriting

consumed input) */

avail_total = d->input_buffer_size - lzs->avail_in;

if (avail_total < len) {

size_t offset = lzs->next_in - d->input_buffer;

uint8_t *tmp;

size_t new_size = d->input_buffer_size + len - avail_now;

/* Assign to temporary variable first, so we don't

lose address of allocated buffer if realloc fails */

tmp = PyMem_Realloc(d->input_buffer, new_size);

if (tmp == NULL) {

PyErr_SetNone(PyExc_MemoryError);

return NULL;

}

d->input_buffer = tmp;