Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Doc/library/lzma.rst
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,12 @@ Compressing and decompressing data in memory
fail with an :class:`LZMAError` if it is not possible to decompress the input
within the given memory limit.

When *memlimit* is omitted, a default limit of 1.5 GiB (the maximum
dictionary size supported by liblzma) is used. The dictionary is allocated
by liblzma up front from the stream's header, so an untrusted stream can
otherwise force a multi-gigabyte allocation from a few bytes of input. Pass
an explicit *memlimit* to adjust the bound for trusted input.

The *filters* argument specifies the filter chain that was used to create
the stream being decompressed. This argument is required if *format* is
:const:`FORMAT_RAW`, but should not be used for other formats.
Expand Down
26 changes: 26 additions & 0 deletions Lib/test/test_lzma.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,32 @@ def test_decompressor_memlimit(self):
lzd = LZMADecompressor(lzma.FORMAT_ALONE, memlimit=1024)
self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_ALONE)

def test_decompressor_rejects_huge_dictionary(self):
# A dictionary size beyond the maximum supported by liblzma must be
# rejected up front: liblzma allocates the full dictionary when the
# decompressor is created, so an unbounded dict_size lets a tiny
# untrusted stream trigger a multi-gigabyte allocation.
huge = 1 << 31
with self.assertRaises(LZMAError):
LZMADecompressor(lzma.FORMAT_RAW,
filters=[{"id": lzma.FILTER_LZMA1,
"dict_size": huge}])
with self.assertRaises(LZMAError):
LZMACompressor(filters=[{"id": lzma.FILTER_LZMA1,
"dict_size": huge}])

def test_decompressor_alone_huge_dictionary_rejected(self):
# The 13-byte LZMA1 "alone" header declares the dictionary size.
# With the default memory limit, a stream demanding more than the
# maximum supported dictionary size must fail without allocating.
header = b"\x66\xff\xff\xff\xff" + b"\xff" * 8 # ~4 GiB dict
lzd = LZMADecompressor()
with self.assertRaises(LZMAError):
lzd.decompress(header)
lzd = LZMADecompressor(lzma.FORMAT_ALONE)
with self.assertRaises(LZMAError):
lzd.decompress(header)

# Test LZMADecompressor on known-good input data.

def _test_decompressor(self, lzd, data, check, unused_data=b""):
Expand Down
12 changes: 12 additions & 0 deletions Lib/test/test_tarfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,18 @@ def test_read_with_file_comment(self):
with tarfile.open(tmpname, mode=self.mode):
pass

def test_open_short_input_with_fake_lzma_header(self):
# A truncated input whose first byte looks like a valid LZMA-alone
# properties byte and whose remaining bytes encode a huge dictionary
# must fail with ReadError like any other unopenable archive.
# Previously the detection loop let liblzma allocate a
# multi-gigabyte dictionary from the 13-byte header alone
# (memory denial of service).
data = b"\x66" + b"\xff" * 12
with self.assertRaises(tarfile.ReadError):
tarfile.open(fileobj=io.BytesIO(data),
ignore_zeros=True, errorlevel=0)


class MiscReadTest(MiscReadTestBase, unittest.TestCase):
test_fail_comp = None
Expand Down
25 changes: 25 additions & 0 deletions Lib/test/test_zipfile/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import struct
import subprocess
import sys
import tempfile
import time
import unittest
import unittest.mock as mock
Expand Down Expand Up @@ -712,6 +713,30 @@ class LzmaTestsWithSourceFile(AbstractTestsWithSourceFile,
unittest.TestCase):
compression = zipfile.ZIP_LZMA

def test_extractall_rejects_huge_lzma_dictionary(self):
# A member whose LZMA properties declare a dictionary larger than
# liblzma's maximum must fail with LZMAError instead of making
# liblzma allocate that memory up front (memory DoS: the
# dictionary is allocated from the 5-byte properties header alone,
# before any compressed data is decompressed).
import lzma
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_LZMA) as zf:
zf.writestr("member.txt", b"hello")
data = bytearray(buf.getvalue())
info = zipfile.ZipFile(io.BytesIO(data)).getinfo("member.txt")
# Local header: 30 bytes + name + extra, then the LZMA stream:
# [2-byte version][2-byte props size][5-byte props][payload].
lzma_stream = (30 + len(info.filename) + len(info.extra)
+ info.header_offset + 4)
bomb_props = b"\x5d\x00\xfc\x7f\xff" # ~2 GiB dictionary
self.assertEqual(len(bomb_props), 5)
data[lzma_stream:lzma_stream + 5] = bomb_props
with zipfile.ZipFile(io.BytesIO(bytes(data))) as zf:
with tempfile.TemporaryDirectory() as tmp_dir:
with self.assertRaises(lzma.LZMAError):
zf.extractall(path=tmp_dir)

@requires_zstd()
class ZstdTestsWithSourceFile(AbstractTestsWithSourceFile,
unittest.TestCase):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Bound the LZMA dictionary size accepted for decompression to the maximum
supported by liblzma (1.5 GiB), and default the decompressor memory limit to
that value. Previously, a tiny attacker-controlled LZMA stream (for example
an LZMA-compressed member of a zip file, or a short input tried as an LZMA
"alone" stream by tarfile's compression detection) could make liblzma
allocate a multi-gigabyte dictionary up front, exhausting memory. An
explicit ``memlimit`` can still be passed to raise or lower the bound.
30 changes: 29 additions & 1 deletion Modules/_lzmamodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,19 @@
#define LZMA_FILTER_RISCV LZMA_VLI_C(0x0B)
#endif

/*
* liblzma's internal maximum supported dictionary size (1.5 GiB). It is
* not exposed through lzma.h, so define it here to bound the amount of
* memory a (potentially attacker-controlled) LZMA stream can make us
* allocate up front: liblzma allocates the full dictionary when a
* decompressor is created, so an unbounded dict_size allows a tiny
* compressed stream to trigger a multi-gigabyte allocation (a memory
* denial of service, e.g. through zipfile or tarfile).
*/
#ifndef LZMA_DICT_SIZE_MAX
#define LZMA_DICT_SIZE_MAX ((UINT32_C(1) << 30) + (UINT32_C(1) << 29))
#endif

/* On success, return value >= 0
On failure, return -1 */
static inline Py_ssize_t
Expand Down Expand Up @@ -295,6 +308,14 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec)
return NULL;
}

if (options->dict_size > LZMA_DICT_SIZE_MAX) {
PyErr_Format(state->error,
"LZMA dictionary size too large: %u",
(unsigned int)options->dict_size);
PyMem_Free(options);
return NULL;
}

return options;
}

Expand Down Expand Up @@ -1213,7 +1234,14 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format,
{
Decompressor *self;
const uint32_t decoder_flags = LZMA_TELL_ANY_CHECK | LZMA_TELL_NO_CHECK;
uint64_t memlimit_ = UINT64_MAX;
/* Bound the default memory usage of a decompressor to the maximum
dictionary size supported by liblzma (1.5 GiB). Without a limit,
a stream whose header declares a huge dictionary (e.g. an LZMA1
"alone" stream, whose dictionary size is only parsed by liblzma)
makes liblzma allocate that much memory up front, allowing a tiny
untrusted stream to exhaust memory. Callers can pass an explicit
memlimit to raise or lower the bound. */
uint64_t memlimit_ = LZMA_DICT_SIZE_MAX;
lzma_ret lzret;
_lzma_state *state = PyType_GetModuleState(type);
assert(state != NULL);
Expand Down
Loading