From 8dbcb31e19ca429283dd24dc81a83a07afca803c Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 29 Sep 2026 16:10:58 +0000 Subject: [PATCH 1/3] zlib: return output inflate still holds after the last input byte decompress_chunks returned before it called inflate when the input was empty, and returned once the input was consumed even when the output buffer was full. Raw deflate has no trailer, so inflate can consume the last input byte and still hold output. Decompress.decompress(b"", n) and Decompress.flush() then returned nothing, and zlib.decompress with a small bufsize raised "incomplete or truncated stream". Mirror CPython's zlibmodule.c: call inflate for empty input, and call it again while it fills the output buffer. A stream with no pending output answers Z_BUF_ERROR, so an empty input before any compressed data still leaves eof false. Assisted-by: Claude Code:claude-opus-5-5 Fixes #8907 --- crates/common/src/compression/zlib.rs | 63 +++++++++++++++++++++++++-- extra_tests/snippets/stdlib_zlib.py | 42 ++++++++++++++++++ 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/crates/common/src/compression/zlib.rs b/crates/common/src/compression/zlib.rs index 6c34d7cbe55..45d1d943023 100644 --- a/crates/common/src/compression/zlib.rs +++ b/crates/common/src/compression/zlib.rs @@ -350,9 +350,9 @@ fn decompress_chunks( max_length: Option, calc_flush: impl Fn(bool) -> InflateFlush, ) -> Result<(Vec, bool), String> { - if data.is_empty() { - return Ok((Vec::new(), false)); - } + // Empty input still reaches inflate, like CPython: inflate can hold output + // after it consumed the last input byte. A stream with no pending output + // answers `BufError`, not `StreamEnd`, so `eof` stays false. let max_length = max_length.unwrap_or(usize::MAX); let mut buf = Vec::new(); @@ -378,6 +378,11 @@ fn decompress_chunks( match res { Ok(status) => { let stream_end = status == Status::StreamEnd; + if produced == additional && !stream_end { + // A full output buffer can leave output pending in + // inflate, even with no input left. + continue 'outer; + } if stream_end || data.is_empty() { buf.shrink_to_fit(); return Ok((buf, stream_end)); @@ -1022,4 +1027,56 @@ mod tests { assert_eq!(d.unconsumed_tail(), b""); assert_eq!(d.unused_data(), b"NEXT"); } + + /// Decode raw deflate of `b"x" * 1168` in 100-byte steps until all + /// input is consumed. Inflate then still holds the last 68 bytes. + fn drained_raw_deflate() -> (Decompressor, Vec) { + let mut co = Compressor::new(6, 8, -MAX_WBITS, 8, 0, None).unwrap(); + let mut raw = co.compress(&[b'x'; 1168]).unwrap(); + raw.extend(co.flush(Z_FINISH).unwrap()); + let mut d = Decompressor::new(-MAX_WBITS, None).unwrap(); + let mut out = d.decompress(&raw, Some(100)).unwrap(); + while !d.unconsumed_tail().is_empty() { + let tail = d.unconsumed_tail().to_vec(); + out.extend(d.decompress(&tail, Some(100)).unwrap()); + } + assert_eq!(out.len(), 1100); + assert!(!d.eof()); + (d, out) + } + + #[test] + fn empty_input_returns_pending_output() { + let (mut d, mut out) = drained_raw_deflate(); + out.extend(d.decompress(b"", Some(100)).unwrap()); + assert_eq!(out, [b'x'; 1168]); + assert!(d.eof()); + } + + #[test] + fn flush_returns_pending_output() { + let (mut d, mut out) = drained_raw_deflate(); + out.extend(d.flush(DEF_BUF_SIZE).unwrap()); + assert_eq!(out, [b'x'; 1168]); + assert!(d.eof()); + } + + #[test] + fn small_bufsize_returns_pending_output() { + let data = [b'x'; 1168]; + let raw = compress(&data, 6, -MAX_WBITS).unwrap(); + for bufsize in [1, 100] { + assert_eq!(decompress(&raw, -MAX_WBITS, bufsize).unwrap(), data); + } + } + + #[test] + fn empty_flush_does_not_finish_a_stream() { + let encoded = compress(b"later input", -1, MAX_WBITS).unwrap(); + let mut d = Decompressor::new(MAX_WBITS, None).unwrap(); + assert_eq!(d.flush(DEF_BUF_SIZE).unwrap(), b""); + assert!(!d.eof()); + assert_eq!(d.decompress(&encoded, None).unwrap(), b"later input"); + assert!(d.eof()); + } } diff --git a/extra_tests/snippets/stdlib_zlib.py b/extra_tests/snippets/stdlib_zlib.py index dfadf7eebdc..cd3f91b61d5 100644 --- a/extra_tests/snippets/stdlib_zlib.py +++ b/extra_tests/snippets/stdlib_zlib.py @@ -79,3 +79,45 @@ assert d.flush() == b"" assert d.unconsumed_tail == tail assert d.unused_data == b"NEXT" + tail + + +# Raw deflate has no trailer, so inflate can consume the last input byte and +# still hold output. decompress(b"", n) and flush() must return that output. +def drained_raw_deflate(): + c = zlib.compressobj(6, zlib.DEFLATED, -15) + raw = c.compress(b"x" * 1168) + c.flush() + d = zlib.decompressobj(-15) + out = d.decompress(raw, 100) + while d.unconsumed_tail: + out += d.decompress(d.unconsumed_tail, 100) + assert len(out) == 1100 + assert not d.eof + return d, out + + +d, out = drained_raw_deflate() +more = d.decompress(b"", 100) +assert len(more) == 68 +assert d.eof +assert d.flush() == b"" +assert out + more == b"x" * 1168 + +d, out = drained_raw_deflate() +rest = d.flush() +assert len(rest) == 68 +assert d.eof +assert out + rest == b"x" * 1168 + +# An empty input before any compressed data does not end the stream. +d = zlib.decompressobj() +assert d.decompress(b"") == b"" +assert not d.eof +assert d.flush() == b"" +assert not d.eof +assert d.decompress(zlib.compress(b"later input")) == b"later input" +assert d.eof + +# The one-shot decompress drains the same pending output with a small bufsize. +raw = zlib.compress(b"x" * 1168, 6, -15) +for bufsize in (1, 100): + assert zlib.decompress(raw, -15, bufsize) == b"x" * 1168 From bb02418fd5a5efa02783f2f0c152c8e70c6ae817 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 29 Sep 2026 16:21:52 +0000 Subject: [PATCH 2/3] zlib: fold the full-buffer case into the loop's exit condition Return when the stream ended, or when the input is used up and inflate left part of the output buffer empty. A full buffer falls through to the existing next iteration, as CPython's loop runs while avail_out is zero, instead of a separate branch ahead of the exit check. Assisted-by: Claude Code:claude-opus-5-5 --- crates/common/src/compression/zlib.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/common/src/compression/zlib.rs b/crates/common/src/compression/zlib.rs index 45d1d943023..6733d99b7b4 100644 --- a/crates/common/src/compression/zlib.rs +++ b/crates/common/src/compression/zlib.rs @@ -378,12 +378,9 @@ fn decompress_chunks( match res { Ok(status) => { let stream_end = status == Status::StreamEnd; - if produced == additional && !stream_end { - // A full output buffer can leave output pending in - // inflate, even with no input left. - continue 'outer; - } - if stream_end || data.is_empty() { + // A full output buffer can leave output pending in inflate, + // even with no input left: only a partial buffer ends it. + if stream_end || (data.is_empty() && produced < additional) { buf.shrink_to_fit(); return Ok((buf, stream_end)); } else if !chunk.is_empty() && consumed == 0 { From 5f763df9caadb222cc339f9f57b0d8e5ca9923ff Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 29 Sep 2026 19:06:20 +0000 Subject: [PATCH 3/3] zlib: drop the comment that stood in for the removed early return Assisted-by: Claude Code:claude-opus-5-5 --- crates/common/src/compression/zlib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/common/src/compression/zlib.rs b/crates/common/src/compression/zlib.rs index 6733d99b7b4..5c90c41d8cb 100644 --- a/crates/common/src/compression/zlib.rs +++ b/crates/common/src/compression/zlib.rs @@ -350,9 +350,6 @@ fn decompress_chunks( max_length: Option, calc_flush: impl Fn(bool) -> InflateFlush, ) -> Result<(Vec, bool), String> { - // Empty input still reaches inflate, like CPython: inflate can hold output - // after it consumed the last input byte. A stream with no pending output - // answers `BufError`, not `StreamEnd`, so `eof` stays false. let max_length = max_length.unwrap_or(usize::MAX); let mut buf = Vec::new();