From b05099f1a19fd2f8d26150089dbf386a71d017e8 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 29 Sep 2026 16:07:23 +0000 Subject: [PATCH 1/3] zlib: clear unconsumed_tail when a bounded decompress reaches stream end When a max_length-bounded Decompress.decompress() call reached the end of the stream, the bytes after the stream went to unused_data, but unconsumed_tail kept the previous call's tail, including input the call had already consumed. A later flush() then moved those stale bytes into unused_data. Empty unconsumed_tail at end of stream, as CPython's save_unconsumed_input() does since gh-158169. CPython releases before that change report the bytes after the stream in unconsumed_tail, so the snippet accepts either value when it runs under CPython. Fixes #8906 Assisted-by: Claude Code:claude-opus-5-5 --- crates/common/src/compression/zlib.rs | 36 +++++++++++++++++++-------- extra_tests/snippets/stdlib_zlib.py | 21 ++++++++++++++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/crates/common/src/compression/zlib.rs b/crates/common/src/compression/zlib.rs index f955f9be7d0..3163a91124f 100644 --- a/crates/common/src/compression/zlib.rs +++ b/crates/common/src/compression/zlib.rs @@ -670,17 +670,15 @@ impl Decompressor { }; let consumed = (d.total_in() - prev_in) as usize; - // save unused input - let unconsumed = &data[consumed..]; - if !unconsumed.is_empty() { - if stream_end { - unused_data.extend_from_slice(unconsumed); - } else { - *unconsumed_tail = unconsumed.to_vec(); - } - } else if !unconsumed_tail.is_empty() { - unconsumed_tail.clear(); + // save unused input: past the end of the stream it is unused_data and + // unconsumed_tail is emptied, otherwise it is the new unconsumed_tail + let mut unconsumed = &data[consumed..]; + if stream_end { + unused_data.extend_from_slice(unconsumed); + unconsumed = &[]; } + unconsumed_tail.clear(); + unconsumed_tail.extend_from_slice(unconsumed); Ok((ret, stream_end)) } @@ -1005,4 +1003,22 @@ mod tests { assert_eq!(d.decompress(&encoded, None).unwrap(), b"later input"); assert!(d.eof()); } + + #[test] + fn stream_end_clears_unconsumed_tail() { + let source = b"x".repeat(20000); + let data = [compress(&source, -1, MAX_WBITS).unwrap(), b"NEXT".to_vec()].concat(); + let mut d = Decompressor::new(MAX_WBITS, None).unwrap(); + let first = d.decompress(&data, Some(100)).unwrap(); + assert!(!d.eof()); + let tail = d.unconsumed_tail().to_vec(); + let rest = d.decompress(&tail, Some(100_000)).unwrap(); + assert!(d.eof()); + assert_eq!([first, rest].concat(), source); + assert_eq!(d.unconsumed_tail(), b""); + assert_eq!(d.unused_data(), b"NEXT"); + assert_eq!(d.flush(DEF_BUF_SIZE).unwrap(), b""); + assert_eq!(d.unconsumed_tail(), b""); + assert_eq!(d.unused_data(), b"NEXT"); + } } diff --git a/extra_tests/snippets/stdlib_zlib.py b/extra_tests/snippets/stdlib_zlib.py index ac2b525b0a7..acf1b39ad52 100644 --- a/extra_tests/snippets/stdlib_zlib.py +++ b/extra_tests/snippets/stdlib_zlib.py @@ -1,3 +1,4 @@ +import sys import zlib from testutils import assert_raises @@ -62,3 +63,23 @@ assert_raises(zlib.error, lambda: zlib.compress(b"123", -40)) assert_raises(zlib.error, lambda: zlib.compress(b"123", 10)) + +# A max_length-bounded call that reaches the end of the stream keeps no +# stale input in unconsumed_tail (gh-8906). CPython before gh-158169 reports +# the bytes after the stream in unconsumed_tail too. +source = b"x" * 20000 +d = zlib.decompressobj() +out = d.decompress(zlib.compress(source) + b"NEXT", 100) +assert not d.eof +out += d.decompress(d.unconsumed_tail, 100000) +assert d.eof +assert out == source +assert d.unused_data == b"NEXT" +tail = d.unconsumed_tail +if sys.implementation.name == "rustpython": + assert tail == b"" +else: + assert tail in (b"", b"NEXT") +assert d.flush() == b"" +assert d.unconsumed_tail == tail +assert d.unused_data == b"NEXT" + tail From 99c0071c66947416626bf32bd0d1528e90fcfbc6 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 29 Sep 2026 16:13:15 +0000 Subject: [PATCH 2/3] zlib: bind the unconsumed input with an if expression Choose between the empty slice and the rest of the input in one expression instead of reassigning a mutable binding. Assisted-by: Claude Code:claude-opus-5-5 --- crates/common/src/compression/zlib.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/common/src/compression/zlib.rs b/crates/common/src/compression/zlib.rs index 3163a91124f..6c34d7cbe55 100644 --- a/crates/common/src/compression/zlib.rs +++ b/crates/common/src/compression/zlib.rs @@ -672,11 +672,12 @@ impl Decompressor { // save unused input: past the end of the stream it is unused_data and // unconsumed_tail is emptied, otherwise it is the new unconsumed_tail - let mut unconsumed = &data[consumed..]; - if stream_end { - unused_data.extend_from_slice(unconsumed); - unconsumed = &[]; - } + let unconsumed: &[u8] = if stream_end { + unused_data.extend_from_slice(&data[consumed..]); + &[] + } else { + &data[consumed..] + }; unconsumed_tail.clear(); unconsumed_tail.extend_from_slice(unconsumed); From 640838050a296ee1d2f72ddb28055f9e1a0b870c Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 29 Sep 2026 19:05:56 +0000 Subject: [PATCH 3/3] zlib: accept both valid tails in the snippet on every implementation The snippet no longer branches on sys.implementation. RustPython and CPython after gh-158169 give an empty tail; CPython before it gives the bytes after the stream. The stale tail of gh-8906 fails either way. Assisted-by: Claude Code:claude-opus-5-5 --- extra_tests/snippets/stdlib_zlib.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/extra_tests/snippets/stdlib_zlib.py b/extra_tests/snippets/stdlib_zlib.py index acf1b39ad52..dfadf7eebdc 100644 --- a/extra_tests/snippets/stdlib_zlib.py +++ b/extra_tests/snippets/stdlib_zlib.py @@ -1,4 +1,3 @@ -import sys import zlib from testutils import assert_raises @@ -65,8 +64,8 @@ assert_raises(zlib.error, lambda: zlib.compress(b"123", 10)) # A max_length-bounded call that reaches the end of the stream keeps no -# stale input in unconsumed_tail (gh-8906). CPython before gh-158169 reports -# the bytes after the stream in unconsumed_tail too. +# stale input in unconsumed_tail (gh-8906). The tail is empty, or, as in +# CPython before gh-158169, the bytes after the stream. source = b"x" * 20000 d = zlib.decompressobj() out = d.decompress(zlib.compress(source) + b"NEXT", 100) @@ -76,10 +75,7 @@ assert out == source assert d.unused_data == b"NEXT" tail = d.unconsumed_tail -if sys.implementation.name == "rustpython": - assert tail == b"" -else: - assert tail in (b"", b"NEXT") +assert tail in (b"", b"NEXT") assert d.flush() == b"" assert d.unconsumed_tail == tail assert d.unused_data == b"NEXT" + tail