Skip to content

Commit fa97ee9

Browse files
DeviLabDeviLab
authored
Make public API specify explicit maxAllocation to prevent OOM (#15005)
Motivation: The current `ZLibCodecFactory` provides `newZlibDecoder` methods without an option to specify a maximum memory limit for decompression. These methods are utilized in various parts of the project, such as the per-message WebSocket extension. As a result, a client could send a small, maliciously crafted compressed message that, upon decompression, would consume all available memory. This can lead to an `OutOfMemoryError` scenario, which can easily be reproduced as follows: ``` Exception in thread "io-compute-15" java.lang.OutOfMemoryError: Java heap space at io.netty.util.internal.PlatformDependent.allocateUninitializedArray(PlatformDependent.java:326) at io.netty.buffer.PoolArena$HeapArena.newByteArray(PoolArena.java:628) at io.netty.buffer.PoolArena$HeapArena.newUnpooledChunk(PoolArena.java:652) at io.netty.buffer.PoolArena.allocateHuge(PoolArena.java:224) at io.netty.buffer.PoolArena.allocate(PoolArena.java:142) at io.netty.buffer.PoolArena.reallocate(PoolArena.java:317) at io.netty.buffer.PooledByteBuf.capacity(PooledByteBuf.java:123) at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:333) at io.netty.handler.codec.compression.ZlibDecoder.prepareDecompressBuffer(ZlibDecoder.java:74) at io.netty.handler.codec.compression.JdkZlibDecoder.decode(JdkZlibDecoder.java:265) at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at io.netty.channel.embedded.EmbeddedChannel.writeInbound(EmbeddedChannel.java:348) at io.netty.handler.codec.http.websocketx.extensions.compression.DeflateDecoder.decompressContent(DeflateDecoder.java:119) at io.netty.handler.codec.http.websocketx.extensions.compression.DeflateDecoder.decode(DeflateDecoder.java:80) at io.netty.handler.codec.http.websocketx.extensions.compression.PerMessageDeflateDecoder.decode(PerMessageDeflateDecoder.java:87) at io.netty.handler.codec.http.websocketx.extensions.compression.PerMessageDeflateDecoder.decode(PerMessageDeflateDecoder.java:31) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:91) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) java.lang.OutOfMemoryError: Java heap space at io.netty.util.internal.PlatformDependent.allocateUninitializedArray(PlatformDependent.java:326) at io.netty.buffer.PoolArena$HeapArena.newByteArray(PoolArena.java:628) at io.netty.buffer.PoolArena$HeapArena.newUnpooledChunk(PoolArena.java:652) ``` Modification: - Introduced new `newZlibDecoder` methods within `ZlibCodecFactory` that include an explicit `maxAllocation` parameter to specify the maximum allowed memory during decompression. - The older methods have been deprecated in favor of the new ones. - Public APIs that invoke `newZlibDecoder` now require the `maxAllocation` parameter as well. Result: This change does not modify the public API behavior, but it encourages users to adopt the updated methods, which include the explicit `maxAllocation` argument, providing more control over memory usage during decompression. Fixes #6663.
1 parent c750775 commit fa97ee9

36 files changed

Lines changed: 623 additions & 136 deletions

File tree

‎codec-compression/src/main/java/io/netty/handler/codec/compression/JZlibDecoder.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ public class JZlibDecoder extends ZlibDecoder {
3434
* Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
3535
*
3636
* @throws DecompressionException if failed to initialize zlib
37+
* @deprecated Use {@link JZlibDecoder#JZlibDecoder(int)}.
3738
*/
39+
@Deprecated
3840
public JZlibDecoder() {
3941
this(ZlibWrapper.ZLIB, 0);
4042
}
@@ -57,7 +59,9 @@ public JZlibDecoder(int maxAllocation) {
5759
* Creates a new instance with the specified wrapper.
5860
*
5961
* @throws DecompressionException if failed to initialize zlib
62+
* @deprecated Use {@link JZlibDecoder#JZlibDecoder(ZlibWrapper, int)}.
6063
*/
64+
@Deprecated
6165
public JZlibDecoder(ZlibWrapper wrapper) {
6266
this(wrapper, 0);
6367
}
@@ -88,7 +92,9 @@ public JZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
8892
* supports the preset dictionary.
8993
*
9094
* @throws DecompressionException if failed to initialize zlib
95+
* @deprecated Use {@link JZlibDecoder#JZlibDecoder(byte[], int)}.
9196
*/
97+
@Deprecated
9298
public JZlibDecoder(byte[] dictionary) {
9399
this(dictionary, 0);
94100
}

‎codec-compression/src/main/java/io/netty/handler/codec/compression/JdkZlibDecoder.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,10 @@ private enum GzipState {
6464

6565
/**
6666
* Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
67+
*
68+
* @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(int)}.
6769
*/
70+
@Deprecated
6871
public JdkZlibDecoder() {
6972
this(ZlibWrapper.ZLIB, null, false, 0);
7073
}
@@ -85,7 +88,10 @@ public JdkZlibDecoder(int maxAllocation) {
8588
* Creates a new instance with the specified preset dictionary. The wrapper
8689
* is always {@link ZlibWrapper#ZLIB} because it is the only format that
8790
* supports the preset dictionary.
91+
*
92+
* @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(byte[], int)}.
8893
*/
94+
@Deprecated
8995
public JdkZlibDecoder(byte[] dictionary) {
9096
this(ZlibWrapper.ZLIB, dictionary, false, 0);
9197
}
@@ -107,7 +113,10 @@ public JdkZlibDecoder(byte[] dictionary, int maxAllocation) {
107113
* Creates a new instance with the specified wrapper.
108114
* Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
109115
* supported atm.
116+
*
117+
* @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, int)}.
110118
*/
119+
@Deprecated
111120
public JdkZlibDecoder(ZlibWrapper wrapper) {
112121
this(wrapper, null, false, 0);
113122
}
@@ -125,6 +134,10 @@ public JdkZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
125134
this(wrapper, null, false, maxAllocation);
126135
}
127136

137+
/**
138+
* @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(ZlibWrapper, boolean, int)}.
139+
*/
140+
@Deprecated
128141
public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated) {
129142
this(wrapper, null, decompressConcatenated, 0);
130143
}
@@ -133,6 +146,10 @@ public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated, int m
133146
this(wrapper, null, decompressConcatenated, maxAllocation);
134147
}
135148

149+
/**
150+
* @deprecated Use {@link JdkZlibDecoder#JdkZlibDecoder(boolean, int)}.
151+
*/
152+
@Deprecated
136153
public JdkZlibDecoder(boolean decompressConcatenated) {
137154
this(ZlibWrapper.GZIP, null, decompressConcatenated, 0);
138155
}

‎codec-compression/src/main/java/io/netty/handler/codec/compression/ZlibCodecFactory.java‎

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -120,27 +120,82 @@ public static ZlibEncoder newZlibEncoder(int compressionLevel, int windowBits, i
120120
}
121121
}
122122

123+
/**
124+
* Create a new decoder instance.
125+
*
126+
* @deprecated Use {@link ZlibCodecFactory#newZlibDecoder(int)}.
127+
*/
128+
@Deprecated
123129
public static ZlibDecoder newZlibDecoder() {
130+
return newZlibDecoder(0);
131+
}
132+
133+
/**
134+
* Create a new decoder instance with specified maximum buffer allocation.
135+
*
136+
* @param maxAllocation
137+
* Maximum size of the decompression buffer. Must be >= 0.
138+
* If zero, maximum size is not limited by decoder.
139+
*/
140+
public static ZlibDecoder newZlibDecoder(int maxAllocation) {
124141
if (noJdkZlibDecoder) {
125-
return new JZlibDecoder();
142+
return new JZlibDecoder(maxAllocation);
126143
} else {
127-
return new JdkZlibDecoder(true);
144+
return new JdkZlibDecoder(true, maxAllocation);
128145
}
129146
}
130147

148+
/**
149+
* Create a new decoder instance with the specified wrapper.
150+
*
151+
* @deprecated Use {@link ZlibCodecFactory#newZlibDecoder(ZlibWrapper, int)}.
152+
*/
153+
@Deprecated
131154
public static ZlibDecoder newZlibDecoder(ZlibWrapper wrapper) {
155+
return newZlibDecoder(wrapper, 0);
156+
}
157+
158+
/**
159+
* Create a new decoder instance with the specified wrapper and maximum buffer allocation.
160+
*
161+
* @param maxAllocation
162+
* Maximum size of the decompression buffer. Must be >= 0.
163+
* If zero, maximum size is not limited by decoder.
164+
*/
165+
public static ZlibDecoder newZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
132166
if (noJdkZlibDecoder) {
133-
return new JZlibDecoder(wrapper);
167+
return new JZlibDecoder(wrapper, maxAllocation);
134168
} else {
135-
return new JdkZlibDecoder(wrapper, true);
169+
return new JdkZlibDecoder(wrapper, true, maxAllocation);
136170
}
137171
}
138172

173+
/**
174+
* Create a new decoder instance with the specified preset dictionary. The wrapper
175+
* is always {@link ZlibWrapper#ZLIB} because it is the only format that
176+
* supports the preset dictionary.
177+
*
178+
* @deprecated Use {@link ZlibCodecFactory#newZlibDecoder(byte[], int)}.
179+
*/
180+
@Deprecated
139181
public static ZlibDecoder newZlibDecoder(byte[] dictionary) {
182+
return newZlibDecoder(dictionary, 0);
183+
}
184+
185+
/**
186+
* Create a new decoder instance with the specified preset dictionary and maximum buffer allocation.
187+
* The wrapper is always {@link ZlibWrapper#ZLIB} because it is the only format that
188+
* supports the preset dictionary.
189+
*
190+
* @param maxAllocation
191+
* Maximum size of the decompression buffer. Must be >= 0.
192+
* If zero, maximum size is not limited by decoder.
193+
*/
194+
public static ZlibDecoder newZlibDecoder(byte[] dictionary, int maxAllocation) {
140195
if (noJdkZlibDecoder) {
141-
return new JZlibDecoder(dictionary);
196+
return new JZlibDecoder(dictionary, maxAllocation);
142197
} else {
143-
return new JdkZlibDecoder(dictionary);
198+
return new JdkZlibDecoder(dictionary, maxAllocation);
144199
}
145200
}
146201

‎codec-compression/src/test/java/io/netty/handler/codec/compression/JdkZlibTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public void testConcatenatedStreamsReadFirstOnly() throws IOException {
8686

8787
@Test
8888
public void testConcatenatedStreamsReadFully() throws IOException {
89-
EmbeddedChannel chDecoderGZip = new EmbeddedChannel(new JdkZlibDecoder(true));
89+
EmbeddedChannel chDecoderGZip = new EmbeddedChannel(new JdkZlibDecoder(true, 0));
9090

9191
try {
9292
byte[] bytes = IOUtils.toByteArray(getClass().getResourceAsStream("/multiple.gz"));
@@ -108,7 +108,7 @@ public void testConcatenatedStreamsReadFully() throws IOException {
108108

109109
@Test
110110
public void testConcatenatedStreamsReadFullyWhenFragmented() throws IOException {
111-
EmbeddedChannel chDecoderGZip = new EmbeddedChannel(new JdkZlibDecoder(true));
111+
EmbeddedChannel chDecoderGZip = new EmbeddedChannel(new JdkZlibDecoder(true, 0));
112112

113113
try {
114114
byte[] bytes = IOUtils.toByteArray(getClass().getResourceAsStream("/multiple.gz"));
@@ -147,7 +147,7 @@ public void testDecodeWithHeaderFollowingFooter() throws Exception {
147147

148148
byte[] compressed = bytesOut.toByteArray();
149149
ByteBuf buffer = Unpooled.buffer().writeBytes(compressed).writeBytes(compressed);
150-
EmbeddedChannel channel = new EmbeddedChannel(new JdkZlibDecoder(ZlibWrapper.GZIP, true));
150+
EmbeddedChannel channel = new EmbeddedChannel(new JdkZlibDecoder(ZlibWrapper.GZIP, true, 0));
151151
// Write it into the Channel in a way that we were able to decompress the first data completely but not the
152152
// whole footer.
153153
assertTrue(channel.writeInbound(buffer.readRetainedSlice(compressed.length - 1)));

‎codec-http/src/main/java/io/netty/handler/codec/http/HttpContentDecompressor.java‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import static io.netty.handler.codec.http.HttpHeaderValues.X_GZIP;
2323
import static io.netty.handler.codec.http.HttpHeaderValues.SNAPPY;
2424
import static io.netty.handler.codec.http.HttpHeaderValues.ZSTD;
25+
import static io.netty.util.internal.ObjectUtil.checkPositiveOrZero;
2526

2627
import io.netty.channel.embedded.EmbeddedChannel;
2728
import io.netty.handler.codec.compression.Brotli;
@@ -40,37 +41,66 @@
4041
public class HttpContentDecompressor extends HttpContentDecoder {
4142

4243
private final boolean strict;
44+
private final int maxAllocation;
4345

4446
/**
4547
* Create a new {@link HttpContentDecompressor} in non-strict mode.
48+
* @deprecated
49+
* Use {@link HttpContentDecompressor#HttpContentDecompressor(int)}.
4650
*/
51+
@Deprecated
4752
public HttpContentDecompressor() {
48-
this(false);
53+
this(false, 0);
54+
}
55+
56+
/**
57+
* Create a new {@link HttpContentDecompressor} in non-strict mode.
58+
* @param maxAllocation
59+
* Maximum size of the decompression buffer. Must be >= 0. If zero, maximum size is not limited.
60+
*/
61+
public HttpContentDecompressor(int maxAllocation) {
62+
this(false, maxAllocation);
4963
}
5064

5165
/**
5266
* Create a new {@link HttpContentDecompressor}.
5367
*
5468
* @param strict if {@code true} use strict handling of deflate if used, otherwise handle it in a
5569
* more lenient fashion.
70+
* @deprecated
71+
* Use {@link HttpContentDecompressor#HttpContentDecompressor(boolean, int)}.
5672
*/
73+
@Deprecated
5774
public HttpContentDecompressor(boolean strict) {
75+
this(strict, 0);
76+
}
77+
78+
/**
79+
* Create a new {@link HttpContentDecompressor}.
80+
*
81+
* @param strict if {@code true} use strict handling of deflate if used, otherwise handle it in a
82+
* more lenient fashion.
83+
* @param maxAllocation
84+
* Maximum size of the decompression buffer. Must be >= 0. If zero, maximum size is not limited.
85+
*/
86+
public HttpContentDecompressor(boolean strict, int maxAllocation) {
5887
this.strict = strict;
88+
this.maxAllocation = checkPositiveOrZero(maxAllocation, "maxAllocation");
5989
}
6090

6191
@Override
6292
protected EmbeddedChannel newContentDecoder(String contentEncoding) throws Exception {
6393
if (GZIP.contentEqualsIgnoreCase(contentEncoding) ||
6494
X_GZIP.contentEqualsIgnoreCase(contentEncoding)) {
6595
return new EmbeddedChannel(ctx.channel().id(), ctx.channel().metadata().hasDisconnect(),
66-
ctx.channel().config(), ZlibCodecFactory.newZlibDecoder(ZlibWrapper.GZIP));
96+
ctx.channel().config(), ZlibCodecFactory.newZlibDecoder(ZlibWrapper.GZIP, maxAllocation));
6797
}
6898
if (DEFLATE.contentEqualsIgnoreCase(contentEncoding) ||
6999
X_DEFLATE.contentEqualsIgnoreCase(contentEncoding)) {
70100
final ZlibWrapper wrapper = strict ? ZlibWrapper.ZLIB : ZlibWrapper.ZLIB_OR_NONE;
71101
// To be strict, 'deflate' means ZLIB, but some servers were not implemented correctly.
72102
return new EmbeddedChannel(ctx.channel().id(), ctx.channel().metadata().hasDisconnect(),
73-
ctx.channel().config(), ZlibCodecFactory.newZlibDecoder(wrapper));
103+
ctx.channel().config(), ZlibCodecFactory.newZlibDecoder(wrapper, maxAllocation));
74104
}
75105
if (Brotli.isAvailable() && BR.contentEqualsIgnoreCase(contentEncoding)) {
76106
return new EmbeddedChannel(ctx.channel().id(), ctx.channel().metadata().hasDisconnect(),

‎codec-http/src/main/java/io/netty/handler/codec/http/websocketx/extensions/compression/DeflateDecoder.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ abstract class DeflateDecoder extends WebSocketExtensionDecoder {
5050

5151
private final boolean noContext;
5252
private final WebSocketExtensionFilter extensionDecoderFilter;
53+
private final int maxAllocation;
5354

5455
private EmbeddedChannel decoder;
5556

@@ -59,9 +60,10 @@ abstract class DeflateDecoder extends WebSocketExtensionDecoder {
5960
* @param noContext true to disable context takeover.
6061
* @param extensionDecoderFilter extension decoder filter.
6162
*/
62-
DeflateDecoder(boolean noContext, WebSocketExtensionFilter extensionDecoderFilter) {
63+
DeflateDecoder(boolean noContext, WebSocketExtensionFilter extensionDecoderFilter, int maxAllocation) {
6364
this.noContext = noContext;
6465
this.extensionDecoderFilter = checkNotNull(extensionDecoderFilter, "extensionDecoderFilter");
66+
this.maxAllocation = maxAllocation;
6567
}
6668

6769
/**
@@ -110,7 +112,7 @@ private ByteBuf decompressContent(ChannelHandlerContext ctx, WebSocketFrame msg)
110112
if (!(msg instanceof TextWebSocketFrame) && !(msg instanceof BinaryWebSocketFrame)) {
111113
throw new CodecException("unexpected initial frame type: " + msg.getClass().getName());
112114
}
113-
decoder = new EmbeddedChannel(ZlibCodecFactory.newZlibDecoder(ZlibWrapper.NONE));
115+
decoder = new EmbeddedChannel(ZlibCodecFactory.newZlibDecoder(ZlibWrapper.NONE, maxAllocation));
114116
}
115117

116118
boolean readable = msg.content().isReadable();

0 commit comments

Comments
 (0)