Snappy: Use unsigned short to handle 2 ^ 16 input size instead of 2 ^ 15 (#13828)

lucamolteni · web-flow · commit 59a8bd070858 · 2024-02-09T16:43:17.000+01:00
Motivation: Snappy compressor to be able to compress a much larger amount than 32767 bytes. Modifications: - Avoid iterating to create correct size hash table (limited to MAX_HT_SIZE) - Configurable snappy frame size for the encoder - Added static factory method for SnappyEncoder supporting 2 ^ 16 size frame slices - Added integration test Result: Fixes #13226
diff --git a/codec/src/main/java/io/netty/handler/codec/compression/Snappy.java b/codec/src/main/java/io/netty/handler/codec/compression/Snappy.java
@@ -16,6 +16,7 @@
 package io.netty.handler.codec.compression;
 
 import io.netty.buffer.ByteBuf;
+import io.netty.util.internal.MathUtil;
 
 /**
  * Uncompresses an input {@link ByteBuf} encoded with Snappy compression into an
@@ -95,7 +96,9 @@ public void encode(final ByteBuf in, final ByteBuf out, final int length) {
 
                     nextHash = hash(in, nextIndex, shift);
 
-                    candidate = baseIndex + table[hash];
+                    // equivalent to Short.toUnsignedInt
+                    // use unsigned short cast to avoid loss precision when 32767 <= length <= 65355
+                    candidate = baseIndex + ((int) table[hash]) & 0xffff;
 
                     table[hash] = (short) (inIndex - baseIndex);
                 }
@@ -157,11 +160,8 @@ private static int hash(ByteBuf in, int index, int shift) {
      * @return An appropriately sized empty hashtable
      */
     private static short[] getHashTable(int inputSize) {
-        int htSize = 256;
-        while (htSize < MAX_HT_SIZE && htSize < inputSize) {
-            htSize <<= 1;
-        }
-        return new short[htSize];
+        int hashTableSize = MathUtil.findNextPositivePowerOfTwo(inputSize);
+        return new short[Math.min(hashTableSize, MAX_HT_SIZE)];
     }
 
     /**
diff --git a/codec/src/main/java/io/netty/handler/codec/compression/SnappyFrameEncoder.java b/codec/src/main/java/io/netty/handler/codec/compression/SnappyFrameEncoder.java
@@ -27,6 +27,17 @@
  * See <a href="https://github.com/google/snappy/blob/master/framing_format.txt">Snappy framing format</a>.
  */
 public class SnappyFrameEncoder extends MessageToByteEncoder<ByteBuf> {
+
+    private static final short SNAPPY_SLICE_SIZE = Short.MAX_VALUE;
+
+    /**
+     * Both
+     * {@value io.netty.handler.codec.compression.SnappyFrameEncoder#SNAPPY_SLICE_SIZE}
+     * and {@value io.netty.handler.codec.compression.SnappyFrameEncoder#SNAPPY_SLICE_JUMBO_SIZE}
+     * are valid lengths for the Snappy framing format
+     */
+    private static final int SNAPPY_SLICE_JUMBO_SIZE = 65535;
+
     /**
      * The minimum amount that we'll consider actually attempting to compress.
      * This value is preamble + the minimum length our Snappy service will
@@ -42,8 +53,26 @@ public class SnappyFrameEncoder extends MessageToByteEncoder<ByteBuf> {
         (byte) 0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59
     };
 
+    public SnappyFrameEncoder() {
+        this(SNAPPY_SLICE_SIZE);
+    }
+
+    /**
+     * Create a new instance with a
+     * {@value io.netty.handler.codec.compression.SnappyFrameEncoder#SNAPPY_SLICE_JUMBO_SIZE}
+     * chunk size.
+     */
+    public static SnappyFrameEncoder snappyEncoderWithJumboFrames() {
+        return new SnappyFrameEncoder(SNAPPY_SLICE_JUMBO_SIZE);
+    }
+
+    private SnappyFrameEncoder(int sliceSize) {
+        this.sliceSize = sliceSize;
+    }
+
     private final Snappy snappy = new Snappy();
     private boolean started;
+    private final int sliceSize;
 
     @Override
     protected void encode(ChannelHandlerContext ctx, ByteBuf in, ByteBuf out) throws Exception {
@@ -67,12 +96,12 @@ protected void encode(ChannelHandlerContext ctx, ByteBuf in, ByteBuf out) throws
                 }
 
                 out.writeInt(0);
-                if (dataLength > Short.MAX_VALUE) {
-                    ByteBuf slice = in.readSlice(Short.MAX_VALUE);
+                if (dataLength > sliceSize) {
+                    ByteBuf slice = in.readSlice(sliceSize);
                     calculateAndWriteChecksum(slice, out);
-                    snappy.encode(slice, out, Short.MAX_VALUE);
+                    snappy.encode(slice, out, sliceSize);
                     setChunkLength(out, lengthIdx);
-                    dataLength -= Short.MAX_VALUE;
+                    dataLength -= sliceSize;
                 } else {
                     ByteBuf slice = in.readSlice(dataLength);
                     calculateAndWriteChecksum(slice, out);
diff --git a/codec/src/test/java/io/netty/handler/codec/compression/SnappyJumboSizeIntegrationTest.java b/codec/src/test/java/io/netty/handler/codec/compression/SnappyJumboSizeIntegrationTest.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2024 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.codec.compression;
+
+import io.netty.channel.embedded.EmbeddedChannel;
+
+public class SnappyJumboSizeIntegrationTest extends AbstractIntegrationTest {
+
+    @Override
+    protected EmbeddedChannel createEncoder() {
+        return new EmbeddedChannel(SnappyFrameEncoder.snappyEncoderWithJumboFrames());
+    }
+
+    @Override
+    protected EmbeddedChannel createDecoder() {
+        return new EmbeddedChannel(new SnappyFrameDecoder());
+    }
+
+}
