WebSocket frame encoding: avoid redundant allocations on frame masking (#12969)

mostroverkhov · web-flow · commit 75aac7c87d82 · 2022-11-09T08:52:58.000+01:00
Motivation:

When masking is enabled, redundant intermediate bytebuffer + array is created for each websocket message.

Modifications:

Remove intermediate bytebuffer used to hold frame mask, use int mask directly.

Result:

No redundant allocations on frame masking
diff --git a/codec-http/src/main/java/io/netty/handler/codec/http/websocketx/WebSocket08FrameEncoder.java b/codec-http/src/main/java/io/netty/handler/codec/http/websocketx/WebSocket08FrameEncoder.java
@@ -106,7 +106,6 @@ public WebSocket08FrameEncoder(boolean maskPayload) {
     @Override
     protected void encode(ChannelHandlerContext ctx, WebSocketFrame msg, List<Object> out) throws Exception {
         final ByteBuf data = msg.content();
-        byte[] mask;
 
         byte opcode;
         if (msg instanceof TextWebSocketFrame) {
@@ -175,9 +174,8 @@ protected void encode(ChannelHandlerContext ctx, WebSocketFrame msg, List<Object
 
             // Write payload
             if (maskPayload) {
-                int random = PlatformDependent.threadLocalRandom().nextInt(Integer.MAX_VALUE);
-                mask = ByteBuffer.allocate(4).putInt(random).array();
-                buf.writeBytes(mask);
+                int mask = PlatformDependent.threadLocalRandom().nextInt(Integer.MAX_VALUE);
+                buf.writeInt(mask);
 
                 if (data.isReadable()) {
 
@@ -189,13 +187,7 @@ protected void encode(ChannelHandlerContext ctx, WebSocketFrame msg, List<Object
 
                     if (srcOrder == dstOrder) {
                         // Use the optimized path only when byte orders match
-                        // Remark: & 0xFF is necessary because Java will do signed expansion from
-                        // byte to int which we don't want.
-                        long longMask = (long) (mask[0] & 0xff) << 24
-                                        | (mask[1] & 0xff) << 16
-                                        | (mask[2] & 0xff) << 8
-                                        | (mask[3] & 0xff);
-                        longMask |= longMask << 32;
+                        long longMask = mask | (long) mask << 32;
 
                         // If the byte order of our buffers it little endian we have to bring our mask
                         // into the same format, because getInt() and writeInt() will use a reversed byte order
@@ -215,7 +207,7 @@ protected void encode(ChannelHandlerContext ctx, WebSocketFrame msg, List<Object
                     int maskOffset = 0;
                     for (; i < end; i++) {
                         byte byteData = data.getByte(i);
-                        buf.writeByte(byteData ^ mask[maskOffset++ & 3]);
+                        buf.writeByte(byteData ^ byteAtIndex(mask, maskOffset++ & 3));
                     }
                 }
                 out.add(buf);
@@ -236,4 +228,8 @@ protected void encode(ChannelHandlerContext ctx, WebSocketFrame msg, List<Object
             }
         }
     }
+
+    private static int byteAtIndex(int mask, int index) {
+        return (mask >> 8 * (3 - index)) & 0xFF;
+    }
 }
