Take care of incomplete multi-bytes at the tail of buffer

komamitsu · komamitsu · commit ac1ad7e8d8cb · 2015-10-27T23:43:49.000+09:00
diff --git a/msgpack-core/src/main/java/org/msgpack/core/MessageUnpacker.java b/msgpack-core/src/main/java/org/msgpack/core/MessageUnpacker.java
@@ -1025,13 +1025,20 @@ public String unpackString()
                 decodeBuffer.clear();
                 StringBuilder sb = new StringBuilder();
 
+                boolean incompleteMultiBytes = false;
                 while (cursor < strLen) {
                     int readLen = Math.min(position < buffer.size() ? buffer.size() - position : buffer.size(), strLen - cursor);
+                    if (incompleteMultiBytes) {
+                        // Prepare enough buffer for decoding multi-bytes character right after running into incomplete one
+                        readLen = Math.min(config.getStringDecoderBufferSize(), strLen - cursor);
+                    }
                     if (!ensure(readLen)) {
                         throw new EOFException();
                     }
 
+                    incompleteMultiBytes = false;
                     ByteBuffer bb = buffer.toByteBuffer(position, readLen);
+                    int startPos = bb.position();
 
                     while (bb.hasRemaining()) {
                         boolean endOfInput = (cursor + readLen) >= strLen;
@@ -1051,10 +1058,9 @@ public String unpackString()
                             if (config.getActionOnMalFormedInput() == CodingErrorAction.REPORT) {
                                 throw new MalformedInputException(strLen);
                             }
-                            // trash truncated bytes
-                            while (bb.hasRemaining()) {
-                                bb.get();
-                            }
+                            incompleteMultiBytes = true;
+                            // Proceed the cursor with the length already decoded successfully.
+                            readLen = bb.position() - startPos;
                         }
 
                         if (cr.isError()) {
@@ -1068,6 +1074,10 @@ public String unpackString()
                         sb.append(decodeBuffer);
 
                         decodeBuffer.clear();
+
+                        if (incompleteMultiBytes) {
+                            break;
+                        }
                     }
 
                     cursor += readLen;
diff --git a/msgpack-core/src/test/scala/org/msgpack/core/MessageUnpackerTest.scala b/msgpack-core/src/test/scala/org/msgpack/core/MessageUnpackerTest.scala
@@ -16,6 +16,7 @@
 package org.msgpack.core
 
 import java.io._
+import java.nio.ByteBuffer
 
 import org.msgpack.core.buffer._
 import org.msgpack.value.ValueType
@@ -663,5 +664,30 @@ class MessageUnpackerTest extends MessagePackSpec {
         unpacker.getTotalReadBytes shouldBe arr.length
       }
     }
+
+    "unpack string crossing end of buffer" in {
+      def check(expected: String, strLen: Int) = {
+        val bytes = new Array[Byte](strLen)
+        val out = new ByteArrayOutputStream
+
+        val packer = MessagePack.newDefaultPacker(out)
+        packer.packBinaryHeader(bytes.length)
+        packer.writePayload(bytes)
+        packer.packString(expected)
+        packer.close
+
+        val unpacker = new MessageUnpacker(new InputStreamBufferInput(new ByteArrayInputStream(out.toByteArray)))
+        val len = unpacker.unpackBinaryHeader
+        unpacker.readPayload(len)
+        val got = unpacker.unpackString
+        unpacker.close
+
+        got shouldBe expected
+      }
+
+      Seq("あ", "aあ", "あa", "あいうえおかきくけこさしすせそ").foreach { s =>
+        Seq(8185, 8186, 8187, 8188, 16377, 16378, 16379, 16380).foreach { n => check(s, n)}
+      }
+    }
   }
 }
