fix large string decode issues

gfx · gfx · commit 9a3c7bf22d5b · 2019-05-23T23:05:40.000+09:00
diff --git a/src/utils/utf8.ts b/src/utils/utf8.ts
@@ -1,5 +1,3 @@
-import { prettyByte } from "./prettyByte";
-
 export function utf8Count(str: string): number {
   const strLength = str.length;
 
@@ -89,6 +87,23 @@ export function utf8Encode(str: string, output: DataView, outputOffset: number):
   }
 }
 
+const CHUNK_SIZE = 0x10_000;
+
+export function safeStringFromCharCode(units: Array<number> | Uint16Array) {
+  if (units.length <= CHUNK_SIZE) {
+    // `String.fromCharCode.apply()` is faster than `String.fromCharCode(...units)`
+    // in case `units` is a typed array
+    return String.fromCharCode.apply(String, units as any);
+  }
+
+  let result = "";
+  for (let i = 0; i < units.length; i++) {
+    const chunk = units.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
+    result += String.fromCharCode.apply(String, chunk as any);
+  }
+  return result;
+}
+
 export function utf8Decode(bytes: Uint8Array, outputOffset: number, byteLength: number): string {
   let offset = outputOffset;
   const out: Array<number> = [];
@@ -123,5 +138,6 @@ export function utf8Decode(bytes: Uint8Array, outputOffset: number, byteLength:
       out.push(byte1);
     }
   }
-  return String.fromCharCode(...out);
+
+  return safeStringFromCharCode(out);
 }
diff --git a/src/wasmFunctions.ts b/src/wasmFunctions.ts
@@ -1,3 +1,5 @@
+import { safeStringFromCharCode } from "./utils/utf8";
+
 // WASM=never - disable WASM functions
 // WASM=force - force to use WASM functions
 const WASM: string = process.env.MSGPACK_WASM || process.env.WASM || "";
@@ -70,10 +72,8 @@ export function utf8DecodeWasm(bytes: Uint8Array, offset: number, byteLength: nu
     setMemoryU8(inputPtr, bytes.subarray(offset, offset + byteLength), byteLength);
 
     const outputArraySize = wm.utf8DecodeToUint16Array(outputPtr, inputPtr, byteLength);
-    const codepoints = new Uint16Array(wm.memory.buffer, outputPtr, outputArraySize);
-
-    // FIXME: split codepoints if it is too long (the maximum size depends on the JS engine, though).
-    return String.fromCharCode.apply(String, codepoints as any);
+    const units = new Uint16Array(wm.memory.buffer, outputPtr, outputArraySize);
+    return safeStringFromCharCode(units);
   } finally {
     wm.free(inputPtr);
     wm.free(outputPtr);
diff --git a/test/msgpack-test-suite.test.ts b/test/msgpack-test-suite.test.ts
@@ -91,8 +91,9 @@ describe("msgpack-test-suite", () => {
       FLOAT64_POSITIVE_INF: Number.POSITIVE_INFINITY,
       FLOAT64_NEGATIVE_INF: Number.NEGATIVE_INFINITY,
       FLOAT64_NAN: Number.NaN,
-      STR16: "x".repeat(0x100),
-      STR32: "x".repeat(0x10000),
+      STR16: "a".repeat(0x100),
+      STR32: "b".repeat(0x10_000),
+      STR32LARGE: "c".repeat(0x100_000), // may cause "RangeError: Maximum call stack size exceeded" in simple implelementions
       BIN16: new Uint8Array(0x100).fill(0xff),
       BIN32: new Uint8Array(0x10000).fill(0xff),
       ARRAY16: new Array<boolean>(0x100).fill(true),
