do not drop lone surrogate

gfx · gfx · commit 5e631cc7caa8 · 2019-05-24T00:21:03.000+09:00
diff --git a/assembly/utf8CountUint16Array.ts b/assembly/utf8CountUint16Array.ts
@@ -29,9 +29,6 @@ export function utf8CountUint16Array(inputPtr: usize, inputLength: usize): usize
             value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;
           }
         }
-        if (value >= 0xd800 && value <= 0xdbff) {
-          continue; // drop lone surrogate
-        }
       }
 
       if ((value & 0xffff0000) === 0) {
diff --git a/assembly/utf8EncodeUint16Array.ts b/assembly/utf8EncodeUint16Array.ts
@@ -59,9 +59,6 @@ export function utf8EncodeUint16Array(outputPtr: usize, inputPtr: usize, inputLe
             value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;
           }
         }
-        if (value >= 0xd800 && value <= 0xdbff) {
-          continue; // drop lone surrogate
-        }
       }
 
       if ((value & 0xffff0000) === 0) {
diff --git a/src/utils/utf8.ts b/src/utils/utf8.ts
@@ -24,9 +24,6 @@ export function utf8Count(str: string): number {
             value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;
           }
         }
-        if (value >= 0xd800 && value <= 0xdbff) {
-          continue; // FIXME: drop lone surrogate
-        }
       }
 
       if ((value & 0xffff0000) === 0) {
@@ -66,9 +63,6 @@ export function utf8Encode(str: string, output: DataView, outputOffset: number):
             value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;
           }
         }
-        if (value >= 0xd800 && value <= 0xdbff) {
-          continue; // FIXME: drop lone surrogate
-        }
       }
 
       if ((value & 0xffff0000) === 0) {
diff --git a/test/msgpack-test-suite.test.ts b/test/msgpack-test-suite.test.ts
@@ -95,6 +95,7 @@ describe("msgpack-test-suite", () => {
       STR32: "b".repeat(0x10_000),
       STR32LARGE: "c".repeat(0x100_000), // may cause "RangeError: Maximum call stack size exceeded" in simple implelementions
       STR_BROKEN_FF: "\xff",
+      STR_LONE_SURROGATE: "\ud800",
       BIN16: new Uint8Array(0x100).fill(0xff),
       BIN32: new Uint8Array(0x10000).fill(0xff),
       ARRAY16: new Array<boolean>(0x100).fill(true),

Original file line number	Diff line number	Diff line change
`@@ -29,9 +29,6 @@ export function utf8CountUint16Array(inputPtr: usize, inputLength: usize): usize`
`29`	`29`	`value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;`
`30`	`30`	`}`
`31`	`31`	`}`
`32`		`- if (value >= 0xd800 && value <= 0xdbff) {`
`33`		`- continue; // drop lone surrogate`
`34`		`- }`
`35`	`32`	`}`
`36`	`33`
`37`	`34`	`if ((value & 0xffff0000) === 0) {`
Original file line number	Diff line number	Diff line change
`@@ -59,9 +59,6 @@ export function utf8EncodeUint16Array(outputPtr: usize, inputPtr: usize, inputLe`
`59`	`59`	`value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;`
`60`	`60`	`}`
`61`	`61`	`}`
`62`		`- if (value >= 0xd800 && value <= 0xdbff) {`
`63`		`- continue; // drop lone surrogate`
`64`		`- }`
`65`	`62`	`}`
`66`	`63`
`67`	`64`	`if ((value & 0xffff0000) === 0) {`
Original file line number	Diff line number	Diff line change
`@@ -24,9 +24,6 @@ export function utf8Count(str: string): number {`
`24`	`24`	`value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;`
`25`	`25`	`}`
`26`	`26`	`}`
`27`		`- if (value >= 0xd800 && value <= 0xdbff) {`
`28`		`- continue; // FIXME: drop lone surrogate`
`29`		`- }`
`30`	`27`	`}`
`31`	`28`
`32`	`29`	`if ((value & 0xffff0000) === 0) {`
`@@ -66,9 +63,6 @@ export function utf8Encode(str: string, output: DataView, outputOffset: number):`
`66`	`63`	`value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;`
`67`	`64`	`}`
`68`	`65`	`}`
`69`		`- if (value >= 0xd800 && value <= 0xdbff) {`
`70`		`- continue; // FIXME: drop lone surrogate`
`71`		`- }`
`72`	`66`	`}`
`73`	`67`
`74`	`68`	`if ((value & 0xffff0000) === 0) {`