re-structured wasm modules

gfx · gfx · commit 9cdac6fcf2dd · 2019-05-16T09:47:50.000+09:00
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -7,7 +7,7 @@
   "types": "./dist/index.d.ts",
   "scripts": {
     "build": "npm publish --dry-run",
-    "prepare": "rm -rf dist dist.* ; tsc -p tsconfig.dist.json && webpack",
+    "prepare": "rm -rf dist dist.* ; tsc -p tsconfig.dist.json && webpack && npm run asbuild",
     "prepublishOnly": "TEST_DIST=true npm run test",
     "clean": "rm -rf build dist dist.*",
     "test": "mocha 'test/**/*.test.ts'",
@@ -24,9 +24,9 @@
     "profile:encode": "rm -f isolate-*.log ; node --prof --require ts-node/register  -e 'require(\"./benchmark/profile-encode\")' && node --prof-process --preprocess -j isolate-*.log | npx flamebearer",
     "profile:decode": "rm -f isolate-*.log ; node --prof --require ts-node/register  -e 'require(\"./benchmark/profile-decode\")' && node --prof-process --preprocess -j isolate-*.log | npx flamebearer",
     "benchmark": "ts-node benchmark/benchmark-from-msgpack-lite.ts",
-    "asbuild:untouched": "asc assembly/index.ts -b build/wasm/untouched.wasm -t build/wasm/untouched.wat --sourceMap --validate --debug",
-    "asbuild:optimized": "asc assembly/index.ts -b build/wasm/optimized.wasm -t build/wasm/optimized.wat --sourceMap --validate -O3",
-    "asbuild": "npm run asbuild:untouched && npm run asbuild:optimized"
+    "asbuild:untouched": "asc assembly/index.ts -b build/wasm/untouched.wasm -t build/wasm/untouched.wat --sourceMap --validate --debug --measure",
+    "asbuild:optimized": "asc assembly/index.ts -b build/wasm/optimized.wasm -t build/wasm/optimized.wat --sourceMap --validate -O3 --measure",
+    "asbuild": "rm -rf build/wasm && npm run asbuild:untouched && npm run asbuild:optimized && ts-node tools/pack-wasm.ts"
   },
   "repository": {
     "type": "git",
@@ -45,6 +45,7 @@
   "homepage": "https://msgpack.org/",
   "devDependencies": {
     "@bitjourney/check-es-version-webpack-plugin": "^1.0.2",
+    "@types/base64-js": "^1.2.5",
     "@types/mocha": "^5.2.6",
     "@types/node": "^11.13.10",
     "@typescript-eslint/eslint-plugin": "^1.9.0",
@@ -75,7 +76,9 @@
     "webpack": "^4.30.0",
     "webpack-cli": "^3.3.1"
   },
-  "dependencies": {},
+  "dependencies": {
+    "base64-js": "^1.3.0"
+  },
   "files": [
     "src/**/*.*",
     "dist/**/*.*",
diff --git a/src/Decoder.ts b/src/Decoder.ts
@@ -2,7 +2,7 @@ import { prettyByte } from "./utils/prettyByte";
 import { ExtensionCodec } from "./ExtensionCodec";
 import { getInt64, getUint64 } from "./utils/int";
 import { utf8Decode } from "./utils/utf8";
-import { utf8Decode2 } from "../wasmModule";
+import { utf8DecodeWasm, WASM_AVAILABLE } from "./wasmFunctions";
 import { createDataView, ensureUint8Array } from "./utils/typedArrays";
 
 enum State {
@@ -47,8 +47,6 @@ export const DataViewIndexOutOfBoundsError: typeof Error = (() => {
 
 const MORE_DATA = new DataViewIndexOutOfBoundsError("Insufficient data");
 
-const USE_WASM = process.env.USE_WASM === "true";
-
 export class Decoder {
   totalPos = 0;
   pos = 0;
@@ -382,8 +380,8 @@ export class Decoder {
     }
 
     let object: string;
-    if (USE_WASM) {
-      object = utf8Decode2(this.bytes, this.pos + headOffset, byteLength);
+    if (WASM_AVAILABLE) {
+      object = utf8DecodeWasm(this.bytes, this.pos + headOffset, byteLength);
     } else {
       object = utf8Decode(this.bytes, this.pos + headOffset, byteLength);
     }
diff --git a/src/wasmFunctions.ts b/src/wasmFunctions.ts
@@ -0,0 +1,51 @@
+let wasmModule: any;
+try {
+  wasmModule = require("../build/wasm/optimized.wasm.js").wasmModule;
+} catch {
+  // WebAssembly is not supported.
+}
+
+declare var WebAssembly: any;
+const WASM_MEMORY_PAGE_SIZE = 0x10000; // 64KiB
+
+const defaultWasmInstance = wasmModule && new WebAssembly.Instance(wasmModule);
+
+export const WASM_AVAILABLE = !!wasmModule && process.env.NO_WASM !== "true";
+
+function copyArrayBuffer(dest: ArrayBuffer, src: Uint8Array) {
+  const destView = new Uint8Array(dest);
+  destView.set(src);
+}
+
+export function utf8DecodeWasm(
+  bytes: Uint8Array,
+  offset: number,
+  byteLength: number,
+  wasmInstance = defaultWasmInstance,
+): string {
+  if (!wasmInstance) {
+    throw new Error("No WebAssembly available");
+  }
+
+  const currentMemorySize: number = wasmInstance.exports.memory.buffer.byteLength;
+  const requiredMemorySize = bytes.length * 3; // input(utf8) + output(utf16)
+  if (currentMemorySize < requiredMemorySize) {
+    const page = Math.ceil((requiredMemorySize - currentMemorySize) / WASM_MEMORY_PAGE_SIZE);
+    wasmInstance.exports.memory.grow(page);
+  }
+
+  copyArrayBuffer(wasmInstance.exports.memory.buffer, bytes.subarray(offset, offset + byteLength));
+  // console.log(instanceMemory.subarray(0, 10));
+
+  const outputStart = Math.ceil(byteLength / Uint16Array.BYTES_PER_ELEMENT) * Uint16Array.BYTES_PER_ELEMENT;
+  const outputEnd = wasmInstance.exports.utf8ToUtf16(byteLength, outputStart);
+  const codepoints = new Uint16Array(
+    wasmInstance.exports.memory.buffer,
+    outputStart,
+    (outputEnd - outputStart) / Uint16Array.BYTES_PER_ELEMENT,
+  );
+  // console.log([byteLength, outputStart, outputEnd]);
+  // console.log(instanceMemory.subarray(0, 10));
+  // console.log(utf16array);
+  return String.fromCharCode.apply(String, codepoints as any);
+}
diff --git a/tools/pack-wasm.ts b/tools/pack-wasm.ts
@@ -0,0 +1,25 @@
+// pack build/wasm/*.wasm
+
+import fs from "fs";
+import { resolve } from "path";
+import base64 from "base64-js";
+
+const artifactDir = resolve(__dirname, "../build/wasm");
+for (const basename of fs.readdirSync(artifactDir)) {
+  const file = resolve(artifactDir, basename);
+  if (!file.endsWith(".wasm")) {
+    continue;
+  }
+
+  const blob = fs.readFileSync(file);
+  fs.writeFileSync(
+    `${file}.js`,
+    `// generated from ${basename}
+var base64 = require("base64-js");
+module.exports.wasmModule = new WebAssembly.Module(
+  base64.toByteArray(
+    ${JSON.stringify(base64.fromByteArray(blob))}
+));
+`,
+  );
+}
diff --git a/wasmModule.ts b/wasmModule.ts
