From ef6938093fde66c3c512391bde004fffe45d11cb Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Tue, 5 May 2020 18:36:29 +0200 Subject: [PATCH 01/21] Added memory benchmarks to CI --- .github/workflows/ci.yml | 63 ++++ .gitignore | 4 + benchmark/README.md | 45 +++ benchmark/benchmark_types.ts | 20 ++ benchmark/dist/json.lua | 388 +++++++++++++++++++++ benchmark/memory_benchmark.ts | 47 +++ benchmark/memory_benchmarks/graph_cylce.ts | 52 +++ benchmark/run.ts | 74 ++++ benchmark/tsconfig.53.json | 11 + benchmark/tsconfig.jit.json | 11 + benchmark/tsconfig.json | 15 + benchmark/util.ts | 5 + package-lock.json | 6 + package.json | 1 + 14 files changed, 742 insertions(+) create mode 100644 benchmark/README.md create mode 100644 benchmark/benchmark_types.ts create mode 100644 benchmark/dist/json.lua create mode 100644 benchmark/memory_benchmark.ts create mode 100644 benchmark/memory_benchmarks/graph_cylce.ts create mode 100644 benchmark/run.ts create mode 100644 benchmark/tsconfig.53.json create mode 100644 benchmark/tsconfig.jit.json create mode 100644 benchmark/tsconfig.json create mode 100644 benchmark/util.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ee8ba432..736ad4bcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,3 +38,66 @@ jobs: CI: true - if: matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v1 + + benchmark: + name: Benchmark + runs-on: ubuntu-latest + steps: + - name: Lua Install + run: sudo apt-get install lua5.3 luajit + - uses: actions/checkout@v2 + - name: Use Node.js 12.13.1 + uses: actions/setup-node@v1 + with: + node-version: 12.13.1 + - run: npm ci + - run: npm run build + # This will never result in a direct cache hit. + # Teherefore benchmark data is always updated. + - name: Cache benchmark data + id: cache-benchmark + uses: actions/cache@v1 + with: + path: ./benchmark/data + key: ${{ runner.os }}-master-benchmark-${{ github.sha }} + restore-keys: ${{ runner.os }}-master-benchmark- + - name: Ensure benchmark data dir exists + run: mkdir -p ./benchmark/data + - name: Build benchmark Lua 5.3 + run: node dist/tstl.js -p benchmark/tsconfig.53.json + - name: Run benchmark Lua 5.3 + id: benchmark-lua + run: echo ::set-output name=info::`lua5.3 -- run.lua ../data/benchmark_master_53.json ${{github.ref}}` + working-directory: benchmark/dist + - name: Build benchmark LuaJIT + run: node dist/tstl.js -p benchmark/tsconfig.jit.json + - name: Run benchmark LuaJIT + id: benchmark-jit + run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_master_jit.json ${{github.ref}}` + working-directory: benchmark/dist + - name: Create benchmark check + uses: actions/github-script@0.9.0 + with: + benchmark-info-lua: ${{steps.benchmark-lua.outputs.info}} + benchmark-info-jit: ${{steps.benchmark-jit.outputs.info}} + script: | + const benchmarkInfoLua = JSON.parse(core.getInput('benchmark-info-lua', { required: true })); + const benchmarkInfoJIT = JSON.parse(core.getInput('benchmark-info-jit', { required: true })); + + const summary = `### Lua5.3\n${benchmarkInfoLua.summary}\n### LuaJIT\n${benchmarkInfoJIT.summary}`; + + const text = `### Lua5.3\n${benchmarkInfoLua.text}\n### LuaJIT\n${benchmarkInfoJIT.text}`; + + github.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: "Benchmark results", + head_sha: context.sha, + status: "completed", + conclusion: "neutral", + output: { + title: "Benchmark results", + summary: summary, + text: text + } + }); diff --git a/.gitignore b/.gitignore index 09a293fc7..5546914da 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ yarn.lock .vscode .idea .DS_Store + +benchmark/data/* +benchmark/dist/* +!benchmark/dist/json.lua diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 000000000..55e58afd0 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,45 @@ +## TSTL Benchmarks + +These benchmarks are written in typescript and transpiled to lua by using tstl. + +### Currently only memory benchmarks are supported + +To add a new benchmark add a new file to `memory_benchmarks` +with a exported function with the following type: `() => void`. + +And add the function to the `memoryBenchmarkInput` inside `run.ts`. + +For example: + +```ts +export default myBenchmark() { + cont n = 123; + for (let i = 0; i < n; i++) { + // Do something memory instensive + } +} +``` + +```ts +import myBenchmark from "./memory_benchmarks/myBenchmark"; + +// ... + +const memoryBenchmarkInput: (() => void)[] = [ + // ... + myBenchmark +]; +``` + +**Goal** + +The goal of memory benchmarks is to track how much (memory) `"garbage"` is created by tstl. +For that reason garabage collection is disabled in the benchmarks. + +You can force the creation of `"garbage"` by creating a lot of anonymous functions or temporary tables (see [lua-users.org](http://lua-users.org/wiki/OptimisingGarbageCollection) for more information). + +To avoid crashes in the CI your benchmark should not use more than 500MB of memory. + +**Running locally** + +`npx typescript-to-lua -p tsconfig.53.json && cd dist && lua -- run.lua ../data/benchmark_master_53.json master` \ No newline at end of file diff --git a/benchmark/benchmark_types.ts b/benchmark/benchmark_types.ts new file mode 100644 index 000000000..3f9a1337e --- /dev/null +++ b/benchmark/benchmark_types.ts @@ -0,0 +1,20 @@ +export enum BenchmarkKind { + Memory = "memory", +} + +export interface BenchmarkResult { + kind: BenchmarkKind +} + +export interface MemoryBenchmarkResult extends BenchmarkResult { + kind: BenchmarkKind.Memory + benchmarkName: string; + preExecMemoryUsage: number, + postExecMemoryUsage: number, + memoryUsedForExec: number, + memoryAfterGC: number, +} + +export function isMemoryBenchmarkResult(result: BenchmarkResult): result is MemoryBenchmarkResult { + return result.kind == BenchmarkKind.Memory; +} \ No newline at end of file diff --git a/benchmark/dist/json.lua b/benchmark/dist/json.lua new file mode 100644 index 000000000..54d444840 --- /dev/null +++ b/benchmark/dist/json.lua @@ -0,0 +1,388 @@ +-- +-- json.lua +-- +-- Copyright (c) 2020 rxi +-- +-- Permission is hereby granted, free of charge, to any person obtaining a copy of +-- this software and associated documentation files (the "Software"), to deal in +-- the Software without restriction, including without limitation the rights to +-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +-- of the Software, and to permit persons to whom the Software is furnished to do +-- so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in all +-- copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +-- SOFTWARE. +-- + +local json = { _version = "0.1.2" } + +------------------------------------------------------------------------------- +-- Encode +------------------------------------------------------------------------------- + +local encode + +local escape_char_map = { + [ "\\" ] = "\\", + [ "\"" ] = "\"", + [ "\b" ] = "b", + [ "\f" ] = "f", + [ "\n" ] = "n", + [ "\r" ] = "r", + [ "\t" ] = "t", +} + +local escape_char_map_inv = { [ "/" ] = "/" } +for k, v in pairs(escape_char_map) do + escape_char_map_inv[v] = k +end + + +local function escape_char(c) + return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte())) +end + + +local function encode_nil(val) + return "null" +end + + +local function encode_table(val, stack) + local res = {} + stack = stack or {} + + -- Circular reference? + if stack[val] then error("circular reference") end + + stack[val] = true + + if rawget(val, 1) ~= nil or next(val) == nil then + -- Treat as array -- check keys are valid and it is not sparse + local n = 0 + for k in pairs(val) do + if type(k) ~= "number" then + error("invalid table: mixed or invalid key types") + end + n = n + 1 + end + if n ~= #val then + error("invalid table: sparse array") + end + -- Encode + for i, v in ipairs(val) do + table.insert(res, encode(v, stack)) + end + stack[val] = nil + return "[" .. table.concat(res, ",") .. "]" + + else + -- Treat as an object + for k, v in pairs(val) do + if type(k) ~= "string" then + error("invalid table: mixed or invalid key types") + end + table.insert(res, encode(k, stack) .. ":" .. encode(v, stack)) + end + stack[val] = nil + return "{" .. table.concat(res, ",") .. "}" + end +end + + +local function encode_string(val) + return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"' +end + + +local function encode_number(val) + -- Check for NaN, -inf and inf + if val ~= val or val <= -math.huge or val >= math.huge then + error("unexpected number value '" .. tostring(val) .. "'") + end + return string.format("%.14g", val) +end + + +local type_func_map = { + [ "nil" ] = encode_nil, + [ "table" ] = encode_table, + [ "string" ] = encode_string, + [ "number" ] = encode_number, + [ "boolean" ] = tostring, +} + + +encode = function(val, stack) + local t = type(val) + local f = type_func_map[t] + if f then + return f(val, stack) + end + error("unexpected type '" .. t .. "'") +end + + +function json.encode(val) + return ( encode(val) ) +end + + +------------------------------------------------------------------------------- +-- Decode +------------------------------------------------------------------------------- + +local parse + +local function create_set(...) + local res = {} + for i = 1, select("#", ...) do + res[ select(i, ...) ] = true + end + return res +end + +local space_chars = create_set(" ", "\t", "\r", "\n") +local delim_chars = create_set(" ", "\t", "\r", "\n", "]", "}", ",") +local escape_chars = create_set("\\", "/", '"', "b", "f", "n", "r", "t", "u") +local literals = create_set("true", "false", "null") + +local literal_map = { + [ "true" ] = true, + [ "false" ] = false, + [ "null" ] = nil, +} + + +local function next_char(str, idx, set, negate) + for i = idx, #str do + if set[str:sub(i, i)] ~= negate then + return i + end + end + return #str + 1 +end + + +local function decode_error(str, idx, msg) + local line_count = 1 + local col_count = 1 + for i = 1, idx - 1 do + col_count = col_count + 1 + if str:sub(i, i) == "\n" then + line_count = line_count + 1 + col_count = 1 + end + end + error( string.format("%s at line %d col %d", msg, line_count, col_count) ) +end + + +local function codepoint_to_utf8(n) + -- http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=iws-appendixa + local f = math.floor + if n <= 0x7f then + return string.char(n) + elseif n <= 0x7ff then + return string.char(f(n / 64) + 192, n % 64 + 128) + elseif n <= 0xffff then + return string.char(f(n / 4096) + 224, f(n % 4096 / 64) + 128, n % 64 + 128) + elseif n <= 0x10ffff then + return string.char(f(n / 262144) + 240, f(n % 262144 / 4096) + 128, + f(n % 4096 / 64) + 128, n % 64 + 128) + end + error( string.format("invalid unicode codepoint '%x'", n) ) +end + + +local function parse_unicode_escape(s) + local n1 = tonumber( s:sub(1, 4), 16 ) + local n2 = tonumber( s:sub(7, 10), 16 ) + -- Surrogate pair? + if n2 then + return codepoint_to_utf8((n1 - 0xd800) * 0x400 + (n2 - 0xdc00) + 0x10000) + else + return codepoint_to_utf8(n1) + end +end + + +local function parse_string(str, i) + local res = "" + local j = i + 1 + local k = j + + while j <= #str do + local x = str:byte(j) + + if x < 32 then + decode_error(str, j, "control character in string") + + elseif x == 92 then -- `\`: Escape + res = res .. str:sub(k, j - 1) + j = j + 1 + local c = str:sub(j, j) + if c == "u" then + local hex = str:match("^[dD][89aAbB]%x%x\\u%x%x%x%x", j + 1) + or str:match("^%x%x%x%x", j + 1) + or decode_error(str, j - 1, "invalid unicode escape in string") + res = res .. parse_unicode_escape(hex) + j = j + #hex + else + if not escape_chars[c] then + decode_error(str, j - 1, "invalid escape char '" .. c .. "' in string") + end + res = res .. escape_char_map_inv[c] + end + k = j + 1 + + elseif x == 34 then -- `"`: End of string + res = res .. str:sub(k, j - 1) + return res, j + 1 + end + + j = j + 1 + end + + decode_error(str, i, "expected closing quote for string") +end + + +local function parse_number(str, i) + local x = next_char(str, i, delim_chars) + local s = str:sub(i, x - 1) + local n = tonumber(s) + if not n then + decode_error(str, i, "invalid number '" .. s .. "'") + end + return n, x +end + + +local function parse_literal(str, i) + local x = next_char(str, i, delim_chars) + local word = str:sub(i, x - 1) + if not literals[word] then + decode_error(str, i, "invalid literal '" .. word .. "'") + end + return literal_map[word], x +end + + +local function parse_array(str, i) + local res = {} + local n = 1 + i = i + 1 + while 1 do + local x + i = next_char(str, i, space_chars, true) + -- Empty / end of array? + if str:sub(i, i) == "]" then + i = i + 1 + break + end + -- Read token + x, i = parse(str, i) + res[n] = x + n = n + 1 + -- Next token + i = next_char(str, i, space_chars, true) + local chr = str:sub(i, i) + i = i + 1 + if chr == "]" then break end + if chr ~= "," then decode_error(str, i, "expected ']' or ','") end + end + return res, i +end + + +local function parse_object(str, i) + local res = {} + i = i + 1 + while 1 do + local key, val + i = next_char(str, i, space_chars, true) + -- Empty / end of object? + if str:sub(i, i) == "}" then + i = i + 1 + break + end + -- Read key + if str:sub(i, i) ~= '"' then + decode_error(str, i, "expected string for key") + end + key, i = parse(str, i) + -- Read ':' delimiter + i = next_char(str, i, space_chars, true) + if str:sub(i, i) ~= ":" then + decode_error(str, i, "expected ':' after key") + end + i = next_char(str, i + 1, space_chars, true) + -- Read value + val, i = parse(str, i) + -- Set + res[key] = val + -- Next token + i = next_char(str, i, space_chars, true) + local chr = str:sub(i, i) + i = i + 1 + if chr == "}" then break end + if chr ~= "," then decode_error(str, i, "expected '}' or ','") end + end + return res, i +end + + +local char_func_map = { + [ '"' ] = parse_string, + [ "0" ] = parse_number, + [ "1" ] = parse_number, + [ "2" ] = parse_number, + [ "3" ] = parse_number, + [ "4" ] = parse_number, + [ "5" ] = parse_number, + [ "6" ] = parse_number, + [ "7" ] = parse_number, + [ "8" ] = parse_number, + [ "9" ] = parse_number, + [ "-" ] = parse_number, + [ "t" ] = parse_literal, + [ "f" ] = parse_literal, + [ "n" ] = parse_literal, + [ "[" ] = parse_array, + [ "{" ] = parse_object, +} + + +parse = function(str, idx) + local chr = str:sub(idx, idx) + local f = char_func_map[chr] + if f then + return f(str, idx) + end + decode_error(str, idx, "unexpected character '" .. chr .. "'") +end + + +function json.decode(str) + if type(str) ~= "string" then + error("expected argument of type string, got " .. type(str)) + end + local res, idx = parse(str, next_char(str, 1, space_chars, true)) + idx = next_char(str, idx, space_chars, true) + if idx <= #str then + decode_error(str, idx, "trailing garbage") + end + return res +end + + +return json \ No newline at end of file diff --git a/benchmark/memory_benchmark.ts b/benchmark/memory_benchmark.ts new file mode 100644 index 000000000..c7b98c8f9 --- /dev/null +++ b/benchmark/memory_benchmark.ts @@ -0,0 +1,47 @@ +import { BenchmarkKind, MemoryBenchmarkResult } from "./benchmark_types"; +import { round, json } from "./util"; + + +export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmarkResult { + let result: MemoryBenchmarkResult = { kind: BenchmarkKind.Memory, benchmarkName: "NO_NAME", preExecMemoryUsage: 0, postExecMemoryUsage: 0, memoryUsedForExec: 0, memoryAfterGC: 0 }; + + collectgarbage('stop') + result.preExecMemoryUsage = collectgarbage("count"); + + benchmarkFunction(); + + result.postExecMemoryUsage = collectgarbage("count"); + result.memoryUsedForExec = result.postExecMemoryUsage - result.preExecMemoryUsage; + + collectgarbage("restart") + collectgarbage("collect") + + result.memoryAfterGC = collectgarbage("count"); + + result.benchmarkName = debug.getinfo(benchmarkFunction).short_src; + + return result; +} + +export function compareMemoryBenchmarks(oldResults: MemoryBenchmarkResult[], updatedResults: MemoryBenchmarkResult[]): [string, string] { + let comparisonTable = "| name | master (kb) | commit (kb) | change (kb) | change (%) |\n| - | - | - | - | - |\n"; + + // we group by the new results in case benchmarks have been added + updatedResults.forEach(newResult => { + const masterResult = oldResults.find(r => r.benchmarkName == newResult.benchmarkName); + if (masterResult) { + const percentageChange = newResult.memoryUsedForExec / masterResult.memoryUsedForExec * 100 - 100; + comparisonTable += `| ${newResult.benchmarkName} | ${round(masterResult.memoryUsedForExec, 3)} | ${round(newResult.memoryUsedForExec, 3)} | ${round(newResult.memoryUsedForExec - masterResult.memoryUsedForExec, 3)} | ${round(percentageChange, 2)} |\n`; + } else { + // No master found => new benchmark + comparisonTable += `| ${newResult.benchmarkName}(new) | / | ${round(newResult.memoryUsedForExec, 3)} | / | / |\n`; + } + }); + + const markdownSummary = `**Memory:**\n${comparisonTable}`; + + const markdownText = + `**master:**\n${json.encode(oldResults)}\n**commit:**\n${json.encode(updatedResults)}`; + + return [markdownSummary, markdownText] +} \ No newline at end of file diff --git a/benchmark/memory_benchmarks/graph_cylce.ts b/benchmark/memory_benchmarks/graph_cylce.ts new file mode 100644 index 000000000..79d1d63f6 --- /dev/null +++ b/benchmark/memory_benchmarks/graph_cylce.ts @@ -0,0 +1,52 @@ +type Graph = Map; + +function range(start: number, end: number): number[] { + if (start > end) return []; + return [start, ...range(start + 1, end)]; +} + +export default function detectCyleBenchmark() { + const n = 500; + const benchmarkGraph = new Map(); + // build a graph with n nodes and no cycle + for (let i = 0; i < n; i++) { + benchmarkGraph.set(i, range(i, n - 1)); + } + detectCycle(benchmarkGraph); +} + +/** + * Detects cycles in an undirected graph + */ +function detectCycle(graph: Graph): boolean { + const visited: Map = new Map(); + + return [...graph.keys()].some((current) => { + if (!visited.get(current)) { + return _detectCycle(graph, current, visited, undefined); + } + }); +} + +function _detectCycle(graph: Graph, current: T, visited: Map, parent: T | undefined): boolean { + visited.set(current, true); + + const neighbours = graph.get(current); + + if (!neighbours) { + throw "Err invalid graph format"; + } + + return neighbours.some((neighbour) => { + if (!visited.get(neighbour)) { + // If an adjacent is not visited, then recur for that adjacent + return _detectCycle(graph, neighbour, visited, current); + } else if (neighbour !== parent) { + /* + * If an adjacent node is visited and not a parent of current vertex, + * then there is a cycle. + */ + return true; + } + }); +} \ No newline at end of file diff --git a/benchmark/run.ts b/benchmark/run.ts new file mode 100644 index 000000000..abed6178d --- /dev/null +++ b/benchmark/run.ts @@ -0,0 +1,74 @@ +import { runMemoryBenchmark, compareMemoryBenchmarks } from "./memory_benchmark"; +import { isMemoryBenchmarkResult, BenchmarkResult } from "./benchmark_types"; +import detectCyleBenchmark from "./memory_benchmarks/graph_cylce"; +import { json } from "./util"; + +// CLI arguments +// arg[0]: path to baseline benchmark data (required because this is also the output path) +// arg[1]: branchname (optional) +declare var arg: any[]; + +function benchmark() { + // Benchnmarks need to run first since we always want to output a new baseline + // even if there was no previous one + + // Memory tests + const memoryBenchmarkInput: (() => void)[] = [ + detectCyleBenchmark + ]; + + const memoryUpdatedResults = memoryBenchmarkInput.map(runMemoryBenchmark); + + // run future benchmarks types here + + const updatedResults = [...memoryUpdatedResults]; + + // Try to read the last benchmark result + const masterContent = loadMasterBenchmarkData(); + if (masterContent) { + const masterResults = json.decode(masterContent) as BenchmarkResult[]; + + const masterResultsMemory = masterResults.filter(isMemoryBenchmarkResult); + + const memoryComparisonInfo = compareMemoryBenchmarks(masterResultsMemory, memoryUpdatedResults); + + const jsonInfo = json.encode({ summary: memoryComparisonInfo[0], text: memoryComparisonInfo[1] }); + + // Output benchmark information to stdout + print(jsonInfo); + } else { + // No master yet, just write the current results to disk and output empty info + print(json.encode({ summary: "new benchmark (no results yet)", text: "" })) + } + + // Only update baseline if we are on master branch + if (arg[1] && string.find(arg[1], "master")[0]) { + const updatedMasterFile = io.open(arg[0], "w+")[0] as LuaFile + updatedMasterFile.write(json.encode(updatedResults)); + } +} +benchmark(); + +function loadMasterBenchmarkData(): (string | undefined) { + const masterFileOpen = io.open(arg[0], "rb"); + + if (masterFileOpen && masterFileOpen[0]) { + const masterFile = masterFileOpen[0]; + let masterContent: (string | undefined)[]; + if (_VERSION == "Lua 5.3") { + // @ts-ignore + masterContent = masterFile.read("a"); + } + else { + // JIT + // @ts-ignore + masterContent = masterFile.read("*a"); + } + masterFile.close(); + + if (masterContent[0]) { + return masterContent[0]; + } + } +} + diff --git a/benchmark/tsconfig.53.json b/benchmark/tsconfig.53.json new file mode 100644 index 000000000..a7851f33f --- /dev/null +++ b/benchmark/tsconfig.53.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": [ + "lua-types/5.3" + ] + }, + "tstl": { + "luaTarget": "5.3" + } +} \ No newline at end of file diff --git a/benchmark/tsconfig.jit.json b/benchmark/tsconfig.jit.json new file mode 100644 index 000000000..e4d393579 --- /dev/null +++ b/benchmark/tsconfig.jit.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": [ + "lua-types/jit" + ] + }, + "tstl": { + "luaTarget": "JIT" + } +} \ No newline at end of file diff --git a/benchmark/tsconfig.json b/benchmark/tsconfig.json new file mode 100644 index 000000000..55dd6b4ee --- /dev/null +++ b/benchmark/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": [ + "esnext" + ], + // Dev types are JIT + "types": [ + "lua-types/jit" + ], + "moduleResolution": "node", + "strict": true, + "outDir": "dist" + } +} \ No newline at end of file diff --git a/benchmark/util.ts b/benchmark/util.ts new file mode 100644 index 000000000..7fb956ddf --- /dev/null +++ b/benchmark/util.ts @@ -0,0 +1,5 @@ +export function round(num: number, decimalPlaces: number = 0) { + return tonumber(string.format(`%.${decimalPlaces}f`, num)) +} + +export const json: { decode: (this: void, str: string) => {}, encode: (this: void, val: any) => string } = require("json"); diff --git a/package-lock.json b/package-lock.json index ad25546ec..d29e3820f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3027,6 +3027,12 @@ "@sinonjs/commons": "^1.7.0" } }, + "lua-types": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/lua-types/-/lua-types-2.8.0.tgz", + "integrity": "sha512-FJY32giHIqD/XW1XGkJnl8XotXIJsJ2M42fj9A2UudttWA6orJioToW1OpgPdayTr+S1/oTO7i+hfBY3UVG8Fg==", + "dev": true + }, "make-dir": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.2.tgz", diff --git a/package.json b/package.json index 7a3656c5d..0284b0902 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "javascript-stringify": "^2.0.1", "jest": "^25.1.0", "jest-circus": "^25.1.0", + "lua-types": "^2.8.0", "prettier": "^1.19.1", "ts-jest": "^25.2.1", "ts-node": "^8.6.2", From 02e2452a69f17e2e414bf5bb9ebe491afa75d6f1 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Tue, 5 May 2020 18:42:26 +0200 Subject: [PATCH 02/21] Fixed linting issues --- benchmark/README.md | 6 +-- benchmark/benchmark_types.ts | 14 +++---- benchmark/memory_benchmark.ts | 43 +++++++++++++++------- benchmark/memory_benchmarks/graph_cylce.ts | 6 +-- benchmark/run.ts | 14 +++---- benchmark/tsconfig.53.json | 6 +-- benchmark/tsconfig.jit.json | 6 +-- benchmark/tsconfig.json | 10 ++--- benchmark/util.ts | 7 +++- 9 files changed, 60 insertions(+), 52 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 55e58afd0..458c44278 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -26,8 +26,8 @@ import myBenchmark from "./memory_benchmarks/myBenchmark"; // ... const memoryBenchmarkInput: (() => void)[] = [ - // ... - myBenchmark + // ... + myBenchmark, ]; ``` @@ -42,4 +42,4 @@ To avoid crashes in the CI your benchmark should not use more than 500MB of memo **Running locally** -`npx typescript-to-lua -p tsconfig.53.json && cd dist && lua -- run.lua ../data/benchmark_master_53.json master` \ No newline at end of file +`npx typescript-to-lua -p tsconfig.53.json && cd dist && lua -- run.lua ../data/benchmark_master_53.json master` diff --git a/benchmark/benchmark_types.ts b/benchmark/benchmark_types.ts index 3f9a1337e..bab33d739 100644 --- a/benchmark/benchmark_types.ts +++ b/benchmark/benchmark_types.ts @@ -3,18 +3,18 @@ export enum BenchmarkKind { } export interface BenchmarkResult { - kind: BenchmarkKind + kind: BenchmarkKind; } export interface MemoryBenchmarkResult extends BenchmarkResult { - kind: BenchmarkKind.Memory + kind: BenchmarkKind.Memory; benchmarkName: string; - preExecMemoryUsage: number, - postExecMemoryUsage: number, - memoryUsedForExec: number, - memoryAfterGC: number, + preExecMemoryUsage: number; + postExecMemoryUsage: number; + memoryUsedForExec: number; + memoryAfterGC: number; } export function isMemoryBenchmarkResult(result: BenchmarkResult): result is MemoryBenchmarkResult { return result.kind == BenchmarkKind.Memory; -} \ No newline at end of file +} diff --git a/benchmark/memory_benchmark.ts b/benchmark/memory_benchmark.ts index c7b98c8f9..1a2c021d2 100644 --- a/benchmark/memory_benchmark.ts +++ b/benchmark/memory_benchmark.ts @@ -1,11 +1,17 @@ import { BenchmarkKind, MemoryBenchmarkResult } from "./benchmark_types"; import { round, json } from "./util"; - export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmarkResult { - let result: MemoryBenchmarkResult = { kind: BenchmarkKind.Memory, benchmarkName: "NO_NAME", preExecMemoryUsage: 0, postExecMemoryUsage: 0, memoryUsedForExec: 0, memoryAfterGC: 0 }; + let result: MemoryBenchmarkResult = { + kind: BenchmarkKind.Memory, + benchmarkName: "NO_NAME", + preExecMemoryUsage: 0, + postExecMemoryUsage: 0, + memoryUsedForExec: 0, + memoryAfterGC: 0, + }; - collectgarbage('stop') + collectgarbage("stop"); result.preExecMemoryUsage = collectgarbage("count"); benchmarkFunction(); @@ -13,8 +19,8 @@ export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmark result.postExecMemoryUsage = collectgarbage("count"); result.memoryUsedForExec = result.postExecMemoryUsage - result.preExecMemoryUsage; - collectgarbage("restart") - collectgarbage("collect") + collectgarbage("restart"); + collectgarbage("collect"); result.memoryAfterGC = collectgarbage("count"); @@ -23,25 +29,36 @@ export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmark return result; } -export function compareMemoryBenchmarks(oldResults: MemoryBenchmarkResult[], updatedResults: MemoryBenchmarkResult[]): [string, string] { +export function compareMemoryBenchmarks( + oldResults: MemoryBenchmarkResult[], + updatedResults: MemoryBenchmarkResult[] +): [string, string] { let comparisonTable = "| name | master (kb) | commit (kb) | change (kb) | change (%) |\n| - | - | - | - | - |\n"; // we group by the new results in case benchmarks have been added updatedResults.forEach(newResult => { const masterResult = oldResults.find(r => r.benchmarkName == newResult.benchmarkName); if (masterResult) { - const percentageChange = newResult.memoryUsedForExec / masterResult.memoryUsedForExec * 100 - 100; - comparisonTable += `| ${newResult.benchmarkName} | ${round(masterResult.memoryUsedForExec, 3)} | ${round(newResult.memoryUsedForExec, 3)} | ${round(newResult.memoryUsedForExec - masterResult.memoryUsedForExec, 3)} | ${round(percentageChange, 2)} |\n`; + const percentageChange = (newResult.memoryUsedForExec / masterResult.memoryUsedForExec) * 100 - 100; + comparisonTable += `| ${newResult.benchmarkName} | ${round(masterResult.memoryUsedForExec, 3)} | ${round( + newResult.memoryUsedForExec, + 3 + )} | ${round(newResult.memoryUsedForExec - masterResult.memoryUsedForExec, 3)} | ${round( + percentageChange, + 2 + )} |\n`; } else { // No master found => new benchmark - comparisonTable += `| ${newResult.benchmarkName}(new) | / | ${round(newResult.memoryUsedForExec, 3)} | / | / |\n`; + comparisonTable += `| ${newResult.benchmarkName}(new) | / | ${round( + newResult.memoryUsedForExec, + 3 + )} | / | / |\n`; } }); const markdownSummary = `**Memory:**\n${comparisonTable}`; - const markdownText = - `**master:**\n${json.encode(oldResults)}\n**commit:**\n${json.encode(updatedResults)}`; + const markdownText = `**master:**\n${json.encode(oldResults)}\n**commit:**\n${json.encode(updatedResults)}`; - return [markdownSummary, markdownText] -} \ No newline at end of file + return [markdownSummary, markdownText]; +} diff --git a/benchmark/memory_benchmarks/graph_cylce.ts b/benchmark/memory_benchmarks/graph_cylce.ts index 79d1d63f6..ae80b4690 100644 --- a/benchmark/memory_benchmarks/graph_cylce.ts +++ b/benchmark/memory_benchmarks/graph_cylce.ts @@ -21,7 +21,7 @@ export default function detectCyleBenchmark() { function detectCycle(graph: Graph): boolean { const visited: Map = new Map(); - return [...graph.keys()].some((current) => { + return [...graph.keys()].some(current => { if (!visited.get(current)) { return _detectCycle(graph, current, visited, undefined); } @@ -37,7 +37,7 @@ function _detectCycle(graph: Graph, current: T, visited: Map, throw "Err invalid graph format"; } - return neighbours.some((neighbour) => { + return neighbours.some(neighbour => { if (!visited.get(neighbour)) { // If an adjacent is not visited, then recur for that adjacent return _detectCycle(graph, neighbour, visited, current); @@ -49,4 +49,4 @@ function _detectCycle(graph: Graph, current: T, visited: Map, return true; } }); -} \ No newline at end of file +} diff --git a/benchmark/run.ts b/benchmark/run.ts index abed6178d..68b3844f1 100644 --- a/benchmark/run.ts +++ b/benchmark/run.ts @@ -13,9 +13,7 @@ function benchmark() { // even if there was no previous one // Memory tests - const memoryBenchmarkInput: (() => void)[] = [ - detectCyleBenchmark - ]; + const memoryBenchmarkInput: (() => void)[] = [detectCyleBenchmark]; const memoryUpdatedResults = memoryBenchmarkInput.map(runMemoryBenchmark); @@ -38,18 +36,18 @@ function benchmark() { print(jsonInfo); } else { // No master yet, just write the current results to disk and output empty info - print(json.encode({ summary: "new benchmark (no results yet)", text: "" })) + print(json.encode({ summary: "new benchmark (no results yet)", text: "" })); } // Only update baseline if we are on master branch if (arg[1] && string.find(arg[1], "master")[0]) { - const updatedMasterFile = io.open(arg[0], "w+")[0] as LuaFile + const updatedMasterFile = io.open(arg[0], "w+")[0] as LuaFile; updatedMasterFile.write(json.encode(updatedResults)); } } benchmark(); -function loadMasterBenchmarkData(): (string | undefined) { +function loadMasterBenchmarkData(): string | undefined { const masterFileOpen = io.open(arg[0], "rb"); if (masterFileOpen && masterFileOpen[0]) { @@ -58,8 +56,7 @@ function loadMasterBenchmarkData(): (string | undefined) { if (_VERSION == "Lua 5.3") { // @ts-ignore masterContent = masterFile.read("a"); - } - else { + } else { // JIT // @ts-ignore masterContent = masterFile.read("*a"); @@ -71,4 +68,3 @@ function loadMasterBenchmarkData(): (string | undefined) { } } } - diff --git a/benchmark/tsconfig.53.json b/benchmark/tsconfig.53.json index a7851f33f..0a7096234 100644 --- a/benchmark/tsconfig.53.json +++ b/benchmark/tsconfig.53.json @@ -1,11 +1,9 @@ { "extends": "./tsconfig.json", "compilerOptions": { - "types": [ - "lua-types/5.3" - ] + "types": ["lua-types/5.3"] }, "tstl": { "luaTarget": "5.3" } -} \ No newline at end of file +} diff --git a/benchmark/tsconfig.jit.json b/benchmark/tsconfig.jit.json index e4d393579..1fe18fa2c 100644 --- a/benchmark/tsconfig.jit.json +++ b/benchmark/tsconfig.jit.json @@ -1,11 +1,9 @@ { "extends": "./tsconfig.json", "compilerOptions": { - "types": [ - "lua-types/jit" - ] + "types": ["lua-types/jit"] }, "tstl": { "luaTarget": "JIT" } -} \ No newline at end of file +} diff --git a/benchmark/tsconfig.json b/benchmark/tsconfig.json index 55dd6b4ee..76fe14876 100644 --- a/benchmark/tsconfig.json +++ b/benchmark/tsconfig.json @@ -1,15 +1,11 @@ { "compilerOptions": { "target": "esnext", - "lib": [ - "esnext" - ], + "lib": ["esnext"], // Dev types are JIT - "types": [ - "lua-types/jit" - ], + "types": ["lua-types/jit"], "moduleResolution": "node", "strict": true, "outDir": "dist" } -} \ No newline at end of file +} diff --git a/benchmark/util.ts b/benchmark/util.ts index 7fb956ddf..a79aae884 100644 --- a/benchmark/util.ts +++ b/benchmark/util.ts @@ -1,5 +1,8 @@ export function round(num: number, decimalPlaces: number = 0) { - return tonumber(string.format(`%.${decimalPlaces}f`, num)) + return tonumber(string.format(`%.${decimalPlaces}f`, num)); } -export const json: { decode: (this: void, str: string) => {}, encode: (this: void, val: any) => string } = require("json"); +export const json: { + decode: (this: void, str: string) => {}; + encode: (this: void, val: any) => string; +} = require("json"); From d333822dd2d5cc90026b57839e52dd628b812646 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Tue, 5 May 2020 18:50:06 +0200 Subject: [PATCH 03/21] Fixed benchmark working directory --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 736ad4bcf..cc350240d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,13 +64,15 @@ jobs: - name: Ensure benchmark data dir exists run: mkdir -p ./benchmark/data - name: Build benchmark Lua 5.3 - run: node dist/tstl.js -p benchmark/tsconfig.53.json + run: node ../dist/tstl.js -p tsconfig.53.json + working-directory: benchmark - name: Run benchmark Lua 5.3 id: benchmark-lua run: echo ::set-output name=info::`lua5.3 -- run.lua ../data/benchmark_master_53.json ${{github.ref}}` working-directory: benchmark/dist - name: Build benchmark LuaJIT - run: node dist/tstl.js -p benchmark/tsconfig.jit.json + run: node ../dist/tstl.js -p tsconfig.jit.json + working-directory: benchmark - name: Run benchmark LuaJIT id: benchmark-jit run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_master_jit.json ${{github.ref}}` From edc37d8e37e0da3be9409eee4ba5737fb9748049 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Wed, 6 May 2020 22:11:55 +0200 Subject: [PATCH 04/21] Load benchmarks from directory --- benchmark/benchmark_types.ts | 2 ++ benchmark/run.ts | 22 ++++--------- benchmark/util.ts | 63 ++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 16 deletions(-) diff --git a/benchmark/benchmark_types.ts b/benchmark/benchmark_types.ts index bab33d739..0373ea6eb 100644 --- a/benchmark/benchmark_types.ts +++ b/benchmark/benchmark_types.ts @@ -6,6 +6,8 @@ export interface BenchmarkResult { kind: BenchmarkKind; } +export type BenchmarkFunction = () => void; + export interface MemoryBenchmarkResult extends BenchmarkResult { kind: BenchmarkKind.Memory; benchmarkName: string; diff --git a/benchmark/run.ts b/benchmark/run.ts index 68b3844f1..e62ea6cbb 100644 --- a/benchmark/run.ts +++ b/benchmark/run.ts @@ -1,7 +1,6 @@ import { runMemoryBenchmark, compareMemoryBenchmarks } from "./memory_benchmark"; -import { isMemoryBenchmarkResult, BenchmarkResult } from "./benchmark_types"; -import detectCyleBenchmark from "./memory_benchmarks/graph_cylce"; -import { json } from "./util"; +import { isMemoryBenchmarkResult, BenchmarkResult, BenchmarkFunction } from "./benchmark_types"; +import { json, readAll, readDir, loadBenchmarksFromDirectory } from "./util"; // CLI arguments // arg[0]: path to baseline benchmark data (required because this is also the output path) @@ -13,8 +12,7 @@ function benchmark() { // even if there was no previous one // Memory tests - const memoryBenchmarkInput: (() => void)[] = [detectCyleBenchmark]; - + const memoryBenchmarkInput = loadBenchmarksFromDirectory("memory_benchmarks"); const memoryUpdatedResults = memoryBenchmarkInput.map(runMemoryBenchmark); // run future benchmarks types here @@ -52,19 +50,11 @@ function loadMasterBenchmarkData(): string | undefined { if (masterFileOpen && masterFileOpen[0]) { const masterFile = masterFileOpen[0]; - let masterContent: (string | undefined)[]; - if (_VERSION == "Lua 5.3") { - // @ts-ignore - masterContent = masterFile.read("a"); - } else { - // JIT - // @ts-ignore - masterContent = masterFile.read("*a"); - } + let masterContent = readAll(masterFile); masterFile.close(); - if (masterContent[0]) { - return masterContent[0]; + if (masterContent) { + return masterContent; } } } diff --git a/benchmark/util.ts b/benchmark/util.ts index a79aae884..d39276d05 100644 --- a/benchmark/util.ts +++ b/benchmark/util.ts @@ -1,3 +1,5 @@ +import { BenchmarkFunction } from "./benchmark_types"; + export function round(num: number, decimalPlaces: number = 0) { return tonumber(string.format(`%.${decimalPlaces}f`, num)); } @@ -6,3 +8,64 @@ export const json: { decode: (this: void, str: string) => {}; encode: (this: void, val: any) => string; } = require("json"); + +export function readAll(file: LuaFile): string | undefined { + let content: (string | undefined)[]; + if (_VERSION == "Lua 5.3") { + // @ts-ignore + content = file.read("a"); + } else { + // JIT + // @ts-ignore + content = file.read("*a"); + } + + if (content && content[0]) { + return content[0]; + } +} + +export function readDir(dir = ""): string[] | undefined { + let isWindows = false; + let [success, findHandle] = pcall(() => io.popen(`find ${dir} -type f -d 1`)); + + if (!success) { + [success, findHandle] = pcall(() => io.popen(`dir /A-D /B ${dir}`)); + isWindows = true; + } + + if (success) { + // appereantly TS can't infer this on it's own + findHandle = findHandle as LuaFile; + + const findResult = readAll(findHandle); + findHandle.close(); + + if (findResult) { + let files = findResult.split("\n"); + if (isWindows) { + // on windows we need to append the directory path + // on unix this is done by find automatically + files = files.map(f => `${dir}/${f}`); + } + return findResult.split("\n").filter(p => p !== ""); + } + } +} + +export function loadBenchmarksFromDirectory(dir = ""): BenchmarkFunction[] { + // Memory tests + const benchmarkPaths = readDir(dir); + + if (!benchmarkPaths) { + return []; + } + + return benchmarkPaths.map(f => { + // replace slashes with dots + let dotPath = string.gsub(f, "%/", ".")[0]; + // remove extension + dotPath = string.gsub(dotPath, ".lua", "")[0]; + return require(dotPath).default as BenchmarkFunction; + }); +} From d09436e746549d719c5b8911a7d8a26b3b4243e5 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Thu, 7 May 2020 12:23:38 +0200 Subject: [PATCH 05/21] Run benchmarks on master and commit Run benchmarks on master and commit Run benchmarks on master and commit --- .github/workflows/ci.yml | 76 +++++++++++++++++++++++++++------------- benchmark/run.ts | 7 ++-- 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc350240d..8bb53a15d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,43 +45,69 @@ jobs: steps: - name: Lua Install run: sudo apt-get install lua5.3 luajit - - uses: actions/checkout@v2 + # Checkout master & commit + - name: Checkout master + uses: actions/checkout@v2 + with: + ref: master + path: master + - name: Checkout commit + uses: actions/checkout@v2 + with: + path: commit - name: Use Node.js 12.13.1 uses: actions/setup-node@v1 with: node-version: 12.13.1 - - run: npm ci - - run: npm run build - # This will never result in a direct cache hit. - # Teherefore benchmark data is always updated. - - name: Cache benchmark data - id: cache-benchmark - uses: actions/cache@v1 - with: - path: ./benchmark/data - key: ${{ runner.os }}-master-benchmark-${{ github.sha }} - restore-keys: ${{ runner.os }}-master-benchmark- + # NPM + - name: NPM master + # TODO Lua types is only added manually to test the benchmark PR this can be removed again once the PR is merged + run: npm ci && npm run build && npm install -D lua-types + working-directory: master + - name: NPM commit + run: npm ci && npm run build + working-directory: commit + # Benchmark estup - name: Ensure benchmark data dir exists run: mkdir -p ./benchmark/data - - name: Build benchmark Lua 5.3 + working-directory: commit + - name: Copy commit benchmark to master + run: rm -rf ./master/benchmark && cp -rf ./commit/benchmark ./master/benchmark + # Run master benchmark first and output to commit benchmark data + - name: Build benchmark Lua 5.3 master + run: node ../dist/tstl.js -p tsconfig.53.json + working-directory: master/benchmark + - name: Run benchmark Lua 5.3 master + id: benchmark-lua-master + run: lua5.3 -- run.lua ../../../commit/benchmark/data/benchmark_53.json + working-directory: master/benchmark/dist + - name: Build benchmark LuaJIT master + run: node ../dist/tstl.js -p tsconfig.jit.json + working-directory: master/benchmark + - name: Run benchmark LuaJIT + id: benchmark-jit-master + run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_jit.json + working-directory: master/benchmark/dist + # Run commit benchmark and compare with master + - name: Build benchmark Lua 5.3 commit run: node ../dist/tstl.js -p tsconfig.53.json - working-directory: benchmark - - name: Run benchmark Lua 5.3 - id: benchmark-lua - run: echo ::set-output name=info::`lua5.3 -- run.lua ../data/benchmark_master_53.json ${{github.ref}}` - working-directory: benchmark/dist - - name: Build benchmark LuaJIT + working-directory: commit/benchmark + - name: Run benchmark Lua 5.3 commit + id: benchmark-lua-commit + run: echo ::set-output name=info::`lua5.3 -- run.lua ../data/benchmark_53.json` + working-directory: commit/benchmark/dist + - name: Build benchmark LuaJIT commit run: node ../dist/tstl.js -p tsconfig.jit.json - working-directory: benchmark + working-directory: commit/benchmark - name: Run benchmark LuaJIT - id: benchmark-jit - run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_master_jit.json ${{github.ref}}` - working-directory: benchmark/dist + id: benchmark-jit-commit + run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_jit.json` + working-directory: commit/benchmark/dist - name: Create benchmark check uses: actions/github-script@0.9.0 with: - benchmark-info-lua: ${{steps.benchmark-lua.outputs.info}} - benchmark-info-jit: ${{steps.benchmark-jit.outputs.info}} + benchmark-info-lua: ${{steps.benchmark-lua-commit.outputs.info}} + benchmark-info-jit: ${{steps.benchmark-jit-commit.outputs.info}} script: | const benchmarkInfoLua = JSON.parse(core.getInput('benchmark-info-lua', { required: true })); const benchmarkInfoJIT = JSON.parse(core.getInput('benchmark-info-jit', { required: true })); diff --git a/benchmark/run.ts b/benchmark/run.ts index e62ea6cbb..f21b74c07 100644 --- a/benchmark/run.ts +++ b/benchmark/run.ts @@ -37,11 +37,8 @@ function benchmark() { print(json.encode({ summary: "new benchmark (no results yet)", text: "" })); } - // Only update baseline if we are on master branch - if (arg[1] && string.find(arg[1], "master")[0]) { - const updatedMasterFile = io.open(arg[0], "w+")[0] as LuaFile; - updatedMasterFile.write(json.encode(updatedResults)); - } + const updatedMasterFile = io.open(arg[0], "w+")[0] as LuaFile; + updatedMasterFile.write(json.encode(updatedResults)); } benchmark(); From 2adb2bb62cd7dd9b50f5f435df5c8f6dbaef9271 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Thu, 7 May 2020 12:30:23 +0200 Subject: [PATCH 06/21] Changed find -d parameter to be unix compliant --- .github/workflows/ci.yml | 4 ++-- benchmark/util.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bb53a15d..89503fa37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: - name: Build benchmark LuaJIT master run: node ../dist/tstl.js -p tsconfig.jit.json working-directory: master/benchmark - - name: Run benchmark LuaJIT + - name: Run benchmark LuaJIT master id: benchmark-jit-master run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_jit.json working-directory: master/benchmark/dist @@ -99,7 +99,7 @@ jobs: - name: Build benchmark LuaJIT commit run: node ../dist/tstl.js -p tsconfig.jit.json working-directory: commit/benchmark - - name: Run benchmark LuaJIT + - name: Run benchmark LuaJIT commit id: benchmark-jit-commit run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_jit.json` working-directory: commit/benchmark/dist diff --git a/benchmark/util.ts b/benchmark/util.ts index d39276d05..7bdda9bed 100644 --- a/benchmark/util.ts +++ b/benchmark/util.ts @@ -27,7 +27,7 @@ export function readAll(file: LuaFile): string | undefined { export function readDir(dir = ""): string[] | undefined { let isWindows = false; - let [success, findHandle] = pcall(() => io.popen(`find ${dir} -type f -d 1`)); + let [success, findHandle] = pcall(() => io.popen(`find '${dir}' -type f -depth 1`)); if (!success) { [success, findHandle] = pcall(() => io.popen(`dir /A-D /B ${dir}`)); From df50d62e950e79e1839d49318bb80f5b14c7d50d Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Thu, 7 May 2020 12:44:18 +0200 Subject: [PATCH 07/21] Strip leading './' in readDir find command --- benchmark/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/util.ts b/benchmark/util.ts index 7bdda9bed..5ebd7c958 100644 --- a/benchmark/util.ts +++ b/benchmark/util.ts @@ -27,7 +27,7 @@ export function readAll(file: LuaFile): string | undefined { export function readDir(dir = ""): string[] | undefined { let isWindows = false; - let [success, findHandle] = pcall(() => io.popen(`find '${dir}' -type f -depth 1`)); + let [success, findHandle] = pcall(() => io.popen(`find '${dir}' -maxdepth 1 -type f | sed "s|^\./||"`)); if (!success) { [success, findHandle] = pcall(() => io.popen(`dir /A-D /B ${dir}`)); From 7aa0db0765aef92bea997f8050a172135918f928 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Thu, 7 May 2020 13:35:44 +0200 Subject: [PATCH 08/21] Improved local development workflow and performed additional cleanup --- .github/workflows/ci.yml | 8 +-- benchmark/README.md | 26 ++++----- benchmark/run.ts | 57 ------------------- benchmark/{ => src}/benchmark_types.ts | 0 benchmark/{ => src}/memory_benchmark.ts | 4 +- .../memory_benchmarks/graph_cylce.ts | 0 benchmark/src/run.ts | 50 ++++++++++++++++ benchmark/{ => src}/util.ts | 14 +++++ benchmark/tsconfig.json | 3 +- 9 files changed, 83 insertions(+), 79 deletions(-) delete mode 100644 benchmark/run.ts rename benchmark/{ => src}/benchmark_types.ts (100%) rename benchmark/{ => src}/memory_benchmark.ts (93%) rename benchmark/{ => src}/memory_benchmarks/graph_cylce.ts (100%) create mode 100644 benchmark/src/run.ts rename benchmark/{ => src}/util.ts (85%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89503fa37..4ccaeca03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,14 +79,14 @@ jobs: working-directory: master/benchmark - name: Run benchmark Lua 5.3 master id: benchmark-lua-master - run: lua5.3 -- run.lua ../../../commit/benchmark/data/benchmark_53.json + run: lua5.3 -- run.lua ../../../commit/benchmark/data/benchmark_master_53.json working-directory: master/benchmark/dist - name: Build benchmark LuaJIT master run: node ../dist/tstl.js -p tsconfig.jit.json working-directory: master/benchmark - name: Run benchmark LuaJIT master id: benchmark-jit-master - run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_jit.json + run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_master_jit.json working-directory: master/benchmark/dist # Run commit benchmark and compare with master - name: Build benchmark Lua 5.3 commit @@ -94,14 +94,14 @@ jobs: working-directory: commit/benchmark - name: Run benchmark Lua 5.3 commit id: benchmark-lua-commit - run: echo ::set-output name=info::`lua5.3 -- run.lua ../data/benchmark_53.json` + run: echo ::set-output name=info::`lua5.3 -- run.lua ../data/benchmark_commit_53.json ../data/benchmark_master_53.json` working-directory: commit/benchmark/dist - name: Build benchmark LuaJIT commit run: node ../dist/tstl.js -p tsconfig.jit.json working-directory: commit/benchmark - name: Run benchmark LuaJIT commit id: benchmark-jit-commit - run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_jit.json` + run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_commit_jit.json ../data/benchmark_master_jit.json` working-directory: commit/benchmark/dist - name: Create benchmark check uses: actions/github-script@0.9.0 diff --git a/benchmark/README.md b/benchmark/README.md index 458c44278..421f63984 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -5,11 +5,9 @@ These benchmarks are written in typescript and transpiled to lua by using tstl. ### Currently only memory benchmarks are supported To add a new benchmark add a new file to `memory_benchmarks` -with a exported function with the following type: `() => void`. +and **default** export a function with the following type: `() => void`. -And add the function to the `memoryBenchmarkInput` inside `run.ts`. - -For example: +For example (memory_benchmarks/my_benchmark.ts): ```ts export default myBenchmark() { @@ -20,17 +18,6 @@ export default myBenchmark() { } ``` -```ts -import myBenchmark from "./memory_benchmarks/myBenchmark"; - -// ... - -const memoryBenchmarkInput: (() => void)[] = [ - // ... - myBenchmark, -]; -``` - **Goal** The goal of memory benchmarks is to track how much (memory) `"garbage"` is created by tstl. @@ -42,4 +29,11 @@ To avoid crashes in the CI your benchmark should not use more than 500MB of memo **Running locally** -`npx typescript-to-lua -p tsconfig.53.json && cd dist && lua -- run.lua ../data/benchmark_master_53.json master` +1. Create a benchmark baseline called "benchmark_baseilne.json": + `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_baseilne.json` +2. Make some changes to tstl. +3. Create an updated benchmark and compare with the baseline: + `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseilne.json` +4. The above command will output comparison data as json to stdout. + If you provide a path as third argument the comparison data will be written to that path instead. + `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseilne.json result.md` diff --git a/benchmark/run.ts b/benchmark/run.ts deleted file mode 100644 index f21b74c07..000000000 --- a/benchmark/run.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { runMemoryBenchmark, compareMemoryBenchmarks } from "./memory_benchmark"; -import { isMemoryBenchmarkResult, BenchmarkResult, BenchmarkFunction } from "./benchmark_types"; -import { json, readAll, readDir, loadBenchmarksFromDirectory } from "./util"; - -// CLI arguments -// arg[0]: path to baseline benchmark data (required because this is also the output path) -// arg[1]: branchname (optional) -declare var arg: any[]; - -function benchmark() { - // Benchnmarks need to run first since we always want to output a new baseline - // even if there was no previous one - - // Memory tests - const memoryBenchmarkInput = loadBenchmarksFromDirectory("memory_benchmarks"); - const memoryUpdatedResults = memoryBenchmarkInput.map(runMemoryBenchmark); - - // run future benchmarks types here - - const updatedResults = [...memoryUpdatedResults]; - - // Try to read the last benchmark result - const masterContent = loadMasterBenchmarkData(); - if (masterContent) { - const masterResults = json.decode(masterContent) as BenchmarkResult[]; - - const masterResultsMemory = masterResults.filter(isMemoryBenchmarkResult); - - const memoryComparisonInfo = compareMemoryBenchmarks(masterResultsMemory, memoryUpdatedResults); - - const jsonInfo = json.encode({ summary: memoryComparisonInfo[0], text: memoryComparisonInfo[1] }); - - // Output benchmark information to stdout - print(jsonInfo); - } else { - // No master yet, just write the current results to disk and output empty info - print(json.encode({ summary: "new benchmark (no results yet)", text: "" })); - } - - const updatedMasterFile = io.open(arg[0], "w+")[0] as LuaFile; - updatedMasterFile.write(json.encode(updatedResults)); -} -benchmark(); - -function loadMasterBenchmarkData(): string | undefined { - const masterFileOpen = io.open(arg[0], "rb"); - - if (masterFileOpen && masterFileOpen[0]) { - const masterFile = masterFileOpen[0]; - let masterContent = readAll(masterFile); - masterFile.close(); - - if (masterContent) { - return masterContent; - } - } -} diff --git a/benchmark/benchmark_types.ts b/benchmark/src/benchmark_types.ts similarity index 100% rename from benchmark/benchmark_types.ts rename to benchmark/src/benchmark_types.ts diff --git a/benchmark/memory_benchmark.ts b/benchmark/src/memory_benchmark.ts similarity index 93% rename from benchmark/memory_benchmark.ts rename to benchmark/src/memory_benchmark.ts index 1a2c021d2..15e86da0c 100644 --- a/benchmark/memory_benchmark.ts +++ b/benchmark/src/memory_benchmark.ts @@ -58,7 +58,9 @@ export function compareMemoryBenchmarks( const markdownSummary = `**Memory:**\n${comparisonTable}`; - const markdownText = `**master:**\n${json.encode(oldResults)}\n**commit:**\n${json.encode(updatedResults)}`; + const markdownText = `**master:**\n\`\`\`json\n${json.encode( + oldResults + )}\n\`\`\`\n**commit:**\n\`\`\`json\n${json.encode(updatedResults)}\n\`\`\``; return [markdownSummary, markdownText]; } diff --git a/benchmark/memory_benchmarks/graph_cylce.ts b/benchmark/src/memory_benchmarks/graph_cylce.ts similarity index 100% rename from benchmark/memory_benchmarks/graph_cylce.ts rename to benchmark/src/memory_benchmarks/graph_cylce.ts diff --git a/benchmark/src/run.ts b/benchmark/src/run.ts new file mode 100644 index 000000000..d998149a6 --- /dev/null +++ b/benchmark/src/run.ts @@ -0,0 +1,50 @@ +import { runMemoryBenchmark, compareMemoryBenchmarks } from "./memory_benchmark"; +import { isMemoryBenchmarkResult, BenchmarkResult } from "./benchmark_types"; +import { json, readAll, readDir, loadBenchmarksFromDirectory, readFile } from "./util"; + +// CLI arguments +// arg[0]: output path for benchmark data +// arg[1]: path to baseline benchmark data (required to generate comparison) +// arg[2]: path to result markdown file (optional) +declare const arg: [string | undefined, string | undefined, string | undefined]; + +function benchmark() { + // Memory tests + const memoryBenchmarkInput = loadBenchmarksFromDirectory("memory_benchmarks"); + const memoryUpdatedResults = memoryBenchmarkInput.map(runMemoryBenchmark); + + // run future benchmarks types here + + const updatedResults = [...memoryUpdatedResults]; + + let comparisonInfo = { summary: "nothing to compare", text: "" }; + + // Try to read the last benchmark result + const inputContent = arg[1] && readFile(arg[1]); + if (inputContent) { + const currentResults = json.decode(inputContent) as BenchmarkResult[]; + + const currentResultsMemory = currentResults.filter(isMemoryBenchmarkResult); + + const memoryComparisonInfo = compareMemoryBenchmarks(currentResultsMemory, memoryUpdatedResults); + + comparisonInfo = { summary: memoryComparisonInfo[0], text: memoryComparisonInfo[1] }; + } + + // Output comparison info + if (!arg[2]) { + // Output to stdout as json by default, this is used by the CI to retrieve the info + print(json.encode(comparisonInfo)); + } else { + // Output to file as markdown if arg[2] is set, this is useful for local development + const updatedResultsFile = io.open(arg[2], "w+")[0] as LuaFile; + updatedResultsFile.write(comparisonInfo.summary + comparisonInfo.text); + } + + // Output benchmark results to json + if (arg[0]) { + const updatedResultsFile = io.open(arg[0], "w+")[0] as LuaFile; + updatedResultsFile.write(json.encode(updatedResults)); + } +} +benchmark(); diff --git a/benchmark/util.ts b/benchmark/src/util.ts similarity index 85% rename from benchmark/util.ts rename to benchmark/src/util.ts index 5ebd7c958..96cd35962 100644 --- a/benchmark/util.ts +++ b/benchmark/src/util.ts @@ -9,6 +9,20 @@ export const json: { encode: (this: void, val: any) => string; } = require("json"); +export function readFile(path: string): string | undefined { + const fileOpenArray = io.open(path, "rb"); + + if (fileOpenArray && fileOpenArray[0]) { + const fileHandle = fileOpenArray[0]; + let fileContent = readAll(fileHandle); + fileHandle.close(); + + if (fileContent) { + return fileContent; + } + } +} + export function readAll(file: LuaFile): string | undefined { let content: (string | undefined)[]; if (_VERSION == "Lua 5.3") { diff --git a/benchmark/tsconfig.json b/benchmark/tsconfig.json index 76fe14876..df29e6853 100644 --- a/benchmark/tsconfig.json +++ b/benchmark/tsconfig.json @@ -6,6 +6,7 @@ "types": ["lua-types/jit"], "moduleResolution": "node", "strict": true, - "outDir": "dist" + "outDir": "dist", + "rootDir": "src" } } From 7dcb7c016e1970a8f83872b55255efae454d9275 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Thu, 7 May 2020 14:10:35 +0200 Subject: [PATCH 09/21] Changed unit from kb to mb Some more cleanup --- benchmark/src/memory_benchmark.ts | 21 ++++++++++----------- benchmark/src/run.ts | 30 +++++++++++++++++++----------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/benchmark/src/memory_benchmark.ts b/benchmark/src/memory_benchmark.ts index 15e86da0c..09f72b6f5 100644 --- a/benchmark/src/memory_benchmark.ts +++ b/benchmark/src/memory_benchmark.ts @@ -33,25 +33,24 @@ export function compareMemoryBenchmarks( oldResults: MemoryBenchmarkResult[], updatedResults: MemoryBenchmarkResult[] ): [string, string] { - let comparisonTable = "| name | master (kb) | commit (kb) | change (kb) | change (%) |\n| - | - | - | - | - |\n"; + let comparisonTable = "| name | master (MB) | commit (MB) | change (MB) | change (%) |\n| - | - | - | - | - |\n"; + + const formatMemory = (memInKB: number) => round(memInKB / 1024, 3); // we group by the new results in case benchmarks have been added updatedResults.forEach(newResult => { const masterResult = oldResults.find(r => r.benchmarkName == newResult.benchmarkName); if (masterResult) { const percentageChange = (newResult.memoryUsedForExec / masterResult.memoryUsedForExec) * 100 - 100; - comparisonTable += `| ${newResult.benchmarkName} | ${round(masterResult.memoryUsedForExec, 3)} | ${round( - newResult.memoryUsedForExec, - 3 - )} | ${round(newResult.memoryUsedForExec - masterResult.memoryUsedForExec, 3)} | ${round( - percentageChange, - 2 - )} |\n`; + comparisonTable += `| ${newResult.benchmarkName} | ${formatMemory( + masterResult.memoryUsedForExec + )} | ${formatMemory(newResult.memoryUsedForExec)} | ${formatMemory( + newResult.memoryUsedForExec - masterResult.memoryUsedForExec + )} | ${round(percentageChange, 2)} |\n`; } else { // No master found => new benchmark - comparisonTable += `| ${newResult.benchmarkName}(new) | / | ${round( - newResult.memoryUsedForExec, - 3 + comparisonTable += `| ${newResult.benchmarkName}(new) | / | ${formatMemory( + newResult.memoryUsedForExec )} | / | / |\n`; } }); diff --git a/benchmark/src/run.ts b/benchmark/src/run.ts index d998149a6..9df85907a 100644 --- a/benchmark/src/run.ts +++ b/benchmark/src/run.ts @@ -17,21 +17,31 @@ function benchmark() { const updatedResults = [...memoryUpdatedResults]; - let comparisonInfo = { summary: "nothing to compare", text: "" }; - - // Try to read the last benchmark result + // Try to read the baseline benchmark result + let previousResults: BenchmarkResult[] = []; const inputContent = arg[1] && readFile(arg[1]); if (inputContent) { - const currentResults = json.decode(inputContent) as BenchmarkResult[]; + previousResults = json.decode(inputContent) as BenchmarkResult[]; + } - const currentResultsMemory = currentResults.filter(isMemoryBenchmarkResult); + // Compare results + const comparisonInfo = compareResults(previousResults, updatedResults); - const memoryComparisonInfo = compareMemoryBenchmarks(currentResultsMemory, memoryUpdatedResults); + // Output comparison info + outputResults(comparisonInfo, updatedResults); +} +benchmark(); - comparisonInfo = { summary: memoryComparisonInfo[0], text: memoryComparisonInfo[1] }; - } +function compareResults(previousResults: BenchmarkResult[], updatedResults: BenchmarkResult[]) { + const previousResultsMemory = previousResults.filter(isMemoryBenchmarkResult); + const updatedResultsMemory = updatedResults.filter(isMemoryBenchmarkResult); - // Output comparison info + const memoryComparisonInfo = compareMemoryBenchmarks(previousResultsMemory, updatedResultsMemory); + + return { summary: memoryComparisonInfo[0], text: memoryComparisonInfo[1] }; +} + +function outputResults(comparisonInfo: { summary: string; text: string }, updatedResults: BenchmarkResult[]) { if (!arg[2]) { // Output to stdout as json by default, this is used by the CI to retrieve the info print(json.encode(comparisonInfo)); @@ -40,11 +50,9 @@ function benchmark() { const updatedResultsFile = io.open(arg[2], "w+")[0] as LuaFile; updatedResultsFile.write(comparisonInfo.summary + comparisonInfo.text); } - // Output benchmark results to json if (arg[0]) { const updatedResultsFile = io.open(arg[0], "w+")[0] as LuaFile; updatedResultsFile.write(json.encode(updatedResults)); } } -benchmark(); From d9ce6433f6fd8ad0f8fc8c5fd0a8a365498bbd99 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Thu, 7 May 2020 17:33:06 +0200 Subject: [PATCH 10/21] Addressed Review Fixed popen behaviour Improved error handling Fixed tslint issues --- benchmark/README.md | 2 +- benchmark/src/benchmark_types.ts | 15 +- benchmark/src/memory_benchmark.ts | 10 +- .../src/memory_benchmarks/graph_cylce.ts | 5 +- benchmark/src/run.ts | 49 ++++--- benchmark/src/util.ts | 128 +++++++++++------- benchmark/tsconfig.json | 8 +- 7 files changed, 131 insertions(+), 86 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 421f63984..c549bcc02 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -11,7 +11,7 @@ For example (memory_benchmarks/my_benchmark.ts): ```ts export default myBenchmark() { - cont n = 123; + const n = 123; for (let i = 0; i < n; i++) { // Do something memory instensive } diff --git a/benchmark/src/benchmark_types.ts b/benchmark/src/benchmark_types.ts index 0373ea6eb..efd75c3e4 100644 --- a/benchmark/src/benchmark_types.ts +++ b/benchmark/src/benchmark_types.ts @@ -2,13 +2,11 @@ export enum BenchmarkKind { Memory = "memory", } -export interface BenchmarkResult { - kind: BenchmarkKind; -} - export type BenchmarkFunction = () => void; -export interface MemoryBenchmarkResult extends BenchmarkResult { +export type BenchmarkResult = MemoryBenchmarkResult; + +export interface MemoryBenchmarkResult { kind: BenchmarkKind.Memory; benchmarkName: string; preExecMemoryUsage: number; @@ -18,5 +16,10 @@ export interface MemoryBenchmarkResult extends BenchmarkResult { } export function isMemoryBenchmarkResult(result: BenchmarkResult): result is MemoryBenchmarkResult { - return result.kind == BenchmarkKind.Memory; + return result.kind === BenchmarkKind.Memory; +} + +export interface ComparisonInfo { + summary: string; + text: string; } diff --git a/benchmark/src/memory_benchmark.ts b/benchmark/src/memory_benchmark.ts index 09f72b6f5..30333e933 100644 --- a/benchmark/src/memory_benchmark.ts +++ b/benchmark/src/memory_benchmark.ts @@ -1,8 +1,8 @@ import { BenchmarkKind, MemoryBenchmarkResult } from "./benchmark_types"; -import { round, json } from "./util"; +import { toFixed, json } from "./util"; export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmarkResult { - let result: MemoryBenchmarkResult = { + const result: MemoryBenchmarkResult = { kind: BenchmarkKind.Memory, benchmarkName: "NO_NAME", preExecMemoryUsage: 0, @@ -35,18 +35,18 @@ export function compareMemoryBenchmarks( ): [string, string] { let comparisonTable = "| name | master (MB) | commit (MB) | change (MB) | change (%) |\n| - | - | - | - | - |\n"; - const formatMemory = (memInKB: number) => round(memInKB / 1024, 3); + const formatMemory = (memInKB: number) => toFixed(memInKB / 1024, 3); // we group by the new results in case benchmarks have been added updatedResults.forEach(newResult => { - const masterResult = oldResults.find(r => r.benchmarkName == newResult.benchmarkName); + const masterResult = oldResults.find(r => r.benchmarkName === newResult.benchmarkName); if (masterResult) { const percentageChange = (newResult.memoryUsedForExec / masterResult.memoryUsedForExec) * 100 - 100; comparisonTable += `| ${newResult.benchmarkName} | ${formatMemory( masterResult.memoryUsedForExec )} | ${formatMemory(newResult.memoryUsedForExec)} | ${formatMemory( newResult.memoryUsedForExec - masterResult.memoryUsedForExec - )} | ${round(percentageChange, 2)} |\n`; + )} | ${toFixed(percentageChange, 2)} |\n`; } else { // No master found => new benchmark comparisonTable += `| ${newResult.benchmarkName}(new) | / | ${formatMemory( diff --git a/benchmark/src/memory_benchmarks/graph_cylce.ts b/benchmark/src/memory_benchmarks/graph_cylce.ts index ae80b4690..3a172a591 100644 --- a/benchmark/src/memory_benchmarks/graph_cylce.ts +++ b/benchmark/src/memory_benchmarks/graph_cylce.ts @@ -5,7 +5,8 @@ function range(start: number, end: number): number[] { return [start, ...range(start + 1, end)]; } -export default function detectCyleBenchmark() { +// tslint:disable-next-line:no-default-export +export default function detectCyleBenchmark(): void { const n = 500; const benchmarkGraph = new Map(); // build a graph with n nodes and no cycle @@ -34,7 +35,7 @@ function _detectCycle(graph: Graph, current: T, visited: Map, const neighbours = graph.get(current); if (!neighbours) { - throw "Err invalid graph format"; + throw Error("Err invalid graph format"); } return neighbours.some(neighbour => { diff --git a/benchmark/src/run.ts b/benchmark/src/run.ts index 9df85907a..c4cd276f5 100644 --- a/benchmark/src/run.ts +++ b/benchmark/src/run.ts @@ -1,6 +1,6 @@ import { runMemoryBenchmark, compareMemoryBenchmarks } from "./memory_benchmark"; -import { isMemoryBenchmarkResult, BenchmarkResult } from "./benchmark_types"; -import { json, readAll, readDir, loadBenchmarksFromDirectory, readFile } from "./util"; +import { isMemoryBenchmarkResult, BenchmarkResult, MemoryBenchmarkResult, ComparisonInfo } from "./benchmark_types"; +import { json, loadBenchmarksFromDirectory, readFile } from "./util"; // CLI arguments // arg[0]: output path for benchmark data @@ -8,31 +8,41 @@ import { json, readAll, readDir, loadBenchmarksFromDirectory, readFile } from ". // arg[2]: path to result markdown file (optional) declare const arg: [string | undefined, string | undefined, string | undefined]; -function benchmark() { +function benchmark(): void { // Memory tests - const memoryBenchmarkInput = loadBenchmarksFromDirectory("memory_benchmarks"); - const memoryUpdatedResults = memoryBenchmarkInput.map(runMemoryBenchmark); + let memoryUpdatedResults: MemoryBenchmarkResult[] = []; + + const loadPreviousMemoryBenchmarksResult = loadBenchmarksFromDirectory("memory_benchmarks"); + + if (loadPreviousMemoryBenchmarksResult.isOk()) { + memoryUpdatedResults = loadPreviousMemoryBenchmarksResult.value.map(runMemoryBenchmark); + } else { + print(loadPreviousMemoryBenchmarksResult.error); + os.exit(1); + } // run future benchmarks types here - const updatedResults = [...memoryUpdatedResults]; + const benchmarkResults = [...memoryUpdatedResults]; // Try to read the baseline benchmark result - let previousResults: BenchmarkResult[] = []; - const inputContent = arg[1] && readFile(arg[1]); - if (inputContent) { - previousResults = json.decode(inputContent) as BenchmarkResult[]; + let previousBenchmarkResults: BenchmarkResult[] = []; + if (arg[1]) { + const readPreviousFileResult = readFile(arg[1]); + if (readPreviousFileResult.isOk()) { + previousBenchmarkResults = json.decode(readPreviousFileResult.value) as BenchmarkResult[]; + } } // Compare results - const comparisonInfo = compareResults(previousResults, updatedResults); + const comparisonInfo = compareBenchmarks(previousBenchmarkResults, benchmarkResults); // Output comparison info - outputResults(comparisonInfo, updatedResults); + outputBenchmarkData(comparisonInfo, benchmarkResults); } benchmark(); -function compareResults(previousResults: BenchmarkResult[], updatedResults: BenchmarkResult[]) { +function compareBenchmarks(previousResults: BenchmarkResult[], updatedResults: BenchmarkResult[]): ComparisonInfo { const previousResultsMemory = previousResults.filter(isMemoryBenchmarkResult); const updatedResultsMemory = updatedResults.filter(isMemoryBenchmarkResult); @@ -41,18 +51,21 @@ function compareResults(previousResults: BenchmarkResult[], updatedResults: Benc return { summary: memoryComparisonInfo[0], text: memoryComparisonInfo[1] }; } -function outputResults(comparisonInfo: { summary: string; text: string }, updatedResults: BenchmarkResult[]) { +function outputBenchmarkData( + comparisonInfo: { summary: string; text: string }, + updatedResults: BenchmarkResult[] +): void { if (!arg[2]) { // Output to stdout as json by default, this is used by the CI to retrieve the info print(json.encode(comparisonInfo)); } else { // Output to file as markdown if arg[2] is set, this is useful for local development - const updatedResultsFile = io.open(arg[2], "w+")[0] as LuaFile; - updatedResultsFile.write(comparisonInfo.summary + comparisonInfo.text); + const markdownDataFile = io.open(arg[2], "w+")[0]!; + markdownDataFile.write(comparisonInfo.summary + comparisonInfo.text); } // Output benchmark results to json if (arg[0]) { - const updatedResultsFile = io.open(arg[0], "w+")[0] as LuaFile; - updatedResultsFile.write(json.encode(updatedResults)); + const jsonDataFile = io.open(arg[0], "w+")[0]!; + jsonDataFile.write(json.encode(updatedResults)); } } diff --git a/benchmark/src/util.ts b/benchmark/src/util.ts index 96cd35962..8d56fa69a 100644 --- a/benchmark/src/util.ts +++ b/benchmark/src/util.ts @@ -1,85 +1,111 @@ import { BenchmarkFunction } from "./benchmark_types"; -export function round(num: number, decimalPlaces: number = 0) { - return tonumber(string.format(`%.${decimalPlaces}f`, num)); +export type Result = Ok | Err; + +export class Ok { + constructor(readonly value: T) {} + + public isOk(): this is Ok { + return true; + } + + public isError(): this is Err { + return false; + } } +export class Err { + constructor(readonly error: E) {} + + public isOk(): this is Ok { + return false; + } + + public isError(): this is Err { + return true; + } +} + +export const ok = (value: T): Ok => new Ok(value); + +export const err = (err: E): Err => new Err(err); + +export function toFixed(num: number, decimalPlaces = 0): string { + return string.format(`%.${decimalPlaces}f`, num); +} + +// @ts-ignore +export const isWindows = package.config.sub(1, 1) === "\\"; + export const json: { decode: (this: void, str: string) => {}; encode: (this: void, val: any) => string; } = require("json"); -export function readFile(path: string): string | undefined { +export function readFile(path: string): Result { const fileOpenArray = io.open(path, "rb"); if (fileOpenArray && fileOpenArray[0]) { const fileHandle = fileOpenArray[0]; - let fileContent = readAll(fileHandle); + const readAllResult = readAll(fileHandle); fileHandle.close(); - if (fileContent) { - return fileContent; + if (readAllResult.isOk()) { + return ok(readAllResult); + } else { + return readAllResult; } } + return err(`Can't open file ${path}`); } -export function readAll(file: LuaFile): string | undefined { - let content: (string | undefined)[]; - if (_VERSION == "Lua 5.3") { - // @ts-ignore - content = file.read("a"); - } else { - // JIT - // @ts-ignore - content = file.read("*a"); - } +export function readAll(file: LuaFile): Result { + const content = file.read(_VERSION === "Lua 5.3" ? "a" : ("*a" as any)) as [string | undefined]; - if (content && content[0]) { - return content[0]; + if (content[0]) { + return ok(content[0]); } + return err(`Can't readAll for file ${file}`); } -export function readDir(dir = ""): string[] | undefined { - let isWindows = false; - let [success, findHandle] = pcall(() => io.popen(`find '${dir}' -maxdepth 1 -type f | sed "s|^\./||"`)); +export function readDir(dir: string): Result { + const findHandle = isWindows ? io.popen(`dir /A-D /B ${dir}`) : io.popen(`find '${dir}' -maxdepth 1 -type f`); + const findReadAllResult = readAll(findHandle); - if (!success) { - [success, findHandle] = pcall(() => io.popen(`dir /A-D /B ${dir}`)); - isWindows = true; + if (!findHandle.close()) { + return err(`readDir popen failed for dir ${dir} see stdout for more information.`); } - if (success) { - // appereantly TS can't infer this on it's own - findHandle = findHandle as LuaFile; - - const findResult = readAll(findHandle); - findHandle.close(); - - if (findResult) { - let files = findResult.split("\n"); - if (isWindows) { - // on windows we need to append the directory path - // on unix this is done by find automatically - files = files.map(f => `${dir}/${f}`); - } - return findResult.split("\n").filter(p => p !== ""); + if (findReadAllResult.isOk()) { + let files = findReadAllResult.value.split("\n"); + if (isWindows) { + // on windows we need to append the directory path + // on unix this is done by find automatically + files = files.map(f => `${dir}/${f}`); + } else { + // strip leading "./" on unix + files = files.map(f => (f[0] === "." && f[1] === "/" ? f.substr(2) : f)); } + return ok(files.filter(p => p !== "")); + } else { + return findReadAllResult; } } -export function loadBenchmarksFromDirectory(dir = ""): BenchmarkFunction[] { - // Memory tests - const benchmarkPaths = readDir(dir); +export function loadBenchmarksFromDirectory(benchmarkDir: string): Result { + const readBenchmarkDirResult = readDir(benchmarkDir); - if (!benchmarkPaths) { - return []; + if (readBenchmarkDirResult.isError()) { + return readBenchmarkDirResult; } - return benchmarkPaths.map(f => { - // replace slashes with dots - let dotPath = string.gsub(f, "%/", ".")[0]; - // remove extension - dotPath = string.gsub(dotPath, ".lua", "")[0]; - return require(dotPath).default as BenchmarkFunction; - }); + return ok( + readBenchmarkDirResult.value.map(f => { + // replace slashes with dots + let dotPath = string.gsub(f, "%/", ".")[0]; + // remove extension + dotPath = string.gsub(dotPath, ".lua", "")[0]; + return require(dotPath).default as BenchmarkFunction; + }) + ); } diff --git a/benchmark/tsconfig.json b/benchmark/tsconfig.json index df29e6853..673557039 100644 --- a/benchmark/tsconfig.json +++ b/benchmark/tsconfig.json @@ -3,10 +3,12 @@ "target": "esnext", "lib": ["esnext"], // Dev types are JIT - "types": ["lua-types/jit"], + "types": ["lua-types/5.3"], "moduleResolution": "node", - "strict": true, "outDir": "dist", - "rootDir": "src" + "rootDir": "src", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true } } From 02c1dadf640ebb6c2ad1a15b9ab4fa1c39863651 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Thu, 7 May 2020 17:45:39 +0200 Subject: [PATCH 11/21] Fixed readFile returning incorrect result Reverted lua-types dev version back to jit --- benchmark/src/util.ts | 10 +++------- benchmark/tsconfig.json | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/benchmark/src/util.ts b/benchmark/src/util.ts index 8d56fa69a..456eb9e81 100644 --- a/benchmark/src/util.ts +++ b/benchmark/src/util.ts @@ -50,11 +50,7 @@ export function readFile(path: string): Result { const readAllResult = readAll(fileHandle); fileHandle.close(); - if (readAllResult.isOk()) { - return ok(readAllResult); - } else { - return readAllResult; - } + return readAllResult; } return err(`Can't open file ${path}`); } @@ -88,7 +84,7 @@ export function readDir(dir: string): Result { } return ok(files.filter(p => p !== "")); } else { - return findReadAllResult; + return err(findReadAllResult.error); } } @@ -96,7 +92,7 @@ export function loadBenchmarksFromDirectory(benchmarkDir: string): Result Date: Thu, 7 May 2020 18:22:45 +0200 Subject: [PATCH 12/21] Consistent variable names --- benchmark/src/memory_benchmark.ts | 28 ++++++++++----------- benchmark/src/run.ts | 41 ++++++++++++++----------------- 2 files changed, 33 insertions(+), 36 deletions(-) diff --git a/benchmark/src/memory_benchmark.ts b/benchmark/src/memory_benchmark.ts index 30333e933..23109ae56 100644 --- a/benchmark/src/memory_benchmark.ts +++ b/benchmark/src/memory_benchmark.ts @@ -1,4 +1,4 @@ -import { BenchmarkKind, MemoryBenchmarkResult } from "./benchmark_types"; +import { BenchmarkKind, MemoryBenchmarkResult, ComparisonInfo } from "./benchmark_types"; import { toFixed, json } from "./util"; export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmarkResult { @@ -31,21 +31,21 @@ export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmark export function compareMemoryBenchmarks( oldResults: MemoryBenchmarkResult[], - updatedResults: MemoryBenchmarkResult[] -): [string, string] { + newResults: MemoryBenchmarkResult[] +): ComparisonInfo { let comparisonTable = "| name | master (MB) | commit (MB) | change (MB) | change (%) |\n| - | - | - | - | - |\n"; const formatMemory = (memInKB: number) => toFixed(memInKB / 1024, 3); // we group by the new results in case benchmarks have been added - updatedResults.forEach(newResult => { - const masterResult = oldResults.find(r => r.benchmarkName === newResult.benchmarkName); - if (masterResult) { - const percentageChange = (newResult.memoryUsedForExec / masterResult.memoryUsedForExec) * 100 - 100; + newResults.forEach(newResult => { + const oldResult = oldResults.find(r => r.benchmarkName === newResult.benchmarkName); + if (oldResult) { + const percentageChange = (newResult.memoryUsedForExec / oldResult.memoryUsedForExec) * 100 - 100; comparisonTable += `| ${newResult.benchmarkName} | ${formatMemory( - masterResult.memoryUsedForExec + oldResult.memoryUsedForExec )} | ${formatMemory(newResult.memoryUsedForExec)} | ${formatMemory( - newResult.memoryUsedForExec - masterResult.memoryUsedForExec + newResult.memoryUsedForExec - oldResult.memoryUsedForExec )} | ${toFixed(percentageChange, 2)} |\n`; } else { // No master found => new benchmark @@ -55,11 +55,11 @@ export function compareMemoryBenchmarks( } }); - const markdownSummary = `**Memory:**\n${comparisonTable}`; + const summary = `**Memory:**\n${comparisonTable}`; - const markdownText = `**master:**\n\`\`\`json\n${json.encode( - oldResults - )}\n\`\`\`\n**commit:**\n\`\`\`json\n${json.encode(updatedResults)}\n\`\`\``; + const text = `**master:**\n\`\`\`json\n${json.encode(oldResults)}\n\`\`\`\n**commit:**\n\`\`\`json\n${json.encode( + newResults + )}\n\`\`\``; - return [markdownSummary, markdownText]; + return { summary, text }; } diff --git a/benchmark/src/run.ts b/benchmark/src/run.ts index c4cd276f5..1144b515a 100644 --- a/benchmark/src/run.ts +++ b/benchmark/src/run.ts @@ -10,51 +10,48 @@ declare const arg: [string | undefined, string | undefined, string | undefined]; function benchmark(): void { // Memory tests - let memoryUpdatedResults: MemoryBenchmarkResult[] = []; + let memoryBenchmarkNewResults: MemoryBenchmarkResult[] = []; - const loadPreviousMemoryBenchmarksResult = loadBenchmarksFromDirectory("memory_benchmarks"); + const memoryBenchmarks = loadBenchmarksFromDirectory("memory_benchmarks"); - if (loadPreviousMemoryBenchmarksResult.isOk()) { - memoryUpdatedResults = loadPreviousMemoryBenchmarksResult.value.map(runMemoryBenchmark); + if (memoryBenchmarks.isOk()) { + memoryBenchmarkNewResults = memoryBenchmarks.value.map(runMemoryBenchmark); } else { - print(loadPreviousMemoryBenchmarksResult.error); + print(memoryBenchmarks.error); os.exit(1); } // run future benchmarks types here - const benchmarkResults = [...memoryUpdatedResults]; + const newBenchmarkResults = [...memoryBenchmarkNewResults]; // Try to read the baseline benchmark result - let previousBenchmarkResults: BenchmarkResult[] = []; + let oldBenchmarkResults: BenchmarkResult[] = []; if (arg[1]) { - const readPreviousFileResult = readFile(arg[1]); - if (readPreviousFileResult.isOk()) { - previousBenchmarkResults = json.decode(readPreviousFileResult.value) as BenchmarkResult[]; + const oldBenchmarkData = readFile(arg[1]); + if (oldBenchmarkData.isOk()) { + oldBenchmarkResults = json.decode(oldBenchmarkData.value) as BenchmarkResult[]; } } // Compare results - const comparisonInfo = compareBenchmarks(previousBenchmarkResults, benchmarkResults); + const comparisonInfo = compareBenchmarks(oldBenchmarkResults, newBenchmarkResults); // Output comparison info - outputBenchmarkData(comparisonInfo, benchmarkResults); + outputBenchmarkData(comparisonInfo, newBenchmarkResults); } benchmark(); -function compareBenchmarks(previousResults: BenchmarkResult[], updatedResults: BenchmarkResult[]): ComparisonInfo { - const previousResultsMemory = previousResults.filter(isMemoryBenchmarkResult); - const updatedResultsMemory = updatedResults.filter(isMemoryBenchmarkResult); +function compareBenchmarks(oldResults: BenchmarkResult[], newResults: BenchmarkResult[]): ComparisonInfo { + const oldResultsMemory = oldResults.filter(isMemoryBenchmarkResult); + const newResultsMemory = newResults.filter(isMemoryBenchmarkResult); - const memoryComparisonInfo = compareMemoryBenchmarks(previousResultsMemory, updatedResultsMemory); + const memoryComparisonInfo = compareMemoryBenchmarks(oldResultsMemory, newResultsMemory); - return { summary: memoryComparisonInfo[0], text: memoryComparisonInfo[1] }; + return { summary: memoryComparisonInfo.summary, text: memoryComparisonInfo.text }; } -function outputBenchmarkData( - comparisonInfo: { summary: string; text: string }, - updatedResults: BenchmarkResult[] -): void { +function outputBenchmarkData(comparisonInfo: { summary: string; text: string }, newResults: BenchmarkResult[]): void { if (!arg[2]) { // Output to stdout as json by default, this is used by the CI to retrieve the info print(json.encode(comparisonInfo)); @@ -66,6 +63,6 @@ function outputBenchmarkData( // Output benchmark results to json if (arg[0]) { const jsonDataFile = io.open(arg[0], "w+")[0]!; - jsonDataFile.write(json.encode(updatedResults)); + jsonDataFile.write(json.encode(newResults)); } } From 1611e67b6132922cc980f441445b20a944a387a4 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Thu, 7 May 2020 22:27:57 +0200 Subject: [PATCH 13/21] Reduced error handling boilerplate & Added some becnhmarks --- .../src/memory_benchmarks/array_concat.ts | 9 +++ .../src/memory_benchmarks/array_every.ts | 10 +++ benchmark/src/memory_benchmarks/array_push.ts | 8 +++ .../src/memory_benchmarks/class_creation.ts | 22 +++++++ benchmark/src/run.ts | 4 +- benchmark/src/util.ts | 65 ++++++------------- 6 files changed, 70 insertions(+), 48 deletions(-) create mode 100644 benchmark/src/memory_benchmarks/array_concat.ts create mode 100644 benchmark/src/memory_benchmarks/array_every.ts create mode 100644 benchmark/src/memory_benchmarks/array_push.ts create mode 100644 benchmark/src/memory_benchmarks/class_creation.ts diff --git a/benchmark/src/memory_benchmarks/array_concat.ts b/benchmark/src/memory_benchmarks/array_concat.ts new file mode 100644 index 000000000..26a02b2f5 --- /dev/null +++ b/benchmark/src/memory_benchmarks/array_concat.ts @@ -0,0 +1,9 @@ +// tslint:disable-next-line:no-default-export +export default function arrayConcat(): void { + let arr1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; + const arr2 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; + const n = 500; + for (let i = 0; i < n; i++) { + arr1 = arr1.concat(arr2); + } +} diff --git a/benchmark/src/memory_benchmarks/array_every.ts b/benchmark/src/memory_benchmarks/array_every.ts new file mode 100644 index 000000000..a48d36aef --- /dev/null +++ b/benchmark/src/memory_benchmarks/array_every.ts @@ -0,0 +1,10 @@ +// tslint:disable-next-line:no-default-export +export default function arrayEvery(): boolean { + const arr = []; + const n = 500; + for (let i = 0; i < n; i++) { + arr[i] = i; + } + const isSmallerN = arr.every(e => e < n); + return isSmallerN; +} diff --git a/benchmark/src/memory_benchmarks/array_push.ts b/benchmark/src/memory_benchmarks/array_push.ts new file mode 100644 index 000000000..056cf070a --- /dev/null +++ b/benchmark/src/memory_benchmarks/array_push.ts @@ -0,0 +1,8 @@ +// tslint:disable-next-line:no-default-export +export default function arrayPush(): void { + const n = 500; + const numberList: number[] = []; + for (let i = 0; i < n; i++) { + numberList.push(i * i); + } +} diff --git a/benchmark/src/memory_benchmarks/class_creation.ts b/benchmark/src/memory_benchmarks/class_creation.ts new file mode 100644 index 000000000..300a9d0b2 --- /dev/null +++ b/benchmark/src/memory_benchmarks/class_creation.ts @@ -0,0 +1,22 @@ +class A { + constructor(x: number) { + this.z = x * x; + } + + public z: number; +} + +class B extends A { + constructor(y: number) { + super(y); + } +} + +// tslint:disable-next-line:no-default-export +export default function classCreation(): void { + const arr1 = []; + const n = 500; + for (let i = 0; i < n; i++) { + arr1.push(new B(i)); + } +} diff --git a/benchmark/src/run.ts b/benchmark/src/run.ts index 1144b515a..a0200aad6 100644 --- a/benchmark/src/run.ts +++ b/benchmark/src/run.ts @@ -14,7 +14,7 @@ function benchmark(): void { const memoryBenchmarks = loadBenchmarksFromDirectory("memory_benchmarks"); - if (memoryBenchmarks.isOk()) { + if (memoryBenchmarks.success) { memoryBenchmarkNewResults = memoryBenchmarks.value.map(runMemoryBenchmark); } else { print(memoryBenchmarks.error); @@ -29,7 +29,7 @@ function benchmark(): void { let oldBenchmarkResults: BenchmarkResult[] = []; if (arg[1]) { const oldBenchmarkData = readFile(arg[1]); - if (oldBenchmarkData.isOk()) { + if (oldBenchmarkData.success) { oldBenchmarkResults = json.decode(oldBenchmarkData.value) as BenchmarkResult[]; } } diff --git a/benchmark/src/util.ts b/benchmark/src/util.ts index 456eb9e81..015715620 100644 --- a/benchmark/src/util.ts +++ b/benchmark/src/util.ts @@ -1,34 +1,6 @@ import { BenchmarkFunction } from "./benchmark_types"; -export type Result = Ok | Err; - -export class Ok { - constructor(readonly value: T) {} - - public isOk(): this is Ok { - return true; - } - - public isError(): this is Err { - return false; - } -} - -export class Err { - constructor(readonly error: E) {} - - public isOk(): this is Ok { - return false; - } - - public isError(): this is Err { - return true; - } -} - -export const ok = (value: T): Ok => new Ok(value); - -export const err = (err: E): Err => new Err(err); +type Result = { success: true; value: T } | { success: false; error: string }; export function toFixed(num: number, decimalPlaces = 0): string { return string.format(`%.${decimalPlaces}f`, num); @@ -42,7 +14,7 @@ export const json: { encode: (this: void, val: any) => string; } = require("json"); -export function readFile(path: string): Result { +export function readFile(path: string): Result { const fileOpenArray = io.open(path, "rb"); if (fileOpenArray && fileOpenArray[0]) { @@ -52,27 +24,27 @@ export function readFile(path: string): Result { return readAllResult; } - return err(`Can't open file ${path}`); + return { success: false, error: `Can't open file ${path}` }; } -export function readAll(file: LuaFile): Result { +export function readAll(file: LuaFile): Result { const content = file.read(_VERSION === "Lua 5.3" ? "a" : ("*a" as any)) as [string | undefined]; if (content[0]) { - return ok(content[0]); + return { success: true, value: content[0] }; } - return err(`Can't readAll for file ${file}`); + return { success: false, error: `Can't readAll for file ${file}` }; } -export function readDir(dir: string): Result { +export function readDir(dir: string): Result { const findHandle = isWindows ? io.popen(`dir /A-D /B ${dir}`) : io.popen(`find '${dir}' -maxdepth 1 -type f`); const findReadAllResult = readAll(findHandle); if (!findHandle.close()) { - return err(`readDir popen failed for dir ${dir} see stdout for more information.`); + return { success: false, error: `readDir popen failed for dir ${dir} see stdout for more information.` }; } - if (findReadAllResult.isOk()) { + if (findReadAllResult.success) { let files = findReadAllResult.value.split("\n"); if (isWindows) { // on windows we need to append the directory path @@ -82,26 +54,27 @@ export function readDir(dir: string): Result { // strip leading "./" on unix files = files.map(f => (f[0] === "." && f[1] === "/" ? f.substr(2) : f)); } - return ok(files.filter(p => p !== "")); + return { success: true, value: files.filter(p => p !== "") }; } else { - return err(findReadAllResult.error); + return { success: false, error: findReadAllResult.error }; } } -export function loadBenchmarksFromDirectory(benchmarkDir: string): Result { +export function loadBenchmarksFromDirectory(benchmarkDir: string): Result { const readBenchmarkDirResult = readDir(benchmarkDir); - if (readBenchmarkDirResult.isError()) { - return err(readBenchmarkDirResult.error); + if (!readBenchmarkDirResult.success) { + return { success: false, error: readBenchmarkDirResult.error }; } - return ok( - readBenchmarkDirResult.value.map(f => { + return { + success: true, + value: readBenchmarkDirResult.value.map(f => { // replace slashes with dots let dotPath = string.gsub(f, "%/", ".")[0]; // remove extension dotPath = string.gsub(dotPath, ".lua", "")[0]; return require(dotPath).default as BenchmarkFunction; - }) - ); + }), + }; } From 495bb2e4b0f233a0ee5ab946a5afd35de7c0eb7a Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Fri, 8 May 2020 11:43:00 +0200 Subject: [PATCH 14/21] Track amount of garbage collected --- benchmark/src/benchmark_types.ts | 10 +- benchmark/src/memory_benchmark.ts | 107 +++++++++++++----- .../src/memory_benchmarks/array_concat.ts | 5 +- .../src/memory_benchmarks/array_every.ts | 2 +- benchmark/src/memory_benchmarks/array_push.ts | 5 +- .../src/memory_benchmarks/class_creation.ts | 5 +- .../src/memory_benchmarks/graph_cylce.ts | 4 +- benchmark/src/util.ts | 4 + 8 files changed, 99 insertions(+), 43 deletions(-) diff --git a/benchmark/src/benchmark_types.ts b/benchmark/src/benchmark_types.ts index efd75c3e4..6ec80efe6 100644 --- a/benchmark/src/benchmark_types.ts +++ b/benchmark/src/benchmark_types.ts @@ -6,13 +6,15 @@ export type BenchmarkFunction = () => void; export type BenchmarkResult = MemoryBenchmarkResult; +export enum MemoryBenchmarkCategory { + TotalMemory = "totalMemory", + Garbage = "garbage", +} + export interface MemoryBenchmarkResult { kind: BenchmarkKind.Memory; + categories: Record; benchmarkName: string; - preExecMemoryUsage: number; - postExecMemoryUsage: number; - memoryUsedForExec: number; - memoryAfterGC: number; } export function isMemoryBenchmarkResult(result: BenchmarkResult): result is MemoryBenchmarkResult { diff --git a/benchmark/src/memory_benchmark.ts b/benchmark/src/memory_benchmark.ts index 23109ae56..a085fedfd 100644 --- a/benchmark/src/memory_benchmark.ts +++ b/benchmark/src/memory_benchmark.ts @@ -1,65 +1,112 @@ -import { BenchmarkKind, MemoryBenchmarkResult, ComparisonInfo } from "./benchmark_types"; -import { toFixed, json } from "./util"; +import { BenchmarkKind, MemoryBenchmarkResult, ComparisonInfo, MemoryBenchmarkCategory } from "./benchmark_types"; +import { toFixed, json, calculatePercentageChange } from "./util"; export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmarkResult { const result: MemoryBenchmarkResult = { kind: BenchmarkKind.Memory, benchmarkName: "NO_NAME", - preExecMemoryUsage: 0, - postExecMemoryUsage: 0, - memoryUsedForExec: 0, - memoryAfterGC: 0, + categories: { + [MemoryBenchmarkCategory.Garbage]: 0, + [MemoryBenchmarkCategory.TotalMemory]: 0, + }, }; + // collect before running benchmark + collectgarbage("collect"); + + // sop automatic gc collectgarbage("stop"); - result.preExecMemoryUsage = collectgarbage("count"); - benchmarkFunction(); + const preExecMemoryUsage = collectgarbage("count"); + + // Store return value this allows benchmark to preent data from being garbage collected + let temp = benchmarkFunction(); - result.postExecMemoryUsage = collectgarbage("count"); - result.memoryUsedForExec = result.postExecMemoryUsage - result.preExecMemoryUsage; + const postExecMemoryUsage = collectgarbage("count"); collectgarbage("restart"); collectgarbage("collect"); - result.memoryAfterGC = collectgarbage("count"); + // get the amount of garbage collected + result.categories[MemoryBenchmarkCategory.Garbage] = postExecMemoryUsage - collectgarbage("count"); + + // make sure result isn't garbage collected until now and supress unused var warning + temp = temp; result.benchmarkName = debug.getinfo(benchmarkFunction).short_src; + result.categories[MemoryBenchmarkCategory.TotalMemory] = postExecMemoryUsage - preExecMemoryUsage; + return result; } +const formatMemory = (memInKB: number) => toFixed(memInKB / 1024, 3); +const makeMarkdownTableRow = (cells: string[]) => `| ${cells.join(" | ")} |\n`; +const makeBold = (input: string) => `**${input}**`; + export function compareMemoryBenchmarks( oldResults: MemoryBenchmarkResult[], newResults: MemoryBenchmarkResult[] ): ComparisonInfo { - let comparisonTable = "| name | master (MB) | commit (MB) | change (MB) | change (%) |\n| - | - | - | - | - |\n"; + // Cant use Object.values because we want a fixed order. + const categories = [MemoryBenchmarkCategory.TotalMemory, MemoryBenchmarkCategory.Garbage]; + + const summary = categories + .map(category => { + return `${makeBold(category)}\n${compareCategory(newResults, oldResults, category)}`; + }) + .join("\n"); + + const text = `**master:**\n\`\`\`json\n${json.encode(oldResults)}\n\`\`\`\n**commit:**\n\`\`\`json\n${json.encode( + newResults + )}\n\`\`\``; - const formatMemory = (memInKB: number) => toFixed(memInKB / 1024, 3); + return { summary, text }; +} - // we group by the new results in case benchmarks have been added +function compareCategory( + newResults: MemoryBenchmarkResult[], + oldResults: MemoryBenchmarkResult[], + category: MemoryBenchmarkCategory +): string { + let comparisonTable = makeMarkdownTableRow(["name", "master (mb)", "commit (mb)", "change (mb)", "change (%)"]); + comparisonTable += makeMarkdownTableRow(["-", "-", "-", "-", "-"]); + let oldValueSum = 0; + let newValueSum = 0; newResults.forEach(newResult => { const oldResult = oldResults.find(r => r.benchmarkName === newResult.benchmarkName); if (oldResult) { - const percentageChange = (newResult.memoryUsedForExec / oldResult.memoryUsedForExec) * 100 - 100; - comparisonTable += `| ${newResult.benchmarkName} | ${formatMemory( - oldResult.memoryUsedForExec - )} | ${formatMemory(newResult.memoryUsedForExec)} | ${formatMemory( - newResult.memoryUsedForExec - oldResult.memoryUsedForExec - )} | ${toFixed(percentageChange, 2)} |\n`; + const oldValue = oldResult.categories[category]; + const newValue = newResult.categories[category]; + const percentageChange = calculatePercentageChange( + newResult.categories[category], + oldResult.categories[category] + ); + const change = newResult.categories[category] - oldResult.categories[category]; + const row = [ + newResult.benchmarkName, + formatMemory(oldValue), + formatMemory(newValue), + formatMemory(change), + toFixed(percentageChange, 2), + ]; + comparisonTable += makeMarkdownTableRow(row); + oldValueSum += oldValue; + newValueSum += newValue; } else { // No master found => new benchmark - comparisonTable += `| ${newResult.benchmarkName}(new) | / | ${formatMemory( - newResult.memoryUsedForExec - )} | / | / |\n`; + const row = [newResult.benchmarkName, formatMemory(newResult.categories[category]), "/", "/", "/"]; + comparisonTable += makeMarkdownTableRow(row); } }); + const sumPercentageChange = calculatePercentageChange(oldValueSum, newValueSum); + comparisonTable += makeMarkdownTableRow([ + makeBold("sum"), + makeBold(formatMemory(oldValueSum)), + makeBold(formatMemory(newValueSum)), + makeBold(formatMemory(newValueSum - oldValueSum)), + makeBold(toFixed(sumPercentageChange, 2)), + ]); - const summary = `**Memory:**\n${comparisonTable}`; - - const text = `**master:**\n\`\`\`json\n${json.encode(oldResults)}\n\`\`\`\n**commit:**\n\`\`\`json\n${json.encode( - newResults - )}\n\`\`\``; - - return { summary, text }; + return comparisonTable; } diff --git a/benchmark/src/memory_benchmarks/array_concat.ts b/benchmark/src/memory_benchmarks/array_concat.ts index 26a02b2f5..9a97dfa1d 100644 --- a/benchmark/src/memory_benchmarks/array_concat.ts +++ b/benchmark/src/memory_benchmarks/array_concat.ts @@ -1,9 +1,10 @@ // tslint:disable-next-line:no-default-export -export default function arrayConcat(): void { +export default function arrayConcat(): number[] { let arr1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; const arr2 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; - const n = 500; + const n = 1000; for (let i = 0; i < n; i++) { arr1 = arr1.concat(arr2); } + return arr1; } diff --git a/benchmark/src/memory_benchmarks/array_every.ts b/benchmark/src/memory_benchmarks/array_every.ts index a48d36aef..b13c14a13 100644 --- a/benchmark/src/memory_benchmarks/array_every.ts +++ b/benchmark/src/memory_benchmarks/array_every.ts @@ -1,7 +1,7 @@ // tslint:disable-next-line:no-default-export export default function arrayEvery(): boolean { const arr = []; - const n = 500; + const n = 10000; for (let i = 0; i < n; i++) { arr[i] = i; } diff --git a/benchmark/src/memory_benchmarks/array_push.ts b/benchmark/src/memory_benchmarks/array_push.ts index 056cf070a..bf7eb9dbd 100644 --- a/benchmark/src/memory_benchmarks/array_push.ts +++ b/benchmark/src/memory_benchmarks/array_push.ts @@ -1,8 +1,9 @@ // tslint:disable-next-line:no-default-export -export default function arrayPush(): void { - const n = 500; +export default function arrayPush(): number[] { + const n = 10000; const numberList: number[] = []; for (let i = 0; i < n; i++) { numberList.push(i * i); } + return numberList; } diff --git a/benchmark/src/memory_benchmarks/class_creation.ts b/benchmark/src/memory_benchmarks/class_creation.ts index 300a9d0b2..cc3a03e7c 100644 --- a/benchmark/src/memory_benchmarks/class_creation.ts +++ b/benchmark/src/memory_benchmarks/class_creation.ts @@ -13,10 +13,11 @@ class B extends A { } // tslint:disable-next-line:no-default-export -export default function classCreation(): void { +export default function classCreation(): B[] { const arr1 = []; - const n = 500; + const n = 10000; for (let i = 0; i < n; i++) { arr1.push(new B(i)); } + return arr1; } diff --git a/benchmark/src/memory_benchmarks/graph_cylce.ts b/benchmark/src/memory_benchmarks/graph_cylce.ts index 3a172a591..e752e0131 100644 --- a/benchmark/src/memory_benchmarks/graph_cylce.ts +++ b/benchmark/src/memory_benchmarks/graph_cylce.ts @@ -6,14 +6,14 @@ function range(start: number, end: number): number[] { } // tslint:disable-next-line:no-default-export -export default function detectCyleBenchmark(): void { +export default function detectCyleBenchmark(): boolean { const n = 500; const benchmarkGraph = new Map(); // build a graph with n nodes and no cycle for (let i = 0; i < n; i++) { benchmarkGraph.set(i, range(i, n - 1)); } - detectCycle(benchmarkGraph); + return detectCycle(benchmarkGraph); } /** diff --git a/benchmark/src/util.ts b/benchmark/src/util.ts index 015715620..89cd2d38e 100644 --- a/benchmark/src/util.ts +++ b/benchmark/src/util.ts @@ -6,6 +6,10 @@ export function toFixed(num: number, decimalPlaces = 0): string { return string.format(`%.${decimalPlaces}f`, num); } +export function calculatePercentageChange(oldValue: number, newValue: number): number { + return (newValue / oldValue) * 100 - 100; +} + // @ts-ignore export const isWindows = package.config.sub(1, 1) === "\\"; From 15e5d4a234aa83f6fbfcd4ab0427e3b4c49daf79 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Fri, 8 May 2020 11:44:50 +0200 Subject: [PATCH 15/21] Typos --- benchmark/src/memory_benchmark.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/benchmark/src/memory_benchmark.ts b/benchmark/src/memory_benchmark.ts index a085fedfd..9bc8d043c 100644 --- a/benchmark/src/memory_benchmark.ts +++ b/benchmark/src/memory_benchmark.ts @@ -14,12 +14,13 @@ export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmark // collect before running benchmark collectgarbage("collect"); - // sop automatic gc + // stop automatic gc collectgarbage("stop"); const preExecMemoryUsage = collectgarbage("count"); - // Store return value this allows benchmark to preent data from being garbage collected + // store return value this allows benchmark functions + // to prevent "useful" result data from being garbage collected let temp = benchmarkFunction(); const postExecMemoryUsage = collectgarbage("count"); @@ -48,7 +49,7 @@ export function compareMemoryBenchmarks( oldResults: MemoryBenchmarkResult[], newResults: MemoryBenchmarkResult[] ): ComparisonInfo { - // Cant use Object.values because we want a fixed order. + // Can not use Object.values because we want a fixed order. const categories = [MemoryBenchmarkCategory.TotalMemory, MemoryBenchmarkCategory.Garbage]; const summary = categories @@ -71,8 +72,10 @@ function compareCategory( ): string { let comparisonTable = makeMarkdownTableRow(["name", "master (mb)", "commit (mb)", "change (mb)", "change (%)"]); comparisonTable += makeMarkdownTableRow(["-", "-", "-", "-", "-"]); + let oldValueSum = 0; let newValueSum = 0; + newResults.forEach(newResult => { const oldResult = oldResults.find(r => r.benchmarkName === newResult.benchmarkName); if (oldResult) { @@ -99,6 +102,7 @@ function compareCategory( comparisonTable += makeMarkdownTableRow(row); } }); + const sumPercentageChange = calculatePercentageChange(oldValueSum, newValueSum); comparisonTable += makeMarkdownTableRow([ makeBold("sum"), From fafa7e89dda129cddbf64dcc84e7f1a3e4f73cbc Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Sun, 10 May 2020 23:25:54 +0200 Subject: [PATCH 16/21] Addressed final review Changed error handling to use throw instead of monads Added eslint and fixed linting issues --- .eslintrc.js | 11 ++- .github/workflows/ci.yml | 2 +- benchmark/src/memory_benchmark.ts | 4 +- .../src/memory_benchmarks/class_creation.ts | 6 +- benchmark/src/run.ts | 11 +-- benchmark/src/util.ts | 69 ++++++++----------- benchmark/tsconfig.json | 3 +- 7 files changed, 45 insertions(+), 61 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 0ffbec9c9..fb8ec7488 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -67,7 +67,10 @@ const typescriptBase = { module.exports = { extends: ["plugin:jest/recommended", "plugin:jest/style"], - parserOptions: { sourceType: "module", project: ["test/tsconfig.json", "src/lualib/tsconfig.json"] }, + parserOptions: { + sourceType: "module", + project: ["test/tsconfig.json", "src/lualib/tsconfig.json", "benchmark/tsconfig.json"], + }, env: { es6: true, node: true }, plugins: ["import"], rules: { @@ -186,5 +189,11 @@ module.exports = { "@typescript-eslint/prefer-optional-chain": "off", }, }, + { + files: "benchmark/src/memory_benchmarks/**/*.ts", + rules: { + "import/no-default-export": "off", + }, + }, ], }; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ccaeca03..6ba373041 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,7 @@ jobs: - name: NPM commit run: npm ci && npm run build working-directory: commit - # Benchmark estup + # Benchmark directory setup - name: Ensure benchmark data dir exists run: mkdir -p ./benchmark/data working-directory: commit diff --git a/benchmark/src/memory_benchmark.ts b/benchmark/src/memory_benchmark.ts index 9bc8d043c..10934757a 100644 --- a/benchmark/src/memory_benchmark.ts +++ b/benchmark/src/memory_benchmark.ts @@ -53,9 +53,7 @@ export function compareMemoryBenchmarks( const categories = [MemoryBenchmarkCategory.TotalMemory, MemoryBenchmarkCategory.Garbage]; const summary = categories - .map(category => { - return `${makeBold(category)}\n${compareCategory(newResults, oldResults, category)}`; - }) + .map(category => `${makeBold(category)}\n${compareCategory(newResults, oldResults, category)}`) .join("\n"); const text = `**master:**\n\`\`\`json\n${json.encode(oldResults)}\n\`\`\`\n**commit:**\n\`\`\`json\n${json.encode( diff --git a/benchmark/src/memory_benchmarks/class_creation.ts b/benchmark/src/memory_benchmarks/class_creation.ts index cc3a03e7c..df14995f3 100644 --- a/benchmark/src/memory_benchmarks/class_creation.ts +++ b/benchmark/src/memory_benchmarks/class_creation.ts @@ -6,11 +6,7 @@ class A { public z: number; } -class B extends A { - constructor(y: number) { - super(y); - } -} +class B extends A {} // tslint:disable-next-line:no-default-export export default function classCreation(): B[] { diff --git a/benchmark/src/run.ts b/benchmark/src/run.ts index a0200aad6..c632efe39 100644 --- a/benchmark/src/run.ts +++ b/benchmark/src/run.ts @@ -14,12 +14,7 @@ function benchmark(): void { const memoryBenchmarks = loadBenchmarksFromDirectory("memory_benchmarks"); - if (memoryBenchmarks.success) { - memoryBenchmarkNewResults = memoryBenchmarks.value.map(runMemoryBenchmark); - } else { - print(memoryBenchmarks.error); - os.exit(1); - } + memoryBenchmarkNewResults = memoryBenchmarks.map(runMemoryBenchmark); // run future benchmarks types here @@ -29,9 +24,7 @@ function benchmark(): void { let oldBenchmarkResults: BenchmarkResult[] = []; if (arg[1]) { const oldBenchmarkData = readFile(arg[1]); - if (oldBenchmarkData.success) { - oldBenchmarkResults = json.decode(oldBenchmarkData.value) as BenchmarkResult[]; - } + oldBenchmarkResults = json.decode(oldBenchmarkData) as BenchmarkResult[]; } // Compare results diff --git a/benchmark/src/util.ts b/benchmark/src/util.ts index 89cd2d38e..133ad89a3 100644 --- a/benchmark/src/util.ts +++ b/benchmark/src/util.ts @@ -1,7 +1,5 @@ import { BenchmarkFunction } from "./benchmark_types"; -type Result = { success: true; value: T } | { success: false; error: string }; - export function toFixed(num: number, decimalPlaces = 0): string { return string.format(`%.${decimalPlaces}f`, num); } @@ -18,67 +16,56 @@ export const json: { encode: (this: void, val: any) => string; } = require("json"); -export function readFile(path: string): Result { +export function readFile(path: string): string { const fileOpenArray = io.open(path, "rb"); - if (fileOpenArray && fileOpenArray[0]) { + if (fileOpenArray?.[0]) { const fileHandle = fileOpenArray[0]; - const readAllResult = readAll(fileHandle); + const fileContent = readAll(fileHandle); fileHandle.close(); - return readAllResult; + return fileContent; } - return { success: false, error: `Can't open file ${path}` }; + throw Error(`Can't open file ${path}`); } -export function readAll(file: LuaFile): Result { +export function readAll(file: LuaFile): string { const content = file.read(_VERSION === "Lua 5.3" ? "a" : ("*a" as any)) as [string | undefined]; if (content[0]) { - return { success: true, value: content[0] }; + return content[0]; } - return { success: false, error: `Can't readAll for file ${file}` }; + throw Error(`Can't readAll for file ${file}`); } -export function readDir(dir: string): Result { +export function readDir(dir: string): string[] { const findHandle = isWindows ? io.popen(`dir /A-D /B ${dir}`) : io.popen(`find '${dir}' -maxdepth 1 -type f`); - const findReadAllResult = readAll(findHandle); + const findResult = readAll(findHandle); if (!findHandle.close()) { - return { success: false, error: `readDir popen failed for dir ${dir} see stdout for more information.` }; + throw Error(`readDir popen failed for dir ${dir} see stdout for more information.`); } - if (findReadAllResult.success) { - let files = findReadAllResult.value.split("\n"); - if (isWindows) { - // on windows we need to append the directory path - // on unix this is done by find automatically - files = files.map(f => `${dir}/${f}`); - } else { - // strip leading "./" on unix - files = files.map(f => (f[0] === "." && f[1] === "/" ? f.substr(2) : f)); - } - return { success: true, value: files.filter(p => p !== "") }; + let files = findResult.split("\n"); + if (isWindows) { + // on windows we need to append the directory path + // on unix this is done by find automatically + files = files.map(f => `${dir}/${f}`); } else { - return { success: false, error: findReadAllResult.error }; + // strip leading "./" on unix + files = files.map(f => (f.startsWith(".") && f[1] === "/" ? f.substr(2) : f)); } + return files.filter(p => p !== ""); } -export function loadBenchmarksFromDirectory(benchmarkDir: string): Result { - const readBenchmarkDirResult = readDir(benchmarkDir); - - if (!readBenchmarkDirResult.success) { - return { success: false, error: readBenchmarkDirResult.error }; - } +export function loadBenchmarksFromDirectory(benchmarkDir: string): BenchmarkFunction[] { + const benchmarkFiles = readDir(benchmarkDir); - return { - success: true, - value: readBenchmarkDirResult.value.map(f => { - // replace slashes with dots - let dotPath = string.gsub(f, "%/", ".")[0]; - // remove extension - dotPath = string.gsub(dotPath, ".lua", "")[0]; - return require(dotPath).default as BenchmarkFunction; - }), - }; + return benchmarkFiles.map(f => { + // replace slashes with dots + let dotPath = string.gsub(f, "%/", ".")[0]; + // remove extension + dotPath = string.gsub(dotPath, ".lua", "")[0]; + return require(dotPath).default as BenchmarkFunction; + }); } diff --git a/benchmark/tsconfig.json b/benchmark/tsconfig.json index a2d64d893..aefc13ae9 100644 --- a/benchmark/tsconfig.json +++ b/benchmark/tsconfig.json @@ -10,5 +10,6 @@ "strict": true, "noUnusedLocals": true, "noUnusedParameters": true - } + }, + "include": ["src"] } From 1f4172b39ac53d52378f5c0fffe6a35063ce4da4 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Sun, 10 May 2020 23:27:03 +0200 Subject: [PATCH 17/21] Added missing function to readme --- benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/README.md b/benchmark/README.md index c549bcc02..128206236 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -10,7 +10,7 @@ and **default** export a function with the following type: `() => void`. For example (memory_benchmarks/my_benchmark.ts): ```ts -export default myBenchmark() { +export default function myBenchmark() { const n = 123; for (let i = 0; i < n; i++) { // Do something memory instensive From 4c4dc9ffd838913028b73490caf74c28d27e37d4 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Sun, 10 May 2020 23:29:36 +0200 Subject: [PATCH 18/21] Removed tslint comments --- benchmark/README.md | 8 ++++---- benchmark/src/memory_benchmarks/array_concat.ts | 1 - benchmark/src/memory_benchmarks/array_every.ts | 1 - benchmark/src/memory_benchmarks/array_push.ts | 1 - benchmark/src/memory_benchmarks/class_creation.ts | 1 - benchmark/src/memory_benchmarks/graph_cylce.ts | 1 - 6 files changed, 4 insertions(+), 9 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 128206236..37145a3f3 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -11,10 +11,10 @@ For example (memory_benchmarks/my_benchmark.ts): ```ts export default function myBenchmark() { - const n = 123; - for (let i = 0; i < n; i++) { - // Do something memory instensive - } + const n = 123; + for (let i = 0; i < n; i++) { + // Do something memory instensive + } } ``` diff --git a/benchmark/src/memory_benchmarks/array_concat.ts b/benchmark/src/memory_benchmarks/array_concat.ts index 9a97dfa1d..fbe149703 100644 --- a/benchmark/src/memory_benchmarks/array_concat.ts +++ b/benchmark/src/memory_benchmarks/array_concat.ts @@ -1,4 +1,3 @@ -// tslint:disable-next-line:no-default-export export default function arrayConcat(): number[] { let arr1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; const arr2 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; diff --git a/benchmark/src/memory_benchmarks/array_every.ts b/benchmark/src/memory_benchmarks/array_every.ts index b13c14a13..0ba21334f 100644 --- a/benchmark/src/memory_benchmarks/array_every.ts +++ b/benchmark/src/memory_benchmarks/array_every.ts @@ -1,4 +1,3 @@ -// tslint:disable-next-line:no-default-export export default function arrayEvery(): boolean { const arr = []; const n = 10000; diff --git a/benchmark/src/memory_benchmarks/array_push.ts b/benchmark/src/memory_benchmarks/array_push.ts index bf7eb9dbd..15c0a7dd1 100644 --- a/benchmark/src/memory_benchmarks/array_push.ts +++ b/benchmark/src/memory_benchmarks/array_push.ts @@ -1,4 +1,3 @@ -// tslint:disable-next-line:no-default-export export default function arrayPush(): number[] { const n = 10000; const numberList: number[] = []; diff --git a/benchmark/src/memory_benchmarks/class_creation.ts b/benchmark/src/memory_benchmarks/class_creation.ts index df14995f3..aaa57742f 100644 --- a/benchmark/src/memory_benchmarks/class_creation.ts +++ b/benchmark/src/memory_benchmarks/class_creation.ts @@ -8,7 +8,6 @@ class A { class B extends A {} -// tslint:disable-next-line:no-default-export export default function classCreation(): B[] { const arr1 = []; const n = 10000; diff --git a/benchmark/src/memory_benchmarks/graph_cylce.ts b/benchmark/src/memory_benchmarks/graph_cylce.ts index e752e0131..c78812dc5 100644 --- a/benchmark/src/memory_benchmarks/graph_cylce.ts +++ b/benchmark/src/memory_benchmarks/graph_cylce.ts @@ -5,7 +5,6 @@ function range(start: number, end: number): number[] { return [start, ...range(start + 1, end)]; } -// tslint:disable-next-line:no-default-export export default function detectCyleBenchmark(): boolean { const n = 500; const benchmarkGraph = new Map(); From 2a95680fd54431fa553d3b9c88cf3f4a627ce7f3 Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Sun, 10 May 2020 23:36:55 +0200 Subject: [PATCH 19/21] Updated readme regarding non garbage results --- benchmark/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmark/README.md b/benchmark/README.md index 37145a3f3..1f93b809e 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -6,15 +6,19 @@ These benchmarks are written in typescript and transpiled to lua by using tstl. To add a new benchmark add a new file to `memory_benchmarks` and **default** export a function with the following type: `() => void`. +To prevent the benchmark from reporting "useful" results of your benchmark function as garbage, simply return the result. +The memory used by the returned result wont count towards the total garbage amount. For example (memory_benchmarks/my_benchmark.ts): ```ts export default function myBenchmark() { const n = 123; + const result = []; for (let i = 0; i < n; i++) { // Do something memory instensive } + return result; // Return results so they wont be counted as garbage } ``` From a77e46a719837e2bf3c4b77d502b998ad99cdb3b Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Sun, 10 May 2020 23:45:47 +0200 Subject: [PATCH 20/21] Improved readabilty --- benchmark/src/util.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmark/src/util.ts b/benchmark/src/util.ts index 133ad89a3..d1c9fcc1d 100644 --- a/benchmark/src/util.ts +++ b/benchmark/src/util.ts @@ -9,7 +9,7 @@ export function calculatePercentageChange(oldValue: number, newValue: number): n } // @ts-ignore -export const isWindows = package.config.sub(1, 1) === "\\"; +export const isWindows = package.config.startsWith("\\"); export const json: { decode: (this: void, str: string) => {}; @@ -17,16 +17,16 @@ export const json: { } = require("json"); export function readFile(path: string): string { - const fileOpenArray = io.open(path, "rb"); + const [fileHandle] = io.open(path, "rb"); - if (fileOpenArray?.[0]) { - const fileHandle = fileOpenArray[0]; - const fileContent = readAll(fileHandle); - fileHandle.close(); - - return fileContent; + if (!fileHandle) { + throw Error(`Can't open file ${path}`); } - throw Error(`Can't open file ${path}`); + + const fileContent = readAll(fileHandle); + fileHandle.close(); + + return fileContent; } export function readAll(file: LuaFile): string { @@ -39,7 +39,7 @@ export function readAll(file: LuaFile): string { } export function readDir(dir: string): string[] { - const findHandle = isWindows ? io.popen(`dir /A-D /B ${dir}`) : io.popen(`find '${dir}' -maxdepth 1 -type f`); + const findHandle = io.popen(isWindows ? `dir /A-D /B ${dir}` : `find '${dir}' -maxdepth 1 -type f`); const findResult = readAll(findHandle); if (!findHandle.close()) { From da3185d6ea44740ba8940767531f6388738ec02a Mon Sep 17 00:00:00 2001 From: Lorenz Junglas Date: Mon, 11 May 2020 18:58:44 +0200 Subject: [PATCH 21/21] Fixed typos --- benchmark/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 1f93b809e..c7b7d39c2 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -33,11 +33,11 @@ To avoid crashes in the CI your benchmark should not use more than 500MB of memo **Running locally** -1. Create a benchmark baseline called "benchmark_baseilne.json": - `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_baseilne.json` +1. Create a benchmark baseline called "benchmark_baseline.json": + `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_baseline.json` 2. Make some changes to tstl. 3. Create an updated benchmark and compare with the baseline: - `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseilne.json` + `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseline.json` 4. The above command will output comparison data as json to stdout. If you provide a path as third argument the comparison data will be written to that path instead. - `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseilne.json result.md` + `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseline.json result.md`