diff --git a/.eslintrc.js b/.eslintrc.js index 0ffbec9c9..fb8ec7488 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -67,7 +67,10 @@ const typescriptBase = { module.exports = { extends: ["plugin:jest/recommended", "plugin:jest/style"], - parserOptions: { sourceType: "module", project: ["test/tsconfig.json", "src/lualib/tsconfig.json"] }, + parserOptions: { + sourceType: "module", + project: ["test/tsconfig.json", "src/lualib/tsconfig.json", "benchmark/tsconfig.json"], + }, env: { es6: true, node: true }, plugins: ["import"], rules: { @@ -186,5 +189,11 @@ module.exports = { "@typescript-eslint/prefer-optional-chain": "off", }, }, + { + files: "benchmark/src/memory_benchmarks/**/*.ts", + rules: { + "import/no-default-export": "off", + }, + }, ], }; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ee8ba432..6ba373041 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,3 +38,94 @@ jobs: CI: true - if: matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v1 + + benchmark: + name: Benchmark + runs-on: ubuntu-latest + steps: + - name: Lua Install + run: sudo apt-get install lua5.3 luajit + # Checkout master & commit + - name: Checkout master + uses: actions/checkout@v2 + with: + ref: master + path: master + - name: Checkout commit + uses: actions/checkout@v2 + with: + path: commit + - name: Use Node.js 12.13.1 + uses: actions/setup-node@v1 + with: + node-version: 12.13.1 + # NPM + - name: NPM master + # TODO Lua types is only added manually to test the benchmark PR this can be removed again once the PR is merged + run: npm ci && npm run build && npm install -D lua-types + working-directory: master + - name: NPM commit + run: npm ci && npm run build + working-directory: commit + # Benchmark directory setup + - name: Ensure benchmark data dir exists + run: mkdir -p ./benchmark/data + working-directory: commit + - name: Copy commit benchmark to master + run: rm -rf ./master/benchmark && cp -rf ./commit/benchmark ./master/benchmark + # Run master benchmark first and output to commit benchmark data + - name: Build benchmark Lua 5.3 master + run: node ../dist/tstl.js -p tsconfig.53.json + working-directory: master/benchmark + - name: Run benchmark Lua 5.3 master + id: benchmark-lua-master + run: lua5.3 -- run.lua ../../../commit/benchmark/data/benchmark_master_53.json + working-directory: master/benchmark/dist + - name: Build benchmark LuaJIT master + run: node ../dist/tstl.js -p tsconfig.jit.json + working-directory: master/benchmark + - name: Run benchmark LuaJIT master + id: benchmark-jit-master + run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_master_jit.json + working-directory: master/benchmark/dist + # Run commit benchmark and compare with master + - name: Build benchmark Lua 5.3 commit + run: node ../dist/tstl.js -p tsconfig.53.json + working-directory: commit/benchmark + - name: Run benchmark Lua 5.3 commit + id: benchmark-lua-commit + run: echo ::set-output name=info::`lua5.3 -- run.lua ../data/benchmark_commit_53.json ../data/benchmark_master_53.json` + working-directory: commit/benchmark/dist + - name: Build benchmark LuaJIT commit + run: node ../dist/tstl.js -p tsconfig.jit.json + working-directory: commit/benchmark + - name: Run benchmark LuaJIT commit + id: benchmark-jit-commit + run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_commit_jit.json ../data/benchmark_master_jit.json` + working-directory: commit/benchmark/dist + - name: Create benchmark check + uses: actions/github-script@0.9.0 + with: + benchmark-info-lua: ${{steps.benchmark-lua-commit.outputs.info}} + benchmark-info-jit: ${{steps.benchmark-jit-commit.outputs.info}} + script: | + const benchmarkInfoLua = JSON.parse(core.getInput('benchmark-info-lua', { required: true })); + const benchmarkInfoJIT = JSON.parse(core.getInput('benchmark-info-jit', { required: true })); + + const summary = `### Lua5.3\n${benchmarkInfoLua.summary}\n### LuaJIT\n${benchmarkInfoJIT.summary}`; + + const text = `### Lua5.3\n${benchmarkInfoLua.text}\n### LuaJIT\n${benchmarkInfoJIT.text}`; + + github.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: "Benchmark results", + head_sha: context.sha, + status: "completed", + conclusion: "neutral", + output: { + title: "Benchmark results", + summary: summary, + text: text + } + }); diff --git a/.gitignore b/.gitignore index 09a293fc7..5546914da 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ yarn.lock .vscode .idea .DS_Store + +benchmark/data/* +benchmark/dist/* +!benchmark/dist/json.lua diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 000000000..c7b7d39c2 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,43 @@ +## TSTL Benchmarks + +These benchmarks are written in typescript and transpiled to lua by using tstl. + +### Currently only memory benchmarks are supported + +To add a new benchmark add a new file to `memory_benchmarks` +and **default** export a function with the following type: `() => void`. +To prevent the benchmark from reporting "useful" results of your benchmark function as garbage, simply return the result. +The memory used by the returned result wont count towards the total garbage amount. + +For example (memory_benchmarks/my_benchmark.ts): + +```ts +export default function myBenchmark() { + const n = 123; + const result = []; + for (let i = 0; i < n; i++) { + // Do something memory instensive + } + return result; // Return results so they wont be counted as garbage +} +``` + +**Goal** + +The goal of memory benchmarks is to track how much (memory) `"garbage"` is created by tstl. +For that reason garabage collection is disabled in the benchmarks. + +You can force the creation of `"garbage"` by creating a lot of anonymous functions or temporary tables (see [lua-users.org](http://lua-users.org/wiki/OptimisingGarbageCollection) for more information). + +To avoid crashes in the CI your benchmark should not use more than 500MB of memory. + +**Running locally** + +1. Create a benchmark baseline called "benchmark_baseline.json": + `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_baseline.json` +2. Make some changes to tstl. +3. Create an updated benchmark and compare with the baseline: + `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseline.json` +4. The above command will output comparison data as json to stdout. + If you provide a path as third argument the comparison data will be written to that path instead. + `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseline.json result.md` diff --git a/benchmark/dist/json.lua b/benchmark/dist/json.lua new file mode 100644 index 000000000..54d444840 --- /dev/null +++ b/benchmark/dist/json.lua @@ -0,0 +1,388 @@ +-- +-- json.lua +-- +-- Copyright (c) 2020 rxi +-- +-- Permission is hereby granted, free of charge, to any person obtaining a copy of +-- this software and associated documentation files (the "Software"), to deal in +-- the Software without restriction, including without limitation the rights to +-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +-- of the Software, and to permit persons to whom the Software is furnished to do +-- so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in all +-- copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +-- SOFTWARE. +-- + +local json = { _version = "0.1.2" } + +------------------------------------------------------------------------------- +-- Encode +------------------------------------------------------------------------------- + +local encode + +local escape_char_map = { + [ "\\" ] = "\\", + [ "\"" ] = "\"", + [ "\b" ] = "b", + [ "\f" ] = "f", + [ "\n" ] = "n", + [ "\r" ] = "r", + [ "\t" ] = "t", +} + +local escape_char_map_inv = { [ "/" ] = "/" } +for k, v in pairs(escape_char_map) do + escape_char_map_inv[v] = k +end + + +local function escape_char(c) + return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte())) +end + + +local function encode_nil(val) + return "null" +end + + +local function encode_table(val, stack) + local res = {} + stack = stack or {} + + -- Circular reference? + if stack[val] then error("circular reference") end + + stack[val] = true + + if rawget(val, 1) ~= nil or next(val) == nil then + -- Treat as array -- check keys are valid and it is not sparse + local n = 0 + for k in pairs(val) do + if type(k) ~= "number" then + error("invalid table: mixed or invalid key types") + end + n = n + 1 + end + if n ~= #val then + error("invalid table: sparse array") + end + -- Encode + for i, v in ipairs(val) do + table.insert(res, encode(v, stack)) + end + stack[val] = nil + return "[" .. table.concat(res, ",") .. "]" + + else + -- Treat as an object + for k, v in pairs(val) do + if type(k) ~= "string" then + error("invalid table: mixed or invalid key types") + end + table.insert(res, encode(k, stack) .. ":" .. encode(v, stack)) + end + stack[val] = nil + return "{" .. table.concat(res, ",") .. "}" + end +end + + +local function encode_string(val) + return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"' +end + + +local function encode_number(val) + -- Check for NaN, -inf and inf + if val ~= val or val <= -math.huge or val >= math.huge then + error("unexpected number value '" .. tostring(val) .. "'") + end + return string.format("%.14g", val) +end + + +local type_func_map = { + [ "nil" ] = encode_nil, + [ "table" ] = encode_table, + [ "string" ] = encode_string, + [ "number" ] = encode_number, + [ "boolean" ] = tostring, +} + + +encode = function(val, stack) + local t = type(val) + local f = type_func_map[t] + if f then + return f(val, stack) + end + error("unexpected type '" .. t .. "'") +end + + +function json.encode(val) + return ( encode(val) ) +end + + +------------------------------------------------------------------------------- +-- Decode +------------------------------------------------------------------------------- + +local parse + +local function create_set(...) + local res = {} + for i = 1, select("#", ...) do + res[ select(i, ...) ] = true + end + return res +end + +local space_chars = create_set(" ", "\t", "\r", "\n") +local delim_chars = create_set(" ", "\t", "\r", "\n", "]", "}", ",") +local escape_chars = create_set("\\", "/", '"', "b", "f", "n", "r", "t", "u") +local literals = create_set("true", "false", "null") + +local literal_map = { + [ "true" ] = true, + [ "false" ] = false, + [ "null" ] = nil, +} + + +local function next_char(str, idx, set, negate) + for i = idx, #str do + if set[str:sub(i, i)] ~= negate then + return i + end + end + return #str + 1 +end + + +local function decode_error(str, idx, msg) + local line_count = 1 + local col_count = 1 + for i = 1, idx - 1 do + col_count = col_count + 1 + if str:sub(i, i) == "\n" then + line_count = line_count + 1 + col_count = 1 + end + end + error( string.format("%s at line %d col %d", msg, line_count, col_count) ) +end + + +local function codepoint_to_utf8(n) + -- http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=iws-appendixa + local f = math.floor + if n <= 0x7f then + return string.char(n) + elseif n <= 0x7ff then + return string.char(f(n / 64) + 192, n % 64 + 128) + elseif n <= 0xffff then + return string.char(f(n / 4096) + 224, f(n % 4096 / 64) + 128, n % 64 + 128) + elseif n <= 0x10ffff then + return string.char(f(n / 262144) + 240, f(n % 262144 / 4096) + 128, + f(n % 4096 / 64) + 128, n % 64 + 128) + end + error( string.format("invalid unicode codepoint '%x'", n) ) +end + + +local function parse_unicode_escape(s) + local n1 = tonumber( s:sub(1, 4), 16 ) + local n2 = tonumber( s:sub(7, 10), 16 ) + -- Surrogate pair? + if n2 then + return codepoint_to_utf8((n1 - 0xd800) * 0x400 + (n2 - 0xdc00) + 0x10000) + else + return codepoint_to_utf8(n1) + end +end + + +local function parse_string(str, i) + local res = "" + local j = i + 1 + local k = j + + while j <= #str do + local x = str:byte(j) + + if x < 32 then + decode_error(str, j, "control character in string") + + elseif x == 92 then -- `\`: Escape + res = res .. str:sub(k, j - 1) + j = j + 1 + local c = str:sub(j, j) + if c == "u" then + local hex = str:match("^[dD][89aAbB]%x%x\\u%x%x%x%x", j + 1) + or str:match("^%x%x%x%x", j + 1) + or decode_error(str, j - 1, "invalid unicode escape in string") + res = res .. parse_unicode_escape(hex) + j = j + #hex + else + if not escape_chars[c] then + decode_error(str, j - 1, "invalid escape char '" .. c .. "' in string") + end + res = res .. escape_char_map_inv[c] + end + k = j + 1 + + elseif x == 34 then -- `"`: End of string + res = res .. str:sub(k, j - 1) + return res, j + 1 + end + + j = j + 1 + end + + decode_error(str, i, "expected closing quote for string") +end + + +local function parse_number(str, i) + local x = next_char(str, i, delim_chars) + local s = str:sub(i, x - 1) + local n = tonumber(s) + if not n then + decode_error(str, i, "invalid number '" .. s .. "'") + end + return n, x +end + + +local function parse_literal(str, i) + local x = next_char(str, i, delim_chars) + local word = str:sub(i, x - 1) + if not literals[word] then + decode_error(str, i, "invalid literal '" .. word .. "'") + end + return literal_map[word], x +end + + +local function parse_array(str, i) + local res = {} + local n = 1 + i = i + 1 + while 1 do + local x + i = next_char(str, i, space_chars, true) + -- Empty / end of array? + if str:sub(i, i) == "]" then + i = i + 1 + break + end + -- Read token + x, i = parse(str, i) + res[n] = x + n = n + 1 + -- Next token + i = next_char(str, i, space_chars, true) + local chr = str:sub(i, i) + i = i + 1 + if chr == "]" then break end + if chr ~= "," then decode_error(str, i, "expected ']' or ','") end + end + return res, i +end + + +local function parse_object(str, i) + local res = {} + i = i + 1 + while 1 do + local key, val + i = next_char(str, i, space_chars, true) + -- Empty / end of object? + if str:sub(i, i) == "}" then + i = i + 1 + break + end + -- Read key + if str:sub(i, i) ~= '"' then + decode_error(str, i, "expected string for key") + end + key, i = parse(str, i) + -- Read ':' delimiter + i = next_char(str, i, space_chars, true) + if str:sub(i, i) ~= ":" then + decode_error(str, i, "expected ':' after key") + end + i = next_char(str, i + 1, space_chars, true) + -- Read value + val, i = parse(str, i) + -- Set + res[key] = val + -- Next token + i = next_char(str, i, space_chars, true) + local chr = str:sub(i, i) + i = i + 1 + if chr == "}" then break end + if chr ~= "," then decode_error(str, i, "expected '}' or ','") end + end + return res, i +end + + +local char_func_map = { + [ '"' ] = parse_string, + [ "0" ] = parse_number, + [ "1" ] = parse_number, + [ "2" ] = parse_number, + [ "3" ] = parse_number, + [ "4" ] = parse_number, + [ "5" ] = parse_number, + [ "6" ] = parse_number, + [ "7" ] = parse_number, + [ "8" ] = parse_number, + [ "9" ] = parse_number, + [ "-" ] = parse_number, + [ "t" ] = parse_literal, + [ "f" ] = parse_literal, + [ "n" ] = parse_literal, + [ "[" ] = parse_array, + [ "{" ] = parse_object, +} + + +parse = function(str, idx) + local chr = str:sub(idx, idx) + local f = char_func_map[chr] + if f then + return f(str, idx) + end + decode_error(str, idx, "unexpected character '" .. chr .. "'") +end + + +function json.decode(str) + if type(str) ~= "string" then + error("expected argument of type string, got " .. type(str)) + end + local res, idx = parse(str, next_char(str, 1, space_chars, true)) + idx = next_char(str, idx, space_chars, true) + if idx <= #str then + decode_error(str, idx, "trailing garbage") + end + return res +end + + +return json \ No newline at end of file diff --git a/benchmark/src/benchmark_types.ts b/benchmark/src/benchmark_types.ts new file mode 100644 index 000000000..6ec80efe6 --- /dev/null +++ b/benchmark/src/benchmark_types.ts @@ -0,0 +1,27 @@ +export enum BenchmarkKind { + Memory = "memory", +} + +export type BenchmarkFunction = () => void; + +export type BenchmarkResult = MemoryBenchmarkResult; + +export enum MemoryBenchmarkCategory { + TotalMemory = "totalMemory", + Garbage = "garbage", +} + +export interface MemoryBenchmarkResult { + kind: BenchmarkKind.Memory; + categories: Record; + benchmarkName: string; +} + +export function isMemoryBenchmarkResult(result: BenchmarkResult): result is MemoryBenchmarkResult { + return result.kind === BenchmarkKind.Memory; +} + +export interface ComparisonInfo { + summary: string; + text: string; +} diff --git a/benchmark/src/memory_benchmark.ts b/benchmark/src/memory_benchmark.ts new file mode 100644 index 000000000..10934757a --- /dev/null +++ b/benchmark/src/memory_benchmark.ts @@ -0,0 +1,114 @@ +import { BenchmarkKind, MemoryBenchmarkResult, ComparisonInfo, MemoryBenchmarkCategory } from "./benchmark_types"; +import { toFixed, json, calculatePercentageChange } from "./util"; + +export function runMemoryBenchmark(benchmarkFunction: Function): MemoryBenchmarkResult { + const result: MemoryBenchmarkResult = { + kind: BenchmarkKind.Memory, + benchmarkName: "NO_NAME", + categories: { + [MemoryBenchmarkCategory.Garbage]: 0, + [MemoryBenchmarkCategory.TotalMemory]: 0, + }, + }; + + // collect before running benchmark + collectgarbage("collect"); + + // stop automatic gc + collectgarbage("stop"); + + const preExecMemoryUsage = collectgarbage("count"); + + // store return value this allows benchmark functions + // to prevent "useful" result data from being garbage collected + let temp = benchmarkFunction(); + + const postExecMemoryUsage = collectgarbage("count"); + + collectgarbage("restart"); + collectgarbage("collect"); + + // get the amount of garbage collected + result.categories[MemoryBenchmarkCategory.Garbage] = postExecMemoryUsage - collectgarbage("count"); + + // make sure result isn't garbage collected until now and supress unused var warning + temp = temp; + + result.benchmarkName = debug.getinfo(benchmarkFunction).short_src; + + result.categories[MemoryBenchmarkCategory.TotalMemory] = postExecMemoryUsage - preExecMemoryUsage; + + return result; +} + +const formatMemory = (memInKB: number) => toFixed(memInKB / 1024, 3); +const makeMarkdownTableRow = (cells: string[]) => `| ${cells.join(" | ")} |\n`; +const makeBold = (input: string) => `**${input}**`; + +export function compareMemoryBenchmarks( + oldResults: MemoryBenchmarkResult[], + newResults: MemoryBenchmarkResult[] +): ComparisonInfo { + // Can not use Object.values because we want a fixed order. + const categories = [MemoryBenchmarkCategory.TotalMemory, MemoryBenchmarkCategory.Garbage]; + + const summary = categories + .map(category => `${makeBold(category)}\n${compareCategory(newResults, oldResults, category)}`) + .join("\n"); + + const text = `**master:**\n\`\`\`json\n${json.encode(oldResults)}\n\`\`\`\n**commit:**\n\`\`\`json\n${json.encode( + newResults + )}\n\`\`\``; + + return { summary, text }; +} + +function compareCategory( + newResults: MemoryBenchmarkResult[], + oldResults: MemoryBenchmarkResult[], + category: MemoryBenchmarkCategory +): string { + let comparisonTable = makeMarkdownTableRow(["name", "master (mb)", "commit (mb)", "change (mb)", "change (%)"]); + comparisonTable += makeMarkdownTableRow(["-", "-", "-", "-", "-"]); + + let oldValueSum = 0; + let newValueSum = 0; + + newResults.forEach(newResult => { + const oldResult = oldResults.find(r => r.benchmarkName === newResult.benchmarkName); + if (oldResult) { + const oldValue = oldResult.categories[category]; + const newValue = newResult.categories[category]; + const percentageChange = calculatePercentageChange( + newResult.categories[category], + oldResult.categories[category] + ); + const change = newResult.categories[category] - oldResult.categories[category]; + const row = [ + newResult.benchmarkName, + formatMemory(oldValue), + formatMemory(newValue), + formatMemory(change), + toFixed(percentageChange, 2), + ]; + comparisonTable += makeMarkdownTableRow(row); + oldValueSum += oldValue; + newValueSum += newValue; + } else { + // No master found => new benchmark + const row = [newResult.benchmarkName, formatMemory(newResult.categories[category]), "/", "/", "/"]; + comparisonTable += makeMarkdownTableRow(row); + } + }); + + const sumPercentageChange = calculatePercentageChange(oldValueSum, newValueSum); + comparisonTable += makeMarkdownTableRow([ + makeBold("sum"), + makeBold(formatMemory(oldValueSum)), + makeBold(formatMemory(newValueSum)), + makeBold(formatMemory(newValueSum - oldValueSum)), + makeBold(toFixed(sumPercentageChange, 2)), + ]); + + return comparisonTable; +} diff --git a/benchmark/src/memory_benchmarks/array_concat.ts b/benchmark/src/memory_benchmarks/array_concat.ts new file mode 100644 index 000000000..fbe149703 --- /dev/null +++ b/benchmark/src/memory_benchmarks/array_concat.ts @@ -0,0 +1,9 @@ +export default function arrayConcat(): number[] { + let arr1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; + const arr2 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; + const n = 1000; + for (let i = 0; i < n; i++) { + arr1 = arr1.concat(arr2); + } + return arr1; +} diff --git a/benchmark/src/memory_benchmarks/array_every.ts b/benchmark/src/memory_benchmarks/array_every.ts new file mode 100644 index 000000000..0ba21334f --- /dev/null +++ b/benchmark/src/memory_benchmarks/array_every.ts @@ -0,0 +1,9 @@ +export default function arrayEvery(): boolean { + const arr = []; + const n = 10000; + for (let i = 0; i < n; i++) { + arr[i] = i; + } + const isSmallerN = arr.every(e => e < n); + return isSmallerN; +} diff --git a/benchmark/src/memory_benchmarks/array_push.ts b/benchmark/src/memory_benchmarks/array_push.ts new file mode 100644 index 000000000..15c0a7dd1 --- /dev/null +++ b/benchmark/src/memory_benchmarks/array_push.ts @@ -0,0 +1,8 @@ +export default function arrayPush(): number[] { + const n = 10000; + const numberList: number[] = []; + for (let i = 0; i < n; i++) { + numberList.push(i * i); + } + return numberList; +} diff --git a/benchmark/src/memory_benchmarks/class_creation.ts b/benchmark/src/memory_benchmarks/class_creation.ts new file mode 100644 index 000000000..aaa57742f --- /dev/null +++ b/benchmark/src/memory_benchmarks/class_creation.ts @@ -0,0 +1,18 @@ +class A { + constructor(x: number) { + this.z = x * x; + } + + public z: number; +} + +class B extends A {} + +export default function classCreation(): B[] { + const arr1 = []; + const n = 10000; + for (let i = 0; i < n; i++) { + arr1.push(new B(i)); + } + return arr1; +} diff --git a/benchmark/src/memory_benchmarks/graph_cylce.ts b/benchmark/src/memory_benchmarks/graph_cylce.ts new file mode 100644 index 000000000..c78812dc5 --- /dev/null +++ b/benchmark/src/memory_benchmarks/graph_cylce.ts @@ -0,0 +1,52 @@ +type Graph = Map; + +function range(start: number, end: number): number[] { + if (start > end) return []; + return [start, ...range(start + 1, end)]; +} + +export default function detectCyleBenchmark(): boolean { + const n = 500; + const benchmarkGraph = new Map(); + // build a graph with n nodes and no cycle + for (let i = 0; i < n; i++) { + benchmarkGraph.set(i, range(i, n - 1)); + } + return detectCycle(benchmarkGraph); +} + +/** + * Detects cycles in an undirected graph + */ +function detectCycle(graph: Graph): boolean { + const visited: Map = new Map(); + + return [...graph.keys()].some(current => { + if (!visited.get(current)) { + return _detectCycle(graph, current, visited, undefined); + } + }); +} + +function _detectCycle(graph: Graph, current: T, visited: Map, parent: T | undefined): boolean { + visited.set(current, true); + + const neighbours = graph.get(current); + + if (!neighbours) { + throw Error("Err invalid graph format"); + } + + return neighbours.some(neighbour => { + if (!visited.get(neighbour)) { + // If an adjacent is not visited, then recur for that adjacent + return _detectCycle(graph, neighbour, visited, current); + } else if (neighbour !== parent) { + /* + * If an adjacent node is visited and not a parent of current vertex, + * then there is a cycle. + */ + return true; + } + }); +} diff --git a/benchmark/src/run.ts b/benchmark/src/run.ts new file mode 100644 index 000000000..c632efe39 --- /dev/null +++ b/benchmark/src/run.ts @@ -0,0 +1,61 @@ +import { runMemoryBenchmark, compareMemoryBenchmarks } from "./memory_benchmark"; +import { isMemoryBenchmarkResult, BenchmarkResult, MemoryBenchmarkResult, ComparisonInfo } from "./benchmark_types"; +import { json, loadBenchmarksFromDirectory, readFile } from "./util"; + +// CLI arguments +// arg[0]: output path for benchmark data +// arg[1]: path to baseline benchmark data (required to generate comparison) +// arg[2]: path to result markdown file (optional) +declare const arg: [string | undefined, string | undefined, string | undefined]; + +function benchmark(): void { + // Memory tests + let memoryBenchmarkNewResults: MemoryBenchmarkResult[] = []; + + const memoryBenchmarks = loadBenchmarksFromDirectory("memory_benchmarks"); + + memoryBenchmarkNewResults = memoryBenchmarks.map(runMemoryBenchmark); + + // run future benchmarks types here + + const newBenchmarkResults = [...memoryBenchmarkNewResults]; + + // Try to read the baseline benchmark result + let oldBenchmarkResults: BenchmarkResult[] = []; + if (arg[1]) { + const oldBenchmarkData = readFile(arg[1]); + oldBenchmarkResults = json.decode(oldBenchmarkData) as BenchmarkResult[]; + } + + // Compare results + const comparisonInfo = compareBenchmarks(oldBenchmarkResults, newBenchmarkResults); + + // Output comparison info + outputBenchmarkData(comparisonInfo, newBenchmarkResults); +} +benchmark(); + +function compareBenchmarks(oldResults: BenchmarkResult[], newResults: BenchmarkResult[]): ComparisonInfo { + const oldResultsMemory = oldResults.filter(isMemoryBenchmarkResult); + const newResultsMemory = newResults.filter(isMemoryBenchmarkResult); + + const memoryComparisonInfo = compareMemoryBenchmarks(oldResultsMemory, newResultsMemory); + + return { summary: memoryComparisonInfo.summary, text: memoryComparisonInfo.text }; +} + +function outputBenchmarkData(comparisonInfo: { summary: string; text: string }, newResults: BenchmarkResult[]): void { + if (!arg[2]) { + // Output to stdout as json by default, this is used by the CI to retrieve the info + print(json.encode(comparisonInfo)); + } else { + // Output to file as markdown if arg[2] is set, this is useful for local development + const markdownDataFile = io.open(arg[2], "w+")[0]!; + markdownDataFile.write(comparisonInfo.summary + comparisonInfo.text); + } + // Output benchmark results to json + if (arg[0]) { + const jsonDataFile = io.open(arg[0], "w+")[0]!; + jsonDataFile.write(json.encode(newResults)); + } +} diff --git a/benchmark/src/util.ts b/benchmark/src/util.ts new file mode 100644 index 000000000..d1c9fcc1d --- /dev/null +++ b/benchmark/src/util.ts @@ -0,0 +1,71 @@ +import { BenchmarkFunction } from "./benchmark_types"; + +export function toFixed(num: number, decimalPlaces = 0): string { + return string.format(`%.${decimalPlaces}f`, num); +} + +export function calculatePercentageChange(oldValue: number, newValue: number): number { + return (newValue / oldValue) * 100 - 100; +} + +// @ts-ignore +export const isWindows = package.config.startsWith("\\"); + +export const json: { + decode: (this: void, str: string) => {}; + encode: (this: void, val: any) => string; +} = require("json"); + +export function readFile(path: string): string { + const [fileHandle] = io.open(path, "rb"); + + if (!fileHandle) { + throw Error(`Can't open file ${path}`); + } + + const fileContent = readAll(fileHandle); + fileHandle.close(); + + return fileContent; +} + +export function readAll(file: LuaFile): string { + const content = file.read(_VERSION === "Lua 5.3" ? "a" : ("*a" as any)) as [string | undefined]; + + if (content[0]) { + return content[0]; + } + throw Error(`Can't readAll for file ${file}`); +} + +export function readDir(dir: string): string[] { + const findHandle = io.popen(isWindows ? `dir /A-D /B ${dir}` : `find '${dir}' -maxdepth 1 -type f`); + const findResult = readAll(findHandle); + + if (!findHandle.close()) { + throw Error(`readDir popen failed for dir ${dir} see stdout for more information.`); + } + + let files = findResult.split("\n"); + if (isWindows) { + // on windows we need to append the directory path + // on unix this is done by find automatically + files = files.map(f => `${dir}/${f}`); + } else { + // strip leading "./" on unix + files = files.map(f => (f.startsWith(".") && f[1] === "/" ? f.substr(2) : f)); + } + return files.filter(p => p !== ""); +} + +export function loadBenchmarksFromDirectory(benchmarkDir: string): BenchmarkFunction[] { + const benchmarkFiles = readDir(benchmarkDir); + + return benchmarkFiles.map(f => { + // replace slashes with dots + let dotPath = string.gsub(f, "%/", ".")[0]; + // remove extension + dotPath = string.gsub(dotPath, ".lua", "")[0]; + return require(dotPath).default as BenchmarkFunction; + }); +} diff --git a/benchmark/tsconfig.53.json b/benchmark/tsconfig.53.json new file mode 100644 index 000000000..0a7096234 --- /dev/null +++ b/benchmark/tsconfig.53.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": ["lua-types/5.3"] + }, + "tstl": { + "luaTarget": "5.3" + } +} diff --git a/benchmark/tsconfig.jit.json b/benchmark/tsconfig.jit.json new file mode 100644 index 000000000..1fe18fa2c --- /dev/null +++ b/benchmark/tsconfig.jit.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": ["lua-types/jit"] + }, + "tstl": { + "luaTarget": "JIT" + } +} diff --git a/benchmark/tsconfig.json b/benchmark/tsconfig.json new file mode 100644 index 000000000..aefc13ae9 --- /dev/null +++ b/benchmark/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["esnext"], + // Dev types are JIT + "types": ["lua-types/jit"], + "moduleResolution": "node", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true + }, + "include": ["src"] +} diff --git a/package-lock.json b/package-lock.json index f3d427049..e17d11a90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3798,6 +3798,12 @@ "@sinonjs/commons": "^1.7.0" } }, + "lua-types": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/lua-types/-/lua-types-2.8.0.tgz", + "integrity": "sha512-FJY32giHIqD/XW1XGkJnl8XotXIJsJ2M42fj9A2UudttWA6orJioToW1OpgPdayTr+S1/oTO7i+hfBY3UVG8Fg==", + "dev": true + }, "make-dir": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.2.tgz", diff --git a/package.json b/package.json index 6651b3109..3008b5b67 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "javascript-stringify": "^2.0.1", "jest": "^25.1.0", "jest-circus": "^25.1.0", + "lua-types": "^2.8.0", "prettier": "^1.19.1", "ts-jest": "^25.2.1", "ts-node": "^8.6.2"