Skip to content

Commit 11a9e40

Browse files
GlassBricksGlassBricks
committed
Merge branch 'runtime-benchmark' into optimize-array-operations
2 parents 939a134 + 50cf286 commit 11a9e40

4 files changed

Lines changed: 103 additions & 87 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 77 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -39,80 +39,80 @@ jobs:
3939
- if: matrix.os == 'ubuntu-latest'
4040
uses: codecov/codecov-action@v1
4141

42-
# benchmark:
43-
# name: Benchmark
44-
# runs-on: ubuntu-latest
45-
# steps:
46-
# - name: Lua Install
47-
# run: sudo apt-get install lua5.3 luajit
48-
# - name: Glow Install
49-
# run: brew install glow
50-
# # Checkout master & commit
51-
# - name: Checkout master
52-
# uses: actions/checkout@v2
53-
# with:
54-
# ref: master
55-
# path: master
56-
# - name: Checkout commit
57-
# uses: actions/checkout@v2
58-
# with:
59-
# path: commit
60-
# - name: Use Node.js 12.13.1
61-
# uses: actions/setup-node@v1
62-
# with:
63-
# node-version: 12.13.1
64-
# # NPM
65-
# - name: NPM master
66-
# run: npm ci && npm run build
67-
# working-directory: master
68-
# - name: NPM commit
69-
# run: npm ci && npm run build
70-
# working-directory: commit
71-
# # Benchmark directory setup
72-
# - name: Ensure benchmark data dir exists
73-
# run: mkdir -p ./benchmark/data
74-
# working-directory: commit
75-
# - name: Copy commit benchmark to master
76-
# run: rm -rf ./master/benchmark && cp -rf ./commit/benchmark ./master/benchmark
77-
# # Run master benchmark first and output to commit benchmark data
78-
# - name: Build benchmark Lua 5.3 master
79-
# run: node ../../commit/dist/tstl.js -p tsconfig.53.json
80-
# working-directory: master/benchmark
81-
# - name: Run benchmark Lua 5.3 master
82-
# id: benchmark-lua-master
83-
# run: lua5.3 -- run.lua ../../../commit/benchmark/data/benchmark_master_53.json
84-
# working-directory: master/benchmark/dist
85-
# - name: Build benchmark LuaJIT master
86-
# run: node ../../commit/dist/tstl.js -p tsconfig.jit.json
87-
# working-directory: master/benchmark
88-
# - name: Run benchmark LuaJIT master
89-
# id: benchmark-jit-master
90-
# run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_master_jit.json
91-
# working-directory: master/benchmark/dist
92-
# # Run commit benchmark and compare with master
93-
# - name: Build benchmark Lua 5.3 commit
94-
# run: node ../../commit/dist/tstl.js -p tsconfig.53.json
95-
# working-directory: commit/benchmark
96-
# - name: Run benchmark Lua 5.3 commit
97-
# id: benchmark-lua-commit
98-
# run: lua5.3 -- run.lua ../data/benchmark_master_vs_commit_53.json ../data/benchmark_master_53.json
99-
# working-directory: commit/benchmark/dist
100-
# - name: Build benchmark LuaJIT commit
101-
# run: node ../../commit/dist/tstl.js -p tsconfig.jit.json
102-
# working-directory: commit/benchmark
103-
# - name: Run benchmark LuaJIT commit
104-
# id: benchmark-jit-commit
105-
# run: luajit -- run.lua ../data/benchmark_master_vs_commit_jit.json ../data/benchmark_master_jit.json
106-
# working-directory: commit/benchmark/dist
107-
# - name: Combine benchmark results
108-
# id: script-combine-results
109-
# uses: actions/github-script@v3
110-
# with:
111-
# benchmark-result-path-lua: commit/benchmark/data/benchmark_master_vs_commit_53.json
112-
# benchmark-result-path-jit: commit/benchmark/data/benchmark_master_vs_commit_jit.json
113-
# result-encoding: string
114-
# script: |
115-
# const createBenchmarkCheck = require(`${process.env.GITHUB_WORKSPACE}/commit/.github/scripts/create_benchmark_check.js`);
116-
# return createBenchmarkCheck({ github, context, core });
117-
# - name: Benchmark results
118-
# run: echo "${{steps.script-combine-results.outputs.result}}" | glow -s dark -w 120 -
42+
benchmark:
43+
name: Benchmark
44+
runs-on: ubuntu-latest
45+
steps:
46+
- name: Lua Install
47+
run: sudo apt-get install lua5.3 luajit
48+
- name: Glow Install
49+
run: brew install glow
50+
# Checkout master & commit
51+
- name: Checkout master
52+
uses: actions/checkout@v2
53+
with:
54+
ref: master
55+
path: master
56+
- name: Checkout commit
57+
uses: actions/checkout@v2
58+
with:
59+
path: commit
60+
- name: Use Node.js 12.13.1
61+
uses: actions/setup-node@v1
62+
with:
63+
node-version: 12.13.1
64+
# NPM
65+
- name: NPM master
66+
run: npm ci && npm run build
67+
working-directory: master
68+
- name: NPM commit
69+
run: npm ci && npm run build
70+
working-directory: commit
71+
# Benchmark directory setup
72+
- name: Ensure benchmark data dir exists
73+
run: mkdir -p ./benchmark/data
74+
working-directory: commit
75+
- name: Copy commit benchmark to master
76+
run: rm -rf ./master/benchmark && cp -rf ./commit/benchmark ./master/benchmark
77+
# Run master benchmark first and output to commit benchmark data
78+
- name: Build benchmark Lua 5.3 master
79+
run: node ../../commit/dist/tstl.js -p tsconfig.53.json
80+
working-directory: master/benchmark
81+
- name: Run benchmark Lua 5.3 master
82+
id: benchmark-lua-master
83+
run: lua5.3 -- run.lua ../../../commit/benchmark/data/benchmark_master_53.json
84+
working-directory: master/benchmark/dist
85+
- name: Build benchmark LuaJIT master
86+
run: node ../../commit/dist/tstl.js -p tsconfig.jit.json
87+
working-directory: master/benchmark
88+
- name: Run benchmark LuaJIT master
89+
id: benchmark-jit-master
90+
run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_master_jit.json
91+
working-directory: master/benchmark/dist
92+
# Run commit benchmark and compare with master
93+
- name: Build benchmark Lua 5.3 commit
94+
run: node ../../commit/dist/tstl.js -p tsconfig.53.json
95+
working-directory: commit/benchmark
96+
- name: Run benchmark Lua 5.3 commit
97+
id: benchmark-lua-commit
98+
run: lua5.3 -- run.lua ../data/benchmark_master_vs_commit_53.json ../data/benchmark_master_53.json
99+
working-directory: commit/benchmark/dist
100+
- name: Build benchmark LuaJIT commit
101+
run: node ../../commit/dist/tstl.js -p tsconfig.jit.json
102+
working-directory: commit/benchmark
103+
- name: Run benchmark LuaJIT commit
104+
id: benchmark-jit-commit
105+
run: luajit -- run.lua ../data/benchmark_master_vs_commit_jit.json ../data/benchmark_master_jit.json
106+
working-directory: commit/benchmark/dist
107+
- name: Combine benchmark results
108+
id: script-combine-results
109+
uses: actions/github-script@v3
110+
with:
111+
benchmark-result-path-lua: commit/benchmark/data/benchmark_master_vs_commit_53.json
112+
benchmark-result-path-jit: commit/benchmark/data/benchmark_master_vs_commit_jit.json
113+
result-encoding: string
114+
script: |
115+
const createBenchmarkCheck = require(`${process.env.GITHUB_WORKSPACE}/commit/.github/scripts/create_benchmark_check.js`);
116+
return createBenchmarkCheck({ github, context, core });
117+
- name: Benchmark results
118+
run: echo "${{steps.script-combine-results.outputs.result}}" | glow -s dark -w 120 -

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,3 @@ yarn.lock
99

1010
benchmark/data/*
1111
benchmark/dist/*
12-
!benchmark/dist/json.lua

‎benchmark/src/benchmark_types.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ export function isMemoryBenchmarkResult(result: BenchmarkResult): result is Memo
2424

2525
export interface RuntimeBenchmarkResult {
2626
kind: BenchmarkKind.Runtime;
27-
// seconds, as returned by os.clock()
2827
time: number;
2928
benchmarkName: string;
3029
}

‎benchmark/src/run.ts‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
ComparisonInfo,
77
RuntimeBenchmarkResult,
88
isRuntimeBenchmarkResult,
9+
BenchmarkKind,
910
} from "./benchmark_types";
1011
import { runRuntimeBenchmark, compareRuntimeBenchmarks } from "./runtime_benchmark";
1112
import { json, loadBenchmarksFromDirectory, readFile } from "./util";
@@ -37,6 +38,8 @@ function benchmark(): void {
3738
}
3839

3940
// Output comparison info
41+
oldBenchmarkResults.sort(sortByName);
42+
newBenchmarkResults.sort(sortByName);
4043
outputBenchmarkData(oldBenchmarkResults, newBenchmarkResults);
4144
}
4245
benchmark();
@@ -47,23 +50,38 @@ function sortByName({ benchmarkName: a }: BenchmarkResult, { benchmarkName: b }:
4750
return 0;
4851
}
4952

50-
function compareBenchmarks(oldResults: BenchmarkResult[], newResults: BenchmarkResult[]): ComparisonInfo {
51-
const oldResultsMemory = oldResults.filter(isMemoryBenchmarkResult).sort(sortByName);
52-
const newResultsMemory = newResults.filter(isMemoryBenchmarkResult).sort(sortByName);
53+
function compareBenchmarks(
54+
oldResults: BenchmarkResult[],
55+
newResults: BenchmarkResult[]
56+
): Record<BenchmarkKind, ComparisonInfo> {
57+
const oldResultsMemory = oldResults.filter(isMemoryBenchmarkResult);
58+
const newResultsMemory = newResults.filter(isMemoryBenchmarkResult);
5359

5460
const memoryComparisonInfo = compareMemoryBenchmarks(oldResultsMemory, newResultsMemory);
5561

56-
const oldResultsRuntime = oldResults.filter(isRuntimeBenchmarkResult).sort(sortByName);
57-
const newResultsRuntime = newResults.filter(isRuntimeBenchmarkResult).sort(sortByName);
62+
const oldResultsRuntime = oldResults.filter(isRuntimeBenchmarkResult);
63+
const newResultsRuntime = newResults.filter(isRuntimeBenchmarkResult);
5864

5965
const runtimeComparisonInfo = compareRuntimeBenchmarks(oldResultsRuntime, newResultsRuntime);
6066

6167
return {
62-
summary: memoryComparisonInfo.summary + "\n" + runtimeComparisonInfo.summary,
63-
text: memoryComparisonInfo.text + "\n" + runtimeComparisonInfo.text,
68+
[BenchmarkKind.Memory]: memoryComparisonInfo,
69+
[BenchmarkKind.Runtime]: runtimeComparisonInfo,
6470
};
6571
}
6672

73+
function formatComparisonMarkdownFile(comparisonInfo: Record<BenchmarkKind, ComparisonInfo>): string {
74+
let result = "";
75+
const benchmarkKinds = [BenchmarkKind.Memory, BenchmarkKind.Runtime];
76+
for (const kind of benchmarkKinds) {
77+
result += comparisonInfo[kind].summary + "\n";
78+
}
79+
for (const kind of benchmarkKinds) {
80+
result += comparisonInfo[kind].text + "\n";
81+
}
82+
return result;
83+
}
84+
6785
function outputBenchmarkData(oldResults: BenchmarkResult[], newResults: BenchmarkResult[]): void {
6886
// Output benchmark results to json
6987
if (arg[0]) {
@@ -82,6 +100,6 @@ function outputBenchmarkData(oldResults: BenchmarkResult[], newResults: Benchmar
82100
// Compare results
83101
const comparisonInfo = compareBenchmarks(oldResults, newResults);
84102
const markdownDataFile = io.open(arg[2], "w+")[0]!;
85-
markdownDataFile.write(comparisonInfo.summary + comparisonInfo.text);
103+
markdownDataFile.write(formatComparisonMarkdownFile(comparisonInfo));
86104
}
87105
}

0 commit comments

Comments
 (0)