diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7cc3ff3b..ba6ce737 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,12 @@ jobs: - name: "🔨 Build & test" run: >- nix develop --accept-flake-config --allow-import-from-derivation --command cabal test all --test-show-details=direct + # Deterministic LuaJIT counters over freshly linked bench artifacts + # (FNEW census, trace abort/blacklist state). Wall-clock benchmarks + # stay local — shared runners are too noisy; see bench/README.md. + - name: "📊 Bench counters" + run: >- + nix develop --accept-flake-config --allow-import-from-derivation --command ./bench/ci format: runs-on: ubuntu-latest steps: diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 00000000..ba65b13a --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..5a1fd2da --- /dev/null +++ b/bench/README.md @@ -0,0 +1,78 @@ +# Benchmark harness + +Measures the performance of generated Lua and meters how LuaJIT's tracing +JIT treats it. Two kinds of output with two different purposes: + +- **Wall-clock timings** (`./bench/run`) — local numbers for defending an + optimization win. Too noisy for CI. +- **Deterministic LuaJIT counters** (`./bench/ci`) — byte-stable reports + that CI diffs against the committed oracles in `goldens/`, so a codegen + change that adds closure allocations or blacklists a loop shows up as a + reviewable diff. + +## Layout + +- `micro/` — hand-written Lua pairs: the shape the backend currently + generates for a pattern (`current`) next to the idiomatic Lua it stands + for (`ideal`). These are fixed reference points; they do not change when + codegen changes. +- `macro/` — specs driving real linked modules. Each spec names a `Bench.*` + PureScript module (sources in `test/ps/src/Bench/`, linked by + `./bench/link` into `_build/`), how to drive its export hot, and an + `ideal` hand-written equivalent. +- `tools/` — the runners and meters. `fnew_census.lua` and + `trace_report.lua` require LuaJIT (`jit.util`, `jit.attach`); the timing + runners work under both PUC Lua and LuaJIT. +- `goldens/` — committed counter reports; the CI oracle. + +## Usage + +```bash +./bench/run # all wall-clock benchmarks, all runtimes +lua bench/tools/run_micro.lua bench/micro/curried_apply.lua # one bench +luajit bench/tools/run_macro.lua bench/macro/array_foldl.lua 5e6 # custom n +./bench/ci # regenerate counters, verify against goldens/ +./bench/ci --accept # rewrite goldens/ after a deliberate change +``` + +Timings use `os.clock()` — CPU time, not wall time. That is deliberate: +the benchmarks are pure computation, and CPU time ignores scheduler noise. +It would under-report I/O, so do not reuse the timing helpers for anything +that waits. For quieter numbers, pin the process to a core (`taskset -c N +./bench/run`) and use the `performance` CPU governor; leave ASLR alone — +disabling it trades a real security property for nearly nothing. + +## What the counters mean + +`fnew_census` statically counts `FNEW` bytecodes (closure creation) in a +linked artifact without running it, split by where the instruction lives: +the **main chunk** runs once at load time, so its FNEWs are init cost; a +**function body** runs per call, so its FNEWs are steady-state allocation — +and each one aborts LuaJIT trace recording (`NYI: bytecode FNEW`), which is +what keeps curried hot code interpreted. The census is a pure function of +the artifact, hence byte-stable. + +`trace_report` runs a macro spec hot and reports (a) the *set* of distinct +trace-abort sites with reasons and (b) the end state of loop and +function-entry bytecodes: LuaJIT rewrites an opcode to its `J*` form when +it installs a trace there and to its `I*` form when it blacklists the spot. +Raw abort *counts* are not reported: the retry-penalty step that leads to a +blacklist draws on an entropy-seeded PRNG, so counts jitter across runs +while the final opcode state and the abort-site set do not. Blacklisting is +never logged by `-jv`/`-jdump`; the post-hoc opcode read is the only stable +way to observe it. + +Both reports record the LuaJIT version (`runtime:` header line): the abort +reasons, the NYI set, and the opcode families are properties of a specific +LuaJIT snapshot, so a toolchain bump that moves the counters shows up in +the golden diff as an attributable header change, not a mystery regression. + +Two caveats about golden stability. The trace reports pin source *lines* of +both the linked artifact and the macro spec file itself, so any edit to +`bench/macro/*.lua` — comments included — legitimately moves the goldens; +rerun `./bench/ci --accept` and review the diff. And one residual +nondeterminism channel exists: LuaJIT's hot-counters live in a small hashed +table keyed by bytecode address, so a rare cross-process aliasing change +can alter trace formation order. `./bench/ci` generates every report twice +and compares, precisely so that this manifests as a distinct "reports +differ between runs" failure rather than a confusing golden mismatch. diff --git a/bench/ci b/bench/ci new file mode 100755 index 00000000..8a634d13 --- /dev/null +++ b/bench/ci @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Deterministic LuaJIT counters: a static FNEW census per linked artifact +# and a trace abort/blacklist report per macro spec. Every report is +# generated twice and compared, proving byte-stability, then diffed against +# the committed oracles in bench/goldens/. Pass --accept after a deliberate +# codegen change to rewrite the oracles from the current output. +set -euo pipefail +cd "$(dirname "$0")/.." + +accept=0 +if [ "${1:-}" = "--accept" ]; then + accept=1 +fi + +./bench/link + +out=bench/_build/counters +rm -rf "$out" +mkdir -p "$out" + +for artifact in bench/_build/Bench.*.lua; do + name=$(basename "$artifact" .lua) + luajit bench/tools/fnew_census.lua "$artifact" >"$out/fnew_$name.txt" + luajit bench/tools/fnew_census.lua "$artifact" >"$out/second-run.txt" + cmp "$out/fnew_$name.txt" "$out/second-run.txt" +done + +for spec in bench/macro/*.lua; do + name=$(basename "$spec" .lua) + luajit bench/tools/trace_report.lua "$spec" >"$out/trace_$name.txt" + luajit bench/tools/trace_report.lua "$spec" >"$out/second-run.txt" + cmp "$out/trace_$name.txt" "$out/second-run.txt" +done +rm "$out/second-run.txt" + +if [ "$accept" = 1 ]; then + mkdir -p bench/goldens + rm -f bench/goldens/*.txt + cp "$out"/*.txt bench/goldens/ + echo "accepted $(find "$out" -name '*.txt' | wc -l) counter files into bench/goldens/" +else + diff -ru bench/goldens "$out" + echo "bench counters match goldens" +fi diff --git a/bench/goldens/fnew_Bench.ArrayFoldl.txt b/bench/goldens/fnew_Bench.ArrayFoldl.txt new file mode 100644 index 00000000..f7b79f65 --- /dev/null +++ b/bench/goldens/fnew_Bench.ArrayFoldl.txt @@ -0,0 +1,19 @@ +chunk: Bench.ArrayFoldl.lua +runtime: LuaJIT 2.1.1741730670 +main-chunk FNEW: 7 +function-body FNEW: 12 +total FNEW: 19 +prototypes: 20 +function-body FNEW sites: + Bench.ArrayFoldl.lua:3 + Bench.ArrayFoldl.lua:14 + Bench.ArrayFoldl.lua:13 + Bench.ArrayFoldl.lua:24 + Bench.ArrayFoldl.lua:23 + Bench.ArrayFoldl.lua:28 + Bench.ArrayFoldl.lua:28 + Bench.ArrayFoldl.lua:54 + Bench.ArrayFoldl.lua:53 + Bench.ArrayFoldl.lua:52 + Bench.ArrayFoldl.lua:63 + Bench.ArrayFoldl.lua:62 diff --git a/bench/goldens/fnew_Bench.BindChain.txt b/bench/goldens/fnew_Bench.BindChain.txt new file mode 100644 index 00000000..d814bb96 --- /dev/null +++ b/bench/goldens/fnew_Bench.BindChain.txt @@ -0,0 +1,11 @@ +chunk: Bench.BindChain.lua +runtime: LuaJIT 2.1.1741730670 +main-chunk FNEW: 4 +function-body FNEW: 4 +total FNEW: 8 +prototypes: 9 +function-body FNEW sites: + Bench.BindChain.lua:3 + Bench.BindChain.lua:19 + Bench.BindChain.lua:27 + Bench.BindChain.lua:26 diff --git a/bench/goldens/fnew_Bench.Fib.txt b/bench/goldens/fnew_Bench.Fib.txt new file mode 100644 index 00000000..ebe63662 --- /dev/null +++ b/bench/goldens/fnew_Bench.Fib.txt @@ -0,0 +1,9 @@ +chunk: Bench.Fib.lua +runtime: LuaJIT 2.1.1741730670 +main-chunk FNEW: 3 +function-body FNEW: 2 +total FNEW: 5 +prototypes: 6 +function-body FNEW sites: + Bench.Fib.lua:3 + Bench.Fib.lua:6 diff --git a/bench/goldens/trace_array_foldl.txt b/bench/goldens/trace_array_foldl.txt new file mode 100644 index 00000000..c88bec76 --- /dev/null +++ b/bench/goldens/trace_array_foldl.txt @@ -0,0 +1,18 @@ +spec: array_foldl +runtime: LuaJIT 2.1.1741730670 +workload: n=5000000 reps=2 result=12500002500000 +aborts (distinct site -- reason): + Bench.ArrayFoldl.lua:3 -- NYI: bytecode FNEW + Bench.ArrayFoldl.lua:62 -- NYI: bytecode FNEW +bytecode end state (J*=compiled, I*=blacklisted): + Bench.ArrayFoldl.lua:21 IFORL + Bench.ArrayFoldl.lua:3 IFUNCF + Bench.ArrayFoldl.lua:3 JFUNCF + Bench.ArrayFoldl.lua:35 JLOOP + Bench.ArrayFoldl.lua:59 IFUNCF + Bench.ArrayFoldl.lua:60 IFUNCF + array_foldl.lua:13 JFORI + array_foldl.lua:13 JFORL + array_foldl.lua:17 JFORI + array_foldl.lua:17 JFORL +counts: aborts=2 compiled=6 blacklisted=4 diff --git a/bench/goldens/trace_bind_chain.txt b/bench/goldens/trace_bind_chain.txt new file mode 100644 index 00000000..6c752539 --- /dev/null +++ b/bench/goldens/trace_bind_chain.txt @@ -0,0 +1,19 @@ +spec: bind_chain +runtime: LuaJIT 2.1.1741730670 +workload: n=1000000 reps=2 result=3000000 +aborts (distinct site -- reason): + Bench.BindChain.lua:19 -- NYI: bytecode FNEW + Bench.BindChain.lua:3 -- NYI: bytecode FNEW +bytecode end state (J*=compiled, I*=blacklisted): + Bench.BindChain.lua:22 IFUNCF + Bench.BindChain.lua:23 IFUNCF + Bench.BindChain.lua:24 IFUNCF + Bench.BindChain.lua:3 IFUNCF + Bench.BindChain.lua:3 JFUNCF + Bench.BindChain.lua:5 JFUNCF + Bench.BindChain.lua:8 IFUNCF + Bench.BindChain.lua:9 IFUNCF + bind_chain.lua:10 IFORL + bind_chain.lua:17 JFORI + bind_chain.lua:17 JFORL +counts: aborts=2 compiled=4 blacklisted=7 diff --git a/bench/goldens/trace_fibonacci.txt b/bench/goldens/trace_fibonacci.txt new file mode 100644 index 00000000..488171ab --- /dev/null +++ b/bench/goldens/trace_fibonacci.txt @@ -0,0 +1,15 @@ +spec: fibonacci +runtime: LuaJIT 2.1.1741730670 +workload: n=30 reps=2 result=832040 +aborts (distinct site -- reason): + Bench.Fib.lua:3 -- NYI: bytecode FNEW + Bench.Fib.lua:6 -- NYI: bytecode FNEW + fibonacci.lua:11 -- call unroll limit reached +bytecode end state (J*=compiled, I*=blacklisted): + Bench.Fib.lua:3 IFUNCF + Bench.Fib.lua:3 JFUNCF + Bench.Fib.lua:6 IFUNCF + Bench.Fib.lua:6 JFUNCF + Bench.Fib.lua:8 JFUNCF + fibonacci.lua:11 JFUNCF +counts: aborts=3 compiled=4 blacklisted=2 diff --git a/bench/link b/bench/link new file mode 100755 index 00000000..61c8adba --- /dev/null +++ b/bench/link @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Links the Bench.* PureScript modules into standalone Lua artifacts under +# bench/_build/, building pslua and the CoreFn output first. The module list +# is derived from test/ps/src/Bench/*.purs, and stale artifacts are removed, +# so bench/_build/Bench.*.lua always mirrors the current sources. +set -euo pipefail +cd "$(dirname "$0")/.." +root=$PWD + +(cd test/ps && spago build) + +cabal build -v0 exe:pslua +pslua=$(cabal list-bin pslua) + +mkdir -p bench/_build +rm -f bench/_build/Bench.*.lua +# CoreFn module paths are relative to test/ps (where spago runs), and pslua +# resolves foreign files against them, so it has to run from there too. +for src in test/ps/src/Bench/*.purs; do + module="Bench.$(basename "$src" .purs)" + ( + cd test/ps + "$pslua" \ + --ps-output output \ + --foreign-path foreign \ + --entry "$module" \ + --lua-output-file "$root/bench/_build/$module.lua" + ) +done diff --git a/bench/macro/array_foldl.lua b/bench/macro/array_foldl.lua new file mode 100644 index 00000000..b3fbc671 --- /dev/null +++ b/bench/macro/array_foldl.lua @@ -0,0 +1,22 @@ +-- Foldable `foldl` over an Array with a curried step: the FFI fold loop is +-- a plain Lua `for`, but compiling it means inlining the curried callee, so +-- the tracer aborts on FNEW and blacklists the loop. The ideal variant does +-- the same work — build the array, fold it — with an uncurried step. +return { + artifact = "Bench.ArrayFoldl", + n = 5e6, + drive = function(mod, n) + return mod.run(n) + end, + ideal = function(n) + local t = {} + for i = 1, n do + t[i] = i + end + local acc = 0 + for i = 1, #t do + acc = acc + t[i] + end + return acc + end, +} diff --git a/bench/macro/bind_chain.lua b/bench/macro/bind_chain.lua new file mode 100644 index 00000000..2fe91d37 --- /dev/null +++ b/bench/macro/bind_chain.lua @@ -0,0 +1,22 @@ +-- A three-step Maybe bind chain per iteration: each step goes through the +-- Bind dictionary and allocates a Just constructor. The driver loop is the +-- hot loop here, so this also exercises what happens to a caller loop whose +-- callee cannot be traced. +return { + artifact = "Bench.BindChain", + n = 1e6, + drive = function(mod, n) + local acc = 0 + for _ = 1, n do + acc = mod.run(acc) + end + return acc + end, + ideal = function(n) + local acc = 0 + for _ = 1, n do + acc = acc + 3 + end + return acc + end, +} diff --git a/bench/macro/fibonacci.lua b/bench/macro/fibonacci.lua new file mode 100644 index 00000000..717d4c35 --- /dev/null +++ b/bench/macro/fibonacci.lua @@ -0,0 +1,22 @@ +-- Recursion with typeclass arithmetic: the same `fib` as the +-- Golden.Fibonacci.Test golden, linked as a module so the driver can call +-- it without going through stdout. +return { + artifact = "Bench.Fib", + n = 30, + drive = function(mod, n) + return mod.fib(n) + end, + ideal = function(n) + local function fib(v) + if v == 0 then + return 0 + end + if v == 1 then + return 1 + end + return fib(v - 1) + fib(v - 2) + end + return fib(n) + end, +} diff --git a/bench/micro/ctor_match.lua b/bench/micro/ctor_match.lua new file mode 100644 index 00000000..421b21a5 --- /dev/null +++ b/bench/micro/ctor_match.lua @@ -0,0 +1,34 @@ +-- Constructor allocation plus tag match: the generated encoding keeps the +-- tag and the payload in the table's hash part, keyed by long strings, +-- versus an array-part encoding with a small-integer tag. +return { + n = 2e6, + variants = { + { + name = "current", + fn = function(n) + local acc = 0 + for i = 1, n do + local m = { ["$ctor"] = "Data.Maybe∷Maybe.Just", value0 = i } + if "Data.Maybe∷Maybe.Just" == m["$ctor"] then + acc = acc + m.value0 + end + end + return acc + end, + }, + { + name = "ideal", + fn = function(n) + local acc = 0 + for i = 1, n do + local m = { 1, i } + if 1 == m[1] then + acc = acc + m[2] + end + end + return acc + end, + }, + }, +} diff --git a/bench/micro/curried_apply.lua b/bench/micro/curried_apply.lua new file mode 100644 index 00000000..20d18add --- /dev/null +++ b/bench/micro/curried_apply.lua @@ -0,0 +1,37 @@ +-- Curried application in a hot loop: every partial application allocates a +-- closure (an FNEW bytecode under LuaJIT, which aborts trace recording), so +-- the loop never gets JIT-compiled. The uncurried variant compiles to a +-- single trace. +return { + n = 2e7, + variants = { + { + name = "current", + fn = function(n) + local add = function(x) + return function(y) + return x + y + end + end + local acc = 0 + for i = 1, n do + acc = add(acc)(i) + end + return acc + end, + }, + { + name = "ideal", + fn = function(n) + local add = function(x, y) + return x + y + end + local acc = 0 + for i = 1, n do + acc = add(acc, i) + end + return acc + end, + }, + }, +} diff --git a/bench/micro/dict_compare.lua b/bench/micro/dict_compare.lua new file mode 100644 index 00000000..0efb1818 --- /dev/null +++ b/bench/micro/dict_compare.lua @@ -0,0 +1,60 @@ +-- Dictionary-driven comparison: `greaterThanOrEq ordInt` goes through a +-- dictionary table, a curried three-level call chain, an Ordering +-- constructor allocation and a tag match — versus the native `>=` the +-- whole dance stands for. +local LT = { ["$ctor"] = "Data.Ordering∷Ordering.LT" } +local EQ = { ["$ctor"] = "Data.Ordering∷Ordering.EQ" } +local GT = { ["$ctor"] = "Data.Ordering∷Ordering.GT" } +local M = {} +M.Data_Ord_ordInt = { + compare = function(x) + return function(y) + if x < y then + return LT + elseif x == y then + return EQ + else + return GT + end + end + end, +} +M.Data_Ord_greaterThanOrEq = function(dictOrd) + return function(a1) + return function(a2) + return dictOrd.compare(a1)(a2)["$ctor"] ~= "Data.Ordering∷Ordering.LT" + end + end +end + +return { + n = 5e6, + variants = { + { + name = "current", + fn = function(n) + local half = n / 2 + local count = 0 + for i = 1, n do + if M.Data_Ord_greaterThanOrEq(M.Data_Ord_ordInt)(i)(half) then + count = count + 1 + end + end + return count + end, + }, + { + name = "ideal", + fn = function(n) + local half = n / 2 + local count = 0 + for i = 1, n do + if i >= half then + count = count + 1 + end + end + return count + end, + }, + }, +} diff --git a/bench/micro/field_access.lua b/bench/micro/field_access.lua new file mode 100644 index 00000000..15d3ab54 --- /dev/null +++ b/bench/micro/field_access.lua @@ -0,0 +1,39 @@ +-- Reaching a function through module-table fields on every call, the way +-- linked code does (M.Data_Semiring_foreign.intAdd), versus hoisting it +-- into a local once. Both variants call uncurried so the difference is the +-- field lookups alone. +return { + n = 1e7, + variants = { + { + name = "current", + fn = function(n) + local M = { + Data_Semiring_foreign = { + intAdd = function(x, y) + return x + y + end, + }, + } + local acc = 0 + for i = 1, n do + acc = M.Data_Semiring_foreign.intAdd(acc, i) + end + return acc + end, + }, + { + name = "ideal", + fn = function(n) + local intAdd = function(x, y) + return x + y + end + local acc = 0 + for i = 1, n do + acc = intAdd(acc, i) + end + return acc + end, + }, + }, +} diff --git a/bench/run b/bench/run new file mode 100755 index 00000000..2f49b4bc --- /dev/null +++ b/bench/run @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Wall-clock benchmarks: micro codegen patterns and linked macro modules +# under PUC Lua and LuaJIT (with and without the JIT). Timings are for local +# use — too noisy for CI, which gates on ./bench/ci instead. For quieter +# numbers pin the process (e.g. `taskset -c N ./bench/run`) and use the +# `performance` CPU governor. +set -euo pipefail +cd "$(dirname "$0")/.." + +./bench/link + +echo "== micro: codegen patterns ==" +for bench in bench/micro/*.lua; do + lua bench/tools/run_micro.lua "$bench" + luajit bench/tools/run_micro.lua "$bench" + luajit -joff bench/tools/run_micro.lua "$bench" +done + +echo +echo "== macro: linked modules ==" +for spec in bench/macro/*.lua; do + lua bench/tools/run_macro.lua "$spec" + luajit bench/tools/run_macro.lua "$spec" + luajit -joff bench/tools/run_macro.lua "$spec" +done diff --git a/bench/tools/bc_lib.lua b/bench/tools/bc_lib.lua new file mode 100644 index 00000000..6b1307c3 --- /dev/null +++ b/bench/tools/bc_lib.lua @@ -0,0 +1,50 @@ +-- Shared LuaJIT bytecode introspection for the counter tools: opcode names +-- from jit.vmdef and a prototype walk shaped like jit/bc.lua's — iterate +-- jit.util.funcbc until nil, recurse into child prototypes via negative +-- jit.util.funck indices. The walk starts at pc 0, the FUNC* header slot, +-- so visitors see function-entry opcodes (their J*/I* rewrites are how +-- trace installation and blacklisting are observed). +local jutil = require("jit.util") +local vmdef = require("jit.vmdef") +local band = require("bit").band + +local M = {} + +-- vmdef.bcnames is a string of fixed-width 6-character cells, one per +-- opcode number. +function M.bcname(op) + return (vmdef.bcnames:sub(op * 6 + 1, op * 6 + 6):gsub("%s+$", "")) +end + +function M.basename(source) + return (source:gsub("^@", ""):gsub("^.*/", "")) +end + +-- Depth-first over proto and its children, calling visit(proto, pc, op, +-- depth) for every bytecode. Depth 0 is the outermost (main-chunk) +-- prototype; everything below it is a function body. +function M.walk(proto, visit, depth) + depth = depth or 0 + local pc = 0 + while true do + local ins = jutil.funcbc(proto, pc) + if not ins then + break + end + visit(proto, pc, M.bcname(band(ins, 0xff)), depth) + pc = pc + 1 + end + if jutil.funcinfo(proto).children then + for n = -1, -1e9, -1 do + local k = jutil.funck(proto, n) + if not k then + break + end + if type(k) == "proto" then + M.walk(k, visit, depth + 1) + end + end + end +end + +return M diff --git a/bench/tools/bench_lib.lua b/bench/tools/bench_lib.lua new file mode 100644 index 00000000..0f47406d --- /dev/null +++ b/bench/tools/bench_lib.lua @@ -0,0 +1,63 @@ +-- Shared timing helpers for the wall-clock benchmark runners. +-- +-- os.clock() measures CPU time, not wall time. For these CPU-bound loops +-- that is the right metric (time the process spends scheduled out does not +-- pollute samples), but it would silently under-report anything I/O-bound. +-- luacheck: read globals jit +local M = {} + +-- Runs fn(n) once untimed (lets LuaJIT compile traces: a loop becomes a +-- tracing candidate after 56 iterations, a side exit after 10), then times +-- `samples` runs with the collector stopped, starting each sample from a +-- freshly collected heap. Reports the median so a stray outlier sample +-- cannot move the headline number. +function M.measure(fn, n, samples) + samples = samples or 5 + assert(samples >= 1, "samples must be >= 1") + local checksum = fn(n) + local times = {} + for s = 1, samples do + collectgarbage("collect") + collectgarbage("stop") + local t0 = os.clock() + fn(n) + local t1 = os.clock() + collectgarbage("restart") + times[s] = t1 - t0 + end + table.sort(times) + local mid = math.floor((#times + 1) / 2) + local median = times[mid] + if #times % 2 == 0 then + median = (times[mid] + times[mid + 1]) / 2 + end + return { + median = median, + min = times[1], + max = times[#times], + checksum = checksum, + } +end + +function M.runtime_tag() + if jit then + return jit.status() and "luajit" or "luajit-joff" + end + return (_VERSION:gsub("%s", ""):lower()) +end + +function M.report(bench_name, variant_name, n, r) + io.write(string.format( + "%-13s %-8s %-12s n=%-9.0f median=%8.4fs min=%8.4fs max=%8.4fs result=%s\n", + bench_name, + variant_name, + M.runtime_tag(), + n, + r.median, + r.min, + r.max, + tostring(r.checksum) + )) +end + +return M diff --git a/bench/tools/fnew_census.lua b/bench/tools/fnew_census.lua new file mode 100644 index 00000000..bf952c87 --- /dev/null +++ b/bench/tools/fnew_census.lua @@ -0,0 +1,50 @@ +-- Static FNEW census over a Lua chunk: counts closure-creation bytecodes +-- without executing anything, so the output is a pure function of the file +-- and the LuaJIT version (recorded in the report header). +-- +-- An FNEW in the main chunk runs once, at load time; an FNEW in a function +-- body runs on every call of that function, and aborts LuaJIT trace +-- recording (closure creation is NYI in the 2.1 tracer). The split is a +-- structural property of the prototype an instruction lives in: the main +-- chunk is the outermost prototype, everything reachable through child +-- prototypes is a function body. +-- +-- usage: luajit fnew_census.lua +-- luacheck: read globals jit +local jutil = require("jit.util") +local here = arg[0]:match("^(.*)/[^/]+$") or "." +local bc = dofile(here .. "/bc_lib.lua") + +local counts = { main = 0, body = 0, protos = 0 } +local body_sites = {} + +local function visit(proto, pc, op, depth) + if pc == 0 then + counts.protos = counts.protos + 1 + end + if op == "FNEW" then + if depth == 0 then + counts.main = counts.main + 1 + else + counts.body = counts.body + 1 + local info = jutil.funcinfo(proto, pc) + body_sites[#body_sites + 1] = + string.format("%s:%d", bc.basename(info.source), info.currentline) + end + end +end + +local path = assert(arg[1], "usage: fnew_census.lua ") +local chunk = assert(loadfile(path)) +bc.walk(chunk, visit) + +io.write("chunk: ", bc.basename("@" .. path), "\n") +io.write("runtime: ", jit.version, "\n") +io.write("main-chunk FNEW: ", counts.main, "\n") +io.write("function-body FNEW: ", counts.body, "\n") +io.write("total FNEW: ", counts.main + counts.body, "\n") +io.write("prototypes: ", counts.protos, "\n") +io.write("function-body FNEW sites:\n") +for _, site in ipairs(body_sites) do + io.write(" ", site, "\n") +end diff --git a/bench/tools/run_macro.lua b/bench/tools/run_macro.lua new file mode 100644 index 00000000..499e2b5d --- /dev/null +++ b/bench/tools/run_macro.lua @@ -0,0 +1,20 @@ +-- Times one macrobenchmark spec: the linked pslua artifact (loaded from +-- bench/_build/, produced by bench/link) against the spec's hand-written +-- idiomatic-Lua equivalent. +-- +-- usage: lua|luajit [-joff] run_macro.lua [n] [samples] +local here = arg[0]:match("^(.*)/[^/]+$") or "." +local lib = dofile(here .. "/bench_lib.lua") + +local spec_path = assert(arg[1], "usage: run_macro.lua [n] [samples]") +local spec_name = spec_path:match("([^/]+)%.lua$") +local spec = dofile(spec_path) +local mod = dofile(here .. "/../_build/" .. spec.artifact .. ".lua") +local n = tonumber(arg[2]) or spec.n +local samples = tonumber(arg[3]) + +local linked = function(k) + return spec.drive(mod, k) +end +lib.report(spec_name, "linked", n, lib.measure(linked, n, samples)) +lib.report(spec_name, "ideal", n, lib.measure(spec.ideal, n, samples)) diff --git a/bench/tools/run_micro.lua b/bench/tools/run_micro.lua new file mode 100644 index 00000000..e47e9d34 --- /dev/null +++ b/bench/tools/run_micro.lua @@ -0,0 +1,15 @@ +-- Times every variant of one microbenchmark under the current runtime. +-- +-- usage: lua|luajit [-joff] run_micro.lua [n] [samples] +local here = arg[0]:match("^(.*)/[^/]+$") or "." +local lib = dofile(here .. "/bench_lib.lua") + +local spec_path = assert(arg[1], "usage: run_micro.lua [n] [samples]") +local spec_name = spec_path:match("([^/]+)%.lua$") +local spec = dofile(spec_path) +local n = tonumber(arg[2]) or spec.n +local samples = tonumber(arg[3]) + +for _, variant in ipairs(spec.variants) do + lib.report(spec_name, variant.name, n, lib.measure(variant.fn, n, samples)) +end diff --git a/bench/tools/trace_report.lua b/bench/tools/trace_report.lua new file mode 100644 index 00000000..6a730509 --- /dev/null +++ b/bench/tools/trace_report.lua @@ -0,0 +1,148 @@ +-- Runs a macrobenchmark spec hot under the tracing JIT and reports two +-- deterministic signals per source location: +-- +-- * the set of distinct trace-abort sites (location + reason), collected +-- via jit.attach("trace"). A set, not a count: how many retries happen +-- before a spot is blacklisted depends on LuaJIT's entropy-seeded +-- penalty PRNG, so raw abort counts are not stable across runs. +-- +-- * the end state of loop and function-entry bytecodes after the run, +-- read back with jit.util.funcbc: the JIT rewrites an opcode to its J* +-- form when it installs a trace there and to its I* form when it +-- blacklists the spot. Blacklisting itself is never logged, so this +-- post-hoc read is the only stable way to observe it. +-- +-- usage: luajit trace_report.lua +-- luacheck: read globals jit +local jutil = require("jit.util") +local vmdef = require("jit.vmdef") +local here = arg[0]:match("^(.*)/[^/]+$") or "." +local bc = dofile(here .. "/bc_lib.lua") + +local function location(func, pc) + local info = jutil.funcinfo(func, pc) + return bc.basename(info.source), info.currentline or 0 +end + +-- The J*/I* rewrite families, derived from vmdef.bcnames so a LuaJIT bump +-- that adds a hot-countable pair shows up instead of being silently +-- skipped: an opcode named J or I where X is itself an opcode is a +-- rewrite form of X. (FORI has a J form only — the trace entry at a loop +-- start; the I* fallbacks exist for hot-countable spots: loops and +-- function entries.) +local COMPILED, BLACKLISTED = {}, {} +do + local names = {} + for op = 0, #vmdef.bcnames / 6 - 1 do + names[bc.bcname(op)] = true + end + for name in pairs(names) do + local prefix, base = name:sub(1, 1), name:sub(2) + if names[base] then + if prefix == "J" then + COMPILED[name] = true + elseif prefix == "I" then + BLACKLISTED[name] = true + end + end + end +end + +local spec_path = assert(arg[1], "usage: trace_report.lua ") +local spec_name = spec_path:match("([^/]+)%.lua$") +local spec_chunk = assert(loadfile(spec_path)) +local spec = spec_chunk() +local artifact_path = here .. "/../_build/" .. spec.artifact .. ".lua" +local artifact_chunk = assert(loadfile(artifact_path)) +local mod = artifact_chunk() + +local aborts = {} +local function on_trace(what, _tr, func, pc, code, extra) + if what ~= "abort" then + return + end + local reason = code + if type(code) == "number" then + local msg = vmdef.traceerr[code] or ("trace error " .. code) + -- Whether recording ever runs into an already-blacklisted spot depends + -- on how many PRNG-jittered retries preceded the blacklist, so this + -- abort reason is not stable across runs. It carries no information of + -- its own either: the blacklisted spot shows up as an I* opcode below. + if msg == "blacklisted" then + return + end + if type(extra) == "number" and msg:find("bytecode") then + extra = bc.bcname(extra) + elseif type(extra) == "function" then + -- Builtins carry no source (funcinfo gives ffid/addr instead), so + -- fall back the way jit/dump.lua's fmtfunc does. + local info = jutil.funcinfo(extra) + if info.source then + local file, line = location(extra) + extra = file .. ":" .. line + elseif info.ffid then + extra = vmdef.ffnames[info.ffid] + elseif info.addr then + extra = string.format("C:%x", info.addr) + else + extra = "(?)" + end + end + reason = msg:find("%%") and string.format(msg, extra) or msg + end + local file, line = location(func, pc) + aborts[string.format("%s:%d -- %s", file, line, reason)] = true +end + +jit.attach(on_trace, "trace") +local checksum +for _ = 1, 2 do + checksum = spec.drive(mod, spec.n) + spec.ideal(spec.n) +end +jit.attach(on_trace) + +local states = {} +local function visit(proto, pc, op) + if COMPILED[op] or BLACKLISTED[op] then + local file, line = location(proto, pc) + states[string.format("%s:%d %s", file, line, op)] = op + end +end +bc.walk(artifact_chunk, visit) +bc.walk(spec_chunk, visit) + +local function sorted_keys(set) + local keys = {} + for key in pairs(set) do + keys[#keys + 1] = key + end + table.sort(keys) + return keys +end + +io.write("spec: ", spec_name, "\n") +io.write("runtime: ", jit.version, "\n") +io.write(string.format("workload: n=%.0f reps=2 result=%s\n", spec.n, tostring(checksum))) +io.write("aborts (distinct site -- reason):\n") +local abort_keys = sorted_keys(aborts) +for _, key in ipairs(abort_keys) do + io.write(" ", key, "\n") +end +local compiled, blacklisted = 0, 0 +local state_keys = sorted_keys(states) +io.write("bytecode end state (J*=compiled, I*=blacklisted):\n") +for _, key in ipairs(state_keys) do + if COMPILED[states[key]] then + compiled = compiled + 1 + else + blacklisted = blacklisted + 1 + end + io.write(" ", key, "\n") +end +io.write(string.format( + "counts: aborts=%d compiled=%d blacklisted=%d\n", + #abort_keys, + compiled, + blacklisted +)) diff --git a/changelog.d/20260707_093000_unisay_bench_harness.md b/changelog.d/20260707_093000_unisay_bench_harness.md new file mode 100644 index 00000000..d40224ff --- /dev/null +++ b/changelog.d/20260707_093000_unisay_bench_harness.md @@ -0,0 +1,15 @@ +### Added + +- A `bench/` harness measuring generated code. `./bench/run` times + hand-written micro patterns (curried application, dictionary-driven + comparison, constructor allocation + tag match, module-field access) and + linked `Bench.*` macro modules (fib recursion, `foldl` over an array with + a curried step, a Maybe bind chain) under PUC Lua and LuaJIT, with and + without the JIT. `./bench/ci` regenerates deterministic LuaJIT counters + and diffs them against committed oracles in `bench/goldens/`: a static + FNEW census per linked artifact (split into main-chunk occurrences, which + run once at load, and function-body occurrences, which allocate per call + and abort trace recording) and a trace report of distinct abort sites + plus the end state of loop/function bytecodes (`J*` compiled, `I*` + blacklisted). CI runs `./bench/ci` after the test suite; the dev shell + gains `luajit` (#172). diff --git a/flake.nix b/flake.nix index caf21b14..d5efc3eb 100644 --- a/flake.nix +++ b/flake.nix @@ -65,6 +65,7 @@ spago-bin.spago-1_0_4 lua51Packages.lua lua51Packages.luacheck + luajit nil scriv upx diff --git a/test/Language/PureScript/Backend/Lua/Golden/Spec.hs b/test/Language/PureScript/Backend/Lua/Golden/Spec.hs index 70213e9f..1c60aa38 100644 --- a/test/Language/PureScript/Backend/Lua/Golden/Spec.hs +++ b/test/Language/PureScript/Backend/Lua/Golden/Spec.hs @@ -264,6 +264,11 @@ luacParse src = (code, out) ← readProcessInterleaved (proc "luac" ["-p", toFilePath file]) pure (code, decodeUtf8 out) +{- | Corefn files that participate in golden tests: only Golden.* modules. +Other modules compiled from test/ps/src pass through uncollected — +bench/link relies on this to link the Bench.* corefns without them +entering the suite. +-} collectGoldenCorefns ∷ MonadIO m ⇒ Path Rel Dir → m [Path Abs File] collectGoldenCorefns = walkDirAccum Nothing -- Descend into every directory diff --git a/test/ps/spago.lock b/test/ps/spago.lock index 27a0ac00..257880ed 100644 --- a/test/ps/spago.lock +++ b/test/ps/spago.lock @@ -5,6 +5,7 @@ "path": "./", "core": { "dependencies": [ + "arrays", "console", "effect", "either", diff --git a/test/ps/spago.yaml b/test/ps/spago.yaml index 44a5b46b..384dc10f 100644 --- a/test/ps/spago.yaml +++ b/test/ps/spago.yaml @@ -10,6 +10,7 @@ package: name: test-project dependencies: + - arrays - console - effect - either diff --git a/test/ps/src/Bench/ArrayFoldl.purs b/test/ps/src/Bench/ArrayFoldl.purs new file mode 100644 index 00000000..c811d75f --- /dev/null +++ b/test/ps/src/Bench/ArrayFoldl.purs @@ -0,0 +1,12 @@ +-- Foldable `foldl` over an Array with a curried step function. The FFI fold +-- loop is a plain Lua `for`, but every step application is a curried call +-- chain, so a tracing JIT has to inline closure creation to compile the loop. +module Bench.ArrayFoldl where + +import Prelude + +import Data.Array (range) +import Data.Foldable (foldl) + +run :: Int -> Int +run n = foldl (\acc i -> acc + i) 0 (range 1 n) diff --git a/test/ps/src/Bench/BindChain.purs b/test/ps/src/Bench/BindChain.purs new file mode 100644 index 00000000..2cba46b9 --- /dev/null +++ b/test/ps/src/Bench/BindChain.purs @@ -0,0 +1,14 @@ +-- A three-step Maybe bind chain; the benchmark driver applies `run` in a hot +-- loop, so each iteration goes through the Bind dictionary and constructor +-- allocation for every step. +module Bench.BindChain where + +import Prelude + +import Data.Maybe (Maybe(..), fromMaybe) + +run :: Int -> Int +run x = fromMaybe 0 do + a <- Just (x + 1) + b <- Just (a + 1) + Just (b + 1) diff --git a/test/ps/src/Bench/Fib.purs b/test/ps/src/Bench/Fib.purs new file mode 100644 index 00000000..f0781758 --- /dev/null +++ b/test/ps/src/Bench/Fib.purs @@ -0,0 +1,11 @@ +-- Source-identical to the `fib` in Golden.Fibonacci.Test: recursion with +-- typeclass arithmetic, without the golden's stdout side effect, so a +-- benchmark driver can call it repeatedly and time it in-process. +module Bench.Fib where + +import Prelude + +fib :: Int -> Int +fib 0 = 0 +fib 1 = 1 +fib n = fib (n - 1) + fib (n - 2)