From 5ec752d2b94361697da44c7b727e3b4e1704e818 Mon Sep 17 00:00:00 2001 From: Yura Lazarev Date: Tue, 7 Jul 2026 09:24:38 +0200 Subject: [PATCH 1/6] chore: add luajit to the dev shell (#172) The benchmark harness meters generated code under LuaJIT (static FNEW census via jit.util, trace abort/blacklist reports via jit.attach); the shell so far only carried PUC Lua 5.1. nixpkgs' luajit is a mainline 2.1 snapshot, which is what generated code typically runs under. --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index caf21b14..d5efc3eb 100644 --- a/flake.nix +++ b/flake.nix @@ -65,6 +65,7 @@ spago-bin.spago-1_0_4 lua51Packages.lua lua51Packages.luacheck + luajit nil scriv upx From 0051e48a77ae5a581576e718755c1a6d8d431b51 Mon Sep 17 00:00:00 2001 From: Yura Lazarev Date: Tue, 7 Jul 2026 09:24:47 +0200 Subject: [PATCH 2/6] chore: add Bench.* macrobenchmark sources to the test project (#172) Three small modules exercising the hot generated shapes end-to-end: fib recursion with typeclass arithmetic, Foldable foldl over an Array with a curried step (needs the arrays package, hence the new dependency), and a three-step Maybe bind chain. The golden harness only collects Golden.*-prefixed output directories, so these compile to CoreFn alongside the goldens without joining the golden suite; the bench scripts link them AsModule from that CoreFn. --- test/ps/spago.lock | 1 + test/ps/spago.yaml | 1 + test/ps/src/Bench/ArrayFoldl.purs | 12 ++++++++++++ test/ps/src/Bench/BindChain.purs | 14 ++++++++++++++ test/ps/src/Bench/Fib.purs | 11 +++++++++++ 5 files changed, 39 insertions(+) create mode 100644 test/ps/src/Bench/ArrayFoldl.purs create mode 100644 test/ps/src/Bench/BindChain.purs create mode 100644 test/ps/src/Bench/Fib.purs diff --git a/test/ps/spago.lock b/test/ps/spago.lock index 27a0ac00..257880ed 100644 --- a/test/ps/spago.lock +++ b/test/ps/spago.lock @@ -5,6 +5,7 @@ "path": "./", "core": { "dependencies": [ + "arrays", "console", "effect", "either", diff --git a/test/ps/spago.yaml b/test/ps/spago.yaml index 44a5b46b..384dc10f 100644 --- a/test/ps/spago.yaml +++ b/test/ps/spago.yaml @@ -10,6 +10,7 @@ package: name: test-project dependencies: + - arrays - console - effect - either diff --git a/test/ps/src/Bench/ArrayFoldl.purs b/test/ps/src/Bench/ArrayFoldl.purs new file mode 100644 index 00000000..c811d75f --- /dev/null +++ b/test/ps/src/Bench/ArrayFoldl.purs @@ -0,0 +1,12 @@ +-- Foldable `foldl` over an Array with a curried step function. The FFI fold +-- loop is a plain Lua `for`, but every step application is a curried call +-- chain, so a tracing JIT has to inline closure creation to compile the loop. +module Bench.ArrayFoldl where + +import Prelude + +import Data.Array (range) +import Data.Foldable (foldl) + +run :: Int -> Int +run n = foldl (\acc i -> acc + i) 0 (range 1 n) diff --git a/test/ps/src/Bench/BindChain.purs b/test/ps/src/Bench/BindChain.purs new file mode 100644 index 00000000..2cba46b9 --- /dev/null +++ b/test/ps/src/Bench/BindChain.purs @@ -0,0 +1,14 @@ +-- A three-step Maybe bind chain; the benchmark driver applies `run` in a hot +-- loop, so each iteration goes through the Bind dictionary and constructor +-- allocation for every step. +module Bench.BindChain where + +import Prelude + +import Data.Maybe (Maybe(..), fromMaybe) + +run :: Int -> Int +run x = fromMaybe 0 do + a <- Just (x + 1) + b <- Just (a + 1) + Just (b + 1) diff --git a/test/ps/src/Bench/Fib.purs b/test/ps/src/Bench/Fib.purs new file mode 100644 index 00000000..f0781758 --- /dev/null +++ b/test/ps/src/Bench/Fib.purs @@ -0,0 +1,11 @@ +-- Source-identical to the `fib` in Golden.Fibonacci.Test: recursion with +-- typeclass arithmetic, without the golden's stdout side effect, so a +-- benchmark driver can call it repeatedly and time it in-process. +module Bench.Fib where + +import Prelude + +fib :: Int -> Int +fib 0 = 0 +fib 1 = 1 +fib n = fib (n - 1) + fib (n - 2) From 0d63b250db45161f29f2c3a713596882a9a2eccf Mon Sep 17 00:00:00 2001 From: Yura Lazarev Date: Tue, 7 Jul 2026 09:25:11 +0200 Subject: [PATCH 3/6] chore: add the benchmark harness: timings and LuaJIT counters (#172) ./bench/run times micro codegen patterns (curried application, dictionary-driven comparison, constructor allocation + tag match, module-field access) and the linked Bench.* macro modules under PUC Lua and LuaJIT with and without the JIT. Wall-clock numbers stay local: they are too noisy for shared CI runners. ./bench/ci is the CI-grade check: byte-stable LuaJIT counters diffed against committed oracles in bench/goldens/ (--accept rewrites them after a deliberate codegen change), with every report generated twice and compared to prove stability. Two meters, both built on jit.util instead of parsing -bl/-jv text, whose format is not a contract: * fnew_census statically walks a linked artifact's prototypes and counts FNEW bytecodes, split into main-chunk occurrences (run once at load) and function-body occurrences (allocate per call, and abort trace recording: closure creation is NYI in the 2.1 tracer). * trace_report drives a macro spec hot and reports the set of distinct trace-abort sites plus the end state of loop/function-entry opcodes: the JIT rewrites them to J* forms when it installs a trace and to I* forms when it blacklists a spot. Blacklisting is never logged, and raw abort counts jitter with LuaJIT's entropy-seeded retry-penalty PRNG, so the post-hoc opcode read and the site set are the stable signals (aborts that merely ran into an already-blacklisted spot are filtered out for the same reason). On this machine the harness reproduces the motivating measurements: the curried hot loop runs ~120x slower than its uncurried shape under LuaJIT and no faster with -joff (every partial application FNEWs, the loop is blacklisted), and the FFI fold loop driving foldl with a curried step ends up IFORL-blacklisted while the hand-written fold compiles to a single trace. --- bench/.gitignore | 1 + bench/README.md | 63 +++++++ bench/ci | 44 +++++ bench/goldens/fnew_Bench.ArrayFoldl.txt | 18 ++ bench/goldens/fnew_Bench.BindChain.txt | 10 ++ bench/goldens/fnew_Bench.Fib.txt | 8 + bench/goldens/trace_array_foldl.txt | 17 ++ bench/goldens/trace_bind_chain.txt | 18 ++ bench/goldens/trace_fibonacci.txt | 14 ++ bench/link | 31 ++++ bench/macro/array_foldl.lua | 23 +++ bench/macro/bind_chain.lua | 23 +++ bench/macro/fibonacci.lua | 23 +++ bench/micro/ctor_match.lua | 35 ++++ bench/micro/curried_apply.lua | 38 +++++ bench/micro/dict_compare.lua | 61 +++++++ bench/micro/field_access.lua | 40 +++++ bench/run | 25 +++ bench/tools/bench_lib.lua | 61 +++++++ bench/tools/fnew_census.lua | 74 +++++++++ bench/tools/run_macro.lua | 19 +++ bench/tools/run_micro.lua | 14 ++ bench/tools/trace_report.lua | 157 ++++++++++++++++++ .../20260707_093000_unisay_bench_harness.md | 15 ++ 24 files changed, 832 insertions(+) create mode 100644 bench/.gitignore create mode 100644 bench/README.md create mode 100755 bench/ci create mode 100644 bench/goldens/fnew_Bench.ArrayFoldl.txt create mode 100644 bench/goldens/fnew_Bench.BindChain.txt create mode 100644 bench/goldens/fnew_Bench.Fib.txt create mode 100644 bench/goldens/trace_array_foldl.txt create mode 100644 bench/goldens/trace_bind_chain.txt create mode 100644 bench/goldens/trace_fibonacci.txt create mode 100755 bench/link create mode 100644 bench/macro/array_foldl.lua create mode 100644 bench/macro/bind_chain.lua create mode 100644 bench/macro/fibonacci.lua create mode 100644 bench/micro/ctor_match.lua create mode 100644 bench/micro/curried_apply.lua create mode 100644 bench/micro/dict_compare.lua create mode 100644 bench/micro/field_access.lua create mode 100755 bench/run create mode 100644 bench/tools/bench_lib.lua create mode 100644 bench/tools/fnew_census.lua create mode 100644 bench/tools/run_macro.lua create mode 100644 bench/tools/run_micro.lua create mode 100644 bench/tools/trace_report.lua create mode 100644 changelog.d/20260707_093000_unisay_bench_harness.md diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 00000000..ba65b13a --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..d3b7083e --- /dev/null +++ b/bench/README.md @@ -0,0 +1,63 @@ +# Benchmark harness + +Measures the performance of generated Lua and meters how LuaJIT's tracing +JIT treats it. Two kinds of output with two different purposes: + +- **Wall-clock timings** (`./bench/run`) — local numbers for defending an + optimization win. Too noisy for CI. +- **Deterministic LuaJIT counters** (`./bench/ci`) — byte-stable reports + that CI diffs against the committed oracles in `goldens/`, so a codegen + change that adds closure allocations or blacklists a loop shows up as a + reviewable diff. + +## Layout + +- `micro/` — hand-written Lua pairs: the shape the backend currently + generates for a pattern (`current`) next to the idiomatic Lua it stands + for (`ideal`). These are fixed reference points; they do not change when + codegen changes. +- `macro/` — specs driving real linked modules. Each spec names a `Bench.*` + PureScript module (sources in `test/ps/src/Bench/`, linked by + `./bench/link` into `_build/`), how to drive its export hot, and an + `ideal` hand-written equivalent. +- `tools/` — the runners and meters. `fnew_census.lua` and + `trace_report.lua` require LuaJIT (`jit.util`, `jit.attach`); the timing + runners work under both PUC Lua and LuaJIT. +- `goldens/` — committed counter reports; the CI oracle. + +## Usage + +```bash +./bench/run # all wall-clock benchmarks, all runtimes +lua bench/tools/run_micro.lua bench/micro/curried_apply.lua # one bench +luajit bench/tools/run_macro.lua bench/macro/array_foldl.lua 5e6 # custom n +./bench/ci # regenerate counters, verify against goldens/ +./bench/ci --accept # rewrite goldens/ after a deliberate change +``` + +Timings use `os.clock()` — CPU time, not wall time. That is deliberate: +the benchmarks are pure computation, and CPU time ignores scheduler noise. +It would under-report I/O, so do not reuse the timing helpers for anything +that waits. For quieter numbers, pin the process to a core (`taskset -c N +./bench/run`) and use the `performance` CPU governor; leave ASLR alone — +disabling it trades a real security property for nearly nothing. + +## What the counters mean + +`fnew_census` statically counts `FNEW` bytecodes (closure creation) in a +linked artifact without running it, split by where the instruction lives: +the **main chunk** runs once at load time, so its FNEWs are init cost; a +**function body** runs per call, so its FNEWs are steady-state allocation — +and each one aborts LuaJIT trace recording (`NYI: bytecode FNEW`), which is +what keeps curried hot code interpreted. The census is a pure function of +the artifact, hence byte-stable. + +`trace_report` runs a macro spec hot and reports (a) the *set* of distinct +trace-abort sites with reasons and (b) the end state of loop and +function-entry bytecodes: LuaJIT rewrites an opcode to its `J*` form when +it installs a trace there and to its `I*` form when it blacklists the spot. +Raw abort *counts* are not reported: the retry-penalty step that leads to a +blacklist draws on an entropy-seeded PRNG, so counts jitter across runs +while the final opcode state and the abort-site set do not. Blacklisting is +never logged by `-jv`/`-jdump`; the post-hoc opcode read is the only stable +way to observe it. diff --git a/bench/ci b/bench/ci new file mode 100755 index 00000000..8a634d13 --- /dev/null +++ b/bench/ci @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Deterministic LuaJIT counters: a static FNEW census per linked artifact +# and a trace abort/blacklist report per macro spec. Every report is +# generated twice and compared, proving byte-stability, then diffed against +# the committed oracles in bench/goldens/. Pass --accept after a deliberate +# codegen change to rewrite the oracles from the current output. +set -euo pipefail +cd "$(dirname "$0")/.." + +accept=0 +if [ "${1:-}" = "--accept" ]; then + accept=1 +fi + +./bench/link + +out=bench/_build/counters +rm -rf "$out" +mkdir -p "$out" + +for artifact in bench/_build/Bench.*.lua; do + name=$(basename "$artifact" .lua) + luajit bench/tools/fnew_census.lua "$artifact" >"$out/fnew_$name.txt" + luajit bench/tools/fnew_census.lua "$artifact" >"$out/second-run.txt" + cmp "$out/fnew_$name.txt" "$out/second-run.txt" +done + +for spec in bench/macro/*.lua; do + name=$(basename "$spec" .lua) + luajit bench/tools/trace_report.lua "$spec" >"$out/trace_$name.txt" + luajit bench/tools/trace_report.lua "$spec" >"$out/second-run.txt" + cmp "$out/trace_$name.txt" "$out/second-run.txt" +done +rm "$out/second-run.txt" + +if [ "$accept" = 1 ]; then + mkdir -p bench/goldens + rm -f bench/goldens/*.txt + cp "$out"/*.txt bench/goldens/ + echo "accepted $(find "$out" -name '*.txt' | wc -l) counter files into bench/goldens/" +else + diff -ru bench/goldens "$out" + echo "bench counters match goldens" +fi diff --git a/bench/goldens/fnew_Bench.ArrayFoldl.txt b/bench/goldens/fnew_Bench.ArrayFoldl.txt new file mode 100644 index 00000000..ca6e4e00 --- /dev/null +++ b/bench/goldens/fnew_Bench.ArrayFoldl.txt @@ -0,0 +1,18 @@ +chunk: Bench.ArrayFoldl.lua +main-chunk FNEW: 7 +function-body FNEW: 12 +total FNEW: 19 +prototypes: 20 +function-body FNEW sites: + Bench.ArrayFoldl.lua:3 + Bench.ArrayFoldl.lua:14 + Bench.ArrayFoldl.lua:13 + Bench.ArrayFoldl.lua:24 + Bench.ArrayFoldl.lua:23 + Bench.ArrayFoldl.lua:28 + Bench.ArrayFoldl.lua:28 + Bench.ArrayFoldl.lua:54 + Bench.ArrayFoldl.lua:53 + Bench.ArrayFoldl.lua:52 + Bench.ArrayFoldl.lua:63 + Bench.ArrayFoldl.lua:62 diff --git a/bench/goldens/fnew_Bench.BindChain.txt b/bench/goldens/fnew_Bench.BindChain.txt new file mode 100644 index 00000000..aff7661a --- /dev/null +++ b/bench/goldens/fnew_Bench.BindChain.txt @@ -0,0 +1,10 @@ +chunk: Bench.BindChain.lua +main-chunk FNEW: 4 +function-body FNEW: 4 +total FNEW: 8 +prototypes: 9 +function-body FNEW sites: + Bench.BindChain.lua:3 + Bench.BindChain.lua:19 + Bench.BindChain.lua:27 + Bench.BindChain.lua:26 diff --git a/bench/goldens/fnew_Bench.Fib.txt b/bench/goldens/fnew_Bench.Fib.txt new file mode 100644 index 00000000..29276152 --- /dev/null +++ b/bench/goldens/fnew_Bench.Fib.txt @@ -0,0 +1,8 @@ +chunk: Bench.Fib.lua +main-chunk FNEW: 3 +function-body FNEW: 2 +total FNEW: 5 +prototypes: 6 +function-body FNEW sites: + Bench.Fib.lua:3 + Bench.Fib.lua:6 diff --git a/bench/goldens/trace_array_foldl.txt b/bench/goldens/trace_array_foldl.txt new file mode 100644 index 00000000..4a69cc5d --- /dev/null +++ b/bench/goldens/trace_array_foldl.txt @@ -0,0 +1,17 @@ +spec: array_foldl +workload: n=5000000 reps=2 result=12500002500000 +aborts (distinct site -- reason): + Bench.ArrayFoldl.lua:3 -- NYI: bytecode FNEW + Bench.ArrayFoldl.lua:62 -- NYI: bytecode FNEW +bytecode end state (J*=compiled, I*=blacklisted): + Bench.ArrayFoldl.lua:21 IFORL + Bench.ArrayFoldl.lua:3 IFUNCF + Bench.ArrayFoldl.lua:3 JFUNCF + Bench.ArrayFoldl.lua:35 JLOOP + Bench.ArrayFoldl.lua:59 IFUNCF + Bench.ArrayFoldl.lua:60 IFUNCF + array_foldl.lua:14 JFORI + array_foldl.lua:14 JFORL + array_foldl.lua:18 JFORI + array_foldl.lua:18 JFORL +counts: aborts=2 compiled=6 blacklisted=4 diff --git a/bench/goldens/trace_bind_chain.txt b/bench/goldens/trace_bind_chain.txt new file mode 100644 index 00000000..2145de66 --- /dev/null +++ b/bench/goldens/trace_bind_chain.txt @@ -0,0 +1,18 @@ +spec: bind_chain +workload: n=1000000 reps=2 result=3000000 +aborts (distinct site -- reason): + Bench.BindChain.lua:19 -- NYI: bytecode FNEW + Bench.BindChain.lua:3 -- NYI: bytecode FNEW +bytecode end state (J*=compiled, I*=blacklisted): + Bench.BindChain.lua:22 IFUNCF + Bench.BindChain.lua:23 IFUNCF + Bench.BindChain.lua:24 IFUNCF + Bench.BindChain.lua:3 IFUNCF + Bench.BindChain.lua:3 JFUNCF + Bench.BindChain.lua:5 JFUNCF + Bench.BindChain.lua:8 IFUNCF + Bench.BindChain.lua:9 IFUNCF + bind_chain.lua:11 IFORL + bind_chain.lua:18 JFORI + bind_chain.lua:18 JFORL +counts: aborts=2 compiled=4 blacklisted=7 diff --git a/bench/goldens/trace_fibonacci.txt b/bench/goldens/trace_fibonacci.txt new file mode 100644 index 00000000..9c20f7bd --- /dev/null +++ b/bench/goldens/trace_fibonacci.txt @@ -0,0 +1,14 @@ +spec: fibonacci +workload: n=30 reps=2 result=832040 +aborts (distinct site -- reason): + Bench.Fib.lua:3 -- NYI: bytecode FNEW + Bench.Fib.lua:6 -- NYI: bytecode FNEW + fibonacci.lua:12 -- call unroll limit reached +bytecode end state (J*=compiled, I*=blacklisted): + Bench.Fib.lua:3 IFUNCF + Bench.Fib.lua:3 JFUNCF + Bench.Fib.lua:6 IFUNCF + Bench.Fib.lua:6 JFUNCF + Bench.Fib.lua:8 JFUNCF + fibonacci.lua:12 JFUNCF +counts: aborts=3 compiled=4 blacklisted=2 diff --git a/bench/link b/bench/link new file mode 100755 index 00000000..9eea58cc --- /dev/null +++ b/bench/link @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Links the Bench.* PureScript modules into standalone Lua artifacts under +# bench/_build/, building pslua and the CoreFn output first when needed. +set -euo pipefail +cd "$(dirname "$0")/.." +root=$PWD + +if [ ! -f test/ps/output/Bench.ArrayFoldl/corefn.json ]; then + (cd test/ps && spago build) +fi + +cabal build -v0 exe:pslua +pslua=$(cabal list-bin pslua) + +mkdir -p bench/_build +# CoreFn module paths are relative to test/ps (where spago runs), and pslua +# resolves foreign files against them, so it has to run from there too. +for module in Bench.Fib Bench.ArrayFoldl Bench.BindChain; do + rm -f "bench/_build/$module.lua" + ( + cd test/ps + "$pslua" \ + --ps-output output \ + --foreign-path foreign \ + --entry "$module" \ + --lua-output-file "$root/bench/_build/$module.lua" + ) + # pslua can report a linker error without a failing exit code; an existing + # non-empty artifact is the reliable success signal. + test -s "bench/_build/$module.lua" +done diff --git a/bench/macro/array_foldl.lua b/bench/macro/array_foldl.lua new file mode 100644 index 00000000..4f5c26ba --- /dev/null +++ b/bench/macro/array_foldl.lua @@ -0,0 +1,23 @@ +-- Foldable `foldl` over an Array with a curried step: the FFI fold loop is +-- a plain Lua `for`, but compiling it means inlining the curried callee, so +-- the tracer aborts on FNEW and blacklists the loop. The ideal variant does +-- the same work — build the array, fold it — with an uncurried step. +return { + name = "array_foldl", + artifact = "Bench.ArrayFoldl", + n = 5e6, + drive = function(mod, n) + return mod.run(n) + end, + ideal = function(n) + local t = {} + for i = 1, n do + t[i] = i + end + local acc = 0 + for i = 1, #t do + acc = acc + t[i] + end + return acc + end, +} diff --git a/bench/macro/bind_chain.lua b/bench/macro/bind_chain.lua new file mode 100644 index 00000000..487757c1 --- /dev/null +++ b/bench/macro/bind_chain.lua @@ -0,0 +1,23 @@ +-- A three-step Maybe bind chain per iteration: each step goes through the +-- Bind dictionary and allocates a Just constructor. The driver loop is the +-- hot loop here, so this also exercises what happens to a caller loop whose +-- callee cannot be traced. +return { + name = "bind_chain", + artifact = "Bench.BindChain", + n = 1e6, + drive = function(mod, n) + local acc = 0 + for _ = 1, n do + acc = mod.run(acc) + end + return acc + end, + ideal = function(n) + local acc = 0 + for _ = 1, n do + acc = acc + 3 + end + return acc + end, +} diff --git a/bench/macro/fibonacci.lua b/bench/macro/fibonacci.lua new file mode 100644 index 00000000..260b7883 --- /dev/null +++ b/bench/macro/fibonacci.lua @@ -0,0 +1,23 @@ +-- Recursion with typeclass arithmetic: the same `fib` as the +-- Golden.Fibonacci.Test golden, linked as a module so the driver can call +-- it without going through stdout. +return { + name = "fibonacci", + artifact = "Bench.Fib", + n = 30, + drive = function(mod, n) + return mod.fib(n) + end, + ideal = function(n) + local function fib(v) + if v == 0 then + return 0 + end + if v == 1 then + return 1 + end + return fib(v - 1) + fib(v - 2) + end + return fib(n) + end, +} diff --git a/bench/micro/ctor_match.lua b/bench/micro/ctor_match.lua new file mode 100644 index 00000000..cd2735a8 --- /dev/null +++ b/bench/micro/ctor_match.lua @@ -0,0 +1,35 @@ +-- Constructor allocation plus tag match: the generated encoding keeps the +-- tag and the payload in the table's hash part, keyed by long strings, +-- versus an array-part encoding with a small-integer tag. +return { + name = "ctor_match", + n = 2e6, + variants = { + { + name = "current", + fn = function(n) + local acc = 0 + for i = 1, n do + local m = { ["$ctor"] = "Data.Maybe∷Maybe.Just", value0 = i } + if "Data.Maybe∷Maybe.Just" == m["$ctor"] then + acc = acc + m.value0 + end + end + return acc + end, + }, + { + name = "ideal", + fn = function(n) + local acc = 0 + for i = 1, n do + local m = { 1, i } + if 1 == m[1] then + acc = acc + m[2] + end + end + return acc + end, + }, + }, +} diff --git a/bench/micro/curried_apply.lua b/bench/micro/curried_apply.lua new file mode 100644 index 00000000..df3616d0 --- /dev/null +++ b/bench/micro/curried_apply.lua @@ -0,0 +1,38 @@ +-- Curried application in a hot loop: every partial application allocates a +-- closure (an FNEW bytecode under LuaJIT, which aborts trace recording), so +-- the loop never gets JIT-compiled. The uncurried variant compiles to a +-- single trace. +return { + name = "curried", + n = 2e7, + variants = { + { + name = "current", + fn = function(n) + local add = function(x) + return function(y) + return x + y + end + end + local acc = 0 + for i = 1, n do + acc = add(acc)(i) + end + return acc + end, + }, + { + name = "ideal", + fn = function(n) + local add = function(x, y) + return x + y + end + local acc = 0 + for i = 1, n do + acc = add(acc, i) + end + return acc + end, + }, + }, +} diff --git a/bench/micro/dict_compare.lua b/bench/micro/dict_compare.lua new file mode 100644 index 00000000..985926c7 --- /dev/null +++ b/bench/micro/dict_compare.lua @@ -0,0 +1,61 @@ +-- Dictionary-driven comparison: `greaterThanOrEq ordInt` goes through a +-- dictionary table, a curried three-level call chain, an Ordering +-- constructor allocation and a tag match — versus the native `>=` the +-- whole dance stands for. +local LT = { ["$ctor"] = "Data.Ordering∷Ordering.LT" } +local EQ = { ["$ctor"] = "Data.Ordering∷Ordering.EQ" } +local GT = { ["$ctor"] = "Data.Ordering∷Ordering.GT" } +local M = {} +M.Data_Ord_ordInt = { + compare = function(x) + return function(y) + if x < y then + return LT + elseif x == y then + return EQ + else + return GT + end + end + end, +} +M.Data_Ord_greaterThanOrEq = function(dictOrd) + return function(a1) + return function(a2) + return dictOrd.compare(a1)(a2)["$ctor"] ~= "Data.Ordering∷Ordering.LT" + end + end +end + +return { + name = "dict_gte", + n = 5e6, + variants = { + { + name = "current", + fn = function(n) + local half = n / 2 + local count = 0 + for i = 1, n do + if M.Data_Ord_greaterThanOrEq(M.Data_Ord_ordInt)(i)(half) then + count = count + 1 + end + end + return count + end, + }, + { + name = "ideal", + fn = function(n) + local half = n / 2 + local count = 0 + for i = 1, n do + if i >= half then + count = count + 1 + end + end + return count + end, + }, + }, +} diff --git a/bench/micro/field_access.lua b/bench/micro/field_access.lua new file mode 100644 index 00000000..1ebdf86a --- /dev/null +++ b/bench/micro/field_access.lua @@ -0,0 +1,40 @@ +-- Reaching a function through module-table fields on every call, the way +-- linked code does (M.Data_Semiring_foreign.intAdd), versus hoisting it +-- into a local once. Both variants call uncurried so the difference is the +-- field lookups alone. +return { + name = "field_access", + n = 1e7, + variants = { + { + name = "current", + fn = function(n) + local M = { + Data_Semiring_foreign = { + intAdd = function(x, y) + return x + y + end, + }, + } + local acc = 0 + for i = 1, n do + acc = M.Data_Semiring_foreign.intAdd(acc, i) + end + return acc + end, + }, + { + name = "ideal", + fn = function(n) + local intAdd = function(x, y) + return x + y + end + local acc = 0 + for i = 1, n do + acc = intAdd(acc, i) + end + return acc + end, + }, + }, +} diff --git a/bench/run b/bench/run new file mode 100755 index 00000000..2f49b4bc --- /dev/null +++ b/bench/run @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Wall-clock benchmarks: micro codegen patterns and linked macro modules +# under PUC Lua and LuaJIT (with and without the JIT). Timings are for local +# use — too noisy for CI, which gates on ./bench/ci instead. For quieter +# numbers pin the process (e.g. `taskset -c N ./bench/run`) and use the +# `performance` CPU governor. +set -euo pipefail +cd "$(dirname "$0")/.." + +./bench/link + +echo "== micro: codegen patterns ==" +for bench in bench/micro/*.lua; do + lua bench/tools/run_micro.lua "$bench" + luajit bench/tools/run_micro.lua "$bench" + luajit -joff bench/tools/run_micro.lua "$bench" +done + +echo +echo "== macro: linked modules ==" +for spec in bench/macro/*.lua; do + lua bench/tools/run_macro.lua "$spec" + luajit bench/tools/run_macro.lua "$spec" + luajit -joff bench/tools/run_macro.lua "$spec" +done diff --git a/bench/tools/bench_lib.lua b/bench/tools/bench_lib.lua new file mode 100644 index 00000000..b00a253a --- /dev/null +++ b/bench/tools/bench_lib.lua @@ -0,0 +1,61 @@ +-- Shared timing helpers for the wall-clock benchmark runners. +-- +-- os.clock() measures CPU time, not wall time. For these CPU-bound loops +-- that is the right metric (time the process spends scheduled out does not +-- pollute samples), but it would silently under-report anything I/O-bound. +local M = {} + +-- Runs fn(n) once untimed (lets LuaJIT compile traces: a loop becomes a +-- tracing candidate after 56 iterations, a side exit after 10), then times +-- `samples` runs with the collector stopped, starting each sample from a +-- freshly collected heap. Reports the median so a stray outlier sample +-- cannot move the headline number. +function M.measure(fn, n, samples) + samples = samples or 5 + local checksum = fn(n) + local times = {} + for s = 1, samples do + collectgarbage("collect") + collectgarbage("stop") + local t0 = os.clock() + fn(n) + local t1 = os.clock() + collectgarbage("restart") + times[s] = t1 - t0 + end + table.sort(times) + local mid = math.floor((#times + 1) / 2) + local median = times[mid] + if #times % 2 == 0 then + median = (times[mid] + times[mid + 1]) / 2 + end + return { + median = median, + min = times[1], + max = times[#times], + checksum = checksum, + } +end + +function M.runtime_tag() + if jit then + return jit.status() and "luajit" or "luajit-joff" + end + return (_VERSION:gsub("%s", ""):lower()) +end + +function M.report(bench_name, variant_name, n, r) + io.write(string.format( + "%-12s %-8s %-12s n=%-9.0f median=%8.4fs min=%8.4fs max=%8.4fs result=%s\n", + bench_name, + variant_name, + M.runtime_tag(), + n, + r.median, + r.min, + r.max, + tostring(r.checksum) + )) +end + +return M diff --git a/bench/tools/fnew_census.lua b/bench/tools/fnew_census.lua new file mode 100644 index 00000000..bbc03439 --- /dev/null +++ b/bench/tools/fnew_census.lua @@ -0,0 +1,74 @@ +-- Static FNEW census over a Lua chunk: counts closure-creation bytecodes +-- without executing anything, so the output is a pure function of the file. +-- +-- An FNEW in the main chunk runs once, at load time; an FNEW in a function +-- body runs on every call of that function, and aborts LuaJIT trace +-- recording (closure creation is NYI in the 2.1 tracer). The split is a +-- structural property of the prototype an instruction lives in: the main +-- chunk is the outermost prototype, everything reachable through child +-- prototypes is a function body. Prototypes are walked the way jit/bc.lua +-- does: iterate jit.util.funcbc until nil, recurse into children via +-- negative jit.util.funck indices. +-- +-- usage: luajit fnew_census.lua +local jutil = require("jit.util") +local vmdef = require("jit.vmdef") +local band = require("bit").band + +local function bcname(op) + return (vmdef.bcnames:sub(op * 6 + 1, op * 6 + 6):gsub("%s+$", "")) +end + +local function basename(source) + return (source:gsub("^@", ""):gsub("^.*/", "")) +end + +local counts = { main = 0, body = 0, protos = 0 } +local body_sites = {} + +local function walk(proto, in_main) + counts.protos = counts.protos + 1 + local pc = 1 + while true do + local ins = jutil.funcbc(proto, pc) + if not ins then + break + end + if bcname(band(ins, 0xff)) == "FNEW" then + if in_main then + counts.main = counts.main + 1 + else + counts.body = counts.body + 1 + local info = jutil.funcinfo(proto, pc) + body_sites[#body_sites + 1] = + string.format("%s:%d", basename(info.source), info.currentline) + end + end + pc = pc + 1 + end + if jutil.funcinfo(proto).children then + for n = -1, -1e9, -1 do + local k = jutil.funck(proto, n) + if not k then + break + end + if type(k) == "proto" then + walk(k, false) + end + end + end +end + +local path = assert(arg[1], "usage: fnew_census.lua ") +local chunk = assert(loadfile(path)) +walk(chunk, true) + +io.write("chunk: ", basename("@" .. path), "\n") +io.write("main-chunk FNEW: ", counts.main, "\n") +io.write("function-body FNEW: ", counts.body, "\n") +io.write("total FNEW: ", counts.main + counts.body, "\n") +io.write("prototypes: ", counts.protos, "\n") +io.write("function-body FNEW sites:\n") +for _, site in ipairs(body_sites) do + io.write(" ", site, "\n") +end diff --git a/bench/tools/run_macro.lua b/bench/tools/run_macro.lua new file mode 100644 index 00000000..a24a31c0 --- /dev/null +++ b/bench/tools/run_macro.lua @@ -0,0 +1,19 @@ +-- Times one macrobenchmark spec: the linked pslua artifact (loaded from +-- bench/_build/, produced by bench/link) against the spec's hand-written +-- idiomatic-Lua equivalent. +-- +-- usage: lua|luajit [-joff] run_macro.lua [n] [samples] +local here = arg[0]:match("^(.*)/[^/]+$") or "." +local lib = dofile(here .. "/bench_lib.lua") + +local spec_path = assert(arg[1], "usage: run_macro.lua [n] [samples]") +local spec = dofile(spec_path) +local mod = dofile(here .. "/../_build/" .. spec.artifact .. ".lua") +local n = tonumber(arg[2]) or spec.n +local samples = tonumber(arg[3]) + +local linked = function(k) + return spec.drive(mod, k) +end +lib.report(spec.name, "linked", n, lib.measure(linked, n, samples)) +lib.report(spec.name, "ideal", n, lib.measure(spec.ideal, n, samples)) diff --git a/bench/tools/run_micro.lua b/bench/tools/run_micro.lua new file mode 100644 index 00000000..c0de22da --- /dev/null +++ b/bench/tools/run_micro.lua @@ -0,0 +1,14 @@ +-- Times every variant of one microbenchmark under the current runtime. +-- +-- usage: lua|luajit [-joff] run_micro.lua [n] [samples] +local here = arg[0]:match("^(.*)/[^/]+$") or "." +local lib = dofile(here .. "/bench_lib.lua") + +local spec_path = assert(arg[1], "usage: run_micro.lua [n] [samples]") +local spec = dofile(spec_path) +local n = tonumber(arg[2]) or spec.n +local samples = tonumber(arg[3]) + +for _, variant in ipairs(spec.variants) do + lib.report(spec.name, variant.name, n, lib.measure(variant.fn, n, samples)) +end diff --git a/bench/tools/trace_report.lua b/bench/tools/trace_report.lua new file mode 100644 index 00000000..1ce11c33 --- /dev/null +++ b/bench/tools/trace_report.lua @@ -0,0 +1,157 @@ +-- Runs a macrobenchmark spec hot under the tracing JIT and reports two +-- deterministic signals per source location: +-- +-- * the set of distinct trace-abort sites (location + reason), collected +-- via jit.attach("trace"). A set, not a count: how many retries happen +-- before a spot is blacklisted depends on LuaJIT's entropy-seeded +-- penalty PRNG, so raw abort counts are not stable across runs. +-- +-- * the end state of loop and function-entry bytecodes after the run, +-- read back with jit.util.funcbc: the JIT rewrites an opcode to its J* +-- form when it installs a trace there and to its I* form when it +-- blacklists the spot. Blacklisting itself is never logged, so this +-- post-hoc read is the only stable way to observe it. +-- +-- usage: luajit trace_report.lua +local jutil = require("jit.util") +local vmdef = require("jit.vmdef") +local band = require("bit").band + +local function bcname(op) + return (vmdef.bcnames:sub(op * 6 + 1, op * 6 + 6):gsub("%s+$", "")) +end + +local function basename(source) + return (source:gsub("^@", ""):gsub("^.*/", "")) +end + +local function location(func, pc) + local info = jutil.funcinfo(func, pc) + return basename(info.source), info.currentline or 0 +end + +-- The J*/I* opcode families observable after a run. FORI has a J form only +-- (trace entry at a loop start); the interpreted-fallback forms exist for +-- hot-countable spots: loops and function entries. +local COMPILED = { + JFORI = true, + JFORL = true, + JITERL = true, + JLOOP = true, + JFUNCF = true, + JFUNCV = true, +} +local BLACKLISTED = { + IFORL = true, + IITERL = true, + ILOOP = true, + IFUNCF = true, + IFUNCV = true, +} + +local spec_path = assert(arg[1], "usage: trace_report.lua ") +local spec_chunk = assert(loadfile(spec_path)) +local spec = spec_chunk() +local here = arg[0]:match("^(.*)/[^/]+$") or "." +local artifact_path = here .. "/../_build/" .. spec.artifact .. ".lua" +local artifact_chunk = assert(loadfile(artifact_path)) +local mod = artifact_chunk() + +local aborts = {} +local function on_trace(what, _tr, func, pc, code, extra) + if what ~= "abort" then + return + end + local reason = code + if type(code) == "number" then + local msg = vmdef.traceerr[code] or ("trace error " .. code) + -- Whether recording ever runs into an already-blacklisted spot depends + -- on how many PRNG-jittered retries preceded the blacklist, so this + -- abort reason is not stable across runs. It carries no information of + -- its own either: the blacklisted spot shows up as an I* opcode below. + if msg == "blacklisted" then + return + end + if type(extra) == "number" and msg:find("bytecode") then + extra = bcname(extra) + elseif type(extra) == "function" then + local file, line = location(extra) + extra = file .. ":" .. line + end + reason = msg:find("%%") and string.format(msg, extra) or msg + end + local file, line = location(func, pc) + aborts[string.format("%s:%d -- %s", file, line, reason)] = true +end + +jit.attach(on_trace, "trace") +local checksum +for _ = 1, 2 do + checksum = spec.drive(mod, spec.n) + spec.ideal(spec.n) +end +jit.attach(on_trace) + +local states = {} +local function walk(proto) + local pc = 0 + while true do + local ins = jutil.funcbc(proto, pc) + if not ins then + break + end + local op = bcname(band(ins, 0xff)) + if COMPILED[op] or BLACKLISTED[op] then + local file, line = location(proto, pc) + states[string.format("%s:%d %s", file, line, op)] = op + end + pc = pc + 1 + end + if jutil.funcinfo(proto).children then + for n = -1, -1e9, -1 do + local k = jutil.funck(proto, n) + if not k then + break + end + if type(k) == "proto" then + walk(k) + end + end + end +end +walk(artifact_chunk) +walk(spec_chunk) + +local function sorted_keys(set) + local keys = {} + for key in pairs(set) do + keys[#keys + 1] = key + end + table.sort(keys) + return keys +end + +io.write("spec: ", spec.name, "\n") +io.write(string.format("workload: n=%.0f reps=2 result=%s\n", spec.n, tostring(checksum))) +io.write("aborts (distinct site -- reason):\n") +local abort_keys = sorted_keys(aborts) +for _, key in ipairs(abort_keys) do + io.write(" ", key, "\n") +end +local compiled, blacklisted = 0, 0 +local state_keys = sorted_keys(states) +io.write("bytecode end state (J*=compiled, I*=blacklisted):\n") +for _, key in ipairs(state_keys) do + if COMPILED[states[key]] then + compiled = compiled + 1 + else + blacklisted = blacklisted + 1 + end + io.write(" ", key, "\n") +end +io.write(string.format( + "counts: aborts=%d compiled=%d blacklisted=%d\n", + #abort_keys, + compiled, + blacklisted +)) diff --git a/changelog.d/20260707_093000_unisay_bench_harness.md b/changelog.d/20260707_093000_unisay_bench_harness.md new file mode 100644 index 00000000..d40224ff --- /dev/null +++ b/changelog.d/20260707_093000_unisay_bench_harness.md @@ -0,0 +1,15 @@ +### Added + +- A `bench/` harness measuring generated code. `./bench/run` times + hand-written micro patterns (curried application, dictionary-driven + comparison, constructor allocation + tag match, module-field access) and + linked `Bench.*` macro modules (fib recursion, `foldl` over an array with + a curried step, a Maybe bind chain) under PUC Lua and LuaJIT, with and + without the JIT. `./bench/ci` regenerates deterministic LuaJIT counters + and diffs them against committed oracles in `bench/goldens/`: a static + FNEW census per linked artifact (split into main-chunk occurrences, which + run once at load, and function-body occurrences, which allocate per call + and abort trace recording) and a trace report of distinct abort sites + plus the end state of loop/function bytecodes (`J*` compiled, `I*` + blacklisted). CI runs `./bench/ci` after the test suite; the dev shell + gains `luajit` (#172). From 91bf04dedbe580eb2403f194882e524f40f5c77d Mon Sep 17 00:00:00 2001 From: Yura Lazarev Date: Tue, 7 Jul 2026 09:25:22 +0200 Subject: [PATCH 4/6] chore: run the deterministic bench counters in CI (#172) The step runs after the test suite on purpose: the suite's spago build leaves the full CoreFn output (including dependencies) on disk, which is what ./bench/ci links the Bench.* artifacts from. --- .github/workflows/ci.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7cc3ff3b..ba6ce737 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,12 @@ jobs: - name: "🔨 Build & test" run: >- nix develop --accept-flake-config --allow-import-from-derivation --command cabal test all --test-show-details=direct + # Deterministic LuaJIT counters over freshly linked bench artifacts + # (FNEW census, trace abort/blacklist state). Wall-clock benchmarks + # stay local — shared runners are too noisy; see bench/README.md. + - name: "📊 Bench counters" + run: >- + nix develop --accept-flake-config --allow-import-from-derivation --command ./bench/ci format: runs-on: ubuntu-latest steps: From cf795ebc56101d19d3a086e5db00d186784853e8 Mon Sep 17 00:00:00 2001 From: Yura Lazarev Date: Tue, 7 Jul 2026 10:02:26 +0200 Subject: [PATCH 5/6] chore: drop bench/link's needless post-hoc artifact check (#172) pslua does exit non-zero on a linker error; the earlier '$?' check that seemed to say otherwise was reading tail's exit code through a pipe, not pslua's. bench/link already runs under 'set -e', so a real failure stops it without help. --- bench/link | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bench/link b/bench/link index 9eea58cc..9dd3f25e 100755 --- a/bench/link +++ b/bench/link @@ -16,7 +16,6 @@ mkdir -p bench/_build # CoreFn module paths are relative to test/ps (where spago runs), and pslua # resolves foreign files against them, so it has to run from there too. for module in Bench.Fib Bench.ArrayFoldl Bench.BindChain; do - rm -f "bench/_build/$module.lua" ( cd test/ps "$pslua" \ @@ -25,7 +24,4 @@ for module in Bench.Fib Bench.ArrayFoldl Bench.BindChain; do --entry "$module" \ --lua-output-file "$root/bench/_build/$module.lua" ) - # pslua can report a linker error without a failing exit code; an existing - # non-empty artifact is the reliable success signal. - test -s "bench/_build/$module.lua" done From 2075836b9b6646ca627d03444ea66d0f8e533bab Mon Sep 17 00:00:00 2001 From: Yura Lazarev Date: Tue, 7 Jul 2026 10:47:24 +0200 Subject: [PATCH 6/6] chore: harden the bench harness against code-review findings (#172) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bench/link now rebuilds CoreFn unconditionally, clears stale _build artifacts, and derives its module list from test/ps/src/Bench/*.purs — the prior existence-check guard could silently link stale or missing CoreFn after editing a bench source. fnew_census.lua and trace_report.lua share their bytecode-walking logic via a new bc_lib.lua, derive the J*/I* opcode families from vmdef.bcnames instead of a hardcoded snapshot, and stamp jit.version into every report so a LuaJIT bump shows up as an attributable golden diff instead of a mystery one. The abort-reason formatter now falls back correctly for builtin functions, which carry no source info. Spec name fields are dropped in favor of deriving them from the file path, removing a filename/name drift the report headers depended on. bench_lib.measure guards against a zero/negative sample count. Goldens are regenerated to match. --- bench/README.md | 15 +++ bench/goldens/fnew_Bench.ArrayFoldl.txt | 1 + bench/goldens/fnew_Bench.BindChain.txt | 1 + bench/goldens/fnew_Bench.Fib.txt | 1 + bench/goldens/trace_array_foldl.txt | 9 +- bench/goldens/trace_bind_chain.txt | 7 +- bench/goldens/trace_fibonacci.txt | 5 +- bench/link | 12 +- bench/macro/array_foldl.lua | 1 - bench/macro/bind_chain.lua | 1 - bench/macro/fibonacci.lua | 1 - bench/micro/ctor_match.lua | 1 - bench/micro/curried_apply.lua | 1 - bench/micro/dict_compare.lua | 1 - bench/micro/field_access.lua | 1 - bench/tools/bc_lib.lua | 50 ++++++++ bench/tools/bench_lib.lua | 4 +- bench/tools/fnew_census.lua | 64 ++++------ bench/tools/run_macro.lua | 5 +- bench/tools/run_micro.lua | 3 +- bench/tools/trace_report.lua | 109 ++++++++---------- .../PureScript/Backend/Lua/Golden/Spec.hs | 5 + 22 files changed, 170 insertions(+), 128 deletions(-) create mode 100644 bench/tools/bc_lib.lua diff --git a/bench/README.md b/bench/README.md index d3b7083e..5a1fd2da 100644 --- a/bench/README.md +++ b/bench/README.md @@ -61,3 +61,18 @@ blacklist draws on an entropy-seeded PRNG, so counts jitter across runs while the final opcode state and the abort-site set do not. Blacklisting is never logged by `-jv`/`-jdump`; the post-hoc opcode read is the only stable way to observe it. + +Both reports record the LuaJIT version (`runtime:` header line): the abort +reasons, the NYI set, and the opcode families are properties of a specific +LuaJIT snapshot, so a toolchain bump that moves the counters shows up in +the golden diff as an attributable header change, not a mystery regression. + +Two caveats about golden stability. The trace reports pin source *lines* of +both the linked artifact and the macro spec file itself, so any edit to +`bench/macro/*.lua` — comments included — legitimately moves the goldens; +rerun `./bench/ci --accept` and review the diff. And one residual +nondeterminism channel exists: LuaJIT's hot-counters live in a small hashed +table keyed by bytecode address, so a rare cross-process aliasing change +can alter trace formation order. `./bench/ci` generates every report twice +and compares, precisely so that this manifests as a distinct "reports +differ between runs" failure rather than a confusing golden mismatch. diff --git a/bench/goldens/fnew_Bench.ArrayFoldl.txt b/bench/goldens/fnew_Bench.ArrayFoldl.txt index ca6e4e00..f7b79f65 100644 --- a/bench/goldens/fnew_Bench.ArrayFoldl.txt +++ b/bench/goldens/fnew_Bench.ArrayFoldl.txt @@ -1,4 +1,5 @@ chunk: Bench.ArrayFoldl.lua +runtime: LuaJIT 2.1.1741730670 main-chunk FNEW: 7 function-body FNEW: 12 total FNEW: 19 diff --git a/bench/goldens/fnew_Bench.BindChain.txt b/bench/goldens/fnew_Bench.BindChain.txt index aff7661a..d814bb96 100644 --- a/bench/goldens/fnew_Bench.BindChain.txt +++ b/bench/goldens/fnew_Bench.BindChain.txt @@ -1,4 +1,5 @@ chunk: Bench.BindChain.lua +runtime: LuaJIT 2.1.1741730670 main-chunk FNEW: 4 function-body FNEW: 4 total FNEW: 8 diff --git a/bench/goldens/fnew_Bench.Fib.txt b/bench/goldens/fnew_Bench.Fib.txt index 29276152..ebe63662 100644 --- a/bench/goldens/fnew_Bench.Fib.txt +++ b/bench/goldens/fnew_Bench.Fib.txt @@ -1,4 +1,5 @@ chunk: Bench.Fib.lua +runtime: LuaJIT 2.1.1741730670 main-chunk FNEW: 3 function-body FNEW: 2 total FNEW: 5 diff --git a/bench/goldens/trace_array_foldl.txt b/bench/goldens/trace_array_foldl.txt index 4a69cc5d..c88bec76 100644 --- a/bench/goldens/trace_array_foldl.txt +++ b/bench/goldens/trace_array_foldl.txt @@ -1,4 +1,5 @@ spec: array_foldl +runtime: LuaJIT 2.1.1741730670 workload: n=5000000 reps=2 result=12500002500000 aborts (distinct site -- reason): Bench.ArrayFoldl.lua:3 -- NYI: bytecode FNEW @@ -10,8 +11,8 @@ bytecode end state (J*=compiled, I*=blacklisted): Bench.ArrayFoldl.lua:35 JLOOP Bench.ArrayFoldl.lua:59 IFUNCF Bench.ArrayFoldl.lua:60 IFUNCF - array_foldl.lua:14 JFORI - array_foldl.lua:14 JFORL - array_foldl.lua:18 JFORI - array_foldl.lua:18 JFORL + array_foldl.lua:13 JFORI + array_foldl.lua:13 JFORL + array_foldl.lua:17 JFORI + array_foldl.lua:17 JFORL counts: aborts=2 compiled=6 blacklisted=4 diff --git a/bench/goldens/trace_bind_chain.txt b/bench/goldens/trace_bind_chain.txt index 2145de66..6c752539 100644 --- a/bench/goldens/trace_bind_chain.txt +++ b/bench/goldens/trace_bind_chain.txt @@ -1,4 +1,5 @@ spec: bind_chain +runtime: LuaJIT 2.1.1741730670 workload: n=1000000 reps=2 result=3000000 aborts (distinct site -- reason): Bench.BindChain.lua:19 -- NYI: bytecode FNEW @@ -12,7 +13,7 @@ bytecode end state (J*=compiled, I*=blacklisted): Bench.BindChain.lua:5 JFUNCF Bench.BindChain.lua:8 IFUNCF Bench.BindChain.lua:9 IFUNCF - bind_chain.lua:11 IFORL - bind_chain.lua:18 JFORI - bind_chain.lua:18 JFORL + bind_chain.lua:10 IFORL + bind_chain.lua:17 JFORI + bind_chain.lua:17 JFORL counts: aborts=2 compiled=4 blacklisted=7 diff --git a/bench/goldens/trace_fibonacci.txt b/bench/goldens/trace_fibonacci.txt index 9c20f7bd..488171ab 100644 --- a/bench/goldens/trace_fibonacci.txt +++ b/bench/goldens/trace_fibonacci.txt @@ -1,14 +1,15 @@ spec: fibonacci +runtime: LuaJIT 2.1.1741730670 workload: n=30 reps=2 result=832040 aborts (distinct site -- reason): Bench.Fib.lua:3 -- NYI: bytecode FNEW Bench.Fib.lua:6 -- NYI: bytecode FNEW - fibonacci.lua:12 -- call unroll limit reached + fibonacci.lua:11 -- call unroll limit reached bytecode end state (J*=compiled, I*=blacklisted): Bench.Fib.lua:3 IFUNCF Bench.Fib.lua:3 JFUNCF Bench.Fib.lua:6 IFUNCF Bench.Fib.lua:6 JFUNCF Bench.Fib.lua:8 JFUNCF - fibonacci.lua:12 JFUNCF + fibonacci.lua:11 JFUNCF counts: aborts=3 compiled=4 blacklisted=2 diff --git a/bench/link b/bench/link index 9dd3f25e..61c8adba 100755 --- a/bench/link +++ b/bench/link @@ -1,21 +1,23 @@ #!/usr/bin/env bash # Links the Bench.* PureScript modules into standalone Lua artifacts under -# bench/_build/, building pslua and the CoreFn output first when needed. +# bench/_build/, building pslua and the CoreFn output first. The module list +# is derived from test/ps/src/Bench/*.purs, and stale artifacts are removed, +# so bench/_build/Bench.*.lua always mirrors the current sources. set -euo pipefail cd "$(dirname "$0")/.." root=$PWD -if [ ! -f test/ps/output/Bench.ArrayFoldl/corefn.json ]; then - (cd test/ps && spago build) -fi +(cd test/ps && spago build) cabal build -v0 exe:pslua pslua=$(cabal list-bin pslua) mkdir -p bench/_build +rm -f bench/_build/Bench.*.lua # CoreFn module paths are relative to test/ps (where spago runs), and pslua # resolves foreign files against them, so it has to run from there too. -for module in Bench.Fib Bench.ArrayFoldl Bench.BindChain; do +for src in test/ps/src/Bench/*.purs; do + module="Bench.$(basename "$src" .purs)" ( cd test/ps "$pslua" \ diff --git a/bench/macro/array_foldl.lua b/bench/macro/array_foldl.lua index 4f5c26ba..b3fbc671 100644 --- a/bench/macro/array_foldl.lua +++ b/bench/macro/array_foldl.lua @@ -3,7 +3,6 @@ -- the tracer aborts on FNEW and blacklists the loop. The ideal variant does -- the same work — build the array, fold it — with an uncurried step. return { - name = "array_foldl", artifact = "Bench.ArrayFoldl", n = 5e6, drive = function(mod, n) diff --git a/bench/macro/bind_chain.lua b/bench/macro/bind_chain.lua index 487757c1..2fe91d37 100644 --- a/bench/macro/bind_chain.lua +++ b/bench/macro/bind_chain.lua @@ -3,7 +3,6 @@ -- hot loop here, so this also exercises what happens to a caller loop whose -- callee cannot be traced. return { - name = "bind_chain", artifact = "Bench.BindChain", n = 1e6, drive = function(mod, n) diff --git a/bench/macro/fibonacci.lua b/bench/macro/fibonacci.lua index 260b7883..717d4c35 100644 --- a/bench/macro/fibonacci.lua +++ b/bench/macro/fibonacci.lua @@ -2,7 +2,6 @@ -- Golden.Fibonacci.Test golden, linked as a module so the driver can call -- it without going through stdout. return { - name = "fibonacci", artifact = "Bench.Fib", n = 30, drive = function(mod, n) diff --git a/bench/micro/ctor_match.lua b/bench/micro/ctor_match.lua index cd2735a8..421b21a5 100644 --- a/bench/micro/ctor_match.lua +++ b/bench/micro/ctor_match.lua @@ -2,7 +2,6 @@ -- tag and the payload in the table's hash part, keyed by long strings, -- versus an array-part encoding with a small-integer tag. return { - name = "ctor_match", n = 2e6, variants = { { diff --git a/bench/micro/curried_apply.lua b/bench/micro/curried_apply.lua index df3616d0..20d18add 100644 --- a/bench/micro/curried_apply.lua +++ b/bench/micro/curried_apply.lua @@ -3,7 +3,6 @@ -- the loop never gets JIT-compiled. The uncurried variant compiles to a -- single trace. return { - name = "curried", n = 2e7, variants = { { diff --git a/bench/micro/dict_compare.lua b/bench/micro/dict_compare.lua index 985926c7..0efb1818 100644 --- a/bench/micro/dict_compare.lua +++ b/bench/micro/dict_compare.lua @@ -28,7 +28,6 @@ M.Data_Ord_greaterThanOrEq = function(dictOrd) end return { - name = "dict_gte", n = 5e6, variants = { { diff --git a/bench/micro/field_access.lua b/bench/micro/field_access.lua index 1ebdf86a..15d3ab54 100644 --- a/bench/micro/field_access.lua +++ b/bench/micro/field_access.lua @@ -3,7 +3,6 @@ -- into a local once. Both variants call uncurried so the difference is the -- field lookups alone. return { - name = "field_access", n = 1e7, variants = { { diff --git a/bench/tools/bc_lib.lua b/bench/tools/bc_lib.lua new file mode 100644 index 00000000..6b1307c3 --- /dev/null +++ b/bench/tools/bc_lib.lua @@ -0,0 +1,50 @@ +-- Shared LuaJIT bytecode introspection for the counter tools: opcode names +-- from jit.vmdef and a prototype walk shaped like jit/bc.lua's — iterate +-- jit.util.funcbc until nil, recurse into child prototypes via negative +-- jit.util.funck indices. The walk starts at pc 0, the FUNC* header slot, +-- so visitors see function-entry opcodes (their J*/I* rewrites are how +-- trace installation and blacklisting are observed). +local jutil = require("jit.util") +local vmdef = require("jit.vmdef") +local band = require("bit").band + +local M = {} + +-- vmdef.bcnames is a string of fixed-width 6-character cells, one per +-- opcode number. +function M.bcname(op) + return (vmdef.bcnames:sub(op * 6 + 1, op * 6 + 6):gsub("%s+$", "")) +end + +function M.basename(source) + return (source:gsub("^@", ""):gsub("^.*/", "")) +end + +-- Depth-first over proto and its children, calling visit(proto, pc, op, +-- depth) for every bytecode. Depth 0 is the outermost (main-chunk) +-- prototype; everything below it is a function body. +function M.walk(proto, visit, depth) + depth = depth or 0 + local pc = 0 + while true do + local ins = jutil.funcbc(proto, pc) + if not ins then + break + end + visit(proto, pc, M.bcname(band(ins, 0xff)), depth) + pc = pc + 1 + end + if jutil.funcinfo(proto).children then + for n = -1, -1e9, -1 do + local k = jutil.funck(proto, n) + if not k then + break + end + if type(k) == "proto" then + M.walk(k, visit, depth + 1) + end + end + end +end + +return M diff --git a/bench/tools/bench_lib.lua b/bench/tools/bench_lib.lua index b00a253a..0f47406d 100644 --- a/bench/tools/bench_lib.lua +++ b/bench/tools/bench_lib.lua @@ -3,6 +3,7 @@ -- os.clock() measures CPU time, not wall time. For these CPU-bound loops -- that is the right metric (time the process spends scheduled out does not -- pollute samples), but it would silently under-report anything I/O-bound. +-- luacheck: read globals jit local M = {} -- Runs fn(n) once untimed (lets LuaJIT compile traces: a loop becomes a @@ -12,6 +13,7 @@ local M = {} -- cannot move the headline number. function M.measure(fn, n, samples) samples = samples or 5 + assert(samples >= 1, "samples must be >= 1") local checksum = fn(n) local times = {} for s = 1, samples do @@ -46,7 +48,7 @@ end function M.report(bench_name, variant_name, n, r) io.write(string.format( - "%-12s %-8s %-12s n=%-9.0f median=%8.4fs min=%8.4fs max=%8.4fs result=%s\n", + "%-13s %-8s %-12s n=%-9.0f median=%8.4fs min=%8.4fs max=%8.4fs result=%s\n", bench_name, variant_name, M.runtime_tag(), diff --git a/bench/tools/fnew_census.lua b/bench/tools/fnew_census.lua index bbc03439..bf952c87 100644 --- a/bench/tools/fnew_census.lua +++ b/bench/tools/fnew_census.lua @@ -1,69 +1,45 @@ -- Static FNEW census over a Lua chunk: counts closure-creation bytecodes --- without executing anything, so the output is a pure function of the file. +-- without executing anything, so the output is a pure function of the file +-- and the LuaJIT version (recorded in the report header). -- -- An FNEW in the main chunk runs once, at load time; an FNEW in a function -- body runs on every call of that function, and aborts LuaJIT trace -- recording (closure creation is NYI in the 2.1 tracer). The split is a -- structural property of the prototype an instruction lives in: the main -- chunk is the outermost prototype, everything reachable through child --- prototypes is a function body. Prototypes are walked the way jit/bc.lua --- does: iterate jit.util.funcbc until nil, recurse into children via --- negative jit.util.funck indices. +-- prototypes is a function body. -- -- usage: luajit fnew_census.lua +-- luacheck: read globals jit local jutil = require("jit.util") -local vmdef = require("jit.vmdef") -local band = require("bit").band - -local function bcname(op) - return (vmdef.bcnames:sub(op * 6 + 1, op * 6 + 6):gsub("%s+$", "")) -end - -local function basename(source) - return (source:gsub("^@", ""):gsub("^.*/", "")) -end +local here = arg[0]:match("^(.*)/[^/]+$") or "." +local bc = dofile(here .. "/bc_lib.lua") local counts = { main = 0, body = 0, protos = 0 } local body_sites = {} -local function walk(proto, in_main) - counts.protos = counts.protos + 1 - local pc = 1 - while true do - local ins = jutil.funcbc(proto, pc) - if not ins then - break - end - if bcname(band(ins, 0xff)) == "FNEW" then - if in_main then - counts.main = counts.main + 1 - else - counts.body = counts.body + 1 - local info = jutil.funcinfo(proto, pc) - body_sites[#body_sites + 1] = - string.format("%s:%d", basename(info.source), info.currentline) - end - end - pc = pc + 1 +local function visit(proto, pc, op, depth) + if pc == 0 then + counts.protos = counts.protos + 1 end - if jutil.funcinfo(proto).children then - for n = -1, -1e9, -1 do - local k = jutil.funck(proto, n) - if not k then - break - end - if type(k) == "proto" then - walk(k, false) - end + if op == "FNEW" then + if depth == 0 then + counts.main = counts.main + 1 + else + counts.body = counts.body + 1 + local info = jutil.funcinfo(proto, pc) + body_sites[#body_sites + 1] = + string.format("%s:%d", bc.basename(info.source), info.currentline) end end end local path = assert(arg[1], "usage: fnew_census.lua ") local chunk = assert(loadfile(path)) -walk(chunk, true) +bc.walk(chunk, visit) -io.write("chunk: ", basename("@" .. path), "\n") +io.write("chunk: ", bc.basename("@" .. path), "\n") +io.write("runtime: ", jit.version, "\n") io.write("main-chunk FNEW: ", counts.main, "\n") io.write("function-body FNEW: ", counts.body, "\n") io.write("total FNEW: ", counts.main + counts.body, "\n") diff --git a/bench/tools/run_macro.lua b/bench/tools/run_macro.lua index a24a31c0..499e2b5d 100644 --- a/bench/tools/run_macro.lua +++ b/bench/tools/run_macro.lua @@ -7,6 +7,7 @@ local here = arg[0]:match("^(.*)/[^/]+$") or "." local lib = dofile(here .. "/bench_lib.lua") local spec_path = assert(arg[1], "usage: run_macro.lua [n] [samples]") +local spec_name = spec_path:match("([^/]+)%.lua$") local spec = dofile(spec_path) local mod = dofile(here .. "/../_build/" .. spec.artifact .. ".lua") local n = tonumber(arg[2]) or spec.n @@ -15,5 +16,5 @@ local samples = tonumber(arg[3]) local linked = function(k) return spec.drive(mod, k) end -lib.report(spec.name, "linked", n, lib.measure(linked, n, samples)) -lib.report(spec.name, "ideal", n, lib.measure(spec.ideal, n, samples)) +lib.report(spec_name, "linked", n, lib.measure(linked, n, samples)) +lib.report(spec_name, "ideal", n, lib.measure(spec.ideal, n, samples)) diff --git a/bench/tools/run_micro.lua b/bench/tools/run_micro.lua index c0de22da..e47e9d34 100644 --- a/bench/tools/run_micro.lua +++ b/bench/tools/run_micro.lua @@ -5,10 +5,11 @@ local here = arg[0]:match("^(.*)/[^/]+$") or "." local lib = dofile(here .. "/bench_lib.lua") local spec_path = assert(arg[1], "usage: run_micro.lua [n] [samples]") +local spec_name = spec_path:match("([^/]+)%.lua$") local spec = dofile(spec_path) local n = tonumber(arg[2]) or spec.n local samples = tonumber(arg[3]) for _, variant in ipairs(spec.variants) do - lib.report(spec.name, variant.name, n, lib.measure(variant.fn, n, samples)) + lib.report(spec_name, variant.name, n, lib.measure(variant.fn, n, samples)) end diff --git a/bench/tools/trace_report.lua b/bench/tools/trace_report.lua index 1ce11c33..6a730509 100644 --- a/bench/tools/trace_report.lua +++ b/bench/tools/trace_report.lua @@ -13,46 +13,45 @@ -- post-hoc read is the only stable way to observe it. -- -- usage: luajit trace_report.lua +-- luacheck: read globals jit local jutil = require("jit.util") local vmdef = require("jit.vmdef") -local band = require("bit").band - -local function bcname(op) - return (vmdef.bcnames:sub(op * 6 + 1, op * 6 + 6):gsub("%s+$", "")) -end - -local function basename(source) - return (source:gsub("^@", ""):gsub("^.*/", "")) -end +local here = arg[0]:match("^(.*)/[^/]+$") or "." +local bc = dofile(here .. "/bc_lib.lua") local function location(func, pc) local info = jutil.funcinfo(func, pc) - return basename(info.source), info.currentline or 0 + return bc.basename(info.source), info.currentline or 0 end --- The J*/I* opcode families observable after a run. FORI has a J form only --- (trace entry at a loop start); the interpreted-fallback forms exist for --- hot-countable spots: loops and function entries. -local COMPILED = { - JFORI = true, - JFORL = true, - JITERL = true, - JLOOP = true, - JFUNCF = true, - JFUNCV = true, -} -local BLACKLISTED = { - IFORL = true, - IITERL = true, - ILOOP = true, - IFUNCF = true, - IFUNCV = true, -} +-- The J*/I* rewrite families, derived from vmdef.bcnames so a LuaJIT bump +-- that adds a hot-countable pair shows up instead of being silently +-- skipped: an opcode named J or I where X is itself an opcode is a +-- rewrite form of X. (FORI has a J form only — the trace entry at a loop +-- start; the I* fallbacks exist for hot-countable spots: loops and +-- function entries.) +local COMPILED, BLACKLISTED = {}, {} +do + local names = {} + for op = 0, #vmdef.bcnames / 6 - 1 do + names[bc.bcname(op)] = true + end + for name in pairs(names) do + local prefix, base = name:sub(1, 1), name:sub(2) + if names[base] then + if prefix == "J" then + COMPILED[name] = true + elseif prefix == "I" then + BLACKLISTED[name] = true + end + end + end +end local spec_path = assert(arg[1], "usage: trace_report.lua ") +local spec_name = spec_path:match("([^/]+)%.lua$") local spec_chunk = assert(loadfile(spec_path)) local spec = spec_chunk() -local here = arg[0]:match("^(.*)/[^/]+$") or "." local artifact_path = here .. "/../_build/" .. spec.artifact .. ".lua" local artifact_chunk = assert(loadfile(artifact_path)) local mod = artifact_chunk() @@ -73,10 +72,21 @@ local function on_trace(what, _tr, func, pc, code, extra) return end if type(extra) == "number" and msg:find("bytecode") then - extra = bcname(extra) + extra = bc.bcname(extra) elseif type(extra) == "function" then - local file, line = location(extra) - extra = file .. ":" .. line + -- Builtins carry no source (funcinfo gives ffid/addr instead), so + -- fall back the way jit/dump.lua's fmtfunc does. + local info = jutil.funcinfo(extra) + if info.source then + local file, line = location(extra) + extra = file .. ":" .. line + elseif info.ffid then + extra = vmdef.ffnames[info.ffid] + elseif info.addr then + extra = string.format("C:%x", info.addr) + else + extra = "(?)" + end end reason = msg:find("%%") and string.format(msg, extra) or msg end @@ -93,34 +103,14 @@ end jit.attach(on_trace) local states = {} -local function walk(proto) - local pc = 0 - while true do - local ins = jutil.funcbc(proto, pc) - if not ins then - break - end - local op = bcname(band(ins, 0xff)) - if COMPILED[op] or BLACKLISTED[op] then - local file, line = location(proto, pc) - states[string.format("%s:%d %s", file, line, op)] = op - end - pc = pc + 1 - end - if jutil.funcinfo(proto).children then - for n = -1, -1e9, -1 do - local k = jutil.funck(proto, n) - if not k then - break - end - if type(k) == "proto" then - walk(k) - end - end +local function visit(proto, pc, op) + if COMPILED[op] or BLACKLISTED[op] then + local file, line = location(proto, pc) + states[string.format("%s:%d %s", file, line, op)] = op end end -walk(artifact_chunk) -walk(spec_chunk) +bc.walk(artifact_chunk, visit) +bc.walk(spec_chunk, visit) local function sorted_keys(set) local keys = {} @@ -131,7 +121,8 @@ local function sorted_keys(set) return keys end -io.write("spec: ", spec.name, "\n") +io.write("spec: ", spec_name, "\n") +io.write("runtime: ", jit.version, "\n") io.write(string.format("workload: n=%.0f reps=2 result=%s\n", spec.n, tostring(checksum))) io.write("aborts (distinct site -- reason):\n") local abort_keys = sorted_keys(aborts) diff --git a/test/Language/PureScript/Backend/Lua/Golden/Spec.hs b/test/Language/PureScript/Backend/Lua/Golden/Spec.hs index 70213e9f..1c60aa38 100644 --- a/test/Language/PureScript/Backend/Lua/Golden/Spec.hs +++ b/test/Language/PureScript/Backend/Lua/Golden/Spec.hs @@ -264,6 +264,11 @@ luacParse src = (code, out) ← readProcessInterleaved (proc "luac" ["-p", toFilePath file]) pure (code, decodeUtf8 out) +{- | Corefn files that participate in golden tests: only Golden.* modules. +Other modules compiled from test/ps/src pass through uncollected — +bench/link relies on this to link the Bench.* corefns without them +entering the suite. +-} collectGoldenCorefns ∷ MonadIO m ⇒ Path Rel Dir → m [Path Abs File] collectGoldenCorefns = walkDirAccum Nothing -- Descend into every directory