From dfa59f34ac3ceca7c6e0f4ef7e54ea691043cd57 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Fri, 31 Jul 2026 14:32:00 -0500 Subject: [PATCH] Add an asv benchmark suite 101 benchmarks over the library's hot paths: scalar access (getitem, get and contains on hit and miss, setitem), index parsing, small-op overhead on 10/100/1000-element objects, large kernels at ~1e6 nnz (mxm, mxv, ewise, reduce, single-threaded via OMP_NUM_THREADS=1), coo/dense/scipy conversions, cold import and first-use timings in fresh subprocesses, and repr costs. `benchmarks/_verify.py` is a standalone checker rather than an asv benchmark: it imports every module the way asv does and calls each benchmark once across all parameter combinations. It reports 101 passed, 0 failed, 0 skipped. No timing numbers are claimed; what this commit establishes is that the benchmarks exist and run. `asv.conf.json` uses `//` comments. That is asv's own documented config format and its loader strips them, but the repo's `check-json` hook is a strict JSON parser and rejects the file, so that one path is excluded from the hook and the hook's stale "no JSON files yet" comment is replaced. Verified with `pre-commit run check-json --all-files`, which passes. Two alternatives were considered and not taken. Renaming to `asv.conf.jsonc` needs no exclude and asv 0.6.5 does resolve it (`Config.load` accepts `.json` and `.jsonc`), but asv is not a pinned dependency here and older versions look only for the `.json` name, which would fail as "No `asv.conf` file found". Stripping the comments would delete the only explanation of why each setting is set as it is. The exclude names the single path, so every other JSON file is still checked. asv is deliberately not added to `dev-requirements.txt` or `environment.yml`. Nothing in the test suite or CI invokes it, and this repo ties dependency changes to `scripts/check_versions.sh` and the CI version pools, which a benchmarks-only change should not be editing. The README carries the `pip install asv` line and the reasoning. CI wiring is also left alone: a weekly cron on a fixed machine suits asv far better than per-PR runs on shared runners. The lint configuration for benchmarks/ rides in this commit because the directory it configures arrives here: flake8 and ruff ignore B015/B018 (a bare expression IS what an asv benchmark measures) and T201 for the benchmark CLI, and codespell learns "bu". --- .flake8 | 1 + .pre-commit-config.yaml | 6 +- asv.conf.json | 73 ++++++++++++++++ benchmarks/README.md | 91 +++++++++++++++++++ benchmarks/__init__.py | 12 +++ benchmarks/_verify.py | 107 +++++++++++++++++++++++ benchmarks/common.py | 91 +++++++++++++++++++ benchmarks/conversions.py | 98 +++++++++++++++++++++ benchmarks/imports.py | 35 ++++++++ benchmarks/index_parse.py | 90 +++++++++++++++++++ benchmarks/large_kernels.py | 73 ++++++++++++++++ benchmarks/repr_bench.py | 51 +++++++++++ benchmarks/scalar_access.py | 168 ++++++++++++++++++++++++++++++++++++ benchmarks/small_ops.py | 72 ++++++++++++++++ pyproject.toml | 3 +- 15 files changed, 969 insertions(+), 2 deletions(-) create mode 100644 asv.conf.json create mode 100644 benchmarks/README.md create mode 100644 benchmarks/__init__.py create mode 100755 benchmarks/_verify.py create mode 100644 benchmarks/common.py create mode 100644 benchmarks/conversions.py create mode 100644 benchmarks/imports.py create mode 100644 benchmarks/index_parse.py create mode 100644 benchmarks/large_kernels.py create mode 100644 benchmarks/repr_bench.py create mode 100644 benchmarks/scalar_access.py create mode 100644 benchmarks/small_ops.py diff --git a/.flake8 b/.flake8 index 959e9a22c..9ba580871 100644 --- a/.flake8 +++ b/.flake8 @@ -10,6 +10,7 @@ extend-ignore = SIM401, # E203 whitespace before ':' (to be compatible with black) per-file-ignores = + benchmarks/*.py:B015,B018, scripts/create_pickle.py:F403,F405, graphblas/tests/*.py:T201,B043, graphblas/core/ss/matrix.py:SIM113, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3582142d2..b9603a6b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,11 @@ repos: - id: check-illegal-windows-names - id: check-merge-conflict - id: check-ast - - id: check-json # no JSON files yet; enabled for the future + # `asv.conf.json` is JSONC: asv documents `//` comments and strips them + # in its own loader, but this hook is a strict JSON parser. Excluding + # that one path keeps the hook live for every other JSON file. + - id: check-json + exclude: ^asv\.conf\.json$ - id: check-toml - id: check-yaml - id: check-executables-have-shebangs diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 000000000..acd395737 --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,73 @@ +{ + // asv (airspeed velocity) config for python-graphblas. + // + // Placement: this file and the benchmarks/ directory are intended to live at the + // repo root (next to pyproject.toml). The paths below are relative to that root. + // asv's config loader strips these // line comments; a strict JSON linter will + // not. If the repo's pre-commit `check-json` hook is enabled, exclude this file + // (asv.conf.json is asv's own format, and its docs use // comments). + // + // Typical use: + // asv run # benchmark current commit + // asv continuous main HEAD # compare a branch against main + // asv publish && asv preview # build/serve the HTML report + + "version": 1, + "project": "python-graphblas", + "project_url": "https://github.com/python-graphblas/python-graphblas", + + // "." means the repo that contains this file. python-graphblas is pure Python, so + // building each commit is a cheap editable install; the C library comes from the + // separate suitesparse-graphblas package, pinned in the matrix below. + "repo": ".", + "branches": ["main"], + "dvcs": "git", + + // conda matches the project's primary dev path and lets us pin the C library + // cleanly from conda-forge. asv will use mamba automatically if it is installed. + "environment_type": "conda", + "conda_channels": ["conda-forge"], + + // PINNING NOTE: python-suitesparse-graphblas (the C library) is pinned so a run + // measures python-graphblas-side regressions in isolation. To instead track + // end-to-end performance including C-library changes, drop the pin (set it to []) + // or bump it deliberately. Keep this in sync with the PSG version the suite is + // calibrated against (see scripts/ci_pick_versions.py PSG pools). + "matrix": { + "req": { + "python-suitesparse-graphblas": ["10.3.1.0"], + "numpy": [], + "scipy": [], + "pandas": [], + "numba": [] + } + }, + + // python-graphblas is pure Python: a no-build-isolation editable install is fast + // and reuses the already-solved conda env (which supplies the C library plus + // numba/scipy/pandas). asv substitutes {build_dir} for the checkout of each commit. + "build_command": [], + "install_command": [ + "in-dir={build_dir} python -m pip install --no-build-isolation --no-deps -e ." + ], + "uninstall_command": [ + "in-dir={build_dir} python -m pip uninstall -y python-graphblas" + ], + + "benchmark_dir": "benchmarks", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + + // Give the C-library install room to solve/download. + "install_timeout": 900, + + // Isolate JIT/compiler and thread-count noise so numbers are comparable across + // runs. GraphBLAS is multithreaded by default; pin to 1 thread for reproducible + // single-core timings (raise this in a dedicated run if you want to benchmark + // parallel scaling instead). + "environment_variables": { + "OMP_NUM_THREADS": "1", + "GRAPHBLAS_TEST_SEED": "0" + } +} diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..5962927f7 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,91 @@ +# asv benchmark scaffold for python-graphblas + +A starting airspeed velocity (asv) benchmark suite covering the library's hot +paths. Built as a scaffold: review, then move `asv.conf.json` and `benchmarks/` +to the repo root (next to `pyproject.toml`). `_verify.py` is a local sanity +checker and does not need to ship. + +## Layout + +``` +asv.conf.json asv config, tuned for this repo (see caveats below). + JSONC: asv documents `//` comments and strips them in + its loader, so `.pre-commit-config.yaml` excludes this + one path from the strict `check-json` hook. asv also + accepts `asv.conf.jsonc`, which would need no exclude, + but asv is not a pinned dependency here and older + versions resolve only the `.json` name. +benchmarks/ + __init__.py package marker + note on the dual-import shim + common.py shared, seeded data builders (called from setup, not timed) + scalar_access.py single-element get / extract / assign, hit vs miss + index_parse.py parse_index int fast lane vs numpy-int lane (index build) + small_ops.py ewise / apply / reduce on ~100-element objects (overhead) + large_kernels.py mxm / mxv / ewise / reduce on ~1e6 nnz (C-library bound) + conversions.py from_coo/to_coo, from_dense/to_dense, scipy interop + imports.py cold import + first-use timing (timeraw, fresh subprocess) + repr_bench.py repr / _repr_html_ for small and large objects +_verify.py standalone check: runs every benchmark once (not an asv file) +``` + +## Running + +```bash +pip install asv +asv machine --yes # one-time: record machine info +asv run # benchmark the current commit +asv continuous main HEAD # compare a branch against main, flag regressions +asv publish && asv preview # build and serve the HTML report +asv run --bench scalar_access # run a subset by regex +``` + +Quick correctness check without asv (runs each benchmark once): + +```bash +python _verify.py +``` + +## Design choices + +- `setup()` builds all data once, outside the timed region; every builder is + seeded so runs are comparable across commits. +- Sizes: "small" is 10 to 1000 elements (overhead-dominated, so we track + Python-side cost); "large" is ~1e6 nonzeros (C-library dominated). Average + matrix degree is ~1 so `mxm` output stays near the input size instead of + exploding into quadratic fill-in. Verified per-call times on one dev machine: + large kernels ran in 1 to 5 ms, conversions under 10 ms, cold `import +graphblas` ~32 ms, init/first-operator ~0.2 s. Bump `common.LARGE_NNZ` or add a + higher-degree matrix if you want heavier `mxm`. +- Large kernels use `number = 1`, `warmup_time = 0`, small `repeat`, and a + `timeout`, so asv does not batch them into multi-second runs. +- `imports.py` uses `timeraw_*`, which runs the returned code string in a fresh + subprocess. That is the only way to measure true cold import cost; a normal + benchmark would read ~0 because the module is already imported. +- `asv.conf.json` pins `python-suitesparse-graphblas` so a run isolates + python-graphblas-side regressions from C-library changes. Drop the pin to track + end-to-end (library + C) performance. Keep the pinned version in sync with the + PSG pools in `scripts/ci_pick_versions.py`. +- `OMP_NUM_THREADS=1` pins GraphBLAS to one thread for reproducible single-core + timings. Raise it in a dedicated run to benchmark parallel scaling. + +## Caveats to resolve before merging + +- **dev deps**: `asv` is deliberately NOT added to `dev-requirements.txt` or + `environment.yml`. Nothing in the test suite or CI invokes it, so adding it + would make every contributor's dev install fetch a package only used by + people who opt into benchmarking; and this repo ties dependency changes to + `scripts/check_versions.sh` and the CI version pools, which a benchmarks-only + change has no business editing. `pip install asv` is in the usage section + above. Adding a `benchmark` optional-dependencies group (so that + `pip install python-graphblas[benchmark]` works) is the natural move if the + suite graduates from scaffold to something CI runs. +- **CI wiring**: asv is not wired into CI here. A weekly cron running `asv +continuous` against a fixed machine (or asv's own regression detection) is the + natural follow-up; per-PR benchmarking on shared GitHub runners is too noisy to + gate on. +- **First-run JIT/compile noise**: the first use of some operators triggers + numba/JIT work. `setup()` touches the operators the benchmark uses, but the + very first `asv run` on a fresh env may still show inflated one-off numbers; + asv's repeats and its own warmup mitigate this. +- **pandas dependency**: `repr_bench.py` needs pandas (the default/test extras + include it). The env matrix installs pandas, so this is satisfied. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..dd8e37ffb --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1,12 @@ +"""python-graphblas benchmark suite (airspeed velocity). + +Benchmark modules live alongside this file. Shared data builders are in +``common.py``. Each benchmark module imports ``common`` with a dual path so it +works whether asv imports the files as a package or adds ``benchmark_dir`` to +``sys.path`` and imports them flat: + + try: + from . import common + except ImportError: # imported flat, not as a package + import common +""" diff --git a/benchmarks/_verify.py b/benchmarks/_verify.py new file mode 100755 index 000000000..cb9ccd5d4 --- /dev/null +++ b/benchmarks/_verify.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +"""Standalone sanity check for the asv benchmark suite (not an asv benchmark). + +Imports every benchmark module the way asv does (benchmark_dir on sys.path), +then for each benchmark class runs setup() and calls each benchmark method once +across all parameter combinations. timeraw_* methods return a code string, which +is executed in a fresh subprocess. Prints PASS/FAIL per benchmark and a timing so +we can spot anything accidentally slow. + +Usage: python _verify.py +""" + +import importlib +import inspect +import itertools +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +BENCH_DIR = HERE / "benchmarks" +sys.path.insert(0, str(BENCH_DIR)) # flat import mode, as asv does + +MODULES = [ + "scalar_access", + "index_parse", + "small_ops", + "large_kernels", + "conversions", + "imports", + "repr_bench", +] +PREFIXES = ("time_", "timeraw_", "peakmem_", "mem_", "track_") + + +def param_combos(cls): + params = getattr(cls, "params", None) + if not params: + return [()] + # asv: a flat list is a single parameter; a list of lists is a product. + if params and isinstance(params[0], (list, tuple)): + return list(itertools.product(*params)) + return [(p,) for p in params] + + +def run_one(cls, combo): + results = [] + obj = cls() + if hasattr(obj, "setup"): + obj.setup(*combo) + for name in sorted(dir(obj)): + if not name.startswith(PREFIXES): + continue + method = getattr(obj, name) + if not callable(method): + continue + label = f"{cls.__module__}.{cls.__name__}.{name}{combo or ''}" + t0 = time.perf_counter() + try: + if name.startswith("timeraw_"): + code = method() + setup_code = "" + if isinstance(code, tuple): + code, setup_code = code + script = (setup_code + "\n" + code) if setup_code else code + subprocess.run( + [sys.executable, "-c", script], check=True, capture_output=True, timeout=180 + ) + else: + method(*combo) + dt = time.perf_counter() - t0 + results.append((label, dt, None)) + except Exception as exc: + dt = time.perf_counter() - t0 + results.append((label, dt, repr(exc))) + if hasattr(obj, "teardown"): + obj.teardown(*combo) + return results + + +def main(): + all_results = [] + for modname in MODULES: + mod = importlib.import_module(modname) + classes = [ + c for _, c in inspect.getmembers(mod, inspect.isclass) if c.__module__ == modname + ] + for cls in classes: + for combo in param_combos(cls): + all_results.extend(run_one(cls, combo)) + + fails = [r for r in all_results if r[2] is not None] + for label, dt, err in all_results: + status = "PASS" if err is None else "FAIL" + line = f"[{status}] {dt:7.3f}s {label}" + if err is not None: + line += f" -> {err}" + print(line) + print( + f"\n{len(all_results) - len(fails)}/{len(all_results)} benchmarks ran; {len(fails)} failed" + ) + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/common.py b/benchmarks/common.py new file mode 100644 index 000000000..44db815fa --- /dev/null +++ b/benchmarks/common.py @@ -0,0 +1,91 @@ +"""Shared data builders for the python-graphblas asv benchmark suite. + +Each benchmark module imports these helpers and calls them from ``setup`` so the +input data is constructed once per benchmark (outside the timed region). All +builders take a seed and use ``numpy.random.default_rng`` so runs are +deterministic and comparable across commits. + +Size constants are chosen so the "large" kernels land near ~1e6 nonzeros (enough +to exercise the C library) while still finishing in well under a second each, and +so no dense intermediate blows up memory (a 1e6 x 1e6 dense matrix is never +materialized; dense conversions use a separate, small, fully dense matrix). +""" + +import numpy as np + +import graphblas as gb +from graphblas import Matrix, Vector + +# Sizes + +# "Large" sparse operands: square matrix and vector near 1e6 nonzeros. Average +# degree ~1 keeps mxm output bounded (A @ A stays near 1e6 nnz) so the kernel +# benchmarks do not accidentally measure quadratic fill-in. +LARGE_N = 1_000_000 +LARGE_NNZ = 1_000_000 + +# "Small" operands: overhead-dominated. At this size the wall time is almost all +# Python-side expression machinery, which is exactly what we want to track. +SMALL_SIZE = 100 + +# Dense conversions use a fully dense square matrix small enough to materialize. +DENSE_DIM = 1000 # 1e6 dense elements + +# scipy interop matrix: ~1e6 nnz at 1% density. +SCIPY_DIM = 10_000 +SCIPY_DENSITY = 0.01 + + +# Builders + + +def make_coo(n, nnz, seed=0, dtype="FP64"): + """Return (rows, cols, vals) numpy arrays for an n x n matrix with ~nnz entries. + + Duplicates are possible; callers that build a Matrix should pass a ``dup_op``. + """ + rng = np.random.default_rng(seed) + rows = rng.integers(0, n, nnz, dtype=np.uint64) + cols = rng.integers(0, n, nnz, dtype=np.uint64) + if dtype == "BOOL": + vals = rng.integers(0, 2, nnz, dtype=np.bool_) + elif dtype in ("INT64", "INT32"): + vals = rng.integers(1, 100, nnz, dtype=np.int64) + else: + vals = rng.random(nnz) + return rows, cols, vals + + +def make_matrix(n=LARGE_N, nnz=LARGE_NNZ, seed=0, dtype="FP64"): + """Square Matrix with ~nnz entries (duplicates summed).""" + rows, cols, vals = make_coo(n, nnz, seed=seed, dtype=dtype) + dup = gb.binary.lor if dtype == "BOOL" else gb.binary.plus + return Matrix.from_coo(rows, cols, vals, nrows=n, ncols=n, dtype=dtype, dup_op=dup) + + +def make_vector(size=LARGE_N, nnz=LARGE_NNZ, seed=1, dtype="FP64"): + """Vector of length ``size`` with ~nnz entries (duplicates summed).""" + rng = np.random.default_rng(seed) + idx = rng.integers(0, size, nnz, dtype=np.uint64) + vals = rng.random(nnz) if dtype not in ("INT64", "INT32") else rng.integers(1, 100, nnz) + dup = gb.binary.plus + return Vector.from_coo(idx, vals, size=size, dtype=dtype, dup_op=dup) + + +def make_dense_vector(size, seed=2, dtype="FP64"): + """Fully dense Vector (no missing entries), e.g. an mxv operand.""" + rng = np.random.default_rng(seed) + return Vector.from_dense(rng.random(size)) + + +def make_dense_matrix(dim=DENSE_DIM, seed=3): + """Fully dense square Matrix built from a numpy array.""" + rng = np.random.default_rng(seed) + return Matrix.from_dense(rng.random((dim, dim))) + + +def make_scipy(dim=SCIPY_DIM, density=SCIPY_DENSITY, seed=4, fmt="csr"): + """A scipy.sparse matrix for interop benchmarks.""" + import scipy.sparse as sp + + return sp.random(dim, dim, density=density, format=fmt, random_state=seed) diff --git a/benchmarks/conversions.py b/benchmarks/conversions.py new file mode 100644 index 000000000..90be6c986 --- /dev/null +++ b/benchmarks/conversions.py @@ -0,0 +1,98 @@ +"""Import/export conversions between graphblas objects and numpy/scipy formats. + +Sparse conversions (from_coo/to_coo, scipy interop) run on ~1e6 nonzeros. Dense +conversions use a separate fully dense 1000x1000 matrix so nothing materializes a +1e6 x 1e6 dense array. +""" + +import numpy as np + +from graphblas import Matrix, Vector, io + +try: + from . import common +except ImportError: + import common + + +class MatrixCoo: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.rows, self.cols, self.vals = common.make_coo(common.LARGE_N, common.LARGE_NNZ, seed=40) + self.M = common.make_matrix(seed=40) + + def time_from_coo(self): + Matrix.from_coo( + self.rows, + self.cols, + self.vals, + nrows=common.LARGE_N, + ncols=common.LARGE_N, + dup_op="plus", + ) + + def time_to_coo(self): + self.M.to_coo() + + +class MatrixDense: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.dense = np.random.default_rng(41).random((common.DENSE_DIM, common.DENSE_DIM)) + self.M = Matrix.from_dense(self.dense) + + def time_from_dense(self): + Matrix.from_dense(self.dense) + + def time_to_dense(self): + self.M.to_dense() + + +class MatrixScipy: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.sp = common.make_scipy(seed=42) + self.M = io.from_scipy_sparse(self.sp) + + def time_from_scipy_sparse(self): + io.from_scipy_sparse(self.sp) + + def time_to_scipy_sparse(self): + io.to_scipy_sparse(self.M, format="csr") + + +class VectorConvert: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.v = common.make_vector(seed=43) + self.idx, self.vals = self.v.to_coo() + self.dense_arr = np.random.default_rng(44).random(common.LARGE_N) + self.dv = Vector.from_dense(self.dense_arr) + + def time_from_coo(self): + Vector.from_coo(self.idx, self.vals, size=common.LARGE_N, dup_op="plus") + + def time_to_coo(self): + self.v.to_coo() + + def time_from_dense(self): + Vector.from_dense(self.dense_arr) + + def time_to_dense(self): + self.dv.to_dense() diff --git a/benchmarks/imports.py b/benchmarks/imports.py new file mode 100644 index 000000000..9204b26a2 --- /dev/null +++ b/benchmarks/imports.py @@ -0,0 +1,35 @@ +"""Import and cold-start timing. + +These use asv's ``timeraw_*`` form: each returns a code string that asv runs in a +*fresh* subprocess, so the measurement is the true cold cost (module already +imported into the benchmark process would otherwise read as ~0). ``timeraw`` +benchmarks cannot see ``setup`` state or this module's imports, so everything +they need must be inside the returned string. + +``graphblas`` initializes lazily: ``import graphblas`` is cheap, and the real +cost (loading the C library, building operator namespaces) is deferred until +first use. The staged benchmarks below separate those phases. +""" + + +class ImportTiming: + # A little headroom: importing numba/llvmlite on first operator use is not fast. + timeout = 120 + + def timeraw_import_graphblas(self): + return "import graphblas" + + def timeraw_from_import_core_types(self): + return "from graphblas import Matrix, Vector, Scalar" + + def timeraw_import_then_init(self): + # Force backend initialization (loads the C library). + return "import graphblas as gb; gb.init('suitesparse')" + + def timeraw_first_operator_access(self): + # First attribute access on an operator namespace triggers its lazy build. + return "import graphblas as gb; gb.binary.plus" + + def timeraw_first_matrix(self): + # End-to-end cold path: import, init, and build one tiny Matrix. + return "import graphblas as gb; gb.Matrix.from_coo([0], [0], [1.0], nrows=1, ncols=1)" diff --git a/benchmarks/index_parse.py b/benchmarks/index_parse.py new file mode 100644 index 000000000..5b5b0c856 --- /dev/null +++ b/benchmarks/index_parse.py @@ -0,0 +1,90 @@ +"""Index parsing overhead: the ``parse_index`` int fast lane. + +Every ``v[i]`` / ``A[i, j]`` builds an ``IndexerResolver`` that runs each index +through ``parse_index``. A plain Python ``int`` takes a dedicated fast lane that +skips two ``np.issubdtype`` checks (~110ns each) used by the numpy-integer lane. +These benchmarks isolate that parsing cost from element extraction (no ``.new()``, +no value read), so a regression in the fast lane, or an index accidentally +falling out of it, is visible on its own. + +Two granularities are measured: + +* the public path (``v[i]`` returns an index expression without resolving it), and +* ``IndexerResolver(obj, idx)`` directly, the tightest view of ``parse_index``. + +The plain-int and numpy-int variants sit side by side so the fast lane's margin +over the general lane is tracked directly. +""" + +import numpy as np + +from graphblas.core.expr import IndexerResolver + +try: + from . import common +except ImportError: # imported flat by asv, not as a package + import common + + +class VectorIndexParse: + """Parse a single vector index: plain-int fast lane vs numpy-int lane.""" + + def setup(self): + self.v = common.make_vector(size=10_000, nnz=1_000, seed=12) + self.i = 4321 # in range, positive + self.neg = -1 # in range, triggers the negative-wrap branch + self.npi = np.int64(4321) + for _ in range(3): + self.v[self.i] + self.v[self.neg] + self.v[self.npi] + IndexerResolver(self.v, self.i) + IndexerResolver(self.v, self.npi) + + # Public path: build the index expression (parse_index + expression object). + def time_getitem_int(self): + self.v[self.i] + + def time_getitem_int_negative(self): + self.v[self.neg] + + def time_getitem_numpy_int(self): + self.v[self.npi] + + # Tightest view: just the resolver (parse_index, no expression object). + def time_resolver_int(self): + IndexerResolver(self.v, self.i) + + def time_resolver_numpy_int(self): + IndexerResolver(self.v, self.npi) + + +class MatrixIndexParse: + """Parse a two-axis matrix index: plain-int fast lane vs numpy-int lane.""" + + def setup(self): + self.M = common.make_matrix(n=10_000, nnz=1_000, seed=13) + self.ij = (4321, 8765) + self.neg = (-1, -1) + self.npij = (np.int64(4321), np.int64(8765)) + for _ in range(3): + self.M[self.ij[0], self.ij[1]] + self.M[self.neg[0], self.neg[1]] + self.M[self.npij[0], self.npij[1]] + IndexerResolver(self.M, self.ij) + IndexerResolver(self.M, self.npij) + + def time_getitem_int(self): + self.M[self.ij[0], self.ij[1]] + + def time_getitem_int_negative(self): + self.M[self.neg[0], self.neg[1]] + + def time_getitem_numpy_int(self): + self.M[self.npij[0], self.npij[1]] + + def time_resolver_int(self): + IndexerResolver(self.M, self.ij) + + def time_resolver_numpy_int(self): + IndexerResolver(self.M, self.npij) diff --git a/benchmarks/large_kernels.py b/benchmarks/large_kernels.py new file mode 100644 index 000000000..f0c278409 --- /dev/null +++ b/benchmarks/large_kernels.py @@ -0,0 +1,73 @@ +"""Large kernels (~1e6 nonzeros): dominated by the C library, so these catch +suitesparse-graphblas-side regressions (and any python-graphblas overhead that +scales with data). + +Each op runs once per sample (``number = 1``) with no warmup, because the inputs +are large enough that a single call is well above timer resolution and we do not +want asv auto-tuning ``number`` up into multi-second batches. ``repeat`` and the +timeout keep total time bounded. Average degree ~1 keeps ``mxm`` output near the +input size instead of exploding into quadratic fill-in. +""" + +from graphblas import binary, monoid, semiring, unary + +try: + from . import common +except ImportError: + import common + + +class VectorLarge: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.v = common.make_vector(seed=30) + self.u = common.make_vector(seed=31) + + def time_ewise_mult(self): + self.v.ewise_mult(self.u, binary.times).new() + + def time_ewise_add(self): + self.v.ewise_add(self.u, monoid.plus).new() + + def time_apply(self): + self.v.apply(unary.abs).new() + + def time_reduce(self): + self.v.reduce(monoid.plus).new() + + def peakmem_ewise_add(self): + self.v.ewise_add(self.u, monoid.plus).new() + + +class MatrixLarge: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.A = common.make_matrix(seed=32) + self.B = common.make_matrix(seed=33) + self.x = common.make_dense_vector(common.LARGE_N, seed=34) + + def time_mxv(self): + self.A.mxv(self.x, semiring.plus_times).new() + + def time_mxm(self): + self.A.mxm(self.B, semiring.plus_times).new() + + def time_ewise_add(self): + self.A.ewise_add(self.B, monoid.plus).new() + + def time_reduce_rowwise(self): + self.A.reduce_rowwise(monoid.plus).new() + + def time_reduce_scalar(self): + self.A.reduce_scalar(monoid.plus).new() + + def peakmem_mxm(self): + self.A.mxm(self.B, semiring.plus_times).new() diff --git a/benchmarks/repr_bench.py b/benchmarks/repr_bench.py new file mode 100644 index 000000000..4f138a354 --- /dev/null +++ b/benchmarks/repr_bench.py @@ -0,0 +1,51 @@ +"""repr / _repr_html_ rendering. + +Object display goes through pandas and, for large objects, the formatting code +that decides what to elide. Small reprs are an overhead microbenchmark; large +reprs check that the truncation path stays cheap (it must not render every one of +~1e6 nonzeros). +""" + +from graphblas import Matrix, Vector + +try: + from . import common +except ImportError: + import common + + +class ReprSmall: + def setup(self): + self.M = Matrix.from_coo([0, 1, 2], [0, 1, 2], [1.0, 2.0, 3.0], nrows=4, ncols=4) + self.v = Vector.from_coo([0, 2], [1.0, 2.0], size=5) + + def time_repr_matrix(self): + repr(self.M) + + def time_repr_vector(self): + repr(self.v) + + def time_repr_html_matrix(self): + self.M._repr_html_() + + def time_repr_html_vector(self): + self.v._repr_html_() + + +class ReprLarge: + # Should be bounded by truncation, but give it room in case a regression makes + # it render the whole object. + timeout = 120 + + def setup(self): + self.M = common.make_matrix(seed=50) + self.v = common.make_vector(seed=51) + + def time_repr_matrix(self): + repr(self.M) + + def time_repr_vector(self): + repr(self.v) + + def time_repr_html_matrix(self): + self.M._repr_html_() diff --git a/benchmarks/scalar_access.py b/benchmarks/scalar_access.py new file mode 100644 index 000000000..0de089802 --- /dev/null +++ b/benchmarks/scalar_access.py @@ -0,0 +1,168 @@ +"""Single-element access: the tightest hot loops in the library. + +Extracting or assigning one element goes through the full expression + descriptor +machinery, so these times are almost entirely Python overhead. "hit" indexes an +element that is present; "miss" indexes an empty slot (the extract returns an +empty Scalar / ``None`` value). + +Several of these paths gained dedicated fast lanes (single-element extract, +``get``, ``__contains__``, integer ``__setitem__``); the benchmarks below track +each one so a regression that re-routes through the slow expression path shows up. +Every ``setup`` warms the path once so the first timed sample is not inflated by +one-off numba/dtype/operator cache population. +""" + +import graphblas as gb +from graphblas import Scalar + +try: + from . import common +except ImportError: # imported flat by asv, not as a package + import common + + +class ScalarObject: + """Scalar construction and .value round-trips.""" + + def setup(self): + self.s = Scalar.from_value(3.14, dtype=gb.dtypes.FP64) + # Warm construction / value round-trip caches. + for _ in range(3): + Scalar.from_value(3.14, dtype=gb.dtypes.FP64) + self.s.value + self.s.value = 2.0 + self.s.dup() + + def time_from_value(self): + Scalar.from_value(3.14, dtype=gb.dtypes.FP64) + + def time_get_value(self): + self.s.value + + def time_set_value(self): + self.s.value = 2.0 + + def time_dup(self): + self.s.dup() + + +class VectorElement: + """Get / extract / assign a single vector element, present vs missing.""" + + def setup(self): + # size 10000, ~1000 present entries; index 0 forced present, 1 forced empty + self.v = common.make_vector(size=10_000, nnz=1_000, seed=10) + self.v[0] = 1.0 + del self.v[1] + self.hit = 0 + self.miss = 1 + # Warm each timed path once (extract, get, contains, setitem, the value / + # float / int single-extract fast paths) so no timed sample pays the + # first-call numba compile or cache-fill cost. + for _ in range(3): + self.v[self.hit].new() + self.v[self.miss].new() + self.v[self.hit].value + self.v[self.miss].value + float(self.v[self.hit]) + int(self.v[self.hit]) + self.v.get(self.hit) + self.v.get(self.miss) + self.v.get(self.miss, 0.0) + _ = self.hit in self.v + _ = self.miss in self.v + self.v[self.hit] = 5.0 + + def time_getitem_hit(self): + self.v[self.hit].new() + + def time_getitem_miss(self): + self.v[self.miss].new() + + def time_value_hit(self): + self.v[self.hit].value + + def time_value_miss(self): + self.v[self.miss].value + + def time_float_hit(self): + float(self.v[self.hit]) + + def time_int_hit(self): + int(self.v[self.hit]) + + def time_get_hit(self): + self.v.get(self.hit) + + def time_get_miss(self): + self.v.get(self.miss) + + def time_get_miss_default(self): + # `get` with an explicit default; the miss returns the default rather than None. + self.v.get(self.miss, 0.0) + + def time_contains_hit(self): + self.hit in self.v + + def time_contains_miss(self): + self.miss in self.v + + def time_setitem(self): + # Overwrites an existing slot, so state is stable across repeated calls. + self.v[self.hit] = 5.0 + + +class MatrixElement: + """Get / extract / assign a single matrix element, present vs missing.""" + + def setup(self): + self.M = common.make_matrix(n=10_000, nnz=1_000, seed=11) + self.M[0, 0] = 1.0 + del self.M[1, 1] + self.hit = (0, 0) + self.miss = (1, 1) + for _ in range(3): + self.M[self.hit[0], self.hit[1]].new() + self.M[self.miss[0], self.miss[1]].new() + self.M[self.hit[0], self.hit[1]].value + self.M[self.miss[0], self.miss[1]].value + float(self.M[self.hit[0], self.hit[1]]) + self.M.get(self.hit[0], self.hit[1]) + self.M.get(self.miss[0], self.miss[1]) + self.M.get(self.miss[0], self.miss[1], 0.0) + _ = self.hit in self.M + _ = self.miss in self.M + self.M[self.hit[0], self.hit[1]] = 5.0 + + def time_getitem_hit(self): + self.M[self.hit[0], self.hit[1]].new() + + def time_getitem_miss(self): + self.M[self.miss[0], self.miss[1]].new() + + def time_value_hit(self): + self.M[self.hit[0], self.hit[1]].value + + def time_value_miss(self): + self.M[self.miss[0], self.miss[1]].value + + def time_float_hit(self): + float(self.M[self.hit[0], self.hit[1]]) + + def time_get_hit(self): + self.M.get(self.hit[0], self.hit[1]) + + def time_get_miss(self): + self.M.get(self.miss[0], self.miss[1]) + + def time_get_miss_default(self): + self.M.get(self.miss[0], self.miss[1], 0.0) + + def time_contains_hit(self): + self.hit in self.M + + def time_contains_miss(self): + self.miss in self.M + + def time_setitem(self): + self.M[self.hit[0], self.hit[1]] = 5.0 diff --git a/benchmarks/small_ops.py b/benchmarks/small_ops.py new file mode 100644 index 000000000..ce2c040a6 --- /dev/null +++ b/benchmarks/small_ops.py @@ -0,0 +1,72 @@ +"""Small-operand op overhead. + +On ~100-element objects the C kernels are trivially fast, so these times isolate +the Python-side cost of building an expression and resolving operators. The +``time_build_*`` benchmarks stop at the expression object (no ``.new()``) to +separate expression construction from evaluation. +""" + +from graphblas import binary, monoid, semiring, unary + +try: + from . import common +except ImportError: + import common + + +class SmallVector: + params = [10, 100, 1000] + param_names = ["size"] + + def setup(self, size): + # Dense vectors so every op touches ``size`` elements. + self.v = common.make_dense_vector(size, seed=20) + self.u = common.make_dense_vector(size, seed=21) + + def time_build_ewise_mult(self, size): + # Expression only, not evaluated: pure construction overhead. + self.v.ewise_mult(self.u, binary.times) + + def time_ewise_mult(self, size): + self.v.ewise_mult(self.u, binary.times).new() + + def time_ewise_add(self, size): + self.v.ewise_add(self.u, monoid.plus).new() + + def time_apply(self, size): + self.v.apply(unary.abs).new() + + def time_apply_bind_scalar(self, size): + self.v.apply(binary.plus, right=1.0).new() + + def time_reduce(self, size): + self.v.reduce(monoid.plus).new() + + def time_assign_into(self, size): + # The `<<` update path: evaluate into an existing object with no mask/accum. + self.v << self.v.ewise_mult(self.u, binary.times) + + +class SmallMatrix: + params = [10, 100] + param_names = ["dim"] + + def setup(self, dim): + # Dense dim x dim (dim**2 entries): 100 or 10000 nonzeros. + self.A = common.make_dense_matrix(dim=dim, seed=22) + self.B = common.make_dense_matrix(dim=dim, seed=23) + + def time_ewise_mult(self, dim): + self.A.ewise_mult(self.B, binary.times).new() + + def time_apply(self, dim): + self.A.apply(unary.abs).new() + + def time_reduce_rowwise(self, dim): + self.A.reduce_rowwise(monoid.plus).new() + + def time_reduce_scalar(self, dim): + self.A.reduce_scalar(monoid.plus).new() + + def time_mxm(self, dim): + self.A.mxm(self.B, semiring.plus_times).new() diff --git a/pyproject.toml b/pyproject.toml index 9fe272c98..0475fd5e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,7 +225,7 @@ skip_empty = true exclude_lines = ["pragma: no cover", "raise AssertionError", "raise NotImplementedError"] [tool.codespell] -ignore-words-list = "coo,ba" +ignore-words-list = "bu,coo,ba" [tool.ruff] # https://github.com/charliermarsh/ruff/ @@ -386,6 +386,7 @@ ignore = [ "graphblas/core/operator/base.py" = ["S102"] # exec is used for UDF "graphblas/core/operator/udt_utils.py" = ["S102"] # exec is used for UDT op codegen "graphblas/monoid/numpy.py" = ["PLW0108"] # lambda is needed for numba.njit +"benchmarks/*.py" = ["T201", "B015", "B018"] # asv: bare exprs are the measurement; _verify prints "graphblas/core/ss/matrix.py" = [ "NPY002", # numba doesn't support rng generator yet "PLR1730",