diff --git a/.flake8 b/.flake8 index 959e9a22c..9ba580871 100644 --- a/.flake8 +++ b/.flake8 @@ -10,6 +10,7 @@ extend-ignore = SIM401, # E203 whitespace before ':' (to be compatible with black) per-file-ignores = + benchmarks/*.py:B015,B018, scripts/create_pickle.py:F403,F405, graphblas/tests/*.py:T201,B043, graphblas/core/ss/matrix.py:SIM113, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3582142d2..b9603a6b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,11 @@ repos: - id: check-illegal-windows-names - id: check-merge-conflict - id: check-ast - - id: check-json # no JSON files yet; enabled for the future + # `asv.conf.json` is JSONC: asv documents `//` comments and strips them + # in its own loader, but this hook is a strict JSON parser. Excluding + # that one path keeps the hook live for every other JSON file. + - id: check-json + exclude: ^asv\.conf\.json$ - id: check-toml - id: check-yaml - id: check-executables-have-shebangs diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 000000000..acd395737 --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,73 @@ +{ + // asv (airspeed velocity) config for python-graphblas. + // + // Placement: this file and the benchmarks/ directory are intended to live at the + // repo root (next to pyproject.toml). The paths below are relative to that root. + // asv's config loader strips these // line comments; a strict JSON linter will + // not. If the repo's pre-commit `check-json` hook is enabled, exclude this file + // (asv.conf.json is asv's own format, and its docs use // comments). + // + // Typical use: + // asv run # benchmark current commit + // asv continuous main HEAD # compare a branch against main + // asv publish && asv preview # build/serve the HTML report + + "version": 1, + "project": "python-graphblas", + "project_url": "https://github.com/python-graphblas/python-graphblas", + + // "." means the repo that contains this file. python-graphblas is pure Python, so + // building each commit is a cheap editable install; the C library comes from the + // separate suitesparse-graphblas package, pinned in the matrix below. + "repo": ".", + "branches": ["main"], + "dvcs": "git", + + // conda matches the project's primary dev path and lets us pin the C library + // cleanly from conda-forge. asv will use mamba automatically if it is installed. + "environment_type": "conda", + "conda_channels": ["conda-forge"], + + // PINNING NOTE: python-suitesparse-graphblas (the C library) is pinned so a run + // measures python-graphblas-side regressions in isolation. To instead track + // end-to-end performance including C-library changes, drop the pin (set it to []) + // or bump it deliberately. Keep this in sync with the PSG version the suite is + // calibrated against (see scripts/ci_pick_versions.py PSG pools). + "matrix": { + "req": { + "python-suitesparse-graphblas": ["10.3.1.0"], + "numpy": [], + "scipy": [], + "pandas": [], + "numba": [] + } + }, + + // python-graphblas is pure Python: a no-build-isolation editable install is fast + // and reuses the already-solved conda env (which supplies the C library plus + // numba/scipy/pandas). asv substitutes {build_dir} for the checkout of each commit. + "build_command": [], + "install_command": [ + "in-dir={build_dir} python -m pip install --no-build-isolation --no-deps -e ." + ], + "uninstall_command": [ + "in-dir={build_dir} python -m pip uninstall -y python-graphblas" + ], + + "benchmark_dir": "benchmarks", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + + // Give the C-library install room to solve/download. + "install_timeout": 900, + + // Isolate JIT/compiler and thread-count noise so numbers are comparable across + // runs. GraphBLAS is multithreaded by default; pin to 1 thread for reproducible + // single-core timings (raise this in a dedicated run if you want to benchmark + // parallel scaling instead). + "environment_variables": { + "OMP_NUM_THREADS": "1", + "GRAPHBLAS_TEST_SEED": "0" + } +} diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..5962927f7 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,91 @@ +# asv benchmark scaffold for python-graphblas + +A starting airspeed velocity (asv) benchmark suite covering the library's hot +paths. Built as a scaffold: review, then move `asv.conf.json` and `benchmarks/` +to the repo root (next to `pyproject.toml`). `_verify.py` is a local sanity +checker and does not need to ship. + +## Layout + +``` +asv.conf.json asv config, tuned for this repo (see caveats below). + JSONC: asv documents `//` comments and strips them in + its loader, so `.pre-commit-config.yaml` excludes this + one path from the strict `check-json` hook. asv also + accepts `asv.conf.jsonc`, which would need no exclude, + but asv is not a pinned dependency here and older + versions resolve only the `.json` name. +benchmarks/ + __init__.py package marker + note on the dual-import shim + common.py shared, seeded data builders (called from setup, not timed) + scalar_access.py single-element get / extract / assign, hit vs miss + index_parse.py parse_index int fast lane vs numpy-int lane (index build) + small_ops.py ewise / apply / reduce on ~100-element objects (overhead) + large_kernels.py mxm / mxv / ewise / reduce on ~1e6 nnz (C-library bound) + conversions.py from_coo/to_coo, from_dense/to_dense, scipy interop + imports.py cold import + first-use timing (timeraw, fresh subprocess) + repr_bench.py repr / _repr_html_ for small and large objects +_verify.py standalone check: runs every benchmark once (not an asv file) +``` + +## Running + +```bash +pip install asv +asv machine --yes # one-time: record machine info +asv run # benchmark the current commit +asv continuous main HEAD # compare a branch against main, flag regressions +asv publish && asv preview # build and serve the HTML report +asv run --bench scalar_access # run a subset by regex +``` + +Quick correctness check without asv (runs each benchmark once): + +```bash +python _verify.py +``` + +## Design choices + +- `setup()` builds all data once, outside the timed region; every builder is + seeded so runs are comparable across commits. +- Sizes: "small" is 10 to 1000 elements (overhead-dominated, so we track + Python-side cost); "large" is ~1e6 nonzeros (C-library dominated). Average + matrix degree is ~1 so `mxm` output stays near the input size instead of + exploding into quadratic fill-in. Verified per-call times on one dev machine: + large kernels ran in 1 to 5 ms, conversions under 10 ms, cold `import +graphblas` ~32 ms, init/first-operator ~0.2 s. Bump `common.LARGE_NNZ` or add a + higher-degree matrix if you want heavier `mxm`. +- Large kernels use `number = 1`, `warmup_time = 0`, small `repeat`, and a + `timeout`, so asv does not batch them into multi-second runs. +- `imports.py` uses `timeraw_*`, which runs the returned code string in a fresh + subprocess. That is the only way to measure true cold import cost; a normal + benchmark would read ~0 because the module is already imported. +- `asv.conf.json` pins `python-suitesparse-graphblas` so a run isolates + python-graphblas-side regressions from C-library changes. Drop the pin to track + end-to-end (library + C) performance. Keep the pinned version in sync with the + PSG pools in `scripts/ci_pick_versions.py`. +- `OMP_NUM_THREADS=1` pins GraphBLAS to one thread for reproducible single-core + timings. Raise it in a dedicated run to benchmark parallel scaling. + +## Caveats to resolve before merging + +- **dev deps**: `asv` is deliberately NOT added to `dev-requirements.txt` or + `environment.yml`. Nothing in the test suite or CI invokes it, so adding it + would make every contributor's dev install fetch a package only used by + people who opt into benchmarking; and this repo ties dependency changes to + `scripts/check_versions.sh` and the CI version pools, which a benchmarks-only + change has no business editing. `pip install asv` is in the usage section + above. Adding a `benchmark` optional-dependencies group (so that + `pip install python-graphblas[benchmark]` works) is the natural move if the + suite graduates from scaffold to something CI runs. +- **CI wiring**: asv is not wired into CI here. A weekly cron running `asv +continuous` against a fixed machine (or asv's own regression detection) is the + natural follow-up; per-PR benchmarking on shared GitHub runners is too noisy to + gate on. +- **First-run JIT/compile noise**: the first use of some operators triggers + numba/JIT work. `setup()` touches the operators the benchmark uses, but the + very first `asv run` on a fresh env may still show inflated one-off numbers; + asv's repeats and its own warmup mitigate this. +- **pandas dependency**: `repr_bench.py` needs pandas (the default/test extras + include it). The env matrix installs pandas, so this is satisfied. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..dd8e37ffb --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1,12 @@ +"""python-graphblas benchmark suite (airspeed velocity). + +Benchmark modules live alongside this file. Shared data builders are in +``common.py``. Each benchmark module imports ``common`` with a dual path so it +works whether asv imports the files as a package or adds ``benchmark_dir`` to +``sys.path`` and imports them flat: + + try: + from . import common + except ImportError: # imported flat, not as a package + import common +""" diff --git a/benchmarks/_verify.py b/benchmarks/_verify.py new file mode 100755 index 000000000..cb9ccd5d4 --- /dev/null +++ b/benchmarks/_verify.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +"""Standalone sanity check for the asv benchmark suite (not an asv benchmark). + +Imports every benchmark module the way asv does (benchmark_dir on sys.path), +then for each benchmark class runs setup() and calls each benchmark method once +across all parameter combinations. timeraw_* methods return a code string, which +is executed in a fresh subprocess. Prints PASS/FAIL per benchmark and a timing so +we can spot anything accidentally slow. + +Usage: python _verify.py +""" + +import importlib +import inspect +import itertools +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +BENCH_DIR = HERE / "benchmarks" +sys.path.insert(0, str(BENCH_DIR)) # flat import mode, as asv does + +MODULES = [ + "scalar_access", + "index_parse", + "small_ops", + "large_kernels", + "conversions", + "imports", + "repr_bench", +] +PREFIXES = ("time_", "timeraw_", "peakmem_", "mem_", "track_") + + +def param_combos(cls): + params = getattr(cls, "params", None) + if not params: + return [()] + # asv: a flat list is a single parameter; a list of lists is a product. + if params and isinstance(params[0], (list, tuple)): + return list(itertools.product(*params)) + return [(p,) for p in params] + + +def run_one(cls, combo): + results = [] + obj = cls() + if hasattr(obj, "setup"): + obj.setup(*combo) + for name in sorted(dir(obj)): + if not name.startswith(PREFIXES): + continue + method = getattr(obj, name) + if not callable(method): + continue + label = f"{cls.__module__}.{cls.__name__}.{name}{combo or ''}" + t0 = time.perf_counter() + try: + if name.startswith("timeraw_"): + code = method() + setup_code = "" + if isinstance(code, tuple): + code, setup_code = code + script = (setup_code + "\n" + code) if setup_code else code + subprocess.run( + [sys.executable, "-c", script], check=True, capture_output=True, timeout=180 + ) + else: + method(*combo) + dt = time.perf_counter() - t0 + results.append((label, dt, None)) + except Exception as exc: + dt = time.perf_counter() - t0 + results.append((label, dt, repr(exc))) + if hasattr(obj, "teardown"): + obj.teardown(*combo) + return results + + +def main(): + all_results = [] + for modname in MODULES: + mod = importlib.import_module(modname) + classes = [ + c for _, c in inspect.getmembers(mod, inspect.isclass) if c.__module__ == modname + ] + for cls in classes: + for combo in param_combos(cls): + all_results.extend(run_one(cls, combo)) + + fails = [r for r in all_results if r[2] is not None] + for label, dt, err in all_results: + status = "PASS" if err is None else "FAIL" + line = f"[{status}] {dt:7.3f}s {label}" + if err is not None: + line += f" -> {err}" + print(line) + print( + f"\n{len(all_results) - len(fails)}/{len(all_results)} benchmarks ran; {len(fails)} failed" + ) + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/common.py b/benchmarks/common.py new file mode 100644 index 000000000..44db815fa --- /dev/null +++ b/benchmarks/common.py @@ -0,0 +1,91 @@ +"""Shared data builders for the python-graphblas asv benchmark suite. + +Each benchmark module imports these helpers and calls them from ``setup`` so the +input data is constructed once per benchmark (outside the timed region). All +builders take a seed and use ``numpy.random.default_rng`` so runs are +deterministic and comparable across commits. + +Size constants are chosen so the "large" kernels land near ~1e6 nonzeros (enough +to exercise the C library) while still finishing in well under a second each, and +so no dense intermediate blows up memory (a 1e6 x 1e6 dense matrix is never +materialized; dense conversions use a separate, small, fully dense matrix). +""" + +import numpy as np + +import graphblas as gb +from graphblas import Matrix, Vector + +# Sizes + +# "Large" sparse operands: square matrix and vector near 1e6 nonzeros. Average +# degree ~1 keeps mxm output bounded (A @ A stays near 1e6 nnz) so the kernel +# benchmarks do not accidentally measure quadratic fill-in. +LARGE_N = 1_000_000 +LARGE_NNZ = 1_000_000 + +# "Small" operands: overhead-dominated. At this size the wall time is almost all +# Python-side expression machinery, which is exactly what we want to track. +SMALL_SIZE = 100 + +# Dense conversions use a fully dense square matrix small enough to materialize. +DENSE_DIM = 1000 # 1e6 dense elements + +# scipy interop matrix: ~1e6 nnz at 1% density. +SCIPY_DIM = 10_000 +SCIPY_DENSITY = 0.01 + + +# Builders + + +def make_coo(n, nnz, seed=0, dtype="FP64"): + """Return (rows, cols, vals) numpy arrays for an n x n matrix with ~nnz entries. + + Duplicates are possible; callers that build a Matrix should pass a ``dup_op``. + """ + rng = np.random.default_rng(seed) + rows = rng.integers(0, n, nnz, dtype=np.uint64) + cols = rng.integers(0, n, nnz, dtype=np.uint64) + if dtype == "BOOL": + vals = rng.integers(0, 2, nnz, dtype=np.bool_) + elif dtype in ("INT64", "INT32"): + vals = rng.integers(1, 100, nnz, dtype=np.int64) + else: + vals = rng.random(nnz) + return rows, cols, vals + + +def make_matrix(n=LARGE_N, nnz=LARGE_NNZ, seed=0, dtype="FP64"): + """Square Matrix with ~nnz entries (duplicates summed).""" + rows, cols, vals = make_coo(n, nnz, seed=seed, dtype=dtype) + dup = gb.binary.lor if dtype == "BOOL" else gb.binary.plus + return Matrix.from_coo(rows, cols, vals, nrows=n, ncols=n, dtype=dtype, dup_op=dup) + + +def make_vector(size=LARGE_N, nnz=LARGE_NNZ, seed=1, dtype="FP64"): + """Vector of length ``size`` with ~nnz entries (duplicates summed).""" + rng = np.random.default_rng(seed) + idx = rng.integers(0, size, nnz, dtype=np.uint64) + vals = rng.random(nnz) if dtype not in ("INT64", "INT32") else rng.integers(1, 100, nnz) + dup = gb.binary.plus + return Vector.from_coo(idx, vals, size=size, dtype=dtype, dup_op=dup) + + +def make_dense_vector(size, seed=2, dtype="FP64"): + """Fully dense Vector (no missing entries), e.g. an mxv operand.""" + rng = np.random.default_rng(seed) + return Vector.from_dense(rng.random(size)) + + +def make_dense_matrix(dim=DENSE_DIM, seed=3): + """Fully dense square Matrix built from a numpy array.""" + rng = np.random.default_rng(seed) + return Matrix.from_dense(rng.random((dim, dim))) + + +def make_scipy(dim=SCIPY_DIM, density=SCIPY_DENSITY, seed=4, fmt="csr"): + """A scipy.sparse matrix for interop benchmarks.""" + import scipy.sparse as sp + + return sp.random(dim, dim, density=density, format=fmt, random_state=seed) diff --git a/benchmarks/conversions.py b/benchmarks/conversions.py new file mode 100644 index 000000000..90be6c986 --- /dev/null +++ b/benchmarks/conversions.py @@ -0,0 +1,98 @@ +"""Import/export conversions between graphblas objects and numpy/scipy formats. + +Sparse conversions (from_coo/to_coo, scipy interop) run on ~1e6 nonzeros. Dense +conversions use a separate fully dense 1000x1000 matrix so nothing materializes a +1e6 x 1e6 dense array. +""" + +import numpy as np + +from graphblas import Matrix, Vector, io + +try: + from . import common +except ImportError: + import common + + +class MatrixCoo: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.rows, self.cols, self.vals = common.make_coo(common.LARGE_N, common.LARGE_NNZ, seed=40) + self.M = common.make_matrix(seed=40) + + def time_from_coo(self): + Matrix.from_coo( + self.rows, + self.cols, + self.vals, + nrows=common.LARGE_N, + ncols=common.LARGE_N, + dup_op="plus", + ) + + def time_to_coo(self): + self.M.to_coo() + + +class MatrixDense: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.dense = np.random.default_rng(41).random((common.DENSE_DIM, common.DENSE_DIM)) + self.M = Matrix.from_dense(self.dense) + + def time_from_dense(self): + Matrix.from_dense(self.dense) + + def time_to_dense(self): + self.M.to_dense() + + +class MatrixScipy: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.sp = common.make_scipy(seed=42) + self.M = io.from_scipy_sparse(self.sp) + + def time_from_scipy_sparse(self): + io.from_scipy_sparse(self.sp) + + def time_to_scipy_sparse(self): + io.to_scipy_sparse(self.M, format="csr") + + +class VectorConvert: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.v = common.make_vector(seed=43) + self.idx, self.vals = self.v.to_coo() + self.dense_arr = np.random.default_rng(44).random(common.LARGE_N) + self.dv = Vector.from_dense(self.dense_arr) + + def time_from_coo(self): + Vector.from_coo(self.idx, self.vals, size=common.LARGE_N, dup_op="plus") + + def time_to_coo(self): + self.v.to_coo() + + def time_from_dense(self): + Vector.from_dense(self.dense_arr) + + def time_to_dense(self): + self.dv.to_dense() diff --git a/benchmarks/imports.py b/benchmarks/imports.py new file mode 100644 index 000000000..9204b26a2 --- /dev/null +++ b/benchmarks/imports.py @@ -0,0 +1,35 @@ +"""Import and cold-start timing. + +These use asv's ``timeraw_*`` form: each returns a code string that asv runs in a +*fresh* subprocess, so the measurement is the true cold cost (module already +imported into the benchmark process would otherwise read as ~0). ``timeraw`` +benchmarks cannot see ``setup`` state or this module's imports, so everything +they need must be inside the returned string. + +``graphblas`` initializes lazily: ``import graphblas`` is cheap, and the real +cost (loading the C library, building operator namespaces) is deferred until +first use. The staged benchmarks below separate those phases. +""" + + +class ImportTiming: + # A little headroom: importing numba/llvmlite on first operator use is not fast. + timeout = 120 + + def timeraw_import_graphblas(self): + return "import graphblas" + + def timeraw_from_import_core_types(self): + return "from graphblas import Matrix, Vector, Scalar" + + def timeraw_import_then_init(self): + # Force backend initialization (loads the C library). + return "import graphblas as gb; gb.init('suitesparse')" + + def timeraw_first_operator_access(self): + # First attribute access on an operator namespace triggers its lazy build. + return "import graphblas as gb; gb.binary.plus" + + def timeraw_first_matrix(self): + # End-to-end cold path: import, init, and build one tiny Matrix. + return "import graphblas as gb; gb.Matrix.from_coo([0], [0], [1.0], nrows=1, ncols=1)" diff --git a/benchmarks/index_parse.py b/benchmarks/index_parse.py new file mode 100644 index 000000000..5b5b0c856 --- /dev/null +++ b/benchmarks/index_parse.py @@ -0,0 +1,90 @@ +"""Index parsing overhead: the ``parse_index`` int fast lane. + +Every ``v[i]`` / ``A[i, j]`` builds an ``IndexerResolver`` that runs each index +through ``parse_index``. A plain Python ``int`` takes a dedicated fast lane that +skips two ``np.issubdtype`` checks (~110ns each) used by the numpy-integer lane. +These benchmarks isolate that parsing cost from element extraction (no ``.new()``, +no value read), so a regression in the fast lane, or an index accidentally +falling out of it, is visible on its own. + +Two granularities are measured: + +* the public path (``v[i]`` returns an index expression without resolving it), and +* ``IndexerResolver(obj, idx)`` directly, the tightest view of ``parse_index``. + +The plain-int and numpy-int variants sit side by side so the fast lane's margin +over the general lane is tracked directly. +""" + +import numpy as np + +from graphblas.core.expr import IndexerResolver + +try: + from . import common +except ImportError: # imported flat by asv, not as a package + import common + + +class VectorIndexParse: + """Parse a single vector index: plain-int fast lane vs numpy-int lane.""" + + def setup(self): + self.v = common.make_vector(size=10_000, nnz=1_000, seed=12) + self.i = 4321 # in range, positive + self.neg = -1 # in range, triggers the negative-wrap branch + self.npi = np.int64(4321) + for _ in range(3): + self.v[self.i] + self.v[self.neg] + self.v[self.npi] + IndexerResolver(self.v, self.i) + IndexerResolver(self.v, self.npi) + + # Public path: build the index expression (parse_index + expression object). + def time_getitem_int(self): + self.v[self.i] + + def time_getitem_int_negative(self): + self.v[self.neg] + + def time_getitem_numpy_int(self): + self.v[self.npi] + + # Tightest view: just the resolver (parse_index, no expression object). + def time_resolver_int(self): + IndexerResolver(self.v, self.i) + + def time_resolver_numpy_int(self): + IndexerResolver(self.v, self.npi) + + +class MatrixIndexParse: + """Parse a two-axis matrix index: plain-int fast lane vs numpy-int lane.""" + + def setup(self): + self.M = common.make_matrix(n=10_000, nnz=1_000, seed=13) + self.ij = (4321, 8765) + self.neg = (-1, -1) + self.npij = (np.int64(4321), np.int64(8765)) + for _ in range(3): + self.M[self.ij[0], self.ij[1]] + self.M[self.neg[0], self.neg[1]] + self.M[self.npij[0], self.npij[1]] + IndexerResolver(self.M, self.ij) + IndexerResolver(self.M, self.npij) + + def time_getitem_int(self): + self.M[self.ij[0], self.ij[1]] + + def time_getitem_int_negative(self): + self.M[self.neg[0], self.neg[1]] + + def time_getitem_numpy_int(self): + self.M[self.npij[0], self.npij[1]] + + def time_resolver_int(self): + IndexerResolver(self.M, self.ij) + + def time_resolver_numpy_int(self): + IndexerResolver(self.M, self.npij) diff --git a/benchmarks/large_kernels.py b/benchmarks/large_kernels.py new file mode 100644 index 000000000..f0c278409 --- /dev/null +++ b/benchmarks/large_kernels.py @@ -0,0 +1,73 @@ +"""Large kernels (~1e6 nonzeros): dominated by the C library, so these catch +suitesparse-graphblas-side regressions (and any python-graphblas overhead that +scales with data). + +Each op runs once per sample (``number = 1``) with no warmup, because the inputs +are large enough that a single call is well above timer resolution and we do not +want asv auto-tuning ``number`` up into multi-second batches. ``repeat`` and the +timeout keep total time bounded. Average degree ~1 keeps ``mxm`` output near the +input size instead of exploding into quadratic fill-in. +""" + +from graphblas import binary, monoid, semiring, unary + +try: + from . import common +except ImportError: + import common + + +class VectorLarge: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.v = common.make_vector(seed=30) + self.u = common.make_vector(seed=31) + + def time_ewise_mult(self): + self.v.ewise_mult(self.u, binary.times).new() + + def time_ewise_add(self): + self.v.ewise_add(self.u, monoid.plus).new() + + def time_apply(self): + self.v.apply(unary.abs).new() + + def time_reduce(self): + self.v.reduce(monoid.plus).new() + + def peakmem_ewise_add(self): + self.v.ewise_add(self.u, monoid.plus).new() + + +class MatrixLarge: + number = 1 + repeat = 5 + warmup_time = 0 + timeout = 300 + + def setup(self): + self.A = common.make_matrix(seed=32) + self.B = common.make_matrix(seed=33) + self.x = common.make_dense_vector(common.LARGE_N, seed=34) + + def time_mxv(self): + self.A.mxv(self.x, semiring.plus_times).new() + + def time_mxm(self): + self.A.mxm(self.B, semiring.plus_times).new() + + def time_ewise_add(self): + self.A.ewise_add(self.B, monoid.plus).new() + + def time_reduce_rowwise(self): + self.A.reduce_rowwise(monoid.plus).new() + + def time_reduce_scalar(self): + self.A.reduce_scalar(monoid.plus).new() + + def peakmem_mxm(self): + self.A.mxm(self.B, semiring.plus_times).new() diff --git a/benchmarks/repr_bench.py b/benchmarks/repr_bench.py new file mode 100644 index 000000000..4f138a354 --- /dev/null +++ b/benchmarks/repr_bench.py @@ -0,0 +1,51 @@ +"""repr / _repr_html_ rendering. + +Object display goes through pandas and, for large objects, the formatting code +that decides what to elide. Small reprs are an overhead microbenchmark; large +reprs check that the truncation path stays cheap (it must not render every one of +~1e6 nonzeros). +""" + +from graphblas import Matrix, Vector + +try: + from . import common +except ImportError: + import common + + +class ReprSmall: + def setup(self): + self.M = Matrix.from_coo([0, 1, 2], [0, 1, 2], [1.0, 2.0, 3.0], nrows=4, ncols=4) + self.v = Vector.from_coo([0, 2], [1.0, 2.0], size=5) + + def time_repr_matrix(self): + repr(self.M) + + def time_repr_vector(self): + repr(self.v) + + def time_repr_html_matrix(self): + self.M._repr_html_() + + def time_repr_html_vector(self): + self.v._repr_html_() + + +class ReprLarge: + # Should be bounded by truncation, but give it room in case a regression makes + # it render the whole object. + timeout = 120 + + def setup(self): + self.M = common.make_matrix(seed=50) + self.v = common.make_vector(seed=51) + + def time_repr_matrix(self): + repr(self.M) + + def time_repr_vector(self): + repr(self.v) + + def time_repr_html_matrix(self): + self.M._repr_html_() diff --git a/benchmarks/scalar_access.py b/benchmarks/scalar_access.py new file mode 100644 index 000000000..0de089802 --- /dev/null +++ b/benchmarks/scalar_access.py @@ -0,0 +1,168 @@ +"""Single-element access: the tightest hot loops in the library. + +Extracting or assigning one element goes through the full expression + descriptor +machinery, so these times are almost entirely Python overhead. "hit" indexes an +element that is present; "miss" indexes an empty slot (the extract returns an +empty Scalar / ``None`` value). + +Several of these paths gained dedicated fast lanes (single-element extract, +``get``, ``__contains__``, integer ``__setitem__``); the benchmarks below track +each one so a regression that re-routes through the slow expression path shows up. +Every ``setup`` warms the path once so the first timed sample is not inflated by +one-off numba/dtype/operator cache population. +""" + +import graphblas as gb +from graphblas import Scalar + +try: + from . import common +except ImportError: # imported flat by asv, not as a package + import common + + +class ScalarObject: + """Scalar construction and .value round-trips.""" + + def setup(self): + self.s = Scalar.from_value(3.14, dtype=gb.dtypes.FP64) + # Warm construction / value round-trip caches. + for _ in range(3): + Scalar.from_value(3.14, dtype=gb.dtypes.FP64) + self.s.value + self.s.value = 2.0 + self.s.dup() + + def time_from_value(self): + Scalar.from_value(3.14, dtype=gb.dtypes.FP64) + + def time_get_value(self): + self.s.value + + def time_set_value(self): + self.s.value = 2.0 + + def time_dup(self): + self.s.dup() + + +class VectorElement: + """Get / extract / assign a single vector element, present vs missing.""" + + def setup(self): + # size 10000, ~1000 present entries; index 0 forced present, 1 forced empty + self.v = common.make_vector(size=10_000, nnz=1_000, seed=10) + self.v[0] = 1.0 + del self.v[1] + self.hit = 0 + self.miss = 1 + # Warm each timed path once (extract, get, contains, setitem, the value / + # float / int single-extract fast paths) so no timed sample pays the + # first-call numba compile or cache-fill cost. + for _ in range(3): + self.v[self.hit].new() + self.v[self.miss].new() + self.v[self.hit].value + self.v[self.miss].value + float(self.v[self.hit]) + int(self.v[self.hit]) + self.v.get(self.hit) + self.v.get(self.miss) + self.v.get(self.miss, 0.0) + _ = self.hit in self.v + _ = self.miss in self.v + self.v[self.hit] = 5.0 + + def time_getitem_hit(self): + self.v[self.hit].new() + + def time_getitem_miss(self): + self.v[self.miss].new() + + def time_value_hit(self): + self.v[self.hit].value + + def time_value_miss(self): + self.v[self.miss].value + + def time_float_hit(self): + float(self.v[self.hit]) + + def time_int_hit(self): + int(self.v[self.hit]) + + def time_get_hit(self): + self.v.get(self.hit) + + def time_get_miss(self): + self.v.get(self.miss) + + def time_get_miss_default(self): + # `get` with an explicit default; the miss returns the default rather than None. + self.v.get(self.miss, 0.0) + + def time_contains_hit(self): + self.hit in self.v + + def time_contains_miss(self): + self.miss in self.v + + def time_setitem(self): + # Overwrites an existing slot, so state is stable across repeated calls. + self.v[self.hit] = 5.0 + + +class MatrixElement: + """Get / extract / assign a single matrix element, present vs missing.""" + + def setup(self): + self.M = common.make_matrix(n=10_000, nnz=1_000, seed=11) + self.M[0, 0] = 1.0 + del self.M[1, 1] + self.hit = (0, 0) + self.miss = (1, 1) + for _ in range(3): + self.M[self.hit[0], self.hit[1]].new() + self.M[self.miss[0], self.miss[1]].new() + self.M[self.hit[0], self.hit[1]].value + self.M[self.miss[0], self.miss[1]].value + float(self.M[self.hit[0], self.hit[1]]) + self.M.get(self.hit[0], self.hit[1]) + self.M.get(self.miss[0], self.miss[1]) + self.M.get(self.miss[0], self.miss[1], 0.0) + _ = self.hit in self.M + _ = self.miss in self.M + self.M[self.hit[0], self.hit[1]] = 5.0 + + def time_getitem_hit(self): + self.M[self.hit[0], self.hit[1]].new() + + def time_getitem_miss(self): + self.M[self.miss[0], self.miss[1]].new() + + def time_value_hit(self): + self.M[self.hit[0], self.hit[1]].value + + def time_value_miss(self): + self.M[self.miss[0], self.miss[1]].value + + def time_float_hit(self): + float(self.M[self.hit[0], self.hit[1]]) + + def time_get_hit(self): + self.M.get(self.hit[0], self.hit[1]) + + def time_get_miss(self): + self.M.get(self.miss[0], self.miss[1]) + + def time_get_miss_default(self): + self.M.get(self.miss[0], self.miss[1], 0.0) + + def time_contains_hit(self): + self.hit in self.M + + def time_contains_miss(self): + self.miss in self.M + + def time_setitem(self): + self.M[self.hit[0], self.hit[1]] = 5.0 diff --git a/benchmarks/small_ops.py b/benchmarks/small_ops.py new file mode 100644 index 000000000..ce2c040a6 --- /dev/null +++ b/benchmarks/small_ops.py @@ -0,0 +1,72 @@ +"""Small-operand op overhead. + +On ~100-element objects the C kernels are trivially fast, so these times isolate +the Python-side cost of building an expression and resolving operators. The +``time_build_*`` benchmarks stop at the expression object (no ``.new()``) to +separate expression construction from evaluation. +""" + +from graphblas import binary, monoid, semiring, unary + +try: + from . import common +except ImportError: + import common + + +class SmallVector: + params = [10, 100, 1000] + param_names = ["size"] + + def setup(self, size): + # Dense vectors so every op touches ``size`` elements. + self.v = common.make_dense_vector(size, seed=20) + self.u = common.make_dense_vector(size, seed=21) + + def time_build_ewise_mult(self, size): + # Expression only, not evaluated: pure construction overhead. + self.v.ewise_mult(self.u, binary.times) + + def time_ewise_mult(self, size): + self.v.ewise_mult(self.u, binary.times).new() + + def time_ewise_add(self, size): + self.v.ewise_add(self.u, monoid.plus).new() + + def time_apply(self, size): + self.v.apply(unary.abs).new() + + def time_apply_bind_scalar(self, size): + self.v.apply(binary.plus, right=1.0).new() + + def time_reduce(self, size): + self.v.reduce(monoid.plus).new() + + def time_assign_into(self, size): + # The `<<` update path: evaluate into an existing object with no mask/accum. + self.v << self.v.ewise_mult(self.u, binary.times) + + +class SmallMatrix: + params = [10, 100] + param_names = ["dim"] + + def setup(self, dim): + # Dense dim x dim (dim**2 entries): 100 or 10000 nonzeros. + self.A = common.make_dense_matrix(dim=dim, seed=22) + self.B = common.make_dense_matrix(dim=dim, seed=23) + + def time_ewise_mult(self, dim): + self.A.ewise_mult(self.B, binary.times).new() + + def time_apply(self, dim): + self.A.apply(unary.abs).new() + + def time_reduce_rowwise(self, dim): + self.A.reduce_rowwise(monoid.plus).new() + + def time_reduce_scalar(self, dim): + self.A.reduce_scalar(monoid.plus).new() + + def time_mxm(self, dim): + self.A.mxm(self.B, semiring.plus_times).new() diff --git a/pyproject.toml b/pyproject.toml index 9fe272c98..0475fd5e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,7 +225,7 @@ skip_empty = true exclude_lines = ["pragma: no cover", "raise AssertionError", "raise NotImplementedError"] [tool.codespell] -ignore-words-list = "coo,ba" +ignore-words-list = "bu,coo,ba" [tool.ruff] # https://github.com/charliermarsh/ruff/ @@ -386,6 +386,7 @@ ignore = [ "graphblas/core/operator/base.py" = ["S102"] # exec is used for UDF "graphblas/core/operator/udt_utils.py" = ["S102"] # exec is used for UDT op codegen "graphblas/monoid/numpy.py" = ["PLW0108"] # lambda is needed for numba.njit +"benchmarks/*.py" = ["T201", "B015", "B018"] # asv: bare exprs are the measurement; _verify prints "graphblas/core/ss/matrix.py" = [ "NPY002", # numba doesn't support rng generator yet "PLR1730",