python-graphblas · eriknw · Jul 31, 2026
diff --git a/.flake8 b/.flake8
@@ -10,6 +10,7 @@ extend-ignore =
     SIM401,
 # E203 whitespace before ':' (to be compatible with black)
 per-file-ignores =
+    benchmarks/*.py:B015,B018,
     scripts/create_pickle.py:F403,F405,
     graphblas/tests/*.py:T201,B043,
     graphblas/core/ss/matrix.py:SIM113,

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -26,7 +26,11 @@ repos:
       - id: check-illegal-windows-names
       - id: check-merge-conflict
       - id: check-ast
-      - id: check-json # no JSON files yet; enabled for the future
+      # `asv.conf.json` is JSONC: asv documents `//` comments and strips them
+      # in its own loader, but this hook is a strict JSON parser. Excluding
+      # that one path keeps the hook live for every other JSON file.
+      - id: check-json
+        exclude: ^asv\.conf\.json$
       - id: check-toml
       - id: check-yaml
       - id: check-executables-have-shebangs

diff --git a/asv.conf.json b/asv.conf.json
@@ -0,0 +1,73 @@
+{
+  // asv (airspeed velocity) config for python-graphblas.
+  //
+  // Placement: this file and the benchmarks/ directory are intended to live at the
+  // repo root (next to pyproject.toml). The paths below are relative to that root.
+  // asv's config loader strips these // line comments; a strict JSON linter will
+  // not. If the repo's pre-commit `check-json` hook is enabled, exclude this file
+  // (asv.conf.json is asv's own format, and its docs use // comments).
+  //
+  // Typical use:
+  //   asv run                     # benchmark current commit
+  //   asv continuous main HEAD    # compare a branch against main
+  //   asv publish && asv preview  # build/serve the HTML report
+
+  "version": 1,
+  "project": "python-graphblas",
+  "project_url": "https://github.com/python-graphblas/python-graphblas",
+
+  // "." means the repo that contains this file. python-graphblas is pure Python, so
+  // building each commit is a cheap editable install; the C library comes from the
+  // separate suitesparse-graphblas package, pinned in the matrix below.
+  "repo": ".",
+  "branches": ["main"],
+  "dvcs": "git",
+
+  // conda matches the project's primary dev path and lets us pin the C library
+  // cleanly from conda-forge. asv will use mamba automatically if it is installed.
+  "environment_type": "conda",
+  "conda_channels": ["conda-forge"],
+
+  // PINNING NOTE: python-suitesparse-graphblas (the C library) is pinned so a run
+  // measures python-graphblas-side regressions in isolation. To instead track
+  // end-to-end performance including C-library changes, drop the pin (set it to [])
+  // or bump it deliberately. Keep this in sync with the PSG version the suite is
+  // calibrated against (see scripts/ci_pick_versions.py PSG pools).
+  "matrix": {
+    "req": {
+      "python-suitesparse-graphblas": ["10.3.1.0"],
+      "numpy": [],
+      "scipy": [],
+      "pandas": [],
+      "numba": []
+    }
+  },
+
+  // python-graphblas is pure Python: a no-build-isolation editable install is fast
+  // and reuses the already-solved conda env (which supplies the C library plus
+  // numba/scipy/pandas). asv substitutes {build_dir} for the checkout of each commit.
+  "build_command": [],
+  "install_command": [
+    "in-dir={build_dir} python -m pip install --no-build-isolation --no-deps -e ."
+  ],
+  "uninstall_command": [
+    "in-dir={build_dir} python -m pip uninstall -y python-graphblas"
+  ],
+
+  "benchmark_dir": "benchmarks",
+  "env_dir": ".asv/env",
+  "results_dir": ".asv/results",
+  "html_dir": ".asv/html",
+
+  // Give the C-library install room to solve/download.
+  "install_timeout": 900,
+
+  // Isolate JIT/compiler and thread-count noise so numbers are comparable across
+  // runs. GraphBLAS is multithreaded by default; pin to 1 thread for reproducible
+  // single-core timings (raise this in a dedicated run if you want to benchmark
+  // parallel scaling instead).
+  "environment_variables": {
+    "OMP_NUM_THREADS": "1",
+    "GRAPHBLAS_TEST_SEED": "0"
+  }
+}
diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -0,0 +1,91 @@
+# asv benchmark scaffold for python-graphblas
+
+A starting airspeed velocity (asv) benchmark suite covering the library's hot
+paths. Built as a scaffold: review, then move `asv.conf.json` and `benchmarks/`
+to the repo root (next to `pyproject.toml`). `_verify.py` is a local sanity
+checker and does not need to ship.
+
+## Layout
+
+```
+asv.conf.json            asv config, tuned for this repo (see caveats below).
+                         JSONC: asv documents `//` comments and strips them in
+                         its loader, so `.pre-commit-config.yaml` excludes this
+                         one path from the strict `check-json` hook. asv also
+                         accepts `asv.conf.jsonc`, which would need no exclude,
+                         but asv is not a pinned dependency here and older
+                         versions resolve only the `.json` name.
+benchmarks/
+  __init__.py            package marker + note on the dual-import shim
+  common.py              shared, seeded data builders (called from setup, not timed)
+  scalar_access.py       single-element get / extract / assign, hit vs miss
+  index_parse.py         parse_index int fast lane vs numpy-int lane (index build)
+  small_ops.py           ewise / apply / reduce on ~100-element objects (overhead)
+  large_kernels.py       mxm / mxv / ewise / reduce on ~1e6 nnz (C-library bound)
+  conversions.py         from_coo/to_coo, from_dense/to_dense, scipy interop
+  imports.py             cold import + first-use timing (timeraw, fresh subprocess)
+  repr_bench.py          repr / _repr_html_ for small and large objects
+_verify.py               standalone check: runs every benchmark once (not an asv file)
+```
+
+## Running
+
+```bash
+pip install asv
+asv machine --yes            # one-time: record machine info
+asv run                      # benchmark the current commit
+asv continuous main HEAD     # compare a branch against main, flag regressions
+asv publish && asv preview   # build and serve the HTML report
+asv run --bench scalar_access # run a subset by regex
+```
+
+Quick correctness check without asv (runs each benchmark once):
+
+```bash
+python _verify.py
+```
+
+## Design choices
+
+- `setup()` builds all data once, outside the timed region; every builder is
+  seeded so runs are comparable across commits.
+- Sizes: "small" is 10 to 1000 elements (overhead-dominated, so we track
+  Python-side cost); "large" is ~1e6 nonzeros (C-library dominated). Average
+  matrix degree is ~1 so `mxm` output stays near the input size instead of
+  exploding into quadratic fill-in. Verified per-call times on one dev machine:
+  large kernels ran in 1 to 5 ms, conversions under 10 ms, cold `import
+graphblas` ~32 ms, init/first-operator ~0.2 s. Bump `common.LARGE_NNZ` or add a
+  higher-degree matrix if you want heavier `mxm`.
+- Large kernels use `number = 1`, `warmup_time = 0`, small `repeat`, and a
+  `timeout`, so asv does not batch them into multi-second runs.
+- `imports.py` uses `timeraw_*`, which runs the returned code string in a fresh
+  subprocess. That is the only way to measure true cold import cost; a normal
+  benchmark would read ~0 because the module is already imported.
+- `asv.conf.json` pins `python-suitesparse-graphblas` so a run isolates
+  python-graphblas-side regressions from C-library changes. Drop the pin to track
+  end-to-end (library + C) performance. Keep the pinned version in sync with the
+  PSG pools in `scripts/ci_pick_versions.py`.
+- `OMP_NUM_THREADS=1` pins GraphBLAS to one thread for reproducible single-core
+  timings. Raise it in a dedicated run to benchmark parallel scaling.
+
+## Caveats to resolve before merging
+
+- **dev deps**: `asv` is deliberately NOT added to `dev-requirements.txt` or
+  `environment.yml`. Nothing in the test suite or CI invokes it, so adding it
+  would make every contributor's dev install fetch a package only used by
+  people who opt into benchmarking; and this repo ties dependency changes to
+  `scripts/check_versions.sh` and the CI version pools, which a benchmarks-only
+  change has no business editing. `pip install asv` is in the usage section
+  above. Adding a `benchmark` optional-dependencies group (so that
+  `pip install python-graphblas[benchmark]` works) is the natural move if the
+  suite graduates from scaffold to something CI runs.
+- **CI wiring**: asv is not wired into CI here. A weekly cron running `asv
+continuous` against a fixed machine (or asv's own regression detection) is the
+  natural follow-up; per-PR benchmarking on shared GitHub runners is too noisy to
+  gate on.
+- **First-run JIT/compile noise**: the first use of some operators triggers
+  numba/JIT work. `setup()` touches the operators the benchmark uses, but the
+  very first `asv run` on a fresh env may still show inflated one-off numbers;
+  asv's repeats and its own warmup mitigate this.
+- **pandas dependency**: `repr_bench.py` needs pandas (the default/test extras
+  include it). The env matrix installs pandas, so this is satisfied.
diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py
@@ -0,0 +1,12 @@
+"""python-graphblas benchmark suite (airspeed velocity).
+
+Benchmark modules live alongside this file. Shared data builders are in
+``common.py``. Each benchmark module imports ``common`` with a dual path so it
+works whether asv imports the files as a package or adds ``benchmark_dir`` to
+``sys.path`` and imports them flat:
+
+    try:
+        from . import common
+    except ImportError:  # imported flat, not as a package
+        import common
+"""
diff --git a/benchmarks/_verify.py b/benchmarks/_verify.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+"""Standalone sanity check for the asv benchmark suite (not an asv benchmark).
+
+Imports every benchmark module the way asv does (benchmark_dir on sys.path),
+then for each benchmark class runs setup() and calls each benchmark method once
+across all parameter combinations. timeraw_* methods return a code string, which
+is executed in a fresh subprocess. Prints PASS/FAIL per benchmark and a timing so
+we can spot anything accidentally slow.
+
+Usage:  python _verify.py
+"""
+
+import importlib
+import inspect
+import itertools
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+BENCH_DIR = HERE / "benchmarks"
+sys.path.insert(0, str(BENCH_DIR))  # flat import mode, as asv does
+
+MODULES = [
+    "scalar_access",
+    "index_parse",
+    "small_ops",
+    "large_kernels",
+    "conversions",
+    "imports",
+    "repr_bench",
+]
+PREFIXES = ("time_", "timeraw_", "peakmem_", "mem_", "track_")
+
+
+def param_combos(cls):
+    params = getattr(cls, "params", None)
+    if not params:
+        return [()]
+    # asv: a flat list is a single parameter; a list of lists is a product.
+    if params and isinstance(params[0], (list, tuple)):
+        return list(itertools.product(*params))
+    return [(p,) for p in params]
+
+
+def run_one(cls, combo):
+    results = []
+    obj = cls()
+    if hasattr(obj, "setup"):
+        obj.setup(*combo)
+    for name in sorted(dir(obj)):
+        if not name.startswith(PREFIXES):
+            continue
+        method = getattr(obj, name)
+        if not callable(method):
+            continue
+        label = f"{cls.__module__}.{cls.__name__}.{name}{combo or ''}"
+        t0 = time.perf_counter()
+        try:
+            if name.startswith("timeraw_"):
+                code = method()
+                setup_code = ""
+                if isinstance(code, tuple):
+                    code, setup_code = code
+                script = (setup_code + "\n" + code) if setup_code else code
+                subprocess.run(
+                    [sys.executable, "-c", script], check=True, capture_output=True, timeout=180
+                )
+            else:
+                method(*combo)
+            dt = time.perf_counter() - t0
+            results.append((label, dt, None))
+        except Exception as exc:
+            dt = time.perf_counter() - t0
+            results.append((label, dt, repr(exc)))
+    if hasattr(obj, "teardown"):
+        obj.teardown(*combo)
+    return results
+
+
+def main():
+    all_results = []
+    for modname in MODULES:
+        mod = importlib.import_module(modname)
+        classes = [
+            c for _, c in inspect.getmembers(mod, inspect.isclass) if c.__module__ == modname
+        ]
+        for cls in classes:
+            for combo in param_combos(cls):
+                all_results.extend(run_one(cls, combo))
+
+    fails = [r for r in all_results if r[2] is not None]
+    for label, dt, err in all_results:
+        status = "PASS" if err is None else "FAIL"
+        line = f"[{status}] {dt:7.3f}s  {label}"
+        if err is not None:
+            line += f"   -> {err}"
+        print(line)
+    print(
+        f"\n{len(all_results) - len(fails)}/{len(all_results)} benchmarks ran; {len(fails)} failed"
+    )
+    return 1 if fails else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/benchmarks/common.py b/benchmarks/common.py
@@ -0,0 +1,91 @@
+"""Shared data builders for the python-graphblas asv benchmark suite.
+
+Each benchmark module imports these helpers and calls them from ``setup`` so the
+input data is constructed once per benchmark (outside the timed region). All
+builders take a seed and use ``numpy.random.default_rng`` so runs are
+deterministic and comparable across commits.
+
+Size constants are chosen so the "large" kernels land near ~1e6 nonzeros (enough
+to exercise the C library) while still finishing in well under a second each, and
+so no dense intermediate blows up memory (a 1e6 x 1e6 dense matrix is never
+materialized; dense conversions use a separate, small, fully dense matrix).
+"""
+
+import numpy as np
+
+import graphblas as gb
+from graphblas import Matrix, Vector
+
+# Sizes
+
+# "Large" sparse operands: square matrix and vector near 1e6 nonzeros. Average
+# degree ~1 keeps mxm output bounded (A @ A stays near 1e6 nnz) so the kernel
+# benchmarks do not accidentally measure quadratic fill-in.
+LARGE_N = 1_000_000
+LARGE_NNZ = 1_000_000
+
+# "Small" operands: overhead-dominated. At this size the wall time is almost all
+# Python-side expression machinery, which is exactly what we want to track.
+SMALL_SIZE = 100
+
+# Dense conversions use a fully dense square matrix small enough to materialize.
+DENSE_DIM = 1000  # 1e6 dense elements
+
+# scipy interop matrix: ~1e6 nnz at 1% density.
+SCIPY_DIM = 10_000
+SCIPY_DENSITY = 0.01
+
+
+# Builders
+
+
+def make_coo(n, nnz, seed=0, dtype="FP64"):
+    """Return (rows, cols, vals) numpy arrays for an n x n matrix with ~nnz entries.
+
+    Duplicates are possible; callers that build a Matrix should pass a ``dup_op``.
+    """
+    rng = np.random.default_rng(seed)
+    rows = rng.integers(0, n, nnz, dtype=np.uint64)
+    cols = rng.integers(0, n, nnz, dtype=np.uint64)
+    if dtype == "BOOL":
+        vals = rng.integers(0, 2, nnz, dtype=np.bool_)
+    elif dtype in ("INT64", "INT32"):
+        vals = rng.integers(1, 100, nnz, dtype=np.int64)
+    else:
+        vals = rng.random(nnz)
+    return rows, cols, vals
+
+
+def make_matrix(n=LARGE_N, nnz=LARGE_NNZ, seed=0, dtype="FP64"):
+    """Square Matrix with ~nnz entries (duplicates summed)."""
+    rows, cols, vals = make_coo(n, nnz, seed=seed, dtype=dtype)
+    dup = gb.binary.lor if dtype == "BOOL" else gb.binary.plus
+    return Matrix.from_coo(rows, cols, vals, nrows=n, ncols=n, dtype=dtype, dup_op=dup)
+
+
+def make_vector(size=LARGE_N, nnz=LARGE_NNZ, seed=1, dtype="FP64"):
+    """Vector of length ``size`` with ~nnz entries (duplicates summed)."""
+    rng = np.random.default_rng(seed)
+    idx = rng.integers(0, size, nnz, dtype=np.uint64)
+    vals = rng.random(nnz) if dtype not in ("INT64", "INT32") else rng.integers(1, 100, nnz)
+    dup = gb.binary.plus
+    return Vector.from_coo(idx, vals, size=size, dtype=dtype, dup_op=dup)
+
+
+def make_dense_vector(size, seed=2, dtype="FP64"):
+    """Fully dense Vector (no missing entries), e.g. an mxv operand."""
+    rng = np.random.default_rng(seed)
+    return Vector.from_dense(rng.random(size))
+
+
+def make_dense_matrix(dim=DENSE_DIM, seed=3):
+    """Fully dense square Matrix built from a numpy array."""
+    rng = np.random.default_rng(seed)
+    return Matrix.from_dense(rng.random((dim, dim)))
+
+
+def make_scipy(dim=SCIPY_DIM, density=SCIPY_DENSITY, seed=4, fmt="csr"):
+    """A scipy.sparse matrix for interop benchmarks."""
+    import scipy.sparse as sp
+
+    return sp.random(dim, dim, density=density, format=fmt, random_state=seed)