Fix benchmark selection.

python · ericsnowcurrently · Dec 8, 2021 · Jun 18, 2021 · Jun 18, 2021 · Jun 21, 2021
commit f30421d9fced159db8499a26c8c1f51048f24d14
diff --git a/pyperformance/_utils/__init__.py b/pyperformance/_utils/__init__.py
@@ -4,6 +4,12 @@
 from ._fs import (
     temporary_file,
 )
+from ._misc import (
+    check_name,
+    parse_name_pattern,
+    parse_tag_pattern,
+    parse_selections,
+)
 from ._platform import (
     MS_WINDOWS,
 )
diff --git a/pyperformance/_utils/_misc.py b/pyperformance/_utils/_misc.py
@@ -0,0 +1,51 @@
+
+def check_name(name, *, loose=False):
+    if not name or not isinstance(name, str):
+        raise ValueError(f'bad name {name!r}')
+    if not loose:
+        if name.startswith('-'):
+            raise ValueError(name)
+        if not name.replace('-', '_').isidentifier():
+            raise ValueError(name)
+
+
+def parse_name_pattern(text, *, fail=True):
+    name = text
+    # XXX Support globs and/or regexes?  (return a callable)
+    try:
+        check_name('_' + name)
+    except Exception:
+        if fail:
+            raise  # re-raise
+        return None
+    return name
+
+
+def parse_tag_pattern(text):
+    if not text.startswith('<'):
+        return None
+    if not text.endswith('>'):
+        return None
+    tag = text[1:-1]
+    # XXX Support globs and/or regexes?  (return a callable)
+    check_name(tag)
+    return tag
+
+
+def parse_selections(selections, parse_entry=None):
+    if isinstance(selections, str):
+        selections = selections.split(',')
+    if parse_entry is None:
+        parse_entry = (lambda o, e: (o, e, None, e))
+
+    for entry in selections:
+        entry = entry.strip()
+        if not entry:
+            continue
+
+        op = '+'
+        if entry.startswith('-'):
+            op = '-'
+            entry = entry[1:]
+
+        yield parse_entry(op, entry)
diff --git a/pyperformance/benchmark/__init__.py b/pyperformance/benchmark/__init__.py
@@ -1,4 +1,4 @@
 
 # aliases
-from ._spec import BenchmarkSpec, parse_benchmark
+from ._spec import BenchmarkSpec, parse_benchmark, check_name
 from ._benchmark import Benchmark
diff --git a/pyperformance/benchmark/_benchmark.py b/pyperformance/benchmark/_benchmark.py
@@ -1,14 +1,17 @@
-from ._spec import parse_benchmark
+from ._spec import BenchmarkSpec
 
 
 class Benchmark:
 
-    def __init__(self, spec, run):
-        if isinstance(spec, str):
-            spec = parse_benchmark(spec)
+    def __init__(self, spec, metafile):
+        spec, _metafile = BenchmarkSpec.from_raw(spec)
+        if not metafile:
+            if not _metafile:
+                raise ValueError(f'missing metafile for {spec!r}')
+            metafile = _metafile
 
         self.spec = spec
-        self.run = run
+        self.metafile = metafile
 
     def __repr__(self):
         return f'{type(self).__name__}(spec={self.spec}, run={self.run})'
@@ -32,3 +35,6 @@ def __gt__(self, other):
         except AttributeError:
             return NotImplemented
         return self.spec > other_spec
+
+    def run(self, *args):
+        return self._func(*args)
diff --git a/pyperformance/benchmark/_spec.py b/pyperformance/benchmark/_spec.py
@@ -1,14 +1,10 @@
 from collections import namedtuple
 
+from .. import _utils
 
-class BenchmarkSpec(namedtuple('BenchmarkSpec', 'name version origin')):
-
-    metafile = None
 
-    def __new__(cls, name, version=None, origin=None, metafile=None):
-        self = super().__new__(cls, name, version, origin)
-        self.metafile = metafile
-        return self
+def check_name(name):
+    _utils.check_name('_' + name)
 
 
 def parse_benchmark(entry):
@@ -18,4 +14,18 @@ def parse_benchmark(entry):
     metafile = None
     if not f'_{name}'.isidentifier():
         raise ValueError(f'unsupported benchmark name in {entry!r}')
-    return BenchmarkSpec(name, version, origin, metafile)
+    bench = BenchmarkSpec(name, version, origin)
+    return bench, metafile
+
+
+class BenchmarkSpec(namedtuple('BenchmarkSpec', 'name version origin')):
+    __slots__ = ()
+
+    @classmethod
+    def from_raw(cls, raw):
+        if isinstance(raw, BenchmarkSpec):
+            return raw, None
+        elif isinstance(raw, str):
+            return parse_benchmark(raw)
+        else:
+            raise ValueError(f'unsupported raw spec {raw!r}')
diff --git a/pyperformance/benchmarks/__init__.py b/pyperformance/benchmarks/__init__.py
@@ -4,8 +4,9 @@
 from .. import _benchmarks, benchmark as _benchmark
 from . import _manifest
 
-# an alias (but also used here)
-from ._parse import parse_benchmarks
+# aliases
+from ._manifest import expand_benchmark_groups
+from ._selections import parse_selection, iter_selections
 
 
 DEFAULTS_DIR = os.path.join(
@@ -19,14 +20,23 @@ def load_manifest(filename, *, resolve=None):
         filename = DEFAULT_MANIFEST
         if resolve is None:
             def resolve(bench):
-                if not bench.version:
-                    bench = bench._replace(version=__version__)
-                if not bench.origin:
-                    bench = bench._replace(origin='<default>')
+                if isinstance(bench, _benchmark.Benchmark):
+                    spec = bench.spec
+                else:
+                    spec = bench
+                    bench = _benchmark.Benchmark(spec, '<bogus>')
+                    bench.metafile = None
+
+                if not spec.version:
+                    spec = spec._replace(version=__version__)
+                if not spec.origin:
+                    spec = spec._replace(origin='<default>')
+                bench.spec = spec
+
                 if not bench.metafile:
                     metafile = os.path.join(DEFAULTS_DIR,
                                             f'bm_{bench.name}',
-                                            'METADATA')
+                                            'pyproject.toml')
                     #bench = bench._replace(metafile=metafile)
                     bench.metafile = metafile
                 return bench
@@ -37,53 +47,10 @@ def resolve(bench):
 def iter_benchmarks(manifest):
     # XXX Use the benchmark's "run" script.
     funcs, _ = _benchmarks.get_benchmarks()
-    for spec in manifest.benchmarks:
-        func = funcs[spec.name]
-        yield _benchmark.Benchmark(spec, func)
+    for bench in manifest.benchmarks:
+        bench._func = funcs[bench.name]
+        yield bench
 
 
 def get_benchmarks(manifest):
     return list(iter_benchmarks(manifest))
-
-
-def get_benchmark_groups(manifest):
-    return dict(manifest.groups)
-
-
-def expand_benchmark_groups(parsed, groups):
-    if isinstance(parsed, str):
-        parsed = _benchmark.parse_benchmark(parsed)
-
-    if not groups:
-        yield parsed
-    elif parsed.name not in groups:
-        yield parsed
-    else:
-        benchmarks = groups[parsed.name]
-        for bench in benchmarks or ():
-            yield from expand_benchmark_groups(bench, groups)
-
-
-def select_benchmarks(raw, manifest, *,
-                      expand=None,
-                      known=None,
-                      ):
-    if expand is None:
-        groups = get_benchmark_groups(manifest)
-        expand = lambda n: expand_benchmark_groups(n, groups)
-    if known is None:
-        known = get_benchmarks(manifest)
-    benchmarks = {b.spec: b for b in get_benchmarks(manifest)}
-
-    included, excluded = parse_benchmarks(raw, expand=expand, known=known)
-    if not included:
-        included = set(expand('default', 'add'))
-
-    selected = set()
-    for spec in included:
-        bench = benchmarks[spec]
-        selected.add(bench)
-    for spec in excluded:
-        bench = benchmarks[spec]
-        selected.remove(bench)
-    return selected
diff --git a/pyperformance/benchmarks/_manifest.py b/pyperformance/benchmarks/_manifest.py
@@ -1,6 +1,6 @@
 from collections import namedtuple
 
-from .. import benchmark as _benchmark
+from .. import benchmark as _benchmark, _utils
 
 
 BENCH_COLUMNS = ('name', 'version', 'origin', 'metafile')
@@ -26,9 +26,33 @@ def parse_manifest(text, *, resolve=None):
         elif section.startswith('group '):
             _, _, group = section.partition(' ')
             groups[group] = _parse_group(group, seclines, benchmarks)
+    _check_groups(groups)
+    # XXX Update tags for each benchmark with member groups.
     return BenchmarksManifest(benchmarks, groups)
 
 
+def expand_benchmark_groups(bench, groups):
+    if isinstance(bench, str):
+        spec, metafile = _benchmark.parse_benchmark(bench)
+        if metafile:
+            bench = _benchmark.Benchmark(spec, metafile)
+        else:
+            bench = spec
+    elif isinstance(bench, _benchmark.Benchmark):
+        spec = bench.spec
+    else:
+        spec = bench
+
+    if not groups:
+        yield bench
+    elif bench.name not in groups:
+        yield bench
+    else:
+        benchmarks = groups[bench.name]
+        for bench in benchmarks or ():
+            yield from expand_benchmark_groups(bench, groups)
+
+
 def _iter_sections(lines):
     lines = (line.split('#')[0].strip()
              for line in lines)
@@ -63,66 +87,48 @@ def _parse_benchmarks(lines, resolve):
     benchmarks = []
     for line in lines:
         try:
-            name, version, origin, metafile = line.split('\t')
+            name, version, origin, metafile = (None if l == '-' else l
+                                               for l in line.split('\t'))
         except ValueError:
             raise ValueError(f'bad benchmark line {line!r}')
-        if not version or version == '-':
-            version = None
-        if not origin or origin == '-':
-            origin = None
-        if not metafile or metafile == '-':
-            metafile = None
-        bench = _benchmark.BenchmarkSpec(name, version, origin, metafile)
+        spec = _benchmark.BenchmarkSpec(name or None,
+                                        version or None,
+                                        origin or None,
+                                        )
+        if metafile:
+            bench = _benchmark.Benchmark(spec, metafile)
+        else:
+            bench = spec
         if resolve is not None:
             bench = resolve(bench)
         benchmarks.append(bench)
     return benchmarks
 
 
 def _parse_group(name, lines, benchmarks):
-    benchmarks = set(benchmarks)
     byname = {b.name: b for b in benchmarks}
+    if name in byname:
+        raise ValueError(f'a group and a benchmark have the same name ({name})')
+
     group = []
+    seen = set()
     for line in lines:
-        bench = _benchmark.parse_benchmark(line)
-        if bench not in benchmarks:
-            try:
-                bench = byname[bench.name]
-            except KeyError:
-                raise ValueError(f'unknown benchmark {bench.name!r} ({name})')
-        group.append(bench)
+        benchname = line
+        _benchmark.check_name(benchname)
+        if benchname in seen:
+            continue
+        if benchname in byname:
+            group.append(byname[benchname])
+        else:
+            # It may be a group.  We check later.
+            group.append(benchname)
     return group
 
 
-#def render_manifest(manifest):
-#    if isinstance(manifest, str):
-#        raise NotImplementedError
-#        manifest = manifest.splitlines()
-#    yield BENCH_HEADER
-#    for row in manifest:
-#        if isinstance(row, str):
-#            row = _parse_manifest_row(row)
-#            if isinstance(row, str):
-#                yield row
-#                continue
-#        line _render_manifest_row(row)
-#
-#    raise NotImplementedError
-#
-#
-#def parse_group_manifest(text):
-#    ...
-#
-#
-#def render_group_manifest(group, benchmarks):
-#    # (manifest file, bm name)
-#    ...
-#
-#
-#def parse_bench_from_manifest(line):
-#    raise NotImplementedError
-#
-#
-#def render_bench_for_manifest(benchmark, columns):
-#    raise NotImplementedError
-#    name, origin, version, metafile = info
+def _check_groups(groups):
+    for group, benchmarks in groups.items():
+        for bench in benchmarks:
+            if not isinstance(bench, str):
+                continue
+            elif bench not in groups:
+                raise ValueError(f'unknown benchmark {name!r} (in group {group!r})')