Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/benchx.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,9 @@ neither asv nor asv_runner is used. PATH is a directory holding `asv.conf.json`
`benchmark_dir` is the suite) or the benchmark directory itself, and its checkout is
the target (`bx bench . --adapter asv --suite benchmarks` is the same order).
`--filter` is asv's `--bench`: searched in the name, and in `name(p0, p1)` for each
parameter combination; a CASE is an exact name, `name(p0, p1)` for a combination. `adapters/asv_runtime.py` discovers
parameter combination, the arguments in the order of `param_names`, each as its `repr`; a CASE is an exact name,
`name(p0, p1)` for a combination. A filter that names a parametrized benchmark and none of its combinations is
refused with that order and one case as it is searched. `adapters/asv_runtime.py` discovers
the benchmarks as asv does (names, `params`, `setup`/`teardown`/`setup_cache`, skips,
and asv's `version` hash all agree with asv_runner's; a test checks it). The
benchmarks run in the `python` first on `PATH`, which must import pyperf and the
Expand Down
2 changes: 1 addition & 1 deletion src/benchx/adapters/asv.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]:
for case, index in benchx_params.expand(info["name"], info["params"], case_filter)]
if not selected:
if benchx_params.matches_name_only(info["name"], info["params"], case_filter):
unmatched.append(info["name"])
unmatched.append(info)
continue
if info["type"] != "time":
other[info["type"]] = other.get(info["type"], 0) + 1
Expand Down
63 changes: 48 additions & 15 deletions src/benchx/adapters/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,15 @@
attempt. For a harness that repeats at several levels.
"""

import difflib
import os
import re
import signal
import subprocess
import textwrap

from ..core import Refusal, Rejected
from .benchx_params import combination
from .benchx_params import combination, search_name


class Unsupported(Rejected):
Expand All @@ -97,25 +100,55 @@ def parametrized_workload(case, runnable) -> tuple[str, dict]:
return name, dict(zip(info["param_names"], combination(info["params"], index)))


def name_only_note(names) -> str:
"""For a refusal: the filter matched parametrized benchmarks by name, but not their
arguments (see `benchx_params.matches_name_only`)."""
return (f"the filter matches the name of a parametrized benchmark ({', '.join(names[:3])}) but "
"not its arguments, and those are matched as name(p0, p1)")


def no_cases(where, case_filter, available=(), notes=(), limit=10) -> Unsupported:
def name_only_note(found) -> str:
"""For a refusal: the filter is about parametrized benchmarks (`found`, their listing entries)
but matches none of their arguments (see `benchx_params.matches_name_only`). Says how a case
is searched, with the first benchmark's parameters, in order, and one case as it is searched."""
first = found[0]
names = ", ".join(info["name"] for info in found[:3])
return "\n".join([
f"the filter matches the name of a parametrized benchmark ({names}) but not its arguments.",
"They are searched as name(p0, p1), each as its repr, in the order of param_names:",
f" {first['name']}({', '.join(first['param_names'])})",
"for example:",
f" {search_name(first['name'], first['params'], 0)}"])


def similar(case_filter, names, limit=5, cutoff=0.75) -> list[str]:
"""The names a filter may have meant: those that have, as a dotted part (module, class, method), the
most of its words, or words close to them, then the closest first. A word is a run of letters, digits
and `_` of three characters or more, so an escape such as `\\d` says nothing, and the case of a letter
is not a difference. A filter that names a module and misspells its class thus finds that class's
benchmarks, and not the whole module."""
words = [w.lower() for w in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", case_filter or "") if len(w) >= 3]
scored = []
for name in names:
parts = name.lower().split(".")
best = (max(difflib.SequenceMatcher(None, word, part).ratio() for part in parts) for word in words)
hits = [score for score in best if score >= cutoff]
if hits:
scored.append((-len(hits), -sum(hits), name))
scored.sort()
return [name for count, _, name in scored[:limit] if count == scored[0][0]]


def no_cases(where, case_filter, available=(), notes=(), limit=5) -> Unsupported:
"""The refusal for a plan with no case in it. Nothing would run, and a filter with a
typo must not end in `0 result(s)` and a success status. `available` are the names
that could have been selected (the first `limit` are shown) and `notes` say what else
the adapter knows about why the plan is empty."""
head = f"filter {case_filter!r} selects no case in {where}" if case_filter else f"no case to run in {where}"
parts = [head]
the adapter knows about why the plan is empty, each on its own lines (a note may have several).
The last line is the names that are similar to the filter (`similar`), else the first available.
The filter is shown as given, not as a Python string, so its backslashes are the ones typed."""
head = f"filter '{case_filter}' selects no case in {where}" if case_filter else f"no case to run in {where}"
lines = [head] + [textwrap.indent(note, " ") for note in notes]
names = sorted(available)
if names:
close = similar(case_filter, names, limit)
if close:
lines.append(f" similar: {', '.join(close)}")
elif names:
more = f", … ({len(names) - limit} more)" if len(names) > limit else ""
parts.append(f"available: {', '.join(names[:limit])}{more}")
return Unsupported("; ".join(parts + list(notes)))
lines.append(f" available: {', '.join(names[:limit])}{more}")
return Unsupported("\n".join(lines))


def run_bounded(args, *, timeout, what, **kwargs) -> subprocess.CompletedProcess:
Expand Down
16 changes: 13 additions & 3 deletions src/benchx/adapters/benchx_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,16 @@ def expand(name, ids, case_filter):


def matches_name_only(name, ids, case_filter) -> bool:
"""Whether a filter that selected none of a benchmark's cases matches its name alone:
a parametrized benchmark is searched in `name(p0, p1)`, so `name$` is a likely slip."""
return bool(ids and case_filter and re.search(case_filter, name))
"""Whether a filter that selected none of a benchmark's cases is about this benchmark, though
not about any of its combinations: it matches the name alone (`name$` is a likely slip), or the
part of it before its first `\\(` does, so what it gives after the name, the arguments, is
what fails (in the wrong order, or a value the benchmark does not have)."""
if not (ids and case_filter):
return False
named = case_filter.split(r"\(")[0]
if not named.strip("^"): # it starts with its arguments: no name to go by
return False
try:
return bool(re.search(named, name))
except re.error:
return False
2 changes: 1 addition & 1 deletion src/benchx/adapters/pyperf.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]:
selected = benchx_params.expand(name, info["params"], case_filter)
if not selected:
if benchx_params.matches_name_only(name, info["params"], case_filter):
unmatched.append(name)
unmatched.append(info)
continue
if "error" in info:
raise Unsupported(f"cannot read the params of {name} in {where}: {info['error']}")
Expand Down
33 changes: 32 additions & 1 deletion tests/test_asv.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,10 +197,18 @@ def test_a_filter_that_selects_nothing_is_refused(suite, capsys):
assert cli.main([*ASV, "--filter", "time_plian", "--run-key", "typo", "--no-ingest", *FAST]) == cli.EXIT_REFUSED
err = capsys.readouterr().err
assert "refused: filter 'time_plian' selects no case in" in err
assert "available: bm.Cached.time_sum, bm.Mutating.time_pop," in err
assert "\n similar: bm.time_plain\n" in err # what the typo is close to, not the first of the alphabet
assert "available" not in err
assert results(suite, "typo") == {}


def test_a_filter_that_is_like_no_name_lists_the_first_available(suite, capsys):
assert cli.main([*ASV, "--filter", "zzzzzz", "--run-key", "unlike", "--no-ingest", *FAST]) == cli.EXIT_REFUSED
err = capsys.readouterr().err
assert "available: bm.Cached.time_sum, bm.Mutating.time_pop," in err
assert "similar" not in err


def test_an_anchored_name_of_a_parametrized_benchmark_is_refused_with_a_hint(suite, capsys):
# asv searches `name(p0, p1)`, so `time_sorted$` matches none of the combinations.
assert cli.main([*ASV, "--filter", "time_sorted$", "--run-key", "anchored", "--no-ingest",
Expand All @@ -209,6 +217,16 @@ def test_an_anchored_name_of_a_parametrized_benchmark_is_refused_with_a_hint(sui
in capsys.readouterr().err)


def test_arguments_in_the_wrong_order_are_refused_with_the_order_and_an_example(suite, capsys):
# `Sort` takes (n, key); `('a', 10)` names the benchmark and selects none of its combinations.
assert cli.main([*ASV, "--filter", r"time_sorted\('a', 10\)", "--run-key", "order", "--no-ingest",
*FAST]) == cli.EXIT_REFUSED
err = capsys.readouterr().err
assert "the filter matches the name of a parametrized benchmark (bm.Sort.time_sorted) but not its arguments" in err
assert "\n bm.Sort.time_sorted(n, key)\n for example:\n bm.Sort.time_sorted(10, 'a')\n" in err
assert results(suite, "order") == {}


def test_a_filter_that_selects_only_benchmarks_that_are_not_run_is_refused(suite, capsys):
assert cli.main([*ASV, "--filter", "peakmem_sorted", "--run-key", "notrun", "--no-ingest",
*FAST]) == cli.EXIT_REFUSED
Expand Down Expand Up @@ -419,3 +437,16 @@ def test_missing_asv_runner_is_refused(suite, monkeypatch, tmp, capsys):
monkeypatch.setenv("PATH", str(bare))
assert cli.main([*ASV, "--no-ingest"]) == cli.EXIT_REFUSED
assert "pyperf is not importable" in capsys.readouterr().err


def test_a_refusal_shows_the_filter_as_it_was_typed(suite, capsys):
"""Backslashes are not doubled (not a Python repr), and the refusal is one fact per line."""
assert cli.main([*ASV, "--filter", r"time_sorted\('a', 10\)", "--run-key", "typed", "--no-ingest",
*FAST]) == cli.EXIT_REFUSED
lines = [line for line in capsys.readouterr().err.splitlines() if line.startswith(("refused:", " "))]
assert lines[0].startswith(r"refused: filter 'time_sorted\('a', 10\)' selects no case in ")
assert [line.split(":")[0].strip() for line in lines[1:]] == [
"the filter matches the name of a parametrized benchmark (bm.Sort.time_sorted) but not its arguments.",
"They are searched as name(p0, p1), each as its repr, in the order of param_names",
"bm.Sort.time_sorted(n, key)", "for example", "bm.Sort.time_sorted(10, 'a')",
"1 module(s) failed to import", "similar"] # the suite has a module that does not import
2 changes: 1 addition & 1 deletion tests/test_bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def ran(run_key):
assert cli.main(["bench", "mod.py", "bench_sort(100, 'a')", "--run-key", "one", "--no-ingest", *FAST]) == cli.EXIT_OK
assert ran("one") == ["100'a'"]
assert cli.main(["bench", "mod.py", "bench_srot", "--run-key", "typo", "--no-ingest", *FAST]) == cli.EXIT_REFUSED
assert "selects no case in mod.py; available: bench_sort" in capsys.readouterr().err
assert "selects no case in mod.py\n similar: bench_sort\n" in capsys.readouterr().err


def test_shorthand_order_is_stable(tree):
Expand Down
45 changes: 44 additions & 1 deletion tests/test_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,17 +76,60 @@ def test_a_filter_that_matches_only_the_name_is_noticed():
assert not params.matches_name_only("m.C.time_x", IDS, None)


def test_a_filter_that_gives_the_arguments_in_the_wrong_order_is_noticed():
"""The part before the first `\\(` is the benchmark's name; what follows it selected nothing."""
assert params.expand("m.C.time_x", IDS, r"C\.time_x\('a', 10\)") == []
assert params.matches_name_only("m.C.time_x", IDS, r"C\.time_x\('a', 10\)")
assert params.matches_name_only("m.C.time_x", IDS, r"^m\.C\.time_x\(100, 'c'\)$")
assert not params.matches_name_only("m.C.time_x", IDS, r"C\.time_y\(10, 'a'\)") # another benchmark
assert not params.matches_name_only("m.C.time_x", IDS, r"\(10, ") # no name to go by
assert not params.matches_name_only("m.C.time_x", IDS, r"^\(10, ")
assert not params.matches_name_only("m.C.time_x", IDS, r"(unbalanced\(10") # not a regex: no hint, no error


def test_the_workload_of_a_case_comes_from_the_table_not_from_the_id():
runnable = {"cases": {"m.C.time_x-2": ("m.C.time_x", 2), "m.time_y-1": ("m.time_y", None)},
"benchmarks": {"m.C.time_x": {"params": IDS, "param_names": ["n", "key"]}, "m.time_y": {}}}
assert base.parametrized_workload("m.C.time_x-2", runnable) == ("m.C.time_x", {"n": "100", "key": "'a'"})
assert base.parametrized_workload("m.time_y-1", runnable) == ("m.time_y", {}) # a name that looks like -1


NAMES = ["bench_function_base.Partition.time_argpartition", "bench_function_base.Partition.time_partition",
"bench_function_base.Sort.time_sort", "bench_linalg.Linalg.time_norm", "bench_ufunc.UFunc.time_add"]


def test_a_slip_in_a_word_of_the_filter_finds_the_names_that_have_it():
assert base.similar("Partiton", NAMES) == NAMES[:2] # a class, as spelled in the names
assert base.similar("time_argpartiton", NAMES) == NAMES[:2] # a method: its own first (0.97), then time_partition (0.87)
assert base.similar("bench_linalgg", NAMES) == [NAMES[3]] # a module
assert base.similar("partition", NAMES) == NAMES[:2] # the case of a letter is no difference


def test_the_words_of_a_regex_are_what_is_compared():
assert base.similar(r"^bench_function_base\.Partiton\.time_x\(10, 'a'\)$", NAMES) == NAMES[:2]
assert base.similar(r"Partiton|time_ad\b", NAMES) == [NAMES[0], NAMES[1], NAMES[4]] # 0.94, 0.94, then 0.93
assert base.similar(r"\d+\.\w", NAMES) == [] # escapes and one letter say nothing


def test_nothing_is_similar_to_what_is_not_like_any_name():
assert base.similar("zzzz", NAMES) == []
assert base.similar(None, NAMES) == []
assert base.similar("Partiton", []) == []


def test_the_most_similar_names_come_first_and_at_most_limit_are_returned():
names = ["m.Alpha.time_a", "m.Alphb.time_a", "m.Alpxx.time_a", "m.Alpha.time_b"]
assert base.similar("Alpha", names) == ["m.Alpha.time_a", "m.Alpha.time_b", "m.Alphb.time_a"] # exact, then near
assert base.similar("Alpha", names, limit=1) == ["m.Alpha.time_a"]


def test_the_note_names_the_benchmarks_it_means():
note = base.name_only_note(["a", "b", "c", "d"])
found = [{"name": name, "param_names": ["n", "key"], "params": IDS} for name in "abcd"]
note = base.name_only_note(found)
assert "(a, b, c)" in note # three at most
assert "name(p0, p1)" in note
# the first one's parameters, in order, and a case of it, each on a line of its own to be copied
assert note.splitlines()[2:] == [" a(n, key)", "for example:", " a(10, 'a')"]


def test_search_name_is_exactly_what_expand_searches():
Expand Down
20 changes: 19 additions & 1 deletion tests/test_pyperf.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,10 +183,28 @@ def test_a_filter_that_matches_a_name_but_no_arguments_is_refused_with_a_hint(pa
with pytest.raises(adapter.Unsupported) as refused:
planned(param_tree, "bench_sort$")
message = str(refused.value.problems[0])
assert "filter 'bench_sort$' selects no case in mod.py; available: bench_plain, bench_sort;" in message
assert message.startswith("filter 'bench_sort$' selects no case in mod.py\n")
assert "\n similar: bench_sort" in message
assert "the filter matches the name of a parametrized benchmark (bench_sort) but not its arguments" in message


def test_a_refusal_says_how_the_arguments_of_a_case_are_searched(param_tree):
"""Arguments in the wrong order: the filter names the function, so the note gives the order
of `param_names` and a case as it is searched."""
with pytest.raises(adapter.Unsupported) as refused:
planned(param_tree, r"bench_sort\('a', 10\)")
message = str(refused.value.problems[0])
assert "the filter matches the name of a parametrized benchmark (bench_sort) but not its arguments" in message
assert message.splitlines()[3:6] == [" bench_sort(n, key)", " for example:", " bench_sort(10, 'a')"]
assert message.index("not its arguments") < message.index("similar:") # the hint comes before the list


def test_a_filter_with_no_name_in_it_gets_no_note_about_arguments(param_tree):
with pytest.raises(adapter.Unsupported) as refused:
planned(param_tree, r"\(7, ")
assert "not its arguments" not in str(refused.value.problems[0])


def test_params_that_cannot_be_read_matter_only_to_the_function_that_has_them(param_tree):
with pytest.raises(adapter.Unsupported, match="cannot read the params of helper in mod.py: TypeError"):
planned(param_tree, "helper")
Expand Down
Loading