diff --git a/docs/benchx.md b/docs/benchx.md index b6ad371..09ef460 100644 --- a/docs/benchx.md +++ b/docs/benchx.md @@ -183,7 +183,9 @@ neither asv nor asv_runner is used. PATH is a directory holding `asv.conf.json` `benchmark_dir` is the suite) or the benchmark directory itself, and its checkout is the target (`bx bench . --adapter asv --suite benchmarks` is the same order). `--filter` is asv's `--bench`: searched in the name, and in `name(p0, p1)` for each -parameter combination; a CASE is an exact name, `name(p0, p1)` for a combination. `adapters/asv_runtime.py` discovers +parameter combination, the arguments in the order of `param_names`, each as its `repr`; a CASE is an exact name, +`name(p0, p1)` for a combination. A filter that names a parametrized benchmark and none of its combinations is +refused with that order and one case as it is searched. `adapters/asv_runtime.py` discovers the benchmarks as asv does (names, `params`, `setup`/`teardown`/`setup_cache`, skips, and asv's `version` hash all agree with asv_runner's; a test checks it). The benchmarks run in the `python` first on `PATH`, which must import pyperf and the diff --git a/src/benchx/adapters/asv.py b/src/benchx/adapters/asv.py index 248b594..289477a 100644 --- a/src/benchx/adapters/asv.py +++ b/src/benchx/adapters/asv.py @@ -199,7 +199,7 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: for case, index in benchx_params.expand(info["name"], info["params"], case_filter)] if not selected: if benchx_params.matches_name_only(info["name"], info["params"], case_filter): - unmatched.append(info["name"]) + unmatched.append(info) continue if info["type"] != "time": other[info["type"]] = other.get(info["type"], 0) + 1 diff --git a/src/benchx/adapters/base.py b/src/benchx/adapters/base.py index 6239708..ed957eb 100644 --- a/src/benchx/adapters/base.py +++ b/src/benchx/adapters/base.py @@ -67,12 +67,15 @@ attempt. For a harness that repeats at several levels. """ +import difflib import os +import re import signal import subprocess +import textwrap from ..core import Refusal, Rejected -from .benchx_params import combination +from .benchx_params import combination, search_name class Unsupported(Rejected): @@ -97,25 +100,55 @@ def parametrized_workload(case, runnable) -> tuple[str, dict]: return name, dict(zip(info["param_names"], combination(info["params"], index))) -def name_only_note(names) -> str: - """For a refusal: the filter matched parametrized benchmarks by name, but not their - arguments (see `benchx_params.matches_name_only`).""" - return (f"the filter matches the name of a parametrized benchmark ({', '.join(names[:3])}) but " - "not its arguments, and those are matched as name(p0, p1)") - - -def no_cases(where, case_filter, available=(), notes=(), limit=10) -> Unsupported: +def name_only_note(found) -> str: + """For a refusal: the filter is about parametrized benchmarks (`found`, their listing entries) + but matches none of their arguments (see `benchx_params.matches_name_only`). Says how a case + is searched, with the first benchmark's parameters, in order, and one case as it is searched.""" + first = found[0] + names = ", ".join(info["name"] for info in found[:3]) + return "\n".join([ + f"the filter matches the name of a parametrized benchmark ({names}) but not its arguments.", + "They are searched as name(p0, p1), each as its repr, in the order of param_names:", + f" {first['name']}({', '.join(first['param_names'])})", + "for example:", + f" {search_name(first['name'], first['params'], 0)}"]) + + +def similar(case_filter, names, limit=5, cutoff=0.75) -> list[str]: + """The names a filter may have meant: those that have, as a dotted part (module, class, method), the + most of its words, or words close to them, then the closest first. A word is a run of letters, digits + and `_` of three characters or more, so an escape such as `\\d` says nothing, and the case of a letter + is not a difference. A filter that names a module and misspells its class thus finds that class's + benchmarks, and not the whole module.""" + words = [w.lower() for w in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", case_filter or "") if len(w) >= 3] + scored = [] + for name in names: + parts = name.lower().split(".") + best = (max(difflib.SequenceMatcher(None, word, part).ratio() for part in parts) for word in words) + hits = [score for score in best if score >= cutoff] + if hits: + scored.append((-len(hits), -sum(hits), name)) + scored.sort() + return [name for count, _, name in scored[:limit] if count == scored[0][0]] + + +def no_cases(where, case_filter, available=(), notes=(), limit=5) -> Unsupported: """The refusal for a plan with no case in it. Nothing would run, and a filter with a typo must not end in `0 result(s)` and a success status. `available` are the names that could have been selected (the first `limit` are shown) and `notes` say what else - the adapter knows about why the plan is empty.""" - head = f"filter {case_filter!r} selects no case in {where}" if case_filter else f"no case to run in {where}" - parts = [head] + the adapter knows about why the plan is empty, each on its own lines (a note may have several). + The last line is the names that are similar to the filter (`similar`), else the first available. + The filter is shown as given, not as a Python string, so its backslashes are the ones typed.""" + head = f"filter '{case_filter}' selects no case in {where}" if case_filter else f"no case to run in {where}" + lines = [head] + [textwrap.indent(note, " ") for note in notes] names = sorted(available) - if names: + close = similar(case_filter, names, limit) + if close: + lines.append(f" similar: {', '.join(close)}") + elif names: more = f", … ({len(names) - limit} more)" if len(names) > limit else "" - parts.append(f"available: {', '.join(names[:limit])}{more}") - return Unsupported("; ".join(parts + list(notes))) + lines.append(f" available: {', '.join(names[:limit])}{more}") + return Unsupported("\n".join(lines)) def run_bounded(args, *, timeout, what, **kwargs) -> subprocess.CompletedProcess: diff --git a/src/benchx/adapters/benchx_params.py b/src/benchx/adapters/benchx_params.py index bc27cd5..247cb3d 100644 --- a/src/benchx/adapters/benchx_params.py +++ b/src/benchx/adapters/benchx_params.py @@ -84,6 +84,16 @@ def expand(name, ids, case_filter): def matches_name_only(name, ids, case_filter) -> bool: - """Whether a filter that selected none of a benchmark's cases matches its name alone: - a parametrized benchmark is searched in `name(p0, p1)`, so `name$` is a likely slip.""" - return bool(ids and case_filter and re.search(case_filter, name)) + """Whether a filter that selected none of a benchmark's cases is about this benchmark, though + not about any of its combinations: it matches the name alone (`name$` is a likely slip), or the + part of it before its first `\\(` does, so what it gives after the name, the arguments, is + what fails (in the wrong order, or a value the benchmark does not have).""" + if not (ids and case_filter): + return False + named = case_filter.split(r"\(")[0] + if not named.strip("^"): # it starts with its arguments: no name to go by + return False + try: + return bool(re.search(named, name)) + except re.error: + return False diff --git a/src/benchx/adapters/pyperf.py b/src/benchx/adapters/pyperf.py index e99373a..95dac9a 100644 --- a/src/benchx/adapters/pyperf.py +++ b/src/benchx/adapters/pyperf.py @@ -196,7 +196,7 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: selected = benchx_params.expand(name, info["params"], case_filter) if not selected: if benchx_params.matches_name_only(name, info["params"], case_filter): - unmatched.append(name) + unmatched.append(info) continue if "error" in info: raise Unsupported(f"cannot read the params of {name} in {where}: {info['error']}") diff --git a/tests/test_asv.py b/tests/test_asv.py index 90c8827..9ae425b 100644 --- a/tests/test_asv.py +++ b/tests/test_asv.py @@ -197,10 +197,18 @@ def test_a_filter_that_selects_nothing_is_refused(suite, capsys): assert cli.main([*ASV, "--filter", "time_plian", "--run-key", "typo", "--no-ingest", *FAST]) == cli.EXIT_REFUSED err = capsys.readouterr().err assert "refused: filter 'time_plian' selects no case in" in err - assert "available: bm.Cached.time_sum, bm.Mutating.time_pop," in err + assert "\n similar: bm.time_plain\n" in err # what the typo is close to, not the first of the alphabet + assert "available" not in err assert results(suite, "typo") == {} +def test_a_filter_that_is_like_no_name_lists_the_first_available(suite, capsys): + assert cli.main([*ASV, "--filter", "zzzzzz", "--run-key", "unlike", "--no-ingest", *FAST]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "available: bm.Cached.time_sum, bm.Mutating.time_pop," in err + assert "similar" not in err + + def test_an_anchored_name_of_a_parametrized_benchmark_is_refused_with_a_hint(suite, capsys): # asv searches `name(p0, p1)`, so `time_sorted$` matches none of the combinations. assert cli.main([*ASV, "--filter", "time_sorted$", "--run-key", "anchored", "--no-ingest", @@ -209,6 +217,16 @@ def test_an_anchored_name_of_a_parametrized_benchmark_is_refused_with_a_hint(sui in capsys.readouterr().err) +def test_arguments_in_the_wrong_order_are_refused_with_the_order_and_an_example(suite, capsys): + # `Sort` takes (n, key); `('a', 10)` names the benchmark and selects none of its combinations. + assert cli.main([*ASV, "--filter", r"time_sorted\('a', 10\)", "--run-key", "order", "--no-ingest", + *FAST]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "the filter matches the name of a parametrized benchmark (bm.Sort.time_sorted) but not its arguments" in err + assert "\n bm.Sort.time_sorted(n, key)\n for example:\n bm.Sort.time_sorted(10, 'a')\n" in err + assert results(suite, "order") == {} + + def test_a_filter_that_selects_only_benchmarks_that_are_not_run_is_refused(suite, capsys): assert cli.main([*ASV, "--filter", "peakmem_sorted", "--run-key", "notrun", "--no-ingest", *FAST]) == cli.EXIT_REFUSED @@ -419,3 +437,16 @@ def test_missing_asv_runner_is_refused(suite, monkeypatch, tmp, capsys): monkeypatch.setenv("PATH", str(bare)) assert cli.main([*ASV, "--no-ingest"]) == cli.EXIT_REFUSED assert "pyperf is not importable" in capsys.readouterr().err + + +def test_a_refusal_shows_the_filter_as_it_was_typed(suite, capsys): + """Backslashes are not doubled (not a Python repr), and the refusal is one fact per line.""" + assert cli.main([*ASV, "--filter", r"time_sorted\('a', 10\)", "--run-key", "typed", "--no-ingest", + *FAST]) == cli.EXIT_REFUSED + lines = [line for line in capsys.readouterr().err.splitlines() if line.startswith(("refused:", " "))] + assert lines[0].startswith(r"refused: filter 'time_sorted\('a', 10\)' selects no case in ") + assert [line.split(":")[0].strip() for line in lines[1:]] == [ + "the filter matches the name of a parametrized benchmark (bm.Sort.time_sorted) but not its arguments.", + "They are searched as name(p0, p1), each as its repr, in the order of param_names", + "bm.Sort.time_sorted(n, key)", "for example", "bm.Sort.time_sorted(10, 'a')", + "1 module(s) failed to import", "similar"] # the suite has a module that does not import diff --git a/tests/test_bench.py b/tests/test_bench.py index bd5663f..221fb2a 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -99,7 +99,7 @@ def ran(run_key): assert cli.main(["bench", "mod.py", "bench_sort(100, 'a')", "--run-key", "one", "--no-ingest", *FAST]) == cli.EXIT_OK assert ran("one") == ["100'a'"] assert cli.main(["bench", "mod.py", "bench_srot", "--run-key", "typo", "--no-ingest", *FAST]) == cli.EXIT_REFUSED - assert "selects no case in mod.py; available: bench_sort" in capsys.readouterr().err + assert "selects no case in mod.py\n similar: bench_sort\n" in capsys.readouterr().err def test_shorthand_order_is_stable(tree): diff --git a/tests/test_params.py b/tests/test_params.py index 0e3e791..29399b0 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -76,6 +76,17 @@ def test_a_filter_that_matches_only_the_name_is_noticed(): assert not params.matches_name_only("m.C.time_x", IDS, None) +def test_a_filter_that_gives_the_arguments_in_the_wrong_order_is_noticed(): + """The part before the first `\\(` is the benchmark's name; what follows it selected nothing.""" + assert params.expand("m.C.time_x", IDS, r"C\.time_x\('a', 10\)") == [] + assert params.matches_name_only("m.C.time_x", IDS, r"C\.time_x\('a', 10\)") + assert params.matches_name_only("m.C.time_x", IDS, r"^m\.C\.time_x\(100, 'c'\)$") + assert not params.matches_name_only("m.C.time_x", IDS, r"C\.time_y\(10, 'a'\)") # another benchmark + assert not params.matches_name_only("m.C.time_x", IDS, r"\(10, ") # no name to go by + assert not params.matches_name_only("m.C.time_x", IDS, r"^\(10, ") + assert not params.matches_name_only("m.C.time_x", IDS, r"(unbalanced\(10") # not a regex: no hint, no error + + def test_the_workload_of_a_case_comes_from_the_table_not_from_the_id(): runnable = {"cases": {"m.C.time_x-2": ("m.C.time_x", 2), "m.time_y-1": ("m.time_y", None)}, "benchmarks": {"m.C.time_x": {"params": IDS, "param_names": ["n", "key"]}, "m.time_y": {}}} @@ -83,10 +94,42 @@ def test_the_workload_of_a_case_comes_from_the_table_not_from_the_id(): assert base.parametrized_workload("m.time_y-1", runnable) == ("m.time_y", {}) # a name that looks like -1 +NAMES = ["bench_function_base.Partition.time_argpartition", "bench_function_base.Partition.time_partition", + "bench_function_base.Sort.time_sort", "bench_linalg.Linalg.time_norm", "bench_ufunc.UFunc.time_add"] + + +def test_a_slip_in_a_word_of_the_filter_finds_the_names_that_have_it(): + assert base.similar("Partiton", NAMES) == NAMES[:2] # a class, as spelled in the names + assert base.similar("time_argpartiton", NAMES) == NAMES[:2] # a method: its own first (0.97), then time_partition (0.87) + assert base.similar("bench_linalgg", NAMES) == [NAMES[3]] # a module + assert base.similar("partition", NAMES) == NAMES[:2] # the case of a letter is no difference + + +def test_the_words_of_a_regex_are_what_is_compared(): + assert base.similar(r"^bench_function_base\.Partiton\.time_x\(10, 'a'\)$", NAMES) == NAMES[:2] + assert base.similar(r"Partiton|time_ad\b", NAMES) == [NAMES[0], NAMES[1], NAMES[4]] # 0.94, 0.94, then 0.93 + assert base.similar(r"\d+\.\w", NAMES) == [] # escapes and one letter say nothing + + +def test_nothing_is_similar_to_what_is_not_like_any_name(): + assert base.similar("zzzz", NAMES) == [] + assert base.similar(None, NAMES) == [] + assert base.similar("Partiton", []) == [] + + +def test_the_most_similar_names_come_first_and_at_most_limit_are_returned(): + names = ["m.Alpha.time_a", "m.Alphb.time_a", "m.Alpxx.time_a", "m.Alpha.time_b"] + assert base.similar("Alpha", names) == ["m.Alpha.time_a", "m.Alpha.time_b", "m.Alphb.time_a"] # exact, then near + assert base.similar("Alpha", names, limit=1) == ["m.Alpha.time_a"] + + def test_the_note_names_the_benchmarks_it_means(): - note = base.name_only_note(["a", "b", "c", "d"]) + found = [{"name": name, "param_names": ["n", "key"], "params": IDS} for name in "abcd"] + note = base.name_only_note(found) assert "(a, b, c)" in note # three at most assert "name(p0, p1)" in note + # the first one's parameters, in order, and a case of it, each on a line of its own to be copied + assert note.splitlines()[2:] == [" a(n, key)", "for example:", " a(10, 'a')"] def test_search_name_is_exactly_what_expand_searches(): diff --git a/tests/test_pyperf.py b/tests/test_pyperf.py index 72d1cf1..0b65deb 100644 --- a/tests/test_pyperf.py +++ b/tests/test_pyperf.py @@ -183,10 +183,28 @@ def test_a_filter_that_matches_a_name_but_no_arguments_is_refused_with_a_hint(pa with pytest.raises(adapter.Unsupported) as refused: planned(param_tree, "bench_sort$") message = str(refused.value.problems[0]) - assert "filter 'bench_sort$' selects no case in mod.py; available: bench_plain, bench_sort;" in message + assert message.startswith("filter 'bench_sort$' selects no case in mod.py\n") + assert "\n similar: bench_sort" in message assert "the filter matches the name of a parametrized benchmark (bench_sort) but not its arguments" in message +def test_a_refusal_says_how_the_arguments_of_a_case_are_searched(param_tree): + """Arguments in the wrong order: the filter names the function, so the note gives the order + of `param_names` and a case as it is searched.""" + with pytest.raises(adapter.Unsupported) as refused: + planned(param_tree, r"bench_sort\('a', 10\)") + message = str(refused.value.problems[0]) + assert "the filter matches the name of a parametrized benchmark (bench_sort) but not its arguments" in message + assert message.splitlines()[3:6] == [" bench_sort(n, key)", " for example:", " bench_sort(10, 'a')"] + assert message.index("not its arguments") < message.index("similar:") # the hint comes before the list + + +def test_a_filter_with_no_name_in_it_gets_no_note_about_arguments(param_tree): + with pytest.raises(adapter.Unsupported) as refused: + planned(param_tree, r"\(7, ") + assert "not its arguments" not in str(refused.value.problems[0]) + + def test_params_that_cannot_be_read_matter_only_to_the_function_that_has_them(param_tree): with pytest.raises(adapter.Unsupported, match="cannot read the params of helper in mod.py: TypeError"): planned(param_tree, "helper")