diff --git a/docs/benchx.md b/docs/benchx.md index b6ad371..88d5eb4 100644 --- a/docs/benchx.md +++ b/docs/benchx.md @@ -55,7 +55,7 @@ bx compare BASELINE CONTENDER --profile revisions|environments --suite NAME [--filter REGEX] [--quantity wall-time] --rounds R [-e NAME=VALUE ...] [--repetitions N] [--min-time S] [--project P] [--label NAME=BASE,CONTENDER] - [--run-key KEY] [--out DIR] [--no-build] [--k 3] [--json] + [--run-key KEY] [--out DIR] [--no-build] [--k 3] [--json | --tables [--all]] # measure mode, one target: alternate the two values of a benchmark parameter bx compare TARGET --profile variants --adapter pyperf|asv --suite NAME --rounds R @@ -63,7 +63,7 @@ bx compare TARGET --profile variants --adapter pyperf|asv --suite NAME --rounds # read mode: compare results already measured bx compare --run KEY --profile revisions|environments|variants --baseline VALUE - [--label NAME] [--vary NAME] [--results DIR] [--k 3] [--json] + [--label NAME] [--vary NAME] [--results DIR] [--k 3] [--json | --tables [--all]] # low-level command: run a handwritten work oreder bx run [--out DIR] [--no-ingest] @@ -183,7 +183,12 @@ neither asv nor asv_runner is used. PATH is a directory holding `asv.conf.json` `benchmark_dir` is the suite) or the benchmark directory itself, and its checkout is the target (`bx bench . --adapter asv --suite benchmarks` is the same order). `--filter` is asv's `--bench`: searched in the name, and in `name(p0, p1)` for each -parameter combination; a CASE is an exact name, `name(p0, p1)` for a combination. `adapters/asv_runtime.py` discovers +parameter combination, the arguments in the order of `param_names`, each as its `repr`; a CASE is an exact name, +`name(p0, p1)` for a combination. A filter that names a parametrized benchmark and none of its combinations is +refused with that order and one case as it is searched. The runner records the order in +`provenance.info.parameter_names` (the store keeps a document with its keys sorted), and `bx bench`, `bx head --table` +and `bx compare` show the parameters in it, so what they print can be given to `--filter`; results made before have +no such list and show them sorted. `adapters/asv_runtime.py` discovers the benchmarks as asv does (names, `params`, `setup`/`teardown`/`setup_cache`, skips, and asv's `version` hash all agree with asv_runner's; a test checks it). The benchmarks run in the `python` first on `PATH`, which must import pyperf and the @@ -263,6 +268,15 @@ every result is pyperf. Results also record the installed project's version (`side-failed`). `--results` compares result files through a throwaway store: the ad hoc path. Condition measurements (`measurement.kind: condition`) are not compared, and deterministic quantities are deferred. +- **The tables** (`--tables`, `benchx/tables.py`) print, after the usual summary, what + `asv continuous` prints, laid out and rounded as asv does: a table of each side (a value + `4.71±0.08μs` per parameter combination, every cell centered), then one of the changes + (Change, Before, After, Ratio, Noise, Verdict, Benchmark), only the changed rows unless `--all`. + They are recomputed for display from the run's results; the comparison document is the + same. A side's value is the median of its per-round medians and its `±` their standard + deviation (not asv's `±`); `Change` and `Verdict` are the comparator's verdict (`+` + regressed, `-` improved), `Noise` the one it compares the effect with (a change is more than *k* + times it), and `Ratio` is After over Before. A text view: not with `--json`. ## Run it diff --git a/docs/design/comparator.md b/docs/design/comparator.md index 9da057c..a52783f 100644 --- a/docs/design/comparator.md +++ b/docs/design/comparator.md @@ -136,11 +136,15 @@ side are equal, and sides alternate by `procedure.slot`; under `revisions`, that each side has exactly one tree id and the two differ; under `revisions` and `variants`, that a separately recorded benchmark has one `benchmark_tree` across both sides and all rounds (`one-benchmark-tree`, schema §5.5). A side that is not clean does -not fail the run; it makes the comparison local-only (UC-03 §10). A run that -fails an invariant is not compared: the comparison document records the failed invariant -and contains no verdicts for the units it affects. - -Excluded points and their reasons remain visible in the output. Changes in +not fail the run; it makes the comparison local-only (UC-03 §10). An invariant of the +run (the sides, their tree ids, the alternation, the benchmark file) that fails leaves no unit compared. One of a +single unit (the matched facts agree, attempt counts are equal) takes that unit out and the others are compared: the +document records the failed invariant with its workload, and has no verdict for that unit. The matched facts are +those of the attempts that succeeded; a failed attempt reports none of them, and is excluded and listed. + +Excluded points and their reasons remain visible in the output: the reason of an excluded result ends with what the +attempt said when it said something (a failed pyperf case's exception line, an asv benchmark's skip reason), and +the rendered listing is one line per reason with a count. Changes in facts the projection ignores do not split a series by themselves, but differences between the target and its reference are carried as caveats so that a kernel, load, or other recorded condition is not hidden from the reader. diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index 04e267b..ab6590e 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -21,7 +21,7 @@ bx series [--workload … --quantity …] # list series bx history # one series' points (a query) bx head [-n 10] [--project P] [--workload W] [--json|--table] # the latest results, newest first bx compare --run KEY --profile revisions|environments|variants --baseline VALUE - [--label NAME] [--vary NAME] [--results DIR] [--k 3] [--min-rounds 3] [--json] + [--label NAME] [--vary NAME] [--results DIR] [--k 3] [--min-rounds 3] [--json | --tables [--all]] # read mode: compare results already measured bx target describe|prepare TARGET # a target's description (a provider's answer, or a build directory's sidecar) bx compare BASELINE CONTENDER --profile revisions|environments @@ -33,7 +33,7 @@ bx compare BASELINE CONTENDER --profile revisions|environments [--min-time S] [--project P | --no-project] [--label NAME=BASE,CONTENDER] [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI] - [--k 3] [--min-rounds 3] [--json] + [--k 3] [--min-rounds 3] [--json | --tables [--all]] # measure mode: run both sides, then compare bx compare TARGET --profile variants [--vary NAME] [--baseline VALUE] --suite NAME --rounds R [--filter REGEX] [...] # the options above, except --label @@ -270,7 +270,10 @@ criterion 1 holds by construction. `--json` emits this; the default output renders it as the reviewer table (regressed, improved, no change detected, and indeterminate sections, then -the missing, failed-invariant, and excluded listings with counts). In any +the missing, failed-invariant, and excluded listings with counts, in words: a failed invariant names the unit +it affects, units that failed alike are one line, and results excluded for the same reason are one line with +their number, the reason carrying what the failed attempt said, such as `error: harness.error: ValueError: …`; +which results they are is in `--json`). In any profile, a comparison in which either side is not `clean` has `local_only: true` (schema §5.5), is headed `[local-only]`, and names the sides that are not clean on the `caveats:` line; the `revisions` profile fails a side with no @@ -514,7 +517,8 @@ matched facts agree (`sides-agree`) and the invariants the profile names over the run: exactly two sides, which for `environments` differ in more than their label, equal attempt counts per side, and sides alternating by `procedure.slot`. Results the order did not ask for are not compared. A failed invariant is recorded and the affected units get no -verdict; under `revisions` the same happens to a side with no tree id (the +verdict (`sides-agree` and `equal-attempts` affect one unit, and the others are still compared; the rest +affect every unit; `sides-agree` looks at the attempts that succeeded); under `revisions` the same happens to a side with no tree id (the `sides` invariant), to a tree edited during the run (a third side: `two-sides`), and to a benchmark recorded apart from the subject that is not one file on both sides and in every round (`one-benchmark-tree`, schema §5.5), as it does diff --git a/src/benchx/adapters/asv.py b/src/benchx/adapters/asv.py index 248b594..289477a 100644 --- a/src/benchx/adapters/asv.py +++ b/src/benchx/adapters/asv.py @@ -199,7 +199,7 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: for case, index in benchx_params.expand(info["name"], info["params"], case_filter)] if not selected: if benchx_params.matches_name_only(info["name"], info["params"], case_filter): - unmatched.append(info["name"]) + unmatched.append(info) continue if info["type"] != "time": other[info["type"]] = other.get(info["type"], 0) + 1 diff --git a/src/benchx/adapters/base.py b/src/benchx/adapters/base.py index 6239708..ed957eb 100644 --- a/src/benchx/adapters/base.py +++ b/src/benchx/adapters/base.py @@ -67,12 +67,15 @@ attempt. For a harness that repeats at several levels. """ +import difflib import os +import re import signal import subprocess +import textwrap from ..core import Refusal, Rejected -from .benchx_params import combination +from .benchx_params import combination, search_name class Unsupported(Rejected): @@ -97,25 +100,55 @@ def parametrized_workload(case, runnable) -> tuple[str, dict]: return name, dict(zip(info["param_names"], combination(info["params"], index))) -def name_only_note(names) -> str: - """For a refusal: the filter matched parametrized benchmarks by name, but not their - arguments (see `benchx_params.matches_name_only`).""" - return (f"the filter matches the name of a parametrized benchmark ({', '.join(names[:3])}) but " - "not its arguments, and those are matched as name(p0, p1)") - - -def no_cases(where, case_filter, available=(), notes=(), limit=10) -> Unsupported: +def name_only_note(found) -> str: + """For a refusal: the filter is about parametrized benchmarks (`found`, their listing entries) + but matches none of their arguments (see `benchx_params.matches_name_only`). Says how a case + is searched, with the first benchmark's parameters, in order, and one case as it is searched.""" + first = found[0] + names = ", ".join(info["name"] for info in found[:3]) + return "\n".join([ + f"the filter matches the name of a parametrized benchmark ({names}) but not its arguments.", + "They are searched as name(p0, p1), each as its repr, in the order of param_names:", + f" {first['name']}({', '.join(first['param_names'])})", + "for example:", + f" {search_name(first['name'], first['params'], 0)}"]) + + +def similar(case_filter, names, limit=5, cutoff=0.75) -> list[str]: + """The names a filter may have meant: those that have, as a dotted part (module, class, method), the + most of its words, or words close to them, then the closest first. A word is a run of letters, digits + and `_` of three characters or more, so an escape such as `\\d` says nothing, and the case of a letter + is not a difference. A filter that names a module and misspells its class thus finds that class's + benchmarks, and not the whole module.""" + words = [w.lower() for w in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", case_filter or "") if len(w) >= 3] + scored = [] + for name in names: + parts = name.lower().split(".") + best = (max(difflib.SequenceMatcher(None, word, part).ratio() for part in parts) for word in words) + hits = [score for score in best if score >= cutoff] + if hits: + scored.append((-len(hits), -sum(hits), name)) + scored.sort() + return [name for count, _, name in scored[:limit] if count == scored[0][0]] + + +def no_cases(where, case_filter, available=(), notes=(), limit=5) -> Unsupported: """The refusal for a plan with no case in it. Nothing would run, and a filter with a typo must not end in `0 result(s)` and a success status. `available` are the names that could have been selected (the first `limit` are shown) and `notes` say what else - the adapter knows about why the plan is empty.""" - head = f"filter {case_filter!r} selects no case in {where}" if case_filter else f"no case to run in {where}" - parts = [head] + the adapter knows about why the plan is empty, each on its own lines (a note may have several). + The last line is the names that are similar to the filter (`similar`), else the first available. + The filter is shown as given, not as a Python string, so its backslashes are the ones typed.""" + head = f"filter '{case_filter}' selects no case in {where}" if case_filter else f"no case to run in {where}" + lines = [head] + [textwrap.indent(note, " ") for note in notes] names = sorted(available) - if names: + close = similar(case_filter, names, limit) + if close: + lines.append(f" similar: {', '.join(close)}") + elif names: more = f", … ({len(names) - limit} more)" if len(names) > limit else "" - parts.append(f"available: {', '.join(names[:limit])}{more}") - return Unsupported("; ".join(parts + list(notes))) + lines.append(f" available: {', '.join(names[:limit])}{more}") + return Unsupported("\n".join(lines)) def run_bounded(args, *, timeout, what, **kwargs) -> subprocess.CompletedProcess: diff --git a/src/benchx/adapters/benchx_params.py b/src/benchx/adapters/benchx_params.py index bc27cd5..247cb3d 100644 --- a/src/benchx/adapters/benchx_params.py +++ b/src/benchx/adapters/benchx_params.py @@ -84,6 +84,16 @@ def expand(name, ids, case_filter): def matches_name_only(name, ids, case_filter) -> bool: - """Whether a filter that selected none of a benchmark's cases matches its name alone: - a parametrized benchmark is searched in `name(p0, p1)`, so `name$` is a likely slip.""" - return bool(ids and case_filter and re.search(case_filter, name)) + """Whether a filter that selected none of a benchmark's cases is about this benchmark, though + not about any of its combinations: it matches the name alone (`name$` is a likely slip), or the + part of it before its first `\\(` does, so what it gives after the name, the arguments, is + what fails (in the wrong order, or a value the benchmark does not have).""" + if not (ids and case_filter): + return False + named = case_filter.split(r"\(")[0] + if not named.strip("^"): # it starts with its arguments: no name to go by + return False + try: + return bool(re.search(named, name)) + except re.error: + return False diff --git a/src/benchx/adapters/pyperf.py b/src/benchx/adapters/pyperf.py index e99373a..c41c82e 100644 --- a/src/benchx/adapters/pyperf.py +++ b/src/benchx/adapters/pyperf.py @@ -196,7 +196,7 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: selected = benchx_params.expand(name, info["params"], case_filter) if not selected: if benchx_params.matches_name_only(name, info["params"], case_filter): - unmatched.append(name) + unmatched.append(info) continue if "error" in info: raise Unsupported(f"cannot read the params of {name} in {where}: {info['error']}") @@ -214,6 +214,8 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: workload = parametrized_workload # the function, and its parameters as value strings +# The combination index is bound to a name before it is tested: `0 is not None` is a SyntaxWarning, which +# Python prints on the stderr of every worker. DRIVER = '''\ # Generated by benchx. Runs one function under pyperf: one case, which for a function # that declares params is one combination of them, passed as its arguments. @@ -247,10 +249,11 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: sys.modules["benchx_suite"] = module spec.loader.exec_module(module) func = getattr(module, {function!r}) +index = {index!r} args = () -if {index!r} is not None: +if index is not None: values, _, _ = params.normalize(getattr(func, "params", []), getattr(func, "param_names", [])) - args = params.combination(values, {index!r}) + args = params.combination(values, index) runner.bench_func({case!r}, func, *args) ''' @@ -305,12 +308,22 @@ def run_case(runnable, case, flags, env, timeout, native_path, env_names=None) - def _failure_message(stderr: str) -> str: - """The worker's exception line. The manager then raises its own - " failed with exit code N" after a second traceback; that wrapper - only counts when nothing else was said.""" - lines = [l for l in stderr.splitlines() if l and not l[0].isspace() and not l.startswith("Traceback")] - own = [l for l in lines if "failed with exit code" not in l] - return (own or lines or [stderr.strip().splitlines()[-1]])[0] + """The worker's exception line: the line that ends a traceback (the first unindented line after + "Traceback"), the last one when exceptions were chained. A warning printed before is not it, and + neither is the manager's own " failed with exit code N" after a second traceback: that wrapper + only counts when nothing else was said. Without a traceback (a killed process), the first line said.""" + lines = [line for line in stderr.splitlines() if line.strip()] + ended, in_traceback = [], False + for line in lines: + if line.startswith("Traceback"): + in_traceback = True + elif in_traceback and not line[0].isspace(): + ended.append(line) + in_traceback = False + own = [line for line in ended if "failed with exit code" not in line] + if own or ended: + return (own or ended)[-1] + return next((line for line in lines if not line[0].isspace()), lines[-1] if lines else stderr.strip()) def attempted(applied: dict) -> int: diff --git a/src/benchx/bench.py b/src/benchx/bench.py index c54c4fd..d4c1c79 100644 --- a/src/benchx/bench.py +++ b/src/benchx/bench.py @@ -14,7 +14,7 @@ from . import core, snapshot from .errors import SessionError -from .projection import workload_label +from .projection import declared_workload, said, workload_label from .store import warning_codes @@ -77,7 +77,7 @@ def row_of(document: dict) -> dict: # An observation is a number, or an object carrying one with its metadata (pyperf's). values = [o["value"] if isinstance(o, dict) else o for o in measurement.get("observations", [])] return { - "case": workload_label(document["coordinates"]["workload"]), + "case": workload_label(declared_workload(document)), "status": measurement["status"] + (f" ({measurement['reason']})" if measurement.get("reason") else ""), "n": len(values), "median": statistics.median(values) if values else None, @@ -87,6 +87,7 @@ def row_of(document: dict) -> dict: "max": max(values) if values else None, "loops": document.get("procedure", {}).get("inner_iterations"), "warnings": warning_codes(document), + "message": None if measurement["status"] == "success" else said(document), } @@ -110,3 +111,18 @@ def render(rows, leading=()) -> str: widths = [max(len(row[i]) for row in [header, *body]) for i in range(len(header))] lines = [" ".join(cell.ljust(w) for cell, w in zip(row, widths)).rstrip() for row in [header, *body]] return "\n".join([lines[0], " ".join("-" * w for w in widths), *lines[1:]]) + + +def failures(rows, names=3) -> str: + """The section under the table for the cases that were not measured: what each said, `case: message`, + cases with the same message as one line, `message (K cases: a, b, c, …)`. Empty when none has a message + (the status column is then all there is). A row's message is one line already (`projection.said`).""" + said_by = {} + for row in rows: + if row.get("message"): + said_by.setdefault(row["message"], []).append(row["case"]) + if not said_by: + return "" + lines = [f" {message} ({len(cases)} cases: {', '.join(cases[:names])}{', …' if len(cases) > names else ''})" + if len(cases) > 1 else f" {cases[0]}: {message}" for message, cases in said_by.items()] + return "\n".join(["", f"not measured ({sum(len(c) for c in said_by.values())})", *lines]) diff --git a/src/benchx/cli.py b/src/benchx/cli.py index a651d9f..12fd56b 100644 --- a/src/benchx/cli.py +++ b/src/benchx/cli.py @@ -18,6 +18,7 @@ from . import compare as compare_mod from . import config as config_mod from . import core, harnesses, orders, runner, session, snapshot +from . import tables as tables_mod from . import target as target_mod from .core.validation import Kind from .store import Store, warning_codes @@ -126,12 +127,14 @@ def _common(cfg, args, *, forced_adapter=None, forced_suite=None) -> dict: """The settings `bx bench` and `bx compare` share, each from the command line, else the file, else its default (config.Config.pick); ConfigError when a required one has none. The file's adapter block belongs to the adapter it - names, so naming another is refused.""" + names and to the others of its family (pyperf's serves asv), so naming an + adapter of another family is refused.""" block = cfg.adapter given = forced_adapter or args.adapter - if given and block.get("name") and given != block["name"]: + if given and block.get("name") and not harnesses.same_family(given, block["name"]): raise config_mod.ConfigError(f"adapter {given!r}, but {cfg.where} sets adapter.name to {block['name']!r}, " - "and its suite and precision belong to that harness") + "and its precision belongs to that harness and the ones that share it " + "(pyperf and asv)") quantities = config_mod.parse_quantities(args.quantity) if args.quantity else block.get("quantities", ["wall-time"]) if args.project and args.no_project: raise UsageError("give --project or --no-project, not both") @@ -198,7 +201,10 @@ def cmd_bench(args): return EXIT_REFUSED except runner.Refused as e: return _refused(e) - print(bench_mod.render(bench_mod.summarize(summary["files"]))) + rows = bench_mod.summarize(summary["files"]) + print(bench_mod.render(rows)) + if failed := bench_mod.failures(rows): + print(failed) print(f"\n{summary['results']} result(s) -> {summary['out']} (order {summary['order'][:19]})") if store is None: return EXIT_OK @@ -385,6 +391,7 @@ def cmd_compare(args): policy = _policy(cfg, args) mode = _mode(args, policy["profile"]) _profile_flags(args, policy["profile"]) + _view_flags(args) except (UsageError, config_mod.ConfigError) as e: print(f"bx compare: error: {e}", file=sys.stderr) return EXIT_REFUSED @@ -412,6 +419,14 @@ def _profile_flags(args, profile): "comparison are two values of a parameter (--vary)") +def _view_flags(args): + """--tables is a text view of the comparison, and --all a switch of it.""" + if args.tables and args.json: + raise UsageError("--tables is a text view: not with --json") + if args.all and not args.tables: + raise UsageError("--all goes with --tables") + + def _read(cfg, args, policy): if args.results: with tempfile.TemporaryDirectory() as tmp: # the ad hoc path: a throwaway store @@ -449,10 +464,20 @@ def _measure(cfg, args, policy): except (session.SessionError, compare_mod.CompareError) as e: print(f"cannot compare: {e}", file=sys.stderr) return EXIT_REFUSED - print(json.dumps(doc, indent=1) if args.json else compare_mod.render(doc)) + _show(doc, args, store.documents(doc["run_key"]) if args.tables else None) return EXIT_OK +def _show(doc, args, documents): + """The comparison as JSON, or as text; with --tables, the asv-style tables follow the summary.""" + if args.json: + print(json.dumps(doc, indent=1)) + return + print(compare_mod.render(doc)) + if args.tables: + print(tables_mod.render(doc, documents, all_rows=args.all)) + + def _compare(store, args, policy): try: documents = store.documents(args.run) @@ -467,7 +492,7 @@ def _compare(store, args, policy): except compare_mod.CompareError as e: print(f"cannot compare: {e}", file=sys.stderr) return EXIT_REFUSED - print(json.dumps(doc, indent=1) if args.json else compare_mod.render(doc)) + _show(doc, args, documents) return EXIT_OK @@ -625,6 +650,10 @@ def main(argv=None): p.add_argument("--k", type=float, help="noise multiplier (default: compare.k, else 3)") p.add_argument("--min-rounds", type=int, help="fewest paired rounds for a verdict (default: compare.min_rounds, else 3)") p.add_argument("--json", action="store_true") + p.add_argument("--tables", action="store_true", + help="after the summary, asv-style tables: one per side, then the changes") + p.add_argument("--all", action="store_true", + help="with --tables: every row of the change table, not only the changed ones (as asv continuous)") p.add_argument("--label", help="read mode: the label that names the sides (environments); " "measure mode: NAME=BASE,CONTENDER") p.add_argument("--baseline", help="revisions: a revision or tree id prefix; environments: the label value; " diff --git a/src/benchx/compare.py b/src/benchx/compare.py index 39b2b98..92e64c7 100644 --- a/src/benchx/compare.py +++ b/src/benchx/compare.py @@ -16,7 +16,7 @@ from . import identity from .core import canonical, catalog -from .projection import Projection, axis_order, get, workload_label +from .projection import Projection, axis_order, declared_workload, get, said, workload_label from .snapshot import DIRTY_WARNINGS from .store import store_project @@ -224,26 +224,33 @@ def compare(documents, *, run_key, profile, baseline, label=None, parameter=None "detail": "sides do not alternate by procedure.slot: run the sides in the same " "order every round (a, b, a, b, ...)" + how}) - # Units: one workload variant and quantity each. Everything else the - # projection matches must agree across both sides (schema §5.5). + # Units: one workload variant and quantity each. Everything else the projection matches must agree + # across both sides (schema §5.5), in the attempts that succeeded: a failed attempt reports none of + # the facts (runtime, harness version) and is excluded, and listed, below. units = {} for d, p, s in zip(documents, projected, sides): variant = {ptr: v for ptr, v in p.identity.items() if ptr == WORKLOAD or ptr.startswith(WORKLOAD + "/")} key = (_c(variant), d["measurement"]["quantity"]["name"]) units.setdefault(key, {"docs": {base: [], contender: []}, "identities": []}) units[key]["docs"][s].append(d) - units[key]["identities"].append(p.identity) - for (_, quantity), unit in units.items(): + if d["measurement"]["status"] == "success": + units[key]["identities"].append(p.identity) + run_failed = bool(failed) # the invariants above are about the run, and no unit is compared + disagreeing = set() # `sides-agree` is about one unit: it gets no verdict, and the others are compared + for key, unit in units.items(): differing = _differing(unit["identities"]) if differing: failed.append({"invariant": "sides-agree", "workload": _workload(unit["docs"], projection.axis), - "quantity": quantity, "detail": f"{', '.join(differing)} differ"}) - if failed: + "quantity": key[1], "detail": f"{', '.join(differing)} differ"}) + disagreeing.add(key) + if run_failed: return doc - for (_, quantity), unit in sorted( + for (variant_key, quantity), unit in sorted( units.items(), key=lambda item: _unit_order(_workload(item[1]["docs"], projection.axis), item[0][1])): + if (variant_key, quantity) in disagreeing: + continue by_side = unit["docs"] where = {"workload": _workload(by_side, projection.axis), "quantity": quantity} absent = [role for role, s in (("baseline", base), ("contender", contender)) if not by_side[s]] @@ -261,14 +268,16 @@ def compare(documents, *, run_key, profile, baseline, label=None, parameter=None if d["measurement"]["status"] != "success": status, reason = d["measurement"]["status"], d["measurement"].get("reason") result = {"producer": d["producer"]["name"], "ingest_key": d["ingest_key"]} - doc["exclusions"].append({"result": result, "reason": f"{status}: {reason}" if reason else status}) + why = f"{status}: {reason}" if reason else status + doc["exclusions"].append({"result": result, "reason": f"{why}: {said(d)}" if said(d) else why}) side_failed = side_failed or (profile in STRICT_STATUS and status in FAILED_STATUSES) continue rounds.setdefault(d.get("procedure", {}).get("round"), {})[s] = d pairs = [(r[base], r[contender]) for _, r in sorted(rounds.items(), key=lambda x: (x[0] is None, x[0])) if base in r and contender in r] unit_doc = _unit(where, pairs, by_side, base, contender, k, min_rounds, side_failed) - unit_doc["unmatched_differences"] = _differences(projection, by_side[base], by_side[contender]) + succeeded = {s: [d for d in by_side[s] if d["measurement"]["status"] == "success"] for s in (base, contender)} + unit_doc["unmatched_differences"] = _differences(projection, succeeded[base], succeeded[contender]) if caveats := _caveats(by_side, base, contender): unit_doc["caveats"] = caveats doc["units"].append(unit_doc) @@ -314,17 +323,19 @@ def _benchmark_file(documents, names, sides) -> list[dict]: def _workload(by_side, axis=()) -> dict: """The unit's workload variant, as the comparison document names it: what both sides share, so the parameters on the profile's axis (the ones that tell the sides apart) are left out.""" - workload = next(d for docs in by_side.values() for d in docs)["coordinates"]["workload"] + workload = declared_workload(next(d for docs in by_side.values() for d in docs)) varied = {path.removeprefix(PARAMETERS) for path in axis if path.startswith(PARAMETERS)} parameters = {k: v for k, v in (workload.get("parameters") or {}).items() if k not in varied} return {"name": workload["name"], **({"parameters": parameters} if parameters else {})} def _unit_order(workload: dict, quantity: str) -> tuple: - """The order units are listed in: by workload name, then by parameter name and value, then by - quantity, with numbers inside names and values in natural order (`axis_order`), so that `n=50` - comes before `n=100` and `bench_2` before `bench_10`, as people read them.""" - parameters = sorted((workload.get("parameters") or {}).items()) + """The order units are listed in: by workload name, then by the values of its parameters, the first + declared changing slowest (the order of the workload's parameters, see `declared_workload`: sorted by + name for a result that recorded none), then by quantity, with numbers inside names and values in + natural order (`axis_order`), so that `n=50` comes before `n=100` and `bench_2` before `bench_10`, as + people read them.""" + parameters = list((workload.get("parameters") or {}).items()) return (axis_order(workload["name"]), tuple((name, axis_order(value)) for name, value in parameters), quantity) @@ -428,6 +439,45 @@ def seen(found, pointer): return [p for p in _differing(a + b) if seen(a, p) != seen(b, p)] +def _few(names, shown=3, count=False) -> str: + """The first `shown` names; `, …` when there are more, and with `count` how many.""" + more = len(names) - shown + return ", ".join(names[:shown]) + ((f", … (+{more} more)" if count else ", …") if more > 0 else "") + + +def _failed(items) -> list[str]: + """Failed invariants in words, `invariant: detail`: the unit it affects first when it affects one, and + units that failed alike as one line, `(K units: a, b, c, …)`.""" + groups = {} + for item in items: + label = workload_label(item["workload"]) if "workload" in item else None + groups.setdefault((item["invariant"], item["detail"]), []).append(label) + lines = [] + for (invariant, detail), labels in groups.items(): + units = list(dict.fromkeys(label for label in labels if label)) + text = f"{invariant}: {detail}" + lines.append(f"{units[0]}: {text}" if len(units) == 1 else + f"{text} ({len(units)} units: {_few(units)})" if units else text) + return lines + + +def _excluded(items) -> list[str]: + """Results left out, one line for each reason (which has what the result said): `reason (K results)`. + Which results is in the document (`--json`), not here.""" + counts = {} + for item in items: + counts[item["reason"]] = counts.get(item["reason"], 0) + 1 + return [f"{reason} ({n} result{'s' if n != 1 else ''})" for reason, n in counts.items()] + + +def _missing(items) -> list[str]: + """Units a side has no result for, one line for each side: `not on contender: a, b, c, … (+N more)`.""" + sides = {} + for item in items: + sides.setdefault(item["side"], []).append(workload_label(item["workload"])) + return [f"not on {side}: {_few(list(dict.fromkeys(labels)), count=True)}" for side, labels in sorted(sides.items())] + + def render(doc: dict) -> str: """The reviewer-speed table (comparator.md §5).""" lines = [f"run {doc['run_key']} profile {doc['profile']} " @@ -458,9 +508,9 @@ def render(doc: dict) -> str: reason = f" ({u['reason']})" if "reason" in u else "" lines.append(f" {workload_label(u['workload']):<32} {u['quantity']:<10} effect {effect} noise {noise}" f" rounds={u['rounds']}{reason}") - for key, title in (("missing", "missing"), ("failed_invariants", "failed invariants"), - ("exclusions", "excluded")): - if doc[key]: - lines.append(f"\n{title} ({len(doc[key])})") - lines.extend(f" {item}" for item in doc[key]) + for title, items, say in (("missing", doc["missing"], _missing), ("failed invariants", doc["failed_invariants"], _failed), + ("excluded", doc["exclusions"], _excluded)): + if items: + lines.append(f"\n{title} ({len(items)})") + lines.extend(f" {line}" for line in say(items)) return "\n".join(lines) diff --git a/src/benchx/harnesses.py b/src/benchx/harnesses.py index 4fa05ca..d8f8a04 100644 --- a/src/benchx/harnesses.py +++ b/src/benchx/harnesses.py @@ -43,6 +43,7 @@ def exact_filter(names, arguments=False) -> str | None: class Harness: name: str + FAMILY: str # harnesses of one family take the same precision, so one `adapter` block of the config serves all FLAGS: tuple[str, ...] = () BARE_PATH_KIND: str NAMES_TAKE_ARGUMENTS = False @@ -83,6 +84,7 @@ class GoogleBenchmark(Harness): Google Benchmark's default, and the order states it.""" name = "google-benchmark" + FAMILY = "google-benchmark" FLAGS = ("repetitions", "min_time") BARE_PATH_KIND = "build" # the binary is built; BUILD_DIR[:SOURCE_DIR] DEFAULT_REPETITIONS = adapters.gbench.DEFAULT_REPETITIONS @@ -105,6 +107,7 @@ class Pyperf(Harness): flags leave out.""" name = "pyperf" + FAMILY = "pyperf" # asv, timed by pyperf, inherits it FLAGS = ("processes", "values", "min_time", "loops", "warmups") BARE_PATH_KIND = "working_tree" # the module is run as found; the path is its own source NAMES_TAKE_ARGUMENTS = True # a function with params is one case per combination @@ -150,6 +153,13 @@ def describe_precision(self, precision, config_path=None): FLAG_NAMES = tuple(dict.fromkeys(name for h in HARNESSES.values() for name in h.FLAGS)) +def same_family(a: str, b: str) -> bool: + """Whether harnesses `a` and `b` take the same precision, as pyperf and asv do (asv is timed by + pyperf). An unknown name is in no family.""" + first, second = HARNESSES.get(a), HARNESSES.get(b) + return first is not None and second is not None and first.FAMILY == second.FAMILY + + def get(name: str) -> Harness: """The harness registered under `name`; SessionError if there is none.""" try: diff --git a/src/benchx/projection.py b/src/benchx/projection.py index 5995d1c..1a18f9d 100644 --- a/src/benchx/projection.py +++ b/src/benchx/projection.py @@ -252,12 +252,37 @@ def axis_order(value) -> tuple: return (3, canonical(value).decode()) +def declared_workload(document: dict) -> dict: + """The document's workload with its parameters in the order the benchmark declared them. The store keeps + a document in canonical form, with its keys sorted, so the runner records the order as data + (`provenance.info.parameter_names`): the names it lists first, then any it does not name, as stored. A + result without the list (made before, or by another producer) is as stored.""" + workload = document["coordinates"]["workload"] + parameters = workload.get("parameters") + names = ((document.get("provenance") or {}).get("info") or {}).get("parameter_names") + if not parameters or not names: + return workload + ordered = {name: parameters[name] for name in names if name in parameters} + return {**workload, "parameters": {**ordered, **{k: v for k, v in parameters.items() if k not in ordered}}} + + def workload_label(workload: dict) -> str: """A workload as people read it: its name, then its parameters, `name [k=v, ...]`.""" parameters = ", ".join(f"{k}={v}" for k, v in (workload.get("parameters") or {}).items()) return workload["name"] + (f" [{parameters}]" if parameters else "") +def said(document: dict, width: int = 160) -> str | None: + """What the producer said about a result that is not a measurement (`provenance.info.message`): for a + failed pyperf case the exception line, for a skipped asv benchmark the reason. Its first line, cut at + `width` characters; None when it said nothing.""" + message = str(((document.get("provenance") or {}).get("info") or {}).get("message") or "").strip() + line = message.splitlines()[0] if message else "" + if not line: + return None + return line if len(line) <= width else line[:width - 1] + "…" + + def reported_coordinates(doc: dict) -> dict: """Everything reported about what was measured, minus the axis revision and the quantity (§4.3).""" return {key: doc[key] for key in REPORTED if key in doc} diff --git a/src/benchx/runner.py b/src/benchx/runner.py index 95507d5..0a0aade 100644 --- a/src/benchx/runner.py +++ b/src/benchx/runner.py @@ -273,6 +273,8 @@ def run(order_path, out_dir, allow_collectors=False) -> dict: workload = {"name": workload_name} if case_parameters: workload["parameters"] = case_parameters + if len(case_parameters) > 1: # the store sorts keys: the order is data (projection.declared_workload) + info["parameter_names"] = list(case_parameters) subject = {"name": _subject_name(order), "components": [{"role": "primary"}]} if configuration: subject["configuration"] = configuration diff --git a/src/benchx/schemas/config/0.1.0/schema.json b/src/benchx/schemas/config/0.1.0/schema.json index f9f5007..8918112 100644 --- a/src/benchx/schemas/config/0.1.0/schema.json +++ b/src/benchx/schemas/config/0.1.0/schema.json @@ -36,7 +36,7 @@ "additionalProperties": false, "required": ["name"], "properties": { - "name": { "description": "pyperf or google-benchmark.", "type": "string", "minLength": 1 }, + "name": { "description": "pyperf, asv or google-benchmark. The block's precision belongs to this adapter and to the others that take the same precision: a pyperf block also serves `--adapter asv`, which is timed by pyperf.", "type": "string", "minLength": 1 }, "suite": { "description": "The benchmark program in the target: pyperf, the module's path in the working tree; google-benchmark, the binary in the build directory.", "type": "string", diff --git a/src/benchx/tables.py b/src/benchx/tables.py new file mode 100644 index 0000000..d74effb --- /dev/null +++ b/src/benchx/tables.py @@ -0,0 +1,210 @@ +"""asv-style tables for `bx compare --tables` (TABLES_PLAN.md): one table per side, then the changes. + +Display only. The per-side numbers are recomputed here from the run's stored results, with the +projection that made the comparison document, so the document and its schema are unchanged. A +side's value is the median of its per-round medians (the estimator the verdict uses) and its `±` +the sd of those medians: benchx's own meaning, not asv's. +""" + +import math +import statistics + +from .compare import WORKLOAD, _c, _estimate, _run_documents, _unit_order, _workload, profile_projection +from .snapshot import DIRTY_WARNINGS +from .store import store_project + +ROLES = ("baseline", "contender") +CHANGED = {"regressed": "+", "improved": "-", "changed": "~"} # as asv continuous marks them +MARKS = {**CHANGED, "no-change-detected": " ", "indeterminate": "?"} +SHA = 8 # asv's width +# asv_runner.util._human_time_units: the unit is the largest one the value does not reach. +TIME_UNITS = (("ns", 1e-9), ("μs", 1e-6), ("ms", 1e-3), ("s", 1), ("m", 60), ("h", 3600), ("d", 86400), + ("w", 604800), ("M", 2620800), ("q", 7862400), ("y", 31449600), ("C", 3144960000)) + + +def _float(value: float, significant: int, truncate_small: int | None = None, zeros: bool = False) -> str: + """`value` to `significant` digits, as asv's `human_float`: scientific notation outside 1e-5 and 1e9; + trailing zeros dropped unless `zeros`, which also pads `3` to `3.00`; with `truncate_small`, the + zeros after the point of a number below 1 count as digits, up to that many.""" + if value == 0: + return "0" + if math.isinf(value) or math.isnan(value): + return str(value) + sign, value = ("-", -value) if value < 0 else ("", value) + magnitude = math.floor(math.log10(value)) + 1 + if truncate_small is not None: + magnitude = max(magnitude, 1 - truncate_small) + decimals = significant - magnitude + scientific = magnitude <= -5 or magnitude >= 9 + if scientific: + text = f"{value:.{significant}e}" + elif value == int(value) or decimals <= 0: + text = str(int(round(value, decimals))) + else: + text = f"{value:.{decimals}f}" + if not zeros and "." in text and not scientific: + text = text.rstrip("0").removesuffix(".") + text = sign + text + if zeros and "." not in text and len(text) < significant: + text += "." + "0" * (significant - len(text)) + return text + + +def human(value, spread, unit) -> str: + """`4.71±0.08μs`, as asv writes a time (`human_time`): the value to 3 digits with its zeros, the + spread to 1. Any other unit is written after the number.""" + if value is None or value != value: + return "n/a" + if unit != "s": + return _float(value, 3) + (f"±{_float(spread, 1, 2)}" if spread is not None else "") + f" {unit}" + scale = spread if value == 0 and spread is not None else value + units = TIME_UNITS if scale != 0 else (("s", 1), ("m", 60)) + for (name, size), (_, bigger) in zip(units, units[1:]): + if scale < bigger: + return (_float(value / size, 3, zeros=True) + + (f"±{_float(spread / size, 1, truncate_small=2)}" if spread is not None else "") + name) + return "~0" + + +def _projection(doc): + """The projection that made `doc`: its `varying` names the label or parameter that names the sides.""" + varying = doc["varying"] + label = varying.removeprefix("provenance.labels.") if varying.startswith("provenance.labels.") else None + parameter = (varying.removeprefix("coordinates.workload.parameters.") + if varying.startswith("coordinates.workload.parameters.") else None) + return profile_projection(doc["profile"], label, parameter) + + +def _side(docs) -> dict: + """One side of one unit: the median of its per-round medians, their sd, and what went wrong if + no round succeeded.""" + medians = [m for d in docs if d["measurement"]["status"] == "success" and (m := _estimate(d)) is not None] + failed = sorted({d["measurement"]["status"] for d in docs if d["measurement"]["status"] != "success"}) + return {"value": statistics.median(medians) if medians else None, + "spread": statistics.stdev(medians) if len(medians) > 1 else None, + "status": "ok" if medians else (failed[0] if failed else "n/a")} + + +def summarize(doc, documents) -> tuple[list[dict], dict]: + """The comparison's units, sorted as the document lists them, each with its workload, quantity and + the two sides' numbers; and what each side is (commit, project, rounds, dirty).""" + projection = _projection(doc) + names = {_c(doc["baseline"]): "baseline", _c(doc["contender"]): "contender"} + grouped, sides = {}, {role: [] for role in ROLES} + for d in _run_documents(documents, doc["run_key"]): + projected = projection.apply(d, store_project(d)) + role = names.get(_c(projected.axis[0])) + if role is None: + continue + variant = {ptr: v for ptr, v in projected.identity.items() if ptr == WORKLOAD or ptr.startswith(WORKLOAD + "/")} + quantity = d["measurement"]["quantity"] + unit = grouped.setdefault((_c(variant), quantity["name"]), + {"quantity": quantity, "docs": {r: [] for r in ROLES}}) + unit["docs"][role].append(d) + sides[role].append(d) + units = [] + for unit in grouped.values(): + units.append({"workload": _workload(unit["docs"], projection.axis), "quantity": unit["quantity"], + **{role: _side(unit["docs"][role]) for role in ROLES}}) + units.sort(key=lambda u: _unit_order(u["workload"], u["quantity"]["name"])) + varying = doc["varying"].rsplit(".", 1)[-1] # the label or parameter that names the sides + info = {} + for role, docs in sides.items(): + project = store_project(docs[0]) if docs else "" + commit = docs[0]["revision"]["key"][:SHA] if docs else "?" + revisions = doc["profile"] == "revisions" # its sides are two commits; the others', two values + info[role] = {"tag": commit if revisions else str(doc[role]), + "heading": f"{project} commit {commit}" if revisions else f"{project} {varying}={doc[role]}", + "rounds": len({d.get("procedure", {}).get("round") for d in docs}), + "dirty": any(d["provenance"].get(flag) == "dirty" for d in docs for flag, _ in DIRTY_WARNINGS)} + return units, info + + +def _grid(headers, rows) -> list[str]: + """asv's table (`format_text_table`): each column its widest cell plus 2, every cell centered, a + space between columns, `=` borders and `-` under the header. Lines are not trimmed.""" + cells = [[str(cell) for cell in row] for row in (headers, *rows)] + widths = [max(len(row[j]) for row in cells) + 2 for j in range(len(headers))] + lines = [" ".join(cell.center(width) for cell, width in zip(row, widths)) for row in cells] + rule = lambda char: " ".join(char * width for width in widths) # noqa: E731 + return [rule("="), lines[0], rule("-"), *lines[1:], rule("=")] + + +def _cell(unit, role) -> str: + """A side's value, or why it has none.""" + side = unit[role] + return human(side["value"], side["spread"], unit["quantity"].get("unit")) if side["value"] is not None \ + else side["status"] + + +def side_tables(units, role, info) -> str: + """The tables of one side: a block per benchmark, a row per parameter combination.""" + side = info[role] + lines = [f"For {side['heading']}" + (" [dirty tree]" if side["dirty"] else "") + f" ({side['rounds']} rounds):"] + benchmarks = {} + for unit in units: + benchmarks.setdefault((unit["workload"]["name"], unit["quantity"]["name"]), []).append(unit) + for (name, quantity), group in benchmarks.items(): + title = name + ("" if quantity == "wall-time" else f" ({quantity})") + names = list(dict.fromkeys(p for u in group for p in (u["workload"].get("parameters") or {}))) + cell = lambda u: _cell(u, role) # noqa: E731 + if not names: + lines.append(f"{title} {cell(group[0])}") + continue + bad = [u[role]["status"] for u in group if u[role]["status"] != "ok"] + lines.append(f"{title} {bad[0] if bad else 'ok'}") + rows = [[(u["workload"].get("parameters") or {}).get(n, "") for n in names] + [cell(u)] for u in group] + lines.extend(" " + line for line in _grid([*names, ""], rows)) # under the title + return "\n".join(lines) + + +def _label(workload) -> str: + """asv's `name(p1, p2)`.""" + parameters = workload.get("parameters") or {} + return workload["name"] + (f"({', '.join(str(v) for v in parameters.values())})" if parameters else "") + + +def _pipe(headers, rows, right=()) -> str: + """asv's `tabulate` "github" table: a column is as wide as its widest cell or its header plus 2; + text is left-aligned, and the `right` columns (numbers) right-aligned.""" + widths = [max(len(header) + 2, *(len(row[j]) for row in rows)) for j, header in enumerate(headers)] + align = lambda cell, j: cell.rjust(widths[j]) if j in right else cell.ljust(widths[j]) # noqa: E731 + line = lambda cells: "| " + " | ".join(align(cell, j) for j, cell in enumerate(cells)) + " |" # noqa: E731 + return "\n".join([line(headers), "|" + "|".join("-" * (width + 2) for width in widths) + "|", + *(line(row) for row in rows)]) + + +def _verdict(unit) -> str: + """The comparator's verdict in the words of its summary, with the reason when there is no verdict.""" + return unit["verdict"].replace("-", " ") + (f" ({unit['reason']})" if unit.get("reason") else "") + + +def change_table(doc, units, info, *, all_rows=False) -> str: + """asv's comparison table, and the verdict. As `asv continuous` does, only the rows that + changed, unless `all_rows`.""" + by_workload = {(_c(u["workload"]), u["quantity"]["name"]): u for u in units} + rows = [] + for verdict_unit in doc["units"]: + mark = MARKS[verdict_unit["verdict"]] + if mark not in CHANGED.values() and not all_rows: + continue + unit = by_workload[(_c(verdict_unit["workload"]), verdict_unit["quantity"])] + before, after = unit["baseline"], unit["contender"] + ratio = f"{after['value'] / before['value']:.2f}" if before["value"] and after["value"] is not None else "n/a" + noise = f"{verdict_unit['noise']:.2%}" if "noise" in verdict_unit else "n/a" + rows.append([mark, *(_cell(unit, role) for role in ROLES), ratio, noise, _verdict(verdict_unit), + _label(unit["workload"])]) + if not rows: + return "No benchmark changed (the verdict is benchx's; --all lists every row)." + headers = ["Change", f"Before [{info['baseline']['tag']}]", f"After [{info['contender']['tag']}]", + "Ratio", "Noise", "Verdict", "Benchmark (Parameter)"] + return _pipe(headers, rows, right={j for j in (3, 4) if all(row[j] != "n/a" for row in rows)}) + + +def render(doc, documents, *, all_rows=False) -> str: + """The tables of a comparison, or nothing when the document has no units (its invariants failed).""" + if not doc.get("units") or "contender" not in doc: + return "" + units, info = summarize(doc, documents) + return "\n" + "\n\n".join([*(side_tables(units, role, info) for role in ROLES), + change_table(doc, units, info, all_rows=all_rows)]) diff --git a/tests/test_asv.py b/tests/test_asv.py index 90c8827..9ae425b 100644 --- a/tests/test_asv.py +++ b/tests/test_asv.py @@ -197,10 +197,18 @@ def test_a_filter_that_selects_nothing_is_refused(suite, capsys): assert cli.main([*ASV, "--filter", "time_plian", "--run-key", "typo", "--no-ingest", *FAST]) == cli.EXIT_REFUSED err = capsys.readouterr().err assert "refused: filter 'time_plian' selects no case in" in err - assert "available: bm.Cached.time_sum, bm.Mutating.time_pop," in err + assert "\n similar: bm.time_plain\n" in err # what the typo is close to, not the first of the alphabet + assert "available" not in err assert results(suite, "typo") == {} +def test_a_filter_that_is_like_no_name_lists_the_first_available(suite, capsys): + assert cli.main([*ASV, "--filter", "zzzzzz", "--run-key", "unlike", "--no-ingest", *FAST]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "available: bm.Cached.time_sum, bm.Mutating.time_pop," in err + assert "similar" not in err + + def test_an_anchored_name_of_a_parametrized_benchmark_is_refused_with_a_hint(suite, capsys): # asv searches `name(p0, p1)`, so `time_sorted$` matches none of the combinations. assert cli.main([*ASV, "--filter", "time_sorted$", "--run-key", "anchored", "--no-ingest", @@ -209,6 +217,16 @@ def test_an_anchored_name_of_a_parametrized_benchmark_is_refused_with_a_hint(sui in capsys.readouterr().err) +def test_arguments_in_the_wrong_order_are_refused_with_the_order_and_an_example(suite, capsys): + # `Sort` takes (n, key); `('a', 10)` names the benchmark and selects none of its combinations. + assert cli.main([*ASV, "--filter", r"time_sorted\('a', 10\)", "--run-key", "order", "--no-ingest", + *FAST]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "the filter matches the name of a parametrized benchmark (bm.Sort.time_sorted) but not its arguments" in err + assert "\n bm.Sort.time_sorted(n, key)\n for example:\n bm.Sort.time_sorted(10, 'a')\n" in err + assert results(suite, "order") == {} + + def test_a_filter_that_selects_only_benchmarks_that_are_not_run_is_refused(suite, capsys): assert cli.main([*ASV, "--filter", "peakmem_sorted", "--run-key", "notrun", "--no-ingest", *FAST]) == cli.EXIT_REFUSED @@ -419,3 +437,16 @@ def test_missing_asv_runner_is_refused(suite, monkeypatch, tmp, capsys): monkeypatch.setenv("PATH", str(bare)) assert cli.main([*ASV, "--no-ingest"]) == cli.EXIT_REFUSED assert "pyperf is not importable" in capsys.readouterr().err + + +def test_a_refusal_shows_the_filter_as_it_was_typed(suite, capsys): + """Backslashes are not doubled (not a Python repr), and the refusal is one fact per line.""" + assert cli.main([*ASV, "--filter", r"time_sorted\('a', 10\)", "--run-key", "typed", "--no-ingest", + *FAST]) == cli.EXIT_REFUSED + lines = [line for line in capsys.readouterr().err.splitlines() if line.startswith(("refused:", " "))] + assert lines[0].startswith(r"refused: filter 'time_sorted\('a', 10\)' selects no case in ") + assert [line.split(":")[0].strip() for line in lines[1:]] == [ + "the filter matches the name of a parametrized benchmark (bm.Sort.time_sorted) but not its arguments.", + "They are searched as name(p0, p1), each as its repr, in the order of param_names", + "bm.Sort.time_sorted(n, key)", "for example", "bm.Sort.time_sorted(10, 'a')", + "1 module(s) failed to import", "similar"] # the suite has a module that does not import diff --git a/tests/test_bench.py b/tests/test_bench.py index bd5663f..221fb2a 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -99,7 +99,7 @@ def ran(run_key): assert cli.main(["bench", "mod.py", "bench_sort(100, 'a')", "--run-key", "one", "--no-ingest", *FAST]) == cli.EXIT_OK assert ran("one") == ["100'a'"] assert cli.main(["bench", "mod.py", "bench_srot", "--run-key", "typo", "--no-ingest", *FAST]) == cli.EXIT_REFUSED - assert "selects no case in mod.py; available: bench_sort" in capsys.readouterr().err + assert "selects no case in mod.py\n similar: bench_sort\n" in capsys.readouterr().err def test_shorthand_order_is_stable(tree): diff --git a/tests/test_bench_failures.py b/tests/test_bench_failures.py new file mode 100644 index 0000000..e03f166 --- /dev/null +++ b/tests/test_bench_failures.py @@ -0,0 +1,133 @@ +"""What `bx bench` says about a result that is not a measurement (FAILURE_MESSAGES_PLAN.md).""" + +import textwrap + +import pytest + +from benchx import bench, cli + + +def result(name="bench_boom", status="error", reason="harness.error", message="ValueError: boom", parameters=None): + workload = {"name": name, **({"parameters": parameters} if parameters else {})} + document = {"coordinates": {"workload": workload}, + "measurement": {"status": status, **({"reason": reason} if reason else {})}, + "provenance": {"info": {"message": message} if message else {}}} + if status == "success": + document["measurement"]["observations"] = [1.0, 2.0] + return document + + +def rows_of(*documents): + return [bench.row_of(d) for d in documents] + + +def test_a_row_carries_what_a_failed_result_said(): + assert bench.row_of(result())["message"] == "ValueError: boom" + assert bench.row_of(result(status="skipped", reason="harness.skipped", message="no GPU"))["message"] == "no GPU" + + +def test_a_row_has_no_message_when_there_is_none_to_give(): + assert bench.row_of(result(status="success", reason=None, message="a note"))["message"] is None # measured + assert bench.row_of(result(message=None))["message"] is None # failed without a word + no_provenance = result() + del no_provenance["provenance"] # a result from another producer + assert bench.row_of(no_provenance)["message"] is None + + +def test_nothing_is_said_when_every_case_was_measured(): + assert bench.failures([]) == "" + assert bench.failures(rows_of(result(status="success", reason=None, message=None))) == "" + assert bench.failures(rows_of(result(message=None))) == "" # no message: the status column is all there is + + +def test_a_failed_case_is_named_with_what_it_said(): + assert bench.failures(rows_of(result())) == "\nnot measured (1)\n bench_boom: ValueError: boom" + + +def test_cases_that_said_the_same_are_one_line_with_three_names_at_most(): + rows = rows_of(*(result("bench_boom", parameters={"n": str(n)}) for n in (1, 10, 100, 1000, 10000))) + assert bench.failures(rows) == ("\nnot measured (5)\n" + " ValueError: boom (5 cases: bench_boom [n=1], bench_boom [n=10], bench_boom [n=100], …)") + three = rows_of(*(result("bench_boom", parameters={"n": str(n)}) for n in (1, 10, 100))) + assert bench.failures(three).endswith("(3 cases: bench_boom [n=1], bench_boom [n=10], bench_boom [n=100])") + + +def test_different_messages_are_different_lines_in_the_order_they_came(): + rows = rows_of(result("a", message="KeyError: 'x'"), result("b", message="ValueError: boom"), + result("c", status="skipped", reason="harness.skipped", message="KeyError: 'x'")) + assert bench.failures(rows) == ("\nnot measured (3)\n KeyError: 'x' (2 cases: a, c)\n b: ValueError: boom") + + +def test_a_message_is_one_line_and_not_longer_than_a_line_should_be(): + long = bench.failures(rows_of(result(message="ValueError: " + "x" * 300))) + assert long.endswith("…") and len(long.splitlines()[-1]) == len(" bench_boom: ") + 160 + several = bench.failures(rows_of(result(message="RuntimeError: first\nsecond line\nthird"))) + assert several == "\nnot measured (1)\n bench_boom: RuntimeError: first" + + +def test_bx_bench_says_why_a_benchmark_failed(tmp, monkeypatch, capsys): + pytest.importorskip("pyperf") + tree = tmp / "tree" + tree.mkdir() + (tree / "mod.py").write_text(textwrap.dedent(''' + def bench_ok(): + sum(range(100)) + + def bench_boom(): + raise ValueError("array has no axis 3") + ''')) + monkeypatch.chdir(tree) + cli.main(["bench", "mod.py", "--adapter", "pyperf", "--run-key", "boom", "--no-ingest", + "--processes", "2", "--values", "2", "--loops", "20"]) + out = capsys.readouterr().out + assert "error (harness.error)" in out # the status column is as it was + assert "\nnot measured (1)\n bench_boom: ValueError: array has no axis 3\n" in out + assert "bench_ok" not in out.split("not measured")[1] # what was measured is not in it + + +TRACEBACK = """Traceback (most recent call last): + File "driver.py", line 37, in + runner.bench_func('bench_axes', func, *args) + File "mod.py", line 7, in bench_axes + raise KeyError("axis") +{last} +""" +MANAGER = """Traceback (most recent call last): + File "pyperf/_manager.py", line 118, in spawn_worker + raise RuntimeError("%s failed with exit code %s" +RuntimeError: /usr/bin/python failed with exit code 1 +""" +WARNING = """driver.py:34: SyntaxWarning: "is not" with 'int' literal. Did you mean "!="? + if 0 is not None: +""" + + +def test_the_message_is_the_exception_that_ended_the_workers_traceback(): + from benchx.adapters import pyperf + stderr = TRACEBACK.format(last="KeyError: 'axis'") + MANAGER + assert pyperf._failure_message(stderr) == "KeyError: 'axis'" # not the manager's wrapper after it + + +def test_a_warning_printed_before_the_traceback_is_not_the_message(): + from benchx.adapters import pyperf + stderr = WARNING + WARNING + TRACEBACK.format(last="KeyError: 'axis'") + MANAGER + assert pyperf._failure_message(stderr) == "KeyError: 'axis'" + + +def test_a_chained_exception_gives_the_one_that_was_raised_last(): + from benchx.adapters import pyperf + chained = (TRACEBACK.format(last="KeyError: 'axis'") + + "\nThe above exception was the direct cause of the following exception:\n\n" + + TRACEBACK.format(last="ValueError: no such axis")) + assert pyperf._failure_message(chained + MANAGER) == "ValueError: no such axis" + + +def test_the_wrapper_is_the_message_only_when_nothing_else_was_said(): + from benchx.adapters import pyperf + assert pyperf._failure_message(MANAGER) == "RuntimeError: /usr/bin/python failed with exit code 1" + + +def test_without_a_traceback_the_first_line_said_is_the_message(): + from benchx.adapters import pyperf + assert pyperf._failure_message("Killed\n") == "Killed" + assert pyperf._failure_message(WARNING) == WARNING.splitlines()[0] # nothing better was said diff --git a/tests/test_build_import_path.py b/tests/test_build_import_path.py index c27039a..1f0f0b4 100644 --- a/tests/test_build_import_path.py +++ b/tests/test_build_import_path.py @@ -93,7 +93,8 @@ def test_the_build_root_imports_nothing(tmp, toy, monkeypatch): unit = doc["units"][0] assert unit["rounds"] == 0 assert unit["verdict"] == "indeterminate" - assert doc["exclusions"] and {e["reason"] for e in doc["exclusions"]} == {"error: harness.error"} + assert doc["exclusions"] and {e["reason"] for e in doc["exclusions"]} == { + "error: harness.error: ModuleNotFoundError: No module named 'toy'"} # and says what the worker said def test_one_pythonpath_measures_one_build_on_both_sides(tmp, toy, monkeypatch): diff --git a/tests/test_comparator.py b/tests/test_comparator.py index d5dcbea..7bb6696 100644 --- a/tests/test_comparator.py +++ b/tests/test_comparator.py @@ -619,3 +619,148 @@ def test_units_with_more_than_one_other_parameter_are_ordered_by_name_then_value doc = vevaluate(extra) assert [(u["workload"]["parameters"]["n"], u["workload"]["parameters"]["size"]) for u in doc["units"]] == [ ("2", "20"), ("2", "100"), ("10", "20"), ("10", "100")] + + +# --- one benchmark that cannot be compared does not stop the others ------------------------- + +def run_two(rounds=3, ratio=1.1): + """Two workloads measured in every call; the head takes `ratio` times the base's time.""" + docs, slot = [], 0 + for r in range(rounds): + for side, factor in ((BASE, 1.0), (HEAD, ratio)): + docs += [result(side, r, slot, (r + 1.0) * factor, workload=name) for name in ("BM_A", "BM_B")] + slot += 1 + return docs + + +def of_head(docs, workload): + return [d for d in docs if d["revision"]["key"] == HEAD and d["coordinates"]["workload"]["name"] == workload] + + +def by_name(doc): + return {u["workload"]["name"]: u for u in doc["units"]} + + +def test_an_edited_benchmark_fails_its_own_unit_only(): + """A benchmark whose code differs between the sides (asv's benchmark version) is a new series and is + not comparable; the other benchmark is compared.""" + docs = run_two() + for d in docs: + if d["coordinates"]["workload"]["name"] == "BM_B": + version = "v2" if d["revision"]["key"] == HEAD else "v1" + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": version} + doc = evaluate(docs) + assert invariants(doc) == ["sides-agree"] + failure = doc["failed_invariants"][0] + assert failure["workload"] == {"name": "BM_B"} and "benchmark_version differ" in failure["detail"] + assert list(by_name(doc)) == ["BM_A"] and by_name(doc)["BM_A"]["verdict"] == "regressed" + + +def test_a_failed_attempt_is_excluded_before_the_facts_are_matched(): + """A benchmark that raises reports no harness version or runtime, so its attempts differ from the other + side's in those facts. They are excluded and listed, not matched: the benchmark has no verdict + (`insufficient-rounds`) and the other one is compared.""" + docs = run_two() + failing = of_head(docs, "BM_B") + for d in docs: + if d not in failing: # the CPU configuration is a fact the projection ignores: what is reported, differs + d["coordinates"]["environment"]["system"]["cpu"] = {"config": "performance"} + for d in failing: + d["measurement"] = {"quantity": d["measurement"]["quantity"], "status": "error", "reason": "harness.error"} + d["coordinates"]["protocol"]["harness"] = {"name": "google-benchmark", "version": "1.9"} + d["coordinates"]["environment"]["system"]["runtime"] = {"python": "3.12"} + doc = evaluate(docs) + assert doc["failed_invariants"] == [] + assert by_name(doc)["BM_A"]["verdict"] == "regressed" + failed = by_name(doc)["BM_B"] + assert failed["verdict"] == "indeterminate" and failed["reason"] == "insufficient-rounds" + assert [e["reason"] for e in doc["exclusions"]] == ["error: harness.error"] * 3 + assert by_name(doc)["BM_B"].get("unmatched_differences", []) == [] # what it did not report is no difference + assert by_name(doc)["BM_A"].get("unmatched_differences", []) == [] + + +def test_a_failure_of_the_whole_run_still_leaves_no_units(): + """A third side (a tree edited during the run) is about the run, so no benchmark is compared.""" + docs = run_two() + docs[-1]["provenance"]["subject_tree"] = "c" * 40 + doc = evaluate(docs) + assert invariants(doc) == ["two-sides"] and doc["units"] == [] + + +# --- what the output says about what was left out (FAILURE_MESSAGES_PLAN.md, decisions 8 and 9) --- + +def failing_head(docs, workload="BM_B", message="ValueError: array has no axis 3"): + """The head's attempts of a workload fail, the worker having said `message`.""" + for d in of_head(docs, workload): + d["measurement"] = {"quantity": d["measurement"]["quantity"], "status": "error", "reason": "harness.error"} + if message: + d["provenance"] = {**d["provenance"], "info": {"message": message}} + return docs + + +def test_an_exclusion_says_what_the_failed_attempt_said(): + doc = evaluate(failing_head(run_two())) + assert [e["reason"] for e in doc["exclusions"]] == ["error: harness.error: ValueError: array has no axis 3"] * 3 + + +def test_an_exclusion_of_an_attempt_that_said_nothing_is_as_it_was(): + doc = evaluate(failing_head(run_two(), message=None)) + assert [e["reason"] for e in doc["exclusions"]] == ["error: harness.error"] * 3 + + +def test_excluded_results_that_failed_alike_are_one_line_with_their_number(): + text = compare.render(evaluate(failing_head(run_two()))) + assert text.endswith("\nexcluded (3)\n error: harness.error: ValueError: array has no axis 3 (3 results)") + assert "ingest_key" not in text and "{'" not in text # words, not the document's dicts + + +def test_excluded_results_that_failed_differently_are_lines_of_their_own(): + docs = failing_head(run_two(), message="ValueError: array has no axis 3") + docs[docs.index(of_head(docs, "BM_B")[0])]["provenance"]["info"]["message"] = "KeyError: 'x'" + lines = compare.render(evaluate(docs)).splitlines() + assert lines[-3:] == ["excluded (3)", " error: harness.error: KeyError: 'x' (1 result)", + " error: harness.error: ValueError: array has no axis 3 (2 results)"] + + +def test_a_message_in_the_reason_is_one_line_and_cut(): + docs = failing_head(run_two(), message="RuntimeError: " + "x" * 300 + "\nthe second line") + [reason, *_] = {e["reason"] for e in evaluate(docs)["exclusions"]} + assert reason.startswith("error: harness.error: RuntimeError: xxx") and reason.endswith("…") + assert "second line" not in reason and len(reason) == len("error: harness.error: ") + 160 + + +def test_a_failed_invariant_of_one_unit_names_the_unit_in_words(): + docs = run_two() + for d in docs: + if d["coordinates"]["workload"]["name"] == "BM_B": + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": "v2" if d["revision"]["key"] == HEAD else "v1"} + text = compare.render(evaluate(docs)) + assert text.endswith("\nfailed invariants (1)\n BM_B: sides-agree: /coordinates/protocol/asv/benchmark_version differ") + assert "'invariant'" not in text + + +def test_units_that_failed_an_invariant_alike_are_one_line(): + docs = run_two() + for d in docs: + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": "v2" if d["revision"]["key"] == HEAD else "v1"} + text = compare.render(evaluate(docs)) + assert text.endswith("\nfailed invariants (2)\n" + " sides-agree: /coordinates/protocol/asv/benchmark_version differ (2 units: BM_A, BM_B)") + + +def test_a_failed_invariant_of_the_run_is_in_words_too(): + doc = {"run_key": "r", "profile": "revisions", "baseline": BASE, "contender": HEAD, "local_only": False, "units": [], + "missing": [], "exclusions": [], + "failed_invariants": [{"invariant": "equal-attempts", "detail": "3 against 2: run the same number of rounds"}]} + assert compare.render(doc).endswith("\nfailed invariants (1)\n equal-attempts: 3 against 2: run the same number of rounds") + + +def test_missing_results_are_one_line_for_each_side_with_a_few_names(): + where = lambda name, side: {"workload": {"name": name, "parameters": {"n": "1"}}, "quantity": "wall-time", "side": side} + doc = {"run_key": "r", "profile": "revisions", "baseline": BASE, "contender": HEAD, "local_only": False, "units": [], + "failed_invariants": [], "exclusions": [], + "missing": [where("a", "contender"), where("b", "contender"), where("c", "contender"), where("d", "contender"), + where("e", "baseline")]} + assert compare.render(doc).endswith( + "\nmissing (5)\n not on baseline: e [n=1]\n not on contender: a [n=1], b [n=1], c [n=1], … (+1 more)") + diff --git a/tests/test_config.py b/tests/test_config.py index c12e38d..b7b6f18 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -179,3 +179,50 @@ def test_show_says_where_each_value_comes_from(tree, monkeypatch, capsys): assert '"demo" (file)' in out and '"mod.py" (file)' in out assert "compare.rounds - (unset)" in out assert "timeouts.case_s 60 (default)" in out + + +# --- one file for the adapters that share a precision --------------------------------- + +TIMING = {"quantities": ["wall-time"], + "precision": {"repetitions": {"mode": "fixed", "levels": [{"unit": "process", "n": 5}, + {"unit": "value", "n": 3}]}, + "calibration": {"mode": "adaptive", "minimum_sample_seconds": 0.1}, + "warmup": {"mode": "count", "n_warmup": 1}}} + + +def settle(tree, monkeypatch, *argv): + """What `bx bench` settles on, from the file in `tree` and `argv`, without running anything.""" + seen = [] + monkeypatch.setattr(cli, "cmd_bench", lambda args: seen.append(args) or cli.EXIT_OK) + monkeypatch.chdir(tree) + assert cli.main(["bench", "TARGET", *argv]) == cli.EXIT_OK + return cli._common(config.load(tree), seen[0]) + + +def test_asv_is_timed_by_pyperf_so_the_two_share_a_family(): + assert harnesses.same_family("pyperf", "asv") and harnesses.same_family("asv", "pyperf") + assert harnesses.same_family("asv", "asv") and harnesses.same_family("google-benchmark", "google-benchmark") + assert not harnesses.same_family("pyperf", "google-benchmark") + assert not harnesses.same_family("asv", "google-benchmark") + assert not harnesses.same_family("pyperf", "nonesuch") and not harnesses.same_family("nonesuch", "pyperf") + + +@pytest.mark.parametrize("named, given", [("pyperf", "asv"), ("asv", "pyperf"), ("pyperf", "pyperf"), ("asv", "asv")]) +def test_a_file_serves_the_adapters_that_share_its_precision(tree, monkeypatch, named, given): + write_config(tree, adapter={"name": named, **TIMING}) + settings = settle(tree, monkeypatch, "--adapter", given, "--suite", "s") + assert settings["adapter"] == given and settings["suite"] == "s" + assert levels(harnesses.get(given).precision({}, settings["file_precision"])) == {"process": 5, "value": 3} + + +def test_without_an_adapter_flag_the_files_adapter_is_used(tree, monkeypatch): + write_config(tree, adapter={"name": "asv", **TIMING}) + assert settle(tree, monkeypatch, "--suite", "s")["adapter"] == "asv" + + +@pytest.mark.parametrize("named, given", [("google-benchmark", "pyperf"), ("google-benchmark", "asv"), + ("pyperf", "google-benchmark"), ("asv", "google-benchmark")]) +def test_an_adapter_of_another_family_is_refused(tree, monkeypatch, named, given): + write_config(tree, adapter={"name": named}) + with pytest.raises(config.ConfigError, match=rf"adapter {given!r}, but .* sets adapter\.name to {named!r}.*precision"): + settle(tree, monkeypatch, "--adapter", given, "--suite", "s") diff --git a/tests/test_parameter_order.py b/tests/test_parameter_order.py new file mode 100644 index 0000000..6a1fb91 --- /dev/null +++ b/tests/test_parameter_order.py @@ -0,0 +1,120 @@ +"""Parameters in the order the benchmark declared them (PARAMETER_ORDER_PLAN.md). + +The store keeps a document in canonical form, with its keys sorted, so a benchmark's parameters come back +alphabetical: `array_type, dtype, k` for a benchmark that declares `dtype, array_type, k`, which is the order a +filter searches and asv shows. The runner records the declared order in `provenance.info.parameter_names`, and what +shows a workload reads it back. +""" + +import json + +from benchx import bench, compare, core, tables +from benchx.projection import declared_workload, workload_label +from test_comparator import BASE, HEAD, result + +NAME = "bm.Partition.time_argpartition" +DECLARED = ["dtype", "array_type", "k"] # not alphabetical +COMBINATIONS = [{"dtype": dtype, "array_type": array_type, "k": "10"} + for dtype in ("'float64'", "'int16'") for array_type in ("('ordered',)", "('random',)")] + + +def stored(document): + """A document as the store gives it back: canonical, so its keys sorted.""" + return json.loads(core.canonical(document)) + + +def case(side, round_, slot, value, parameters, names=True): + d = result(side, round_, slot, value, workload=NAME) + d["coordinates"]["workload"]["parameters"] = parameters # in the order the benchmark declared them + d["ingest_key"] += f":{parameters}" + if names: + d["provenance"]["info"] = {"parameter_names": list(parameters)} + return stored(d) + + +def run_of(names=True, rounds=5, ratio=1.5): + """Every combination, base and head alternating; the head is `ratio` times the base.""" + docs, slot = [], 0 + for r in range(rounds): + for side in (BASE, HEAD): + for parameters in COMBINATIONS: + docs.append(case(side, r, slot, (1.0 + 0.01 * r) * (ratio if side == HEAD else 1.0), parameters, names)) + slot += 1 + return docs + + +def evaluate(docs): + return compare.compare(docs, run_key="run", profile="revisions", baseline=BASE) + + +def test_the_store_gives_the_parameters_back_alphabetical(): + """The fault: what a result file says in one order, the stored document says in another.""" + document = case(BASE, 0, 0, 1.0, COMBINATIONS[0]) + assert list(document["coordinates"]["workload"]["parameters"]) == sorted(DECLARED) + assert document["provenance"]["info"]["parameter_names"] == DECLARED # which is why the order is recorded + + +def test_a_workload_is_read_in_the_order_its_result_recorded(): + document = case(BASE, 0, 0, 1.0, COMBINATIONS[0]) + assert list(declared_workload(document)["parameters"]) == DECLARED + assert declared_workload(document)["name"] == NAME + assert list(document["coordinates"]["workload"]["parameters"]) == sorted(DECLARED) # the document is not changed + + +def test_a_result_that_recorded_no_order_has_its_parameters_as_stored(): + """Made before the order was recorded, or by another producer.""" + document = case(BASE, 0, 0, 1.0, COMBINATIONS[0], names=False) + assert declared_workload(document) == document["coordinates"]["workload"] + assert declared_workload(result(BASE, 0, 0, 1.0)) == {"name": "BM_W"} # nor parameters + + +def test_a_recorded_order_that_does_not_fit_the_parameters_is_not_trusted_blindly(): + document = case(BASE, 0, 0, 1.0, COMBINATIONS[0]) + document["provenance"]["info"]["parameter_names"] = ["k", "gone"] # one is not a parameter, two are not named + assert list(declared_workload(document)["parameters"]) == ["k", "array_type", "dtype"] + + +def test_a_comparison_names_the_parameters_in_the_order_they_were_declared(): + doc = evaluate(run_of()) + assert [list(u["workload"]["parameters"]) for u in doc["units"]] == [DECLARED] * 4 + assert compare.render(doc).count("[dtype='int16', array_type=('ordered',), k=10]") == 1 + + +def test_the_units_are_listed_in_the_order_the_parameters_vary_as_declared(): + """The first declared parameter changes slowest, as asv lists a product of params.""" + doc = evaluate(run_of()) + assert [(u["workload"]["parameters"]["dtype"], u["workload"]["parameters"]["array_type"]) for u in doc["units"]] == [ + ("'float64'", "('ordered',)"), ("'float64'", "('random',)"), ("'int16'", "('ordered',)"), ("'int16'", "('random',)")] + + +def test_results_without_the_order_are_compared_and_listed_as_before(): + doc = evaluate(run_of(names=False)) + assert [list(u["workload"]["parameters"]) for u in doc["units"]] == [sorted(DECLARED)] * 4 + assert [(u["workload"]["parameters"]["array_type"], u["workload"]["parameters"]["dtype"]) for u in doc["units"]] == [ + ("('ordered',)", "'float64'"), ("('ordered',)", "'int16'"), ("('random',)", "'float64'"), ("('random',)", "'int16'")] + + +def test_a_failed_invariant_names_the_parameters_in_the_declared_order_too(): + docs = run_of() + for d in docs: + if d["coordinates"]["workload"]["parameters"]["dtype"] == "'int16'": + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": "v2" if d["revision"]["key"] == HEAD else "v1"} + doc = evaluate(docs) + assert [list(f["workload"]["parameters"]) for f in doc["failed_invariants"]] == [DECLARED] * 2 + assert compare.render(doc).splitlines()[-1] == ( + " sides-agree: /coordinates/protocol/asv/benchmark_version differ (2 units: " + f"{NAME} [dtype='int16', array_type=('ordered',), k=10], {NAME} [dtype='int16', array_type=('random',), k=10])") + + +def test_the_tables_have_the_columns_and_the_values_in_the_declared_order(): + docs = run_of() + text = tables.render(evaluate(docs), docs) + header = next(line for line in text.splitlines() if "array_type" in line) + assert header.split() == DECLARED + assert f"{NAME}('float64', ('ordered',), 10)" in text # the label of the change table: values, in that order + + +def test_bx_bench_and_bx_head_label_a_stored_result_in_the_declared_order(): + document = case(BASE, 0, 0, 1.0, COMBINATIONS[3]) + assert bench.row_of(document)["case"] == f"{NAME} [dtype='int16', array_type=('random',), k=10]" + assert workload_label(declared_workload(document)) == bench.row_of(document)["case"] diff --git a/tests/test_params.py b/tests/test_params.py index 0e3e791..29399b0 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -76,6 +76,17 @@ def test_a_filter_that_matches_only_the_name_is_noticed(): assert not params.matches_name_only("m.C.time_x", IDS, None) +def test_a_filter_that_gives_the_arguments_in_the_wrong_order_is_noticed(): + """The part before the first `\\(` is the benchmark's name; what follows it selected nothing.""" + assert params.expand("m.C.time_x", IDS, r"C\.time_x\('a', 10\)") == [] + assert params.matches_name_only("m.C.time_x", IDS, r"C\.time_x\('a', 10\)") + assert params.matches_name_only("m.C.time_x", IDS, r"^m\.C\.time_x\(100, 'c'\)$") + assert not params.matches_name_only("m.C.time_x", IDS, r"C\.time_y\(10, 'a'\)") # another benchmark + assert not params.matches_name_only("m.C.time_x", IDS, r"\(10, ") # no name to go by + assert not params.matches_name_only("m.C.time_x", IDS, r"^\(10, ") + assert not params.matches_name_only("m.C.time_x", IDS, r"(unbalanced\(10") # not a regex: no hint, no error + + def test_the_workload_of_a_case_comes_from_the_table_not_from_the_id(): runnable = {"cases": {"m.C.time_x-2": ("m.C.time_x", 2), "m.time_y-1": ("m.time_y", None)}, "benchmarks": {"m.C.time_x": {"params": IDS, "param_names": ["n", "key"]}, "m.time_y": {}}} @@ -83,10 +94,42 @@ def test_the_workload_of_a_case_comes_from_the_table_not_from_the_id(): assert base.parametrized_workload("m.time_y-1", runnable) == ("m.time_y", {}) # a name that looks like -1 +NAMES = ["bench_function_base.Partition.time_argpartition", "bench_function_base.Partition.time_partition", + "bench_function_base.Sort.time_sort", "bench_linalg.Linalg.time_norm", "bench_ufunc.UFunc.time_add"] + + +def test_a_slip_in_a_word_of_the_filter_finds_the_names_that_have_it(): + assert base.similar("Partiton", NAMES) == NAMES[:2] # a class, as spelled in the names + assert base.similar("time_argpartiton", NAMES) == NAMES[:2] # a method: its own first (0.97), then time_partition (0.87) + assert base.similar("bench_linalgg", NAMES) == [NAMES[3]] # a module + assert base.similar("partition", NAMES) == NAMES[:2] # the case of a letter is no difference + + +def test_the_words_of_a_regex_are_what_is_compared(): + assert base.similar(r"^bench_function_base\.Partiton\.time_x\(10, 'a'\)$", NAMES) == NAMES[:2] + assert base.similar(r"Partiton|time_ad\b", NAMES) == [NAMES[0], NAMES[1], NAMES[4]] # 0.94, 0.94, then 0.93 + assert base.similar(r"\d+\.\w", NAMES) == [] # escapes and one letter say nothing + + +def test_nothing_is_similar_to_what_is_not_like_any_name(): + assert base.similar("zzzz", NAMES) == [] + assert base.similar(None, NAMES) == [] + assert base.similar("Partiton", []) == [] + + +def test_the_most_similar_names_come_first_and_at_most_limit_are_returned(): + names = ["m.Alpha.time_a", "m.Alphb.time_a", "m.Alpxx.time_a", "m.Alpha.time_b"] + assert base.similar("Alpha", names) == ["m.Alpha.time_a", "m.Alpha.time_b", "m.Alphb.time_a"] # exact, then near + assert base.similar("Alpha", names, limit=1) == ["m.Alpha.time_a"] + + def test_the_note_names_the_benchmarks_it_means(): - note = base.name_only_note(["a", "b", "c", "d"]) + found = [{"name": name, "param_names": ["n", "key"], "params": IDS} for name in "abcd"] + note = base.name_only_note(found) assert "(a, b, c)" in note # three at most assert "name(p0, p1)" in note + # the first one's parameters, in order, and a case of it, each on a line of its own to be copied + assert note.splitlines()[2:] == [" a(n, key)", "for example:", " a(10, 'a')"] def test_search_name_is_exactly_what_expand_searches(): diff --git a/tests/test_projection.py b/tests/test_projection.py index 59f0dfd..fc9cd67 100644 --- a/tests/test_projection.py +++ b/tests/test_projection.py @@ -8,7 +8,7 @@ from benchx import compare, core, identity from benchx.core import catalog -from benchx.projection import Projection, ProjectionError, workload_label +from benchx.projection import Projection, ProjectionError, said, workload_label POLICIES = catalog.SCHEMAS / "identity-policy" / "0.1.0" / "examples" DEFAULT = json.loads((POLICIES / "default.json").read_text()) @@ -171,3 +171,20 @@ def test_the_quantity_is_never_a_coordinate(): assert history().apply(ARROW).identity == history().apply(other).identity assert identity.series_fingerprint(ARROW, "arrow", identity.MEDIAN, history()) != \ identity.series_fingerprint(other, "arrow", identity.MEDIAN, history()) + + +def test_what_a_result_said_is_its_first_line_cut_to_a_width(): + def document(message): + return {"provenance": {"info": {"message": message}}} + + assert said(document("ValueError: boom")) == "ValueError: boom" + assert said(document(" RuntimeError: first\nsecond\nthird\n")) == "RuntimeError: first" + assert said(document("x" * 300)) == "x" * 159 + "…" + assert said(document("x" * 20), width=10) == "x" * 9 + "…" + assert said(document("")) is None and said(document("\n\n")) is None + + +def test_a_result_that_said_nothing_has_no_message(): + assert said({"provenance": {"info": {}}}) is None + assert said({"provenance": {}}) is None + assert said({}) is None # a result from another producer diff --git a/tests/test_pyperf.py b/tests/test_pyperf.py index 72d1cf1..66e3747 100644 --- a/tests/test_pyperf.py +++ b/tests/test_pyperf.py @@ -12,6 +12,7 @@ import subprocess import sys import textwrap +import warnings from pathlib import Path import pytest @@ -183,10 +184,41 @@ def test_a_filter_that_matches_a_name_but_no_arguments_is_refused_with_a_hint(pa with pytest.raises(adapter.Unsupported) as refused: planned(param_tree, "bench_sort$") message = str(refused.value.problems[0]) - assert "filter 'bench_sort$' selects no case in mod.py; available: bench_plain, bench_sort;" in message + assert message.startswith("filter 'bench_sort$' selects no case in mod.py\n") + assert "\n similar: bench_sort" in message assert "the filter matches the name of a parametrized benchmark (bench_sort) but not its arguments" in message +def test_a_refusal_says_how_the_arguments_of_a_case_are_searched(param_tree): + """Arguments in the wrong order: the filter names the function, so the note gives the order + of `param_names` and a case as it is searched.""" + with pytest.raises(adapter.Unsupported) as refused: + planned(param_tree, r"bench_sort\('a', 10\)") + message = str(refused.value.problems[0]) + assert "the filter matches the name of a parametrized benchmark (bench_sort) but not its arguments" in message + assert message.splitlines()[3:6] == [" bench_sort(n, key)", " for example:", " bench_sort(10, 'a')"] + assert message.index("not its arguments") < message.index("similar:") # the hint comes before the list + + +def test_a_filter_with_no_name_in_it_gets_no_note_about_arguments(param_tree): + with pytest.raises(adapter.Unsupported) as refused: + planned(param_tree, r"\(7, ") + assert "not its arguments" not in str(refused.value.problems[0]) + + +def test_the_driver_of_a_case_gives_python_nothing_to_warn_about(tmp, param_tree): + """The driver is a script of its own, run once per worker. It has `if is not None:` for a + parametrized case, and Python warns about `is not` with a literal: a SyntaxWarning on the + stderr of every worker, which a failing case's message was once taken from.""" + run_order(tmp, pyperf_order(param_tree, "quiet", case_filter="bench_sort|bench_plain"), "quiet") + drivers = sorted((tmp / "quiet").rglob("*.driver.py")) + assert len(drivers) == 5 # four combinations and a function without parameters + for driver in drivers: + with warnings.catch_warnings(): + warnings.simplefilter("error") + compile(driver.read_text(), str(driver), "exec") + + def test_params_that_cannot_be_read_matter_only_to_the_function_that_has_them(param_tree): with pytest.raises(adapter.Unsupported, match="cannot read the params of helper in mod.py: TypeError"): planned(param_tree, "helper") @@ -500,3 +532,13 @@ def test_relative_output_directory(tmp, tree, monkeypatch): runner.run(order_path, Path("rel-out")) docs = results(elsewhere / "rel-out") assert [d["measurement"]["status"] for d in docs] == ["success"] + + +def test_the_runner_records_the_order_the_parameters_were_declared_in(tmp, param_tree): + """The store keeps a document with its keys sorted, so `n, key` would come back as `key, n` + (PARAMETER_ORDER_PLAN.md): the result says in which order they were declared.""" + docs = run_order(tmp, pyperf_order(param_tree, "declared", case_filter="bench_sort|bench_plain"), "declared") + sort = [d for d in docs if d["coordinates"]["workload"]["name"] == "bench_sort"] + plain = [d for d in docs if d["coordinates"]["workload"]["name"] == "bench_plain"] + assert sort and all(d["provenance"]["info"]["parameter_names"] == ["n", "key"] for d in sort) + assert plain and all("parameter_names" not in d["provenance"]["info"] for d in plain) # nothing to put in order diff --git a/tests/test_tables.py b/tests/test_tables.py new file mode 100644 index 0000000..e87488a --- /dev/null +++ b/tests/test_tables.py @@ -0,0 +1,231 @@ +"""`bx compare --tables`: the asv-style tables (TABLES_PLAN.md), on hand-built results and through the CLI.""" + +import pytest + +from benchx import cli, compare, tables +from conftest import cases, make_build +from test_comparator import BASE, HEAD, result + +MEAN = "bench_a.Mean.time_mean_axis" +PLAIN = "bench_b.Plain.time_x" +WORKLOADS = ((MEAN, {"size": 1}, 5.2e-6), (MEAN, {"size": 10}, 5.6e-6), (PLAIN, None, 2.0e-3)) + + +def make(ratios=None, rounds=5, **changes): + """A `revisions` run of three cases over `rounds` rounds. The head takes `ratios[workload]` times + the base's time (default 1.05); `changes` apply to the head's results.""" + ratios = ratios or {} + docs, slot = [], 0 + for r in range(rounds): + for side in (BASE, HEAD): + for name, parameters, seconds in WORKLOADS: + ratio = ratios.get(name, 1.05) if side == HEAD else 1.0 + d = result(side, r, slot, seconds * (1 + 0.01 * r) * ratio, workload=name, **(changes if side == HEAD else {})) + if parameters: + d["coordinates"]["workload"]["parameters"] = parameters + d["ingest_key"] += f":{parameters}" + docs.append(d) + slot += 1 + return docs + + +def evaluate(docs): + return compare.compare(docs, run_key="run", profile="revisions", baseline=BASE) + + +@pytest.mark.parametrize("value, spread, unit, expected", [ + (5.2e-6, 8e-8, "s", "5.20±0.08μs"), + (96.3e-6, 1e-7, "s", "96.3±0.1μs"), + (97.6e-6, 2e-6, "s", "97.6±2μs"), + (2.08e-3, 6e-5, "s", "2.08±0.06ms"), + (1.5, None, "s", "1.50s"), + (100e-6, None, "s", "100μs"), + (10e-6, None, "s", "10.0μs"), + (3e-9, 1e-10, "s", "3.00±0.1ns"), + (5.2e-6, 4e-9, "s", "5.20±0μs"), # a spread below 0.005 of the unit shows as 0, as in asv + (90.0, 3.0, "s", "1.50±0.05m"), + (0.0, None, "s", "0s"), + (1500.0, 20.0, "B", "1500±20 B"), + (None, None, "s", "n/a"), +]) +def test_values_are_written_as_asv_writes_them(value, spread, unit, expected): + assert tables.human(value, spread, unit) == expected + + +def test_a_revisions_run_has_a_table_per_commit_and_one_of_changes(): + """asv's layout (format_text_table, tabulate "github"), the grids indented by two spaces: every cell centered, a column + its widest cell plus 2, lines not trimmed; Ratio and Noise right-aligned; the verdict repeated.""" + docs = make() + assert tables.render(evaluate(docs), docs).split("\n") == [ + "", + "For local/host commit aaaaaaaa (5 rounds):", + "bench_a.Mean.time_mean_axis ok", + " ====== =============", + " size ", + " ------ -------------", + " 1 5.30±0.08μs ", + " 10 5.71±0.09μs ", + " ====== =============", + "bench_b.Plain.time_x 2.04±0.03ms", + "", + "For local/host commit bbbbbbbb (5 rounds):", + "bench_a.Mean.time_mean_axis ok", + " ====== =============", + " size ", + " ------ -------------", + " 1 5.57±0.09μs ", + " 10 6.00±0.09μs ", + " ====== =============", + "bench_b.Plain.time_x 2.14±0.03ms", + "", + "| Change | Before [aaaaaaaa] | After [bbbbbbbb] | Ratio | Noise | Verdict | Benchmark (Parameter) |", + "|----------|---------------------|--------------------|---------|---------|-----------|---------------------------------|", + "| + | 5.30±0.08μs | 5.57±0.09μs | 1.05 | 0.00% | regressed | bench_a.Mean.time_mean_axis(1) |", + "| + | 5.71±0.09μs | 6.00±0.09μs | 1.05 | 0.00% | regressed | bench_a.Mean.time_mean_axis(10) |", + "| + | 2.04±0.03ms | 2.14±0.03ms | 1.05 | 0.00% | regressed | bench_b.Plain.time_x |", + ] + + +def test_the_noise_of_a_row_is_the_comparators(): + """Rounds whose ratios alternate 1.05 and 1.06: the paired differences .05, .06, .05, .06, .05 have + sd 0.005477, and the noise, their standard error, is sd / sqrt(5) = 0.2449%.""" + docs, slot = [], 0 + for r in range(5): + for side, ratio in ((BASE, 1.0), (HEAD, 1.05 + 0.01 * (r % 2))): + docs.append(result(side, r, slot, 2.0e-3 * ratio, workload=PLAIN)) + slot += 1 + doc = evaluate(docs) + assert doc["units"][0]["noise"] == pytest.approx(0.0024494897, rel=1e-6) + row = next(line for line in tables.render(doc, docs).splitlines() if line.startswith("| +")) + assert "| 0.24% | regressed |" in row + + +def test_only_changed_rows_unless_all(): + """As asv continuous does: the unchanged benchmark is listed only with --all, with a blank mark.""" + docs = make({PLAIN: 1.0}) + doc = evaluate(docs) + changes = tables.render(doc, docs).splitlines() + assert [line for line in changes if line.startswith("| ")][1:] and not any(PLAIN in line for line in changes if line.startswith("| ")) + everything = [line for line in tables.render(doc, docs, all_rows=True).splitlines() if line.startswith("| ")] + assert len(everything) == 4 # the header and three rows + assert everything[-1].startswith("| | 2.04±0.03ms") and "1.00" in everything[-1] and PLAIN in everything[-1] + assert "| no change detected |" in everything[-1] and "| regressed " in everything[1] + + +def test_a_run_without_changes_says_so(): + docs = make({MEAN: 1.0, PLAIN: 1.0}) + shown = tables.render(evaluate(docs), docs) + assert shown.endswith("No benchmark changed (the verdict is benchx's; --all lists every row).") + assert "| Change" not in shown + assert "| Change" in tables.render(evaluate(docs), docs, all_rows=True) + + +def test_an_improvement_is_marked_with_a_minus(): + docs = make({MEAN: 0.8}) + rows = [line for line in tables.render(evaluate(docs), docs).splitlines() if line.startswith("| -")] + assert len(rows) == 2 and all("0.80" in line and "| improved " in line for line in rows) + + +def test_a_side_that_failed_shows_its_status(): + docs = [d for d in make() if d["coordinates"]["workload"]["name"] != PLAIN or d["revision"]["key"] == BASE] + docs += [result(HEAD, r, 2 * r + 1, 1.0, workload=PLAIN, measurement__status="error") for r in range(5)] + for d in docs[-5:]: + d["ingest_key"] += ":failed" + doc = evaluate(docs) + assert next(u for u in doc["units"] if u["workload"]["name"] == PLAIN)["verdict"] == "indeterminate" + shown = tables.render(doc, docs, all_rows=True) + assert "bench_b.Plain.time_x error" in shown + assert "| ? | 2.04±0.03ms" in shown and "| error " in shown and "n/a" in shown + assert "| indeterminate (" in shown + + +def test_a_benchmark_that_raises_on_one_side_leaves_the_others_compared(): + """The failing benchmark reports no harness version or runtime. It shows its status in the tables and is + `indeterminate` in the change table; the other benchmarks have their rows.""" + docs = make() + for d in docs: + if d["revision"]["key"] == HEAD and d["coordinates"]["workload"]["name"] == PLAIN: + d["measurement"] = {"quantity": d["measurement"]["quantity"], "status": "error", "reason": "harness.error"} + d["coordinates"]["environment"]["system"]["runtime"] = {"python": "3.12"} + doc = evaluate(docs) + assert doc["failed_invariants"] == [] + shown = tables.render(doc, docs, all_rows=True) + assert "bench_b.Plain.time_x 2.04±0.03ms" in shown and "bench_b.Plain.time_x error" in shown + rows = [line for line in shown.splitlines() if line.startswith("| ")] + assert sum("| regressed " in line for line in rows) == 2 + assert any("bench_b.Plain.time_x" in line and "| ?" in line and "error" in line and "n/a" in line + and "indeterminate (insufficient-rounds)" in line for line in rows) + + +def test_an_edited_benchmark_has_no_row_but_the_others_do(): + """Its two sides ran different code, so it is not compared (`sides-agree`): no row in the change table, + while the table of each side still shows what it measured.""" + docs = make() + for d in docs: + if d["coordinates"]["workload"]["name"] == MEAN: + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": "v2" if d["revision"]["key"] == HEAD else "v1"} + doc = evaluate(docs) + assert [f["invariant"] for f in doc["failed_invariants"]] == ["sides-agree"] * 2 + shown = tables.render(doc, docs) + rows = [line for line in shown.splitlines() if line.startswith("| ")] + assert len(rows) == 2 and PLAIN in rows[1] and not any(MEAN in line for line in rows) + assert shown.count(f"{MEAN} ok") == 2 # the grid of each side + + +def test_a_dirty_side_is_named_in_its_heading(): + docs = make(provenance__subject_dirty="dirty") + shown = tables.render(evaluate(docs), docs) + assert "For local/host commit bbbbbbbb [dirty tree] (5 rounds):" in shown + assert "commit aaaaaaaa (5 rounds):" in shown + + +def test_a_comparison_without_units_has_no_tables(): + docs = [d for d in make() if d["revision"]["key"] == BASE] # one side: the two-sides invariant fails + doc = evaluate(docs) + assert doc["failed_invariants"] and tables.render(doc, docs) == "" + + +@pytest.mark.parametrize("profile, varying, axis", [ + ("environments", "provenance.labels.env", "/provenance/labels/env"), + ("variants", "coordinates.workload.parameters.role", "/coordinates/workload/parameters/role"), + ("revisions", "provenance.subject_tree", "/provenance/subject_tree"), +]) +def test_the_projection_is_rebuilt_from_the_document(profile, varying, axis): + assert tables._projection({"profile": profile, "varying": varying}).axis[0] == axis + + +# Through the CLI, on the fake Google Benchmark of the session tests. + +@pytest.fixture +def builds(repo, tmp): + base = make_build(tmp / "build-base", repo["wt-base"], cases()) + head = make_build(tmp / "build-head", repo["wt-head"], cases(quiet=1.02e-3, noisy=2.06e-3)) + return {"baseline": f"{base}:{repo['wt-base']}", "contender": f"{head}:{repo['wt-head']}"} + + +def test_tables_after_a_measure_mode_comparison_and_the_same_in_read_mode(repo, tmp, builds, capsys): + common = ["--profile", "revisions"] + assert cli.main(["compare", builds["baseline"], builds["contender"], *common, "--adapter", "google-benchmark", + "--suite", "demo-bench", "--rounds", "5", "--run-key", "t-1", "--out", str(tmp / "o"), + "--tables"]) == cli.EXIT_OK + measured = capsys.readouterr().out + assert measured.index("REGRESSED") < measured.index(f"commit {repo['base'][:8]}") < measured.index("| Change") + assert f"commit {repo['head'][:8]} (5 rounds):" in measured and "BM_Quiet/1024" in measured + assert f"| + " in measured and f"Before [{repo['base'][:8]}]" in measured + assert cli.main(["compare", "--run", "t-1", *common, "--baseline", repo["base"][:10], "--tables"]) == cli.EXIT_OK + assert capsys.readouterr().out == measured + + +def test_without_tables_the_output_is_what_it_was(repo, tmp, builds, capsys): + cli.main(["compare", builds["baseline"], builds["contender"], "--profile", "revisions", "--adapter", + "google-benchmark", "--suite", "demo-bench", "--rounds", "3", "--run-key", "t-2", "--out", str(tmp / "o")]) + assert "For " not in capsys.readouterr().out + + +@pytest.mark.parametrize("argv, message", [ + (["--run", "k", "--profile", "revisions", "--baseline", "x", "--tables", "--json"], "--tables is a text view"), + (["--run", "k", "--profile", "revisions", "--baseline", "x", "--all"], "--all goes with --tables"), +]) +def test_tables_flags_that_do_not_fit_are_refused(argv, message, capsys): + assert cli.main(["compare", *argv]) == cli.EXIT_REFUSED + assert message in capsys.readouterr().err diff --git a/tests/test_uc02.py b/tests/test_uc02.py index 4d70f3d..f8fb969 100644 --- a/tests/test_uc02.py +++ b/tests/test_uc02.py @@ -245,3 +245,13 @@ def test_measure_mode_refuses_a_baseline_that_is_not_a_value_before_running(tree assert code == cli.EXIT_REFUSED and doc is None assert "baseline 'variant' is not a value of role; use one of: 'baseline', 'variant'" in err assert not (tree / "results").exists() # nothing ran + + +def test_tables_name_the_sides_by_the_parameter_value(tree, capsys): + loop(["mod.py"], by_case([BASELINE, VARIANT]), PYPERF, rounds=3) + capsys.readouterr() + code = cli.main(["compare", "--run", RUN, "--profile", "variants", "--baseline", BASELINE, "--tables"]) + out = capsys.readouterr().out + assert code == cli.EXIT_OK + assert "role='baseline' (3 rounds):" in out and "role='variant' (3 rounds):" in out + assert f"Before [{BASELINE}]" in out and f"After [{VARIANT}]" in out and "| + " in out diff --git a/tests/test_uc04.py b/tests/test_uc04.py index 0d7fdfb..1bc0ab0 100644 --- a/tests/test_uc04.py +++ b/tests/test_uc04.py @@ -130,3 +130,15 @@ def test_a_failed_check_is_shown_in_the_comparison_and_changes_nothing_else(tree assert "contender: collector-mismatch" in caveats # beside any dirty-tree caveats the fixture tree earns assert "baseline: collector-mismatch" not in out assert "failed invariants" not in out and "bench_a" in out + + +def test_tables_name_the_sides_by_their_label(tree, monkeypatch, capsys): + write_config(tree, {}) + loop({"a": {"dep": "1.0"}, "b": {"dep": "2.0"}}, monkeypatch) + capsys.readouterr() + code = cli.main(["compare", "--run", "S", "--profile", "environments", "--label", "env", "--baseline", "a", + "--tables"]) + out = capsys.readouterr().out + assert code == cli.EXIT_OK + assert "env=a" in out and "env=b" in out and out.count("(3 rounds):") == 2 + assert "Before [a]" in out or "No benchmark changed" in out