From 49cb2cfcf80ad87c9fa779831ab619b8fcfa212f Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 18 Nov 2026 08:57:01 +0000 Subject: [PATCH 1/9] ENH: let one adapter block of benchx.conf.json serve pyperf and asv asv is timed by pyperf and takes the same precision, but `bx bench --adapter asv` was refused when the file's `adapter.name` was `pyperf`, so one benchx.conf.json could not hold the timing for both kinds of suite. Harnesses now have a precision family: pyperf and asv (which subclasses it) are one, google-benchmark is another. `--adapter` may name any adapter of the file's block family. Across families it is still refused, with a message that says so. No schema change: `adapter.name` stays required, and only its description is corrected (it did not mention asv). Co-Authored-By: Claude Sonnet 5.5 --- src/benchx/cli.py | 8 ++-- src/benchx/harnesses.py | 10 +++++ src/benchx/schemas/config/0.1.0/schema.json | 2 +- tests/test_config.py | 47 +++++++++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/src/benchx/cli.py b/src/benchx/cli.py index a651d9f..645f0b8 100644 --- a/src/benchx/cli.py +++ b/src/benchx/cli.py @@ -126,12 +126,14 @@ def _common(cfg, args, *, forced_adapter=None, forced_suite=None) -> dict: """The settings `bx bench` and `bx compare` share, each from the command line, else the file, else its default (config.Config.pick); ConfigError when a required one has none. The file's adapter block belongs to the adapter it - names, so naming another is refused.""" + names and to the others of its family (pyperf's serves asv), so naming an + adapter of another family is refused.""" block = cfg.adapter given = forced_adapter or args.adapter - if given and block.get("name") and given != block["name"]: + if given and block.get("name") and not harnesses.same_family(given, block["name"]): raise config_mod.ConfigError(f"adapter {given!r}, but {cfg.where} sets adapter.name to {block['name']!r}, " - "and its suite and precision belong to that harness") + "and its precision belongs to that harness and the ones that share it " + "(pyperf and asv)") quantities = config_mod.parse_quantities(args.quantity) if args.quantity else block.get("quantities", ["wall-time"]) if args.project and args.no_project: raise UsageError("give --project or --no-project, not both") diff --git a/src/benchx/harnesses.py b/src/benchx/harnesses.py index 4fa05ca..d8f8a04 100644 --- a/src/benchx/harnesses.py +++ b/src/benchx/harnesses.py @@ -43,6 +43,7 @@ def exact_filter(names, arguments=False) -> str | None: class Harness: name: str + FAMILY: str # harnesses of one family take the same precision, so one `adapter` block of the config serves all FLAGS: tuple[str, ...] = () BARE_PATH_KIND: str NAMES_TAKE_ARGUMENTS = False @@ -83,6 +84,7 @@ class GoogleBenchmark(Harness): Google Benchmark's default, and the order states it.""" name = "google-benchmark" + FAMILY = "google-benchmark" FLAGS = ("repetitions", "min_time") BARE_PATH_KIND = "build" # the binary is built; BUILD_DIR[:SOURCE_DIR] DEFAULT_REPETITIONS = adapters.gbench.DEFAULT_REPETITIONS @@ -105,6 +107,7 @@ class Pyperf(Harness): flags leave out.""" name = "pyperf" + FAMILY = "pyperf" # asv, timed by pyperf, inherits it FLAGS = ("processes", "values", "min_time", "loops", "warmups") BARE_PATH_KIND = "working_tree" # the module is run as found; the path is its own source NAMES_TAKE_ARGUMENTS = True # a function with params is one case per combination @@ -150,6 +153,13 @@ def describe_precision(self, precision, config_path=None): FLAG_NAMES = tuple(dict.fromkeys(name for h in HARNESSES.values() for name in h.FLAGS)) +def same_family(a: str, b: str) -> bool: + """Whether harnesses `a` and `b` take the same precision, as pyperf and asv do (asv is timed by + pyperf). An unknown name is in no family.""" + first, second = HARNESSES.get(a), HARNESSES.get(b) + return first is not None and second is not None and first.FAMILY == second.FAMILY + + def get(name: str) -> Harness: """The harness registered under `name`; SessionError if there is none.""" try: diff --git a/src/benchx/schemas/config/0.1.0/schema.json b/src/benchx/schemas/config/0.1.0/schema.json index f9f5007..8918112 100644 --- a/src/benchx/schemas/config/0.1.0/schema.json +++ b/src/benchx/schemas/config/0.1.0/schema.json @@ -36,7 +36,7 @@ "additionalProperties": false, "required": ["name"], "properties": { - "name": { "description": "pyperf or google-benchmark.", "type": "string", "minLength": 1 }, + "name": { "description": "pyperf, asv or google-benchmark. The block's precision belongs to this adapter and to the others that take the same precision: a pyperf block also serves `--adapter asv`, which is timed by pyperf.", "type": "string", "minLength": 1 }, "suite": { "description": "The benchmark program in the target: pyperf, the module's path in the working tree; google-benchmark, the binary in the build directory.", "type": "string", diff --git a/tests/test_config.py b/tests/test_config.py index c12e38d..b7b6f18 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -179,3 +179,50 @@ def test_show_says_where_each_value_comes_from(tree, monkeypatch, capsys): assert '"demo" (file)' in out and '"mod.py" (file)' in out assert "compare.rounds - (unset)" in out assert "timeouts.case_s 60 (default)" in out + + +# --- one file for the adapters that share a precision --------------------------------- + +TIMING = {"quantities": ["wall-time"], + "precision": {"repetitions": {"mode": "fixed", "levels": [{"unit": "process", "n": 5}, + {"unit": "value", "n": 3}]}, + "calibration": {"mode": "adaptive", "minimum_sample_seconds": 0.1}, + "warmup": {"mode": "count", "n_warmup": 1}}} + + +def settle(tree, monkeypatch, *argv): + """What `bx bench` settles on, from the file in `tree` and `argv`, without running anything.""" + seen = [] + monkeypatch.setattr(cli, "cmd_bench", lambda args: seen.append(args) or cli.EXIT_OK) + monkeypatch.chdir(tree) + assert cli.main(["bench", "TARGET", *argv]) == cli.EXIT_OK + return cli._common(config.load(tree), seen[0]) + + +def test_asv_is_timed_by_pyperf_so_the_two_share_a_family(): + assert harnesses.same_family("pyperf", "asv") and harnesses.same_family("asv", "pyperf") + assert harnesses.same_family("asv", "asv") and harnesses.same_family("google-benchmark", "google-benchmark") + assert not harnesses.same_family("pyperf", "google-benchmark") + assert not harnesses.same_family("asv", "google-benchmark") + assert not harnesses.same_family("pyperf", "nonesuch") and not harnesses.same_family("nonesuch", "pyperf") + + +@pytest.mark.parametrize("named, given", [("pyperf", "asv"), ("asv", "pyperf"), ("pyperf", "pyperf"), ("asv", "asv")]) +def test_a_file_serves_the_adapters_that_share_its_precision(tree, monkeypatch, named, given): + write_config(tree, adapter={"name": named, **TIMING}) + settings = settle(tree, monkeypatch, "--adapter", given, "--suite", "s") + assert settings["adapter"] == given and settings["suite"] == "s" + assert levels(harnesses.get(given).precision({}, settings["file_precision"])) == {"process": 5, "value": 3} + + +def test_without_an_adapter_flag_the_files_adapter_is_used(tree, monkeypatch): + write_config(tree, adapter={"name": "asv", **TIMING}) + assert settle(tree, monkeypatch, "--suite", "s")["adapter"] == "asv" + + +@pytest.mark.parametrize("named, given", [("google-benchmark", "pyperf"), ("google-benchmark", "asv"), + ("pyperf", "google-benchmark"), ("asv", "google-benchmark")]) +def test_an_adapter_of_another_family_is_refused(tree, monkeypatch, named, given): + write_config(tree, adapter={"name": named}) + with pytest.raises(config.ConfigError, match=rf"adapter {given!r}, but .* sets adapter\.name to {named!r}.*precision"): + settle(tree, monkeypatch, "--adapter", given, "--suite", "s") From 56adac8411bd832b52cd637213d04c8cac658890 Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 18 Nov 2026 09:04:14 +0000 Subject: [PATCH 2/9] ENH: add asv-style tables to `bx compare` (--tables, --all) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `bx compare --tables` prints, after the usual summary, what `spin bench --compare` (asv) prints: a table of each side's results, then a table of the changes with Ratio, Noise and the comparator's verdict. As with `asv continuous`, only the changed rows are listed unless `--all` is given. The layout and the number formatting follow asv's (`format_text_table`: every cell centered; `tabulate`'s github format; `human_time` and `human_float` for values). The `±` is benchx's own: the standard deviation of the per-round medians, and a side's value is their median. The per-side numbers are recomputed from the run's results for display, so the comparison document and its schema are unchanged. `--tables` is a text view: it is refused together with `--json`. Co-Authored-By: Claude Sonnet 5.5 --- docs/benchx.md | 13 +- docs/design/prototype-design.md | 4 +- src/benchx/cli.py | 28 ++++- src/benchx/tables.py | 210 ++++++++++++++++++++++++++++++++ tests/test_tables.py | 198 ++++++++++++++++++++++++++++++ tests/test_uc02.py | 10 ++ tests/test_uc04.py | 12 ++ 7 files changed, 469 insertions(+), 6 deletions(-) create mode 100644 src/benchx/tables.py create mode 100644 tests/test_tables.py diff --git a/docs/benchx.md b/docs/benchx.md index b6ad371..2ba68ef 100644 --- a/docs/benchx.md +++ b/docs/benchx.md @@ -55,7 +55,7 @@ bx compare BASELINE CONTENDER --profile revisions|environments --suite NAME [--filter REGEX] [--quantity wall-time] --rounds R [-e NAME=VALUE ...] [--repetitions N] [--min-time S] [--project P] [--label NAME=BASE,CONTENDER] - [--run-key KEY] [--out DIR] [--no-build] [--k 3] [--json] + [--run-key KEY] [--out DIR] [--no-build] [--k 3] [--json | --tables [--all]] # measure mode, one target: alternate the two values of a benchmark parameter bx compare TARGET --profile variants --adapter pyperf|asv --suite NAME --rounds R @@ -63,7 +63,7 @@ bx compare TARGET --profile variants --adapter pyperf|asv --suite NAME --rounds # read mode: compare results already measured bx compare --run KEY --profile revisions|environments|variants --baseline VALUE - [--label NAME] [--vary NAME] [--results DIR] [--k 3] [--json] + [--label NAME] [--vary NAME] [--results DIR] [--k 3] [--json | --tables [--all]] # low-level command: run a handwritten work oreder bx run [--out DIR] [--no-ingest] @@ -263,6 +263,15 @@ every result is pyperf. Results also record the installed project's version (`side-failed`). `--results` compares result files through a throwaway store: the ad hoc path. Condition measurements (`measurement.kind: condition`) are not compared, and deterministic quantities are deferred. +- **The tables** (`--tables`, `benchx/tables.py`) print, after the usual summary, what + `asv continuous` prints, laid out and rounded as asv does: a table of each side (a value + `4.71±0.08μs` per parameter combination, every cell centered), then one of the changes + (Change, Before, After, Ratio, Noise, Verdict, Benchmark), only the changed rows unless `--all`. + They are recomputed for display from the run's results; the comparison document is the + same. A side's value is the median of its per-round medians and its `±` their standard + deviation (not asv's `±`); `Change` and `Verdict` are the comparator's verdict (`+` + regressed, `-` improved), `Noise` the one it compares the effect with (a change is more than *k* + times it), and `Ratio` is After over Before. A text view: not with `--json`. ## Run it diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index 04e267b..d0dbe6e 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -21,7 +21,7 @@ bx series [--workload … --quantity …] # list series bx history # one series' points (a query) bx head [-n 10] [--project P] [--workload W] [--json|--table] # the latest results, newest first bx compare --run KEY --profile revisions|environments|variants --baseline VALUE - [--label NAME] [--vary NAME] [--results DIR] [--k 3] [--min-rounds 3] [--json] + [--label NAME] [--vary NAME] [--results DIR] [--k 3] [--min-rounds 3] [--json | --tables [--all]] # read mode: compare results already measured bx target describe|prepare TARGET # a target's description (a provider's answer, or a build directory's sidecar) bx compare BASELINE CONTENDER --profile revisions|environments @@ -33,7 +33,7 @@ bx compare BASELINE CONTENDER --profile revisions|environments [--min-time S] [--project P | --no-project] [--label NAME=BASE,CONTENDER] [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI] - [--k 3] [--min-rounds 3] [--json] + [--k 3] [--min-rounds 3] [--json | --tables [--all]] # measure mode: run both sides, then compare bx compare TARGET --profile variants [--vary NAME] [--baseline VALUE] --suite NAME --rounds R [--filter REGEX] [...] # the options above, except --label diff --git a/src/benchx/cli.py b/src/benchx/cli.py index 645f0b8..1e1cd14 100644 --- a/src/benchx/cli.py +++ b/src/benchx/cli.py @@ -18,6 +18,7 @@ from . import compare as compare_mod from . import config as config_mod from . import core, harnesses, orders, runner, session, snapshot +from . import tables as tables_mod from . import target as target_mod from .core.validation import Kind from .store import Store, warning_codes @@ -387,6 +388,7 @@ def cmd_compare(args): policy = _policy(cfg, args) mode = _mode(args, policy["profile"]) _profile_flags(args, policy["profile"]) + _view_flags(args) except (UsageError, config_mod.ConfigError) as e: print(f"bx compare: error: {e}", file=sys.stderr) return EXIT_REFUSED @@ -414,6 +416,14 @@ def _profile_flags(args, profile): "comparison are two values of a parameter (--vary)") +def _view_flags(args): + """--tables is a text view of the comparison, and --all a switch of it.""" + if args.tables and args.json: + raise UsageError("--tables is a text view: not with --json") + if args.all and not args.tables: + raise UsageError("--all goes with --tables") + + def _read(cfg, args, policy): if args.results: with tempfile.TemporaryDirectory() as tmp: # the ad hoc path: a throwaway store @@ -451,10 +461,20 @@ def _measure(cfg, args, policy): except (session.SessionError, compare_mod.CompareError) as e: print(f"cannot compare: {e}", file=sys.stderr) return EXIT_REFUSED - print(json.dumps(doc, indent=1) if args.json else compare_mod.render(doc)) + _show(doc, args, store.documents(doc["run_key"]) if args.tables else None) return EXIT_OK +def _show(doc, args, documents): + """The comparison as JSON, or as text; with --tables, the asv-style tables follow the summary.""" + if args.json: + print(json.dumps(doc, indent=1)) + return + print(compare_mod.render(doc)) + if args.tables: + print(tables_mod.render(doc, documents, all_rows=args.all)) + + def _compare(store, args, policy): try: documents = store.documents(args.run) @@ -469,7 +489,7 @@ def _compare(store, args, policy): except compare_mod.CompareError as e: print(f"cannot compare: {e}", file=sys.stderr) return EXIT_REFUSED - print(json.dumps(doc, indent=1) if args.json else compare_mod.render(doc)) + _show(doc, args, documents) return EXIT_OK @@ -627,6 +647,10 @@ def main(argv=None): p.add_argument("--k", type=float, help="noise multiplier (default: compare.k, else 3)") p.add_argument("--min-rounds", type=int, help="fewest paired rounds for a verdict (default: compare.min_rounds, else 3)") p.add_argument("--json", action="store_true") + p.add_argument("--tables", action="store_true", + help="after the summary, asv-style tables: one per side, then the changes") + p.add_argument("--all", action="store_true", + help="with --tables: every row of the change table, not only the changed ones (as asv continuous)") p.add_argument("--label", help="read mode: the label that names the sides (environments); " "measure mode: NAME=BASE,CONTENDER") p.add_argument("--baseline", help="revisions: a revision or tree id prefix; environments: the label value; " diff --git a/src/benchx/tables.py b/src/benchx/tables.py new file mode 100644 index 0000000..9dfec3d --- /dev/null +++ b/src/benchx/tables.py @@ -0,0 +1,210 @@ +"""asv-style tables for `bx compare --tables` (TABLES_PLAN.md): one table per side, then the changes. + +Display only. The per-side numbers are recomputed here from the run's stored results, with the +projection that made the comparison document, so the document and its schema are unchanged. A +side's value is the median of its per-round medians (the estimator the verdict uses) and its `±` +the sd of those medians: benchx's own meaning, not asv's. +""" + +import math +import statistics + +from .compare import WORKLOAD, _c, _estimate, _run_documents, _unit_order, _workload, profile_projection +from .snapshot import DIRTY_WARNINGS +from .store import store_project + +ROLES = ("baseline", "contender") +CHANGED = {"regressed": "+", "improved": "-", "changed": "~"} # as asv continuous marks them +MARKS = {**CHANGED, "no-change-detected": " ", "indeterminate": "?"} +SHA = 8 # asv's width +# asv_runner.util._human_time_units: the unit is the largest one the value does not reach. +TIME_UNITS = (("ns", 1e-9), ("μs", 1e-6), ("ms", 1e-3), ("s", 1), ("m", 60), ("h", 3600), ("d", 86400), + ("w", 604800), ("M", 2620800), ("q", 7862400), ("y", 31449600), ("C", 3144960000)) + + +def _float(value: float, significant: int, truncate_small: int | None = None, zeros: bool = False) -> str: + """`value` to `significant` digits, as asv's `human_float`: scientific notation outside 1e-5 and 1e9; + trailing zeros dropped unless `zeros`, which also pads `3` to `3.00`; with `truncate_small`, the + zeros after the point of a number below 1 count as digits, up to that many.""" + if value == 0: + return "0" + if math.isinf(value) or math.isnan(value): + return str(value) + sign, value = ("-", -value) if value < 0 else ("", value) + magnitude = math.floor(math.log10(value)) + 1 + if truncate_small is not None: + magnitude = max(magnitude, 1 - truncate_small) + decimals = significant - magnitude + scientific = magnitude <= -5 or magnitude >= 9 + if scientific: + text = f"{value:.{significant}e}" + elif value == int(value) or decimals <= 0: + text = str(int(round(value, decimals))) + else: + text = f"{value:.{decimals}f}" + if not zeros and "." in text and not scientific: + text = text.rstrip("0").removesuffix(".") + text = sign + text + if zeros and "." not in text and len(text) < significant: + text += "." + "0" * (significant - len(text)) + return text + + +def human(value, spread, unit) -> str: + """`4.71±0.08μs`, as asv writes a time (`human_time`): the value to 3 digits with its zeros, the + spread to 1. Any other unit is written after the number.""" + if value is None or value != value: + return "n/a" + if unit != "s": + return _float(value, 3) + (f"±{_float(spread, 1, 2)}" if spread is not None else "") + f" {unit}" + scale = spread if value == 0 and spread is not None else value + units = TIME_UNITS if scale != 0 else (("s", 1), ("m", 60)) + for (name, size), (_, bigger) in zip(units, units[1:]): + if scale < bigger: + return (_float(value / size, 3, zeros=True) + + (f"±{_float(spread / size, 1, truncate_small=2)}" if spread is not None else "") + name) + return "~0" + + +def _projection(doc): + """The projection that made `doc`: its `varying` names the label or parameter that names the sides.""" + varying = doc["varying"] + label = varying.removeprefix("provenance.labels.") if varying.startswith("provenance.labels.") else None + parameter = (varying.removeprefix("coordinates.workload.parameters.") + if varying.startswith("coordinates.workload.parameters.") else None) + return profile_projection(doc["profile"], label, parameter) + + +def _side(docs) -> dict: + """One side of one unit: the median of its per-round medians, their sd, and what went wrong if + no round succeeded.""" + medians = [m for d in docs if d["measurement"]["status"] == "success" and (m := _estimate(d)) is not None] + failed = sorted({d["measurement"]["status"] for d in docs if d["measurement"]["status"] != "success"}) + return {"value": statistics.median(medians) if medians else None, + "spread": statistics.stdev(medians) if len(medians) > 1 else None, + "status": "ok" if medians else (failed[0] if failed else "n/a")} + + +def summarize(doc, documents) -> tuple[list[dict], dict]: + """The comparison's units, sorted as the document lists them, each with its workload, quantity and + the two sides' numbers; and what each side is (commit, project, rounds, dirty).""" + projection = _projection(doc) + names = {_c(doc["baseline"]): "baseline", _c(doc["contender"]): "contender"} + grouped, sides = {}, {role: [] for role in ROLES} + for d in _run_documents(documents, doc["run_key"]): + projected = projection.apply(d, store_project(d)) + role = names.get(_c(projected.axis[0])) + if role is None: + continue + variant = {ptr: v for ptr, v in projected.identity.items() if ptr == WORKLOAD or ptr.startswith(WORKLOAD + "/")} + quantity = d["measurement"]["quantity"] + unit = grouped.setdefault((_c(variant), quantity["name"]), + {"quantity": quantity, "docs": {r: [] for r in ROLES}}) + unit["docs"][role].append(d) + sides[role].append(d) + units = [] + for unit in grouped.values(): + units.append({"workload": _workload(unit["docs"], projection.axis), "quantity": unit["quantity"], + **{role: _side(unit["docs"][role]) for role in ROLES}}) + units.sort(key=lambda u: _unit_order(u["workload"], u["quantity"]["name"])) + varying = doc["varying"].rsplit(".", 1)[-1] # the label or parameter that names the sides + info = {} + for role, docs in sides.items(): + project = store_project(docs[0]) if docs else "" + commit = docs[0]["revision"]["key"][:SHA] if docs else "?" + revisions = doc["profile"] == "revisions" # its sides are two commits; the others', two values + info[role] = {"tag": commit if revisions else str(doc[role]), + "heading": f"{project} commit {commit}" if revisions else f"{project} {varying}={doc[role]}", + "rounds": len({d.get("procedure", {}).get("round") for d in docs}), + "dirty": any(d["provenance"].get(flag) == "dirty" for d in docs for flag, _ in DIRTY_WARNINGS)} + return units, info + + +def _grid(headers, rows) -> list[str]: + """asv's table (`format_text_table`): each column its widest cell plus 2, every cell centered, a + space between columns, `=` borders and `-` under the header. Lines are not trimmed.""" + cells = [[str(cell) for cell in row] for row in (headers, *rows)] + widths = [max(len(row[j]) for row in cells) + 2 for j in range(len(headers))] + lines = [" ".join(cell.center(width) for cell, width in zip(row, widths)) for row in cells] + rule = lambda char: " ".join(char * width for width in widths) # noqa: E731 + return [rule("="), lines[0], rule("-"), *lines[1:], rule("=")] + + +def _cell(unit, role) -> str: + """A side's value, or why it has none.""" + side = unit[role] + return human(side["value"], side["spread"], unit["quantity"].get("unit")) if side["value"] is not None \ + else side["status"] + + +def side_tables(units, role, info) -> str: + """The tables of one side: a block per benchmark, a row per parameter combination.""" + side = info[role] + lines = [f"For {side['heading']}" + (" [dirty tree]" if side["dirty"] else "") + f" ({side['rounds']} rounds):"] + benchmarks = {} + for unit in units: + benchmarks.setdefault((unit["workload"]["name"], unit["quantity"]["name"]), []).append(unit) + for (name, quantity), group in benchmarks.items(): + title = name + ("" if quantity == "wall-time" else f" ({quantity})") + names = list(dict.fromkeys(p for u in group for p in (u["workload"].get("parameters") or {}))) + cell = lambda u: _cell(u, role) # noqa: E731 + if not names: + lines.append(f"{title} {cell(group[0])}") + continue + bad = [u[role]["status"] for u in group if u[role]["status"] != "ok"] + lines.append(f"{title} {bad[0] if bad else 'ok'}") + rows = [[(u["workload"].get("parameters") or {}).get(n, "") for n in names] + [cell(u)] for u in group] + lines.extend(_grid([*names, ""], rows)) + return "\n".join(lines) + + +def _label(workload) -> str: + """asv's `name(p1, p2)`.""" + parameters = workload.get("parameters") or {} + return workload["name"] + (f"({', '.join(str(v) for v in parameters.values())})" if parameters else "") + + +def _pipe(headers, rows, right=()) -> str: + """asv's `tabulate` "github" table: a column is as wide as its widest cell or its header plus 2; + text is left-aligned, and the `right` columns (numbers) right-aligned.""" + widths = [max(len(header) + 2, *(len(row[j]) for row in rows)) for j, header in enumerate(headers)] + align = lambda cell, j: cell.rjust(widths[j]) if j in right else cell.ljust(widths[j]) # noqa: E731 + line = lambda cells: "| " + " | ".join(align(cell, j) for j, cell in enumerate(cells)) + " |" # noqa: E731 + return "\n".join([line(headers), "|" + "|".join("-" * (width + 2) for width in widths) + "|", + *(line(row) for row in rows)]) + + +def _verdict(unit) -> str: + """The comparator's verdict in the words of its summary, with the reason when there is no verdict.""" + return unit["verdict"].replace("-", " ") + (f" ({unit['reason']})" if unit.get("reason") else "") + + +def change_table(doc, units, info, *, all_rows=False) -> str: + """asv's comparison table, and the verdict. As `asv continuous` does, only the rows that + changed, unless `all_rows`.""" + by_workload = {(_c(u["workload"]), u["quantity"]["name"]): u for u in units} + rows = [] + for verdict_unit in doc["units"]: + mark = MARKS[verdict_unit["verdict"]] + if mark not in CHANGED.values() and not all_rows: + continue + unit = by_workload[(_c(verdict_unit["workload"]), verdict_unit["quantity"])] + before, after = unit["baseline"], unit["contender"] + ratio = f"{after['value'] / before['value']:.2f}" if before["value"] and after["value"] is not None else "n/a" + noise = f"{verdict_unit['noise']:.2%}" if "noise" in verdict_unit else "n/a" + rows.append([mark, *(_cell(unit, role) for role in ROLES), ratio, noise, _verdict(verdict_unit), + _label(unit["workload"])]) + if not rows: + return "No benchmark changed (the verdict is benchx's; --all lists every row)." + headers = ["Change", f"Before [{info['baseline']['tag']}]", f"After [{info['contender']['tag']}]", + "Ratio", "Noise", "Verdict", "Benchmark (Parameter)"] + return _pipe(headers, rows, right={j for j in (3, 4) if all(row[j] != "n/a" for row in rows)}) + + +def render(doc, documents, *, all_rows=False) -> str: + """The tables of a comparison, or nothing when the document has no units (its invariants failed).""" + if not doc.get("units") or "contender" not in doc: + return "" + units, info = summarize(doc, documents) + return "\n" + "\n\n".join([*(side_tables(units, role, info) for role in ROLES), + change_table(doc, units, info, all_rows=all_rows)]) diff --git a/tests/test_tables.py b/tests/test_tables.py new file mode 100644 index 0000000..e532ec2 --- /dev/null +++ b/tests/test_tables.py @@ -0,0 +1,198 @@ +"""`bx compare --tables`: the asv-style tables (TABLES_PLAN.md), on hand-built results and through the CLI.""" + +import pytest + +from benchx import cli, compare, tables +from conftest import cases, make_build +from test_comparator import BASE, HEAD, result + +MEAN = "bench_a.Mean.time_mean_axis" +PLAIN = "bench_b.Plain.time_x" +WORKLOADS = ((MEAN, {"size": 1}, 5.2e-6), (MEAN, {"size": 10}, 5.6e-6), (PLAIN, None, 2.0e-3)) + + +def make(ratios=None, rounds=5, **changes): + """A `revisions` run of three cases over `rounds` rounds. The head takes `ratios[workload]` times + the base's time (default 1.05); `changes` apply to the head's results.""" + ratios = ratios or {} + docs, slot = [], 0 + for r in range(rounds): + for side in (BASE, HEAD): + for name, parameters, seconds in WORKLOADS: + ratio = ratios.get(name, 1.05) if side == HEAD else 1.0 + d = result(side, r, slot, seconds * (1 + 0.01 * r) * ratio, workload=name, **(changes if side == HEAD else {})) + if parameters: + d["coordinates"]["workload"]["parameters"] = parameters + d["ingest_key"] += f":{parameters}" + docs.append(d) + slot += 1 + return docs + + +def evaluate(docs): + return compare.compare(docs, run_key="run", profile="revisions", baseline=BASE) + + +@pytest.mark.parametrize("value, spread, unit, expected", [ + (5.2e-6, 8e-8, "s", "5.20±0.08μs"), + (96.3e-6, 1e-7, "s", "96.3±0.1μs"), + (97.6e-6, 2e-6, "s", "97.6±2μs"), + (2.08e-3, 6e-5, "s", "2.08±0.06ms"), + (1.5, None, "s", "1.50s"), + (100e-6, None, "s", "100μs"), + (10e-6, None, "s", "10.0μs"), + (3e-9, 1e-10, "s", "3.00±0.1ns"), + (5.2e-6, 4e-9, "s", "5.20±0μs"), # a spread below 0.005 of the unit shows as 0, as in asv + (90.0, 3.0, "s", "1.50±0.05m"), + (0.0, None, "s", "0s"), + (1500.0, 20.0, "B", "1500±20 B"), + (None, None, "s", "n/a"), +]) +def test_values_are_written_as_asv_writes_them(value, spread, unit, expected): + assert tables.human(value, spread, unit) == expected + + +def test_a_revisions_run_has_a_table_per_commit_and_one_of_changes(): + """asv's layout (format_text_table, tabulate "github"): every cell of a side's table centered, a column + its widest cell plus 2, lines not trimmed; Ratio and Noise right-aligned; the verdict repeated.""" + docs = make() + assert tables.render(evaluate(docs), docs).split("\n") == [ + "", + "For local/host commit aaaaaaaa (5 rounds):", + "bench_a.Mean.time_mean_axis ok", + "====== =============", + " size ", + "------ -------------", + " 1 5.30±0.08μs ", + " 10 5.71±0.09μs ", + "====== =============", + "bench_b.Plain.time_x 2.04±0.03ms", + "", + "For local/host commit bbbbbbbb (5 rounds):", + "bench_a.Mean.time_mean_axis ok", + "====== =============", + " size ", + "------ -------------", + " 1 5.57±0.09μs ", + " 10 6.00±0.09μs ", + "====== =============", + "bench_b.Plain.time_x 2.14±0.03ms", + "", + "| Change | Before [aaaaaaaa] | After [bbbbbbbb] | Ratio | Noise | Verdict | Benchmark (Parameter) |", + "|----------|---------------------|--------------------|---------|---------|-----------|---------------------------------|", + "| + | 5.30±0.08μs | 5.57±0.09μs | 1.05 | 0.00% | regressed | bench_a.Mean.time_mean_axis(1) |", + "| + | 5.71±0.09μs | 6.00±0.09μs | 1.05 | 0.00% | regressed | bench_a.Mean.time_mean_axis(10) |", + "| + | 2.04±0.03ms | 2.14±0.03ms | 1.05 | 0.00% | regressed | bench_b.Plain.time_x |", + ] + + +def test_the_noise_of_a_row_is_the_comparators(): + """Rounds whose ratios alternate 1.05 and 1.06: the paired differences .05, .06, .05, .06, .05 have + sd 0.005477, and the noise, their standard error, is sd / sqrt(5) = 0.2449%.""" + docs, slot = [], 0 + for r in range(5): + for side, ratio in ((BASE, 1.0), (HEAD, 1.05 + 0.01 * (r % 2))): + docs.append(result(side, r, slot, 2.0e-3 * ratio, workload=PLAIN)) + slot += 1 + doc = evaluate(docs) + assert doc["units"][0]["noise"] == pytest.approx(0.0024494897, rel=1e-6) + row = next(line for line in tables.render(doc, docs).splitlines() if line.startswith("| +")) + assert "| 0.24% | regressed |" in row + + +def test_only_changed_rows_unless_all(): + """As asv continuous does: the unchanged benchmark is listed only with --all, with a blank mark.""" + docs = make({PLAIN: 1.0}) + doc = evaluate(docs) + changes = tables.render(doc, docs).splitlines() + assert [line for line in changes if line.startswith("| ")][1:] and not any(PLAIN in line for line in changes if line.startswith("| ")) + everything = [line for line in tables.render(doc, docs, all_rows=True).splitlines() if line.startswith("| ")] + assert len(everything) == 4 # the header and three rows + assert everything[-1].startswith("| | 2.04±0.03ms") and "1.00" in everything[-1] and PLAIN in everything[-1] + assert "| no change detected |" in everything[-1] and "| regressed " in everything[1] + + +def test_a_run_without_changes_says_so(): + docs = make({MEAN: 1.0, PLAIN: 1.0}) + shown = tables.render(evaluate(docs), docs) + assert shown.endswith("No benchmark changed (the verdict is benchx's; --all lists every row).") + assert "| Change" not in shown + assert "| Change" in tables.render(evaluate(docs), docs, all_rows=True) + + +def test_an_improvement_is_marked_with_a_minus(): + docs = make({MEAN: 0.8}) + rows = [line for line in tables.render(evaluate(docs), docs).splitlines() if line.startswith("| -")] + assert len(rows) == 2 and all("0.80" in line and "| improved " in line for line in rows) + + +def test_a_side_that_failed_shows_its_status(): + docs = [d for d in make() if d["coordinates"]["workload"]["name"] != PLAIN or d["revision"]["key"] == BASE] + docs += [result(HEAD, r, 2 * r + 1, 1.0, workload=PLAIN, measurement__status="error") for r in range(5)] + for d in docs[-5:]: + d["ingest_key"] += ":failed" + doc = evaluate(docs) + assert next(u for u in doc["units"] if u["workload"]["name"] == PLAIN)["verdict"] == "indeterminate" + shown = tables.render(doc, docs, all_rows=True) + assert "bench_b.Plain.time_x error" in shown + assert "| ? | 2.04±0.03ms" in shown and "| error " in shown and "n/a" in shown + assert "| indeterminate (" in shown + + +def test_a_dirty_side_is_named_in_its_heading(): + docs = make(provenance__subject_dirty="dirty") + shown = tables.render(evaluate(docs), docs) + assert "For local/host commit bbbbbbbb [dirty tree] (5 rounds):" in shown + assert "commit aaaaaaaa (5 rounds):" in shown + + +def test_a_comparison_without_units_has_no_tables(): + docs = [d for d in make() if d["revision"]["key"] == BASE] # one side: the two-sides invariant fails + doc = evaluate(docs) + assert doc["failed_invariants"] and tables.render(doc, docs) == "" + + +@pytest.mark.parametrize("profile, varying, axis", [ + ("environments", "provenance.labels.env", "/provenance/labels/env"), + ("variants", "coordinates.workload.parameters.role", "/coordinates/workload/parameters/role"), + ("revisions", "provenance.subject_tree", "/provenance/subject_tree"), +]) +def test_the_projection_is_rebuilt_from_the_document(profile, varying, axis): + assert tables._projection({"profile": profile, "varying": varying}).axis[0] == axis + + +# Through the CLI, on the fake Google Benchmark of the session tests. + +@pytest.fixture +def builds(repo, tmp): + base = make_build(tmp / "build-base", repo["wt-base"], cases()) + head = make_build(tmp / "build-head", repo["wt-head"], cases(quiet=1.02e-3, noisy=2.06e-3)) + return {"baseline": f"{base}:{repo['wt-base']}", "contender": f"{head}:{repo['wt-head']}"} + + +def test_tables_after_a_measure_mode_comparison_and_the_same_in_read_mode(repo, tmp, builds, capsys): + common = ["--profile", "revisions"] + assert cli.main(["compare", builds["baseline"], builds["contender"], *common, "--adapter", "google-benchmark", + "--suite", "demo-bench", "--rounds", "5", "--run-key", "t-1", "--out", str(tmp / "o"), + "--tables"]) == cli.EXIT_OK + measured = capsys.readouterr().out + assert measured.index("REGRESSED") < measured.index(f"commit {repo['base'][:8]}") < measured.index("| Change") + assert f"commit {repo['head'][:8]} (5 rounds):" in measured and "BM_Quiet/1024" in measured + assert f"| + " in measured and f"Before [{repo['base'][:8]}]" in measured + assert cli.main(["compare", "--run", "t-1", *common, "--baseline", repo["base"][:10], "--tables"]) == cli.EXIT_OK + assert capsys.readouterr().out == measured + + +def test_without_tables_the_output_is_what_it_was(repo, tmp, builds, capsys): + cli.main(["compare", builds["baseline"], builds["contender"], "--profile", "revisions", "--adapter", + "google-benchmark", "--suite", "demo-bench", "--rounds", "3", "--run-key", "t-2", "--out", str(tmp / "o")]) + assert "For " not in capsys.readouterr().out + + +@pytest.mark.parametrize("argv, message", [ + (["--run", "k", "--profile", "revisions", "--baseline", "x", "--tables", "--json"], "--tables is a text view"), + (["--run", "k", "--profile", "revisions", "--baseline", "x", "--all"], "--all goes with --tables"), +]) +def test_tables_flags_that_do_not_fit_are_refused(argv, message, capsys): + assert cli.main(["compare", *argv]) == cli.EXIT_REFUSED + assert message in capsys.readouterr().err diff --git a/tests/test_uc02.py b/tests/test_uc02.py index 4d70f3d..f8fb969 100644 --- a/tests/test_uc02.py +++ b/tests/test_uc02.py @@ -245,3 +245,13 @@ def test_measure_mode_refuses_a_baseline_that_is_not_a_value_before_running(tree assert code == cli.EXIT_REFUSED and doc is None assert "baseline 'variant' is not a value of role; use one of: 'baseline', 'variant'" in err assert not (tree / "results").exists() # nothing ran + + +def test_tables_name_the_sides_by_the_parameter_value(tree, capsys): + loop(["mod.py"], by_case([BASELINE, VARIANT]), PYPERF, rounds=3) + capsys.readouterr() + code = cli.main(["compare", "--run", RUN, "--profile", "variants", "--baseline", BASELINE, "--tables"]) + out = capsys.readouterr().out + assert code == cli.EXIT_OK + assert "role='baseline' (3 rounds):" in out and "role='variant' (3 rounds):" in out + assert f"Before [{BASELINE}]" in out and f"After [{VARIANT}]" in out and "| + " in out diff --git a/tests/test_uc04.py b/tests/test_uc04.py index 0d7fdfb..1bc0ab0 100644 --- a/tests/test_uc04.py +++ b/tests/test_uc04.py @@ -130,3 +130,15 @@ def test_a_failed_check_is_shown_in_the_comparison_and_changes_nothing_else(tree assert "contender: collector-mismatch" in caveats # beside any dirty-tree caveats the fixture tree earns assert "baseline: collector-mismatch" not in out assert "failed invariants" not in out and "bench_a" in out + + +def test_tables_name_the_sides_by_their_label(tree, monkeypatch, capsys): + write_config(tree, {}) + loop({"a": {"dep": "1.0"}, "b": {"dep": "2.0"}}, monkeypatch) + capsys.readouterr() + code = cli.main(["compare", "--run", "S", "--profile", "environments", "--label", "env", "--baseline", "a", + "--tables"]) + out = capsys.readouterr().out + assert code == cli.EXIT_OK + assert "env=a" in out and "env=b" in out and out.count("(3 rounds):") == 2 + assert "Before [a]" in out or "No benchmark changed" in out From db2b36c194983ea7398dcb0b5eeccc1ecb4efe41 Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 18 Nov 2026 09:18:03 +0000 Subject: [PATCH 3/9] ENH: indent the per-benchmark tables of `bx compare --tables` by two spaces The grids sit two spaces in under the benchmark's title line. The title, the one-line value of a benchmark without parameters, and the change table are unchanged. Co-Authored-By: Claude Sonnet 5.5 --- src/benchx/tables.py | 2 +- tests/test_tables.py | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/benchx/tables.py b/src/benchx/tables.py index 9dfec3d..d74effb 100644 --- a/src/benchx/tables.py +++ b/src/benchx/tables.py @@ -154,7 +154,7 @@ def side_tables(units, role, info) -> str: bad = [u[role]["status"] for u in group if u[role]["status"] != "ok"] lines.append(f"{title} {bad[0] if bad else 'ok'}") rows = [[(u["workload"].get("parameters") or {}).get(n, "") for n in names] + [cell(u)] for u in group] - lines.extend(_grid([*names, ""], rows)) + lines.extend(" " + line for line in _grid([*names, ""], rows)) # under the title return "\n".join(lines) diff --git a/tests/test_tables.py b/tests/test_tables.py index e532ec2..e4604b3 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -53,29 +53,29 @@ def test_values_are_written_as_asv_writes_them(value, spread, unit, expected): def test_a_revisions_run_has_a_table_per_commit_and_one_of_changes(): - """asv's layout (format_text_table, tabulate "github"): every cell of a side's table centered, a column + """asv's layout (format_text_table, tabulate "github"), the grids indented by two spaces: every cell centered, a column its widest cell plus 2, lines not trimmed; Ratio and Noise right-aligned; the verdict repeated.""" docs = make() assert tables.render(evaluate(docs), docs).split("\n") == [ "", "For local/host commit aaaaaaaa (5 rounds):", "bench_a.Mean.time_mean_axis ok", - "====== =============", - " size ", - "------ -------------", - " 1 5.30±0.08μs ", - " 10 5.71±0.09μs ", - "====== =============", + " ====== =============", + " size ", + " ------ -------------", + " 1 5.30±0.08μs ", + " 10 5.71±0.09μs ", + " ====== =============", "bench_b.Plain.time_x 2.04±0.03ms", "", "For local/host commit bbbbbbbb (5 rounds):", "bench_a.Mean.time_mean_axis ok", - "====== =============", - " size ", - "------ -------------", - " 1 5.57±0.09μs ", - " 10 6.00±0.09μs ", - "====== =============", + " ====== =============", + " size ", + " ------ -------------", + " 1 5.57±0.09μs ", + " 10 6.00±0.09μs ", + " ====== =============", "bench_b.Plain.time_x 2.14±0.03ms", "", "| Change | Before [aaaaaaaa] | After [bbbbbbbb] | Ratio | Noise | Verdict | Benchmark (Parameter) |", From ba72addde5c8acf747263a4664df4f4217f94993 Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 18 Nov 2026 10:05:39 +0000 Subject: [PATCH 4/9] FIX: one failing or edited benchmark no longer stops the others being compared MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `bx compare` returned no verdict at all when the matched facts of one benchmark disagreed between the sides. The `sides-agree` check is about one unit, but any disagreement made `compare()` return a document with no units, so a benchmark that raised on one side (it reports no harness version or runtime) stopped `time_mean` from being compared, though it ran fine on both sides. `equal-attempts` already dropped only its unit, and the design says a failed invariant leaves "no verdicts for the units it affects". - `sides-agree` now drops only the unit that disagrees: it is recorded in `failed_invariants` with its workload, and the other units are compared. The invariants about the whole run (the sides, their tree ids, the alternation, the benchmark file) still leave no unit compared. - The matched facts are those of the attempts that succeeded. A failed attempt reports none of them: it is excluded and listed, and its benchmark is `indeterminate` (`insufficient-rounds`). A benchmark edited between the revisions, so that its benchmark version differs on two successful sides, still fails `sides-agree`, for itself only. - A unit's `unmatched_differences` (the `differs:` line) also counts successful attempts only, so the facts a failed attempt did not report are not differences. No schema change. `comparator.md` and `prototype-design.md` say which invariants affect the run and which one unit. An example output: ``` bench_function_base.Mean.time_mean_axis error ======== ======= size -------- ------- 1 error 10 error 100000 error ======== ======= | Change | Before [6e1aca7e] | After [7b3b54e0] | Ratio | Noise | Verdict | Benchmark (Parameter) | |----------|---------------------|--------------------|---------|---------|-------------------------------------|-------------------------------------------------| | | 7.04±0.1μs | 7.01±0.1μs | 1.00 | 1.33% | no change detected | bench_function_base.Mean.time_mean(1) | | | 7.21±0.1μs | 7.21±0.2μs | 1.00 | 1.06% | no change detected | bench_function_base.Mean.time_mean(10) | | | 106±0.8μs | 106±0.3μs | 1.00 | 0.34% | no change detected | bench_function_base.Mean.time_mean(100000) | | ? | 7.71±0.05μs | error | n/a | n/a | indeterminate (insufficient-rounds) | bench_function_base.Mean.time_mean_axis(1) | | ? | 7.91±0.1μs | error | n/a | n/a | indeterminate (insufficient-rounds) | bench_function_base.Mean.time_mean_axis(10) | | ? | 108±1μs | error | n/a | n/a | indeterminate (insufficient-rounds) | bench_function_base.Mean.time_mean_axis(100000) | ``` Co-Authored-By: Claude Sonnet 5.5 --- docs/design/comparator.md | 8 ++-- docs/design/prototype-design.md | 3 +- src/benchx/compare.py | 24 ++++++++---- tests/test_comparator.py | 66 +++++++++++++++++++++++++++++++++ tests/test_tables.py | 33 +++++++++++++++++ 5 files changed, 122 insertions(+), 12 deletions(-) diff --git a/docs/design/comparator.md b/docs/design/comparator.md index 9da057c..a05ca0f 100644 --- a/docs/design/comparator.md +++ b/docs/design/comparator.md @@ -136,9 +136,11 @@ side are equal, and sides alternate by `procedure.slot`; under `revisions`, that each side has exactly one tree id and the two differ; under `revisions` and `variants`, that a separately recorded benchmark has one `benchmark_tree` across both sides and all rounds (`one-benchmark-tree`, schema §5.5). A side that is not clean does -not fail the run; it makes the comparison local-only (UC-03 §10). A run that -fails an invariant is not compared: the comparison document records the failed invariant -and contains no verdicts for the units it affects. +not fail the run; it makes the comparison local-only (UC-03 §10). An invariant of the +run (the sides, their tree ids, the alternation, the benchmark file) that fails leaves no unit compared. One of a +single unit (the matched facts agree, attempt counts are equal) takes that unit out and the others are compared: the +document records the failed invariant with its workload, and has no verdict for that unit. The matched facts are +those of the attempts that succeeded; a failed attempt reports none of them, and is excluded and listed. Excluded points and their reasons remain visible in the output. Changes in facts the projection ignores do not split a series by themselves, but differences between diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index d0dbe6e..0e90968 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -514,7 +514,8 @@ matched facts agree (`sides-agree`) and the invariants the profile names over the run: exactly two sides, which for `environments` differ in more than their label, equal attempt counts per side, and sides alternating by `procedure.slot`. Results the order did not ask for are not compared. A failed invariant is recorded and the affected units get no -verdict; under `revisions` the same happens to a side with no tree id (the +verdict (`sides-agree` and `equal-attempts` affect one unit, and the others are still compared; the rest +affect every unit; `sides-agree` looks at the attempts that succeeded); under `revisions` the same happens to a side with no tree id (the `sides` invariant), to a tree edited during the run (a third side: `two-sides`), and to a benchmark recorded apart from the subject that is not one file on both sides and in every round (`one-benchmark-tree`, schema §5.5), as it does diff --git a/src/benchx/compare.py b/src/benchx/compare.py index 39b2b98..7ce745f 100644 --- a/src/benchx/compare.py +++ b/src/benchx/compare.py @@ -224,26 +224,33 @@ def compare(documents, *, run_key, profile, baseline, label=None, parameter=None "detail": "sides do not alternate by procedure.slot: run the sides in the same " "order every round (a, b, a, b, ...)" + how}) - # Units: one workload variant and quantity each. Everything else the - # projection matches must agree across both sides (schema §5.5). + # Units: one workload variant and quantity each. Everything else the projection matches must agree + # across both sides (schema §5.5), in the attempts that succeeded: a failed attempt reports none of + # the facts (runtime, harness version) and is excluded, and listed, below. units = {} for d, p, s in zip(documents, projected, sides): variant = {ptr: v for ptr, v in p.identity.items() if ptr == WORKLOAD or ptr.startswith(WORKLOAD + "/")} key = (_c(variant), d["measurement"]["quantity"]["name"]) units.setdefault(key, {"docs": {base: [], contender: []}, "identities": []}) units[key]["docs"][s].append(d) - units[key]["identities"].append(p.identity) - for (_, quantity), unit in units.items(): + if d["measurement"]["status"] == "success": + units[key]["identities"].append(p.identity) + run_failed = bool(failed) # the invariants above are about the run, and no unit is compared + disagreeing = set() # `sides-agree` is about one unit: it gets no verdict, and the others are compared + for key, unit in units.items(): differing = _differing(unit["identities"]) if differing: failed.append({"invariant": "sides-agree", "workload": _workload(unit["docs"], projection.axis), - "quantity": quantity, "detail": f"{', '.join(differing)} differ"}) - if failed: + "quantity": key[1], "detail": f"{', '.join(differing)} differ"}) + disagreeing.add(key) + if run_failed: return doc - for (_, quantity), unit in sorted( + for (variant_key, quantity), unit in sorted( units.items(), key=lambda item: _unit_order(_workload(item[1]["docs"], projection.axis), item[0][1])): + if (variant_key, quantity) in disagreeing: + continue by_side = unit["docs"] where = {"workload": _workload(by_side, projection.axis), "quantity": quantity} absent = [role for role, s in (("baseline", base), ("contender", contender)) if not by_side[s]] @@ -268,7 +275,8 @@ def compare(documents, *, run_key, profile, baseline, label=None, parameter=None pairs = [(r[base], r[contender]) for _, r in sorted(rounds.items(), key=lambda x: (x[0] is None, x[0])) if base in r and contender in r] unit_doc = _unit(where, pairs, by_side, base, contender, k, min_rounds, side_failed) - unit_doc["unmatched_differences"] = _differences(projection, by_side[base], by_side[contender]) + succeeded = {s: [d for d in by_side[s] if d["measurement"]["status"] == "success"] for s in (base, contender)} + unit_doc["unmatched_differences"] = _differences(projection, succeeded[base], succeeded[contender]) if caveats := _caveats(by_side, base, contender): unit_doc["caveats"] = caveats doc["units"].append(unit_doc) diff --git a/tests/test_comparator.py b/tests/test_comparator.py index d5dcbea..8cecaba 100644 --- a/tests/test_comparator.py +++ b/tests/test_comparator.py @@ -619,3 +619,69 @@ def test_units_with_more_than_one_other_parameter_are_ordered_by_name_then_value doc = vevaluate(extra) assert [(u["workload"]["parameters"]["n"], u["workload"]["parameters"]["size"]) for u in doc["units"]] == [ ("2", "20"), ("2", "100"), ("10", "20"), ("10", "100")] + + +# --- one benchmark that cannot be compared does not stop the others ------------------------- + +def run_two(rounds=3, ratio=1.1): + """Two workloads measured in every call; the head takes `ratio` times the base's time.""" + docs, slot = [], 0 + for r in range(rounds): + for side, factor in ((BASE, 1.0), (HEAD, ratio)): + docs += [result(side, r, slot, (r + 1.0) * factor, workload=name) for name in ("BM_A", "BM_B")] + slot += 1 + return docs + + +def of_head(docs, workload): + return [d for d in docs if d["revision"]["key"] == HEAD and d["coordinates"]["workload"]["name"] == workload] + + +def by_name(doc): + return {u["workload"]["name"]: u for u in doc["units"]} + + +def test_an_edited_benchmark_fails_its_own_unit_only(): + """A benchmark whose code differs between the sides (asv's benchmark version) is a new series and is + not comparable; the other benchmark is compared.""" + docs = run_two() + for d in docs: + if d["coordinates"]["workload"]["name"] == "BM_B": + version = "v2" if d["revision"]["key"] == HEAD else "v1" + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": version} + doc = evaluate(docs) + assert invariants(doc) == ["sides-agree"] + failure = doc["failed_invariants"][0] + assert failure["workload"] == {"name": "BM_B"} and "benchmark_version differ" in failure["detail"] + assert list(by_name(doc)) == ["BM_A"] and by_name(doc)["BM_A"]["verdict"] == "regressed" + + +def test_a_failed_attempt_is_excluded_before_the_facts_are_matched(): + """A benchmark that raises reports no harness version or runtime, so its attempts differ from the other + side's in those facts. They are excluded and listed, not matched: the benchmark has no verdict + (`insufficient-rounds`) and the other one is compared.""" + docs = run_two() + failing = of_head(docs, "BM_B") + for d in docs: + if d not in failing: # the CPU configuration is a fact the projection ignores: what is reported, differs + d["coordinates"]["environment"]["system"]["cpu"] = {"config": "performance"} + for d in failing: + d["measurement"] = {"quantity": d["measurement"]["quantity"], "status": "error", "reason": "harness.error"} + d["coordinates"]["protocol"]["harness"] = {"name": "google-benchmark", "version": "1.9"} + d["coordinates"]["environment"]["system"]["runtime"] = {"python": "3.12"} + doc = evaluate(docs) + assert doc["failed_invariants"] == [] + assert by_name(doc)["BM_A"]["verdict"] == "regressed" + failed = by_name(doc)["BM_B"] + assert failed["verdict"] == "indeterminate" and failed["reason"] == "insufficient-rounds" + assert [e["reason"] for e in doc["exclusions"]] == ["error: harness.error"] * 3 + assert by_name(doc)["BM_B"].get("unmatched_differences", []) == [] # what it did not report is no difference + assert by_name(doc)["BM_A"].get("unmatched_differences", []) == [] + + +def test_a_failure_of_the_whole_run_still_leaves_no_units(): + """A third side (a tree edited during the run) is about the run, so no benchmark is compared.""" + docs = run_two() + docs[-1]["provenance"]["subject_tree"] = "c" * 40 + doc = evaluate(docs) + assert invariants(doc) == ["two-sides"] and doc["units"] == [] diff --git a/tests/test_tables.py b/tests/test_tables.py index e4604b3..e87488a 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -139,6 +139,39 @@ def test_a_side_that_failed_shows_its_status(): assert "| indeterminate (" in shown +def test_a_benchmark_that_raises_on_one_side_leaves_the_others_compared(): + """The failing benchmark reports no harness version or runtime. It shows its status in the tables and is + `indeterminate` in the change table; the other benchmarks have their rows.""" + docs = make() + for d in docs: + if d["revision"]["key"] == HEAD and d["coordinates"]["workload"]["name"] == PLAIN: + d["measurement"] = {"quantity": d["measurement"]["quantity"], "status": "error", "reason": "harness.error"} + d["coordinates"]["environment"]["system"]["runtime"] = {"python": "3.12"} + doc = evaluate(docs) + assert doc["failed_invariants"] == [] + shown = tables.render(doc, docs, all_rows=True) + assert "bench_b.Plain.time_x 2.04±0.03ms" in shown and "bench_b.Plain.time_x error" in shown + rows = [line for line in shown.splitlines() if line.startswith("| ")] + assert sum("| regressed " in line for line in rows) == 2 + assert any("bench_b.Plain.time_x" in line and "| ?" in line and "error" in line and "n/a" in line + and "indeterminate (insufficient-rounds)" in line for line in rows) + + +def test_an_edited_benchmark_has_no_row_but_the_others_do(): + """Its two sides ran different code, so it is not compared (`sides-agree`): no row in the change table, + while the table of each side still shows what it measured.""" + docs = make() + for d in docs: + if d["coordinates"]["workload"]["name"] == MEAN: + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": "v2" if d["revision"]["key"] == HEAD else "v1"} + doc = evaluate(docs) + assert [f["invariant"] for f in doc["failed_invariants"]] == ["sides-agree"] * 2 + shown = tables.render(doc, docs) + rows = [line for line in shown.splitlines() if line.startswith("| ")] + assert len(rows) == 2 and PLAIN in rows[1] and not any(MEAN in line for line in rows) + assert shown.count(f"{MEAN} ok") == 2 # the grid of each side + + def test_a_dirty_side_is_named_in_its_heading(): docs = make(provenance__subject_dirty="dirty") shown = tables.render(evaluate(docs), docs) From 36829b74e6d8fd34a0ce31baaae575a41dc81b73 Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 18 Nov 2026 13:07:31 +0000 Subject: [PATCH 5/9] ENH: `bx bench` says why a benchmark was not measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A benchmark that raised was shown as `error (harness.error)` and nothing more, though the worker's exception line is stored in the result (`provenance.info.message`). Under the table `bx bench` now has a section: not measured (5) KeyError: 'axis' (4 cases: bench_axes [n=1], bench_axes [n=10], bench_axes [n=100], …) bench_boom: ValueError: array has no axis 3 one line per message, with up to three case names; skipped asv benchmarks show their reason there too. The first line of a message, cut at 160 characters, is `projection.said`, shared with `bx compare` (next commit). FIX: the pyperf adapter took as the message the first unindented line of the worker's stderr, so a warning printed before the traceback was shown instead of the exception. It is now the line that ends the traceback (the last one when exceptions are chained), not the manager's "failed with exit code" wrapper; without a traceback, the first line said. Results stored before keep the message they have. Co-Authored-By: Claude Sonnet 5.5 --- src/benchx/adapters/pyperf.py | 22 ++++-- src/benchx/bench.py | 18 ++++- src/benchx/cli.py | 5 +- src/benchx/projection.py | 11 +++ tests/test_bench_failures.py | 133 ++++++++++++++++++++++++++++++++++ tests/test_projection.py | 19 ++++- 6 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 tests/test_bench_failures.py diff --git a/src/benchx/adapters/pyperf.py b/src/benchx/adapters/pyperf.py index e99373a..3555786 100644 --- a/src/benchx/adapters/pyperf.py +++ b/src/benchx/adapters/pyperf.py @@ -305,12 +305,22 @@ def run_case(runnable, case, flags, env, timeout, native_path, env_names=None) - def _failure_message(stderr: str) -> str: - """The worker's exception line. The manager then raises its own - " failed with exit code N" after a second traceback; that wrapper - only counts when nothing else was said.""" - lines = [l for l in stderr.splitlines() if l and not l[0].isspace() and not l.startswith("Traceback")] - own = [l for l in lines if "failed with exit code" not in l] - return (own or lines or [stderr.strip().splitlines()[-1]])[0] + """The worker's exception line: the line that ends a traceback (the first unindented line after + "Traceback"), the last one when exceptions were chained. A warning printed before is not it, and + neither is the manager's own " failed with exit code N" after a second traceback: that wrapper + only counts when nothing else was said. Without a traceback (a killed process), the first line said.""" + lines = [line for line in stderr.splitlines() if line.strip()] + ended, in_traceback = [], False + for line in lines: + if line.startswith("Traceback"): + in_traceback = True + elif in_traceback and not line[0].isspace(): + ended.append(line) + in_traceback = False + own = [line for line in ended if "failed with exit code" not in line] + if own or ended: + return (own or ended)[-1] + return next((line for line in lines if not line[0].isspace()), lines[-1] if lines else stderr.strip()) def attempted(applied: dict) -> int: diff --git a/src/benchx/bench.py b/src/benchx/bench.py index c54c4fd..3d0921d 100644 --- a/src/benchx/bench.py +++ b/src/benchx/bench.py @@ -14,7 +14,7 @@ from . import core, snapshot from .errors import SessionError -from .projection import workload_label +from .projection import said, workload_label from .store import warning_codes @@ -87,6 +87,7 @@ def row_of(document: dict) -> dict: "max": max(values) if values else None, "loops": document.get("procedure", {}).get("inner_iterations"), "warnings": warning_codes(document), + "message": None if measurement["status"] == "success" else said(document), } @@ -110,3 +111,18 @@ def render(rows, leading=()) -> str: widths = [max(len(row[i]) for row in [header, *body]) for i in range(len(header))] lines = [" ".join(cell.ljust(w) for cell, w in zip(row, widths)).rstrip() for row in [header, *body]] return "\n".join([lines[0], " ".join("-" * w for w in widths), *lines[1:]]) + + +def failures(rows, names=3) -> str: + """The section under the table for the cases that were not measured: what each said, `case: message`, + cases with the same message as one line, `message (K cases: a, b, c, …)`. Empty when none has a message + (the status column is then all there is). A row's message is one line already (`projection.said`).""" + said_by = {} + for row in rows: + if row.get("message"): + said_by.setdefault(row["message"], []).append(row["case"]) + if not said_by: + return "" + lines = [f" {message} ({len(cases)} cases: {', '.join(cases[:names])}{', …' if len(cases) > names else ''})" + if len(cases) > 1 else f" {cases[0]}: {message}" for message, cases in said_by.items()] + return "\n".join(["", f"not measured ({sum(len(c) for c in said_by.values())})", *lines]) diff --git a/src/benchx/cli.py b/src/benchx/cli.py index 1e1cd14..12fd56b 100644 --- a/src/benchx/cli.py +++ b/src/benchx/cli.py @@ -201,7 +201,10 @@ def cmd_bench(args): return EXIT_REFUSED except runner.Refused as e: return _refused(e) - print(bench_mod.render(bench_mod.summarize(summary["files"]))) + rows = bench_mod.summarize(summary["files"]) + print(bench_mod.render(rows)) + if failed := bench_mod.failures(rows): + print(failed) print(f"\n{summary['results']} result(s) -> {summary['out']} (order {summary['order'][:19]})") if store is None: return EXIT_OK diff --git a/src/benchx/projection.py b/src/benchx/projection.py index 5995d1c..ca6e57c 100644 --- a/src/benchx/projection.py +++ b/src/benchx/projection.py @@ -258,6 +258,17 @@ def workload_label(workload: dict) -> str: return workload["name"] + (f" [{parameters}]" if parameters else "") +def said(document: dict, width: int = 160) -> str | None: + """What the producer said about a result that is not a measurement (`provenance.info.message`): for a + failed pyperf case the exception line, for a skipped asv benchmark the reason. Its first line, cut at + `width` characters; None when it said nothing.""" + message = str(((document.get("provenance") or {}).get("info") or {}).get("message") or "").strip() + line = message.splitlines()[0] if message else "" + if not line: + return None + return line if len(line) <= width else line[:width - 1] + "…" + + def reported_coordinates(doc: dict) -> dict: """Everything reported about what was measured, minus the axis revision and the quantity (§4.3).""" return {key: doc[key] for key in REPORTED if key in doc} diff --git a/tests/test_bench_failures.py b/tests/test_bench_failures.py new file mode 100644 index 0000000..e03f166 --- /dev/null +++ b/tests/test_bench_failures.py @@ -0,0 +1,133 @@ +"""What `bx bench` says about a result that is not a measurement (FAILURE_MESSAGES_PLAN.md).""" + +import textwrap + +import pytest + +from benchx import bench, cli + + +def result(name="bench_boom", status="error", reason="harness.error", message="ValueError: boom", parameters=None): + workload = {"name": name, **({"parameters": parameters} if parameters else {})} + document = {"coordinates": {"workload": workload}, + "measurement": {"status": status, **({"reason": reason} if reason else {})}, + "provenance": {"info": {"message": message} if message else {}}} + if status == "success": + document["measurement"]["observations"] = [1.0, 2.0] + return document + + +def rows_of(*documents): + return [bench.row_of(d) for d in documents] + + +def test_a_row_carries_what_a_failed_result_said(): + assert bench.row_of(result())["message"] == "ValueError: boom" + assert bench.row_of(result(status="skipped", reason="harness.skipped", message="no GPU"))["message"] == "no GPU" + + +def test_a_row_has_no_message_when_there_is_none_to_give(): + assert bench.row_of(result(status="success", reason=None, message="a note"))["message"] is None # measured + assert bench.row_of(result(message=None))["message"] is None # failed without a word + no_provenance = result() + del no_provenance["provenance"] # a result from another producer + assert bench.row_of(no_provenance)["message"] is None + + +def test_nothing_is_said_when_every_case_was_measured(): + assert bench.failures([]) == "" + assert bench.failures(rows_of(result(status="success", reason=None, message=None))) == "" + assert bench.failures(rows_of(result(message=None))) == "" # no message: the status column is all there is + + +def test_a_failed_case_is_named_with_what_it_said(): + assert bench.failures(rows_of(result())) == "\nnot measured (1)\n bench_boom: ValueError: boom" + + +def test_cases_that_said_the_same_are_one_line_with_three_names_at_most(): + rows = rows_of(*(result("bench_boom", parameters={"n": str(n)}) for n in (1, 10, 100, 1000, 10000))) + assert bench.failures(rows) == ("\nnot measured (5)\n" + " ValueError: boom (5 cases: bench_boom [n=1], bench_boom [n=10], bench_boom [n=100], …)") + three = rows_of(*(result("bench_boom", parameters={"n": str(n)}) for n in (1, 10, 100))) + assert bench.failures(three).endswith("(3 cases: bench_boom [n=1], bench_boom [n=10], bench_boom [n=100])") + + +def test_different_messages_are_different_lines_in_the_order_they_came(): + rows = rows_of(result("a", message="KeyError: 'x'"), result("b", message="ValueError: boom"), + result("c", status="skipped", reason="harness.skipped", message="KeyError: 'x'")) + assert bench.failures(rows) == ("\nnot measured (3)\n KeyError: 'x' (2 cases: a, c)\n b: ValueError: boom") + + +def test_a_message_is_one_line_and_not_longer_than_a_line_should_be(): + long = bench.failures(rows_of(result(message="ValueError: " + "x" * 300))) + assert long.endswith("…") and len(long.splitlines()[-1]) == len(" bench_boom: ") + 160 + several = bench.failures(rows_of(result(message="RuntimeError: first\nsecond line\nthird"))) + assert several == "\nnot measured (1)\n bench_boom: RuntimeError: first" + + +def test_bx_bench_says_why_a_benchmark_failed(tmp, monkeypatch, capsys): + pytest.importorskip("pyperf") + tree = tmp / "tree" + tree.mkdir() + (tree / "mod.py").write_text(textwrap.dedent(''' + def bench_ok(): + sum(range(100)) + + def bench_boom(): + raise ValueError("array has no axis 3") + ''')) + monkeypatch.chdir(tree) + cli.main(["bench", "mod.py", "--adapter", "pyperf", "--run-key", "boom", "--no-ingest", + "--processes", "2", "--values", "2", "--loops", "20"]) + out = capsys.readouterr().out + assert "error (harness.error)" in out # the status column is as it was + assert "\nnot measured (1)\n bench_boom: ValueError: array has no axis 3\n" in out + assert "bench_ok" not in out.split("not measured")[1] # what was measured is not in it + + +TRACEBACK = """Traceback (most recent call last): + File "driver.py", line 37, in + runner.bench_func('bench_axes', func, *args) + File "mod.py", line 7, in bench_axes + raise KeyError("axis") +{last} +""" +MANAGER = """Traceback (most recent call last): + File "pyperf/_manager.py", line 118, in spawn_worker + raise RuntimeError("%s failed with exit code %s" +RuntimeError: /usr/bin/python failed with exit code 1 +""" +WARNING = """driver.py:34: SyntaxWarning: "is not" with 'int' literal. Did you mean "!="? + if 0 is not None: +""" + + +def test_the_message_is_the_exception_that_ended_the_workers_traceback(): + from benchx.adapters import pyperf + stderr = TRACEBACK.format(last="KeyError: 'axis'") + MANAGER + assert pyperf._failure_message(stderr) == "KeyError: 'axis'" # not the manager's wrapper after it + + +def test_a_warning_printed_before_the_traceback_is_not_the_message(): + from benchx.adapters import pyperf + stderr = WARNING + WARNING + TRACEBACK.format(last="KeyError: 'axis'") + MANAGER + assert pyperf._failure_message(stderr) == "KeyError: 'axis'" + + +def test_a_chained_exception_gives_the_one_that_was_raised_last(): + from benchx.adapters import pyperf + chained = (TRACEBACK.format(last="KeyError: 'axis'") + + "\nThe above exception was the direct cause of the following exception:\n\n" + + TRACEBACK.format(last="ValueError: no such axis")) + assert pyperf._failure_message(chained + MANAGER) == "ValueError: no such axis" + + +def test_the_wrapper_is_the_message_only_when_nothing_else_was_said(): + from benchx.adapters import pyperf + assert pyperf._failure_message(MANAGER) == "RuntimeError: /usr/bin/python failed with exit code 1" + + +def test_without_a_traceback_the_first_line_said_is_the_message(): + from benchx.adapters import pyperf + assert pyperf._failure_message("Killed\n") == "Killed" + assert pyperf._failure_message(WARNING) == WARNING.splitlines()[0] # nothing better was said diff --git a/tests/test_projection.py b/tests/test_projection.py index 59f0dfd..fc9cd67 100644 --- a/tests/test_projection.py +++ b/tests/test_projection.py @@ -8,7 +8,7 @@ from benchx import compare, core, identity from benchx.core import catalog -from benchx.projection import Projection, ProjectionError, workload_label +from benchx.projection import Projection, ProjectionError, said, workload_label POLICIES = catalog.SCHEMAS / "identity-policy" / "0.1.0" / "examples" DEFAULT = json.loads((POLICIES / "default.json").read_text()) @@ -171,3 +171,20 @@ def test_the_quantity_is_never_a_coordinate(): assert history().apply(ARROW).identity == history().apply(other).identity assert identity.series_fingerprint(ARROW, "arrow", identity.MEDIAN, history()) != \ identity.series_fingerprint(other, "arrow", identity.MEDIAN, history()) + + +def test_what_a_result_said_is_its_first_line_cut_to_a_width(): + def document(message): + return {"provenance": {"info": {"message": message}}} + + assert said(document("ValueError: boom")) == "ValueError: boom" + assert said(document(" RuntimeError: first\nsecond\nthird\n")) == "RuntimeError: first" + assert said(document("x" * 300)) == "x" * 159 + "…" + assert said(document("x" * 20), width=10) == "x" * 9 + "…" + assert said(document("")) is None and said(document("\n\n")) is None + + +def test_a_result_that_said_nothing_has_no_message(): + assert said({"provenance": {"info": {}}}) is None + assert said({"provenance": {}}) is None + assert said({}) is None # a result from another producer From df10a5ad02d79772f4deb54cb1befb762bd73bec Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 18 Nov 2026 13:07:31 +0000 Subject: [PATCH 6/9] ENH: `bx compare` says what was left out in words, not as raw dicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `missing`, `failed invariants` and `excluded` listings printed the document's dicts: one line for each failed attempt (15 for one failing benchmark with three sizes over five rounds), with an ingest key and the category `harness.error` and nothing about what went wrong. - excluded: the reason of an exclusion ends with what the failed attempt said (`error: harness.error: ValueError: array has no axis 3`; unchanged when it said nothing), and results excluded for the same reason are one line with their number. Which results they are is in `--json`. - failed invariants: `invariant: detail`, naming the unit it affects, and units that failed alike are one line, `(K units: a, b, c, …)`. - missing: `not on contender: a, b, c, … (+N more)`, one line for each side. No schema change: the reason is free text. Naming the benchmark on an excluded line would need fields of the exclusion, a schema addition left for later. Co-Authored-By: Claude Sonnet 5.5 --- docs/design/comparator.md | 4 +- docs/design/prototype-design.md | 5 ++- src/benchx/compare.py | 54 +++++++++++++++++++--- tests/test_build_import_path.py | 3 +- tests/test_comparator.py | 79 +++++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 10 deletions(-) diff --git a/docs/design/comparator.md b/docs/design/comparator.md index a05ca0f..a52783f 100644 --- a/docs/design/comparator.md +++ b/docs/design/comparator.md @@ -142,7 +142,9 @@ single unit (the matched facts agree, attempt counts are equal) takes that unit document records the failed invariant with its workload, and has no verdict for that unit. The matched facts are those of the attempts that succeeded; a failed attempt reports none of them, and is excluded and listed. -Excluded points and their reasons remain visible in the output. Changes in +Excluded points and their reasons remain visible in the output: the reason of an excluded result ends with what the +attempt said when it said something (a failed pyperf case's exception line, an asv benchmark's skip reason), and +the rendered listing is one line per reason with a count. Changes in facts the projection ignores do not split a series by themselves, but differences between the target and its reference are carried as caveats so that a kernel, load, or other recorded condition is not hidden from the reader. diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index 0e90968..ab6590e 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -270,7 +270,10 @@ criterion 1 holds by construction. `--json` emits this; the default output renders it as the reviewer table (regressed, improved, no change detected, and indeterminate sections, then -the missing, failed-invariant, and excluded listings with counts). In any +the missing, failed-invariant, and excluded listings with counts, in words: a failed invariant names the unit +it affects, units that failed alike are one line, and results excluded for the same reason are one line with +their number, the reason carrying what the failed attempt said, such as `error: harness.error: ValueError: …`; +which results they are is in `--json`). In any profile, a comparison in which either side is not `clean` has `local_only: true` (schema §5.5), is headed `[local-only]`, and names the sides that are not clean on the `caveats:` line; the `revisions` profile fails a side with no diff --git a/src/benchx/compare.py b/src/benchx/compare.py index 7ce745f..b04b893 100644 --- a/src/benchx/compare.py +++ b/src/benchx/compare.py @@ -16,7 +16,7 @@ from . import identity from .core import canonical, catalog -from .projection import Projection, axis_order, get, workload_label +from .projection import Projection, axis_order, get, said, workload_label from .snapshot import DIRTY_WARNINGS from .store import store_project @@ -268,7 +268,8 @@ def compare(documents, *, run_key, profile, baseline, label=None, parameter=None if d["measurement"]["status"] != "success": status, reason = d["measurement"]["status"], d["measurement"].get("reason") result = {"producer": d["producer"]["name"], "ingest_key": d["ingest_key"]} - doc["exclusions"].append({"result": result, "reason": f"{status}: {reason}" if reason else status}) + why = f"{status}: {reason}" if reason else status + doc["exclusions"].append({"result": result, "reason": f"{why}: {said(d)}" if said(d) else why}) side_failed = side_failed or (profile in STRICT_STATUS and status in FAILED_STATUSES) continue rounds.setdefault(d.get("procedure", {}).get("round"), {})[s] = d @@ -436,6 +437,45 @@ def seen(found, pointer): return [p for p in _differing(a + b) if seen(a, p) != seen(b, p)] +def _few(names, shown=3, count=False) -> str: + """The first `shown` names; `, …` when there are more, and with `count` how many.""" + more = len(names) - shown + return ", ".join(names[:shown]) + ((f", … (+{more} more)" if count else ", …") if more > 0 else "") + + +def _failed(items) -> list[str]: + """Failed invariants in words, `invariant: detail`: the unit it affects first when it affects one, and + units that failed alike as one line, `(K units: a, b, c, …)`.""" + groups = {} + for item in items: + label = workload_label(item["workload"]) if "workload" in item else None + groups.setdefault((item["invariant"], item["detail"]), []).append(label) + lines = [] + for (invariant, detail), labels in groups.items(): + units = list(dict.fromkeys(label for label in labels if label)) + text = f"{invariant}: {detail}" + lines.append(f"{units[0]}: {text}" if len(units) == 1 else + f"{text} ({len(units)} units: {_few(units)})" if units else text) + return lines + + +def _excluded(items) -> list[str]: + """Results left out, one line for each reason (which has what the result said): `reason (K results)`. + Which results is in the document (`--json`), not here.""" + counts = {} + for item in items: + counts[item["reason"]] = counts.get(item["reason"], 0) + 1 + return [f"{reason} ({n} result{'s' if n != 1 else ''})" for reason, n in counts.items()] + + +def _missing(items) -> list[str]: + """Units a side has no result for, one line for each side: `not on contender: a, b, c, … (+N more)`.""" + sides = {} + for item in items: + sides.setdefault(item["side"], []).append(workload_label(item["workload"])) + return [f"not on {side}: {_few(list(dict.fromkeys(labels)), count=True)}" for side, labels in sorted(sides.items())] + + def render(doc: dict) -> str: """The reviewer-speed table (comparator.md §5).""" lines = [f"run {doc['run_key']} profile {doc['profile']} " @@ -466,9 +506,9 @@ def render(doc: dict) -> str: reason = f" ({u['reason']})" if "reason" in u else "" lines.append(f" {workload_label(u['workload']):<32} {u['quantity']:<10} effect {effect} noise {noise}" f" rounds={u['rounds']}{reason}") - for key, title in (("missing", "missing"), ("failed_invariants", "failed invariants"), - ("exclusions", "excluded")): - if doc[key]: - lines.append(f"\n{title} ({len(doc[key])})") - lines.extend(f" {item}" for item in doc[key]) + for title, items, say in (("missing", doc["missing"], _missing), ("failed invariants", doc["failed_invariants"], _failed), + ("excluded", doc["exclusions"], _excluded)): + if items: + lines.append(f"\n{title} ({len(items)})") + lines.extend(f" {line}" for line in say(items)) return "\n".join(lines) diff --git a/tests/test_build_import_path.py b/tests/test_build_import_path.py index c27039a..1f0f0b4 100644 --- a/tests/test_build_import_path.py +++ b/tests/test_build_import_path.py @@ -93,7 +93,8 @@ def test_the_build_root_imports_nothing(tmp, toy, monkeypatch): unit = doc["units"][0] assert unit["rounds"] == 0 assert unit["verdict"] == "indeterminate" - assert doc["exclusions"] and {e["reason"] for e in doc["exclusions"]} == {"error: harness.error"} + assert doc["exclusions"] and {e["reason"] for e in doc["exclusions"]} == { + "error: harness.error: ModuleNotFoundError: No module named 'toy'"} # and says what the worker said def test_one_pythonpath_measures_one_build_on_both_sides(tmp, toy, monkeypatch): diff --git a/tests/test_comparator.py b/tests/test_comparator.py index 8cecaba..7bb6696 100644 --- a/tests/test_comparator.py +++ b/tests/test_comparator.py @@ -685,3 +685,82 @@ def test_a_failure_of_the_whole_run_still_leaves_no_units(): docs[-1]["provenance"]["subject_tree"] = "c" * 40 doc = evaluate(docs) assert invariants(doc) == ["two-sides"] and doc["units"] == [] + + +# --- what the output says about what was left out (FAILURE_MESSAGES_PLAN.md, decisions 8 and 9) --- + +def failing_head(docs, workload="BM_B", message="ValueError: array has no axis 3"): + """The head's attempts of a workload fail, the worker having said `message`.""" + for d in of_head(docs, workload): + d["measurement"] = {"quantity": d["measurement"]["quantity"], "status": "error", "reason": "harness.error"} + if message: + d["provenance"] = {**d["provenance"], "info": {"message": message}} + return docs + + +def test_an_exclusion_says_what_the_failed_attempt_said(): + doc = evaluate(failing_head(run_two())) + assert [e["reason"] for e in doc["exclusions"]] == ["error: harness.error: ValueError: array has no axis 3"] * 3 + + +def test_an_exclusion_of_an_attempt_that_said_nothing_is_as_it_was(): + doc = evaluate(failing_head(run_two(), message=None)) + assert [e["reason"] for e in doc["exclusions"]] == ["error: harness.error"] * 3 + + +def test_excluded_results_that_failed_alike_are_one_line_with_their_number(): + text = compare.render(evaluate(failing_head(run_two()))) + assert text.endswith("\nexcluded (3)\n error: harness.error: ValueError: array has no axis 3 (3 results)") + assert "ingest_key" not in text and "{'" not in text # words, not the document's dicts + + +def test_excluded_results_that_failed_differently_are_lines_of_their_own(): + docs = failing_head(run_two(), message="ValueError: array has no axis 3") + docs[docs.index(of_head(docs, "BM_B")[0])]["provenance"]["info"]["message"] = "KeyError: 'x'" + lines = compare.render(evaluate(docs)).splitlines() + assert lines[-3:] == ["excluded (3)", " error: harness.error: KeyError: 'x' (1 result)", + " error: harness.error: ValueError: array has no axis 3 (2 results)"] + + +def test_a_message_in_the_reason_is_one_line_and_cut(): + docs = failing_head(run_two(), message="RuntimeError: " + "x" * 300 + "\nthe second line") + [reason, *_] = {e["reason"] for e in evaluate(docs)["exclusions"]} + assert reason.startswith("error: harness.error: RuntimeError: xxx") and reason.endswith("…") + assert "second line" not in reason and len(reason) == len("error: harness.error: ") + 160 + + +def test_a_failed_invariant_of_one_unit_names_the_unit_in_words(): + docs = run_two() + for d in docs: + if d["coordinates"]["workload"]["name"] == "BM_B": + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": "v2" if d["revision"]["key"] == HEAD else "v1"} + text = compare.render(evaluate(docs)) + assert text.endswith("\nfailed invariants (1)\n BM_B: sides-agree: /coordinates/protocol/asv/benchmark_version differ") + assert "'invariant'" not in text + + +def test_units_that_failed_an_invariant_alike_are_one_line(): + docs = run_two() + for d in docs: + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": "v2" if d["revision"]["key"] == HEAD else "v1"} + text = compare.render(evaluate(docs)) + assert text.endswith("\nfailed invariants (2)\n" + " sides-agree: /coordinates/protocol/asv/benchmark_version differ (2 units: BM_A, BM_B)") + + +def test_a_failed_invariant_of_the_run_is_in_words_too(): + doc = {"run_key": "r", "profile": "revisions", "baseline": BASE, "contender": HEAD, "local_only": False, "units": [], + "missing": [], "exclusions": [], + "failed_invariants": [{"invariant": "equal-attempts", "detail": "3 against 2: run the same number of rounds"}]} + assert compare.render(doc).endswith("\nfailed invariants (1)\n equal-attempts: 3 against 2: run the same number of rounds") + + +def test_missing_results_are_one_line_for_each_side_with_a_few_names(): + where = lambda name, side: {"workload": {"name": name, "parameters": {"n": "1"}}, "quantity": "wall-time", "side": side} + doc = {"run_key": "r", "profile": "revisions", "baseline": BASE, "contender": HEAD, "local_only": False, "units": [], + "failed_invariants": [], "exclusions": [], + "missing": [where("a", "contender"), where("b", "contender"), where("c", "contender"), where("d", "contender"), + where("e", "baseline")]} + assert compare.render(doc).endswith( + "\nmissing (5)\n not on baseline: e [n=1]\n not on contender: a [n=1], b [n=1], c [n=1], … (+1 more)") + From 587a3bd7581d357dab3977c1f239b8b9b14a8799 Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 18 Nov 2026 12:22:47 +0000 Subject: [PATCH 7/9] ENH: make the refusal of a filter that selects no case readable and helpful A filter that named a parametrized benchmark but gave its arguments in the wrong order was refused with the first ten benchmark names, on one long line, with the filter shown as a Python string (`\.` as `\\.`). The refusal is now one fact per line: - the filter as typed; - when the filter, up to its first `\(`, matches the name of a parametrized benchmark, how its cases are searched: `name(p0, p1)`, each as its repr, in the order of `param_names`, with the parameters of that benchmark and one case; - the benchmarks similar to the filter (words of 3+ characters compared with the dotted parts of each name, difflib ratio >= 0.75, the names matching the most words first, 5 at most), else the first 5 available. Only the text of the refusal changes: not what a filter selects, no schema, no new option. Co-Authored-By: Claude Sonnet 5.5 --- docs/benchx.md | 4 +- src/benchx/adapters/asv.py | 2 +- src/benchx/adapters/base.py | 63 +++++++++++++++++++++------- src/benchx/adapters/benchx_params.py | 16 +++++-- src/benchx/adapters/pyperf.py | 2 +- tests/test_asv.py | 33 ++++++++++++++- tests/test_bench.py | 2 +- tests/test_params.py | 45 +++++++++++++++++++- tests/test_pyperf.py | 20 ++++++++- 9 files changed, 162 insertions(+), 25 deletions(-) diff --git a/docs/benchx.md b/docs/benchx.md index 2ba68ef..d02f6fa 100644 --- a/docs/benchx.md +++ b/docs/benchx.md @@ -183,7 +183,9 @@ neither asv nor asv_runner is used. PATH is a directory holding `asv.conf.json` `benchmark_dir` is the suite) or the benchmark directory itself, and its checkout is the target (`bx bench . --adapter asv --suite benchmarks` is the same order). `--filter` is asv's `--bench`: searched in the name, and in `name(p0, p1)` for each -parameter combination; a CASE is an exact name, `name(p0, p1)` for a combination. `adapters/asv_runtime.py` discovers +parameter combination, the arguments in the order of `param_names`, each as its `repr`; a CASE is an exact name, +`name(p0, p1)` for a combination. A filter that names a parametrized benchmark and none of its combinations is +refused with that order and one case as it is searched. `adapters/asv_runtime.py` discovers the benchmarks as asv does (names, `params`, `setup`/`teardown`/`setup_cache`, skips, and asv's `version` hash all agree with asv_runner's; a test checks it). The benchmarks run in the `python` first on `PATH`, which must import pyperf and the diff --git a/src/benchx/adapters/asv.py b/src/benchx/adapters/asv.py index 248b594..289477a 100644 --- a/src/benchx/adapters/asv.py +++ b/src/benchx/adapters/asv.py @@ -199,7 +199,7 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: for case, index in benchx_params.expand(info["name"], info["params"], case_filter)] if not selected: if benchx_params.matches_name_only(info["name"], info["params"], case_filter): - unmatched.append(info["name"]) + unmatched.append(info) continue if info["type"] != "time": other[info["type"]] = other.get(info["type"], 0) + 1 diff --git a/src/benchx/adapters/base.py b/src/benchx/adapters/base.py index 6239708..ed957eb 100644 --- a/src/benchx/adapters/base.py +++ b/src/benchx/adapters/base.py @@ -67,12 +67,15 @@ attempt. For a harness that repeats at several levels. """ +import difflib import os +import re import signal import subprocess +import textwrap from ..core import Refusal, Rejected -from .benchx_params import combination +from .benchx_params import combination, search_name class Unsupported(Rejected): @@ -97,25 +100,55 @@ def parametrized_workload(case, runnable) -> tuple[str, dict]: return name, dict(zip(info["param_names"], combination(info["params"], index))) -def name_only_note(names) -> str: - """For a refusal: the filter matched parametrized benchmarks by name, but not their - arguments (see `benchx_params.matches_name_only`).""" - return (f"the filter matches the name of a parametrized benchmark ({', '.join(names[:3])}) but " - "not its arguments, and those are matched as name(p0, p1)") - - -def no_cases(where, case_filter, available=(), notes=(), limit=10) -> Unsupported: +def name_only_note(found) -> str: + """For a refusal: the filter is about parametrized benchmarks (`found`, their listing entries) + but matches none of their arguments (see `benchx_params.matches_name_only`). Says how a case + is searched, with the first benchmark's parameters, in order, and one case as it is searched.""" + first = found[0] + names = ", ".join(info["name"] for info in found[:3]) + return "\n".join([ + f"the filter matches the name of a parametrized benchmark ({names}) but not its arguments.", + "They are searched as name(p0, p1), each as its repr, in the order of param_names:", + f" {first['name']}({', '.join(first['param_names'])})", + "for example:", + f" {search_name(first['name'], first['params'], 0)}"]) + + +def similar(case_filter, names, limit=5, cutoff=0.75) -> list[str]: + """The names a filter may have meant: those that have, as a dotted part (module, class, method), the + most of its words, or words close to them, then the closest first. A word is a run of letters, digits + and `_` of three characters or more, so an escape such as `\\d` says nothing, and the case of a letter + is not a difference. A filter that names a module and misspells its class thus finds that class's + benchmarks, and not the whole module.""" + words = [w.lower() for w in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", case_filter or "") if len(w) >= 3] + scored = [] + for name in names: + parts = name.lower().split(".") + best = (max(difflib.SequenceMatcher(None, word, part).ratio() for part in parts) for word in words) + hits = [score for score in best if score >= cutoff] + if hits: + scored.append((-len(hits), -sum(hits), name)) + scored.sort() + return [name for count, _, name in scored[:limit] if count == scored[0][0]] + + +def no_cases(where, case_filter, available=(), notes=(), limit=5) -> Unsupported: """The refusal for a plan with no case in it. Nothing would run, and a filter with a typo must not end in `0 result(s)` and a success status. `available` are the names that could have been selected (the first `limit` are shown) and `notes` say what else - the adapter knows about why the plan is empty.""" - head = f"filter {case_filter!r} selects no case in {where}" if case_filter else f"no case to run in {where}" - parts = [head] + the adapter knows about why the plan is empty, each on its own lines (a note may have several). + The last line is the names that are similar to the filter (`similar`), else the first available. + The filter is shown as given, not as a Python string, so its backslashes are the ones typed.""" + head = f"filter '{case_filter}' selects no case in {where}" if case_filter else f"no case to run in {where}" + lines = [head] + [textwrap.indent(note, " ") for note in notes] names = sorted(available) - if names: + close = similar(case_filter, names, limit) + if close: + lines.append(f" similar: {', '.join(close)}") + elif names: more = f", … ({len(names) - limit} more)" if len(names) > limit else "" - parts.append(f"available: {', '.join(names[:limit])}{more}") - return Unsupported("; ".join(parts + list(notes))) + lines.append(f" available: {', '.join(names[:limit])}{more}") + return Unsupported("\n".join(lines)) def run_bounded(args, *, timeout, what, **kwargs) -> subprocess.CompletedProcess: diff --git a/src/benchx/adapters/benchx_params.py b/src/benchx/adapters/benchx_params.py index bc27cd5..247cb3d 100644 --- a/src/benchx/adapters/benchx_params.py +++ b/src/benchx/adapters/benchx_params.py @@ -84,6 +84,16 @@ def expand(name, ids, case_filter): def matches_name_only(name, ids, case_filter) -> bool: - """Whether a filter that selected none of a benchmark's cases matches its name alone: - a parametrized benchmark is searched in `name(p0, p1)`, so `name$` is a likely slip.""" - return bool(ids and case_filter and re.search(case_filter, name)) + """Whether a filter that selected none of a benchmark's cases is about this benchmark, though + not about any of its combinations: it matches the name alone (`name$` is a likely slip), or the + part of it before its first `\\(` does, so what it gives after the name, the arguments, is + what fails (in the wrong order, or a value the benchmark does not have).""" + if not (ids and case_filter): + return False + named = case_filter.split(r"\(")[0] + if not named.strip("^"): # it starts with its arguments: no name to go by + return False + try: + return bool(re.search(named, name)) + except re.error: + return False diff --git a/src/benchx/adapters/pyperf.py b/src/benchx/adapters/pyperf.py index 3555786..bc74734 100644 --- a/src/benchx/adapters/pyperf.py +++ b/src/benchx/adapters/pyperf.py @@ -196,7 +196,7 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: selected = benchx_params.expand(name, info["params"], case_filter) if not selected: if benchx_params.matches_name_only(name, info["params"], case_filter): - unmatched.append(name) + unmatched.append(info) continue if "error" in info: raise Unsupported(f"cannot read the params of {name} in {where}: {info['error']}") diff --git a/tests/test_asv.py b/tests/test_asv.py index 90c8827..9ae425b 100644 --- a/tests/test_asv.py +++ b/tests/test_asv.py @@ -197,10 +197,18 @@ def test_a_filter_that_selects_nothing_is_refused(suite, capsys): assert cli.main([*ASV, "--filter", "time_plian", "--run-key", "typo", "--no-ingest", *FAST]) == cli.EXIT_REFUSED err = capsys.readouterr().err assert "refused: filter 'time_plian' selects no case in" in err - assert "available: bm.Cached.time_sum, bm.Mutating.time_pop," in err + assert "\n similar: bm.time_plain\n" in err # what the typo is close to, not the first of the alphabet + assert "available" not in err assert results(suite, "typo") == {} +def test_a_filter_that_is_like_no_name_lists_the_first_available(suite, capsys): + assert cli.main([*ASV, "--filter", "zzzzzz", "--run-key", "unlike", "--no-ingest", *FAST]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "available: bm.Cached.time_sum, bm.Mutating.time_pop," in err + assert "similar" not in err + + def test_an_anchored_name_of_a_parametrized_benchmark_is_refused_with_a_hint(suite, capsys): # asv searches `name(p0, p1)`, so `time_sorted$` matches none of the combinations. assert cli.main([*ASV, "--filter", "time_sorted$", "--run-key", "anchored", "--no-ingest", @@ -209,6 +217,16 @@ def test_an_anchored_name_of_a_parametrized_benchmark_is_refused_with_a_hint(sui in capsys.readouterr().err) +def test_arguments_in_the_wrong_order_are_refused_with_the_order_and_an_example(suite, capsys): + # `Sort` takes (n, key); `('a', 10)` names the benchmark and selects none of its combinations. + assert cli.main([*ASV, "--filter", r"time_sorted\('a', 10\)", "--run-key", "order", "--no-ingest", + *FAST]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "the filter matches the name of a parametrized benchmark (bm.Sort.time_sorted) but not its arguments" in err + assert "\n bm.Sort.time_sorted(n, key)\n for example:\n bm.Sort.time_sorted(10, 'a')\n" in err + assert results(suite, "order") == {} + + def test_a_filter_that_selects_only_benchmarks_that_are_not_run_is_refused(suite, capsys): assert cli.main([*ASV, "--filter", "peakmem_sorted", "--run-key", "notrun", "--no-ingest", *FAST]) == cli.EXIT_REFUSED @@ -419,3 +437,16 @@ def test_missing_asv_runner_is_refused(suite, monkeypatch, tmp, capsys): monkeypatch.setenv("PATH", str(bare)) assert cli.main([*ASV, "--no-ingest"]) == cli.EXIT_REFUSED assert "pyperf is not importable" in capsys.readouterr().err + + +def test_a_refusal_shows_the_filter_as_it_was_typed(suite, capsys): + """Backslashes are not doubled (not a Python repr), and the refusal is one fact per line.""" + assert cli.main([*ASV, "--filter", r"time_sorted\('a', 10\)", "--run-key", "typed", "--no-ingest", + *FAST]) == cli.EXIT_REFUSED + lines = [line for line in capsys.readouterr().err.splitlines() if line.startswith(("refused:", " "))] + assert lines[0].startswith(r"refused: filter 'time_sorted\('a', 10\)' selects no case in ") + assert [line.split(":")[0].strip() for line in lines[1:]] == [ + "the filter matches the name of a parametrized benchmark (bm.Sort.time_sorted) but not its arguments.", + "They are searched as name(p0, p1), each as its repr, in the order of param_names", + "bm.Sort.time_sorted(n, key)", "for example", "bm.Sort.time_sorted(10, 'a')", + "1 module(s) failed to import", "similar"] # the suite has a module that does not import diff --git a/tests/test_bench.py b/tests/test_bench.py index bd5663f..221fb2a 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -99,7 +99,7 @@ def ran(run_key): assert cli.main(["bench", "mod.py", "bench_sort(100, 'a')", "--run-key", "one", "--no-ingest", *FAST]) == cli.EXIT_OK assert ran("one") == ["100'a'"] assert cli.main(["bench", "mod.py", "bench_srot", "--run-key", "typo", "--no-ingest", *FAST]) == cli.EXIT_REFUSED - assert "selects no case in mod.py; available: bench_sort" in capsys.readouterr().err + assert "selects no case in mod.py\n similar: bench_sort\n" in capsys.readouterr().err def test_shorthand_order_is_stable(tree): diff --git a/tests/test_params.py b/tests/test_params.py index 0e3e791..29399b0 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -76,6 +76,17 @@ def test_a_filter_that_matches_only_the_name_is_noticed(): assert not params.matches_name_only("m.C.time_x", IDS, None) +def test_a_filter_that_gives_the_arguments_in_the_wrong_order_is_noticed(): + """The part before the first `\\(` is the benchmark's name; what follows it selected nothing.""" + assert params.expand("m.C.time_x", IDS, r"C\.time_x\('a', 10\)") == [] + assert params.matches_name_only("m.C.time_x", IDS, r"C\.time_x\('a', 10\)") + assert params.matches_name_only("m.C.time_x", IDS, r"^m\.C\.time_x\(100, 'c'\)$") + assert not params.matches_name_only("m.C.time_x", IDS, r"C\.time_y\(10, 'a'\)") # another benchmark + assert not params.matches_name_only("m.C.time_x", IDS, r"\(10, ") # no name to go by + assert not params.matches_name_only("m.C.time_x", IDS, r"^\(10, ") + assert not params.matches_name_only("m.C.time_x", IDS, r"(unbalanced\(10") # not a regex: no hint, no error + + def test_the_workload_of_a_case_comes_from_the_table_not_from_the_id(): runnable = {"cases": {"m.C.time_x-2": ("m.C.time_x", 2), "m.time_y-1": ("m.time_y", None)}, "benchmarks": {"m.C.time_x": {"params": IDS, "param_names": ["n", "key"]}, "m.time_y": {}}} @@ -83,10 +94,42 @@ def test_the_workload_of_a_case_comes_from_the_table_not_from_the_id(): assert base.parametrized_workload("m.time_y-1", runnable) == ("m.time_y", {}) # a name that looks like -1 +NAMES = ["bench_function_base.Partition.time_argpartition", "bench_function_base.Partition.time_partition", + "bench_function_base.Sort.time_sort", "bench_linalg.Linalg.time_norm", "bench_ufunc.UFunc.time_add"] + + +def test_a_slip_in_a_word_of_the_filter_finds_the_names_that_have_it(): + assert base.similar("Partiton", NAMES) == NAMES[:2] # a class, as spelled in the names + assert base.similar("time_argpartiton", NAMES) == NAMES[:2] # a method: its own first (0.97), then time_partition (0.87) + assert base.similar("bench_linalgg", NAMES) == [NAMES[3]] # a module + assert base.similar("partition", NAMES) == NAMES[:2] # the case of a letter is no difference + + +def test_the_words_of_a_regex_are_what_is_compared(): + assert base.similar(r"^bench_function_base\.Partiton\.time_x\(10, 'a'\)$", NAMES) == NAMES[:2] + assert base.similar(r"Partiton|time_ad\b", NAMES) == [NAMES[0], NAMES[1], NAMES[4]] # 0.94, 0.94, then 0.93 + assert base.similar(r"\d+\.\w", NAMES) == [] # escapes and one letter say nothing + + +def test_nothing_is_similar_to_what_is_not_like_any_name(): + assert base.similar("zzzz", NAMES) == [] + assert base.similar(None, NAMES) == [] + assert base.similar("Partiton", []) == [] + + +def test_the_most_similar_names_come_first_and_at_most_limit_are_returned(): + names = ["m.Alpha.time_a", "m.Alphb.time_a", "m.Alpxx.time_a", "m.Alpha.time_b"] + assert base.similar("Alpha", names) == ["m.Alpha.time_a", "m.Alpha.time_b", "m.Alphb.time_a"] # exact, then near + assert base.similar("Alpha", names, limit=1) == ["m.Alpha.time_a"] + + def test_the_note_names_the_benchmarks_it_means(): - note = base.name_only_note(["a", "b", "c", "d"]) + found = [{"name": name, "param_names": ["n", "key"], "params": IDS} for name in "abcd"] + note = base.name_only_note(found) assert "(a, b, c)" in note # three at most assert "name(p0, p1)" in note + # the first one's parameters, in order, and a case of it, each on a line of its own to be copied + assert note.splitlines()[2:] == [" a(n, key)", "for example:", " a(10, 'a')"] def test_search_name_is_exactly_what_expand_searches(): diff --git a/tests/test_pyperf.py b/tests/test_pyperf.py index 72d1cf1..0b65deb 100644 --- a/tests/test_pyperf.py +++ b/tests/test_pyperf.py @@ -183,10 +183,28 @@ def test_a_filter_that_matches_a_name_but_no_arguments_is_refused_with_a_hint(pa with pytest.raises(adapter.Unsupported) as refused: planned(param_tree, "bench_sort$") message = str(refused.value.problems[0]) - assert "filter 'bench_sort$' selects no case in mod.py; available: bench_plain, bench_sort;" in message + assert message.startswith("filter 'bench_sort$' selects no case in mod.py\n") + assert "\n similar: bench_sort" in message assert "the filter matches the name of a parametrized benchmark (bench_sort) but not its arguments" in message +def test_a_refusal_says_how_the_arguments_of_a_case_are_searched(param_tree): + """Arguments in the wrong order: the filter names the function, so the note gives the order + of `param_names` and a case as it is searched.""" + with pytest.raises(adapter.Unsupported) as refused: + planned(param_tree, r"bench_sort\('a', 10\)") + message = str(refused.value.problems[0]) + assert "the filter matches the name of a parametrized benchmark (bench_sort) but not its arguments" in message + assert message.splitlines()[3:6] == [" bench_sort(n, key)", " for example:", " bench_sort(10, 'a')"] + assert message.index("not its arguments") < message.index("similar:") # the hint comes before the list + + +def test_a_filter_with_no_name_in_it_gets_no_note_about_arguments(param_tree): + with pytest.raises(adapter.Unsupported) as refused: + planned(param_tree, r"\(7, ") + assert "not its arguments" not in str(refused.value.problems[0]) + + def test_params_that_cannot_be_read_matter_only_to_the_function_that_has_them(param_tree): with pytest.raises(adapter.Unsupported, match="cannot read the params of helper in mod.py: TypeError"): planned(param_tree, "helper") From 5c125d317a6525586d0b25169111cddf1c25822e Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 18 Nov 2026 12:57:02 +0000 Subject: [PATCH 8/9] FIX: the pyperf driver no longer makes Python warn about `0 is not None` For a parametrized case the generated driver had `if is not None:` with the combination index written in, so `if 0 is not None:`. Python warns about `is not` with a literal: a SyntaxWarning on the stderr of every worker of every parametrized case (pyperf and asv). The warning also came before a failing case's traceback. The index is now bound to a name first (`index = 0`) and the name is tested. Co-Authored-By: Claude Sonnet 5.5 --- src/benchx/adapters/pyperf.py | 7 +++++-- tests/test_pyperf.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/benchx/adapters/pyperf.py b/src/benchx/adapters/pyperf.py index bc74734..c41c82e 100644 --- a/src/benchx/adapters/pyperf.py +++ b/src/benchx/adapters/pyperf.py @@ -214,6 +214,8 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: workload = parametrized_workload # the function, and its parameters as value strings +# The combination index is bound to a name before it is tested: `0 is not None` is a SyntaxWarning, which +# Python prints on the stderr of every worker. DRIVER = '''\ # Generated by benchx. Runs one function under pyperf: one case, which for a function # that declares params is one combination of them, passed as its arguments. @@ -247,10 +249,11 @@ def list_cases(runnable, case_filter, env, timeout=None) -> list[str]: sys.modules["benchx_suite"] = module spec.loader.exec_module(module) func = getattr(module, {function!r}) +index = {index!r} args = () -if {index!r} is not None: +if index is not None: values, _, _ = params.normalize(getattr(func, "params", []), getattr(func, "param_names", [])) - args = params.combination(values, {index!r}) + args = params.combination(values, index) runner.bench_func({case!r}, func, *args) ''' diff --git a/tests/test_pyperf.py b/tests/test_pyperf.py index 0b65deb..3de774a 100644 --- a/tests/test_pyperf.py +++ b/tests/test_pyperf.py @@ -12,6 +12,7 @@ import subprocess import sys import textwrap +import warnings from pathlib import Path import pytest @@ -205,6 +206,19 @@ def test_a_filter_with_no_name_in_it_gets_no_note_about_arguments(param_tree): assert "not its arguments" not in str(refused.value.problems[0]) +def test_the_driver_of_a_case_gives_python_nothing_to_warn_about(tmp, param_tree): + """The driver is a script of its own, run once per worker. It has `if is not None:` for a + parametrized case, and Python warns about `is not` with a literal: a SyntaxWarning on the + stderr of every worker, which a failing case's message was once taken from.""" + run_order(tmp, pyperf_order(param_tree, "quiet", case_filter="bench_sort|bench_plain"), "quiet") + drivers = sorted((tmp / "quiet").rglob("*.driver.py")) + assert len(drivers) == 5 # four combinations and a function without parameters + for driver in drivers: + with warnings.catch_warnings(): + warnings.simplefilter("error") + compile(driver.read_text(), str(driver), "exec") + + def test_params_that_cannot_be_read_matter_only_to_the_function_that_has_them(param_tree): with pytest.raises(adapter.Unsupported, match="cannot read the params of helper in mod.py: TypeError"): planned(param_tree, "helper") From 5a1ff27eea55dc0885d29bf49d7b7a38053cf99e Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 18 Nov 2026 13:44:23 +0000 Subject: [PATCH 9/9] FIX: show a benchmark's parameters in the order it declared them `bx compare` labelled `Partition.time_argpartition` as `[array_type=..., dtype='int16', k=10]` and the per-commit table had the columns `array_type, dtype, k`, though the benchmark declares `dtype, array_type, k`: the order `--filter` searches `name(p0, p1, ...)` in, and asv shows. A filter copied from the output selected nothing. The result file had the order, but the store keeps a document in canonical form, with its keys sorted, and `bx compare --run`, `bx head --table` and the tables read the store. A JSON object has no order a schema promises, so the order is now data: the runner records it in `provenance.info.parameter_names` (free-form, no schema change) for a workload with two or more parameters, and `projection.declared_workload` puts the parameters back in it. `compare` (units, failed invariants, missing, and so the tables) and `bx bench` / `bx head --table` read through it. Units are listed with the first declared parameter changing slowest, as asv lists a product of params; before, by parameter name. Results made before have no list and show as they did. Co-Authored-By: Claude Sonnet 5.5 --- docs/benchx.md | 5 +- src/benchx/bench.py | 4 +- src/benchx/compare.py | 14 ++-- src/benchx/projection.py | 14 ++++ src/benchx/runner.py | 2 + tests/test_parameter_order.py | 120 ++++++++++++++++++++++++++++++++++ tests/test_pyperf.py | 10 +++ 7 files changed, 160 insertions(+), 9 deletions(-) create mode 100644 tests/test_parameter_order.py diff --git a/docs/benchx.md b/docs/benchx.md index d02f6fa..88d5eb4 100644 --- a/docs/benchx.md +++ b/docs/benchx.md @@ -185,7 +185,10 @@ the target (`bx bench . --adapter asv --suite benchmarks` is the same order). `--filter` is asv's `--bench`: searched in the name, and in `name(p0, p1)` for each parameter combination, the arguments in the order of `param_names`, each as its `repr`; a CASE is an exact name, `name(p0, p1)` for a combination. A filter that names a parametrized benchmark and none of its combinations is -refused with that order and one case as it is searched. `adapters/asv_runtime.py` discovers +refused with that order and one case as it is searched. The runner records the order in +`provenance.info.parameter_names` (the store keeps a document with its keys sorted), and `bx bench`, `bx head --table` +and `bx compare` show the parameters in it, so what they print can be given to `--filter`; results made before have +no such list and show them sorted. `adapters/asv_runtime.py` discovers the benchmarks as asv does (names, `params`, `setup`/`teardown`/`setup_cache`, skips, and asv's `version` hash all agree with asv_runner's; a test checks it). The benchmarks run in the `python` first on `PATH`, which must import pyperf and the diff --git a/src/benchx/bench.py b/src/benchx/bench.py index 3d0921d..d4c1c79 100644 --- a/src/benchx/bench.py +++ b/src/benchx/bench.py @@ -14,7 +14,7 @@ from . import core, snapshot from .errors import SessionError -from .projection import said, workload_label +from .projection import declared_workload, said, workload_label from .store import warning_codes @@ -77,7 +77,7 @@ def row_of(document: dict) -> dict: # An observation is a number, or an object carrying one with its metadata (pyperf's). values = [o["value"] if isinstance(o, dict) else o for o in measurement.get("observations", [])] return { - "case": workload_label(document["coordinates"]["workload"]), + "case": workload_label(declared_workload(document)), "status": measurement["status"] + (f" ({measurement['reason']})" if measurement.get("reason") else ""), "n": len(values), "median": statistics.median(values) if values else None, diff --git a/src/benchx/compare.py b/src/benchx/compare.py index b04b893..92e64c7 100644 --- a/src/benchx/compare.py +++ b/src/benchx/compare.py @@ -16,7 +16,7 @@ from . import identity from .core import canonical, catalog -from .projection import Projection, axis_order, get, said, workload_label +from .projection import Projection, axis_order, declared_workload, get, said, workload_label from .snapshot import DIRTY_WARNINGS from .store import store_project @@ -323,17 +323,19 @@ def _benchmark_file(documents, names, sides) -> list[dict]: def _workload(by_side, axis=()) -> dict: """The unit's workload variant, as the comparison document names it: what both sides share, so the parameters on the profile's axis (the ones that tell the sides apart) are left out.""" - workload = next(d for docs in by_side.values() for d in docs)["coordinates"]["workload"] + workload = declared_workload(next(d for docs in by_side.values() for d in docs)) varied = {path.removeprefix(PARAMETERS) for path in axis if path.startswith(PARAMETERS)} parameters = {k: v for k, v in (workload.get("parameters") or {}).items() if k not in varied} return {"name": workload["name"], **({"parameters": parameters} if parameters else {})} def _unit_order(workload: dict, quantity: str) -> tuple: - """The order units are listed in: by workload name, then by parameter name and value, then by - quantity, with numbers inside names and values in natural order (`axis_order`), so that `n=50` - comes before `n=100` and `bench_2` before `bench_10`, as people read them.""" - parameters = sorted((workload.get("parameters") or {}).items()) + """The order units are listed in: by workload name, then by the values of its parameters, the first + declared changing slowest (the order of the workload's parameters, see `declared_workload`: sorted by + name for a result that recorded none), then by quantity, with numbers inside names and values in + natural order (`axis_order`), so that `n=50` comes before `n=100` and `bench_2` before `bench_10`, as + people read them.""" + parameters = list((workload.get("parameters") or {}).items()) return (axis_order(workload["name"]), tuple((name, axis_order(value)) for name, value in parameters), quantity) diff --git a/src/benchx/projection.py b/src/benchx/projection.py index ca6e57c..1a18f9d 100644 --- a/src/benchx/projection.py +++ b/src/benchx/projection.py @@ -252,6 +252,20 @@ def axis_order(value) -> tuple: return (3, canonical(value).decode()) +def declared_workload(document: dict) -> dict: + """The document's workload with its parameters in the order the benchmark declared them. The store keeps + a document in canonical form, with its keys sorted, so the runner records the order as data + (`provenance.info.parameter_names`): the names it lists first, then any it does not name, as stored. A + result without the list (made before, or by another producer) is as stored.""" + workload = document["coordinates"]["workload"] + parameters = workload.get("parameters") + names = ((document.get("provenance") or {}).get("info") or {}).get("parameter_names") + if not parameters or not names: + return workload + ordered = {name: parameters[name] for name in names if name in parameters} + return {**workload, "parameters": {**ordered, **{k: v for k, v in parameters.items() if k not in ordered}}} + + def workload_label(workload: dict) -> str: """A workload as people read it: its name, then its parameters, `name [k=v, ...]`.""" parameters = ", ".join(f"{k}={v}" for k, v in (workload.get("parameters") or {}).items()) diff --git a/src/benchx/runner.py b/src/benchx/runner.py index 95507d5..0a0aade 100644 --- a/src/benchx/runner.py +++ b/src/benchx/runner.py @@ -273,6 +273,8 @@ def run(order_path, out_dir, allow_collectors=False) -> dict: workload = {"name": workload_name} if case_parameters: workload["parameters"] = case_parameters + if len(case_parameters) > 1: # the store sorts keys: the order is data (projection.declared_workload) + info["parameter_names"] = list(case_parameters) subject = {"name": _subject_name(order), "components": [{"role": "primary"}]} if configuration: subject["configuration"] = configuration diff --git a/tests/test_parameter_order.py b/tests/test_parameter_order.py new file mode 100644 index 0000000..6a1fb91 --- /dev/null +++ b/tests/test_parameter_order.py @@ -0,0 +1,120 @@ +"""Parameters in the order the benchmark declared them (PARAMETER_ORDER_PLAN.md). + +The store keeps a document in canonical form, with its keys sorted, so a benchmark's parameters come back +alphabetical: `array_type, dtype, k` for a benchmark that declares `dtype, array_type, k`, which is the order a +filter searches and asv shows. The runner records the declared order in `provenance.info.parameter_names`, and what +shows a workload reads it back. +""" + +import json + +from benchx import bench, compare, core, tables +from benchx.projection import declared_workload, workload_label +from test_comparator import BASE, HEAD, result + +NAME = "bm.Partition.time_argpartition" +DECLARED = ["dtype", "array_type", "k"] # not alphabetical +COMBINATIONS = [{"dtype": dtype, "array_type": array_type, "k": "10"} + for dtype in ("'float64'", "'int16'") for array_type in ("('ordered',)", "('random',)")] + + +def stored(document): + """A document as the store gives it back: canonical, so its keys sorted.""" + return json.loads(core.canonical(document)) + + +def case(side, round_, slot, value, parameters, names=True): + d = result(side, round_, slot, value, workload=NAME) + d["coordinates"]["workload"]["parameters"] = parameters # in the order the benchmark declared them + d["ingest_key"] += f":{parameters}" + if names: + d["provenance"]["info"] = {"parameter_names": list(parameters)} + return stored(d) + + +def run_of(names=True, rounds=5, ratio=1.5): + """Every combination, base and head alternating; the head is `ratio` times the base.""" + docs, slot = [], 0 + for r in range(rounds): + for side in (BASE, HEAD): + for parameters in COMBINATIONS: + docs.append(case(side, r, slot, (1.0 + 0.01 * r) * (ratio if side == HEAD else 1.0), parameters, names)) + slot += 1 + return docs + + +def evaluate(docs): + return compare.compare(docs, run_key="run", profile="revisions", baseline=BASE) + + +def test_the_store_gives_the_parameters_back_alphabetical(): + """The fault: what a result file says in one order, the stored document says in another.""" + document = case(BASE, 0, 0, 1.0, COMBINATIONS[0]) + assert list(document["coordinates"]["workload"]["parameters"]) == sorted(DECLARED) + assert document["provenance"]["info"]["parameter_names"] == DECLARED # which is why the order is recorded + + +def test_a_workload_is_read_in_the_order_its_result_recorded(): + document = case(BASE, 0, 0, 1.0, COMBINATIONS[0]) + assert list(declared_workload(document)["parameters"]) == DECLARED + assert declared_workload(document)["name"] == NAME + assert list(document["coordinates"]["workload"]["parameters"]) == sorted(DECLARED) # the document is not changed + + +def test_a_result_that_recorded_no_order_has_its_parameters_as_stored(): + """Made before the order was recorded, or by another producer.""" + document = case(BASE, 0, 0, 1.0, COMBINATIONS[0], names=False) + assert declared_workload(document) == document["coordinates"]["workload"] + assert declared_workload(result(BASE, 0, 0, 1.0)) == {"name": "BM_W"} # nor parameters + + +def test_a_recorded_order_that_does_not_fit_the_parameters_is_not_trusted_blindly(): + document = case(BASE, 0, 0, 1.0, COMBINATIONS[0]) + document["provenance"]["info"]["parameter_names"] = ["k", "gone"] # one is not a parameter, two are not named + assert list(declared_workload(document)["parameters"]) == ["k", "array_type", "dtype"] + + +def test_a_comparison_names_the_parameters_in_the_order_they_were_declared(): + doc = evaluate(run_of()) + assert [list(u["workload"]["parameters"]) for u in doc["units"]] == [DECLARED] * 4 + assert compare.render(doc).count("[dtype='int16', array_type=('ordered',), k=10]") == 1 + + +def test_the_units_are_listed_in_the_order_the_parameters_vary_as_declared(): + """The first declared parameter changes slowest, as asv lists a product of params.""" + doc = evaluate(run_of()) + assert [(u["workload"]["parameters"]["dtype"], u["workload"]["parameters"]["array_type"]) for u in doc["units"]] == [ + ("'float64'", "('ordered',)"), ("'float64'", "('random',)"), ("'int16'", "('ordered',)"), ("'int16'", "('random',)")] + + +def test_results_without_the_order_are_compared_and_listed_as_before(): + doc = evaluate(run_of(names=False)) + assert [list(u["workload"]["parameters"]) for u in doc["units"]] == [sorted(DECLARED)] * 4 + assert [(u["workload"]["parameters"]["array_type"], u["workload"]["parameters"]["dtype"]) for u in doc["units"]] == [ + ("('ordered',)", "'float64'"), ("('ordered',)", "'int16'"), ("('random',)", "'float64'"), ("('random',)", "'int16'")] + + +def test_a_failed_invariant_names_the_parameters_in_the_declared_order_too(): + docs = run_of() + for d in docs: + if d["coordinates"]["workload"]["parameters"]["dtype"] == "'int16'": + d["coordinates"]["protocol"]["asv"] = {"benchmark_version": "v2" if d["revision"]["key"] == HEAD else "v1"} + doc = evaluate(docs) + assert [list(f["workload"]["parameters"]) for f in doc["failed_invariants"]] == [DECLARED] * 2 + assert compare.render(doc).splitlines()[-1] == ( + " sides-agree: /coordinates/protocol/asv/benchmark_version differ (2 units: " + f"{NAME} [dtype='int16', array_type=('ordered',), k=10], {NAME} [dtype='int16', array_type=('random',), k=10])") + + +def test_the_tables_have_the_columns_and_the_values_in_the_declared_order(): + docs = run_of() + text = tables.render(evaluate(docs), docs) + header = next(line for line in text.splitlines() if "array_type" in line) + assert header.split() == DECLARED + assert f"{NAME}('float64', ('ordered',), 10)" in text # the label of the change table: values, in that order + + +def test_bx_bench_and_bx_head_label_a_stored_result_in_the_declared_order(): + document = case(BASE, 0, 0, 1.0, COMBINATIONS[3]) + assert bench.row_of(document)["case"] == f"{NAME} [dtype='int16', array_type=('random',), k=10]" + assert workload_label(declared_workload(document)) == bench.row_of(document)["case"] diff --git a/tests/test_pyperf.py b/tests/test_pyperf.py index 3de774a..66e3747 100644 --- a/tests/test_pyperf.py +++ b/tests/test_pyperf.py @@ -532,3 +532,13 @@ def test_relative_output_directory(tmp, tree, monkeypatch): runner.run(order_path, Path("rel-out")) docs = results(elsewhere / "rel-out") assert [d["measurement"]["status"] for d in docs] == ["success"] + + +def test_the_runner_records_the_order_the_parameters_were_declared_in(tmp, param_tree): + """The store keeps a document with its keys sorted, so `n, key` would come back as `key, n` + (PARAMETER_ORDER_PLAN.md): the result says in which order they were declared.""" + docs = run_order(tmp, pyperf_order(param_tree, "declared", case_filter="bench_sort|bench_plain"), "declared") + sort = [d for d in docs if d["coordinates"]["workload"]["name"] == "bench_sort"] + plain = [d for d in docs if d["coordinates"]["workload"]["name"] == "bench_plain"] + assert sort and all(d["provenance"]["info"]["parameter_names"] == ["n", "key"] for d in sort) + assert plain and all("parameter_names" not in d["provenance"]["info"] for d in plain) # nothing to put in order