From 9d534bbc8aec0eb57109b1f3eddedddaa99287f7 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 18:46:38 +0900 Subject: [PATCH 1/2] Bench: Interleaved A/B measurement of CPython vs TypedPython @compiled run.py interleaved alternates CPython and the compiled extension pair by pair (order flipped each pair) in one time window and reports the median per-pair speedup with its range and the load at every pair. Load hits both sides alike, so the ratio is usable when the machine is never quiet (leader decision 2026-10-03: the official method if no quiet window appears). Refs #41 --- benchmarks/typedpython/run.py | 67 +++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py index 7aeeac231..288cfd746 100644 --- a/benchmarks/typedpython/run.py +++ b/benchmarks/typedpython/run.py @@ -198,6 +198,62 @@ def measure(args: argparse.Namespace) -> int: return 0 +def interleaved(args: argparse.Namespace) -> int: + """CPython vs TypedPython @compiled, alternated A/B in the same time window (leader decision, + 2026-10-03): when the machine is never quiet, load hits both sides of each pair alike, so the + per-pair ratio is stable even though absolute times are not. Reports the median ratio, never + absolute times as the result, with the load average recorded at every pair.""" + if not args.typedpython: + print("interleaved needs --typedpython") + return 2 + compiled = build_compiled(Path(args.typedpython), args.typedpython_python) + results: dict[str, dict[str, object]] = {} + for name, spec in BENCHMARKS.items(): + entry = compiled.get(name) + if not isinstance(entry, tuple): + results[name] = {"skipped": entry} + continue + size = args.size.get(name, spec[5]) + cmds = commands(name, size, args.python, False, compiled) + a_cmd, b_cmd = cmds["CPython"], cmds["TypedPython @compiled"] + pairs = [] + for i in range(args.pairs): + order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first + load = os.getloadavg() + (t1, o1), (t2, o2) = run_once(order[0]), run_once(order[1]) + t_a, o_a, t_b, o_b = (t1, o1, t2, o2) if order[0] is a_cmd else (t2, o2, t1, o1) + if o_a != o_b: + print(f"{name}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") + return 1 + pairs.append({"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load}) + ratios = [p["ratio"] for p in pairs] + results[name] = {"size": size, "compiled_functions": entry[1], "pairs": pairs, + "median_ratio": statistics.median(ratios), "min_ratio": min(ratios), + "max_ratio": max(ratios)} + print(f"measured {name}", file=sys.stderr) + + print() + print("| benchmark | size | speedup vs CPython (median of pairs) | range | load during pairs (1-min) |") + print("|---|---|---|---|---|") + for name, r in results.items(): + if "skipped" in r: + print(f"| {name} | — | {r['skipped']} | | |") + continue + loads = [p["load"][0] for p in r["pairs"]] # type: ignore[index] + print(f"| {name} | {r['size']} | {r['median_ratio']:.1f}x | {r['min_ratio']:.1f}–{r['max_ratio']:.1f}x " + f"| {min(loads):.1f}–{max(loads):.1f} |") + print(f"\n{args.pairs} A/B pairs per benchmark, order alternated; ratio = CPython time / compiled time.") + if args.write: + out_dir = ROOT / "results" + out_dir.mkdir(exist_ok=True) + path = out_dir / f"{datetime.date.today().isoformat()}-{socket.gethostname()}-interleaved.json" + path.write_text(json.dumps({"date": datetime.datetime.now().isoformat(timespec="seconds"), + "host": socket.gethostname(), "method": "interleaved A/B", + "pairs": args.pairs, "results": results}, indent=2) + "\n") + print(f"wrote {path}") + return 0 + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="cmd", required=True) @@ -205,7 +261,12 @@ def main() -> int: v.add_argument("--python", default=sys.executable) m = sub.add_parser("measure") m.add_argument("--python", default=sys.executable) - for p in (v, m): + ab = sub.add_parser("interleaved", help="CPython vs @compiled alternated A/B; reports median ratios") + ab.add_argument("--python", default=sys.executable) + ab.add_argument("--pairs", type=int, default=11) + ab.add_argument("--write", action="store_true") + ab.add_argument("--size", action="append", default=[], metavar="NAME=N") + for p in (v, m, ab): p.add_argument("--typedpython", metavar="DIR", help="directory holding the typedpython package; enables the @compiled column") p.add_argument("--typedpython-python", default=sys.executable, @@ -216,8 +277,10 @@ def main() -> int: m.add_argument("--size", action="append", default=[], metavar="NAME=N", help="override a benchmark size (for runner smoke tests only)") args = parser.parse_args() - if args.cmd == "measure": + if args.cmd in ("measure", "interleaved"): args.size = dict(s.split("=", 1) for s in args.size) + if args.cmd == "interleaved": + return interleaved(args) return verify(args) if args.cmd == "verify" else measure(args) From 4634d51015b1dc97bc3f8a6dcbfb1a6ff1dfc35e Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 18:54:11 +0900 Subject: [PATCH 2/2] Bench: Official interleaved A/B result for M2 (2026-10-03 18:46) 11 A/B pairs per benchmark at full size, order alternated, under 1-min load 12-36 (the machine had no quiet window all day). Median CPython/compiled speedup: nbody 28.5x (13.6-46.5), spectral-norm 17.9x (1.2-34.0), fannkuch-redux 29.2x (11.1-38.2), wordfreq 1.1x (0.9-1.6); binary-trees not compiled (M2b). Outputs identical in every pair. Refs #41 --- ...-03-irackui-Macmini.local-interleaved.json | 499 ++++++++++++++++++ 1 file changed, 499 insertions(+) create mode 100644 benchmarks/typedpython/results/2026-10-03-irackui-Macmini.local-interleaved.json diff --git a/benchmarks/typedpython/results/2026-10-03-irackui-Macmini.local-interleaved.json b/benchmarks/typedpython/results/2026-10-03-irackui-Macmini.local-interleaved.json new file mode 100644 index 000000000..bc94b7aa9 --- /dev/null +++ b/benchmarks/typedpython/results/2026-10-03-irackui-Macmini.local-interleaved.json @@ -0,0 +1,499 @@ +{ + "date": "2026-10-03T18:53:55", + "host": "irackui-Macmini.local", + "method": "interleaved A/B", + "pairs": 11, + "results": { + "nbody": { + "size": "1000000", + "compiled_functions": [ + "offset_momentum", + "advance", + "energy" + ], + "pairs": [ + { + "cpython_s": 6.8829106669873, + "compiled_s": 0.5073095830157399, + "ratio": 13.56747614754549, + "load": [ + 35.9130859375, + 65.2080078125, + 57.60107421875 + ] + }, + { + "cpython_s": 9.880680541973561, + "compiled_s": 0.21239254099782556, + "ratio": 46.52084529689165, + "load": [ + 34.23828125, + 64.3740234375, + 57.35107421875 + ] + }, + { + "cpython_s": 7.175678791012615, + "compiled_s": 0.22394145803991705, + "ratio": 32.04265460187174, + "load": [ + 29.453125, + 61.8271484375, + 56.56396484375 + ] + }, + { + "cpython_s": 5.955677040969022, + "compiled_s": 0.20967479201499373, + "ratio": 28.4043541130263, + "load": [ + 29.49658203125, + 61.29833984375, + 56.408203125 + ] + }, + { + "cpython_s": 6.802065708092414, + "compiled_s": 0.173232915927656, + "ratio": 39.26543446819906, + "load": [ + 27.69482421875, + 60.396484375, + 56.11865234375 + ] + }, + { + "cpython_s": 7.18817412503995, + "compiled_s": 0.1662108750315383, + "ratio": 43.24731533767572, + "load": [ + 23.97802734375, + 58.5224609375, + 55.50341796875 + ] + }, + { + "cpython_s": 6.393004250014201, + "compiled_s": 0.231951666995883, + "ratio": 27.561794803258184, + "load": [ + 23.1787109375, + 57.783203125, + 55.259765625 + ] + }, + { + "cpython_s": 6.450308125000447, + "compiled_s": 0.40894424996804446, + "ratio": 15.77307450955597, + "load": [ + 22.04296875, + 56.97314453125, + 54.98828125 + ] + }, + { + "cpython_s": 6.900172666995786, + "compiled_s": 0.28140733297914267, + "ratio": 24.520230492739906, + "load": [ + 23.5185546875, + 56.134765625, + 54.7138671875 + ] + }, + { + "cpython_s": 8.56268395902589, + "compiled_s": 0.30038412497378886, + "ratio": 28.505780589347253, + "load": [ + 22.99609375, + 55.48486328125, + 54.49267578125 + ] + }, + { + "cpython_s": 7.601551624946296, + "compiled_s": 0.18389916594605893, + "ratio": 41.33543284897754, + "load": [ + 21.09033203125, + 54.00390625, + 53.978515625 + ] + } + ], + "median_ratio": 28.505780589347253, + "min_ratio": 13.56747614754549, + "max_ratio": 46.52084529689165 + }, + "spectral-norm": { + "size": "800", + "compiled_functions": [ + "eval_a", + "mul_av", + "mul_atv" + ], + "pairs": [ + { + "cpython_s": 4.924908375018276, + "compiled_s": 4.0018195828888565, + "ratio": 1.2306672684786693, + "load": [ + 18.984375, + 52.47119140625, + 53.43505859375 + ] + }, + { + "cpython_s": 9.198930458980612, + "compiled_s": 0.32776887505315244, + "ratio": 28.065295881095707, + "load": [ + 17.8642578125, + 51.6826171875, + 53.15087890625 + ] + }, + { + "cpython_s": 6.263924417085946, + "compiled_s": 0.398825958953239, + "ratio": 15.705909498785582, + "load": [ + 16.84716796875, + 50.34423828125, + 52.658203125 + ] + }, + { + "cpython_s": 6.872534624999389, + "compiled_s": 0.49479795806109905, + "ratio": 13.889577580170105, + "load": [ + 15.78759765625, + 49.01513671875, + 52.15966796875 + ] + }, + { + "cpython_s": 7.051057124976069, + "compiled_s": 0.3943999590119347, + "ratio": 17.877935744822683, + "load": [ + 15.40380859375, + 48.3837890625, + 51.91845703125 + ] + }, + { + "cpython_s": 8.140147833968513, + "compiled_s": 0.2962452080100775, + "ratio": 27.4777367325098, + "load": [ + 15.10107421875, + 47.23388671875, + 51.46875 + ] + }, + { + "cpython_s": 8.249426209018566, + "compiled_s": 0.29253941599745303, + "ratio": 28.199366505504983, + "load": [ + 15.0927734375, + 46.69873046875, + 51.2548828125 + ] + }, + { + "cpython_s": 5.723211084026843, + "compiled_s": 0.3193328339839354, + "ratio": 17.922400940188815, + "load": [ + 16.6826171875, + 46.0, + 50.953125 + ] + }, + { + "cpython_s": 10.588757708086632, + "compiled_s": 0.31157975003588945, + "ratio": 33.984101042724895, + "load": [ + 16.6279296875, + 45.501953125, + 50.748046875 + ] + }, + { + "cpython_s": 5.798110875068232, + "compiled_s": 0.3391578750452027, + "ratio": 17.09561033868214, + "load": [ + 17.20361328125, + 44.6640625, + 50.388671875 + ] + }, + { + "cpython_s": 6.154926875024103, + "compiled_s": 0.3255474999314174, + "ratio": 18.906386552870938, + "load": [ + 21.20751953125, + 44.6103515625, + 50.30224609375 + ] + } + ], + "median_ratio": 17.922400940188815, + "min_ratio": 1.2306672684786693, + "max_ratio": 33.984101042724895 + }, + "fannkuch-redux": { + "size": "10", + "compiled_functions": [ + "fannkuch" + ], + "pairs": [ + { + "cpython_s": 9.915836833999492, + "compiled_s": 0.8964975839480758, + "ratio": 11.060639773652538, + "load": [ + 23.1123046875, + 44.61669921875, + 50.27099609375 + ] + }, + { + "cpython_s": 10.107461291947402, + "compiled_s": 0.32782216591294855, + "ratio": 30.832147252151902, + "load": [ + 21.8037109375, + 43.62548828125, + 49.8525390625 + ] + }, + { + "cpython_s": 9.16554791701492, + "compiled_s": 0.4036110829329118, + "ratio": 22.708860842997257, + "load": [ + 20.40185546875, + 42.60107421875, + 49.41552734375 + ] + }, + { + "cpython_s": 10.748112207977101, + "compiled_s": 0.37328437506221235, + "ratio": 28.79336218181058, + "load": [ + 20.79443359375, + 41.955078125, + 49.1064453125 + ] + }, + { + "cpython_s": 10.753968499950133, + "compiled_s": 0.4365679590264335, + "ratio": 24.632977014465226, + "load": [ + 19.36181640625, + 40.95166015625, + 48.66650390625 + ] + }, + { + "cpython_s": 9.816678624949418, + "compiled_s": 0.4199373750016093, + "ratio": 23.37652995261924, + "load": [ + 18.376953125, + 40.0302734375, + 48.24951171875 + ] + }, + { + "cpython_s": 12.343774041044526, + "compiled_s": 0.4121953749563545, + "ratio": 29.946415682979346, + "load": [ + 17.77099609375, + 39.1884765625, + 47.85498046875 + ] + }, + { + "cpython_s": 14.80802058300469, + "compiled_s": 0.4219444999471307, + "ratio": 35.09471170938387, + "load": [ + 17.26904296875, + 38.0078125, + 47.28076171875 + ] + }, + { + "cpython_s": 11.679267625091597, + "compiled_s": 0.3056710830423981, + "ratio": 38.20861138988285, + "load": [ + 18.07421875, + 37.16015625, + 46.8154296875 + ] + }, + { + "cpython_s": 11.083661083946936, + "compiled_s": 0.34054337500128895, + "ratio": 32.54698783643929, + "load": [ + 20.28173828125, + 37.005859375, + 46.64794921875 + ] + }, + { + "cpython_s": 8.800149749964476, + "compiled_s": 0.3017941250000149, + "ratio": 29.15944685790037, + "load": [ + 18.33642578125, + 35.74951171875, + 46.0302734375 + ] + } + ], + "median_ratio": 29.15944685790037, + "min_ratio": 11.060639773652538, + "max_ratio": 38.20861138988285 + }, + "binary-trees": { + "skipped": "n/a (nothing compiled)" + }, + "wordfreq": { + "size": "3000000", + "compiled_functions": [ + "next_state" + ], + "pairs": [ + { + "cpython_s": 3.5079852089984342, + "compiled_s": 3.2369775410043076, + "ratio": 1.0837224431004373, + "load": [ + 16.8203125, + 34.8515625, + 45.59130859375 + ] + }, + { + "cpython_s": 2.966251999954693, + "compiled_s": 3.1815978749655187, + "ratio": 0.9323151814045137, + "load": [ + 15.95361328125, + 34.37255859375, + 45.35888671875 + ] + }, + { + "cpython_s": 3.2128454169724137, + "compiled_s": 3.2285989999072626, + "ratio": 0.9951206133263061, + "load": [ + 15.396484375, + 33.951171875, + 45.1455078125 + ] + }, + { + "cpython_s": 3.67526158306282, + "compiled_s": 3.305175666930154, + "ratio": 1.1119716328047404, + "load": [ + 14.8037109375, + 33.52001953125, + 44.927734375 + ] + }, + { + "cpython_s": 3.4512535840040073, + "compiled_s": 3.7556770410155877, + "ratio": 0.9189431216563658, + "load": [ + 14.0712890625, + 32.74560546875, + 44.51904296875 + ] + }, + { + "cpython_s": 3.1576330839889124, + "compiled_s": 2.782175458036363, + "ratio": 1.134951095506228, + "load": [ + 13.6650390625, + 32.35107421875, + 44.310546875 + ] + }, + { + "cpython_s": 3.071971583063714, + "compiled_s": 2.6286917090183124, + "ratio": 1.168631366137243, + "load": [ + 13.69140625, + 32.04638671875, + 44.1328125 + ] + }, + { + "cpython_s": 3.205945707974024, + "compiled_s": 3.273688666988164, + "ratio": 0.9793068413324519, + "load": [ + 12.26806640625, + 31.138671875, + 43.66943359375 + ] + }, + { + "cpython_s": 3.042461832985282, + "compiled_s": 2.450655375025235, + "ratio": 1.2414890579847249, + "load": [ + 13.28759765625, + 31.03662109375, + 43.5595703125 + ] + }, + { + "cpython_s": 3.65517991699744, + "compiled_s": 2.2709438339807093, + "ratio": 1.6095421922392157, + "load": [ + 12.78369140625, + 30.63720703125, + 43.34521484375 + ] + }, + { + "cpython_s": 4.242163749993779, + "compiled_s": 4.385258833062835, + "ratio": 0.9673690679350133, + "load": [ + 12.0, + 30.17822265625, + 43.1083984375 + ] + } + ], + "median_ratio": 1.0837224431004373, + "min_ratio": 0.9189431216563658, + "max_ratio": 1.6095421922392157 + } + } +}