Skip to content

Commit a6f0d37

Browse files
committed
Bench: Run the TypedPython @compiled column through the pipeline
With --typedpython <dir> (and --typedpython-python for the compile step), each benchmark is copied with the module marker, compiled by typedpython.pipeline, and its main() runs from the built extension. verify includes the column in the agreement check; benchmarks with nothing compiled report n/a. verify: all 5 agree. Refs #24, #41
1 parent 68d64a1 commit a6f0d37

2 files changed

Lines changed: 74 additions & 9 deletions

File tree

‎benchmarks/typedpython/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ target/
22
java-out/
33
__pycache__/
44
*.class
5+
tp-build/

‎benchmarks/typedpython/run.py‎

Lines changed: 73 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@
77
`verify` runs every implementation at a small size and requires identical output.
88
`measure` reports the median wall time over N runs at benchmark size. The Java column runs
99
pre-compiled classes (javac -d java-out), so javac time is excluded; JVM startup is included.
10-
The "TypedPython @compiled" column is a placeholder: that backend does not exist yet.
10+
The "TypedPython @compiled" column runs only with `--typedpython <dir>` (the directory that holds
11+
the `typedpython` package, python-multiplatform-ksp/src/main/python) and `--typedpython-python` (an
12+
interpreter with the gate's dependencies, e.g. that module's .venv): each benchmark is copied with
13+
the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is
14+
run from the built extension. Without the option the column reports n/a.
1115
"""
1216
import argparse
1317
import datetime
@@ -22,6 +26,7 @@
2226

2327
ROOT = Path(__file__).resolve().parent
2428
TARGET = ROOT / "target"
29+
TP_BUILD = ROOT / "tp-build"
2530
JAVA_OUT = ROOT / "java-out"
2631
CARGO = os.environ.get("CARGO", "/Users/ibrew/.cargo/bin/cargo")
2732
NODE = os.environ.get("NODE", "/Users/ibrew/.local/bin/node")
@@ -54,16 +59,61 @@ def build() -> None:
5459
run_checked(["javac", "-d", str(JAVA_OUT), *sources])
5560

5661

57-
def commands(name: str, size: str, python: str, java_source: bool) -> dict[str, list[str]]:
62+
COMPILE_SNIPPET = """
63+
import json, sys
64+
from pathlib import Path
65+
from typedpython import pipeline
66+
r = pipeline.compile_module(Path(sys.argv[1]), Path(sys.argv[2]))
67+
print(json.dumps({"extension": str(r.extension) if r.extension else None,
68+
"compiled": [f.name for f in r.module.functions], "skipped": r.skipped}))
69+
"""
70+
71+
RUN_SNIPPET = """
72+
import importlib.machinery, importlib.util, sys
73+
name, path = sys.argv[1], sys.argv[2]
74+
loader = importlib.machinery.ExtensionFileLoader(name, path)
75+
spec = importlib.util.spec_from_file_location(name, path, loader=loader)
76+
mod = importlib.util.module_from_spec(spec)
77+
loader.exec_module(mod)
78+
sys.argv = [name] + sys.argv[3:]
79+
mod.main()
80+
"""
81+
82+
83+
def build_compiled(tp_dir: Path, tp_python: str) -> dict[str, tuple[str, list[str]] | str]:
84+
"""Compile each benchmark with TypedPython; name -> (extension, compiled functions) or a reason."""
85+
built: dict[str, tuple[str, list[str]] | str] = {}
86+
env = dict(os.environ, PYTHONPATH=str(tp_dir))
87+
for name, (py, *_rest) in BENCHMARKS.items():
88+
work = TP_BUILD / name
89+
work.mkdir(parents=True, exist_ok=True)
90+
source = work / py
91+
source.write_text("# typedpython: compiled\n" + (ROOT / "py" / py).read_text())
92+
result = subprocess.run([tp_python, "-c", COMPILE_SNIPPET, str(source), str(work / "out")],
93+
capture_output=True, text=True, env=env)
94+
if result.returncode != 0:
95+
built[name] = f"n/a (compile failed: {result.stderr.strip().splitlines()[-1:]})"
96+
continue
97+
info = json.loads(result.stdout.strip().splitlines()[-1])
98+
built[name] = (info["extension"], info["compiled"]) if info["extension"] else "n/a (nothing compiled)"
99+
return built
100+
101+
102+
def commands(name: str, size: str, python: str, java_source: bool,
103+
compiled: dict[str, tuple[str, list[str]] | str] | None = None) -> dict[str, list[str]]:
58104
py, js, cls, rs, _, _ = BENCHMARKS[name]
59105
java = ["java", str(ROOT / "java" / f"{cls}.java"), size] if java_source \
60106
else ["java", "-cp", str(JAVA_OUT), cls, size]
61-
return {
107+
cmds = {
62108
"CPython": [python, str(ROOT / "py" / py), size],
63109
"Node": [NODE, str(ROOT / "js" / js), size],
64110
"Java": java,
65111
"Rust": [str(TARGET / "release" / rs), size],
66112
}
113+
entry = (compiled or {}).get(name)
114+
if isinstance(entry, tuple):
115+
cmds["TypedPython @compiled"] = [python, "-c", RUN_SNIPPET, Path(py).stem, entry[0], size]
116+
return cmds
67117

68118

69119
def run_once(cmd: list[str]) -> tuple[float, str]:
@@ -78,9 +128,11 @@ def run_once(cmd: list[str]) -> tuple[float, str]:
78128

79129
def verify(args: argparse.Namespace) -> int:
80130
build()
131+
compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None
81132
bad = 0
82133
for name, spec in BENCHMARKS.items():
83-
outputs = {col: run_once(cmd)[1] for col, cmd in commands(name, spec[4], args.python, True).items()}
134+
outputs = {col: run_once(cmd)[1]
135+
for col, cmd in commands(name, spec[4], args.python, True, compiled).items()}
84136
distinct = set(outputs.values())
85137
status = "OK " if len(distinct) == 1 else "FAIL"
86138
print(f"{status} {name} (size {spec[4]}): {next(iter(outputs.values())) if len(distinct) == 1 else ''}")
@@ -100,19 +152,25 @@ def measure(args: argparse.Namespace) -> int:
100152
print(f"refusing to measure: 1-minute load {load[0]:.2f} > {LOAD_LIMIT} (use --force to override)")
101153
return 2
102154
build()
155+
compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None
103156
results: dict[str, dict[str, object]] = {}
104157
for name, spec in BENCHMARKS.items():
105158
size = args.size.get(name, spec[5])
106159
row: dict[str, object] = {"size": size}
107-
for col, cmd in commands(name, size, args.python, False).items():
160+
entry = (compiled or {}).get(name, NOT_BUILT)
161+
if isinstance(entry, tuple):
162+
row["compiled_functions"] = entry[1]
163+
for col, cmd in commands(name, size, args.python, False, compiled).items():
108164
runs = [run_once(cmd) for _ in range(args.repeat)]
109165
outs = {out for _, out in runs}
110166
row[col] = {"median_s": statistics.median(t for t, _ in runs),
111167
"runs_s": [t for t, _ in runs], "output": sorted(outs)}
112168
if len(outs) != 1:
113169
print(f"warning: {name}/{col} produced differing outputs across runs")
114-
row["TypedPython @compiled"] = NOT_BUILT
115-
outs_all = {tuple(row[c]["output"]) for c in COLUMNS[:4]} # type: ignore[index]
170+
if not isinstance(entry, tuple):
171+
row["TypedPython @compiled"] = entry
172+
measured = [c for c in COLUMNS if isinstance(row.get(c), dict)]
173+
outs_all = {tuple(row[c]["output"]) for c in measured} # type: ignore[index]
116174
row["outputs_agree"] = len(outs_all) == 1
117175
results[name] = row
118176
print(f"measured {name}", file=sys.stderr)
@@ -121,9 +179,10 @@ def measure(args: argparse.Namespace) -> int:
121179
print("| benchmark | size | " + " | ".join(COLUMNS) + " |")
122180
print("|---|---|" + "---|" * len(COLUMNS))
123181
for name, row in results.items():
124-
cells = [f"{row[c]['median_s']:.3f} s" for c in COLUMNS[:4]] # type: ignore[index]
182+
cells = [f"{row[c]['median_s']:.3f} s" if isinstance(row.get(c), dict) else str(row.get(c, NOT_BUILT))
183+
for c in COLUMNS] # type: ignore[index]
125184
flag = "" if row["outputs_agree"] else " (OUTPUTS DIFFER)"
126-
print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells + [NOT_BUILT]) + " |")
185+
print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells) + " |")
127186
print(f"\nmedian of {args.repeat} runs, wall time; Java excludes javac, includes JVM startup.")
128187

129188
if args.write:
@@ -146,6 +205,11 @@ def main() -> int:
146205
v.add_argument("--python", default=sys.executable)
147206
m = sub.add_parser("measure")
148207
m.add_argument("--python", default=sys.executable)
208+
for p in (v, m):
209+
p.add_argument("--typedpython", metavar="DIR",
210+
help="directory holding the typedpython package; enables the @compiled column")
211+
p.add_argument("--typedpython-python", default=sys.executable,
212+
help="interpreter for compiling (needs pyrefly 1.3.2)")
149213
m.add_argument("--repeat", type=int, default=5)
150214
m.add_argument("--force", action="store_true")
151215
m.add_argument("--write", action="store_true")

0 commit comments

Comments
 (0)