77`verify` runs every implementation at a small size and requires identical output.
88`measure` reports the median wall time over N runs at benchmark size. The Java column runs
99pre-compiled classes (javac -d java-out), so javac time is excluded; JVM startup is included.
10- The "TypedPython @compiled" column is a placeholder: that backend does not exist yet.
10+ The "TypedPython @compiled" column runs only with `--typedpython <dir>` (the directory that holds
11+ the `typedpython` package, python-multiplatform-ksp/src/main/python) and `--typedpython-python` (an
12+ interpreter with the gate's dependencies, e.g. that module's .venv): each benchmark is copied with
13+ the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is
14+ run from the built extension. Without the option the column reports n/a.
1115"""
1216import argparse
1317import datetime
2226
2327ROOT = Path (__file__ ).resolve ().parent
2428TARGET = ROOT / "target"
29+ TP_BUILD = ROOT / "tp-build"
2530JAVA_OUT = ROOT / "java-out"
2631CARGO = os .environ .get ("CARGO" , "/Users/ibrew/.cargo/bin/cargo" )
2732NODE = os .environ .get ("NODE" , "/Users/ibrew/.local/bin/node" )
@@ -54,16 +59,61 @@ def build() -> None:
5459 run_checked (["javac" , "-d" , str (JAVA_OUT ), * sources ])
5560
5661
57- def commands (name : str , size : str , python : str , java_source : bool ) -> dict [str , list [str ]]:
62+ COMPILE_SNIPPET = """
63+ import json, sys
64+ from pathlib import Path
65+ from typedpython import pipeline
66+ r = pipeline.compile_module(Path(sys.argv[1]), Path(sys.argv[2]))
67+ print(json.dumps({"extension": str(r.extension) if r.extension else None,
68+ "compiled": [f.name for f in r.module.functions], "skipped": r.skipped}))
69+ """
70+
71+ RUN_SNIPPET = """
72+ import importlib.machinery, importlib.util, sys
73+ name, path = sys.argv[1], sys.argv[2]
74+ loader = importlib.machinery.ExtensionFileLoader(name, path)
75+ spec = importlib.util.spec_from_file_location(name, path, loader=loader)
76+ mod = importlib.util.module_from_spec(spec)
77+ loader.exec_module(mod)
78+ sys.argv = [name] + sys.argv[3:]
79+ mod.main()
80+ """
81+
82+
83+ def build_compiled (tp_dir : Path , tp_python : str ) -> dict [str , tuple [str , list [str ]] | str ]:
84+ """Compile each benchmark with TypedPython; name -> (extension, compiled functions) or a reason."""
85+ built : dict [str , tuple [str , list [str ]] | str ] = {}
86+ env = dict (os .environ , PYTHONPATH = str (tp_dir ))
87+ for name , (py , * _rest ) in BENCHMARKS .items ():
88+ work = TP_BUILD / name
89+ work .mkdir (parents = True , exist_ok = True )
90+ source = work / py
91+ source .write_text ("# typedpython: compiled\n " + (ROOT / "py" / py ).read_text ())
92+ result = subprocess .run ([tp_python , "-c" , COMPILE_SNIPPET , str (source ), str (work / "out" )],
93+ capture_output = True , text = True , env = env )
94+ if result .returncode != 0 :
95+ built [name ] = f"n/a (compile failed: { result .stderr .strip ().splitlines ()[- 1 :]} )"
96+ continue
97+ info = json .loads (result .stdout .strip ().splitlines ()[- 1 ])
98+ built [name ] = (info ["extension" ], info ["compiled" ]) if info ["extension" ] else "n/a (nothing compiled)"
99+ return built
100+
101+
102+ def commands (name : str , size : str , python : str , java_source : bool ,
103+ compiled : dict [str , tuple [str , list [str ]] | str ] | None = None ) -> dict [str , list [str ]]:
58104 py , js , cls , rs , _ , _ = BENCHMARKS [name ]
59105 java = ["java" , str (ROOT / "java" / f"{ cls } .java" ), size ] if java_source \
60106 else ["java" , "-cp" , str (JAVA_OUT ), cls , size ]
61- return {
107+ cmds = {
62108 "CPython" : [python , str (ROOT / "py" / py ), size ],
63109 "Node" : [NODE , str (ROOT / "js" / js ), size ],
64110 "Java" : java ,
65111 "Rust" : [str (TARGET / "release" / rs ), size ],
66112 }
113+ entry = (compiled or {}).get (name )
114+ if isinstance (entry , tuple ):
115+ cmds ["TypedPython @compiled" ] = [python , "-c" , RUN_SNIPPET , Path (py ).stem , entry [0 ], size ]
116+ return cmds
67117
68118
69119def run_once (cmd : list [str ]) -> tuple [float , str ]:
@@ -78,9 +128,11 @@ def run_once(cmd: list[str]) -> tuple[float, str]:
78128
79129def verify (args : argparse .Namespace ) -> int :
80130 build ()
131+ compiled = build_compiled (Path (args .typedpython ), args .typedpython_python ) if args .typedpython else None
81132 bad = 0
82133 for name , spec in BENCHMARKS .items ():
83- outputs = {col : run_once (cmd )[1 ] for col , cmd in commands (name , spec [4 ], args .python , True ).items ()}
134+ outputs = {col : run_once (cmd )[1 ]
135+ for col , cmd in commands (name , spec [4 ], args .python , True , compiled ).items ()}
84136 distinct = set (outputs .values ())
85137 status = "OK " if len (distinct ) == 1 else "FAIL"
86138 print (f"{ status } { name } (size { spec [4 ]} ): { next (iter (outputs .values ())) if len (distinct ) == 1 else '' } " )
@@ -100,19 +152,25 @@ def measure(args: argparse.Namespace) -> int:
100152 print (f"refusing to measure: 1-minute load { load [0 ]:.2f} > { LOAD_LIMIT } (use --force to override)" )
101153 return 2
102154 build ()
155+ compiled = build_compiled (Path (args .typedpython ), args .typedpython_python ) if args .typedpython else None
103156 results : dict [str , dict [str , object ]] = {}
104157 for name , spec in BENCHMARKS .items ():
105158 size = args .size .get (name , spec [5 ])
106159 row : dict [str , object ] = {"size" : size }
107- for col , cmd in commands (name , size , args .python , False ).items ():
160+ entry = (compiled or {}).get (name , NOT_BUILT )
161+ if isinstance (entry , tuple ):
162+ row ["compiled_functions" ] = entry [1 ]
163+ for col , cmd in commands (name , size , args .python , False , compiled ).items ():
108164 runs = [run_once (cmd ) for _ in range (args .repeat )]
109165 outs = {out for _ , out in runs }
110166 row [col ] = {"median_s" : statistics .median (t for t , _ in runs ),
111167 "runs_s" : [t for t , _ in runs ], "output" : sorted (outs )}
112168 if len (outs ) != 1 :
113169 print (f"warning: { name } /{ col } produced differing outputs across runs" )
114- row ["TypedPython @compiled" ] = NOT_BUILT
115- outs_all = {tuple (row [c ]["output" ]) for c in COLUMNS [:4 ]} # type: ignore[index]
170+ if not isinstance (entry , tuple ):
171+ row ["TypedPython @compiled" ] = entry
172+ measured = [c for c in COLUMNS if isinstance (row .get (c ), dict )]
173+ outs_all = {tuple (row [c ]["output" ]) for c in measured } # type: ignore[index]
116174 row ["outputs_agree" ] = len (outs_all ) == 1
117175 results [name ] = row
118176 print (f"measured { name } " , file = sys .stderr )
@@ -121,9 +179,10 @@ def measure(args: argparse.Namespace) -> int:
121179 print ("| benchmark | size | " + " | " .join (COLUMNS ) + " |" )
122180 print ("|---|---|" + "---|" * len (COLUMNS ))
123181 for name , row in results .items ():
124- cells = [f"{ row [c ]['median_s' ]:.3f} s" for c in COLUMNS [:4 ]] # type: ignore[index]
182+ cells = [f"{ row [c ]['median_s' ]:.3f} s" if isinstance (row .get (c ), dict ) else str (row .get (c , NOT_BUILT ))
183+ for c in COLUMNS ] # type: ignore[index]
125184 flag = "" if row ["outputs_agree" ] else " (OUTPUTS DIFFER)"
126- print (f"| { name } { flag } | { row ['size' ]} | " + " | " .join (cells + [ NOT_BUILT ] ) + " |" )
185+ print (f"| { name } { flag } | { row ['size' ]} | " + " | " .join (cells ) + " |" )
127186 print (f"\n median of { args .repeat } runs, wall time; Java excludes javac, includes JVM startup." )
128187
129188 if args .write :
@@ -146,6 +205,11 @@ def main() -> int:
146205 v .add_argument ("--python" , default = sys .executable )
147206 m = sub .add_parser ("measure" )
148207 m .add_argument ("--python" , default = sys .executable )
208+ for p in (v , m ):
209+ p .add_argument ("--typedpython" , metavar = "DIR" ,
210+ help = "directory holding the typedpython package; enables the @compiled column" )
211+ p .add_argument ("--typedpython-python" , default = sys .executable ,
212+ help = "interpreter for compiling (needs pyrefly 1.3.2)" )
149213 m .add_argument ("--repeat" , type = int , default = 5 )
150214 m .add_argument ("--force" , action = "store_true" )
151215 m .add_argument ("--write" , action = "store_true" )
0 commit comments