Skip to content

Commit 913ab17

Browse files
authored
Merge pull request #253 from thisisthepy/feat/tp-dict-str
Feat: Strings, subscripts, iteration and f-strings compile through the interpreter's C API; cost rule (#150)
2 parents 20944cb + 7f9ae1e commit 913ab17

19 files changed

Lines changed: 2228 additions & 62 deletions
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
{
2+
"wordfreq": {
3+
"size": "3000000",
4+
"pairs": [
5+
{
6+
"before_s": 2.534866082947701,
7+
"after_s": 2.4698741249740124,
8+
"cpython_s": 2.246576082892716,
9+
"after_over_before": 0.9743607923073743,
10+
"speedup_before": 0.8862701260653014,
11+
"speedup_after": 0.9095913270140251,
12+
"load": [
13+
2.22265625,
14+
2.5498046875,
15+
3.80224609375
16+
]
17+
},
18+
{
19+
"after_s": 2.054086625110358,
20+
"before_s": 2.0703248330391943,
21+
"cpython_s": 2.226471666712314,
22+
"after_over_before": 0.9921566859124232,
23+
"speedup_before": 1.0754214175385701,
24+
"speedup_after": 1.0839229658061253,
25+
"load": [
26+
2.1943359375,
27+
2.53173828125,
28+
3.78076171875
29+
]
30+
},
31+
{
32+
"before_s": 2.0203522499650717,
33+
"after_s": 2.0748487496748567,
34+
"cpython_s": 2.2474577920511365,
35+
"after_over_before": 1.0269737614867542,
36+
"speedup_before": 1.112408884188384,
37+
"speedup_after": 1.083191144609133,
38+
"load": [
39+
2.0986328125,
40+
2.505859375,
41+
3.76416015625
42+
]
43+
},
44+
{
45+
"after_s": 2.057975041680038,
46+
"before_s": 2.072377833072096,
47+
"cpython_s": 2.225645957980305,
48+
"after_over_before": 0.9930501131781035,
49+
"speedup_before": 1.0739576164453584,
50+
"speedup_after": 1.081473736514991,
51+
"load": [
52+
2.17041015625,
53+
2.513671875,
54+
3.75927734375
55+
]
56+
},
57+
{
58+
"before_s": 2.0653403750620782,
59+
"after_s": 2.0860109999775887,
60+
"cpython_s": 2.24187570810318,
61+
"after_over_before": 1.0100083381727765,
62+
"speedup_before": 1.0854751764758366,
63+
"speedup_after": 1.0747190250325938,
64+
"load": [
65+
2.39697265625,
66+
2.5546875,
67+
3.76611328125
68+
]
69+
},
70+
{
71+
"after_s": 2.047571041621268,
72+
"before_s": 2.0459034591913223,
73+
"cpython_s": 2.240035249851644,
74+
"after_over_before": 1.0008150836357668,
75+
"speedup_before": 1.0948880504542748,
76+
"speedup_after": 1.093996351930228,
77+
"load": [
78+
2.3291015625,
79+
2.53564453125,
80+
3.74462890625
81+
]
82+
},
83+
{
84+
"before_s": 2.064202000387013,
85+
"after_s": 2.066363917198032,
86+
"cpython_s": 2.2770254579372704,
87+
"after_over_before": 1.0010473378141354,
88+
"speedup_before": 1.1031020498528514,
89+
"speedup_after": 1.1019479381080624,
90+
"load": [
91+
2.22265625,
92+
2.509765625,
93+
3.728515625
94+
]
95+
},
96+
{
97+
"after_s": 2.047510417178273,
98+
"before_s": 2.029783667065203,
99+
"cpython_s": 2.238690916914493,
100+
"after_over_before": 1.0087333199102446,
101+
"speedup_before": 1.1029209433689757,
102+
"speedup_after": 1.093372174389076,
103+
"load": [
104+
2.12451171875,
105+
2.484375,
106+
3.71240234375
107+
]
108+
},
109+
{
110+
"before_s": 2.051607625093311,
111+
"after_s": 2.0582683752290905,
112+
"cpython_s": 2.2774744578637183,
113+
"after_over_before": 1.0032466003997604,
114+
"speedup_before": 1.1100926073815574,
115+
"speedup_after": 1.1065002432494886,
116+
"load": [
117+
1.951171875,
118+
2.43505859375,
119+
3.68017578125
120+
]
121+
},
122+
{
123+
"after_s": 2.0976750422269106,
124+
"before_s": 2.0433855829760432,
125+
"cpython_s": 2.259623875375837,
126+
"after_over_before": 1.0265683871429683,
127+
"speedup_before": 1.1058235382501125,
128+
"speedup_after": 1.0772039662430268,
129+
"load": [
130+
2.03515625,
131+
2.4443359375,
132+
3.67578125
133+
]
134+
},
135+
{
136+
"before_s": 2.05512237502262,
137+
"after_s": 2.072784666903317,
138+
"cpython_s": 2.2494797920808196,
139+
"after_over_before": 1.0085942774480778,
140+
"speedup_before": 1.0945721867565479,
141+
"speedup_after": 1.08524528765522,
142+
"load": [
143+
1.9521484375,
144+
2.419921875,
145+
3.65966796875
146+
]
147+
}
148+
],
149+
"median_after_over_before": 1.0032466003997604,
150+
"min_after_over_before": 0.9743607923073743,
151+
"max_after_over_before": 1.0269737614867542,
152+
"median_speedup_before": 1.0948880504542748,
153+
"median_speedup_after": 1.0839229658061253
154+
}
155+
}
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# wordfreq and object-only compiled code (#150, 2026-10-17)
2+
3+
Quiet windows, interleaved against develop, 11 pairs, CPython 3.15.0rc1t.
4+
5+
| build | after / before (median, range) | vs CPython |
6+
|---|---|---|
7+
| #150 without the cost rule (`word_of` and `main` compiled) | 1.335 (1.314 to 1.767) | 1.1x to **0.8x** |
8+
| #150 with the cost rule (only `next_state` compiled, as on develop) | 1.003 (0.974 to 1.027) | 1.1x to 1.1x |
9+
10+
Split, measured in one process: `word_of` compiled 248.6 ms against 95.7 ms interpreted (2.6x slower);
11+
`main` 332 ms against 345 ms.
12+
13+
Cause (read in the generated C): in an impure function an `int` is a CPython object, so each loop
14+
iteration boxes constants, calls generic `PyNumber_*` / `PyObject_RichCompare`, and looks up the
15+
global `chr` through the module and builtins dicts. CPython 3.15's specialising interpreter
16+
(`BINARY_OP_ADD_INT`, `LOAD_GLOBAL_BUILTIN` caches, the small-int cache) runs that faster.
17+
18+
Consequence: the cost rule (`typedpython/cost.py`) keeps an impure function interpreted when it has
19+
loops and none of them does native work. The real fix, native ints in impure functions with in-place
20+
promotion on overflow, pre-boxed constants and cached builtins, is #232.

‎python-multiplatform-ksp/src/main/python/typedpython/cgen.py‎

Lines changed: 109 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,12 @@ def _exprs_of_stmt(s: ir.Stmt):
272272
elif isinstance(s, ir.FieldSet):
273273
yield s.value
274274
yield s.obj
275+
elif isinstance(s, ir.StoreItem):
276+
yield s.value
277+
yield s.obj
278+
yield s.index
279+
elif isinstance(s, ir.ForObj):
280+
yield s.iter
275281
elif isinstance(s, ir.ExprStmt):
276282
yield s.value
277283
elif isinstance(s, ir.If):
@@ -288,7 +294,7 @@ def _exprs_of_stmt(s: ir.Stmt):
288294
def _children(s: ir.Stmt):
289295
if isinstance(s, ir.If):
290296
return s.then + s.orelse
291-
if isinstance(s, (ir.While, ir.ForRange)):
297+
if isinstance(s, (ir.While, ir.ForRange, ir.ForObj)):
292298
return s.body
293299
return ()
294300

@@ -376,7 +382,8 @@ def _watch_event_stmt() -> str:
376382

377383

378384
def _is_effect(e: ir.Expr) -> bool:
379-
if isinstance(e, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat)):
385+
if isinstance(e, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat,
386+
ir.GetItem, ir.Contains, ir.Format)):
380387
return True
381388
return isinstance(e, ir.BinOp) and e.type is Type.OBJ
382389

@@ -387,7 +394,7 @@ def _is_effect(e: ir.Expr) -> bool:
387394
# Nodes that can run Python code (and so create Python frames the compiled frame count must see):
388395
# their slow paths call getattr, the class, __bool__, __float__, a Python callable, a descriptor.
389396
_RUNS_PYTHON = (ir.CallObject, ir.GetAttr, ir.Truth, ir.CompareObj, ir.ObjToFloat, ir.FieldGet,
390-
ir.FieldSet, ir.New)
397+
ir.FieldSet, ir.New, ir.GetItem, ir.Contains, ir.Format, ir.StoreItem, ir.ForObj)
391398

392399

393400
def _runs_no_python(f: ir.Function) -> bool:
@@ -403,7 +410,7 @@ def _runs_no_python(f: ir.Function) -> bool:
403410
if any(t is Type.OBJ for t in f.locals.values()):
404411
return False
405412
for s in _walk_stmts(f.body):
406-
if isinstance(s, ir.FieldSet):
413+
if isinstance(s, (ir.FieldSet, ir.StoreItem, ir.ForObj)):
407414
return False
408415
for e in _all_exprs(f):
409416
if isinstance(e, _RUNS_PYTHON):
@@ -664,7 +671,7 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None:
664671
# a store into a local array is not an effect (ir.py "Local arrays"); into a parameter it is
665672
stores = [s for s in _walk_stmts(f.body)
666673
if (isinstance(s, ir.StoreIndex) and f.locals.get(s.array) not in ir.ARRAYS)
667-
or isinstance(s, ir.FieldSet)]
674+
or isinstance(s, (ir.FieldSet, ir.StoreItem, ir.ForObj))]
668675
if f.pure and (effects or stores):
669676
what = type(effects[0]).__name__ if effects else type(stores[0]).__name__
670677
raise CGenError(f"{f.name}: marked pure but contains an effect ({what})")
@@ -725,6 +732,8 @@ def _is_closed(f: ir.Function, functions: dict[str, ir.Function], seen=None) ->
725732
if f.name in seen:
726733
return True
727734
seen.add(f.name)
735+
if any(isinstance(s, (ir.StoreItem, ir.ForObj)) for s in _walk_stmts(f.body)):
736+
return False
728737
for e in _all_exprs(f):
729738
if _is_effect(e):
730739
return False
@@ -893,6 +902,9 @@ def ev_Const(self, e: ir.Const) -> str:
893902
return self._own_copy("Py_None")
894903
if isinstance(e.value, bool):
895904
return self._own_copy("Py_True" if e.value else "Py_False")
905+
if type(e.value) is str and "\0" not in e.value:
906+
# a str constant: the interned object the module state holds (names table)
907+
return self._own_copy(f"tp_st->names[{self.mg.name_index(e.value)}]")
896908
raise CGenError(f"{self.f.name}: OBJ constant {e.value!r} (use Box of a scalar Const)")
897909
raise CGenError(f"{self.f.name}: constant of type {e.type}")
898910

@@ -934,7 +946,8 @@ def ev_BinOp(self, e: ir.BinOp) -> str:
934946
return t
935947
if lt is Type.OBJ:
936948
t = self.tmp("PyObject *")
937-
self.emit(f"{t} = tp_binop_obj({a}, {b}, {_OBJ_OP[e.op]});")
949+
op = _OBJ_OP[e.op] if not e.inplace else f"TP_BINOP_I{e.op.name}"
950+
self.emit(f"{t} = tp_binop_obj({a}, {b}, {op});")
938951
self._own_release(a, b)
939952
self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}")
940953
return t
@@ -1411,6 +1424,54 @@ def ev_Tuple(self, e: ir.Tuple) -> str:
14111424
self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}")
14121425
return t
14131426

1427+
def ev_GetItem(self, e: ir.GetItem) -> str:
1428+
o = self._obj_operand(e.obj, "GetItem") # the container, then the index
1429+
k = self._obj_operand(e.index, "GetItem index")
1430+
t = self.tmp("PyObject *")
1431+
self.emit(f"{t} = tp_getitem({o}, {k});")
1432+
self._own_release(o, k)
1433+
self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}")
1434+
return t
1435+
1436+
def ev_Contains(self, e: ir.Contains) -> str:
1437+
a = self._obj_operand(e.left, "Contains") # CONTAINS_OP: left, then right
1438+
b = self._obj_operand(e.right, "Contains")
1439+
t = self.tmp("int")
1440+
self.emit(f"{t} = tp_contains({b}, {a});")
1441+
self._own_release(a, b)
1442+
self.emit(f"if ({t} < 0) {{ tp_rc = -1; goto tp_exit; }}")
1443+
if e.negate:
1444+
self.emit(f"{t} = !{t};")
1445+
return t
1446+
1447+
def ev_NewList(self, e: ir.NewList) -> str:
1448+
return self._own_new("tp_new_list()")
1449+
1450+
def ev_NewDict(self, e: ir.NewDict) -> str:
1451+
return self._own_new("tp_new_dict()")
1452+
1453+
def ev_Format(self, e: ir.Format) -> str:
1454+
if e.conversion not in (-1, 97, 114, 115) or (e.spec is not None and "\0" in e.spec):
1455+
raise CGenError(f"{self.f.name}: Format conversion {e.conversion!r} / spec {e.spec!r}")
1456+
o = self._obj_operand(e.operand, "Format")
1457+
spec = "NULL" if e.spec is None else f"tp_st->names[{self.mg.name_index(e.spec)}]"
1458+
t = self.tmp("PyObject *")
1459+
self.emit(f"{t} = tp_format({o}, {spec}, {e.conversion});")
1460+
self._own_release(o)
1461+
self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}")
1462+
return t
1463+
1464+
def ev_BuildString(self, e: ir.BuildString) -> str:
1465+
if len(e.elements) < 2:
1466+
raise CGenError(f"{self.f.name}: BuildString of {len(e.elements)} elements")
1467+
items = [self._obj_operand(x, "BuildString element") for x in e.elements]
1468+
t = self.tmp("PyObject *")
1469+
self.emit(f"{{ PyObject *tp_av[{len(items)}] = {{{', '.join(items)}}};")
1470+
self.emit(f" {t} = tp_build_string(tp_av, {len(items)}); }}")
1471+
self._own_release(*items)
1472+
self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}")
1473+
return t
1474+
14141475
def _assign_array(self, s: ir.Assign, at: Type) -> None:
14151476
"""`local = NewArray | CopyArray`: build into a zeroed temporary, then (only on success)
14161477
free the old array and move the temporary in."""
@@ -1489,6 +1550,48 @@ def st_StoreIndex(self, s: ir.StoreIndex) -> None:
14891550
if not local: # a local array has no list to write back to
14901551
self.emit(f"{ptr}->dirty[{slot}] = 1;")
14911552

1553+
def st_StoreItem(self, s: ir.StoreItem) -> None:
1554+
v = self._obj_operand(s.value, "StoreItem value") # STORE_SUBSCR: value, container, index
1555+
o = self._obj_operand(s.obj, "StoreItem")
1556+
k = self._obj_operand(s.index, "StoreItem index")
1557+
self.emit(f"tp_s = tp_setitem({o}, {k}, {v});")
1558+
self._own_release(o, k, v)
1559+
self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }")
1560+
1561+
def st_ForObj(self, s: ir.ForObj) -> None:
1562+
if not s.targets or len(set(s.targets)) != len(s.targets) or (not s.unpack and len(s.targets) != 1):
1563+
raise CGenError(f"{self.f.name}: ForObj targets {s.targets!r} (unpack={s.unpack})")
1564+
for name in s.targets:
1565+
if self.types.get(name) is not Type.OBJ:
1566+
raise CGenError(f"{self.f.name}: ForObj target {name} must be an OBJ local")
1567+
o = self._obj_operand(s.iter, "ForObj iterable")
1568+
it = self.tmp("PyObject *") # the iterator: owned by this statement across iterations
1569+
self.emit(f"{it} = tp_get_iter({o});") # GET_ITER
1570+
self._own_release(o)
1571+
self.emit(f"if ({it} == NULL) {{ tp_rc = -1; goto tp_exit; }}")
1572+
item = self.tmp("PyObject *")
1573+
self.emit("for (;;) {")
1574+
self.depth += 1
1575+
self.poll()
1576+
self.emit(f"{item} = tp_iter_next({it});") # FOR_ITER (StopIteration ends the loop)
1577+
self.emit(f"if ({item} == NULL) {{ if (PyErr_Occurred()) {{ tp_rc = -1; goto tp_exit; }} break; }}")
1578+
if not s.unpack:
1579+
self._own_move(_ident("l", s.targets[0]), item)
1580+
else:
1581+
n = len(s.targets)
1582+
self.emit(f"{{ PyObject *tp_u[{n}] = {{0}};")
1583+
self.emit(f" tp_s = tp_unpack({item}, {n}, tp_u);") # UNPACK_SEQUENCE; nothing owned on error
1584+
self._own_release(item)
1585+
self.emit(" if (tp_s < 0) { tp_rc = -1; goto tp_exit; }")
1586+
for i, name in enumerate(s.targets):
1587+
local = _ident("l", name)
1588+
self.emit(f" {{ PyObject *tp_old = {local}; {local} = tp_u[{i}]; tp_release(&tp_old); }}")
1589+
self.emit("}")
1590+
self.stmts(s.body)
1591+
self.depth -= 1
1592+
self.emit("}")
1593+
self._own_release(it) # after the loop and after a `break`; a return/error exits via tp_exit
1594+
14921595
def st_FieldSet(self, s: ir.FieldSet) -> None:
14931596
slot = self.mg.slot(self.f.name, s.cls, s.field)
14941597
v = self._obj_operand(s.value, "FieldSet value") # CPython: value, then the object

0 commit comments

Comments
 (0)