diff --git a/docs/SPEC.md b/docs/SPEC.md index ae73cbc9f..701d589d2 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -437,7 +437,11 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. through the typed IR (`typedpython/ir.py`) to C against the CPython C API and builds an extension with the platform C compiler. Compiled functions return what CPython returns (IR contract: checked i64 with deopt to the interpreted function, Python floor division/modulo, exact int/float comparison, - CPython's exception types and messages; tracebacks are the one accepted difference). `Status: planned`. + CPython's exception types and messages; tracebacks are the one accepted difference). Call cycles + (direct and mutual recursion) are compiled: every compiled function counts its frame (`tp_enter_call` / + `tp_leave_call`, added to the Python frame depth and compared with `sys.getrecursionlimit()`), so recursion + past the limit raises `RecursionError('maximum recursion depth exceeded')` where CPython does, and a C-stack + guard keeps the process alive at any limit (#57). `Status: planned`. - **N-9** TypedPython memory safety is proved on the IR before C is generated (#41; maintainer decision 2026-10-03). Each property, and what happens when it cannot be proved: diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index de89dacfe..5a1b55260 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -169,6 +169,22 @@ Gradle 태스크는 입력·출력을 정확히 선언해 up-to-date 판정과 - **LPython** — 업스트림 활동이 LFortran 쪽으로 옮겨간 것으로 보인다. - **Rust 생성** — 위. +### 4.3.3 재귀 (SPEC N-8, #57) + +호출 사이클(직접·상호 재귀)은 컴파일한다. 컴파일된 프레임에는 파이썬 프레임이 없으므로 모든 컴파일 +함수의 진입에서 `tp_enter_call()`, 모든 출구(ok·error·deopt)에서 `tp_leave_call()` 을 부른다. 진입은 +스레드별 컴파일 프레임 수에 현재 파이썬 프레임 깊이(`PyThreadState_GetFrame`/`PyFrame_GetBack`, 공개 API 만, +최상위 프레임 동일성으로 캐시)를 더해 `Py_GetRecursionLimit()` 와 비교하고, 넘으면 CPython 과 같은 +`RecursionError("maximum recursion depth exceeded")` 를 낸다. 이어서 `Py_EnterRecursiveCall` 이 C 스택을 +지킨다. **측정(3.13.0, 3.14.7, 기본 한도 1000):** 인터프리터와 컴파일 재귀 모두 같은 깊이(997)에서 +실패하고, 인터프리터 프레임 k 개 위에서 컴파일 재귀를 시작해도 `최대 n + (추가 프레임 수)` 가 일정해 +오프셋이 없다. 오프셋이 생기는 곳은 둘이다. (1) 한도를 C 스택이 먼저 닫을 만큼 올리면(예: 100만) C 가드의 +메시지가 나온다 — 3.13 은 `maximum recursion depth exceeded in compiled code`, 3.14 는 +`Stack overflow (used 8144 kB) in compiled code` — 인터프리터는 같은 한도에서 파이썬 한도 메시지를 낸다. +(2) 카운터는 모듈별(헤더가 `static`)이라, 한 컴파일 모듈의 두 프레임 사이에 **다른** 컴파일 모듈의 +프레임이 끼면 그 프레임은 세지 않는다. 테스트는 두 경우 모두 "둘 다 예외를 내고 죽지 않는다" 만 단언하고 +깊이를 기록한다(`test_recursion.py`). + ### 4.4 Kotlin 경계의 직접 호출 이 플랫폼에서만 가능한 이점이다. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index f843bcac9..f5c4f7b6d 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -22,6 +22,10 @@ guard, or the impl's 1 in a pure function) increments `__typedpython_deopts__` and calls `__typedpython_interpreted__[f]` with the original arguments. +Depth (issue #57): every impl function calls `tp_enter_call()` before it owns anything (a refusal +returns -1 directly: RecursionError set, nothing counted) and `tp_leave_call()` in the one exit +block, so ok, error and deopt exits all leave exactly once. + Ownership rule (ir.py "Safety"), implemented by the `_own_*` emit helpers below and nowhere else: 1. every OBJ local — including each OBJ parameter, which the impl copies with a new reference at entry — owns one strong reference, or is NULL; @@ -728,10 +732,8 @@ def ev_Call(self, e: ir.Call) -> str | None: args.append(f"&{t}") self.emit("if (tp_s == 0) {") self.depth += 1 - # C-to-C recursion has no Python frame: guard the C stack as CPython guards its own. - self.emit('if (Py_EnterRecursiveCall(" in compiled code")) { tp_rc = -1; goto tp_exit; }') + # Depth is counted by the callee itself (tp_enter_call at its entry, issue #57). self.emit(f"tp_s = {_ident('tp_impl', e.function)}(tp_module{''.join(', ' + x for x in args)});") - self.emit("Py_LeaveRecursiveCall();") self.depth -= 1 self.emit("}") if e.redo: @@ -1096,10 +1098,14 @@ def generate(self) -> str: out.append(" (void)tp_st; (void)tp_dict; (void)tp_s;" + "".join(f" (void){_ident('l', n)};" for n in self.types if n not in params) + "".join(f" (void){_ident('l', n)};" for n in self.local_arrays)) + # Depth guard (issue #57): nothing is owned yet, so a refusal returns directly and counts + # nothing; every other exit goes through tp_exit, which leaves exactly once. + out.append(" if (tp_enter_call() != 0) return -1;") out += prologue out += self.lines out.append("tp_exit:") out += self._own_release_all() + out.append(" tp_leave_call();") out.append(" return tp_rc;") out.append("}") out.append("") diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 0d841f704..8f78ce128 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -146,12 +146,6 @@ def lower(path: Path) -> ir.Module: except Skip as e: skipped[node.name] = e.reason - for name, cycle in _call_cycles(signatures).items(): - skipped[name] = (f"line {signatures[name].node.lineno}: recursion " - f"({' -> '.join(cycle)}) is not compiled: C has no recursion limit " - "where CPython raises RecursionError") - del signatures[name] - kinds = _inferred_kinds(path, source, tree, [s.node for s in signatures.values()]) # Optimistic start: every candidate lowered, pure, not deopting; then iterate to a fixpoint @@ -182,30 +176,6 @@ def lower(path: Path) -> ir.Module: return ir.Module(path.stem, tuple(functions), skipped) -def _call_cycles(signatures: dict[str, _Signature]) -> dict[str, list[str]]: - """Candidates on a call cycle through calls by name to other candidates (syntactic, so a - call that would end up as CallObject is counted too: conservative).""" - calls = { - name: {n.func.id for n in ast.walk(s.node) if isinstance(n, ast.Call) - and isinstance(n.func, ast.Name) and n.func.id in signatures} - for name, s in signatures.items() - } - cycles: dict[str, list[str]] = {} - for start in signatures: - stack = [(start, [start])] - seen: set[str] = set() - while stack and start not in cycles: - name, path = stack.pop() - for callee in sorted(calls[name]): - if callee == start: - cycles[start] = path + [start] - break - if callee not in seen: - seen.add(callee) - stack.append((callee, path + [callee])) - return cycles - - def _module_info(tree: ast.Module) -> _ModuleInfo: bindings: dict[str, list[tuple[int, str]]] = {} diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 78fae7b96..7d0b891fa 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -240,8 +240,9 @@ class Call(Expr): """A direct call of another compiled function in the same module (C to C). If the callee may deopt, the caller must be pure (the deopt propagates and the outermost compiled call is redone); otherwise the front end calls it as an object (`CallObject(Global(name), ...)`). - Recursion (any call cycle) is rejected by the verifier: compiled C has no recursion limit where - CPython raises RecursionError.""" + A call cycle (direct or mutual recursion) is legal: every compiled function counts its frame at + entry (runtime `tp_enter_call`), so a recursion that CPython stops with RecursionError is stopped + with the same error, and the C stack cannot overflow.""" function: str args: tuple[Expr, ...] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 6bfed5ddd..77bd67469 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -84,3 +84,16 @@ Same structs as above with `list == NULL` and `dirty == NULL`. 0 ok, -1 MemoryEr void tp_f64_array_free(tp_f64_array *a); /* tp_*_array_slot works unchanged on local arrays (same IndexError messages as list). */ PyObject *tp_tuple(PyObject *const *items, Py_ssize_t n); /* new tuple; steals nothing */ + +## Call depth (every compiled impl function: enter at entry, leave at every exit; issue #57) + int tp_enter_call(void); /* 0 ok; -1 RecursionError set, nothing counted (the caller must NOT call leave) */ + void tp_leave_call(void); /* undoes one successful tp_enter_call; exactly once per success, on ok, error and deopt exits alike */ +`tp_enter_call` makes compiled recursion fail where interpreted recursion fails. It counts, per thread, +the compiled frames in flight (they have no Python frame), adds the current Python frame depth +(`PyThreadState_GetFrame` / `PyFrame_GetBack`, public API only; cached while the top frame is the same +object, so a pure compiled recursion pays O(1) per call) and compares the total with +`Py_GetRecursionLimit()`. Over it: `RecursionError("maximum recursion depth exceeded")`, the text +CPython's own Python-frame check uses. It then calls `Py_EnterRecursiveCall(" in compiled code")`, the +C-stack guard: a raised recursion limit can never let compiled C overflow the stack. Counters are +per module (the header is `static`): frames of a *different* compiled module that sit between two +frames of this one are not counted. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 876a33eb8..2f64563d8 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -1008,4 +1008,72 @@ static inline void tp_release(PyObject **slot) Py_CLEAR(*slot); } +/* ------------------------------------------------------------------------------------------ + * Call depth (issue #57): compiled frames have no Python frame, so a recursion made of them would + * neither raise RecursionError nor stop before the C stack ends. tp_enter_call / tp_leave_call + * bracket every compiled impl function. See API.md "Call depth". + * ------------------------------------------------------------------------------------------ */ +#if defined(_MSC_VER) +#define TP_THREAD_LOCAL __declspec(thread) +#else +#define TP_THREAD_LOCAL __thread +#endif + +typedef struct { + Py_ssize_t count; /* compiled frames in flight on this thread (this module) */ + PyObject *frame; /* strong ref: the top Python frame when `pydepth` was measured */ + Py_ssize_t pydepth; /* Python frames below and including `frame` */ +} tp_depth_state; + +static TP_THREAD_LOCAL tp_depth_state tp_depth = {0, NULL, 0}; + +/* Python frame depth of the running thread: public API only. The walk is O(depth), so it is + * cached by the identity of the top frame (kept alive while compiled frames are in flight). */ +static inline Py_ssize_t tp_python_depth(void) +{ + PyFrameObject *top = PyThreadState_GetFrame(PyThreadState_Get()); /* new ref or NULL */ + if (top == NULL) { + return 0; + } + if (tp_depth.frame == (PyObject *)top) { + Py_DECREF(top); + return tp_depth.pydepth; + } + Py_ssize_t depth = 0; + PyFrameObject *cur = top; + Py_INCREF(cur); + while (cur != NULL) { + depth++; + PyFrameObject *back = PyFrame_GetBack(cur); + Py_DECREF(cur); + cur = back; + } + PyObject *old = tp_depth.frame; + tp_depth.frame = (PyObject *)top; /* takes the reference from GetFrame */ + tp_depth.pydepth = depth; + Py_XDECREF(old); + return depth; +} + +static inline int tp_enter_call(void) +{ + if (tp_python_depth() + tp_depth.count + 1 > Py_GetRecursionLimit()) { + PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded"); + return -1; + } + if (Py_EnterRecursiveCall(" in compiled code")) { + return -1; + } + tp_depth.count++; + return 0; +} + +static inline void tp_leave_call(void) +{ + Py_LeaveRecursiveCall(); + if (--tp_depth.count == 0) { + Py_CLEAR(tp_depth.frame); + } +} + #endif /* TP_RUNTIME_H */ diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index b649fb3ff..e9b164799 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -31,8 +31,6 @@ verify/unknown-node a node (or enum field) that `ir` does not define verify/call-unresolved a Call names a function that is not compiled in this module verify/call-unverified a Call names a function that the verifier rejected - verify/recursion the function is on a cycle of Calls (C stack depth is unbounded; - CPython would raise RecursionError instead) verify/condition-only a CompareObj outside a condition (If/While cond, or an And/Or operand that is itself in a condition): its rich-comparison result need not be a bool, so only its truth value may be used @@ -146,7 +144,6 @@ UNKNOWN = "verify/unknown-node" UNRESOLVED = "verify/call-unresolved" UNVERIFIED = "verify/call-unverified" -RECURSION = "verify/recursion" STRUCTURE = "verify/structure" CONDITION_ONLY = "verify/condition-only" PROVEN = "verify/proven-overflow" @@ -1018,45 +1015,6 @@ def _exact(op: BinOpKind, a: tuple[int, int], b: tuple[int, int]) -> tuple[int, return min(products), max(products) -def _cycles(graph: dict[str, set[str]]) -> set[str]: - """Names on a cycle of `graph` (Tarjan's strongly connected components).""" - index: dict[str, int] = {} - low: dict[str, int] = {} - stack: list[str] = [] - on: set[str] = set() - result: set[str] = set() - counter = [0] - - def visit(v: str) -> None: - index[v] = low[v] = counter[0] - counter[0] += 1 - stack.append(v) - on.add(v) - for w in graph.get(v, ()): - if w not in graph: - continue - if w not in index: - visit(w) - low[v] = min(low[v], low[w]) - elif w in on: - low[v] = min(low[v], index[w]) - if low[v] == index[v]: - comp = [] - while True: - w = stack.pop() - on.discard(w) - comp.append(w) - if w == v: - break - if len(comp) > 1 or v in graph.get(v, ()): - result.update(comp) - - for v in graph: - if v not in index: - visit(v) - return result - - def verify(module: Module) -> tuple[Module, list[Diagnostic]]: """Prove each function of `module`; return the module of proved functions (with proven bounds marked) and every diagnostic. Rejected functions move to `skipped` with the reason.""" @@ -1112,9 +1070,11 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: reject(name, g.source_line, ENTRY_OPEN, f"has entry globals but is not closed: {open_why[name]}") - for name in sorted(_cycles(callees)): - reject(name, sigs[name].source_line, RECURSION, - f"'{name}' is on a cycle of calls; compiled recursion has no depth limit") + # A call cycle (direct or mutual recursion) is legal: every compiled impl function counts its + # frame at entry (tp_enter_call), so recursion ends in RecursionError where CPython raises it + # and the C stack cannot overflow (issue #57). Purity and may_deopt flow through the cycle as + # through any Call: each flag is proved on the function's own body against its callees' + # declared flags, so a cycle through a may_deopt callee is legal only in a pure function. # A call into a function that is not compiled cannot be a C call: propagate to a fixpoint. changed = True diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index 791f1bf22..f82cd9379 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -783,9 +783,10 @@ def shrink(lo: float, x: float, hi: float) -> float: "@compiled\ndef f(n: int) -> int:\n return g(n)\n\n" "@compiled\ndef g(n: int) -> int:\n return f(n)\n", ]) -def test_recursion_is_skipped(tmp_path, source): +def test_recursion_is_lowered_not_skipped(tmp_path, source): + # #57: a call cycle is compiled; the run-time depth guard replaces the old refusal. m = lower(tmp_path, source) - assert m.functions == () and "recurs" in m.skipped["f"] + assert {f.name for f in m.functions} >= {"f"} and not any("recurs" in r for r in m.skipped.values()) def test_int_modulo_cannot_be_recorded_as_proven_so_it_deopts(tmp_path): diff --git a/python-multiplatform-ksp/src/test/python/test_recursion.py b/python-multiplatform-ksp/src/test/python/test_recursion.py new file mode 100644 index 000000000..db716afec --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_recursion.py @@ -0,0 +1,308 @@ +"""TypedPython recursion (issue #57, SPEC N-8): direct and mutual call cycles are compiled, and +compiled recursion raises CPython's RecursionError, never crashes the process. + +Differential: every result is compared with the same source run by the interpreter. Builds go +through the whole pipeline (source -> gate -> frontend -> verify -> cgen -> cbuild), in +`.tmp/` of the worktree via `tmp_path`. +""" +import importlib.util +import os +import subprocess +import sys +import sysconfig +import textwrap +from pathlib import Path + +import pytest + +from typedpython import pipeline + +PACKAGE = Path(__file__).resolve().parents[2] / "main" / "python" + +SOURCE = """\ +# typedpython: compiled + + +def fib(n: int) -> int: + if n < 2: + return n + return fib(n - 1) + fib(n - 2) + + +def ack(m: int, n: int) -> int: + if m == 0: + return n + 1 + if n == 0: + return ack(m - 1, 1) + return ack(m - 1, ack(m, n - 1)) + + +def is_even(n: int) -> bool: + if n == 0: + return True + return is_odd(n - 1) + + +def is_odd(n: int) -> bool: + if n == 0: + return False + return is_even(n - 1) + + +def down(n: int) -> int: + if n == 0: + return 0 + return 1 + down(n - 1) + + +def pow2(n: int) -> int: + if n == 0: + return 1 + return 2 * pow2(n - 1) + + +def fact(n: int) -> int: + if n < 2: + return 1 + return n * fact(n - 1) +""" + +NAMES = ("fib", "ack", "is_even", "is_odd", "down", "pow2", "fact") + + +def load_plain(path: Path): + spec = importlib.util.spec_from_file_location(path.stem + "_plain", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def built(tmp_path_factory): + d = tmp_path_factory.mktemp("rec") + path = d / "rec_mod.py" + path.write_text(SOURCE) + result = pipeline.compile_module(path, d / "out") + compiled = pipeline.load(result) if result.extension else None + return result, compiled, load_plain(path) + + +@pytest.fixture +def mod(built): + result, compiled, plain = built + assert result.extension is not None, result.skipped + return compiled, plain + + +def test_every_recursive_function_is_compiled(built): + result = built[0] + assert {f.name for f in result.module.functions} == set(NAMES), result.skipped + assert not any("recurs" in why for why in result.skipped.values()), result.skipped + + +def test_fib_matches_the_interpreter(mod): + compiled, plain = mod + for n in (0, 1, 2, 10, 20): + assert compiled.fib(n) == plain.fib(n) + + +def test_ackermann_matches_the_interpreter(mod): + compiled, plain = mod + for m, n in ((0, 0), (1, 3), (2, 3), (3, 3)): + assert compiled.ack(m, n) == plain.ack(m, n) + + +def test_mutual_recursion_matches_the_interpreter(mod): + compiled, plain = mod + for n in (0, 1, 2, 7, 100, 400): + assert compiled.is_even(n) is plain.is_even(n) + assert compiled.is_odd(n) is plain.is_odd(n) + + +def test_a_pure_recursive_int_function_that_overflows_returns_the_big_int(mod): + compiled, plain = mod + before = compiled.__typedpython_deopts__ + assert compiled.pow2(100) == plain.pow2(100) == 2 ** 100 + assert compiled.fact(25) == plain.fact(25) + assert compiled.__typedpython_deopts__ > before # it really deopted + assert compiled.pow2(10) == 1024 # and the fast path still works + + +def raised(fn, *args): + with pytest.raises(RecursionError) as e: + fn(*args) + return str(e.value) + + +def test_recursion_past_the_limit_raises_the_same_error_as_the_interpreter(mod): + compiled, plain = mod + assert raised(compiled.down, 10 ** 6) == raised(plain.down, 10 ** 6) == \ + "maximum recursion depth exceeded" + assert raised(compiled.is_even, 10 ** 6) == "maximum recursion depth exceeded" + + +def test_recursion_past_a_raised_limit_raises_too(mod): + compiled, plain = mod + old = sys.getrecursionlimit() + sys.setrecursionlimit(old + 3000) + try: + assert compiled.down(old + 1000) == old + 1000 # allowed by the raised limit + assert raised(compiled.down, 10 ** 6) == "maximum recursion depth exceeded" + assert raised(plain.down, 10 ** 6) == "maximum recursion depth exceeded" + finally: + sys.setrecursionlimit(old) + + +def test_a_lowered_limit_applies_to_compiled_frames(mod): + compiled, plain = mod + old = sys.getrecursionlimit() + base = len(_stack()) + sys.setrecursionlimit(base + 50) + try: + assert compiled.down(10) == 10 + assert raised(compiled.down, 200) == "maximum recursion depth exceeded" + assert raised(plain.down, 200) == "maximum recursion depth exceeded" + finally: + sys.setrecursionlimit(old) + + +def _stack(): + import inspect + return inspect.stack(0) + + +def test_the_depth_counter_is_balanced_after_every_error_exit(mod): + # Each RecursionError leaves ~1000 compiled frames; a frame whose exit skipped tp_leave_call + # would stay counted, and after a few errors even a shallow call would fail. + compiled, _ = mod + for _ in range(8): + raised(compiled.down, 10 ** 6) + raised(compiled.is_even, 10 ** 6) + assert compiled.down(500) == 500 + assert compiled.fib(15) == 610 + + +def test_the_depth_counter_is_balanced_after_a_deopt_unwinds(mod): + compiled, _ = mod + for _ in range(2000): # more frames than the limit if any deopt leaked a frame + assert compiled.pow2(70) == 2 ** 70 + assert compiled.down(500) == 500 + + +def max_ok(fn, hi=20_000): + """The largest n for which fn(n) does not raise RecursionError (binary search).""" + lo = 0 + while lo < hi: + mid = (lo + hi + 1) // 2 + try: + fn(mid) + lo = mid + except RecursionError: + hi = mid - 1 + return lo + + +def test_depth_at_which_recursion_error_fires_is_recorded(mod, capsys): + """Measures where RecursionError fires for interpreted, compiled, and mixed frames. Asserts + only that both raise and nothing crashes; the offsets are recorded in the test output + and in docs/design/typedpython.md.""" + compiled, plain = mod + interp_only = max_ok(plain.down) + compiled_only = max_ok(compiled.down) + + def mixed(k): + def run(n): + def inner(j): + if j == 0: + return compiled.down(n) + return inner(j - 1) + return inner(k) + return run + + results = {"interpreted": interp_only, "compiled": compiled_only} + for k in (1, 10, 100): + results[f"mixed k={k} interpreted frames"] = max_ok(mixed(k)) + with capsys.disabled(): + print("\nRECURSION-DEPTH-MEASUREMENT", results) + assert interp_only > 0 and compiled_only > 0 + assert all(v > 0 for v in results.values()) + # both kinds do raise past their depth + raised(plain.down, interp_only + 5) + raised(compiled.down, compiled_only + 5) + + +# --- sanitizers: the RecursionError path --------------------------------------------------------- + +ASAN_SCRIPT = textwrap.dedent('''\ + import sys + sys.path.insert(0, {pkg!r}) + from typedpython import cbuild + m = cbuild.load({name!r}, {so!r}) + for _ in range(50): + for f in (m.down, m.is_even, m.fib): + try: + f(10 ** 6) + except RecursionError as e: + assert str(e) == "maximum recursion depth exceeded", e + assert m.down(300) == 300 and m.fib(15) == 610 and m.pow2(80) == 2 ** 80 + sys.setrecursionlimit(200000) # the C-stack guard must stop it before the stack ends + try: + m.down(10 ** 7) + except RecursionError: + pass + print("ASAN-RUN-OK") + ''') + + +def test_recursion_error_path_under_asan_and_ubsan(tmp_path): + from typedpython import cbuild, cgen, frontend, verify + + cc = sysconfig.get_config_var("CC") or "cc" + if sys.platform != "darwin": + pytest.skip("sanitizer preload recipe written for macOS only (as test_cgen.py)") + probe = subprocess.run(cc.split() + ["-print-file-name=libclang_rt.asan_osx_dynamic.dylib"], + capture_output=True, text=True) + if probe.returncode != 0 or not Path(probe.stdout.strip()).is_file(): + pytest.skip("host compiler has no ASan runtime") + path = tmp_path / "rec_asan.py" + path.write_text(SOURCE) + proved, _ = verify.verify(frontend.lower(path)) + assert {f.name for f in proved.functions} == set(NAMES), proved.skipped + flags = list(cbuild.DEFAULT_FLAGS) + ["-O1", "-g", "-fno-omit-frame-pointer", + "-fsanitize=address,undefined", + "-fno-sanitize-recover=undefined"] + so = cbuild.build(cgen.generate(proved, path), "rec_asan", tmp_path, flags=flags) + launcher_c = tmp_path / "asan_python.c" + launcher_c.write_text("#include \nint main(int argc, char **argv) " + "{ return Py_BytesMain(argc, argv); }\n") + launcher = tmp_path / "asan_python" + link = subprocess.run( + cc.split() + ["-fsanitize=address,undefined", "-g", f"-I{sysconfig.get_paths()['include']}", + str(launcher_c), f"-L{sysconfig.get_config_var('LIBDIR')}", + f"-lpython{sysconfig.get_config_var('LDVERSION')}", "-o", str(launcher)], + capture_output=True, text=True) + if link.returncode != 0: + pytest.skip(f"cannot link a sanitized launcher: {link.stderr[-1500:]}") + env = dict(os.environ, PYTHONHOME=sys.base_prefix, + ASAN_OPTIONS="detect_leaks=0:abort_on_error=0:halt_on_error=1", + UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1") + env.pop("DYLD_INSERT_LIBRARIES", None) + script = ASAN_SCRIPT.format(pkg=str(PACKAGE), name="rec_asan", so=str(so)) + run = subprocess.run([str(launcher), "-c", script], capture_output=True, text=True, env=env) + out = run.stdout + run.stderr + if "violates platform policy" in out or "Interceptors are not working" in out: + pytest.skip(f"ASan runtime could not be loaded: {out[-1500:]}") + assert run.returncode == 0 and "ASAN-RUN-OK" in run.stdout, out[-5000:] + linked = subprocess.run(["otool", "-L", str(so)], capture_output=True, text=True).stdout + assert "libclang_rt.asan" in linked, linked + assert "AddressSanitizer" not in out and "runtime error" not in out, out[-3000:] + + +# --- generated code uses the helpers ------------------------------------------------------------- + +def test_every_compiled_function_enters_and_leaves(built): + from typedpython import cgen + result = built[0] + c = cgen.generate(result.module, result.source) + assert c.count("tp_enter_call()") >= len(NAMES) + assert c.count("tp_leave_call()") >= len(NAMES) diff --git a/python-multiplatform-ksp/src/test/python/test_verify.py b/python-multiplatform-ksp/src/test/python/test_verify.py index c88b0f342..38063296c 100644 --- a/python-multiplatform-ksp/src/test/python/test_verify.py +++ b/python-multiplatform-ksp/src/test/python/test_verify.py @@ -39,7 +39,6 @@ UNKNOWN = "verify/unknown-node" UNRESOLVED = "verify/call-unresolved" UNVERIFIED = "verify/call-unverified" -RECURSION = "verify/recursion" STRUCTURE = "verify/structure" CONDITION_ONLY = "verify/condition-only" PROVEN = "verify/proven-overflow" @@ -703,12 +702,35 @@ def test_caller_of_a_rejected_callee_is_rejected_transitively(): assert UNVERIFIED in rules_for(diags, "h") and UNVERIFIED in rules_for(diags, "k") -def test_recursion_is_rejected(): - assert_rejected(RECURSION, fn("f", returns=I64, body=[Return(Call(I64, "f", ()))])) - g = fn("g", returns=I64, body=[Return(Call(I64, "f", ()))]) - out, diags = assert_rejected(RECURSION, fn("f", returns=I64, body=[Return(Call(I64, "g", ()))]), - g) - assert "g" not in kept(out) +def test_direct_and_mutual_recursion_are_accepted(): + # #57: compiled frames are depth-counted at run time (tp_enter_call), so a call cycle is legal. + out, diags = run(fn("f", params=[Param("n", I64)], returns=I64, pure=True, + body=[Return(Call(I64, "f", (li("n"),)))])) + assert kept(out) == ["f"] and diags == [], diags + out, diags = run(fn("f", params=[Param("n", I64)], returns=I64, pure=True, + body=[Return(Call(I64, "g", (li("n"),)))]), + fn("g", params=[Param("n", I64)], returns=I64, pure=True, + body=[Return(Call(I64, "f", (li("n"),)))])) + assert sorted(kept(out)) == ["f", "g"] and diags == [], diags + + +def test_a_cycle_through_a_may_deopt_callee_is_legal_only_in_pure_functions(): + # pure: the deopt redoes the outermost compiled call, which is unobservable + f = fn("f", params=[Param("n", I64)], returns=I64, pure=True, may_deopt=True, + body=[Return(add(Call(I64, "f", (li("n"),)), i(1)))]) + assert_accepted(f) + # impure: a deopt inside the cycle could redo an effect, as for any Call + g = fn("g", params=[Param("n", I64)], returns=I64, may_deopt=True, + body=[ExprStmt(pycall("print")), Return(add(Call(I64, "g", (li("n"),)), i(1)))]) + out, diags = run(g) + assert kept(out) == [] and DEOPT in rules_for(diags, "g") + + +def test_a_cycle_keeps_every_other_rule(): + bad = fn("f", params=[Param("n", I64)], returns=I64, + body=[Return(Call(I64, "f", (Const(F64, 1.0),)))]) # argument type mismatch + out, diags = run(bad) + assert kept(out) == [] and "f" in out.skipped def test_duplicate_function_names_are_rejected():