From d160b6caac5956c12c195a73aa1251faf10b24a7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 9 Nov 2026 20:36:36 -0500 Subject: [PATCH 01/14] NZ G8-x2 (WIP): tie the omission register to the fit; readers select the measurement Two of #1563's follow-ups: - N3: transport.omission_register@1 now reads the calibration solution beside the problem. It decodes the solution against that problem and its axis, requires every household on the axis to hold a person, and refuses a frame whose person weights differ, bit for bit, from the solution's household weights. The register's totals still use the problem's pre-fit weights. NZ's graph and the toy give the register an edge from nz.calibrate. - Nit 1, readers only: the three income evidence readers now select their `measurement` from income_measurements' own measurement_id, so they move with that resource. The stand-in keeps its literal measurement_id, because R2C-1's proof reads a producer's measurement_id as a literal string in the declared rows and refuses any other value. The toy's income measurements carry the id the same way. Not yet in this commit: the engine pins (753ce7d9) and the input surface refreshed at that engine, the closure re-review, the activation test flips, the regenerated declarations and golden, O2's static test, and the differential docstring fix. Co-Authored-By: Claude Opus 5.5 --- .../transport/income_evidence_kernels.py | 29 +++- .../shared/test_nz_transport_declarations.py | 36 ++-- .../test_transport_income_evidence_kernels.py | 161 ++++++++++++++++-- .../microcosm_build/transport_nz_toy.py | 19 ++- tools/build_nz_transport_graph.py | 21 ++- 5 files changed, 230 insertions(+), 36 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py index ff0c234259..b9342fee43 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py @@ -15,7 +15,9 @@ size exactly as declared), beside the totals of what the measure does include at the calibration problem's pre-fit weights. It adds no model-side figure for an omitted item, because the register's sizes mean - IRD's. + IRD's. It decodes the fit's solution against that problem and refuses a + frame whose weights aren't the solution's, so the problem it reads is the + one this run's fit solved. ``transport.native_band_report@1`` Sets the model, at the run's pre-fit and post-fit weights, beside every native row of the publisher's table: the row's count and amount from the @@ -744,7 +746,7 @@ class OmissionRegisterKernel(KernelBase): "measurement", "weight_entity", ) - _inputs = MappingProxyType({"problem": PROBLEM_TYPE}) + _inputs = MappingProxyType({"problem": PROBLEM_TYPE, "solution": SOLUTION_TYPE}) def implementation_hash(self) -> str: return _implementation_hash(self.ref) @@ -799,6 +801,29 @@ def run(self, context: KernelContext) -> KernelResult: entity=entity, weight_entity=weight_entity, ) + # Tie the problem to this run's fit: the solution must be the one + # solved from it, every weight unit on its axis must hold a record, and + # each record's frame weight (its weight unit's, which the frame + # broadcasts) must be the solution's, bit for bit. + solution = decode_solution( + payloads["solution"], + problem_sha256=problem.sha256, + entity_ids=problem.entity_ids, + ) + if len(set(rows)) != len(problem.entity_ids): + raise ValueError( + f"{ref} frame has a {weight_entity} on the problem's axis with no " + f"{entity}." + ) + frame_weights = context.weights.get(entity) + if frame_weights is None or not np.array_equal( + np.asarray(frame_weights.values, dtype=np.float64), + np.asarray(solution.weights, dtype=np.float64)[rows], + ): + raise ValueError( + f"{ref} frame {entity} weights differ from the calibration " + f"solution's {weight_entity} weights." + ) # The problem's initial weights are the fit's pre-fit weights, on its # own entity axis; each record takes its weight unit's. initial = problem.problem.initial_weights diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_transport_declarations.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_transport_declarations.py index b688b41d06..e41acc8651 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_transport_declarations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_transport_declarations.py @@ -677,11 +677,12 @@ def test_nz_executable_support_keeps_region99_separate_from_ta999(): def _nz_composition_spec_and_config(): - """Supply compiler-only toy engine pins and CREATE's inventory. + """Supply synthetic engine refs and CREATE's inventory. - The packaged graph declares every money producer, so the compiler checks - the complete packaged node and population structure with no synthetic - node. The toy pins certify no runtime result. CREATE's inventory holds + The packaged graph declares every money producer, and the packaged + bindings carry the engine pins, so the compiler checks the complete + packaged node and population structure with no synthetic node or pin. The + synthetic engine refs certify no runtime result. CREATE's inventory holds canonical concepts and unit outputs only: no bridge or region column enters as a root. """ @@ -689,9 +690,7 @@ def _nz_composition_spec_and_config(): spec = _spec() resources = spec["resources"] - resources["axiom_rules_bindings"]["engine"].update( - commit="1" * 40, wheel_sha256="2" * 64 - ) + compose.validate_transport_activation(spec) rows = _instantiated_nz_rows(spec) produced = { @@ -1630,10 +1629,13 @@ def test_the_income_evidence_readers_name_the_measurement_but_dont_declare_it(): measurement_proofs counts every node upstream of a calibration that declares the measurement's id and needs exactly one. The three readers - name the stand-in's id under ``measurement``, and the register reads - the problem's household weights. The readers sit two boundary filters - after the reweight, and only stand-ins write the three proxy columns, so - nz.terminal holds what nz.income.standin wrote at nz.central. + name the stand-in's id under ``measurement``, selected from + income_measurements' own ``measurement_id``. The stand-in keeps the literal + that R2C-1's proof reads, and the composed readers resolve to it. The + register reads the problem's household weights and checks the frame's + against the fit's solution. The readers sit two boundary filters after the + reweight, and only stand-ins write the three proxy columns, so nz.terminal + holds what nz.income.standin wrote at nz.central. """ spec = _spec() rows = spec["resources"]["transport_graph"]["nodes"] @@ -1643,13 +1645,21 @@ def test_the_income_evidence_readers_name_the_measurement_but_dont_declare_it(): ] assert declarers == ["nz.income.standin"] by_id = {row["id"]: row for row in rows} + assert by_id["nz.income.standin"]["params"]["measurement_id"] == _MEASUREMENT + selection = { + "resource": "income_measurements", + "path": ["measurement_id"], + "encoding": "value", + } + _, _, graph, _ = _nz_composed() for producer, _ in _INCOME_EVIDENCE.values(): - assert by_id[producer]["params"]["measurement"] == _MEASUREMENT, producer + assert by_id[producer]["params"]["measurement"] == selection, producer + assert graph.node(producer).params["measurement"] == _MEASUREMENT, producer register = by_id["nz.evidence.omission_register"] assert register["params"]["weight_entity"] == "household" assert [ (edge["name"], edge["producer"]) for edge in register["artifact_inputs"] - ] == [("problem", "nz.targets.problem")] + ] == [("problem", "nz.targets.problem"), ("solution", "nz.calibrate")] for name, base in (("nz.as", "nz.calibrate"), ("nz.terminal", "nz.as")): assert (by_id[name]["kernel"], by_id[name]["base"]) == ( "transport.boundary@1", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py index 1d80349d94..3307dfd35a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py @@ -254,7 +254,12 @@ def _identity_node(declaration, *, identities=IDENTITIES, columns=None): def _register_node( - declaration, *, register=REGISTER, columns=None, problem_type=PROBLEM_TYPE + declaration, + *, + register=REGISTER, + columns=None, + problem_type=PROBLEM_TYPE, + solution_type=SOLUTION_TYPE, ): names = columns or ( *(row["column"] for row in declaration["components"]), @@ -272,11 +277,23 @@ def _register_node( }, artifact_inputs=( ArtifactInput("problem", "toy.problem", "problem", problem_type), + ArtifactInput("solution", "toy.calibrate", "solution", solution_type), ), artifact_outputs=(ArtifactOutput("omission_register", OMISSION_REGISTER_TYPE),), ) +def _encoded_solution(problem, weights, *, entity_ids=None): + """A solution of ``problem`` at ``weights``, on its axis unless given one.""" + + decoded = decode_problem(problem) + return encode_solution( + np.asarray(weights, dtype=np.float64), + entity_ids=decoded.entity_ids if entity_ids is None else entity_ids, + problem_sha256=decoded.sha256, + ) + + def _identity_context(person, declaration, weights=None, **node): return _context(_identity_node(declaration, **node), _frame(person, weights)) @@ -294,29 +311,43 @@ def _register_context( weights=None, *, problem=None, + solution=None, problem_type=PROBLEM_TYPE, + solution_type=SOLUTION_TYPE, edit=None, artifacts=None, **node, ): - """The register's context: ``_frame``'s households, and a problem on them. + """The register's context: ``_frame``'s households, a problem and its fit. By default the problem's pre-fit weights are the frame's own design weights, one household per person, so the register's totals equal - ``measure.combine@1``'s at the frame's weights. ``edit`` changes the node. + ``measure.combine@1``'s at the frame's weights. The default solution + gives each household on the problem's axis its frame weight (1.0 for one + the frame lacks), as the fit's own solution does where the register + runs. ``edit`` changes the node. """ frame = _frame(person, weights) + households = frame.table("household")["household_id"].tolist() + frame_weights = frame.resolve_weights("household").values if problem is None: - problem = _encoded_problem( - frame.table("household")["household_id"].tolist(), - frame.resolve_weights("household").values, + problem = _encoded_problem(households, frame_weights) + if solution is None: + by_id = dict(zip(households, frame_weights, strict=True)) + solution = _encoded_solution( + problem, + [by_id.get(value, 1.0) for value in decode_problem(problem).entity_ids], ) - built = _register_node(declaration, problem_type=problem_type, **node) + built = _register_node( + declaration, problem_type=problem_type, solution_type=solution_type, **node + ) return _context( built if edit is None else edit(built), frame, - _artifact_values({"problem": (problem, problem_type)}) + _artifact_values( + {"problem": (problem, problem_type), "solution": (solution, solution_type)} + ) if artifacts is None else artifacts, ) @@ -654,14 +685,18 @@ def test_the_register_binds_the_declared_items_verbatim() -> None: size = len(person) design = np.linspace(1, 2, size) problem = _encoded_problem(list(range(1, size + 1)), design) - payload, receipt = _register(person, TOY, problem=problem) + solution = _encoded_solution(problem, design) + payload, receipt = _register(person, TOY, problem=problem, solution=solution) document = json.loads(payload) assert document["kind"] == "transport_omission_register" assert document["register"] == REGISTER assert document["register_sha256"] == sha256(canonical_json(REGISTER)).hexdigest() assert document["income_measurements_sha256"] == "c" * 64 assert document["measurement_id"] == "toy_measurement_v1" - assert document["inputs"]["artifacts"] == {"problem": sha256(problem).hexdigest()} + assert document["inputs"]["artifacts"] == { + "problem": sha256(problem).hexdigest(), + "solution": sha256(solution).hexdigest(), + } # boundary_diagnostics' two digests, over the people in record order. assert document["weights"] == { "entity": "person", @@ -675,7 +710,8 @@ def test_the_register_binds_the_declared_items_verbatim() -> None: def test_the_register_reads_pre_fit_weights_from_the_problem() -> None: - # The frame's weights may already be calibrated where the register runs; + # Where the register runs, the frame's weights are the fit's solution + # (the default context's solution gives each household its frame weight); # the totals follow the problem's initial weights, not the frame's. person, receipt = _measured(TOY_COLUMNS, TOY) size = len(person) @@ -697,19 +733,38 @@ def test_the_register_refuses_each_node_edit(edit) -> None: def test_the_register_refuses_a_problem_it_cant_broadcast() -> None: person, _ = _measured(TOY_COLUMNS, TOY) size = len(person) - with pytest.raises(ValueError, match="reads exactly \\['problem'\\] artifacts"): + exactly = "reads exactly \\['problem', 'solution'\\] artifacts" + with pytest.raises(ValueError, match=exactly): _register( person, TOY, edit=lambda node: dataclasses.replace(node, artifact_inputs=()), artifacts={}, ) + # G8-x1's register read the problem alone; that node now refuses. + problem = _encoded_problem(list(range(1, size + 1)), np.linspace(1, 2, size)) + with pytest.raises(ValueError, match=exactly): + _register( + person, + TOY, + problem=problem, + edit=lambda node: dataclasses.replace( + node, artifact_inputs=node.artifact_inputs[:1] + ), + artifacts=_artifact_values({"problem": (problem, PROBLEM_TYPE)}), + ) with pytest.raises( ValueError, match="reads 'problem' as microcosm.calibrate.ordered-problem@1, not " "microcosm.calibrate.ordered-solution@1", ): _register(person, TOY, problem_type=SOLUTION_TYPE) + with pytest.raises( + ValueError, + match="reads 'solution' as microcosm.calibrate.ordered-solution@1, not " + "microcosm.calibrate.ordered-problem@1", + ): + _register(person, TOY, solution_type=PROBLEM_TYPE) with pytest.raises(ValueError, match="but the problem calibrates 'household'"): _register( person, @@ -723,6 +778,88 @@ def test_the_register_refuses_a_problem_it_cant_broadcast() -> None: _register(person, TOY, problem=short) +OFF_THE_SOLUTION = ( + "frame person weights differ from the calibration solution's household weights" +) + + +def test_the_register_refuses_frame_weights_off_the_solution() -> None: + # #1563 N3: the frame carries the fit's weights where the register runs, + # so one household one ulp off its solution weight refuses. + person, _ = _measured(TOY_COLUMNS, TOY) + size = len(person) + design = np.linspace(1, 2, size) + problem = _encoded_problem(list(range(1, size + 1)), design) + fitted = design.copy() + fitted[2] = np.nextafter(fitted[2], np.inf) + with pytest.raises(ValueError, match=OFF_THE_SOLUTION): + _register( + person, TOY, problem=problem, solution=_encoded_solution(problem, fitted) + ) + + +def test_the_register_refuses_a_solution_of_another_problem_or_axis() -> None: + # #1563 N3: the solution must be the one solved from the problem it reads. + person, _ = _measured(TOY_COLUMNS, TOY) + size = len(person) + design = np.linspace(1, 2, size) + ids = list(range(1, size + 1)) + problem = _encoded_problem(ids, design) + other = _encoded_problem(ids, design * 2) + with pytest.raises(ValueError, match="Solution belongs to a different problem"): + _register( + person, TOY, problem=problem, solution=_encoded_solution(other, design) + ) + with pytest.raises(ValueError, match="different ordered entity axis"): + _register( + person, + TOY, + problem=problem, + solution=_encoded_solution(problem, design, entity_ids=ids[::-1]), + ) + + +def test_the_register_refuses_a_household_on_the_axis_with_no_person() -> None: + # #1563 N3: the frame's households are the problem's axis, so every + # household the fit weighted holds a record the register reads. + person, _ = _measured(TOY_COLUMNS, TOY) + size = len(person) + wide = _encoded_problem(list(range(1, size + 2)), np.ones(size + 1)) + with pytest.raises( + ValueError, match="frame has a household on the problem's axis with no person" + ): + _register(person, TOY, problem=wide) + + +_WEIGHT = st.floats(min_value=0.01, max_value=1e6, allow_nan=False) + + +@PROPERTY +@given(case=measures(), data=st.data()) +def test_property_the_register_reads_its_fits_frame_at_pre_fit_weights( + case, data +) -> None: + # #1563 N3: at whatever solution the frame carries, the totals are the + # problem's pre-fit totals; one record's weight one ulp off refuses. + columns, declaration, design = case + person, receipt = _measured(columns, declaration, design) + size = len(person) + problem = _encoded_problem(list(range(1, size + 1)), design) + fitted = np.asarray( + data.draw(st.lists(_WEIGHT, min_size=size, max_size=size)), dtype=np.float64 + ) + solution = _encoded_solution(problem, fitted) + payload, _ = _register( + person, declaration, fitted, problem=problem, solution=solution + ) + _included_matches_receipt(json.loads(payload)["included"], receipt) + off = fitted.copy() + index = data.draw(st.integers(min_value=0, max_value=size - 1)) + off[index] = np.nextafter(off[index], np.inf) + with pytest.raises(ValueError, match=OFF_THE_SOLUTION): + _register(person, declaration, off, problem=problem, solution=solution) + + def test_the_register_refuses_a_measure_off_the_person_entity() -> None: person, _ = _measured(TOY_COLUMNS, TOY) declaration = {**TOY, "entity": "household"} diff --git a/test_support/microcosm_build/transport_nz_toy.py b/test_support/microcosm_build/transport_nz_toy.py index d1097d3fde..dd850b9b45 100644 --- a/test_support/microcosm_build/transport_nz_toy.py +++ b/test_support/microcosm_build/transport_nz_toy.py @@ -817,6 +817,7 @@ def _income_standin(resources, rows, facts, problem): declaration = toy_standin_declaration() resources["toy_income_measurements"] = { "schema_version": 1, + "measurement_id": TOY_MEASUREMENT, "measures": { "central": { "declaration": declaration, @@ -1585,15 +1586,18 @@ def _income_evidence(rows): tools/build_nz_transport_graph.py declares the same three nodes over nz.income.standin's measure, all at nz.terminal. Each reads the stand-in node's own declaration params and names its measurement under - ``measurement``, since only the stand-in declares ``measurement_id``. The - native-band report reads the coarse table the toy calibrates beside the - native table it passes through. + ``measurement``, since only the stand-in declares ``measurement_id``. That + name comes from ``toy_income_measurements``' own ``measurement_id``; the + stand-in keeps the literal, which R2C-1's proof reads. The native-band + report reads the coarse table the toy calibrates beside the native table it + passes through. """ standin = next(row for row in rows if row["id"] == TOY_STANDIN_NODE) + measurement = _select("toy_income_measurements", "value", "measurement_id") measure = { "measure": standin["params"]["measure"], "measure_sha256": standin["params"]["measure_sha256"], - "measurement": standin["params"]["measurement_id"], + "measurement": measurement, } return [ _node( @@ -1635,7 +1639,10 @@ def _income_evidence(rows): **measure, "weight_entity": "household", }, - artifact_inputs=[_edge("problem", "nz.targets.problem")], + artifact_inputs=[ + _edge("problem", "nz.targets.problem"), + _edge("solution", "nz.calibrate"), + ], artifact_outputs=[ _artifact_output("omission_register", OMISSION_REGISTER_TYPE) ], @@ -1664,7 +1671,7 @@ def _income_evidence(rows): ), "chronicle_feed_pin": _select("toy_calibration_feed_pin"), "weight_entity": "household", - "measurement": TOY_MEASUREMENT, + "measurement": measurement, }, artifact_inputs=[ _edge("preparation", "nz.calibration.prepare"), diff --git a/tools/build_nz_transport_graph.py b/tools/build_nz_transport_graph.py index 76b45c8486..a4259d978f 100644 --- a/tools/build_nz_transport_graph.py +++ b/tools/build_nz_transport_graph.py @@ -715,6 +715,18 @@ def measure_params(measure="central"): } +def measurement_selection(): + """The id ``income_measurements`` defines, as a reader's ``measurement``. + + R2C-1's proof reads a producer's ``measurement_id`` as a literal string in + the declared rows (``compose._measurement_declaration``) and refuses any + other value, so the stand-in keeps the literal, and the declarations test + pins it to this resource's id. A reader's ``measurement`` sits outside that + proof, so the readers select the id and move with the resource. + """ + return select("income_measurements", "value", "measurement_id") + + def reader_params(measure="central"): """measure_params for a node that reads the measure, under ``measurement``. @@ -726,7 +738,7 @@ def reader_params(measure="central"): return { "measure": params["measure"], "measure_sha256": params["measure_sha256"], - "measurement": params["measurement_id"], + "measurement": measurement_selection(), } @@ -1586,7 +1598,10 @@ def add_evidence(graph): **reader_params(), "weight_entity": "household", }, - artifact_inputs=[edge("problem", "nz.targets.problem")], + artifact_inputs=[ + edge("problem", "nz.targets.problem"), + edge("solution", "nz.calibrate"), + ], artifact_outputs=[ artifact_output("omission_register", OMISSION_REGISTER_TYPE) ], @@ -1610,7 +1625,7 @@ def add_evidence(graph): **document("diagnostic_references", "income_diagnostic_references"), "chronicle_feed_pin": select("chronicle_feed"), "weight_entity": "household", - "measurement": MEASUREMENT_ID, + "measurement": measurement_selection(), }, artifact_inputs=[ edge("preparation", "nz.targets.prepare"), From d40c004d51f9269a6e7f5665c6971511927c22f2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 9 Nov 2026 21:13:58 -0500 Subject: [PATCH 02/14] Read compiled-artifact format 3 in the input-surface refresh Engine 753ce7d9, G8-x2's pin, emits artifact format 3, and the refresh tool refused it before compiling anything. Format 3 adds program.interface, engine_build and source_closure, which the walk never reads. The tool now reads formats 2 and 3, and each artifact must carry the format the engine's capabilities names. From 4d7ec14f (#292) on, the engine follows a cross-entity dense error with ". " and advice. A cross-entity blocker now also explains its own text followed by ". " and any advice, and the snapshot keeps the walk's text. be.json's seven cross-entity modules need this at 753ce7d9; NZ's one blocker, a versioned formula, keeps its exact text. The docs name each snapshot's own engine (nz 753ce7d9, be 04315d94). New engine-free tests drive a stand-in CLI, and Hypothesis checks the error matcher. Co-Authored-By: Claude Opus 5.5 --- .../test_refresh_axiom_input_surface.py | 284 ++++++++++++++++++ tools/refresh_axiom_input_surface.py | 109 +++++-- 2 files changed, 363 insertions(+), 30 deletions(-) create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_refresh_axiom_input_surface.py diff --git a/packages/microcosm-build/tests/engine_free/shared/test_refresh_axiom_input_surface.py b/packages/microcosm-build/tests/engine_free/shared/test_refresh_axiom_input_surface.py new file mode 100644 index 0000000000..e7c31711e8 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_refresh_axiom_input_surface.py @@ -0,0 +1,284 @@ +"""The input-surface refresh tool reads artifact formats 2 and 3 (G8-x2). + +Engine-free. A stand-in CLI answers ``--version``, ``capabilities`` and +``compile`` the way ``tools/refresh_axiom_input_surface.py`` calls them, and +writes the artifact each test gives it. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import stat +import sys +from pathlib import Path + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from tools import refresh_axiom_input_surface as refresh + +_STAND_IN = """\ +import json +import sys + +ADVERTISED = json.loads({advertised!r}) +ARTIFACT = json.loads({artifact!r}) +args = sys.argv[1:] +if args == ["--version"]: + print("axiom-rules-engine 0.0.0-stand-in") +elif args == ["capabilities"]: + print(json.dumps({{"artifact_format_version": ADVERTISED}})) +elif args[:1] == ["compile"]: + with open(args[args.index("--output") + 1], "w", encoding="utf-8") as handle: + json.dump(ARTIFACT, handle) +else: + sys.exit(f"unexpected arguments {{args}}") +""" + +#: One Person rule that reads input ``a`` and a Scalar rule that reads ``b``. +_PROGRAM = { + "derived": [ + { + "name": "x", + "entity": "Person", + "expr": { + "kind": "add", + "left": {"kind": "input", "name": "a"}, + "right": {"kind": "derived", "name": "s"}, + }, + "versions": [], + }, + { + "name": "s", + "entity": "Scalar", + "expr": {"kind": "input", "name": "b"}, + "versions": [], + }, + ] +} +_FORMAT_2 = { + "artifact_format_version": 2, + "engine_version": "0.0.0", + "program": _PROGRAM, + "metadata": { + "evaluation_order": [], + "fast_path": {}, + "input_catalog": [{"slot": "a", "canonical_request_name": "nz:a"}], + }, +} + + +def _format_3() -> dict: + """The same program as format 3, with the blocks format 3 adds. + + The stand-in ``interface`` entries carry ``kind`` values no expression + has, so a walk that read the interface would refuse them. + """ + artifact = copy.deepcopy(_FORMAT_2) + artifact["artifact_format_version"] = 3 + artifact["engine_build"] = {"commit": "1" * 40, "dirty": False} + artifact["source_closure"] = { + "root": "nz:m", + "modules": [{"target": "nz:m", "sha256": "0" * 64}], + "digest": "sha256:" + "0" * 64, + } + artifact["program"]["interface"] = { + "entities": [{"name": "Person", "kind": "not_an_expression_kind"}], + "inputs": [{"slot": "a", "kind": "not_an_expression_kind"}], + } + return artifact + + +def _engine(tmp_path: Path, advertised, artifact: dict | None = None) -> Path: + script = tmp_path / "stand_in_engine.py" + script.write_text( + _STAND_IN.format( + advertised=json.dumps(advertised), + artifact=json.dumps(_FORMAT_2 if artifact is None else artifact), + ), + encoding="utf-8", + ) + engine = tmp_path / "axiom-rules-engine" + engine.write_text( + f'#!/bin/sh\nexec "{sys.executable}" "{script}" "$@"\n', encoding="utf-8" + ) + engine.chmod(engine.stat().st_mode | stat.S_IXUSR) + return engine + + +_MODULE_TEXT = b"stand-in module; the stand-in CLI ignores it\n" + + +def _rulespec_root(tmp_path: Path) -> Path: + module = tmp_path / "rulespec" / "nz" / "m.yaml" + module.parent.mkdir(parents=True) + module.write_bytes(_MODULE_TEXT) + return tmp_path / "rulespec" + + +def _snapshot(tmp_path: Path, artifact: dict) -> dict: + tmp_path.mkdir() + engine = _engine(tmp_path, artifact["artifact_format_version"], artifact) + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + snapshot, differential = refresh.build_snapshot( + country="nz", + rulespec_root=_rulespec_root(tmp_path), + rulespec_commit="0" * 40, + engine_bin=engine, + engine_commit="1" * 40, + dense_python=None, + artifact_dir=artifacts, + ) + assert differential is None + return snapshot + + +# --------------------------------------------------------------------------- +# The format gate +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("artifact_format", [2, 3]) +def test_the_walk_reads_formats_2_and_3(tmp_path, artifact_format) -> None: + artifact = _FORMAT_2 if artifact_format == 2 else _format_3() + engine = _engine(tmp_path, artifact_format, artifact) + assert refresh.engine_identity(engine) == ( + "axiom-rules-engine 0.0.0-stand-in", + artifact_format, + ) + module = _rulespec_root(tmp_path) / "nz" / "m.yaml" + compiled, error = refresh.compile_module( + engine, module, module.parents[1], tmp_path / "m.json", artifact_format + ) + assert error is None + assert compiled == artifact + + +@pytest.mark.parametrize("advertised", [1, 4, 3.0, True, "3", None]) +def test_an_engine_of_another_format_refuses(tmp_path, advertised) -> None: + # 3.0 equals 3 and true equals 1 in Python; neither is a format. + with pytest.raises(refresh.SurfaceError, match="emits artifact format"): + refresh.engine_identity(_engine(tmp_path, advertised)) + + +@pytest.mark.parametrize("written", [2, 3.0, None]) +def test_an_artifact_off_the_engines_format_refuses(tmp_path, written) -> None: + artifact = _format_3() + artifact["artifact_format_version"] = written + engine = _engine(tmp_path, 3, artifact) + assert refresh.engine_identity(engine)[1] == 3 + module = _rulespec_root(tmp_path) / "nz" / "m.yaml" + with pytest.raises( + refresh.SurfaceError, + match=f"has artifact format {written!r}; .* names format 3", + ): + refresh.compile_module( + engine, module, module.parents[1], tmp_path / "m.json", 3 + ) + + +def test_a_format_3_artifact_gives_the_format_2_snapshot(tmp_path) -> None: + # The walk reads program.derived and metadata.input_catalog only, so the + # blocks format 3 adds change nothing in the snapshot. + two = _snapshot(tmp_path / "two", _FORMAT_2) + three = _snapshot(tmp_path / "three", _format_3()) + assert two == three + assert two["modules"] == [ + { + "path": "nz/m.yaml", + "sha256": hashlib.sha256(_MODULE_TEXT).hexdigest(), + "status": "compiled", + "inputs": {"Person": ["a", "b"], "Scalar": ["b"]}, + "canonical_inputs": {"a": "nz:a", "b": None}, + } + ] + + +# --------------------------------------------------------------------------- +# Dense errors the walk explains +# --------------------------------------------------------------------------- + +_BLOCKER = refresh._CROSS_ENTITY.format( + dependency="hh_income", derived="person_benefit", entity="Household" +) +#: Advice in each of the three forms engine 753ce7d9 writes +#: (``cross_entity_advice``, src/dense.rs:560-590). +_ADVICE = ( + "This dense program's root is the entity-free `Scalar`, whose rows hold " + "no `Household`: compile it for `Household` instead, or compile only " + "outputs that do not reach `person_benefit`", + "These rows' ids are `Household` ids, but dense reads a rule of another " + "entity only through a relation, and none carries this read: compile only " + "outputs that do not reach `person_benefit`", + "Dense evaluates the rows of one entity and reads another entity's rule " + "only through a relation, by design: read `hh_income` with a related " + "aggregate (`only`, `sum`, `min`, `max`, `count_where`) over a relation " + "between `Person` and `Household`, such as `member_of_household`, which " + "this program declares, or compile only outputs that do not reach " + "`person_benefit`", +) + + +def test_a_cross_entity_blocker_explains_its_message_with_or_without_advice() -> None: + assert refresh.explained(_BLOCKER, [_BLOCKER]) + for advice in _ADVICE: + assert refresh.explained(f"{_BLOCKER}. {advice}", [_BLOCKER]) + + +@pytest.mark.parametrize( + "message", + [ + _BLOCKER + " " + _ADVICE[0], + _BLOCKER + "." + _ADVICE[0], + _BLOCKER.replace("`Household`", "`TaxUnit`") + ". " + _ADVICE[0], + _BLOCKER.replace("`Household`", "`HouseholdX`") + ". " + _ADVICE[0], + _BLOCKER.replace("`hh_income`", "`hh_wealth`"), + _BLOCKER[:-1], + ], +) +def test_a_cross_entity_blocker_explains_no_other_message(message) -> None: + assert not refresh.explained(message, [_BLOCKER]) + + +def test_any_other_blocker_explains_only_its_own_text() -> None: + versioned = refresh._VERSIONED.format(name="rr_threshold") + assert refresh.explained(versioned, [versioned]) + assert not refresh.explained(f"{versioned}. {_ADVICE[2]}", [versioned]) + assert not refresh.explained(_BLOCKER, [versioned]) + assert not refresh.explained(_BLOCKER, []) + + +_NAME = st.text( + alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_", + min_size=1, + max_size=12, +) +_TRIPLE = st.tuples(_NAME, _NAME, _NAME) + + +@given(_TRIPLE, _TRIPLE, st.text(max_size=80)) +def test_property_a_cross_entity_blocker_explains_exactly_its_names( + blocked, read, advice +) -> None: + # A message names (dependency, derived, entity); with or without advice, + # a blocker explains it exactly when the blocker names the same three. + def message(names): + dependency, derived, entity = names + return refresh._CROSS_ENTITY.format( + dependency=dependency, derived=derived, entity=entity + ) + + blocker = message(blocked) + same = blocked == read + assert refresh.explained(message(read), [blocker]) is same + assert refresh.explained(f"{message(read)}. {advice}", [blocker]) is same + + +@given(_NAME, st.text(max_size=120)) +def test_property_a_versioned_blocker_explains_only_itself(name, message) -> None: + versioned = refresh._VERSIONED.format(name=name) + assert refresh.explained(message, [versioned]) is (message == versioned) diff --git a/tools/refresh_axiom_input_surface.py b/tools/refresh_axiom_input_surface.py index 93564dd651..a223f8560f 100644 --- a/tools/refresh_axiom_input_surface.py +++ b/tools/refresh_axiom_input_surface.py @@ -67,22 +67,35 @@ one its error names varies from process to process. The snapshot therefore records the lexicographically first blocker, in the engine's own message format, plus a count of the rest. In dense mode every dense error must equal -one of the computed blockers. Anything else aborts the run: a new kind of -dense limitation is a review event, not a silent snapshot change. In CLI-only -mode, only these two blocker kinds are detected. +one of the computed blockers. One exception: engines from 4d7ec14f (#292) on +follow a cross-entity message with ". " and advice that names the relation +to read through (``cross_entity_advice`` in the engine's ``src/dense.rs``). +The walk can't build that advice, so a cross-entity blocker also matches an +error that starts with its text and ". ", and the snapshot keeps the walk's +text. Anything else aborts the run: a new kind of dense limitation is a +review event, not a silent snapshot change. In CLI-only mode, only these two +blocker kinds are detected. + +The walk reads CLI artifact formats 2 and 3, and each artifact must carry +the format the engine's ``capabilities`` names. Format 3 (engine 590d1dc9, +#250) adds ``program.interface``, ``engine_build`` and ``source_closure``, +which the walk doesn't read. The engine's ``docs/artifact-format.md`` says +that, for source without election declarations, its ``program`` without +``interface``, and its ``metadata``, are what format 2 emits. Regenerate ---------- Archive the pinned commits into a scratch directory (never build inside a live checkout), build the CLI and the dense extension from the same engine archive, then run this tool once per country from the repository root. -These are the commands that produced the checked-in snapshots. -``AXIOM`` is the directory holding the three TheAxiomFoundation checkouts, -after a ``git fetch origin`` in each:: +Each snapshot names its own engine commit: nz.json 753ce7d9 and be.json +04315d94. Set ``ENGINE`` to the one the snapshot records, and run only that +country's command. ``AXIOM`` is the directory holding the three +TheAxiomFoundation checkouts, after a ``git fetch origin`` in each:: W= AXIOM= - ENGINE=04315d94d04efd61e27665ab3aa300d64fdf3a66 + ENGINE= NZ=6fe181fc4c65763150fe97501ef3443774e64869 BE=b105e2b3a3086ddd2de447d58a9b951346870dd1 mkdir -p $W/engine $W/rulespec-nz $W/rulespec-be @@ -111,10 +124,11 @@ ```` stands for any GIL-enabled CPython 3.14 interpreter. The wrapper pins Python ``==3.14.*``, and the extension must be built for the interpreter that loads it: a wheel built for free-threaded 3.14t does not -import on GIL-enabled 3.14. ``maturin build`` runs without ``--locked`` -because ``python-ext/Cargo.lock`` at that engine commit still records the -path dependency ``axiom-rules-engine`` as 0.1.0 (the crate is 0.2.2). Cargo -rewrites only that entry. +import on GIL-enabled 3.14. At 04315d94 ``maturin build`` runs without +``--locked``, because ``python-ext/Cargo.lock`` there still records the +path dependency ``axiom-rules-engine`` as 0.1.0 (the crate is 0.2.2), and +Cargo rewrites only that entry. At 753ce7d9 the lock records 0.2.2, and +``maturin build --locked`` builds the wheel. ``--rulespec-git-dir`` checks that every file under ``--rulespec-root`` is byte-identical to the blob at ``--rulespec-commit``. ``--check`` regenerates @@ -152,14 +166,17 @@ ENGINE_REPOSITORY = "TheAxiomFoundation/axiom-rules-engine" SURFACE_DENSE = "dense_root_inputs" SURFACE_CLI = "cli_artifact_walk" -#: The CLI artifact layout the walk below was written against. -ARTIFACT_FORMAT_VERSION = 2 +#: The CLI artifact formats the walk below reads (the module docstring says +#: why format 3 needs nothing new). +ARTIFACT_FORMAT_VERSIONS = frozenset({2, 3}) SCALAR_ENTITY = "Scalar" ERROR_TAIL_CHARACTERS = 300 _FULL_SHA = re.compile(r"[0-9a-f]{40}") -#: Every node ``kind`` in artifact v2 expressions (``ScalarExprSpec``, -#: ``JudgmentExprSpec``, ``ScalarValueSpec`` in the engine's ``src/spec.rs``). +#: Every node ``kind`` in artifact expressions at engine 04315d94 +#: (``ScalarExprSpec``, ``JudgmentExprSpec``, ``ScalarValueSpec`` in the +#: engine's ``src/spec.rs``). Engine 753ce7d9 adds ten kinds and keeps these; +#: the walk knows none of the ten, so an artifact that uses one fails the run. #: The walk refuses an unknown kind rather than guessing its evaluation #: context. _KNOWN_KINDS = frozenset( @@ -213,6 +230,8 @@ "dense compilation only supports dependencies within the same root " "entity; `{dependency}` from `{derived}` crosses into `{entity}`" ) +#: The text every cross-entity message starts with, before its names. +_CROSS_ENTITY_HEAD = _CROSS_ENTITY[: _CROSS_ENTITY.index("`")] _VERSIONED = ( "dense compilation does not yet support versioned derived formulas in " "`{name}`; use generic API execution" @@ -334,8 +353,15 @@ def verify_rulespec_tree(root: Path, git_dir: Path, commit: str) -> int: return len(expected) -def engine_version(engine_bin: Path) -> str: - """The CLI's ``--version`` line, after checking its artifact format.""" +def _artifact_format(value: Any) -> int | None: + """``value`` if it names a format the walk reads; a JSON 3.0 or true doesn't.""" + if type(value) is int and value in ARTIFACT_FORMAT_VERSIONS: + return value + return None + + +def engine_identity(engine_bin: Path) -> tuple[str, int]: + """The CLI's ``--version`` line and the artifact format it emits.""" version = subprocess.run( [str(engine_bin), "--version"], capture_output=True, text=True, check=True ).stdout.strip() @@ -347,13 +373,14 @@ def engine_version(engine_bin: Path) -> str: check=True, ).stdout ) - if capabilities.get("artifact_format_version") != ARTIFACT_FORMAT_VERSION: + found = capabilities.get("artifact_format_version") + artifact_format = _artifact_format(found) + if artifact_format is None: raise SurfaceError( - f"{engine_bin} emits artifact format " - f"{capabilities.get('artifact_format_version')!r}; this walk reads " - f"format {ARTIFACT_FORMAT_VERSION}" + f"{engine_bin} emits artifact format {found!r}; this walk reads " + f"formats {sorted(ARTIFACT_FORMAT_VERSIONS)}" ) - return version + return version, artifact_format # ---------------------------------------------------------------------- @@ -368,9 +395,13 @@ def _sanitize(text: str, replacements: Mapping[str, str]) -> str: def compile_module( - engine_bin: Path, module: Path, root: Path, artifact: Path + engine_bin: Path, module: Path, root: Path, artifact: Path, artifact_format: int ) -> tuple[dict[str, Any] | None, str | None]: - """Compile one module with the CLI: (artifact, None) or (None, stderr).""" + """Compile one module with the CLI: (artifact, None) or (None, stderr). + + The artifact must carry ``artifact_format``, the format the engine's + ``capabilities`` names. + """ process = subprocess.run( [ str(engine_bin), @@ -388,10 +419,11 @@ def compile_module( if process.returncode != 0: return None, (process.stderr.strip() or process.stdout.strip()) compiled = json.loads(artifact.read_text(encoding="utf-8")) - if compiled.get("artifact_format_version") != ARTIFACT_FORMAT_VERSION: + found = compiled.get("artifact_format_version") + if _artifact_format(found) != artifact_format: raise SurfaceError( - f"{artifact} has artifact format " - f"{compiled.get('artifact_format_version')!r}" + f"{artifact} has artifact format {found!r}; {engine_bin} names " + f"format {artifact_format}" ) return compiled, None @@ -489,6 +521,23 @@ def _blocker_error(entity: str, blockers: Sequence[str]) -> str: return message +def explained(message: str, blockers: Iterable[str]) -> bool: + """Whether one of the walk's blockers explains a dense error. + + A blocker explains an error equal to it. A cross-entity blocker also + explains its own text followed by ". " and the advice later engines add + (the module docstring has the detail). + """ + return any( + message == blocker + or ( + blocker.startswith(_CROSS_ENTITY_HEAD) + and message.startswith(blocker + ". ") + ) + for blocker in blockers + ) + + # ---------------------------------------------------------------------- # Dense surface # ---------------------------------------------------------------------- @@ -527,7 +576,7 @@ def build_snapshot( artifact_dir: Path, ) -> tuple[dict[str, Any], dict[str, Any] | None]: """Build the snapshot and, in dense mode, the CLI-walk differential.""" - version = engine_version(engine_bin) + version, artifact_format = engine_identity(engine_bin) replacements = { str(artifact_dir): "", str(rulespec_root): "", @@ -542,7 +591,7 @@ def build_snapshot( } artifact_path = artifact_dir / (relative.replace("/", "__") + ".json") artifact, error = compile_module( - engine_bin, module, rulespec_root, artifact_path + engine_bin, module, rulespec_root, artifact_path, artifact_format ) if artifact is None: record.update( @@ -610,7 +659,7 @@ def build_snapshot( f"{relative}: dense reports no {entity} program " "although the artifact has derived rules on it" ) - if message not in walk_blockers[entity]: + if not explained(message, walk_blockers[entity]): raise SurfaceError( f"{relative} [{entity}]: dense error not explained " f"by the artifact walk: {message}" From 74cd0376e51d65cf458a258aac6555c37d1ab0a1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 9 Nov 2026 21:34:06 -0500 Subject: [PATCH 03/14] NZ G8-x2: pin the engine at 753ce7d9, and the packaged NZ spec activates - axiom_rules_bindings.json: engine.commit 753ce7d9 and wheel_sha256 054c9196, the dense-extension wheel the NZ hub built at that commit (maturin build --release --locked; CPython 3.14, macOS arm64). They were the spec's last two activation gaps. The description's relations sentence now says what assert_no_relations checks. - nz.json: the refresh at 753ce7d9, with the walk that reads format 3, moves only engine.commit. The dense-root differential agreed on every module. - axiom_input_closure.json: the surface engine and the four engine source pins move to 753ce7d9. engine_optional_evidence and the undetermined judgment cite 753ce7d9's lines, each one read again at that commit. - test_nz_axiom_input_closure.py: the closure's surface engine commit equals the bindings' engine pin. - test_transport_composed_graph.py: the packaged spec activates, and each absent engine pin is the named gap. Preparation and the CLI now refuse at the missing RuleSpec tree, after the CLI opens its empty graph store. The inventory-store test drops the pins from its copy of the spec. - The input-surface README names each snapshot's own engine, as the refresh tool's docstring does. - transport_bridge_differential.py: the docstring describes the bridge as it stands (a counterfactual base rate and a solve_zero cutout). It drops the three-evaluation recipe and why_not_solve_zero, which as_rate_bridge.json no longer has. Co-Authored-By: Claude Opus 5.5 --- .../build/nz/axiom_input_closure.json | 14 +-- .../build/nz/axiom_rules_bindings.json | 8 +- .../shared/test_nz_axiom_input_closure.py | 14 ++- .../shared/test_transport_composed_graph.py | 95 +++++++++++-------- .../fixtures/axiom_input_surfaces/README.md | 32 ++++--- .../fixtures/axiom_input_surfaces/nz.json | 2 +- tools/transport_bridge_differential.py | 19 ++-- 7 files changed, 104 insertions(+), 80 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json index a7ff60d54b..00f2cf8ac5 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json +++ b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json @@ -14,7 +14,7 @@ "rulespec_commit": "6fe181fc4c65763150fe97501ef3443774e64869", "engine": { "repository": "TheAxiomFoundation/axiom-rules-engine", - "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66" + "commit": "753ce7d9ac4cfb161f6aa4f281f5a3980e67a57d" } }, "rulespec_pin": { @@ -47,28 +47,28 @@ "A main benefit only where the rates module's income-tested net weekly payment for it is positive: entitlement treats the s 24 income test and the s 21 work gap as met (jobseeker_income_less_than_zero_rate_cutout true, jobseeker_in_full_time_employment false), and the Sole Parent Support and Supported Living Payment entitlement rules have no income term; the rates module tests the person's own income only." ] }, - "engine_optional_evidence": "The axiom-rules-engine commit named in input_surface.engine (the engine that generated the committed input surface) has no optional inputs. A RuleSpec module has no inputs section (RulesDocument holds format, imports, module, units, relations and rules; src/rulespec.rs:361-374), a rule-level default is rejected (RuleSpecError::UnsupportedDefault, src/rulespec.rs:91-94), and the dense batch call collects only the inputs the caller supplies (python-ext/src/lib.rs:458-466). A plain read of an absent input fails every row whose evaluation reaches it with EvalError::MissingInput (src/dense.rs:2790-2803), and that fails the request. The internal input_or_else node that would read a default (src/dense.rs:2805-2811) is never produced from RuleSpec text: formula lowering turns a name no rule defines into a plain input read (src/formula.rs:1400-1413). So no root input of these modules can be verified engine-optional, and this closure classes none as such: an input a run does not encode is defaulted here, with its reason.", + "engine_optional_evidence": "The axiom-rules-engine commit named in input_surface.engine (the engine that generated the committed input surface) gives these modules no optional inputs. A RuleSpec inputs block can't declare a default: the engine keeps it only to check only_related and selected-year reads and to read elections, and every other key stays inert (RulesDocument, src/rulespec.rs:663-687; declared_elections, src/rulespec.rs:1569-1611). No module the committed input surface covers has an inputs block, at the surface's RuleSpec commit or at the pin. The engine rejects a rule-level default (RuleSpecError::UnsupportedDefault, src/rulespec.rs:274-277), and the dense batch call collects only the inputs the caller supplies (python-ext/src/lib.rs:552-563). A plain read of an absent input fails every row whose evaluation reaches it with EvalError::MissingInput (src/dense.rs:5154-5168), and the first row error fails the whole request (src/dense.rs:1539, python-ext/src/lib.rs:404-409). The internal input_or_else node that would read a default (src/dense.rs:5169-5175) never comes from RuleSpec text: formula lowering turns a name that no parameter or rule defines into a plain input read (src/formula.rs:1833-1851). So no root input of these modules can be verified engine-optional, and this closure classes none as such: the closure defaults, with its reason, any input a run doesn't encode.", "engine_source_pins": { "note": "The axiom-rules-engine source files that engine_optional_evidence and undetermined_judgments cite by line, pinned at input_surface.engine.commit so each line citation names one exact file version.", "pins": [ { "repository": "TheAxiomFoundation/axiom-rules-engine", - "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66", + "commit": "753ce7d9ac4cfb161f6aa4f281f5a3980e67a57d", "path": "python-ext/src/lib.rs" }, { "repository": "TheAxiomFoundation/axiom-rules-engine", - "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66", + "commit": "753ce7d9ac4cfb161f6aa4f281f5a3980e67a57d", "path": "src/dense.rs" }, { "repository": "TheAxiomFoundation/axiom-rules-engine", - "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66", + "commit": "753ce7d9ac4cfb161f6aa4f281f5a3980e67a57d", "path": "src/formula.rs" }, { "repository": "TheAxiomFoundation/axiom-rules-engine", - "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66", + "commit": "753ce7d9ac4cfb161f6aa4f281f5a3980e67a57d", "path": "src/rulespec.rs" } ] @@ -80,7 +80,7 @@ "eligible" ], "new_default": true, - "reason": "Entitlement judgments come back as codes 1 (holds), -1 (does not hold) and 0 (undetermined). At the axiom-rules-engine commit named in input_surface.engine, no dense leaf yields an undetermined value: dense input columns are booleans, integers, floats or text, an absent input is an error rather than an unknown, and and/or/not only pass an undetermined item along. With every root input encoded or defaulted by this closure, a 0 cannot arise, so the receipt kernel takeup.assign@1, once implemented, is to refuse any 0 it sees instead of guessing: a 0 would mean the engine or the closure changed. The alternatives a review could choose are treating 0 as not entitled (understates receipt) or as entitled (overstates it). microcosm.frame.input_closure.resolve_judgments implements all three." + "reason": "Entitlement judgments come back as codes 1 (holds), -1 (does not hold) and 0 (undetermined). At the axiom-rules-engine commit named in input_surface.engine, no dense leaf yields an undetermined value: dense input columns hold booleans, integers, decimals, floats, text or dates, an absent input is an error rather than an unknown, and and/or/not only pass an undetermined item along. With every root input encoded or defaulted by this closure, a 0 cannot arise, so the receipt kernel takeup.assign@1, once implemented, is to refuse any 0 it sees instead of guessing: a 0 would mean the engine or the closure changed. The alternatives a review could choose are treating 0 as not entitled (understates receipt) or as entitled (overstates it). microcosm.frame.input_closure.resolve_judgments implements all three." }, "knobs": [ { diff --git a/packages/microcosm-build/src/microcosm/build/nz/axiom_rules_bindings.json b/packages/microcosm-build/src/microcosm/build/nz/axiom_rules_bindings.json index 0fc8156f90..4d89ca4190 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/axiom_rules_bindings.json +++ b/packages/microcosm-build/src/microcosm/build/nz/axiom_rules_bindings.json @@ -1,7 +1,7 @@ { "version": 1, "country": "nz", - "description": "Axiom rules bindings for the New Zealand v0 graph build. Each binding names one rulespec-nz module by repository-relative rulespec_path and SHA-256 at the pinned commit, the Frame entity it runs on, the period label it is evaluated for, and the variables the graph requests. No code interprets these bindings yet. The planned shared composer (package G6) will build one AxiomEngine per binding and register every binding in one simulate.rules_by_ref@1 kernel (to be added by package G1), because a run's kernel registry holds one kernel per ref. The Axiom adapter raises when a module declares relations; every bound module declares none at the pin. No kernel implementation hash is to bind the whole NZ spec.", + "description": "Axiom rules bindings for the New Zealand v0 graph build. Each binding names one rulespec-nz module by repository-relative rulespec_path and SHA-256 at the pinned commit, the Frame entity it runs on, the period label it runs for, and the variables the graph requests. build_transport_registry (transport/registry.py) builds one AxiomEngine per binding, and one simulate.rules_by_ref@1 router serves them all, because a run's kernel registry holds one kernel per ref. A binding that declares no relations goes through assert_no_relations, which refuses a module whose dense program on the binding's entity declares relations; every bound module declares none at the pin. No kernel implementation hash is to bind the whole NZ spec.", "rulespec": { "repository": "TheAxiomFoundation/rulespec-nz", "commit": "8dc2507d5f10a0467c285fed54d018ee37246798", @@ -12,9 +12,9 @@ }, "engine": { "repository": "TheAxiomFoundation/axiom-rules-engine", - "commit": null, - "wheel_sha256": null, - "status": "awaiting the hub engine pin at M0 (D1 method card MC1); the composer (package G6, not yet written) is to refuse to run until both pins are set" + "commit": "753ce7d9ac4cfb161f6aa4f281f5a3980e67a57d", + "wheel_sha256": "054c9196d5371f13bb7c2b28f8161ac92294e5c8543aeb6e221fde243897ed42", + "status": "Pinned for G8-x (D1 method card MC1). commit is the axiom-rules-engine commit that NZ's input surface and input closure record. wheel_sha256 is the SHA-256 of the dense-extension wheel the NZ hub built at that commit (maturin build --release --locked; CPython 3.14, macOS arm64). axiom_engine_ref records both as declared and hashes no installed wheel." }, "entity_names": { "person": "Person", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py index de39d1b953..5be96d73de 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py @@ -218,15 +218,21 @@ def test_every_default_is_cited_and_listed_for_review(self) -> None: ) def test_no_input_is_claimed_engine_optional(self) -> None: - # The axiom-rules-engine commit in input_surface.engine (04315d94) has - # no optional inputs; the closure says why and classes none as - # engine-optional. + # The axiom-rules-engine commit in input_surface.engine (753ce7d9, the + # bindings' engine pin) gives the modules no optional inputs; the + # closure says why and classes none as engine-optional. assert not any( entry.closure_class is ClosureClass.ENGINE_OPTIONAL for entry in CLOSURE.entries ) assert "input_surface.engine" in CLOSURE.engine_optional_evidence - assert CLOSURE.surface_engine_commit.startswith("04315d94") + assert CLOSURE.surface_engine_commit.startswith("753ce7d9") + bindings = json.loads( + (CLOSURE_PATH.parent / "axiom_rules_bindings.json").read_text( + encoding="utf-8" + ) + ) + assert CLOSURE.surface_engine_commit == bindings["engine"]["commit"] assert CLOSURE.surface_engine_repository == ( "TheAxiomFoundation/axiom-rules-engine" ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index a3b2c2e2ce..5b6681b73e 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -2748,48 +2748,51 @@ def test_cli_arguments(): # --------------------------------------------------------------------------- -# The packaged country spec: activation gaps refuse, named +# The packaged country spec: it activates, and each engine pin it needs is a +# named gap when absent # --------------------------------------------------------------------------- -_GAPS = ( +_PINS = ( "axiom_rules_bindings.engine.commit", "axiom_rules_bindings.engine.wheel_sha256", ) +_NO_RULESPEC_TREE = "Source 'rulespec_nz' (the RuleSpec tree) needs a path." -def test_packaged_spec_refuses_with_every_activation_gap_named(): +def _unpinned(spec, pins): + spec = copy.deepcopy(spec) + for pin in pins: + spec["resources"]["axiom_rules_bindings"]["engine"][pin.rsplit(".", 1)[1]] = ( + None + ) + return spec + + +def test_the_packaged_spec_activates(): + # G8-x2 set the engine pins, the last two gaps. Every reference row that + # isn't active is exempt from its selection, so none is a gap either. spec = load_transport_spec("nz") - with pytest.raises(ValueError) as error: - validate_transport_activation(spec) - for gap in _GAPS: - assert gap in str(error.value), gap - assert len( - str(error.value).removeprefix("Transport spec is not activated: ").split("; ") - ) == len(_GAPS) - for removed in ( - "placeholder reference precal_references.", - "placeholder reference target_references.", - "placeholder reference holdout_references.", - "receipt_contract.json", - "scenario S4 rent_stock_factor", - ): - assert removed not in str(error.value), removed - # The dependent-child rule is settled from the Social Security Act 2018 - # Schedule 2, so it is no longer an activation gap. - assert "benefit_unit_rule.dependent_child" not in str(error.value) + validate_transport_activation(spec) selections = compose_module._reference_selections(spec) for resource in ("precal_references", "target_references", "holdout_references"): exempt = {row["name"] for row in selections[resource]["exempt"]} for reference in compose_module.transport_resource(spec, resource)[ "target_references" ]: - if ( - reference["metadata"].get("activation_status", "active") != "active" - and reference["name"] not in exempt - ): - assert f"placeholder reference {resource}.{reference['name']}" in str( - error.value - ) + assert ( + reference["metadata"].get("activation_status", "active") == "active" + or reference["name"] in exempt + ), (resource, reference["name"]) + + +@pytest.mark.parametrize("pins", [_PINS[:1], _PINS[1:], _PINS]) +def test_each_absent_engine_pin_is_the_named_activation_gap(pins): + spec = _unpinned(load_transport_spec("nz"), pins) + with pytest.raises(ValueError) as error: + validate_transport_activation(spec) + assert str(error.value) == ( + "Transport spec is not activated: " + "; ".join(pins) + "." + ) config = TransportGraphConfig( engine_refs={"x": "y"}, create_outputs=(Owned("person", "age", "int64"),) ) @@ -2797,26 +2800,33 @@ def test_packaged_spec_refuses_with_every_activation_gap_named(): compose_transport_graph(spec, config) -def test_packaged_spec_refuses_before_reading_any_source(tmp_path, monkeypatch): - # Inside the packaged factors' validity, so activation decides the refusal. +def test_packaged_spec_needs_its_rulespec_tree_before_reading_any_source( + tmp_path, monkeypatch +): + # Inside the packaged factors' validity, so the source check decides. monkeypatch.setattr(cli_module, "_today", lambda: date(2026, 11, 5)) - missing = tmp_path / "absent" - with pytest.raises(ValueError, match="not activated"): - prepare_transport_build( - load_transport_spec("nz"), sources={"rulespec_nz": missing} - ) + spec = load_transport_spec("nz") + with pytest.raises(ValueError, match=re.escape(_NO_RULESPEC_TREE)): + prepare_transport_build(spec, sources={}) + with pytest.raises(ValueError, match="must resolve to a directory"): + prepare_transport_build(spec, sources={"rulespec_nz": tmp_path / "absent"}) -def test_cli_main_refuses_the_packaged_spec_and_writes_nothing( +def test_cli_main_refuses_the_packaged_spec_without_sources_and_stores_nothing( tmp_path, monkeypatch, capsys ): monkeypatch.setattr(cli_module, "_today", lambda: date(2026, 11, 5)) out = tmp_path / "out" assert cli_module.main(["--country", "nz", "--out", str(out)]) == 1 - message = capsys.readouterr().err - for gap in _GAPS: - assert gap in message, gap - assert not out.exists() + assert _NO_RULESPEC_TREE in capsys.readouterr().err + # Activation passes, so preparation opens the graph store, and the + # source check refuses before anything writes into it. + assert sorted(path.relative_to(out).as_posix() for path in out.rglob("*")) == [ + ".graph-store", + ".graph-store/objects", + ".graph-store/tmp", + ] + assert cli_module._store_is_cold(out / ".graph-store") @pytest.mark.parametrize("guard", ["reference", "absent", "empty"]) @@ -3531,11 +3541,14 @@ def test_prepare_does_not_create_inventory_store_before_activation( tmp_path, monkeypatch ): # Inside the packaged factors' validity, so activation decides the refusal. + # The packaged spec activates, so this copy drops its engine pins. monkeypatch.setattr(cli_module, "_today", lambda: date(2026, 11, 5)) inventory = tmp_path / "inventory" with pytest.raises(ValueError, match="not activated"): prepare_transport_build( - load_transport_spec("nz"), sources={}, inventory_store=inventory + _unpinned(load_transport_spec("nz"), _PINS), + sources={}, + inventory_store=inventory, ) assert not inventory.exists() diff --git a/packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/README.md b/packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/README.md index 11d0781fc2..66b1efee7c 100644 --- a/packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/README.md +++ b/packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/README.md @@ -50,10 +50,15 @@ Run from the repository root. `AXIOM` is the directory holding the `git fetch origin` in each first. Everything is built from `git archive` output in a scratch directory, never inside a checkout. +Each snapshot names its own engine commit: `nz.json` records 753ce7d9 and +`be.json` 04315d94. Set `ENGINE` to the `engine.commit` of the snapshot you +regenerate, and `CC` to that snapshot's country. + ```bash W= AXIOM= -ENGINE=04315d94d04efd61e27665ab3aa300d64fdf3a66 +ENGINE= +CC= NZ=6fe181fc4c65763150fe97501ef3443774e64869 BE=b105e2b3a3086ddd2de447d58a9b951346870dd1 mkdir -p $W/engine $W/rulespec-nz $W/rulespec-be @@ -67,21 +72,20 @@ VIRTUAL_ENV=$W/dense-venv uv pip install maturin $W/dense-venv/bin/maturin build --release --manifest-path python-ext/Cargo.toml \ -i $W/dense-venv/bin/python --out $W/wheels) VIRTUAL_ENV=$W/dense-venv uv pip install $W/wheels/*.whl $W/engine/python -for CC in nz be; do - SHA=$([ $CC = nz ] && echo $NZ || echo $BE) - uv run --no-sync python tools/refresh_axiom_input_surface.py \ - --country $CC --rulespec-root $W/rulespec-$CC --rulespec-commit $SHA \ - --rulespec-git-dir $AXIOM/rulespec-$CC \ - --engine-bin $W/engine/target/release/axiom-rules-engine \ - --engine-commit $ENGINE --dense-python $W/dense-venv/bin/python -done +SHA=$([ $CC = nz ] && echo $NZ || echo $BE) +uv run --no-sync python tools/refresh_axiom_input_surface.py \ + --country $CC --rulespec-root $W/rulespec-$CC --rulespec-commit $SHA \ + --rulespec-git-dir $AXIOM/rulespec-$CC \ + --engine-bin $W/engine/target/release/axiom-rules-engine \ + --engine-commit $ENGINE --dense-python $W/dense-venv/bin/python ``` Add `--check` to compare a fresh run with the checked-in file without writing. It exits 1 and summarizes the differences when they differ. -`maturin build` runs without `--locked` for one reason. At the pinned engine -commit, `python-ext/Cargo.lock` still records the path crate -`axiom-rules-engine` as 0.1.0, and the crate is 0.2.2. The extension must be -built for the interpreter that loads it: a wheel built for free-threaded 3.14t -does not import on GIL-enabled 3.14. +At 04315d94, `maturin build` runs without `--locked` for one reason: +`python-ext/Cargo.lock` there still records the path crate +`axiom-rules-engine` as 0.1.0, and the crate is 0.2.2. At 753ce7d9 the lock +records 0.2.2, and `maturin build --locked` builds the wheel. The extension +must be built for the interpreter that loads it: a wheel built for +free-threaded 3.14t does not import on GIL-enabled 3.14. diff --git a/packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/nz.json b/packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/nz.json index d8f8ad8582..4ac1451dbd 100644 --- a/packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/nz.json +++ b/packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/nz.json @@ -1,7 +1,7 @@ { "country": "nz", "engine": { - "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66", + "commit": "753ce7d9ac4cfb161f6aa4f281f5a3980e67a57d", "repository": "TheAxiomFoundation/axiom-rules-engine", "surface": "dense_root_inputs", "version": "axiom-rules-engine 0.2.2" diff --git a/tools/transport_bridge_differential.py b/tools/transport_bridge_differential.py index 13e64628e3..116500b9a3 100644 --- a/tools/transport_bridge_differential.py +++ b/tools/transport_bridge_differential.py @@ -17,15 +17,16 @@ preparation or engine execution. All numbers come from case data or the oracle; the script has no policy constants. -The golden-08 executable case is unresolved pending an approved declaration. -``build/nz/as_rate_bridge.json`` declares three sole-parent cutout evaluations: -``gross_rate``, ``reduction_at_anchor`` and ``reduction_after_step``, followed by -slope, threshold and cutout composition. Its ``why_not_solve_zero`` explains -why a zero solve on the unit's own Jobseeker schedule cannot reproduce that -harness pairing. This driver has no arithmetic-composition contract for those -evaluations and cannot yet run that recipe. Approved sampled-unit cases may -declare additional small entity-table rows and corresponding oracle arrays. -This command does not read donor HDF5. +The golden-08 case isn't declared yet: it needs an approved case declaration +and an explicit tolerance on each comparison. ``build/nz/as_rate_bridge.json`` +records golden-08's expected reg 17 base rate and reg 18 cutout under +``differential_expectations``, and names the axiom-oracles requests file they +come from. ``build/nz/entitlement_graph.json`` computes the base rate with +``simulate.counterfactual@1`` (``nz.scn.{scenario}.bridge.base_rate``) and the +cutout with ``simulate.solve_zero@1`` (``nz.as.bridge.cutout``). The registry +this driver builds registers both kernels, so a case can call them. Approved +sampled-unit cases may declare additional small entity-table rows and +corresponding oracle arrays. This command does not read donor HDF5. """ from __future__ import annotations From dc0939ffe6d22e60f175de5c0174ea59c9bcf996 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 9 Nov 2026 21:34:16 -0500 Subject: [PATCH 04/14] NZ G8-x2: a static walk checks each income kernel's module list (#1563 O2) The trace test checks the modules a kernel's run() executes, and can't see a module the run only reads constants from. test_support's static_reach walks run()'s source instead: every branch, with default argument values, decorators, class bases and attributes, and the module-level constants it reaches, each name followed through its imports to the module that defines it. It imports nothing from the repository. - test_each_kernel_hashes_every_module_its_run_can_reach: for each of the three kernels, the walk reaches exactly the listed modules outside microcosm.graph. - One cut, which the test names: Target.__post_init__ reads microcosm.calibrate.hierarchy only under `self.hierarchy is not None`. test_no_decoded_target_carries_a_hierarchy checks the premise: _read_target, which builds every decoded target, passes no hierarchy. test_the_walk_cuts_only_the_hierarchy_guard checks that the cut removes that one module and nothing else. - test_the_walk_sees_a_constant_the_trace_cant: with artifact_types dropped from the native report's list, the trace test still passes and the walk names the module. - The kernel module's docstring describes both tests. Co-Authored-By: Claude Opus 5.5 --- .../transport/income_evidence_kernels.py | 9 +- .../test_transport_income_evidence_kernels.py | 81 +++ test_support/microcosm_build/static_reach.py | 494 ++++++++++++++++++ 3 files changed, 581 insertions(+), 3 deletions(-) create mode 100644 test_support/microcosm_build/static_reach.py diff --git a/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py index b9342fee43..30bc0ed672 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py @@ -33,9 +33,12 @@ first-party module whose source its ``run()`` executes, and every first-party module outside ``microcosm.graph`` whose constants it reads. ``microcosm.graph.kernel`` and ``microcosm.graph.decl`` stay out, as in the -evidence kernels' lists. The trace test checks the executed modules and can't -see a constant read, so a comment marks each entry whose module a run only -reads constants from. +evidence kernels' lists. Two tests check each list. A trace of one run checks +the executed modules, and can't see a constant read. A static walk of +``run()``'s source (``test_support/microcosm_build/static_reach.py``) follows +every branch and constant outside ``microcosm.graph``, and leaves out one +guarded branch that no decoded target opens. A comment marks each entry whose +module a run only reads constants from. ``evidence_contract`` hashes only the kernels it lists, and this module stays outside the gate battery's binding closure, so adding it re-keys no battery. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py index 3307dfd35a..16795072c2 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py @@ -11,6 +11,7 @@ import copy import dataclasses +import inspect import json import os import random @@ -65,6 +66,7 @@ MEASURE_COMBINE_REF, ) from microcosm.calibrate import CalibrationProblem, Target +from microcosm.calibrate import artifacts as calibrate_artifacts from microcosm.calibrate.artifacts import ( PROBLEM_TYPE, SOLUTION_TYPE, @@ -87,6 +89,7 @@ from microcosm.graph.kernel import ArtifactValue, NumericScope from test_support.microcosm_build.chronicle_feed import write_pinned_artifact from test_support.microcosm_build.first_party_imports import hashed_module_names +from test_support.microcosm_build.static_reach import Cut, reach from test_support.microcosm_build.transport_graph import toy_fact from test_support.microcosm_build.transport_population import digest from test_support.paths import paths_for @@ -1892,6 +1895,84 @@ def test_the_native_report_runs_the_schema_module(kernel_traces) -> None: } +# --------------------------------------------------------------------------- +# Each kernel hashes every module its run() can reach (#1563 O2): the +# constants half of the module-list rule, which no trace can see +# --------------------------------------------------------------------------- + +#: The one guarded branch the static walk leaves out. Target.__post_init__ +#: names microcosm.calibrate.hierarchy only for a target that carries a +#: hierarchy, and _read_target, which builds every target a problem decodes +#: to, passes none (test_no_decoded_target_carries_a_hierarchy). +HIERARCHY_GUARD = Cut( + "microcosm.calibrate.target", "Target.__post_init__", "self.hierarchy is not None" +) + + +def _outside_graph(modules) -> set[str]: + # The lists leave out microcosm.graph.kernel and microcosm.graph.decl by + # rule, and the trace test checks microcosm.graph.canonical. + return { + name + for name in modules + if name != "microcosm.graph" and not name.startswith("microcosm.graph.") + } + + +def _walk(ref, cuts=(HIERARCHY_GUARD,)): + kernel = type(KERNELS[ref]) + return reach(kernel.__module__, f"{kernel.__qualname__}.run", cuts=cuts) + + +@pytest.mark.parametrize("ref", sorted(KERNELS)) +def test_each_kernel_hashes_every_module_its_run_can_reach(ref) -> None: + # A static walk of run(), over every branch with the defaults, + # decorators, class attributes and module-level constants it reaches, + # names exactly the listed modules outside microcosm.graph. The trace + # test sees only the modules one run executes, so a new constant read + # from an unlisted module would pass it and leave the hash short. + walked = _walk(ref) + listed = _outside_graph(INCOME_EVIDENCE_KERNEL_MODULES[ref]) + reached = _outside_graph(walked.modules) + assert {name: walked.names(name) for name in reached - listed} == {} + assert listed - reached == set() + + +def test_the_walk_cuts_only_the_hierarchy_guard() -> None: + # Without the cut, the register and the native report also reach + # microcosm.calibrate.hierarchy, and nothing else off their lists. + for ref in sorted(KERNELS): + extra = _outside_graph(_walk(ref, cuts=()).modules) - set( + INCOME_EVIDENCE_KERNEL_MODULES[ref] + ) + decodes = ref != INCOME_IDENTITY_REF + assert extra == ({"microcosm.calibrate.hierarchy"} if decodes else set()), ref + assert _walk(ref).cuts_used == ( + frozenset({HIERARCHY_GUARD}) if decodes else frozenset() + ), ref + + +def test_no_decoded_target_carries_a_hierarchy() -> None: + # The cut's premise: _read_target passes no hierarchy, so every target a + # problem decodes to keeps the default None and the guard never opens. + assert "hierarchy" not in inspect.getsource(calibrate_artifacts._read_target) + problem = decode_problem(_encoded_problem([1, 2, 3], np.ones(3))) + assert [target.hierarchy for target in problem.problem.targets] == [None] + + +def test_the_walk_sees_a_constant_the_trace_cant(kernel_traces) -> None: + # Mutant: the native report's list loses + # microcosm.build.transport.artifact_types, whose + # CALIBRATION_PREPARATION_TYPE its run() reads but never calls. The trace + # test still passes; the walk names the dropped module. + ref = NATIVE_BAND_REPORT_REF + dropped = "microcosm.build.transport.artifact_types" + shortened = dict(INCOME_EVIDENCE_KERNEL_MODULES) + shortened[ref] = tuple(name for name in shortened[ref] if name != dropped) + assert _unlisted(ref, kernel_traces[ref], shortened) == set() + assert _outside_graph(_walk(ref).modules) - set(shortened[ref]) == {dropped} + + def test_the_module_stays_outside_the_batterys_binding_closure() -> None: # The registry and the composer import this module, and no battery # binding's identity covers it or them, so adding the kernels re-keys no diff --git a/test_support/microcosm_build/static_reach.py b/test_support/microcosm_build/static_reach.py new file mode 100644 index 0000000000..0ab2d4212a --- /dev/null +++ b/test_support/microcosm_build/static_reach.py @@ -0,0 +1,494 @@ +"""The first-party modules a function can reach, read statically from source. + +A kernel's implementation hash covers the modules its list names. A trace of +one run sees the modules whose functions that run calls. It can't see a +module the run only reads constants from, or a branch the run didn't take. +This walk reads source instead, and over-approximates. From a root function +it follows every branch of each body it reaches, with default argument +values, decorators, class bases, class attributes and module-level constant +values. It resolves each name through imports, at module or function level, +to the module that defines it. An attribute of an object whose type the walk +can't tell reaches every same-named member of each class the walk has +referenced. + +It imports nothing from the repository and runs none of its code: it parses +each first-party module's file under ``packages/*/src``. + +What it can't see: a name built at run time (``getattr`` with a computed +name, ``importlib``), a package's lazy ``__getattr__`` exports, and code that +an argument carries in from outside the walk (the executor's context, for +one). Annotations stay out: a ``def`` evaluates them at import, or never +under ``from __future__ import annotations``. A :class:`Cut` leaves out one +guarded branch, so a test names each exception it relies on rather than +skipping a module. +""" + +from __future__ import annotations + +import ast +from collections import defaultdict +from collections.abc import Iterable, Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path + +from test_support.paths import REPOSITORY_ROOT + +#: Dunder methods Python calls without naming them. A referenced class +#: reaches each one it defines. +IMPLICIT_METHODS = frozenset( + { + "__init__", + "__post_init__", + "__new__", + "__init_subclass__", + "__set_name__", + "__class_getitem__", + "__call__", + "__len__", + "__iter__", + "__next__", + "__getitem__", + "__contains__", + "__bool__", + "__eq__", + "__ne__", + "__hash__", + "__lt__", + "__le__", + "__gt__", + "__ge__", + "__str__", + "__repr__", + "__format__", + "__getattr__", + "__getattribute__", + "__setattr__", + "__delattr__", + "__enter__", + "__exit__", + } +) + +#: One reached thing: (defining module, kind, qualified name or None). +Item = tuple[str, str, str | None] + + +@dataclass(frozen=True) +class Cut: + """One guarded branch the walk leaves out: an ``if`` in one function. + + In ``module``'s ``function`` (a qualname such as ``Target.__post_init__``) + the walk skips the test and the body of each ``if`` whose test's source + contains ``guard``. It still walks the ``else`` branch. + """ + + module: str + function: str + guard: str + + +def module_paths(root: Path = REPOSITORY_ROOT) -> dict[str, Path]: + """Each first-party module's dotted name -> its file under ``packages/*/src``.""" + + paths = {} + for path in sorted(root.glob("packages/*/src/microcosm/**/*.py")): + source_root = next(parent for parent in path.parents if parent.name == "src") + parts = path.relative_to(source_root).with_suffix("").parts + if parts[-1] == "__init__": + parts = parts[:-1] + paths[".".join(parts)] = path + return paths + + +def _statements(body: Iterable[ast.stmt]) -> Iterator[ast.stmt]: + """Module-level statements, through ``if``, ``try`` and ``with`` blocks.""" + + for node in body: + yield node + if isinstance(node, ast.If | ast.With | ast.Try | ast.TryStar): + yield from _statements(node.body) + yield from _statements(getattr(node, "orelse", ())) + yield from _statements(getattr(node, "finalbody", ())) + for handler in getattr(node, "handlers", ()): + yield from _statements(handler.body) + + +class _Module: + """One parsed module's top-level names: imports, functions, classes, values.""" + + def __init__(self, name: str, path: Path) -> None: + self.name = name + self.is_package = path.name == "__init__.py" + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + self.imports: dict[str, tuple[str, str | None]] = {} + self.functions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} + self.classes: dict[str, ast.ClassDef] = {} + self.constants: dict[str, list[ast.AST]] = defaultdict(list) + for node in _statements(tree.body): + if isinstance(node, ast.Import | ast.ImportFrom): + self.imports.update(_bindings(self, node)) + elif isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + self.functions[node.name] = node + elif isinstance(node, ast.ClassDef): + self.classes[node.name] = node + elif isinstance(node, ast.Assign | ast.AnnAssign | ast.AugAssign): + targets = ( + node.targets if isinstance(node, ast.Assign) else [node.target] + ) + if node.value is None: + continue + for target in targets: + for name in ast.walk(target): + if isinstance(name, ast.Name): + self.constants[name.id].append(node.value) + + def absolute(self, level: int, module: str | None) -> str: + """The module a ``from`` import names, relative imports resolved.""" + + if not level: + return module or "" + parts = self.name.split(".") + if not self.is_package: + parts = parts[:-1] + parts = parts[: len(parts) - (level - 1)] + base = ".".join(parts) + return f"{base}.{module}" if module else base + + +def _bindings( + module: _Module, node: ast.Import | ast.ImportFrom +) -> dict[str, tuple[str, str | None]]: + """The names one import statement binds -> (module, attribute or None).""" + + if isinstance(node, ast.Import): + return { + (alias.asname or alias.name.split(".")[0]): ( + alias.name if alias.asname else alias.name.split(".")[0], + None, + ) + for alias in node.names + } + stem = module.absolute(node.level, node.module) + return {(alias.asname or alias.name): (stem, alias.name) for alias in node.names} + + +def _annotations(node: ast.AST) -> Iterator[ast.AST]: + """The annotation expressions inside ``node``.""" + + for child in ast.walk(node): + if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef): + yield from (item for item in [child.returns] if item is not None) + arguments = child.args + for argument in ( + *arguments.posonlyargs, + *arguments.args, + *arguments.kwonlyargs, + arguments.vararg, + arguments.kwarg, + ): + if argument is not None and argument.annotation is not None: + yield argument.annotation + elif isinstance(child, ast.AnnAssign): + yield child.annotation + + +def _walk(node: ast.AST, skipped: frozenset[int]) -> Iterator[ast.AST]: + """Every node under ``node``, pruning each subtree whose id is skipped.""" + + stack = [node] + while stack: + current = stack.pop() + if id(current) in skipped: + continue + yield current + stack.extend(ast.iter_child_nodes(current)) + + +class _Walk: + def __init__(self, paths: Mapping[str, Path], cuts: Iterable[Cut]) -> None: + self.paths = paths + # Every module name and each dotted prefix of one, so a namespace + # package (``microcosm``) resolves on the way to its modules. + self.prefixes = { + ".".join(name.split(".")[:size]) + for name in paths + for size in range(1, name.count(".") + 2) + } + self.cuts = tuple(cuts) + self.modules: dict[str, _Module] = {} + self.reached: dict[Item, set[str]] = defaultdict(set) + self.pending: list[Item] = [] + self.classes: set[tuple[str, str]] = set() + self.attributes: set[str] = set() + self.cuts_used: set[Cut] = set() + + def module(self, name: str) -> _Module | None: + if name not in self.paths: + return None + if name not in self.modules: + self.modules[name] = _Module(name, self.paths[name]) + return self.modules[name] + + def is_module(self, name: str) -> bool: + """A first-party module, or a namespace package that holds one.""" + + return name in self.prefixes + + def push(self, item: Item, site: str) -> None: + # A namespace package has no file, and so nothing to hash. + if item[0] not in self.paths: + return + if item not in self.reached: + self.pending.append(item) + self.reached[item].add(site) + + def resolve_expression(self, module: str, node: ast.AST) -> Item | None: + """The item a name or a ``module.attribute`` chain names, if first-party.""" + + chain = [] + while isinstance(node, ast.Attribute): + chain.append(node.attr) + node = node.value + if not isinstance(node, ast.Name): + return None + item = self.resolve(module, node.id) + for attribute in reversed(chain): + if item is None or item[1] != "module": + return None + target = item[0] + if self.is_module(f"{target}.{attribute}"): + item = (f"{target}.{attribute}", "module", None) + else: + item = self.resolve(target, attribute) + return item + + def resolve(self, module: str, name: str, depth: int = 0) -> Item | None: + """Follow ``name`` in ``module``'s scope to the item that defines it.""" + + found = self.module(module) + if found is None or depth > 32: + return None + if name in found.functions: + return (module, "function", name) + if name in found.classes: + return (module, "class", name) + if name in found.constants: + return (module, "constant", name) + if name in found.imports: + return self.resolve_import(*found.imports[name], depth=depth + 1) + return None + + def resolve_import( + self, target: str, attribute: str | None, *, depth: int = 0 + ) -> Item | None: + if attribute is None: + return (target, "module", None) if self.is_module(target) else None + if self.is_module(f"{target}.{attribute}"): + return (f"{target}.{attribute}", "module", None) + if target in self.paths: + return self.resolve(target, attribute, depth) + return None + + def skipped(self, module: str, qualname: str, node: ast.AST) -> frozenset[int]: + """Annotations, and the subtrees each cut in this function leaves out.""" + + ids = {id(child) for item in _annotations(node) for child in ast.walk(item)} + for cut in self.cuts: + if (cut.module, cut.function) != (module, qualname): + continue + for child in ast.walk(node): + if isinstance(child, ast.If) and cut.guard in ast.unparse(child.test): + self.cuts_used.add(cut) + for part in (child.test, *child.body): + ids.update(id(item) for item in ast.walk(part)) + return frozenset(ids) + + def scan( + self, + module: str, + node: ast.AST, + site: str, + *, + qualname: str = "", + owner: str | None = None, + ) -> None: + """Push every first-party item ``node``'s code can name.""" + + found = self.module(module) + skipped = self.skipped(module, qualname, node) + bound: set[str] = set() + local_imports: dict[str, tuple[str, str | None]] = {} + for child in _walk(node, skipped): + if isinstance(child, ast.Import | ast.ImportFrom): + local_imports.update(_bindings(found, child)) + elif isinstance(child, ast.Name) and isinstance( + child.ctx, ast.Store | ast.Del + ): + bound.add(child.id) + elif isinstance(child, ast.arg): + bound.add(child.arg) + elif ( + isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) + and child is not node + ): + bound.add(child.name) + elif isinstance(child, ast.ExceptHandler) and child.name: + bound.add(child.name) + + def lookup(name: str) -> Item | None: + if name in local_imports: + return self.resolve_import(*local_imports[name]) + if name in bound: + return None + return self.resolve(module, name) + + for child in _walk(node, skipped): + line = getattr(child, "lineno", "?") + if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Load): + item = lookup(child.id) + if item is not None: + self.push(item, f"{site}:{line}") + elif isinstance(child, ast.Attribute): + self.attributes.add(child.attr) + chain = [] + current: ast.AST = child + while isinstance(current, ast.Attribute): + chain.append(current.attr) + current = current.value + if not isinstance(current, ast.Name): + continue + if owner is not None and current.id in {"self", "cls"}: + self.push( + (module, "member", f"{owner}.{chain[-1]}"), f"{site}:{line}" + ) + continue + item = lookup(current.id) + if item is None or item[1] != "module": + continue + target = item[0] + for attribute in reversed(chain): + if self.is_module(f"{target}.{attribute}"): + target = f"{target}.{attribute}" + self.push((target, "module", None), f"{site}:{line}") + continue + member = self.resolve(target, attribute) + if member is not None: + self.push(member, f"{site}:{line}") + break + + def members(self, module: str, name: str) -> dict[str, ast.AST]: + """A class body's methods and attribute values, by name.""" + + out: dict[str, ast.AST] = {} + for node in self.module(module).classes[name].body: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + out[node.name] = node + elif isinstance(node, ast.Assign | ast.AnnAssign): + targets = ( + node.targets if isinstance(node, ast.Assign) else [node.target] + ) + for target in targets: + if isinstance(target, ast.Name): + out[target.id] = node + return out + + def process(self, item: Item) -> None: + module, kind, name = item + found = self.module(module) + if found is None: + return + if kind == "function": + self.scan(module, found.functions[name], f"{module}.{name}", qualname=name) + elif kind == "constant": + for value in found.constants[name]: + self.scan(module, value, f"{module}.{name}=") + elif kind == "class": + self.classes.add((module, name)) + node = found.classes[name] + for part in (*node.bases, *node.keywords, *node.decorator_list): + self.scan(module, part, f"{module}.{name}(class)") + for member, body in self.members(module, name).items(): + if not isinstance(body, ast.FunctionDef | ast.AsyncFunctionDef) or ( + member in IMPLICIT_METHODS + ): + self.push( + (module, "member", f"{name}.{member}"), f"{module}.{name}" + ) + elif kind == "member": + owner, member = name.split(".", 1) + if owner not in found.classes: + return + members = self.members(module, owner) + if member in members: + self.scan( + module, + members[member], + f"{module}.{name}", + qualname=name, + owner=owner, + ) + return + # Inherited: the same name on each first-party base class. + for base in found.classes[owner].bases: + resolved = self.resolve_expression(module, base) + if resolved is not None and resolved[1] == "class": + self.push( + (resolved[0], "member", f"{resolved[2]}.{member}"), + f"{module}.{name}(base)", + ) + + def run(self, module: str, qualname: str) -> None: + if "." in qualname: + self.push((module, "member", qualname), "root") + else: + self.push((module, "function", qualname), "root") + while True: + while self.pending: + self.process(self.pending.pop()) + added = False + for module_name, class_name in sorted(self.classes): + for member in self.members(module_name, class_name): + item = (module_name, "member", f"{class_name}.{member}") + if member in self.attributes and item not in self.reached: + self.push(item, f"{module_name}.{class_name}(attribute)") + added = True + if not added: + return + + +@dataclass(frozen=True) +class Reach: + """What a static walk from one root reached.""" + + #: Each reached item -> the sites (``module.qualname:line``) that named it. + items: Mapping[Item, frozenset[str]] + #: The cuts whose guard the walk found and left out. + cuts_used: frozenset[Cut] + + @property + def modules(self) -> frozenset[str]: + return frozenset(module for module, _, _ in self.items) + + def names(self, module: str) -> list[str]: + """The reached items that ``module`` defines, for failure messages.""" + + return sorted( + f"{kind}:{name}" for found, kind, name in self.items if found == module + ) + + +def reach( + module: str, + qualname: str, + *, + cuts: Iterable[Cut] = (), + paths: Mapping[str, Path] | None = None, +) -> Reach: + """Statically walk ``module``'s ``qualname`` (a function or ``Class.method``).""" + + walk = _Walk(module_paths() if paths is None else paths, cuts) + walk.run(module, qualname) + return Reach( + {item: frozenset(sites) for item, sites in walk.reached.items()}, + frozenset(walk.cuts_used), + ) From 002b6c2f214b5a0a57fd4670c36237b96daea34a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 9 Nov 2026 21:41:29 -0500 Subject: [PATCH 05/14] NZ G8-x2: regenerate the NZ graph, the replay bindings and the golden tools/build_nz_transport_graph.py and the golden writer, run on this branch: - transport_graph.json: the three income evidence readers select their measurement from income_measurements' measurement_id (nit 1), and the omission register gains its solution edge from nz.calibrate (N3). - export_replay_bindings.json: the input closure's new sha256. - tests/golden/nz_country_spec.json: the fingerprint and the hashes of the four resources that changed. Every generator's --check passes on the result. Co-Authored-By: Claude Opus 5.5 --- .../build/nz/export_replay_bindings.json | 2 +- .../microcosm/build/nz/transport_graph.json | 29 +++++++++++++++++-- .../tests/golden/nz_country_spec.json | 10 +++---- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/nz/export_replay_bindings.json b/packages/microcosm-build/src/microcosm/build/nz/export_replay_bindings.json index e0fb5daaad..dcd95b15c2 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/export_replay_bindings.json +++ b/packages/microcosm-build/src/microcosm/build/nz/export_replay_bindings.json @@ -1,4 +1,4 @@ { "export_contract_sha256": "4d388236cc3d3382501f5cfae8eae44b661fba6919681965eaa3fe4e032efe86", - "axiom_input_closure_sha256": "e19c4f4e3903b1e23fdea46fe9227157d908c6ce943fbf53ee49531de9903714" + "axiom_input_closure_sha256": "acd994081b81c21f7b8da4ba80648d92cfd7ddd849db8bfa1db98c2fa2fe3b2e" } diff --git a/packages/microcosm-build/src/microcosm/build/nz/transport_graph.json b/packages/microcosm-build/src/microcosm/build/nz/transport_graph.json index 1c59537eef..af33c941a1 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/transport_graph.json +++ b/packages/microcosm-build/src/microcosm/build/nz/transport_graph.json @@ -4564,7 +4564,13 @@ ], "encoding": "value" }, - "measurement": "ird_taxable_income_ty2027_v1", + "measurement": { + "resource": "income_measurements", + "path": [ + "measurement_id" + ], + "encoding": "value" + }, "identities": { "resource": "income_measurements", "path": [ @@ -4636,7 +4642,13 @@ ], "encoding": "value" }, - "measurement": "ird_taxable_income_ty2027_v1", + "measurement": { + "resource": "income_measurements", + "path": [ + "measurement_id" + ], + "encoding": "value" + }, "weight_entity": "household" }, "artifact_inputs": [ @@ -4644,6 +4656,11 @@ "name": "problem", "producer": "nz.targets.problem", "artifact": "problem" + }, + { + "name": "solution", + "producer": "nz.calibrate", + "artifact": "solution" } ], "artifact_outputs": [ @@ -4702,7 +4719,13 @@ "encoding": "json" }, "weight_entity": "household", - "measurement": "ird_taxable_income_ty2027_v1" + "measurement": { + "resource": "income_measurements", + "path": [ + "measurement_id" + ], + "encoding": "value" + } }, "artifact_inputs": [ { diff --git a/packages/microcosm-build/tests/golden/nz_country_spec.json b/packages/microcosm-build/tests/golden/nz_country_spec.json index dd1568ece9..22e05780ec 100644 --- a/packages/microcosm-build/tests/golden/nz_country_spec.json +++ b/packages/microcosm-build/tests/golden/nz_country_spec.json @@ -1,6 +1,6 @@ { "country": "nz", - "fingerprint": "deaa12c8fbbe4b754d52f2880024d855b50ca7967a92accf3726bb83cb91823a", + "fingerprint": "e631a3cfa065675f5162fa7accf6aa3f5014aa9bac962ad3f8909c72c4d24099", "gate_ids": [ "calibration_per_family_fit", "demography_per_family_fit", @@ -43,8 +43,8 @@ "as_encoding.json": "91e0e9972abfaaa202a13ede6e2dbc815a4c6c9514726c970928a47adfef0e32", "as_rate_bridge.json": "1399321cc7ef44486f7ab407968b036930ff6ae8fbab4c30ce710ea0fff7301c", "atomic_geography_support.json": "6efd09d5cde2f16d4d01e522a721191a829a754d2ce019d75bb0407085ecb4b7", - "axiom_input_closure.json": "e19c4f4e3903b1e23fdea46fe9227157d908c6ce943fbf53ee49531de9903714", - "axiom_rules_bindings.json": "2ec91292e3231a4524c372ba78f1e9fedc62a34f8a4f6e38a274c9f608d138a6", + "axiom_input_closure.json": "acd994081b81c21f7b8da4ba80648d92cfd7ddd849db8bfa1db98c2fa2fe3b2e", + "axiom_rules_bindings.json": "5a83c1dd1778dd33e58b4b4064df4ad5e32239e2bd812e53f4b1aac2e3e64e7f", "benefit_gross_up.json": "5518e7ad23b4053d36df842a3335358f8ff89c494ed065e540eb83585d8c6137", "benefit_unit_rule.json": "c84efba8353c770131256a8c17b82c7d98f738e6dd8ef217e19d30014cb14604", "calibration_bindings.json": "26983da893770a7ea915d9eee70c47f310654ffe0fdffcc0327ac5a34c01bf50", @@ -56,7 +56,7 @@ "earnings_bands.json": "7c9f6ae99bf4346fba0ae315037b3d65b59472cd588a1573e7a3f3091c168342", "entitlement_graph.json": "991c41aa78d64e04ccbdb6f846ee2bc4a0152e102a8dc241d2b1e725cb2300d3", "export_contract.json": "4d388236cc3d3382501f5cfae8eae44b661fba6919681965eaa3fe4e032efe86", - "export_replay_bindings.json": "dfdad54e817d655c278a75bce028481295e6bcf82813e4565105f284f65cb762", + "export_replay_bindings.json": "1618f9935a12ff6128f542c76521fd1140c52448077168d483e7e56a4773f5ed", "family_reconciliation.json": "20e3f9f3ce6266d57828d721c99d15bd3642e6c6314a7076ec38972de02bc69e", "gap_annualisation.json": "54762c18c549d2f1f35a630358080e3d581abab901b52a62b4f279830f3f8536", "gates.json": "4bc5b4ebb2e8a2114d74b7d403e1f2f334913e4446159546aa12ffb5c0c8e201", @@ -92,7 +92,7 @@ "spec/vintages.yaml": "1ecc1a3f952bd25b834c394314e8c933ddb5b095fd64ba492dddb03e3cdd803e", "target_references.json": "6af5258a4e3451d7742ab9f3b454fec9e0bd7cf320fab6e784ff002a8be211c8", "transport_atomic_geography_support.json": "53465827f67c87aaf5a06c305c082cbf1e6cdf28f3843e853d1e012635517d10", - "transport_graph.json": "c7f7a89495fcb7424279ffc1d6ae98bfdf6552eebead4ae69967264deaddb3c4", + "transport_graph.json": "1fc558bedfd88d16f31afd86b148c810be33c11ffb2851379ea7358d3e9c6257", "transport_inputs.json": "16d3957214d6305e3d3f41fe6343143a411d6055c75c43d3f171db3ee659ec2d" }, "resources": [ From cd85cffbed2efc9b01f4e9040eaa7a447bd644c9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 9 Nov 2026 21:49:59 -0500 Subject: [PATCH 06/14] NZ G8-x2: test the refresh tool's dense mode, and settle its properties Belgium's review of d40c004d asked for two changes before landing, and this commit makes both, plus three of their optional ones. - A stand-in --dense-python drives build_snapshot in dense mode. It ignores the probe after -c, records the jobs it reads from stdin, and answers each entity from the outcomes its test gives it. A Person rule that reads a Household rule gives the walk one cross-entity blocker. - With the engine's error as the blocker's text, or that text followed by each of the engine's four advice forms, the module records dense_unsupported with the walk's text, and the differential counts two entity programs and one dense compile. - The tool refuses an error the walk can't explain (another cross-entity read, a new limitation, an advised error on an entity the walk doesn't block), a missing program, another root and duplicate root inputs. - The differential records an input, a status and a catalog mismatch where dense and the walk disagree. - Both Hypothesis properties carry deadline=None and suppress the too_slow health check, as test_be_target_references.py does. The cross-entity property failed that check in 3 of 13 runs on Belgium's host. - The versioned property now checks the blocker's own text, another rule's versioned text, and its text with advice after it. The cross-entity property checks its own text with and without advice. - The off-format test adds an engine that names format 2 and writes 3. - The advice list adds the engine's fourth form (a relation read where the program declares no relation). - The tool's docstring says what the advice suggests (compile for the other entity, read through a relation, or compile only the outputs that don't reach the reading rule) and drops two passives. Co-Authored-By: Claude Opus 5.5 --- .../test_refresh_axiom_input_surface.py | 290 ++++++++++++++++-- tools/refresh_axiom_input_surface.py | 19 +- 2 files changed, 281 insertions(+), 28 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_refresh_axiom_input_surface.py b/packages/microcosm-build/tests/engine_free/shared/test_refresh_axiom_input_surface.py index e7c31711e8..9f5f9836c0 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_refresh_axiom_input_surface.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_refresh_axiom_input_surface.py @@ -2,7 +2,9 @@ Engine-free. A stand-in CLI answers ``--version``, ``capabilities`` and ``compile`` the way ``tools/refresh_axiom_input_surface.py`` calls them, and -writes the artifact each test gives it. +writes the artifact each test gives it. A stand-in ``--dense-python`` reads +the dense probe's jobs from stdin and answers each entity from the outcomes +its test gives it. """ from __future__ import annotations @@ -15,7 +17,7 @@ from pathlib import Path import pytest -from hypothesis import given +from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st from tools import refresh_axiom_input_surface as refresh @@ -92,6 +94,15 @@ def _format_3() -> dict: return artifact +def _wrapper(path: Path, script: Path) -> Path: + """An executable at ``path`` that runs ``script`` under this interpreter.""" + path.write_text( + f'#!/bin/sh\nexec "{sys.executable}" "{script}" "$@"\n', encoding="utf-8" + ) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + return path + + def _engine(tmp_path: Path, advertised, artifact: dict | None = None) -> Path: script = tmp_path / "stand_in_engine.py" script.write_text( @@ -101,12 +112,7 @@ def _engine(tmp_path: Path, advertised, artifact: dict | None = None) -> Path: ), encoding="utf-8", ) - engine = tmp_path / "axiom-rules-engine" - engine.write_text( - f'#!/bin/sh\nexec "{sys.executable}" "{script}" "$@"\n', encoding="utf-8" - ) - engine.chmod(engine.stat().st_mode | stat.S_IXUSR) - return engine + return _wrapper(tmp_path / "axiom-rules-engine", script) _MODULE_TEXT = b"stand-in module; the stand-in CLI ignores it\n" @@ -165,19 +171,23 @@ def test_an_engine_of_another_format_refuses(tmp_path, advertised) -> None: refresh.engine_identity(_engine(tmp_path, advertised)) -@pytest.mark.parametrize("written", [2, 3.0, None]) -def test_an_artifact_off_the_engines_format_refuses(tmp_path, written) -> None: +@pytest.mark.parametrize( + ("advertised", "written"), [(3, 2), (3, 3.0), (3, None), (2, 3)] +) +def test_an_artifact_off_the_engines_format_refuses( + tmp_path, advertised, written +) -> None: artifact = _format_3() artifact["artifact_format_version"] = written - engine = _engine(tmp_path, 3, artifact) - assert refresh.engine_identity(engine)[1] == 3 + engine = _engine(tmp_path, advertised, artifact) + assert refresh.engine_identity(engine)[1] == advertised module = _rulespec_root(tmp_path) / "nz" / "m.yaml" with pytest.raises( refresh.SurfaceError, - match=f"has artifact format {written!r}; .* names format 3", + match=f"has artifact format {written!r}; .* names format {advertised}", ): refresh.compile_module( - engine, module, module.parents[1], tmp_path / "m.json", 3 + engine, module, module.parents[1], tmp_path / "m.json", advertised ) @@ -205,8 +215,9 @@ def test_a_format_3_artifact_gives_the_format_2_snapshot(tmp_path) -> None: _BLOCKER = refresh._CROSS_ENTITY.format( dependency="hh_income", derived="person_benefit", entity="Household" ) -#: Advice in each of the three forms engine 753ce7d9 writes -#: (``cross_entity_advice``, src/dense.rs:560-590). +#: Advice in each form engine 753ce7d9 writes (``cross_entity_advice``, +#: src/dense.rs:560-590): for a Scalar root, for a filtered entity's rows, and +#: for a read through a relation, with one the program declares and with none. _ADVICE = ( "This dense program's root is the entity-free `Scalar`, whose rows hold " "no `Household`: compile it for `Household` instead, or compile only " @@ -220,6 +231,12 @@ def test_a_format_3_artifact_gives_the_format_2_snapshot(tmp_path) -> None: "between `Person` and `Household`, such as `member_of_household`, which " "this program declares, or compile only outputs that do not reach " "`person_benefit`", + "Dense evaluates the rows of one entity and reads another entity's rule " + "only through a relation, by design: read `hh_income` with a related " + "aggregate (`only`, `sum`, `min`, `max`, `count_where`) over a relation " + "between `Person` and `Household` (this program declares none; a " + "`data_relation` whose `arguments` name both entities is one), or compile " + "only outputs that do not reach `person_benefit`", ) @@ -260,6 +277,7 @@ def test_any_other_blocker_explains_only_its_own_text() -> None: _TRIPLE = st.tuples(_NAME, _NAME, _NAME) +@settings(deadline=None, suppress_health_check=[HealthCheck.too_slow]) @given(_TRIPLE, _TRIPLE, st.text(max_size=80)) def test_property_a_cross_entity_blocker_explains_exactly_its_names( blocked, read, advice @@ -273,12 +291,246 @@ def message(names): ) blocker = message(blocked) + assert refresh.explained(blocker, [blocker]) + assert refresh.explained(f"{blocker}. {advice}", [blocker]) same = blocked == read assert refresh.explained(message(read), [blocker]) is same assert refresh.explained(f"{message(read)}. {advice}", [blocker]) is same -@given(_NAME, st.text(max_size=120)) -def test_property_a_versioned_blocker_explains_only_itself(name, message) -> None: +@settings(deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given(_NAME, _NAME, st.text(max_size=120)) +def test_property_a_versioned_blocker_explains_only_itself(name, other, text) -> None: + # Its own text matches. Another rule's versioned text matches only under + # the same name, and no text matches with advice after it. versioned = refresh._VERSIONED.format(name=name) - assert refresh.explained(message, [versioned]) is (message == versioned) + assert refresh.explained(versioned, [versioned]) + assert refresh.explained(refresh._VERSIONED.format(name=other), [versioned]) is ( + other == name + ) + assert refresh.explained(text, [versioned]) is (text == versioned) + assert not refresh.explained(f"{versioned}. {text}", [versioned]) + + +# --------------------------------------------------------------------------- +# Dense mode +# --------------------------------------------------------------------------- + +#: Stands in for ``--dense-python``. It ignores the probe after ``-c``, writes +#: the jobs it reads from stdin to JOBS, and answers every entity a job names +#: from OUTCOMES. +_DENSE_STAND_IN = """\ +import json +import sys + +OUTCOMES = json.loads({outcomes!r}) +if sys.argv[1:2] != ["-c"]: + sys.exit(f"unexpected arguments {{sys.argv[1:]}}") +jobs = json.load(sys.stdin) +with open({jobs!r}, "w", encoding="utf-8") as handle: + json.dump(jobs, handle) +results = [] +for job in jobs: + entities = {{entity: OUTCOMES[entity] for entity in job["entities"]}} + results.append({{"module": job["module"], "entities": entities}}) +json.dump(results, sys.stdout) +""" +_CATALOG = {"h": "nz:h", "p": "nz:p"} +#: Household's dense program as the walk sees it: root input ``h``. +_HOUSEHOLD = { + "root_entity": "Household", + "root_inputs": ["h"], + "input_catalog": _CATALOG, +} + + +def _cross_entity_artifact() -> dict: + """A Person rule that reads a Household rule, as format 3. + + The walk blocks Person's dense program with ``_BLOCKER`` and finds + Household's unblocked. + """ + artifact = _format_3() + artifact["program"]["derived"] = [ + { + "name": "person_benefit", + "entity": "Person", + "expr": { + "kind": "add", + "left": {"kind": "input", "name": "p"}, + "right": {"kind": "derived", "name": "hh_income"}, + }, + "versions": [], + }, + { + "name": "hh_income", + "entity": "Household", + "expr": {"kind": "input", "name": "h"}, + "versions": [], + }, + ] + artifact["metadata"]["input_catalog"] = [ + {"slot": slot, "canonical_request_name": name} + for slot, name in _CATALOG.items() + ] + return artifact + + +def _dense_python(tmp_path: Path, outcomes: dict) -> Path: + script = tmp_path / "stand_in_dense.py" + script.write_text( + _DENSE_STAND_IN.format( + outcomes=json.dumps(outcomes), jobs=str(tmp_path / "dense_jobs.json") + ), + encoding="utf-8", + ) + return _wrapper(tmp_path / "dense-python", script) + + +def _dense_run(tmp_path: Path, outcomes: dict) -> tuple[dict, dict | None]: + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + return refresh.build_snapshot( + country="nz", + rulespec_root=_rulespec_root(tmp_path), + rulespec_commit="0" * 40, + engine_bin=_engine(tmp_path, 3, _cross_entity_artifact()), + engine_commit="1" * 40, + dense_python=_dense_python(tmp_path, outcomes), + artifact_dir=artifacts, + ) + + +@pytest.mark.parametrize("advice", ["", *(f". {advice}" for advice in _ADVICE)]) +def test_dense_mode_records_the_walks_text_for_a_cross_entity_error( + tmp_path, advice +) -> None: + # Engines from 4d7ec14f on add advice to the error. The snapshot keeps + # the walk's text, so a change in the engine's advice can't move it. + snapshot, differential = _dense_run( + tmp_path, {"Household": _HOUSEHOLD, "Person": {"error": _BLOCKER + advice}} + ) + root = tmp_path / "rulespec" + jobs = json.loads((tmp_path / "dense_jobs.json").read_text(encoding="utf-8")) + assert jobs == [ + { + "module": str(root / "nz" / "m.yaml"), + "root": str(root), + "entities": ["Household", "Person"], + } + ] + assert snapshot["engine"]["surface"] == refresh.SURFACE_DENSE + assert snapshot["modules"] == [ + { + "path": "nz/m.yaml", + "sha256": hashlib.sha256(_MODULE_TEXT).hexdigest(), + "status": "dense_unsupported", + "inputs": {"Household": ["h"]}, + "canonical_inputs": _CATALOG, + "error": f"Person: {_BLOCKER}", + "dense_unsupported_entities": ["Person"], + "cli_walk_inputs": {"Person": ["p"]}, + } + ] + assert differential == { + "entity_programs": 2, + "dense_compiled": 1, + "input_mismatches": [], + "status_mismatches": [], + "catalog_mismatches": [], + } + + +@pytest.mark.parametrize( + ("outcomes", "match"), + [ + pytest.param( + { + "Household": _HOUSEHOLD, + "Person": { + "error": _BLOCKER.replace("`hh_income`", "`hh_wealth`") + + f". {_ADVICE[2]}" + }, + }, + r"nz/m\.yaml \[Person\]: dense error not explained by the artifact walk", + id="another-cross-entity-read", + ), + pytest.param( + { + "Household": _HOUSEHOLD, + "Person": {"error": "dense compilation does not yet support `y`"}, + }, + r"nz/m\.yaml \[Person\]: dense error not explained by the artifact walk", + id="a-new-limitation", + ), + pytest.param( + { + "Household": {"error": f"{_BLOCKER}. {_ADVICE[0]}"}, + "Person": {"error": _BLOCKER}, + }, + r"nz/m\.yaml \[Household\]: dense error not explained by the " + r"artifact walk", + id="an-entity-the-walk-doesnt-block", + ), + pytest.param( + { + "Household": {"error": refresh._NO_PROGRAM.format("Household")}, + "Person": {"error": _BLOCKER}, + }, + r"nz/m\.yaml: dense reports no Household program", + id="no-program", + ), + pytest.param( + { + "Household": {**_HOUSEHOLD, "root_entity": "Person"}, + "Person": {"error": _BLOCKER}, + }, + r"nz/m\.yaml: asked for Household, dense rooted at Person", + id="another-root", + ), + pytest.param( + { + "Household": {**_HOUSEHOLD, "root_inputs": ["h", "h"]}, + "Person": {"error": _BLOCKER}, + }, + r"nz/m\.yaml \[Household\]: duplicate root inputs", + id="duplicate-root-inputs", + ), + ], +) +def test_dense_mode_refuses_what_the_walk_cant_account_for( + tmp_path, outcomes, match +) -> None: + with pytest.raises(refresh.SurfaceError, match=match): + _dense_run(tmp_path, outcomes) + + +def test_dense_mode_counts_where_dense_and_the_walk_disagree(tmp_path) -> None: + # Dense compiles Person although the walk blocks it, reads an input the + # walk doesn't, and reports another catalog for Household. + snapshot, differential = _dense_run( + tmp_path, + { + "Household": { + **_HOUSEHOLD, + "root_inputs": ["h", "q"], + "input_catalog": {"h": "nz:h"}, + }, + "Person": { + "root_entity": "Person", + "root_inputs": ["p"], + "input_catalog": _CATALOG, + }, + }, + ) + assert differential == { + "entity_programs": 2, + "dense_compiled": 2, + "input_mismatches": [("nz/m.yaml", "Household", ["q"], [])], + "status_mismatches": [("nz/m.yaml", "Person", _BLOCKER)], + "catalog_mismatches": [("nz/m.yaml", "Household")], + } + (module,) = snapshot["modules"] + assert module["status"] == "compiled" + assert module["inputs"] == {"Household": ["h", "q"], "Person": ["p"]} + assert module["canonical_inputs"] == {"h": "nz:h", "p": "nz:p", "q": None} diff --git a/tools/refresh_axiom_input_surface.py b/tools/refresh_axiom_input_surface.py index a223f8560f..dae0071b69 100644 --- a/tools/refresh_axiom_input_surface.py +++ b/tools/refresh_axiom_input_surface.py @@ -60,21 +60,22 @@ ``where`` of ``sum_related`` and ``count_related``: inputs read there are related-entity (relation) inputs, not root inputs. -Dense-compile blockers are read from the artifact: a reference from E's -closure to a rule on another non-Scalar entity, or a rule in that closure +The walk reads dense-compile blockers from the artifact: a reference from +E's closure to a rule on another non-Scalar entity, or a rule in that closure with more than one version or an end-bounded version. The engine iterates derived rules in ``HashMap`` order, so when several blockers exist, which one its error names varies from process to process. The snapshot therefore records the lexicographically first blocker, in the engine's own message format, plus a count of the rest. In dense mode every dense error must equal one of the computed blockers. One exception: engines from 4d7ec14f (#292) on -follow a cross-entity message with ". " and advice that names the relation -to read through (``cross_entity_advice`` in the engine's ``src/dense.rs``). -The walk can't build that advice, so a cross-entity blocker also matches an -error that starts with its text and ". ", and the snapshot keeps the walk's -text. Anything else aborts the run: a new kind of dense limitation is a -review event, not a silent snapshot change. In CLI-only mode, only these two -blocker kinds are detected. +follow a cross-entity message with ". " and advice (``cross_entity_advice`` +in the engine's ``src/dense.rs``), which suggests compiling for the other +entity, reading the rule through a relation, or compiling only the outputs +that don't reach the reading rule. The walk can't build that advice, so a +cross-entity blocker also matches an error that starts with its text and +". ", and the snapshot keeps the walk's text. Anything else aborts the run: +a new kind of dense limitation is a review event, not a silent snapshot +change. In CLI-only mode, the walk detects only these two blocker kinds. The walk reads CLI artifact formats 2 and 3, and each artifact must carry the format the engine's ``capabilities`` names. Format 3 (engine 590d1dc9, From d403cce285e8a94d3e4e36181b64e97261649277 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 9 Nov 2026 23:43:15 -0500 Subject: [PATCH 07/14] NZ G8-x2: close the static walk's blind spots (#1567 review) The cd85cffbe review found four ways a run can reach a module that static_reach.py missed: a module alias held in a module-level constant, getattr with a literal name, __float__ and __add__. None occurs in the modules the three income kernels reach today, and their lists stand. The walk now - resolves a name to every definition that can bind it, through try/except alternatives, star imports and a package's lazy __getattr__ exports; - follows module-level constants that alias a module, class or function; - runs a module-level statement that changes a reached name in place (X.update(...), X[k] = v) along with that name; - reaches every dunder and sunder method of a referenced class; - reads the literal names in getattr, hasattr, setattr, delattr, attrgetter, methodcaller, import_module and __import__; - sends an attribute of an object it can't type to every same-named member of each referenced class and of each module used as a value; - starts from the root's class as well as its run(); - binds names by scope as Python does: comprehensions, lambdas, class bodies, global, nonlocal and local imports. Its docstring lists each of these and what it still can't see, which now includes a module that only an annotation names. static_reach_cases.py writes 32 synthetic roots, each reaching its targets one of those ways, and a parametrized test checks that each walk reaches exactly its targets. The cd85cffbe walk misses a target on 25 of the 32 and reaches an extra one on a 26th. Turning off any one of the 15 mechanisms fails at least one case. Under the new walk each kernel still reaches exactly its listed modules outside microcosm.graph, and only the hierarchy guard's cut applies. income_evidence_kernels.py: the docstring describes the new walk, and the solution check's comment now names what np.array_equal compares, float64 values (finding 2), instead of "bit for bit". Co-Authored-By: Claude Opus 5.5 --- .../transport/income_evidence_kernels.py | 14 +- .../test_transport_income_evidence_kernels.py | 36 +- test_support/microcosm_build/static_reach.py | 889 +++++++++++++----- .../microcosm_build/static_reach_cases.py | 641 +++++++++++++ 4 files changed, 1333 insertions(+), 247 deletions(-) create mode 100644 test_support/microcosm_build/static_reach_cases.py diff --git a/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py index 30bc0ed672..b6f910f821 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/income_evidence_kernels.py @@ -34,11 +34,13 @@ module outside ``microcosm.graph`` whose constants it reads. ``microcosm.graph.kernel`` and ``microcosm.graph.decl`` stay out, as in the evidence kernels' lists. Two tests check each list. A trace of one run checks -the executed modules, and can't see a constant read. A static walk of -``run()``'s source (``test_support/microcosm_build/static_reach.py``) follows -every branch and constant outside ``microcosm.graph``, and leaves out one -guarded branch that no decoded target opens. A comment marks each entry whose -module a run only reads constants from. +the executed modules, and can't see a constant read. A static walk of the +kernel's class and ``run()`` from source +(``test_support/microcosm_build/static_reach.py``) over-approximates what a run +can reach, in the ways that module's docstring lists. Outside +``microcosm.graph`` it must name exactly the listed modules, less one guarded +branch that no decoded target opens. A comment marks each entry whose module a +run only reads constants from. ``evidence_contract`` hashes only the kernels it lists, and this module stays outside the gate battery's binding closure, so adding it re-keys no battery. @@ -807,7 +809,7 @@ def run(self, context: KernelContext) -> KernelResult: # Tie the problem to this run's fit: the solution must be the one # solved from it, every weight unit on its axis must hold a record, and # each record's frame weight (its weight unit's, which the frame - # broadcasts) must be the solution's, bit for bit. + # broadcasts) must equal the solution's as a float64 value. solution = decode_solution( payloads["solution"], problem_sha256=problem.sha256, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py index 16795072c2..d6ca787ea3 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py @@ -87,6 +87,7 @@ from microcosm.graph.canonical import canonical_json from microcosm.graph.executor import Population, _project_context from microcosm.graph.kernel import ArtifactValue, NumericScope +from test_support.microcosm_build import static_reach_cases from test_support.microcosm_build.chronicle_feed import write_pinned_artifact from test_support.microcosm_build.first_party_imports import hashed_module_names from test_support.microcosm_build.static_reach import Cut, reach @@ -1926,11 +1927,11 @@ def _walk(ref, cuts=(HIERARCHY_GUARD,)): @pytest.mark.parametrize("ref", sorted(KERNELS)) def test_each_kernel_hashes_every_module_its_run_can_reach(ref) -> None: - # A static walk of run(), over every branch with the defaults, - # decorators, class attributes and module-level constants it reaches, - # names exactly the listed modules outside microcosm.graph. The trace - # test sees only the modules one run executes, so a new constant read - # from an unlisted module would pass it and leave the hash short. + # A static walk of run() and its class, which over-approximates what a + # run can reach (static_reach's docstring lists how), names exactly the + # listed modules outside microcosm.graph. The trace test sees only the + # modules one run executes, so a new constant read from an unlisted + # module would pass it and leave the hash short. walked = _walk(ref) listed = _outside_graph(INCOME_EVIDENCE_KERNEL_MODULES[ref]) reached = _outside_graph(walked.modules) @@ -1973,6 +1974,31 @@ def test_the_walk_sees_a_constant_the_trace_cant(kernel_traces) -> None: assert _outside_graph(_walk(ref).modules) - set(shortened[ref]) == {dropped} +@pytest.fixture(scope="module") +def walk_case_paths(tmp_path_factory): + return static_reach_cases.write(tmp_path_factory.mktemp("walk-cases")) + + +@pytest.mark.parametrize( + ("module", "qualname"), + sorted(static_reach_cases.ROOTS), + ids=[module for module, _ in sorted(static_reach_cases.ROOTS)], +) +def test_the_walk_reaches_each_way_code_names_a_module( + walk_case_paths, module, qualname +) -> None: + # Each synthetic root reaches its targets one way the walk's docstring + # lists (an alias, a dunder, a literal getattr, a lazy export, a scope + # rule), and its walk reaches exactly those targets. + walked = reach( + f"{static_reach_cases.PACKAGE}.{module}", qualname, paths=walk_case_paths + ) + assert ( + static_reach_cases.targets(walked) + == static_reach_cases.ROOTS[(module, qualname)] + ) + + def test_the_module_stays_outside_the_batterys_binding_closure() -> None: # The registry and the composer import this module, and no battery # binding's identity covers it or them, so adding the kernels re-keys no diff --git a/test_support/microcosm_build/static_reach.py b/test_support/microcosm_build/static_reach.py index 0ab2d4212a..5cb5a1426c 100644 --- a/test_support/microcosm_build/static_reach.py +++ b/test_support/microcosm_build/static_reach.py @@ -3,24 +3,60 @@ A kernel's implementation hash covers the modules its list names. A trace of one run sees the modules whose functions that run calls. It can't see a module the run only reads constants from, or a branch the run didn't take. -This walk reads source instead, and over-approximates. From a root function -it follows every branch of each body it reaches, with default argument -values, decorators, class bases, class attributes and module-level constant -values. It resolves each name through imports, at module or function level, -to the module that defines it. An attribute of an object whose type the walk -can't tell reaches every same-named member of each class the walk has -referenced. +This walk reads source instead, and over-approximates. + +From a root, a function or a method with its class, the walk follows every +branch of each body it reaches, with default argument values, decorators, +class bases, class attributes and module-level constant values. It resolves +each name to every definition that can bind it, through imports at module or +function level, star imports and a package's lazy exports. Each scope binds +only its own names, as Python scopes them: a comprehension, lambda or nested +function keeps its names to itself, a class body's names stay out of its +methods, and ``global`` and ``nonlocal`` send a name outward. + +Where code reaches a definition without naming it, the walk counts it too: + +- A module-level constant stands for each module, class or function its + value names (``_helper = helper_module``). +- A module-level statement that changes a reached name in place + (``_TABLE.update(...)``, ``_TABLE[key] = value``) runs with it, in any + module the walk has read. +- An attribute of an object whose type the walk can't tell reaches every + same-named member of each class it has referenced. It also reaches every + same-named definition in each module the walk has seen used as a value, + such as an argument or a dict entry. +- ``getattr``, ``hasattr``, ``setattr`` and ``delattr`` with a literal name + read that attribute, and so do the literal names that ``attrgetter`` and + ``methodcaller`` take. +- ``import_module`` and ``__import__`` with a literal name reach that module + as a value. +- A referenced class reaches every dunder and sunder method it defines, + since Python calls them without naming them: operators, conversions, + containers, descriptors, copying, pickling and Enum's ``_missing_``. +- A name that a package doesn't bind but exports lazily, through a + module-level ``__getattr__``, reaches that function and the name in each of + the package's modules. It imports nothing from the repository and runs none of its code: it parses each first-party module's file under ``packages/*/src``. -What it can't see: a name built at run time (``getattr`` with a computed -name, ``importlib``), a package's lazy ``__getattr__`` exports, and code that -an argument carries in from outside the walk (the executor's context, for -one). Annotations stay out: a ``def`` evaluates them at import, or never -under ``from __future__ import annotations``. A :class:`Cut` leaves out one -guarded branch, so a test names each exception it relies on rather than -skipping a module. +What it can't see: + +- a name built at run time (``getattr`` or ``import_module`` with a computed + name, ``globals()``, ``exec``); +- code reached by registration rather than by name: a function that a + decorator records for a framework to call (``singledispatch``, pydantic's + validators), or a statement in a module the walk never reads that fills a + reached registry on import; +- code that an argument carries in from outside the walk (the executor's + context, for one); +- a module that only an annotation names. Before Python 3.14 a ``def`` + statement evaluates its annotations as it runs; from 3.14 they wait until + something reads them (``typing.get_type_hints``); and + ``from __future__ import annotations`` keeps them as strings. + +A :class:`Cut` leaves out one guarded branch, so a test names each exception +it relies on rather than skipping a module. """ from __future__ import annotations @@ -33,44 +69,37 @@ from test_support.paths import REPOSITORY_ROOT -#: Dunder methods Python calls without naming them. A referenced class -#: reaches each one it defines. -IMPLICIT_METHODS = frozenset( - { - "__init__", - "__post_init__", - "__new__", - "__init_subclass__", - "__set_name__", - "__class_getitem__", - "__call__", - "__len__", - "__iter__", - "__next__", - "__getitem__", - "__contains__", - "__bool__", - "__eq__", - "__ne__", - "__hash__", - "__lt__", - "__le__", - "__gt__", - "__ge__", - "__str__", - "__repr__", - "__format__", - "__getattr__", - "__getattribute__", - "__setattr__", - "__delattr__", - "__enter__", - "__exit__", - } -) +#: Builtins that read or write an attribute. With a literal name, the walk +#: reads the call as that attribute. +ATTRIBUTE_BUILTINS = frozenset({"getattr", "hasattr", "setattr", "delattr"}) +#: Callables whose literal string arguments name attributes. +ATTRIBUTE_NAMERS = frozenset({"attrgetter", "methodcaller"}) +#: Callables whose literal first argument names a module to import. +IMPORTERS = frozenset({"import_module", "__import__"}) #: One reached thing: (defining module, kind, qualified name or None). Item = tuple[str, str, str | None] +#: What one name binds in a scope: None for a plain local, ``("import", +#: module, attribute or None)``, or ``("member", class)`` for a name in a +#: top-level class's body. +Binding = tuple[str, ...] | None +Scope = Mapping[str, Binding] + +_FUNCTIONS = (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda) +_DEFS = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) +_COMPREHENSIONS = (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp) +_SCOPES = (*_FUNCTIONS, ast.ClassDef, *_COMPREHENSIONS) + + +def is_implicit(member: str) -> bool: + """Whether Python can call ``member`` without naming it. + + Every dunder (``__add__``, ``__float__``) counts, and so does every + sunder, the single-underscore names Enum calls (``_missing_``). + """ + + core = member.strip("_") + return bool(core) and member in {f"__{core}__", f"_{core}_"} @dataclass(frozen=True) @@ -101,46 +130,128 @@ def module_paths(root: Path = REPOSITORY_ROOT) -> dict[str, Path]: def _statements(body: Iterable[ast.stmt]) -> Iterator[ast.stmt]: - """Module-level statements, through ``if``, ``try`` and ``with`` blocks.""" + """The statements that run in ``body``'s own scope, through every block.""" for node in body: yield node - if isinstance(node, ast.If | ast.With | ast.Try | ast.TryStar): - yield from _statements(node.body) - yield from _statements(getattr(node, "orelse", ())) - yield from _statements(getattr(node, "finalbody", ())) - for handler in getattr(node, "handlers", ()): - yield from _statements(handler.body) + if isinstance(node, _DEFS): + continue + for field in ("body", "orelse", "finalbody"): + yield from _statements(getattr(node, field, ())) + for handler in getattr(node, "handlers", ()): + yield from _statements(handler.body) + for case in getattr(node, "cases", ()): + yield from _statements(case.body) + + +def _stored(target: ast.AST) -> list[str]: + """The names an assignment target binds.""" + + return [ + node.id + for node in ast.walk(target) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) + ] + + +def _root_name(node: ast.AST) -> str | None: + """The name an attribute, subscript or call chain starts from.""" + + while isinstance(node, ast.Attribute | ast.Subscript | ast.Call | ast.Starred): + node = node.func if isinstance(node, ast.Call) else node.value + return node.id if isinstance(node, ast.Name) else None + + +def _add(values: list[ast.AST], node: ast.AST) -> None: + if not any(value is node for value in values): + values.append(node) + + +def _assignments(node: ast.stmt) -> Iterator[tuple[ast.AST, ast.AST]]: + """The (target, value) pairs one statement binds in its own scope.""" + + if isinstance(node, ast.Assign): + for target in node.targets: + yield target, node.value + elif isinstance(node, ast.AnnAssign | ast.AugAssign) and node.value is not None: + yield node.target, node.value + elif isinstance(node, ast.For | ast.AsyncFor): + yield node.target, node.iter + elif isinstance(node, ast.With | ast.AsyncWith): + for item in node.items: + if item.optional_vars is not None: + yield item.optional_vars, item.context_expr + if not isinstance(node, _DEFS): + for child in ast.walk(node): + if isinstance(child, ast.NamedExpr): + yield child.target, child.value + + +def _mutations(node: ast.stmt) -> Iterator[str]: + """The names one statement changes in place. + + ``_TABLE`` for ``_TABLE.update(...)``, ``_TABLE[key] = value`` or + ``_TABLE.field += 1``. + """ + + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call): + func = node.value.func + if isinstance(func, ast.Attribute): + name = _root_name(func.value) + if name is not None: + yield name + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, ast.AnnAssign | ast.AugAssign): + targets = [node.target] + else: + targets = [] + for target in targets: + for part in ast.walk(target): + if isinstance(part, ast.Attribute | ast.Subscript) and isinstance( + part.ctx, ast.Store + ): + name = _root_name(part.value) + if name is not None: + yield name class _Module: - """One parsed module's top-level names: imports, functions, classes, values.""" + """One parsed module's top-level bindings, by name.""" def __init__(self, name: str, path: Path) -> None: self.name = name self.is_package = path.name == "__init__.py" tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - self.imports: dict[str, tuple[str, str | None]] = {} - self.functions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} - self.classes: dict[str, ast.ClassDef] = {} - self.constants: dict[str, list[ast.AST]] = defaultdict(list) + imports: dict[str, list[tuple[str, str | None]]] = defaultdict(list) + functions: dict[str, list[ast.AST]] = defaultdict(list) + classes: dict[str, list[ast.ClassDef]] = defaultdict(list) + constants: dict[str, list[ast.AST]] = defaultdict(list) + self.star_imports: list[str] = [] + #: (name, statement): each module-level statement that changes a + #: name in place, once for each name it changes. + self.mutations: list[tuple[str, ast.stmt]] = [] for node in _statements(tree.body): if isinstance(node, ast.Import | ast.ImportFrom): - self.imports.update(_bindings(self, node)) + for bound, binding in _bindings(self, node).items(): + if bound == "*": + self.star_imports.append(binding[0]) + else: + imports[bound].append(binding) elif isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - self.functions[node.name] = node + functions[node.name].append(node) elif isinstance(node, ast.ClassDef): - self.classes[node.name] = node - elif isinstance(node, ast.Assign | ast.AnnAssign | ast.AugAssign): - targets = ( - node.targets if isinstance(node, ast.Assign) else [node.target] - ) - if node.value is None: - continue - for target in targets: - for name in ast.walk(target): - if isinstance(name, ast.Name): - self.constants[name.id].append(node.value) + classes[node.name].append(node) + else: + for target, value in _assignments(node): + for bound in _stored(target): + _add(constants[bound], value) + for bound in dict.fromkeys(_mutations(node)): + self.mutations.append((bound, node)) + self.imports = dict(imports) + self.functions = dict(functions) + self.classes = dict(classes) + self.constants = dict(constants) def absolute(self, level: int, module: str | None) -> str: """The module a ``from`` import names, relative imports resolved.""" @@ -172,36 +283,218 @@ def _bindings( return {(alias.asname or alias.name): (stem, alias.name) for alias in node.names} +def _members(module: _Module, node: ast.ClassDef) -> dict[str, list[ast.AST]]: + """One class body's names, each with the statements that bind it.""" + + out: dict[str, list[ast.AST]] = defaultdict(list) + for child in _statements(node.body): + if isinstance(child, _DEFS): + out[child.name].append(child) + elif isinstance(child, ast.Import | ast.ImportFrom): + for bound in _bindings(module, child): + out[bound].append(child) + elif isinstance(child, ast.AnnAssign) and child.value is None: + # A field with no default, such as a dataclass's ``rows: int``. + for bound in _stored(child.target): + _add(out[bound], child) + else: + for target, _ in _assignments(child): + for bound in _stored(target): + _add(out[bound], child) + return dict(out) + + def _annotations(node: ast.AST) -> Iterator[ast.AST]: """The annotation expressions inside ``node``.""" for child in ast.walk(node): if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef): - yield from (item for item in [child.returns] if item is not None) - arguments = child.args - for argument in ( - *arguments.posonlyargs, - *arguments.args, - *arguments.kwonlyargs, - arguments.vararg, - arguments.kwarg, - ): - if argument is not None and argument.annotation is not None: + if child.returns is not None: + yield child.returns + for argument in _arguments(child.args): + if argument.annotation is not None: yield argument.annotation elif isinstance(child, ast.AnnAssign): yield child.annotation -def _walk(node: ast.AST, skipped: frozenset[int]) -> Iterator[ast.AST]: - """Every node under ``node``, pruning each subtree whose id is skipped.""" +def _arguments(arguments: ast.arguments) -> Iterator[ast.arg]: + for argument in ( + *arguments.posonlyargs, + *arguments.args, + *arguments.kwonlyargs, + arguments.vararg, + arguments.kwarg, + ): + if argument is not None: + yield argument + + +def _outer_parts(node: ast.AST) -> list[ast.AST]: + """The parts of a nested scope that its enclosing scope evaluates.""" + + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda): + decorators = [] if isinstance(node, ast.Lambda) else node.decorator_list + defaults = [item for item in node.args.kw_defaults if item is not None] + return [*decorators, *node.args.defaults, *defaults] + if isinstance(node, ast.ClassDef): + return [*node.decorator_list, *node.bases, *node.keywords] + if isinstance(node, _COMPREHENSIONS): + return [node.generators[0].iter] + return [] + + +def _inner_parts(node: ast.AST) -> list[ast.AST]: + """The parts of a scope that the scope itself evaluates.""" + + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + return list(node.body) + if isinstance(node, ast.Lambda): + return [node.body] + if isinstance(node, _COMPREHENSIONS): + first, *rest = node.generators + parts: list[ast.AST] = [first.target, *first.ifs] + for generator in rest: + parts.extend([generator.iter, generator.target, *generator.ifs]) + if isinstance(node, ast.DictComp): + parts.extend([node.key, node.value]) + else: + parts.append(node.elt) + return parts + return [] + + +def _same_scope(nodes: Iterable[ast.AST]) -> Iterator[ast.AST]: + """Every node under ``nodes`` that runs in their scope, not a nested one's.""" - stack = [node] + stack = list(nodes) while stack: - current = stack.pop() + node = stack.pop() + yield node + if isinstance(node, _SCOPES): + stack.extend(_outer_parts(node)) + else: + stack.extend(ast.iter_child_nodes(node)) + + +def _walrus_targets(node: ast.AST) -> Iterator[str]: + """A comprehension's walrus targets, which bind in its enclosing scope.""" + + for inner in _same_scope(_inner_parts(node)): + if isinstance(inner, ast.NamedExpr) and isinstance(inner.target, ast.Name): + yield inner.target.id + elif isinstance(inner, _COMPREHENSIONS): + yield from _walrus_targets(inner) + + +def _own_names(module: _Module, scope: ast.AST) -> tuple[dict[str, Binding], set[str]]: + """The names ``scope`` binds for itself, and the names it declares global.""" + + names: dict[str, Binding] = {} + if isinstance(scope, _COMPREHENSIONS): + for generator in scope.generators: + names.update(dict.fromkeys(_stored(generator.target))) + return names, set() + if isinstance(scope, _FUNCTIONS): + names.update(dict.fromkeys(argument.arg for argument in _arguments(scope.args))) + declared_global: set[str] = set() + declared_nonlocal: set[str] = set() + for node in _same_scope(_inner_parts(scope)): + if isinstance(node, ast.Global): + declared_global.update(node.names) + elif isinstance(node, ast.Nonlocal): + declared_nonlocal.update(node.names) + elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store | ast.Del): + names.setdefault(node.id, None) + elif isinstance(node, _DEFS): + names.setdefault(node.name, None) + elif isinstance(node, ast.ExceptHandler | ast.MatchAs | ast.MatchStar): + if node.name: + names.setdefault(node.name, None) + elif isinstance(node, ast.MatchMapping) and node.rest: + names.setdefault(node.rest, None) + elif isinstance(node, ast.Import | ast.ImportFrom): + for bound, binding in _bindings(module, node).items(): + names[bound] = ("import", *binding) + elif isinstance(node, _COMPREHENSIONS): + for target in _walrus_targets(node): + names.setdefault(target, None) + for name in declared_global | declared_nonlocal: + names.pop(name, None) + return names, declared_global + + +def _visits( + module: _Module, + node: ast.AST, + skipped: frozenset[int], + here: Scope, + closure: Scope, +) -> Iterator[tuple[ast.AST, Scope]]: + """Every node under ``node`` with the names local where it runs. + + ``here`` holds the names local at ``node``. ``closure`` holds those a + nested function there would see, which leave out a class body's. A node + comes before the nodes under it. + """ + + stack: list[tuple[ast.AST, Scope, Scope]] = [(node, here, closure)] + while stack: + current, here, closure = stack.pop() if id(current) in skipped: continue - yield current - stack.extend(ast.iter_child_nodes(current)) + yield current, here + if isinstance(current, _SCOPES): + own, declared_global = _own_names(module, current) + inner = { + name: binding + for name, binding in {**closure, **own}.items() + if name not in declared_global + } + # A class body sees its own names; its methods and lambdas don't. + after = closure if isinstance(current, ast.ClassDef) else inner + stack.extend( + (part, inner, after) for part in reversed(_inner_parts(current)) + ) + stack.extend( + (part, here, closure) for part in reversed(_outer_parts(current)) + ) + else: + children = list(ast.iter_child_nodes(current)) + stack.extend((child, here, closure) for child in reversed(children)) + + +def _named(value: ast.AST) -> Iterator[ast.AST]: + """The names and dotted names an expression's value can be.""" + + if isinstance(value, ast.Name | ast.Attribute): + yield value + elif isinstance(value, ast.IfExp): + yield from _named(value.body) + yield from _named(value.orelse) + elif isinstance(value, ast.BoolOp): + for part in value.values: + yield from _named(part) + elif isinstance(value, ast.NamedExpr | ast.Starred): + yield from _named(value.value) + elif isinstance(value, ast.Tuple | ast.List | ast.Set): + # ``a, b = first, second``: over-approximate by element. + for part in value.elts: + yield from _named(part) + + +def _callee(func: ast.AST) -> str | None: + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def _literal(node: ast.AST) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return None class _Walk: @@ -219,8 +512,18 @@ def __init__(self, paths: Mapping[str, Path], cuts: Iterable[Cut]) -> None: self.reached: dict[Item, set[str]] = defaultdict(set) self.pending: list[Item] = [] self.classes: set[tuple[str, str]] = set() + #: Modules the walk has seen used as a value, not as an attribute's root. + self.values: set[str] = set() self.attributes: set[str] = set() self.cuts_used: set[Cut] = set() + self.mutations_run: set[int] = set() + self._members: dict[tuple[str, str], dict[str, list[ast.AST]]] = {} + self._inside: dict[str, list[str]] = {} + # The lookups in progress. A cycle of re-exports or aliases adds + # nothing the outer lookup doesn't already see. + self._resolving: set[tuple[str, str]] = set() + self._lazy: set[tuple[str, str]] = set() + self._aliasing: set[Item] = set() def module(self, name: str) -> _Module | None: if name not in self.paths: @@ -234,6 +537,16 @@ def is_module(self, name: str) -> bool: return name in self.prefixes + def inside(self, package: str) -> list[str]: + """The first-party modules inside ``package``, at any depth.""" + + if package not in self._inside: + prefix = f"{package}." + self._inside[package] = sorted( + name for name in self.paths if name.startswith(prefix) + ) + return self._inside[package] + def push(self, item: Item, site: str) -> None: # A namespace package has no file, and so nothing to hash. if item[0] not in self.paths: @@ -242,52 +555,114 @@ def push(self, item: Item, site: str) -> None: self.pending.append(item) self.reached[item].add(site) - def resolve_expression(self, module: str, node: ast.AST) -> Item | None: - """The item a name or a ``module.attribute`` chain names, if first-party.""" + def members(self, module: str, name: str) -> dict[str, list[ast.AST]]: + """A top-level class's body names, across every ``class`` that defines it.""" + + key = (module, name) + if key not in self._members: + found = self.module(module) + out: dict[str, list[ast.AST]] = defaultdict(list) + for node in found.classes.get(name, ()): + for member, nodes in _members(found, node).items(): + out[member].extend(nodes) + self._members[key] = dict(out) + return self._members[key] + + def resolve(self, module: str, name: str) -> list[Item]: + """Every definition ``name`` can bind in ``module``'s top-level scope.""" + + found = self.module(module) + key = (module, name) + if found is None or key in self._resolving: + return [] + self._resolving.add(key) + try: + items: list[Item] = [ + (module, kind, name) + for kind, table in ( + ("function", found.functions), + ("class", found.classes), + ("constant", found.constants), + ) + if name in table + ] + for binding in found.imports.get(name, ()): + items.extend(self.resolve_import(*binding)) + for star in found.star_imports: + items.extend(self.resolve(star, name)) + finally: + self._resolving.discard(key) + return items + + def resolve_import(self, target: str, attribute: str | None) -> list[Item]: + """What ``import target`` or ``from target import attribute`` binds.""" + + if attribute is None: + return [(target, "module", None)] if self.is_module(target) else [] + return self.exported(target, attribute) + + def exported(self, module: str, name: str) -> list[Item]: + """What ``module.name`` can be, read from outside ``module``.""" + + items: list[Item] = [] + if self.is_module(f"{module}.{name}"): + items.append((f"{module}.{name}", "module", None)) + items.extend(self.resolve(module, name)) + found = self.module(module) + if items or found is None or "__getattr__" not in found.functions: + return items + # A lazy export: the module's __getattr__ and, for a package, the name + # wherever the package's own modules bind it. + items.append((module, "function", "__getattr__")) + key = (module, name) + if found.is_package and key not in self._lazy: + self._lazy.add(key) + try: + for inner in self.inside(module): + items.extend(self.resolve(inner, name)) + finally: + self._lazy.discard(key) + return items + + def resolve_expression(self, module: str, node: ast.AST) -> list[Item]: + """What a name or dotted name in ``module``'s top-level scope names.""" chain = [] while isinstance(node, ast.Attribute): chain.append(node.attr) node = node.value if not isinstance(node, ast.Name): - return None - item = self.resolve(module, node.id) + return [] + items = self.resolve(module, node.id) for attribute in reversed(chain): - if item is None or item[1] != "module": - return None - target = item[0] - if self.is_module(f"{target}.{attribute}"): - item = (f"{target}.{attribute}", "module", None) - else: - item = self.resolve(target, attribute) - return item - - def resolve(self, module: str, name: str, depth: int = 0) -> Item | None: - """Follow ``name`` in ``module``'s scope to the item that defines it.""" - - found = self.module(module) - if found is None or depth > 32: - return None - if name in found.functions: - return (module, "function", name) - if name in found.classes: - return (module, "class", name) - if name in found.constants: - return (module, "constant", name) - if name in found.imports: - return self.resolve_import(*found.imports[name], depth=depth + 1) - return None - - def resolve_import( - self, target: str, attribute: str | None, *, depth: int = 0 - ) -> Item | None: - if attribute is None: - return (target, "module", None) if self.is_module(target) else None - if self.is_module(f"{target}.{attribute}"): - return (f"{target}.{attribute}", "module", None) - if target in self.paths: - return self.resolve(target, attribute, depth) - return None + items = [ + found + for item in items + for start in self.aliased(item) + if start[1] == "module" + for found in self.exported(start[0], attribute) + ] + return items + + def aliased(self, item: Item) -> list[Item]: + """``item``; for a constant, what each of its values names. + + ``_helper = helper_module`` makes ``_helper`` stand for that module. + A constant whose values name nothing stands for itself. + """ + + if item[1] != "constant" or item in self._aliasing: + return [item] + self._aliasing.add(item) + found: list[Item] = [] + try: + for value in self.module(item[0]).constants[item[2]]: + for named in _named(value): + for resolved in self.resolve_expression(item[0], named): + found.extend(self.aliased(resolved)) + finally: + self._aliasing.discard(item) + return found or [item] def skipped(self, module: str, qualname: str, node: ast.AST) -> frozenset[int]: """Annotations, and the subtrees each cut in this function leaves out.""" @@ -312,85 +687,96 @@ def scan( qualname: str = "", owner: str | None = None, ) -> None: - """Push every first-party item ``node``'s code can name.""" + """Push every first-party item ``node``'s code can name. + + With an ``owner``, ``node`` sits in that top-level class's body, so + the body's names resolve to the class's members. + """ found = self.module(module) skipped = self.skipped(module, qualname, node) - bound: set[str] = set() - local_imports: dict[str, tuple[str, str | None]] = {} - for child in _walk(node, skipped): - if isinstance(child, ast.Import | ast.ImportFrom): - local_imports.update(_bindings(found, child)) - elif isinstance(child, ast.Name) and isinstance( - child.ctx, ast.Store | ast.Del - ): - bound.add(child.id) - elif isinstance(child, ast.arg): - bound.add(child.arg) - elif ( - isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) - and child is not node - ): - bound.add(child.name) - elif isinstance(child, ast.ExceptHandler) and child.name: - bound.add(child.name) - - def lookup(name: str) -> Item | None: - if name in local_imports: - return self.resolve_import(*local_imports[name]) - if name in bound: - return None - return self.resolve(module, name) - - for child in _walk(node, skipped): - line = getattr(child, "lineno", "?") + start: Scope = ( + dict.fromkeys(self.members(module, owner), ("member", owner)) + if owner is not None + else {} + ) + # Names that root an attribute chain; any other module name is a value. + rooted: set[int] = set() + + def lookup(name: str, here: Scope) -> list[Item]: + if name not in here: + return self.resolve(module, name) + binding = here[name] + if binding is None: + return [] + if binding[0] == "member": + return [(module, "member", f"{binding[1]}.{name}")] + return self.resolve_import(binding[1], binding[2]) + + def attribute(node: ast.Attribute, here: Scope, at: str) -> None: + self.attributes.add(node.attr) + chain = [] + current: ast.AST = node + while isinstance(current, ast.Attribute): + chain.append(current.attr) + current = current.value + if not isinstance(current, ast.Name): + return + rooted.add(id(current)) + if owner is not None and current.id in {"self", "cls"}: + self.push((module, "member", f"{owner}.{chain[-1]}"), at) + return + starts = [ + start + for item in lookup(current.id, here) + for start in self.aliased(item) + if start[1] == "module" + ] + for name in reversed(chain): + following = [] + for start in starts: + for item in self.exported(start[0], name): + self.push(item, at) + following.extend( + found + for found in self.aliased(item) + if found[1] == "module" + ) + starts = following + + for child, here in _visits(found, node, skipped, start, {}): + at = f"{site}:{getattr(child, 'lineno', '?')}" if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Load): - item = lookup(child.id) - if item is not None: - self.push(item, f"{site}:{line}") + for item in lookup(child.id, here): + self.push(item, at) + if item[1] == "module" and id(child) not in rooted: + self.values.add(item[0]) elif isinstance(child, ast.Attribute): - self.attributes.add(child.attr) - chain = [] - current: ast.AST = child - while isinstance(current, ast.Attribute): - chain.append(current.attr) - current = current.value - if not isinstance(current, ast.Name): - continue - if owner is not None and current.id in {"self", "cls"}: - self.push( - (module, "member", f"{owner}.{chain[-1]}"), f"{site}:{line}" - ) - continue - item = lookup(current.id) - if item is None or item[1] != "module": - continue - target = item[0] - for attribute in reversed(chain): - if self.is_module(f"{target}.{attribute}"): - target = f"{target}.{attribute}" - self.push((target, "module", None), f"{site}:{line}") - continue - member = self.resolve(target, attribute) - if member is not None: - self.push(member, f"{site}:{line}") - break - - def members(self, module: str, name: str) -> dict[str, ast.AST]: - """A class body's methods and attribute values, by name.""" - - out: dict[str, ast.AST] = {} - for node in self.module(module).classes[name].body: - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - out[node.name] = node - elif isinstance(node, ast.Assign | ast.AnnAssign): - targets = ( - node.targets if isinstance(node, ast.Assign) else [node.target] - ) - for target in targets: - if isinstance(target, ast.Name): - out[target.id] = node - return out + attribute(child, here, at) + elif isinstance(child, ast.Import | ast.ImportFrom): + for bound, binding in _bindings(found, child).items(): + if bound != "*": + for item in self.resolve_import(*binding): + self.push(item, at) + elif isinstance(child, ast.Call): + callee = _callee(child.func) + texts = [_literal(argument) for argument in child.args] + if callee in ATTRIBUTE_BUILTINS and len(texts) >= 2 and texts[1]: + read = ast.Attribute(child.args[0], texts[1], ast.Load()) + attribute(ast.copy_location(read, child), here, at) + elif callee in ATTRIBUTE_NAMERS: + for text in filter(None, texts): + self.attributes.update(text.split(".")) + elif callee in IMPORTERS and texts and texts[0]: + # import_module returns the module it names; __import__ + # returns the top package, whose attributes reach the rest. + name = texts[0] + if callee == "__import__": + name = name.partition(".")[0] + if self.is_module(name): + self.push((texts[0], "module", None), at) + self.push((name, "module", None), at) + self.values.add(name) def process(self, item: Item) -> None: module, kind, name = item @@ -398,19 +784,22 @@ def process(self, item: Item) -> None: if found is None: return if kind == "function": - self.scan(module, found.functions[name], f"{module}.{name}", qualname=name) + for node in found.functions.get(name, ()): + self.scan(module, node, f"{module}.{name}", qualname=name) elif kind == "constant": - for value in found.constants[name]: + for value in found.constants.get(name, ()): self.scan(module, value, f"{module}.{name}=") elif kind == "class": self.classes.add((module, name)) - node = found.classes[name] - for part in (*node.bases, *node.keywords, *node.decorator_list): - self.scan(module, part, f"{module}.{name}(class)") - for member, body in self.members(module, name).items(): - if not isinstance(body, ast.FunctionDef | ast.AsyncFunctionDef) or ( - member in IMPLICIT_METHODS - ): + for node in found.classes.get(name, ()): + for part in (*node.decorator_list, *node.bases, *node.keywords): + self.scan(module, part, f"{module}.{name}(class)") + for member, nodes in self.members(module, name).items(): + methods = all( + isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) + for node in nodes + ) + if is_implicit(member) or not methods: self.push( (module, "member", f"{name}.{member}"), f"{module}.{name}" ) @@ -418,41 +807,69 @@ def process(self, item: Item) -> None: owner, member = name.split(".", 1) if owner not in found.classes: return - members = self.members(module, owner) - if member in members: - self.scan( - module, - members[member], - f"{module}.{name}", - qualname=name, - owner=owner, - ) + nodes = self.members(module, owner).get(member) + for node in nodes or (): + self.scan(module, node, f"{module}.{name}", qualname=name, owner=owner) + if nodes: return # Inherited: the same name on each first-party base class. - for base in found.classes[owner].bases: - resolved = self.resolve_expression(module, base) - if resolved is not None and resolved[1] == "class": - self.push( - (resolved[0], "member", f"{resolved[2]}.{member}"), - f"{module}.{name}(base)", - ) + for node in found.classes[owner]: + for base in node.bases: + for resolved in self.resolve_expression(module, base): + for start in self.aliased(resolved): + if start[1] == "class": + self.push( + (start[0], "member", f"{start[2]}.{member}"), + f"{module}.{name}(base)", + ) + + def is_reached(self, item: Item) -> bool: + if item[1] == "module": + return any(found[0] == item[0] for found in self.reached) + return item in self.reached + + def settle(self) -> bool: + """Push what reached names imply; whether anything new came.""" + + added = False + for module_name, class_name in sorted(self.classes): + for member in self.members(module_name, class_name): + item = (module_name, "member", f"{class_name}.{member}") + if member in self.attributes and item not in self.reached: + self.push(item, f"{module_name}.{class_name}(attribute)") + added = True + for module_name in sorted(self.values): + for name in sorted(self.attributes): + for item in self.exported(module_name, name): + if item not in self.reached: + self.push(item, f"{module_name}(value).{name}") + added = True + if item[1] == "module" and item[0] not in self.values: + self.values.add(item[0]) + added = True + for found in list(self.modules.values()): + for bound, node in found.mutations: + if id(node) in self.mutations_run: + continue + if any( + self.is_reached(item) for item in self.resolve(found.name, bound) + ): + self.mutations_run.add(id(node)) + self.scan(found.name, node, f"{found.name}:{node.lineno}(changes)") + added = True + return added def run(self, module: str, qualname: str) -> None: if "." in qualname: + # A method's class, with its bases, decorators and dunders. + self.push((module, "class", qualname.split(".", 1)[0]), "root") self.push((module, "member", qualname), "root") else: self.push((module, "function", qualname), "root") while True: while self.pending: self.process(self.pending.pop()) - added = False - for module_name, class_name in sorted(self.classes): - for member in self.members(module_name, class_name): - item = (module_name, "member", f"{class_name}.{member}") - if member in self.attributes and item not in self.reached: - self.push(item, f"{module_name}.{class_name}(attribute)") - added = True - if not added: + if not self.settle() and not self.pending: return diff --git a/test_support/microcosm_build/static_reach_cases.py b/test_support/microcosm_build/static_reach_cases.py new file mode 100644 index 0000000000..50c13c78e6 --- /dev/null +++ b/test_support/microcosm_build/static_reach_cases.py @@ -0,0 +1,641 @@ +"""Synthetic modules for the static walk's own tests. + +Each root in :data:`ROOTS` reaches its target modules (``t_*``) in one of the +ways that :mod:`test_support.microcosm_build.static_reach` lists, and its walk +must reach exactly those targets. A target holds only constants, so a root +reaches it only through code the walk follows. +""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +from test_support.microcosm_build.static_reach import Reach, module_paths + +#: The package that holds the synthetic modules. +PACKAGE = "microcosm.walkcases" + +TARGETS = ( + "add", + "alias", + "alias_base", + "alternative", + "attrgetter", + "base_attr", + "class_scope", + "comprehension", + "conditional", + "decorator", + "factory", + "float", + "global", + "go", + "import_module", + "lambda", + "lazy", + "local", + "nonlocal", + "override", + "prop", + "registrar", + "root_base", + "setattr", + "setitem", + "shadow", + "shadowed", + "sibling", + "star", + "subclass", + "sunder", + "unpack", + "update", + "value", +) + +#: Each module's source, by its dotted name inside :data:`PACKAGE`. +SOURCES: dict[str, str] = { + "__init__": "", + **{f"t_{name}": "X = 1\nROWS = {}\n" for name in TARGETS}, + # -- Scopes --------------------------------------------------------------- + # A nested function's local leaves the enclosing scope's import alone. + "r_shadow": """ + from microcosm.walkcases import t_shadow as target + + + def run(): + def inner(): + target = 1 + return target + + return target.X, inner() + """, + # A comprehension's target binds only inside the comprehension. + "r_comprehension": """ + from microcosm.walkcases import t_comprehension as target + + + def run(): + values = [target for target in range(3)] + return target.X, values + """, + # A local import binds only in its own function, not in a sibling. + "r_sibling": """ + from microcosm.walkcases import t_sibling as target + + + def run(): + def first(): + from microcosm.walkcases import t_local as target + + return target.X + + def second(): + return target.X + + return first(), second() + """, + # A class body's names stay out of its methods. + "r_class_scope": """ + from microcosm.walkcases import t_class_scope as target + + + class Holder: + target = None + + def run(self): + return target.X + """, + # A method's decorator reads the class body's names first, so the + # module's import of the same name stays unreached. + "r_class_decorator": """ + from microcosm.walkcases import t_shadowed as target + + + class Kernel: + target = staticmethod(lambda function: function) + + @target + def run(self): + return 1 + """, + # ``global`` sends a name to module level, past the function's own store. + "r_global": """ + from microcosm.walkcases import t_global + + _CACHE = t_global.X + + + def run(): + global _CACHE + _CACHE = _CACHE or 1 + return _CACHE + """, + # ``nonlocal`` sends a name to the enclosing function's import. + "nl_holder": """ + from microcosm.walkcases import t_nonlocal + + + def get(): + return t_nonlocal.X + """, + "r_nonlocal": """ + def run(): + from microcosm.walkcases import nl_holder as holder + + def inner(): + nonlocal holder + value = holder.get() + holder = None + return value + + return inner() + """, + # -- Resolution ----------------------------------------------------------- + # A star import binds the names its module defines. + "star_src": """ + from microcosm.walkcases import t_star + + + def starred(): + return t_star.X + """, + "r_star": """ + from microcosm.walkcases.star_src import * # noqa: F403 + + + def run(): + return starred() # noqa: F405 + """, + # A name with two bindings reaches both, whichever runs. + "fast": """ + def impl(): + return 0 + """, + "slow": """ + from microcosm.walkcases import t_alternative + + + def impl(): + return t_alternative.X + """, + "r_alternative": """ + try: + from microcosm.walkcases.slow import impl + except ImportError: + from microcosm.walkcases.fast import impl + + + def run(): + return impl() + """, + # A package's lazy export, through its module-level __getattr__. + "lazy.__init__": """ + def __getattr__(name): + from importlib import import_module + + return getattr(import_module(f"{__name__}.impl"), name) + """, + "lazy.impl": """ + from microcosm.walkcases import t_lazy + + + class Lazy: + def go(self): + return t_lazy.X + """, + "r_lazy": """ + from microcosm.walkcases.lazy import Lazy + + + def run(): + return Lazy().go() + """, + # -- Constants ------------------------------------------------------------ + # A constant that holds a module stands for it. + "helper": """ + from microcosm.walkcases import t_alias + + + def compute(): + return t_alias.X + """, + "r_alias": """ + import microcosm.walkcases.helper as helper_module + + _helper = helper_module + + + def run(): + return _helper.compute() + """, + # ... through tuple unpacking, and through either branch of a condition. + "mod_a": """ + def fn(): + return 0 + """, + "mod_b": """ + from microcosm.walkcases import t_unpack + + + def fn(): + return t_unpack.X + """, + "mod_c": """ + from microcosm.walkcases import t_conditional + + + def fn(): + return t_conditional.X + """, + "r_unpack": """ + from microcosm.walkcases import mod_a, mod_b + + _first, _second = mod_a, mod_b + + + def run(): + return _second.fn() + """, + "r_conditional": """ + from microcosm.walkcases import mod_a, mod_c + + FLAG = True + _impl = mod_c if FLAG else mod_a + + + def run(): + return _impl.fn() + """, + # A class base held in a constant: run() is the base's. + "made3": """ + from microcosm.walkcases import t_alias_base + + + class Base: + def run(self): + return t_alias_base.X + """, + "r_alias_base": """ + from microcosm.walkcases import made3 + + _Base = made3.Base + + + class Kernel(_Base): + pass + """, + # -- Changes in place ----------------------------------------------------- + "r_update": """ + from microcosm.walkcases import t_update + + _TABLE = {} + _TABLE.update(t_update.ROWS) + + + def run(): + return _TABLE + """, + "r_setitem": """ + from microcosm.walkcases import t_setitem + + _TABLE = {} + _TABLE["key"] = t_setitem.X + + + def run(): + return _TABLE + """, + # Another module the walk reads fills the registry run() reads. + "registry": """ + REGISTRY = [] + """, + "registrar": """ + from microcosm.walkcases import t_registrar + from microcosm.walkcases.registry import REGISTRY + + REGISTRY.append(t_registrar.X) + + + def helper(): + return 1 + """, + "r_registry": """ + from microcosm.walkcases import registrar + from microcosm.walkcases.registry import REGISTRY + + + def run(): + return REGISTRY, registrar.helper() + """, + # -- Attributes of objects the walk can't type ------------------------------ + # Thing's dunders count wherever Thing does (see Implicit calls). + "made": """ + from microcosm.walkcases import t_add, t_float, t_go, t_prop + + + class Thing: + def go(self): + return t_go.X + + def __float__(self): + return float(t_float.X) + + def __add__(self, other): + return t_add.X + + @property + def prop(self): + return t_prop.X + + + def make(): + return Thing() + """, + "r_prop": """ + from microcosm.walkcases.made import make + + + def run(): + return make().prop + """, + # A module passed as an argument. + "helper2": """ + from microcosm.walkcases import t_value + + + def compute2(): + return t_value.X + """, + "use": """ + def use(module): + return module.compute2() + """, + "r_value": """ + from microcosm.walkcases import helper2 + from microcosm.walkcases.use import use + + + def run(): + return use(helper2) + """, + # The base's run() calls the root class's override. + "made6": """ + class Base: + def run(self): + return self.step() + + def step(self): + return 0 + """, + "r_override": """ + from microcosm.walkcases import t_override + from microcosm.walkcases.made6 import Base + + + class Kernel(Base): + def step(self): + return t_override.X + """, + # -- Literal names ---------------------------------------------------------- + "r_getattr": """ + from microcosm.walkcases.made import make + + + def run(): + return getattr(make(), "go")() + """, + "made7": """ + from microcosm.walkcases import t_setattr + + + class Thing: + @property + def level(self): + return 0 + + @level.setter + def level(self, value): + assert t_setattr.X + """, + "r_setattr": """ + from microcosm.walkcases.made7 import Thing + + + def run(): + thing = Thing() + setattr(thing, "level", 1) + return thing + """, + "made4": """ + from microcosm.walkcases import t_attrgetter + + + class Thing: + @property + def named(self): + return t_attrgetter.X + """, + "r_attrgetter": """ + from operator import attrgetter + + from microcosm.walkcases.made4 import Thing + + + def run(): + return attrgetter("named")(Thing()) + """, + "imp_mod": """ + from microcosm.walkcases import t_import_module + + + def fn(): + return t_import_module.X + """, + "r_import_module": """ + import importlib + + + def run(): + return importlib.import_module("microcosm.walkcases.imp_mod").fn() + """, + # -- Implicit calls --------------------------------------------------------- + "r_float": """ + from microcosm.walkcases.made import make + + + def run(): + return float(make()) + """, + "r_add": """ + from microcosm.walkcases.made import make + + + def run(): + return make() + 1 + """, + "enum_mod": """ + import enum + + from microcosm.walkcases import t_sunder + + + class Kind(enum.Enum): + A = "a" + + @classmethod + def _missing_(cls, value): + return t_sunder.X + """, + "r_sunder": """ + from microcosm.walkcases.enum_mod import Kind + + + def run(): + return Kind("b") + """, + # A base's __init_subclass__ and class attribute, for a class run() makes. + "made2": """ + from microcosm.walkcases import t_base_attr, t_factory, t_subclass + + + def factory(): + return [t_factory.X] + + + class Base: + KIND = t_base_attr.X + + def __init_subclass__(cls, **kw): + return t_subclass.X + """, + "r_subclass": """ + from microcosm.walkcases.made2 import Base + + + def run(): + class Sub(Base): + pass + + return Sub + """, + # The root class's own base runs __init_subclass__ for it. + "made5": """ + from microcosm.walkcases import t_root_base + + + class Base: + def __init_subclass__(cls, **kw): + return t_root_base.X + """, + "r_root_base": """ + from microcosm.walkcases.made5 import Base + + + class Kernel(Base): + def run(self): + return 1 + """, + "r_factory": """ + from dataclasses import dataclass, field + + from microcosm.walkcases.made2 import factory + + + @dataclass + class Holder: + items: list = field(default_factory=factory) + + + def run(): + return Holder() + """, + "r_lambda": """ + from microcosm.walkcases import t_lambda + + + def run(): + f = lambda: t_lambda.X # noqa: E731 + return f() + """, + # A method's decorator, held in a class attribute. + "deco_mod": """ + from microcosm.walkcases import t_decorator + + + def wrap(function): + assert t_decorator.X + return function + """, + "r_decorator": """ + from microcosm.walkcases import deco_mod + + + class Kernel: + _deco = deco_mod.wrap + + @_deco + def run(self): + return 1 + """, +} + +# A root that makes a Thing reaches every dunder Thing defines, so it reaches +# t_add and t_float as well: the walk over-approximates there by design. +_THING = {"add", "float"} + +#: (root module, qualname) -> the targets its walk reaches, exactly. +ROOTS: dict[tuple[str, str], set[str]] = { + ("r_shadow", "run"): {"shadow"}, + ("r_comprehension", "run"): {"comprehension"}, + ("r_sibling", "run"): {"sibling", "local"}, + ("r_class_scope", "Holder.run"): {"class_scope"}, + ("r_class_decorator", "Kernel.run"): set(), + ("r_global", "run"): {"global"}, + ("r_nonlocal", "run"): {"nonlocal"}, + ("r_star", "run"): {"star"}, + ("r_alternative", "run"): {"alternative"}, + ("r_lazy", "run"): {"lazy"}, + ("r_alias", "run"): {"alias"}, + ("r_unpack", "run"): {"unpack"}, + ("r_conditional", "run"): {"conditional"}, + ("r_alias_base", "Kernel.run"): {"alias_base"}, + ("r_update", "run"): {"update"}, + ("r_setitem", "run"): {"setitem"}, + ("r_registry", "run"): {"registrar"}, + ("r_prop", "run"): {"prop", *_THING}, + ("r_value", "run"): {"value"}, + ("r_override", "Kernel.run"): {"override"}, + ("r_getattr", "run"): {"go", *_THING}, + ("r_setattr", "run"): {"setattr"}, + ("r_attrgetter", "run"): {"attrgetter"}, + ("r_import_module", "run"): {"import_module"}, + ("r_float", "run"): {"float", *_THING}, + ("r_add", "run"): {"add", *_THING}, + ("r_sunder", "run"): {"sunder"}, + ("r_subclass", "run"): {"subclass", "base_attr"}, + ("r_root_base", "Kernel.run"): {"root_base"}, + ("r_factory", "run"): {"factory"}, + ("r_lambda", "run"): {"lambda"}, + ("r_decorator", "Kernel.run"): {"decorator"}, +} + + +def write(root: Path) -> dict[str, Path]: + """Write :data:`SOURCES` under ``root``; return their modules' paths.""" + + base = root / "packages" / "cases" / "src" / "microcosm" / "walkcases" + for name, source in SOURCES.items(): + path = base.joinpath(*name.split(".")).with_suffix(".py") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(source).lstrip("\n"), encoding="utf-8") + return module_paths(root) + + +def targets(walked: Reach) -> set[str]: + """The targets a walk reached, by their names in :data:`TARGETS`.""" + + prefix = f"{PACKAGE}.t_" + return { + name.removeprefix(prefix) for name in walked.modules if name.startswith(prefix) + } From eb236a6f2b4110d71ea6b2ccff710c9538bb0885 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 01:30:31 -0500 Subject: [PATCH 08/14] NZ G8-x2: scope class bodies as Python does; close the walk's misses (#1567) The delta review of d403cce28 found that a class-body name lost its module-level binding. Python reads a name the class body binds from the class namespace only once a top-level statement of the body has bound it. Before that, or when only a branch, a loop or a bare annotation binds it, Python reads the module's binding. The walk now follows that rule, and a class inside a function reads its own binding or the module's, never the function's, on 3.13 and 3.14 alike. The commit also closes each miss the review listed rather than documenting it: statement headers at module and class level; filler calls and other import-time effects in each module that holds a reached item; changes in place to a reached name (method calls, stores, dels and call arguments, also through an alias); dotted module values; function-level alternative imports; subscripted bases; match class patterns; getattr, hasattr, setattr, delattr, attrgetter and methodcaller with a literal or constant name; __getattr__ assigned rather than defined; __import__ with a fromlist; a star import of a lazy package; class-level imports used through self; and a relative import_module. settle now counts only real pushes, and the walk raises after 10,000 rounds without settling, so a namespace sub-package can no longer hang it. The self/cls owner push goes: the class-attribute fallback reaches the same members, and no module set moves on any of NZ's 8 kernels. static_reach_cases.py grows from 32 to 80 roots and gains a runtime oracle. Each target module records the reads Python makes, and python_reads runs every root in one isolated interpreter. A new test asserts that Python reads exactly ROOTS less OVER plus BLIND for each root, so ROOTS can't record a walk's mistake as the answer. Another checks OVER, BLIND and the two decoys. A Hypothesis property composes 2 to 6 roots and checks that the walk loses no target and that Python reads the union of the parts' reads. Belgium's seven import-time forms became roots, and BLIND holds their registry decorator. Each of 71 one-edit knockouts of the walk fails at least one root. The income kernels' lists still equal the walk's reach outside microcosm.graph, with the one cut as before. Co-Authored-By: Claude Opus 5.5 --- .../test_transport_income_evidence_kernels.py | 86 +- test_support/microcosm_build/static_reach.py | 737 ++++++-- .../microcosm_build/static_reach_cases.py | 1604 ++++++++++++++--- 3 files changed, 2045 insertions(+), 382 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py index d6ca787ea3..eb4af4a8e3 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py @@ -1975,30 +1975,90 @@ def test_the_walk_sees_a_constant_the_trace_cant(kernel_traces) -> None: @pytest.fixture(scope="module") -def walk_case_paths(tmp_path_factory): - return static_reach_cases.write(tmp_path_factory.mktemp("walk-cases")) +def walk_case_tree(tmp_path_factory): + root = tmp_path_factory.mktemp("walk-cases") + return root, static_reach_cases.write(root) -@pytest.mark.parametrize( - ("module", "qualname"), - sorted(static_reach_cases.ROOTS), - ids=[module for module, _ in sorted(static_reach_cases.ROOTS)], -) +@pytest.fixture(scope="module") +def walk_case_reads(walk_case_tree): + root, _ = walk_case_tree + return static_reach_cases.python_reads(root) + + +_WALK_CASE_ROOTS = sorted(static_reach_cases.ROOTS) +_WALK_CASE_IDS = [f"{module}:{qualname}" for module, qualname in _WALK_CASE_ROOTS] + + +@pytest.mark.parametrize(("module", "qualname"), _WALK_CASE_ROOTS, ids=_WALK_CASE_IDS) def test_the_walk_reaches_each_way_code_names_a_module( - walk_case_paths, module, qualname + walk_case_tree, module, qualname ) -> None: # Each synthetic root reaches its targets one way the walk's docstring - # lists (an alias, a dunder, a literal getattr, a lazy export, a scope - # rule), and its walk reaches exactly those targets. - walked = reach( - f"{static_reach_cases.PACKAGE}.{module}", qualname, paths=walk_case_paths - ) + # lists, and its walk reaches exactly those targets. + _, paths = walk_case_tree + walked = reach(f"{static_reach_cases.PACKAGE}.{module}", qualname, paths=paths) assert ( static_reach_cases.targets(walked) == static_reach_cases.ROOTS[(module, qualname)] ) +@pytest.mark.parametrize(("module", "qualname"), _WALK_CASE_ROOTS, ids=_WALK_CASE_IDS) +def test_a_run_reads_what_the_walk_reaches(walk_case_reads, module, qualname) -> None: + # The same root, run for real: Python reads exactly the targets its walk + # reaches, less the documented over-reach (OVER) and plus the reads that + # the walk's "What it can't see" list explains (BLIND). So ROOTS can't + # record a walk's mistake as the expected answer. + root = (module, qualname) + assert walk_case_reads[root] == static_reach_cases.expected_reads(root) + + +def test_the_walk_cases_account_for_every_target() -> None: + # OVER only takes away targets a walk reaches, BLIND only adds targets it + # doesn't, and every target is some root's, BLIND's or a decoy. + roots = static_reach_cases.ROOTS + over = { + root: found - roots[root] + for root, found in static_reach_cases.OVER.items() + if found - roots[root] + } + blind = { + root: found & roots[root] + for root, found in static_reach_cases.BLIND.items() + if found & roots[root] + } + assert over == {} + assert blind == {} + used = set().union(*roots.values(), *static_reach_cases.BLIND.values()) + assert set(static_reach_cases.TARGETS) - used == static_reach_cases.DECOYS + assert used <= set(static_reach_cases.TARGETS) + + +@PROPERTY +@given( + picked=st.lists( + st.sampled_from(_WALK_CASE_ROOTS), min_size=2, max_size=6, unique=True + ) +) +def test_property_composing_roots_loses_no_target(walk_case_tree, picked) -> None: + # One run() that runs several roots in turn. Its walk reaches every target + # each part's walk reaches, and may reach more, since the attribute + # fallback spans the whole walk. Python reads exactly what the parts read. + # So no part's state in the walk hides another part's reach. + root, _ = walk_case_tree + composite, paths = static_reach_cases.compose(root, picked) + walked = reach( + f"{static_reach_cases.PACKAGE}.{composite[0]}", composite[1], paths=paths + ) + parts = set().union(*(static_reach_cases.ROOTS[part] for part in picked)) + assert parts <= static_reach_cases.targets(walked) + reads = static_reach_cases.python_reads(root, [composite])[composite] + assert reads == set().union( + *(static_reach_cases.expected_reads(part) for part in picked) + ) + + def test_the_module_stays_outside_the_batterys_binding_closure() -> None: # The registry and the composer import this module, and no battery # binding's identity covers it or them, so adding the kernels re-keys no diff --git a/test_support/microcosm_build/static_reach.py b/test_support/microcosm_build/static_reach.py index 5cb5a1426c..b173edcf04 100644 --- a/test_support/microcosm_build/static_reach.py +++ b/test_support/microcosm_build/static_reach.py @@ -5,49 +5,85 @@ module the run only reads constants from, or a branch the run didn't take. This walk reads source instead, and over-approximates. -From a root, a function or a method with its class, the walk follows every -branch of each body it reaches, with default argument values, decorators, -class bases, class attributes and module-level constant values. It resolves -each name to every definition that can bind it, through imports at module or -function level, star imports and a package's lazy exports. Each scope binds -only its own names, as Python scopes them: a comprehension, lambda or nested -function keeps its names to itself, a class body's names stay out of its -methods, and ``global`` and ``nonlocal`` send a name outward. - -Where code reaches a definition without naming it, the walk counts it too: - -- A module-level constant stands for each module, class or function its - value names (``_helper = helper_module``). -- A module-level statement that changes a reached name in place - (``_TABLE.update(...)``, ``_TABLE[key] = value``) runs with it, in any - module the walk has read. -- An attribute of an object whose type the walk can't tell reaches every - same-named member of each class it has referenced. It also reaches every - same-named definition in each module the walk has seen used as a value, - such as an argument or a dict entry. -- ``getattr``, ``hasattr``, ``setattr`` and ``delattr`` with a literal name - read that attribute, and so do the literal names that ``attrgetter`` and - ``methodcaller`` take. -- ``import_module`` and ``__import__`` with a literal name reach that module - as a value. -- A referenced class reaches every dunder and sunder method it defines, - since Python calls them without naming them: operators, conversions, - containers, descriptors, copying, pickling and Enum's ``_missing_``. -- A name that a package doesn't bind but exports lazily, through a - module-level ``__getattr__``, reaches that function and the name in each of - the package's modules. - -It imports nothing from the repository and runs none of its code: it parses -each first-party module's file under ``packages/*/src``. +It parses each first-party module's file under ``packages/*/src`` and imports +nothing under ``packages/``, so it runs none of the code it walks. + +What the walk follows from a root, a function or a method with its class: + +- Bodies. Every branch of each body it reaches, with default argument values, + decorators, class bases and keywords, class attributes and the values of + module-level constants. +- Bindings. A name resolves to every definition that can bind it: each + alternative of a ``try`` or an ``if``, at module or function level, star + imports, re-exports, and a package's lazy exports through a module-level + ``__getattr__``, defined or assigned. +- Scopes. Each scope binds only its own names, as Python scopes them. A + comprehension, lambda or nested function keeps its names to itself, a class + body's names stay out of its methods, and ``global`` and ``nonlocal`` send a + name outward. A class body reads a name it binds from its own namespace + only once a statement at the body's top level has bound it. Before that, or + when only a branch, a loop or a bare annotation binds it, Python reads the + module's binding, so the walk follows that one too. +- Aliases. A module-level constant stands for each module, class or function + its value names: ``_helper = helper_module``, ``_Base = pkg.Base``, + ``pkg.Base[int]``, either branch of a conditional, each element of a tuple, + and ``getattr`` or ``import_module`` with a literal name. +- Inheritance. A member that a class doesn't define resolves on each + first-party base, through aliases and subscripts (``Base[int]``). +- Self and cls. In a method, a chain from ``self`` or ``cls`` continues + through a module the class body imports (``self.mod.fn()``). The + attribute fallback reaches the member itself. +- Implicit calls. A referenced class reaches every dunder and sunder method + it defines, since Python calls them without naming them: operators, + conversions, containers, descriptors, copying, pickling and Enum's + ``_missing_``. +- Import-time code. Importing a module runs its module-level code, and + creating a class runs its body. In each module that holds an item the walk + reaches, and in each class it reaches, the walk follows every statement + that runs only for its effect, whatever it names: an expression statement + (``_setup()``, ``setattr(Kernel, ...)``), a ``del``, and a store to an item + or an attribute. It also follows the header of each compound statement + there (``if``, ``while``, ``for``, ``with``, ``match``, and an ``except`` + clause's types). +- Changes by name. In any module the walk has read, a statement at module + level or in any class body that changes a reached name in place runs with + it: a method call on the name (``_TABLE.update(...)``, also inside an + assignment), a store to or a ``del`` of its item or attribute, or a call + statement that takes it as an argument (``_fill(_TABLE)``), also through a + constant that aliases it (``_VIEW = _TABLE``). +- Attribute fallback. An attribute of an object whose type the walk can't + tell reaches every same-named member of each class the walk has + referenced, and every same-named definition in each module it has seen + used as a value: an argument, a dict entry, or a dotted name such as + ``pkg.helper``. +- Literal names. ``getattr``, ``hasattr``, ``setattr`` and ``delattr`` read + the attribute a literal names, and so do ``attrgetter``, ``methodcaller`` + and a ``match`` class pattern (its keyword names, and an explicit + ``__match_args__`` for positional patterns). ``import_module`` and + ``__import__`` reach the module a literal names as a value: + ``import_module`` relative to a literal package, ``__package__`` or + ``__name__``, and ``__import__`` with its top package as well. A + module-level constant that holds only strings counts as a literal. What it can't see: - a name built at run time (``getattr`` or ``import_module`` with a computed name, ``globals()``, ``exec``); - code reached by registration rather than by name: a function that a - decorator records for a framework to call (``singledispatch``, pydantic's - validators), or a statement in a module the walk never reads that fills a - reached registry on import; + decorator records for a framework or a registry to call + (``singledispatch``, pydantic's validators); +- the import-time code of a module that holds no item the walk reaches, such + as the ``__init__`` of a package above a reached module, or a module + imported only for its effects, unless a statement there changes a reached + name in place in a module the walk has read; +- the side effects of a call inside a statement that binds a name or tests a + value (``_X = _setup()``, ``assert _setup()``), unless the walk reaches that + statement another way; +- the import-time code of a class or function the walk doesn't reach: its + decorators, default values, class bases and class attributes, unless a + statement in the class body changes a reached name in place; +- a member inherited from a base that the ``class`` statement computes with a + call; - code that an argument carries in from outside the walk (the executor's context, for one); - a module that only an annotation names. Before Python 3.14 a ``def`` @@ -76,19 +112,35 @@ class bases, class attributes and module-level constant values. It resolves ATTRIBUTE_NAMERS = frozenset({"attrgetter", "methodcaller"}) #: Callables whose literal first argument names a module to import. IMPORTERS = frozenset({"import_module", "__import__"}) +#: The most rounds a walk may take to settle. Each round reaches a new item or +#: value module, or runs a new statement, so a walk that needs more has a bug. +MAX_ROUNDS = 10_000 #: One reached thing: (defining module, kind, qualified name or None). Item = tuple[str, str, str | None] -#: What one name binds in a scope: None for a plain local, ``("import", -#: module, attribute or None)``, or ``("member", class)`` for a name in a -#: top-level class's body. -Binding = tuple[str, ...] | None -Scope = Mapping[str, Binding] +#: One way a scope can bind a name: ``("import", module, attribute or None)``, +#: ``("member", class)`` for a name in a top-level class's body, or +#: ``("module",)`` for a class-body name that can also come from the module. +Binding = tuple[str, ...] +#: A scope's names -> every way each can be bound; ``()`` for a plain local. +Scope = Mapping[str, tuple[Binding, ...]] _FUNCTIONS = (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda) _DEFS = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) _COMPREHENSIONS = (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp) _SCOPES = (*_FUNCTIONS, ast.ClassDef, *_COMPREHENSIONS) +_COMPOUND = ( + ast.If, + ast.While, + ast.For, + ast.AsyncFor, + ast.With, + ast.AsyncWith, + ast.Match, + ast.Try, + ast.TryStar, +) +_FROM_MODULE: tuple[Binding, ...] = (("module",),) def is_implicit(member: str) -> bool: @@ -144,6 +196,33 @@ def _statements(body: Iterable[ast.stmt]) -> Iterator[ast.stmt]: yield from _statements(case.body) +def _header(node: ast.stmt) -> list[ast.AST]: + """What a compound statement evaluates itself, outside its bodies.""" + + if isinstance(node, ast.If | ast.While): + return [node.test] + if isinstance(node, ast.For | ast.AsyncFor): + return [node.iter] + if isinstance(node, ast.With | ast.AsyncWith): + return [item.context_expr for item in node.items] + if isinstance(node, ast.Match): + parts: list[ast.AST] = [node.subject] + for case in node.cases: + parts.append(case.pattern) + if case.guard is not None: + parts.append(case.guard) + return parts + if isinstance(node, ast.Try | ast.TryStar): + return [handler.type for handler in node.handlers if handler.type is not None] + return [] + + +def _parts(node: ast.stmt) -> list[ast.AST]: + """What one statement runs itself: a compound statement's header, or all of it.""" + + return _header(node) if isinstance(node, _COMPOUND) else [node] + + def _stored(target: ast.AST) -> list[str]: """The names an assignment target binds.""" @@ -167,6 +246,16 @@ def _add(values: list[ast.AST], node: ast.AST) -> None: values.append(node) +def _captures(pattern: ast.AST) -> Iterator[str]: + """The names a ``match`` pattern captures.""" + + for part in ast.walk(pattern): + if isinstance(part, ast.MatchAs | ast.MatchStar) and part.name: + yield part.name + elif isinstance(part, ast.MatchMapping) and part.rest: + yield part.rest + + def _assignments(node: ast.stmt) -> Iterator[tuple[ast.AST, ast.AST]]: """The (target, value) pairs one statement binds in its own scope.""" @@ -181,39 +270,78 @@ def _assignments(node: ast.stmt) -> Iterator[tuple[ast.AST, ast.AST]]: for item in node.items: if item.optional_vars is not None: yield item.optional_vars, item.context_expr + elif isinstance(node, ast.Match): + # A capture holds the subject or a part of it. + for case in node.cases: + for name in _captures(case.pattern): + yield ast.Name(name, ast.Store()), node.subject if not isinstance(node, _DEFS): - for child in ast.walk(node): - if isinstance(child, ast.NamedExpr): - yield child.target, child.value + for part in _parts(node): + for child in ast.walk(part): + if isinstance(child, ast.NamedExpr): + yield child.target, child.value -def _mutations(node: ast.stmt) -> Iterator[str]: - """The names one statement changes in place. +def _stores_into(node: ast.stmt) -> bool: + """Whether a statement stores to an item or an attribute.""" - ``_TABLE`` for ``_TABLE.update(...)``, ``_TABLE[key] = value`` or - ``_TABLE.field += 1``. - """ - - if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call): - func = node.value.func - if isinstance(func, ast.Attribute): - name = _root_name(func.value) - if name is not None: - yield name if isinstance(node, ast.Assign): targets = node.targets elif isinstance(node, ast.AnnAssign | ast.AugAssign): targets = [node.target] else: - targets = [] - for target in targets: - for part in ast.walk(target): - if isinstance(part, ast.Attribute | ast.Subscript) and isinstance( - part.ctx, ast.Store - ): - name = _root_name(part.value) - if name is not None: - yield name + return False + return any( + isinstance(part, ast.Attribute | ast.Subscript) + and isinstance(part.ctx, ast.Store) + for target in targets + for part in ast.walk(target) + ) + + +def _effects(body: Iterable[ast.stmt]) -> Iterator[ast.AST]: + """What a module or class body runs for its effects, through every block. + + Each statement that runs only for its effect (an expression statement, a + ``del``, a store to an item or an attribute), and each compound + statement's header. + """ + + for node in _statements(body): + if isinstance(node, _COMPOUND): + yield from _header(node) + elif isinstance(node, ast.Expr | ast.Delete) or _stores_into(node): + yield node + + +def _mutations(part: ast.AST) -> Iterator[str]: + """The names one statement (or one header) can change in place. + + ``_TABLE`` for ``_TABLE.update(...)``, ``_ROW = _TABLE.setdefault(...)``, + ``_TABLE[key] = value``, ``_TABLE.field += 1``, ``del _TABLE[key]``, and + for a call statement each name its arguments start from: + ``_fill(_TABLE)`` or ``setattr(_TABLE, ...)``. + """ + + calls = [] + if isinstance(part, ast.Expr) and isinstance(part.value, ast.Call): + calls.append(part.value) + for node in _same_scope([part]): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + name = _root_name(node.func.value) + if name is not None: + yield name + elif isinstance(node, ast.Attribute | ast.Subscript) and isinstance( + node.ctx, ast.Store | ast.Del + ): + name = _root_name(node.value) + if name is not None: + yield name + for call in calls: + for argument in (*call.args, *(keyword.value for keyword in call.keywords)): + name = _root_name(argument) + if name is not None: + yield name class _Module: @@ -228,9 +356,12 @@ def __init__(self, name: str, path: Path) -> None: classes: dict[str, list[ast.ClassDef]] = defaultdict(list) constants: dict[str, list[ast.AST]] = defaultdict(list) self.star_imports: list[str] = [] - #: (name, statement): each module-level statement that changes a - #: name in place, once for each name it changes. - self.mutations: list[tuple[str, ast.stmt]] = [] + #: (name, part, class body or None): each module-level or class-level + #: statement (or compound header) that changes a name in place, once + #: for each name it changes. + self.mutations: list[tuple[str, ast.AST, ast.ClassDef | None]] = [] + #: What the module runs at import beyond its bindings. + self.effects: list[ast.AST] = list(_effects(tree.body)) for node in _statements(tree.body): if isinstance(node, ast.Import | ast.ImportFrom): for bound, binding in _bindings(self, node).items(): @@ -242,17 +373,24 @@ def __init__(self, name: str, path: Path) -> None: functions[node.name].append(node) elif isinstance(node, ast.ClassDef): classes[node.name].append(node) + for child in _statements(node.body): + if not isinstance(child, (*_DEFS, ast.Import, ast.ImportFrom)): + self._changes(child, node) else: for target, value in _assignments(node): for bound in _stored(target): _add(constants[bound], value) - for bound in dict.fromkeys(_mutations(node)): - self.mutations.append((bound, node)) + self._changes(node, None) self.imports = dict(imports) self.functions = dict(functions) self.classes = dict(classes) self.constants = dict(constants) + def _changes(self, node: ast.stmt, body: ast.ClassDef | None) -> None: + for part in _parts(node): + for bound in dict.fromkeys(_mutations(part)): + self.mutations.append((bound, part, body)) + def absolute(self, level: int, module: str | None) -> str: """The module a ``from`` import names, relative imports resolved.""" @@ -265,6 +403,12 @@ def absolute(self, level: int, module: str | None) -> str: base = ".".join(parts) return f"{base}.{module}" if module else base + @property + def package(self) -> str: + """``__package__``: the module's own name for a package, else its parent's.""" + + return self.name if self.is_package else self.name.rpartition(".")[0] + def _bindings( module: _Module, node: ast.Import | ast.ImportFrom @@ -284,7 +428,10 @@ def _bindings( def _members(module: _Module, node: ast.ClassDef) -> dict[str, list[ast.AST]]: - """One class body's names, each with the statements that bind it.""" + """One class body's names, each with the statements that bind it. + + A name the body declares ``global`` binds at module level instead. + """ out: dict[str, list[ast.AST]] = defaultdict(list) for child in _statements(node.body): @@ -301,7 +448,61 @@ def _members(module: _Module, node: ast.ClassDef) -> dict[str, list[ast.AST]]: for target, _ in _assignments(child): for bound in _stored(target): _add(out[bound], child) - return dict(out) + declared = { + name + for child in _same_scope(_inner_parts(node)) + if isinstance(child, ast.Global) + for name in child.names + } + return {name: nodes for name, nodes in out.items() if name not in declared} + + +def _binds(node: ast.stmt) -> set[str]: + """The names a statement binds whenever it runs to its end.""" + + if isinstance(node, _DEFS): + return {node.name} + if isinstance(node, ast.Import | ast.ImportFrom): + return { + alias.asname or alias.name.split(".")[0] + for alias in node.names + if alias.name != "*" + } + if isinstance(node, ast.Assign): + return {name for target in node.targets for name in _stored(target)} + if isinstance(node, ast.AnnAssign | ast.AugAssign) and node.value is not None: + return set(_stored(node.target)) + if isinstance(node, ast.With | ast.AsyncWith): + return { + name + for item in node.items + if item.optional_vars is not None + for name in _stored(item.optional_vars) + } + return set() + + +def _bound_before(body: ast.ClassDef, name: str, node: ast.AST) -> bool: + """Whether the class ``body`` has surely bound ``name`` when ``node`` runs. + + Only a statement at the body's top level that ends before ``node`` starts + binds for sure: a branch, a loop or a ``try`` can skip its bindings, a bare + annotation binds nothing, and a ``del`` can undo a binding. + """ + + if not hasattr(node, "lineno"): + return False + for child in _same_scope(_inner_parts(body)): + if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Del): + if child.id == name: + return False + start = (node.lineno, node.col_offset) + for statement in body.body: + if (statement.end_lineno, statement.end_col_offset) > start: + return False + if name in _binds(statement): + return True + return False def _annotations(node: ast.AST) -> Iterator[ast.AST]: @@ -387,16 +588,28 @@ def _walrus_targets(node: ast.AST) -> Iterator[str]: yield from _walrus_targets(inner) -def _own_names(module: _Module, scope: ast.AST) -> tuple[dict[str, Binding], set[str]]: - """The names ``scope`` binds for itself, and the names it declares global.""" +def _own_names( + module: _Module, scope: ast.AST +) -> tuple[dict[str, tuple[Binding, ...]], set[str]]: + """The names ``scope`` binds for itself, and the names it declares global. + + Each name maps to every import that binds it there: a name that a ``try`` + or an ``if`` imports two ways keeps both. + """ + + names: dict[str, tuple[Binding, ...]] = {} + + def local(name: str) -> None: + names.setdefault(name, ()) - names: dict[str, Binding] = {} if isinstance(scope, _COMPREHENSIONS): for generator in scope.generators: - names.update(dict.fromkeys(_stored(generator.target))) + for name in _stored(generator.target): + local(name) return names, set() if isinstance(scope, _FUNCTIONS): - names.update(dict.fromkeys(argument.arg for argument in _arguments(scope.args))) + for argument in _arguments(scope.args): + local(argument.arg) declared_global: set[str] = set() declared_nonlocal: set[str] = set() for node in _same_scope(_inner_parts(scope)): @@ -405,20 +618,20 @@ def _own_names(module: _Module, scope: ast.AST) -> tuple[dict[str, Binding], set elif isinstance(node, ast.Nonlocal): declared_nonlocal.update(node.names) elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store | ast.Del): - names.setdefault(node.id, None) + local(node.id) elif isinstance(node, _DEFS): - names.setdefault(node.name, None) + local(node.name) elif isinstance(node, ast.ExceptHandler | ast.MatchAs | ast.MatchStar): if node.name: - names.setdefault(node.name, None) + local(node.name) elif isinstance(node, ast.MatchMapping) and node.rest: - names.setdefault(node.rest, None) + local(node.rest) elif isinstance(node, ast.Import | ast.ImportFrom): for bound, binding in _bindings(module, node).items(): - names[bound] = ("import", *binding) + names[bound] = (*names.get(bound, ()), ("import", *binding)) elif isinstance(node, _COMPREHENSIONS): for target in _walrus_targets(node): - names.setdefault(target, None) + local(target) for name in declared_global | declared_nonlocal: names.pop(name, None) return names, declared_global @@ -446,9 +659,16 @@ def _visits( yield current, here if isinstance(current, _SCOPES): own, declared_global = _own_names(module, current) + if isinstance(current, ast.ClassDef): + # Python reads a name the class body binds from the class + # namespace or, before the body binds it, from the module: + # never from an enclosing function. + own = { + name: (*bindings, *_FROM_MODULE) for name, bindings in own.items() + } inner = { - name: binding - for name, binding in {**closure, **own}.items() + name: bindings + for name, bindings in {**closure, **own}.items() if name not in declared_global } # A class body sees its own names; its methods and lambdas don't. @@ -465,10 +685,16 @@ def _visits( def _named(value: ast.AST) -> Iterator[ast.AST]: - """The names and dotted names an expression's value can be.""" + """The names and dotted names an expression's value can be. + + ``getattr(pkg, "mod")`` counts as ``pkg.mod``, and a literal that + ``import_module`` or ``__import__`` takes comes back as that constant. + """ if isinstance(value, ast.Name | ast.Attribute): yield value + elif isinstance(value, ast.Subscript): + yield from _named(value.value) elif isinstance(value, ast.IfExp): yield from _named(value.body) yield from _named(value.orelse) @@ -481,6 +707,16 @@ def _named(value: ast.AST) -> Iterator[ast.AST]: # ``a, b = first, second``: over-approximate by element. for part in value.elts: yield from _named(part) + elif isinstance(value, ast.Call): + callee = _callee(value.func) + texts = [_literal(argument) for argument in value.args] + if callee == "getattr" and len(texts) >= 2 and texts[1]: + read = ast.Attribute(value.args[0], texts[1], ast.Load()) + yield ast.copy_location(read, value) + elif callee in IMPORTERS and texts and texts[0]: + yield ast.Constant(texts[0]) + if callee == "__import__": + yield ast.Constant(texts[0].partition(".")[0]) def _callee(func: ast.AST) -> str | None: @@ -510,14 +746,18 @@ def __init__(self, paths: Mapping[str, Path], cuts: Iterable[Cut]) -> None: self.cuts = tuple(cuts) self.modules: dict[str, _Module] = {} self.reached: dict[Item, set[str]] = defaultdict(set) + self.reached_modules: set[str] = set() self.pending: list[Item] = [] self.classes: set[tuple[str, str]] = set() #: Modules the walk has seen used as a value, not as an attribute's root. self.values: set[str] = set() self.attributes: set[str] = set() self.cuts_used: set[Cut] = set() - self.mutations_run: set[int] = set() + #: The import-time statements and changes in place the walk has run. + self.ran: set[int] = set() self._members: dict[tuple[str, str], dict[str, list[ast.AST]]] = {} + #: A top-level class member's binding statement -> its class body. + self._body_of: dict[int, ast.ClassDef] = {} self._inside: dict[str, list[str]] = {} # The lookups in progress. A cycle of re-exports or aliases adds # nothing the outer lookup doesn't already see. @@ -547,13 +787,18 @@ def inside(self, package: str) -> list[str]: ) return self._inside[package] - def push(self, item: Item, site: str) -> None: + def push(self, item: Item, site: str) -> bool: + """Record that ``site`` names ``item``; whether ``item`` is new.""" + # A namespace package has no file, and so nothing to hash. if item[0] not in self.paths: - return - if item not in self.reached: + return False + new = item not in self.reached + if new: self.pending.append(item) + self.reached_modules.add(item[0]) self.reached[item].add(site) + return new def members(self, module: str, name: str) -> dict[str, list[ast.AST]]: """A top-level class's body names, across every ``class`` that defines it.""" @@ -565,6 +810,8 @@ def members(self, module: str, name: str) -> dict[str, list[ast.AST]]: for node in found.classes.get(name, ()): for member, nodes in _members(found, node).items(): out[member].extend(nodes) + for child in nodes: + self._body_of[id(child)] = node self._members[key] = dict(out) return self._members[key] @@ -589,7 +836,7 @@ def resolve(self, module: str, name: str) -> list[Item]: for binding in found.imports.get(name, ()): items.extend(self.resolve_import(*binding)) for star in found.star_imports: - items.extend(self.resolve(star, name)) + items.extend(self.exported(star, name)) finally: self._resolving.discard(key) return items @@ -609,11 +856,14 @@ def exported(self, module: str, name: str) -> list[Item]: items.append((f"{module}.{name}", "module", None)) items.extend(self.resolve(module, name)) found = self.module(module) - if items or found is None or "__getattr__" not in found.functions: + if items or found is None: + return items + # A lazy export: whatever binds the module's __getattr__ and, for a + # package, the name wherever the package's own modules bind it. + getters = self.resolve(module, "__getattr__") + if not getters: return items - # A lazy export: the module's __getattr__ and, for a package, the name - # wherever the package's own modules bind it. - items.append((module, "function", "__getattr__")) + items.extend(getters) key = (module, name) if found.is_package and key not in self._lazy: self._lazy.add(key) @@ -658,12 +908,32 @@ def aliased(self, item: Item) -> list[Item]: try: for value in self.module(item[0]).constants[item[2]]: for named in _named(value): + if isinstance(named, ast.Constant): + if self.is_module(named.value): + found.append((named.value, "module", None)) + continue for resolved in self.resolve_expression(item[0], named): found.extend(self.aliased(resolved)) finally: self._aliasing.discard(item) return found or [item] + def member_imports(self, module: str, owner: str, member: str) -> list[Item]: + """What a top-level class member is where the class body imports it. + + A member that an assignment binds needs nothing here: the walk reaches + a class's non-method members with the class, and a module that a value + names counts as a value. + """ + + found = self.module(module) + return [ + item + for node in self.members(module, owner).get(member, ()) + if isinstance(node, ast.Import | ast.ImportFrom) + for item in self.resolve_import(*_bindings(found, node)[member]) + ] + def skipped(self, module: str, qualname: str, node: ast.AST) -> frozenset[int]: """Annotations, and the subtrees each cut in this function leaves out.""" @@ -686,35 +956,83 @@ def scan( *, qualname: str = "", owner: str | None = None, + body: ast.ClassDef | None = None, ) -> None: """Push every first-party item ``node``'s code can name. - With an ``owner``, ``node`` sits in that top-level class's body, so - the body's names resolve to the class's members. + With an ``owner``, ``node`` sits in that top-level class's ``body``, so + the body's names resolve to the class's members, and also to the + module's binding wherever the body may not have bound them yet. """ found = self.module(module) skipped = self.skipped(module, qualname, node) start: Scope = ( - dict.fromkeys(self.members(module, owner), ("member", owner)) + dict.fromkeys(self.members(module, owner), (("member", owner),)) if owner is not None else {} ) # Names that root an attribute chain; any other module name is a value. rooted: set[int] = set() + # Attributes that a longer chain extends. + prefixes: set[int] = set() - def lookup(name: str, here: Scope) -> list[Item]: + def lookup(name: str, here: Scope, read: ast.AST) -> list[Item]: if name not in here: return self.resolve(module, name) - binding = here[name] - if binding is None: + items: list[Item] = [] + for binding in here[name]: + if binding[0] == "import": + items.extend(self.resolve_import(binding[1], binding[2])) + elif binding[0] == "module": + items.extend(self.resolve(module, name)) + else: + items.append((module, "member", f"{binding[1]}.{name}")) + if body is None or not _bound_before(body, name, read): + items.extend(self.resolve(module, name)) + return items + + def literal_names(node: ast.AST, here: Scope) -> list[str]: + """The strings ``node`` can be: a literal, or a constant of literals.""" + + text = _literal(node) + if text is not None: + return [text] + if not isinstance(node, ast.Name) or node.id in here: return [] - if binding[0] == "member": - return [(module, "member", f"{binding[1]}.{name}")] - return self.resolve_import(binding[1], binding[2]) + values = [ + value + for item in self.resolve(module, node.id) + if item[1] == "constant" + for value in self.module(item[0]).constants[item[2]] + ] + texts = [_literal(value) for value in values] + return texts if texts and None not in texts else [] + + def named_items(node: ast.AST, here: Scope) -> list[Item]: + """What a name or dotted name names where it runs, without pushing.""" + + chain = [] + while isinstance(node, ast.Attribute): + chain.append(node.attr) + node = node.value + if not isinstance(node, ast.Name): + return [] + items = lookup(node.id, here, node) + for attribute in reversed(chain): + items = [ + item + for start in items + for module_item in self.aliased(start) + if module_item[1] == "module" + for item in self.exported(module_item[0], attribute) + ] + return items def attribute(node: ast.Attribute, here: Scope, at: str) -> None: self.attributes.add(node.attr) + if isinstance(node.value, ast.Attribute): + prefixes.add(id(node.value)) chain = [] current: ast.AST = node while isinstance(current, ast.Attribute): @@ -724,11 +1042,14 @@ def attribute(node: ast.Attribute, here: Scope, at: str) -> None: return rooted.add(id(current)) if owner is not None and current.id in {"self", "cls"}: - self.push((module, "member", f"{owner}.{chain[-1]}"), at) - return + # The member itself comes from the attribute fallback. + member = chain.pop() + found_items = self.member_imports(module, owner, member) + else: + found_items = lookup(current.id, here, current) starts = [ start - for item in lookup(current.id, here) + for item in found_items for start in self.aliased(item) if start[1] == "module" ] @@ -743,11 +1064,47 @@ def attribute(node: ast.Attribute, here: Scope, at: str) -> None: if found[1] == "module" ) starts = following + if chain and id(node) not in prefixes: + # A dotted name for a module, used as a value. + self.values.update(start[0] for start in starts) + + def call(node: ast.Call, here: Scope, at: str) -> None: + callee = _callee(node.func) + if callee in ATTRIBUTE_BUILTINS and len(node.args) >= 2: + for text in literal_names(node.args[1], here): + read = ast.Attribute(node.args[0], text, ast.Load()) + attribute(ast.copy_location(read, node), here, at) + elif callee in ATTRIBUTE_NAMERS: + for argument in node.args: + for text in literal_names(argument, here): + self.attributes.update(text.split(".")) + elif callee in IMPORTERS and node.args: + for text in literal_names(node.args[0], here): + for name in self.absolute(module, text, node, here): + # import_module returns the module it names; + # __import__ returns the top package, or with a + # fromlist the module itself. + named = [name] + if callee == "__import__": + named.append(name.partition(".")[0]) + for value in named: + if self.is_module(value): + self.push((value, "module", None), at) + self.values.add(value) + + def match_class(node: ast.MatchClass, here: Scope) -> None: + self.attributes.update(node.kwd_attrs) + if not node.patterns: + return + for item in named_items(node.cls, here): + for start in self.aliased(item): + if start[1] == "class": + self.attributes.update(self.match_args(start)) for child, here in _visits(found, node, skipped, start, {}): at = f"{site}:{getattr(child, 'lineno', '?')}" if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Load): - for item in lookup(child.id, here): + for item in lookup(child.id, here, child): self.push(item, at) if item[1] == "module" and id(child) not in rooted: self.values.add(item[0]) @@ -759,24 +1116,64 @@ def attribute(node: ast.Attribute, here: Scope, at: str) -> None: for item in self.resolve_import(*binding): self.push(item, at) elif isinstance(child, ast.Call): - callee = _callee(child.func) - texts = [_literal(argument) for argument in child.args] - if callee in ATTRIBUTE_BUILTINS and len(texts) >= 2 and texts[1]: - read = ast.Attribute(child.args[0], texts[1], ast.Load()) - attribute(ast.copy_location(read, child), here, at) - elif callee in ATTRIBUTE_NAMERS: - for text in filter(None, texts): - self.attributes.update(text.split(".")) - elif callee in IMPORTERS and texts and texts[0]: - # import_module returns the module it names; __import__ - # returns the top package, whose attributes reach the rest. - name = texts[0] - if callee == "__import__": - name = name.partition(".")[0] - if self.is_module(name): - self.push((texts[0], "module", None), at) - self.push((name, "module", None), at) - self.values.add(name) + call(child, here, at) + elif isinstance(child, ast.MatchClass): + match_class(child, here) + + def absolute( + self, module: str, text: str, node: ast.Call, here: Scope + ) -> list[str]: + """The module names an import call's literal can mean.""" + + if not text.startswith("."): + return [text] + level = len(text) - len(text.lstrip(".")) + rest = text[level:] + if len(node.args) > 1: + package_node = node.args[1] + else: + package_node = next( + ( + keyword.value + for keyword in node.keywords + if keyword.arg == "package" + ), + None, + ) + if package_node is None: + return [] + found = self.module(module) + if isinstance(package_node, ast.Name) and package_node.id == "__package__": + packages = [found.package] + elif isinstance(package_node, ast.Name) and package_node.id == "__name__": + packages = [module] + else: + text_node = _literal(package_node) + packages = [text_node] if text_node is not None else [] + names = [] + for package in packages: + parts = package.split(".") + if level > len(parts): + continue + base = ".".join(parts[: len(parts) - (level - 1)]) + names.append(f"{base}.{rest}" if rest else base) + return names + + def match_args(self, item: Item) -> list[str]: + """The attributes a class's explicit ``__match_args__`` names. + + A positional ``match`` class pattern reads them. A dataclass's own + ``__match_args__`` names its fields, which hold data, not code. + """ + + module, _, name = item + return [ + text + for node in self.members(module, name).get("__match_args__", ()) + for _, value in _assignments(node) + for element in getattr(value, "elts", ()) + if (text := _literal(element)) is not None + ] def process(self, item: Item) -> None: module, kind, name = item @@ -794,6 +1191,17 @@ def process(self, item: Item) -> None: for node in found.classes.get(name, ()): for part in (*node.decorator_list, *node.bases, *node.keywords): self.scan(module, part, f"{module}.{name}(class)") + # Creating the class runs its body. + for part in _effects(node.body): + if id(part) not in self.ran: + self.ran.add(id(part)) + self.scan( + module, + part, + f"{module}.{name}(runs)", + owner=name, + body=node, + ) for member, nodes in self.members(module, name).items(): methods = all( isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) @@ -809,12 +1217,21 @@ def process(self, item: Item) -> None: return nodes = self.members(module, owner).get(member) for node in nodes or (): - self.scan(module, node, f"{module}.{name}", qualname=name, owner=owner) + self.scan( + module, + node, + f"{module}.{name}", + qualname=name, + owner=owner, + body=self._body_of.get(id(node)), + ) if nodes: return # Inherited: the same name on each first-party base class. for node in found.classes[owner]: for base in node.bases: + while isinstance(base, ast.Subscript): + base = base.value for resolved in self.resolve_expression(module, base): for start in self.aliased(resolved): if start[1] == "class": @@ -825,9 +1242,23 @@ def process(self, item: Item) -> None: def is_reached(self, item: Item) -> bool: if item[1] == "module": - return any(found[0] == item[0] for found in self.reached) + return item[0] in self.reached_modules return item in self.reached + def changes_reached( + self, found: _Module, bound: str, body: ast.ClassDef | None + ) -> bool: + """Whether changing ``bound`` where ``found`` binds it changes a reached item.""" + + items = self.resolve(found.name, bound) + if body is not None and bound in _members(found, body): + items.append((found.name, "member", f"{body.name}.{bound}")) + return any( + self.is_reached(alias) + for item in items + for alias in dict.fromkeys((item, *self.aliased(item))) + ) + def settle(self) -> bool: """Push what reached names imply; whether anything new came.""" @@ -835,28 +1266,47 @@ def settle(self) -> bool: for module_name, class_name in sorted(self.classes): for member in self.members(module_name, class_name): item = (module_name, "member", f"{class_name}.{member}") - if member in self.attributes and item not in self.reached: - self.push(item, f"{module_name}.{class_name}(attribute)") + if ( + member in self.attributes + and item not in self.reached + and self.push(item, f"{module_name}.{class_name}(attribute)") + ): added = True for module_name in sorted(self.values): for name in sorted(self.attributes): for item in self.exported(module_name, name): - if item not in self.reached: - self.push(item, f"{module_name}(value).{name}") + if item not in self.reached and self.push( + item, f"{module_name}(value).{name}" + ): added = True if item[1] == "module" and item[0] not in self.values: self.values.add(item[0]) added = True + # Import-time code, in each module that holds a reached item. + for module_name in sorted(self.reached_modules): + for part in self.module(module_name).effects: + if id(part) not in self.ran: + self.ran.add(id(part)) + self.scan( + module_name, + part, + f"{module_name}:{getattr(part, 'lineno', '?')}(runs)", + ) + added = True + # Changes in place, in every module the walk has read. for found in list(self.modules.values()): - for bound, node in found.mutations: - if id(node) in self.mutations_run: + for bound, part, body in found.mutations: + if id(part) in self.ran or not self.changes_reached(found, bound, body): continue - if any( - self.is_reached(item) for item in self.resolve(found.name, bound) - ): - self.mutations_run.add(id(node)) - self.scan(found.name, node, f"{found.name}:{node.lineno}(changes)") - added = True + self.ran.add(id(part)) + self.scan( + found.name, + part, + f"{found.name}:{getattr(part, 'lineno', '?')}(changes)", + owner=None if body is None else body.name, + body=body, + ) + added = True return added def run(self, module: str, qualname: str) -> None: @@ -866,11 +1316,14 @@ def run(self, module: str, qualname: str) -> None: self.push((module, "member", qualname), "root") else: self.push((module, "function", qualname), "root") - while True: + for _ in range(MAX_ROUNDS): while self.pending: self.process(self.pending.pop()) if not self.settle() and not self.pending: return + raise RuntimeError( + f"the walk from {module}.{qualname} didn't settle in {MAX_ROUNDS} rounds" + ) @dataclass(frozen=True) diff --git a/test_support/microcosm_build/static_reach_cases.py b/test_support/microcosm_build/static_reach_cases.py index 50c13c78e6..6e09b2f3e1 100644 --- a/test_support/microcosm_build/static_reach_cases.py +++ b/test_support/microcosm_build/static_reach_cases.py @@ -2,13 +2,25 @@ Each root in :data:`ROOTS` reaches its target modules (``t_*``) in one of the ways that :mod:`test_support.microcosm_build.static_reach` lists, and its walk -must reach exactly those targets. A target holds only constants, so a root -reaches it only through code the walk follows. +must reach exactly those targets. A target defines nothing but a module +``__getattr__`` that records each read of ``X`` or ``ROWS``. So the walk +reaches a target only through code it follows, and :func:`python_reads` can +run each root for real and report which targets Python read. + +Python must read exactly what the walk reaches, apart from the documented +gaps: :data:`OVER` holds the targets a walk reaches by design and a run +doesn't read, and :data:`BLIND` holds the reads a run makes that the walk's +"What it can't see" list explains. """ from __future__ import annotations +import json +import subprocess +import sys import textwrap +from collections.abc import Iterable +from hashlib import sha256 from pathlib import Path from test_support.microcosm_build.static_reach import Reach, module_paths @@ -21,43 +33,332 @@ "alias", "alias_base", "alternative", + "alternative_first", "attrgetter", "base_attr", + "base_else", + "base_first", + "base_getattr", + "base_if", + "base_import", + "base_second", + "base_subscript", + "body_annotation", + "body_before", + "body_conditional", + "body_decorator", + "body_hidden", + "body_nested", + "class_body_update", + "class_fill_call", + "class_fill_del", + "class_fill_item", + "class_fill_method", + "class_fill_view", + "class_header", + "class_import", "class_scope", "comprehension", "conditional", "decorator", + "default", + "del_hidden", + "del_item", + "delattr", + "delattr_patch", + "dotted", "factory", + "fill_call", + "fill_global", "float", + "getattr_alias", + "getattr_constant", "global", "go", + "hasattr", + "header_for", + "header_if", + "header_match", + "header_try", + "header_while", + "header_with", + "import_from", "import_module", + "import_relative", "lambda", + "lambda_shadow", "lazy", + "lazy_assigned", "local", + "local_alternative", + "local_alternative_first", + "match_keyword", + "match_positional", + "metaclass", + "methodcaller", + "mutation_alias", + "namespace", + "nested_scope", "nonlocal", "override", "prop", "registrar", + "registry_decorator", "root_base", + "self_method", "setattr", + "setattr_patch", + "setdefault", "setitem", + "setup_call", "shadow", "shadowed", "sibling", "star", + "star_lazy", + "store_hidden", "subclass", + "subscript", "sunder", "unpack", "update", "value", ) +#: Targets a case binds so that a wrong walk would reach them. No walk +#: reaches them and no run reads them: t_shadowed behind a name the class +#: body rebinds (r_class_decorator), and t_body_hidden behind the function's +#: import of nest_p, which the class in r_body_nested never reads. +DECOYS = frozenset({"body_hidden", "shadowed"}) + +#: Each target's source: reads of X or ROWS are recorded, anything else fails. +_TARGET = """\ +from walkcase_reads import read + + +def __getattr__(name): + return read(__name__, name) +""" + +#: The recorder, a top-level module outside ``microcosm``, so the walk never +#: parses it. +_READS = """\ +READS = set() + + +def read(module, name): + if name not in ("X", "ROWS"): + raise AttributeError(name) + READS.add(module.rpartition(".t_")[2]) + return 1 if name == "X" else {} +""" + #: Each module's source, by its dotted name inside :data:`PACKAGE`. SOURCES: dict[str, str] = { "__init__": "", - **{f"t_{name}": "X = 1\nROWS = {}\n" for name in TARGETS}, - # -- Scopes --------------------------------------------------------------- + **{f"t_{name}": _TARGET for name in TARGETS}, + # -- Bodies ----------------------------------------------------------------- + # A default value, which ``def`` evaluates as it runs. + "r_default": """ + from microcosm.walkcases import t_default + + + def run(value=t_default.X): + return value + """, + # A class keyword: the metaclass's __init__ runs as the class is made. + "meta_mod": """ + from microcosm.walkcases import t_metaclass + + + class Meta(type): + def __init__(cls, name, bases, namespace): + super().__init__(name, bases, namespace) + assert t_metaclass.X + """, + "r_metaclass": """ + from microcosm.walkcases.meta_mod import Meta + + + class Kernel(metaclass=Meta): + def run(self): + return 1 + """, + # A method's decorator that the class body defines first. + "r_decorator": """ + from microcosm.walkcases import t_decorator + + + class Kernel: + @staticmethod + def _deco(function): + assert t_decorator.X + return function + + @_deco + def run(self): + return 1 + """, + # -- Bindings --------------------------------------------------------------- + # A star import binds the names its module defines. + "star_src": """ + from microcosm.walkcases import t_star + + + def starred(): + return t_star.X + """, + "r_star": """ + from microcosm.walkcases.star_src import * # noqa: F403 + + + def run(): + return starred() # noqa: F405 + """, + # A name with two bindings reaches both, whichever runs: here the live + # import is the second, and in r_alternative_first the first. + "slow": """ + from microcosm.walkcases import t_alternative + + + def impl(): + return t_alternative.X + """, + "r_alternative": """ + try: + from microcosm.walkcases.absent import impl + except ImportError: + from microcosm.walkcases.slow import impl + + + def run(): + return impl() + """, + "slow_first": """ + from microcosm.walkcases import t_alternative_first + + + def impl(): + return t_alternative_first.X + """, + "fast": """ + def impl(): + return 0 + """, + "r_alternative_first": """ + try: + from microcosm.walkcases.slow_first import impl + except ImportError: + from microcosm.walkcases.fast import impl + + + def run(): + return impl() + """, + # The same inside a function, for a module bound two ways. + "alt_impl": """ + from microcosm.walkcases import t_local_alternative + + + def impl(): + return t_local_alternative.X + """, + "r_local_alternative": """ + def run(): + try: + from microcosm.walkcases import absent as backend + except ImportError: + from microcosm.walkcases import alt_impl as backend + return backend.impl() + """, + "alt_first": """ + from microcosm.walkcases import t_local_alternative_first + + + def impl(): + return t_local_alternative_first.X + """, + "r_local_alternative_first": """ + def run(): + try: + from microcosm.walkcases import alt_first as backend + except ImportError: + from microcosm.walkcases import absent as backend + return backend.impl() + """, + # A package's lazy export, through its module-level __getattr__. + "lazy.__init__": """ + def __getattr__(name): + from importlib import import_module + + return getattr(import_module(f"{__name__}.impl"), name) + """, + "lazy.impl": """ + from microcosm.walkcases import t_lazy + + + class Lazy: + def go(self): + return t_lazy.X + """, + "r_lazy": """ + from microcosm.walkcases.lazy import Lazy + + + def run(): + return Lazy().go() + """, + # ... with __getattr__ assigned rather than defined. + "lazy2.__init__": """ + def _lazy(name): + from importlib import import_module + + return getattr(import_module(f"{__name__}.impl"), name) + + + __getattr__ = _lazy + """, + "lazy2.impl": """ + from microcosm.walkcases import t_lazy_assigned + + + class Lazy2: + def go(self): + return t_lazy_assigned.X + """, + "r_lazy_assigned": """ + from microcosm.walkcases.lazy2 import Lazy2 + + + def run(): + return Lazy2().go() + """, + # ... and through a star import, which __all__ lets reach the lazy name. + "lazy3.__init__": """ + __all__ = ["Lazy3"] + + + def __getattr__(name): + from importlib import import_module + + return getattr(import_module(f"{__name__}.impl"), name) + """, + "lazy3.impl": """ + from microcosm.walkcases import t_star_lazy + + + class Lazy3: + def go(self): + return t_star_lazy.X + """, + "r_star_lazy": """ + from microcosm.walkcases.lazy3 import * # noqa: F403 + + + def run(): + return Lazy3().go() # noqa: F405 + """, + # -- Scopes ----------------------------------------------------------------- # A nested function's local leaves the enclosing scope's import alone. "r_shadow": """ from microcosm.walkcases import t_shadow as target @@ -106,8 +407,58 @@ class Holder: def run(self): return target.X """, - # A method's decorator reads the class body's names first, so the - # module's import of the same name stays unreached. + # ... and out of a lambda there. + "r_lambda_shadow": """ + from microcosm.walkcases import t_lambda_shadow as target + + + class Kernel: + target = None + pick = staticmethod(lambda: target.X) + + def run(self): + return self.pick() + """, + # ... and a nested class's names out of its methods, which read the + # enclosing function's binding instead. + "nest_q": """ + from microcosm.walkcases import t_body_nested + + + def fn(): + return t_body_nested.X + """, + "nest_p": """ + from microcosm.walkcases import t_body_hidden + + + def fn(): + return t_body_hidden.X + """, + "nest_s": """ + from microcosm.walkcases import t_nested_scope + + + def fn(): + return t_nested_scope.X + """, + "r_nested_scope": """ + from microcosm.walkcases import nest_q as target + + + def run(): + from microcosm.walkcases import nest_s as target + + class Inner: + target = None + + def get(self): + return target.fn() + + return Inner().get() + """, + # A method's decorator reads the class body's names first: the body has + # bound target above it, so the module's import stays unreached. "r_class_decorator": """ from microcosm.walkcases import t_shadowed as target @@ -119,6 +470,79 @@ class Kernel: def run(self): return 1 """, + # Before the body binds a name, Python reads the module's binding: a read + # above the binding, a binding under an if that doesn't run, a bare + # annotation (which binds nothing), and a decorator above the binding. + "r_body_before": """ + from microcosm.walkcases import t_body_before as target + + + class Kernel: + seen = target.X + target = None + + def run(self): + return self.seen + """, + "r_body_conditional": """ + from microcosm.walkcases import t_body_conditional as target + + ON = False + + + class Kernel: + if ON: + target = None + seen = target.X + + def run(self): + return self.seen + """, + "r_body_annotation": """ + from microcosm.walkcases import t_body_annotation as target + + + class Kernel: + target: object + seen = target.X + + def run(self): + return self.seen + """, + "deco_body": """ + from microcosm.walkcases import t_body_decorator + + + def wrap(function): + assert t_body_decorator.X + return function + """, + "r_body_decorator": """ + from microcosm.walkcases.deco_body import wrap as target + + + class Kernel: + @target + def run(self): + return 1 + + target = None + """, + # A class inside a function reads such a name from the module too, never + # from the function, so nest_p stays unreached. + "r_body_nested": """ + from microcosm.walkcases import nest_q as target + + + def run(): + from microcosm.walkcases import nest_p as target + + class Inner: + seen = target.fn() + target = None + + return Inner.seen + """, # ``global`` sends a name to module level, past the function's own store. "r_global": """ from microcosm.walkcases import t_global @@ -151,141 +575,599 @@ def inner(): return inner() """, - # -- Resolution ----------------------------------------------------------- - # A star import binds the names its module defines. - "star_src": """ - from microcosm.walkcases import t_star + # -- Aliases ---------------------------------------------------------------- + # A constant that holds a module stands for it, through tuple unpacking, + # and through either branch of a condition. (A module used as a value + # reaches these too: see the attribute fallback.) + "helper": """ + from microcosm.walkcases import t_alias - def starred(): - return t_star.X + def compute(): + return t_alias.X """, - "r_star": """ - from microcosm.walkcases.star_src import * # noqa: F403 + "r_alias": """ + import microcosm.walkcases.helper as helper_module + + _helper = helper_module def run(): - return starred() # noqa: F405 + return _helper.compute() """, - # A name with two bindings reaches both, whichever runs. - "fast": """ - def impl(): + "mod_a": """ + def fn(): return 0 """, - "slow": """ - from microcosm.walkcases import t_alternative + "mod_b": """ + from microcosm.walkcases import t_unpack - def impl(): - return t_alternative.X + def fn(): + return t_unpack.X """, - "r_alternative": """ - try: - from microcosm.walkcases.slow import impl - except ImportError: - from microcosm.walkcases.fast import impl + "mod_c": """ + from microcosm.walkcases import t_conditional - def run(): - return impl() + def fn(): + return t_conditional.X """, - # A package's lazy export, through its module-level __getattr__. - "lazy.__init__": """ - def __getattr__(name): - from importlib import import_module + "r_unpack": """ + from microcosm.walkcases import mod_a, mod_b - return getattr(import_module(f"{__name__}.impl"), name) - """, - "lazy.impl": """ - from microcosm.walkcases import t_lazy + _first, _second = mod_a, mod_b - class Lazy: - def go(self): - return t_lazy.X + def run(): + return _second.fn() """, - "r_lazy": """ - from microcosm.walkcases.lazy import Lazy + "r_conditional": """ + from microcosm.walkcases import mod_a, mod_c + + FLAG = True + _impl = mod_c if FLAG else mod_a def run(): - return Lazy().go() + return _impl.fn() """, - # -- Constants ------------------------------------------------------------ - # A constant that holds a module stands for it. - "helper": """ - from microcosm.walkcases import t_alias + # A class base held in a constant: run() is the base's. Only the alias + # leads there, since nothing else names run. + "made3": """ + from microcosm.walkcases import t_alias_base - def compute(): - return t_alias.X + class Base: + def run(self): + return t_alias_base.X """, - "r_alias": """ - import microcosm.walkcases.helper as helper_module + "r_alias_base": """ + from microcosm.walkcases import made3 - _helper = helper_module + _Base = made3.Base - def run(): - return _helper.compute() + class Kernel(_Base): + pass """, - # ... through tuple unpacking, and through either branch of a condition. - "mod_a": """ - def fn(): - return 0 + # ... through each branch of a conditional (the walk follows both), each + # element of a tuple, getattr, a subscript and import_module. + "bases": """ + from microcosm.walkcases import ( + t_base_else, + t_base_first, + t_base_getattr, + t_base_if, + t_base_import, + t_base_second, + t_base_subscript, + ) + + + class IfBase: + def run(self): + return t_base_if.X + + + class ElseBase: + def run(self): + return t_base_else.X + + + class FirstBase: + def run(self): + return t_base_first.X + + + class SecondBase: + def run(self): + return t_base_second.X + + + class AttrBase: + def run(self): + return t_base_getattr.X + + + class SubscriptBase: + def __class_getitem__(cls, item): + return cls + + def run(self): + return t_base_subscript.X + + + class ImportBase: + def run(self): + return t_base_import.X """, - "mod_b": """ - from microcosm.walkcases import t_unpack + "r_base_conditional": """ + from microcosm.walkcases import bases + + FLAG = False + _Base = bases.IfBase if FLAG else bases.ElseBase + + + class Kernel(_Base): + pass + """, + "r_base_tuple": """ + from microcosm.walkcases import bases + + _First, _Second = bases.FirstBase, bases.SecondBase + + + class Kernel(_Second): + pass + """, + "r_base_getattr": """ + from microcosm.walkcases import bases + + _Base = getattr(bases, "AttrBase") + + + class Kernel(_Base): + pass + """, + "r_base_subscript": """ + from microcosm.walkcases import bases + + _Base = bases.SubscriptBase[int] + + + class Kernel(_Base): + pass + """, + "r_import_alias_base": """ + import importlib + + _impl = importlib.import_module("microcosm.walkcases.bases") + + + class Kernel(_impl.ImportBase): + pass + """, + # A module alias through getattr with a literal name. + "gmod": """ + from microcosm.walkcases import t_getattr_alias def fn(): - return t_unpack.X + return t_getattr_alias.X """, - "mod_c": """ - from microcosm.walkcases import t_conditional + "r_getattr_alias": """ + import microcosm.walkcases.gmod # noqa: F401 + from microcosm import walkcases + + _impl = getattr(walkcases, "gmod") + + + def run(): + return _impl.fn() + """, + # -- Inheritance ------------------------------------------------------------ + # The base's run() calls the root class's override. + "made6": """ + class Base: + def run(self): + return self.step() + + def step(self): + return 0 + """, + "r_override": """ + from microcosm.walkcases import t_override + from microcosm.walkcases.made6 import Base + + + class Kernel(Base): + def step(self): + return t_override.X + """, + # A subscripted base in the class statement itself. + "made9": """ + from microcosm.walkcases import t_subscript + + + class Base: + def __class_getitem__(cls, item): + return cls + + def run(self): + return t_subscript.X + """, + "r_subscript": """ + from microcosm.walkcases.made9 import Base + + + class Kernel(Base[int]): + pass + """, + # -- Self and cls ----------------------------------------------------------- + # A method called through self. (The class-attribute fallback reaches + # it as well.) + "r_self_method": """ + from microcosm.walkcases import t_self_method + + + class Kernel: + def run(self): + return self.pick() + + def pick(self): + return t_self_method.X + """, + # A module the class body imports, called through self. + "cmod": """ + from microcosm.walkcases import t_class_import def fn(): - return t_conditional.X + return t_class_import.X """, - "r_unpack": """ - from microcosm.walkcases import mod_a, mod_b + "r_class_import": """ + class Kernel: + from microcosm.walkcases import cmod as mod + + def run(self): + return self.mod.fn() + """, + # -- Implicit calls --------------------------------------------------------- + # Thing's dunders count wherever Thing does. + "made": """ + from microcosm.walkcases import t_add, t_float, t_go, t_prop + + + class Thing: + def go(self): + return t_go.X + + def __float__(self): + return float(t_float.X) + + def __add__(self, other): + return t_add.X + + @property + def prop(self): + return t_prop.X + + + def make(): + return Thing() + """, + "r_float": """ + from microcosm.walkcases.made import make + + + def run(): + return float(make()) + """, + "r_add": """ + from microcosm.walkcases.made import make + + + def run(): + return make() + 1 + """, + "enum_mod": """ + import enum + + from microcosm.walkcases import t_sunder + + + class Kind(enum.Enum): + A = "a" + + @classmethod + def _missing_(cls, value): + return cls.A if t_sunder.X else None + """, + "r_sunder": """ + from microcosm.walkcases.enum_mod import Kind + + + def run(): + return Kind("b") + """, + # A base's __init_subclass__ and class attribute, for a class run() makes. + "made2": """ + from microcosm.walkcases import t_base_attr, t_factory, t_subclass + + + def factory(): + return [t_factory.X] + + + class Base: + KIND = t_base_attr.X + + def __init_subclass__(cls, **kw): + return t_subclass.X + """, + "r_subclass": """ + from microcosm.walkcases.made2 import Base + + + def run(): + class Sub(Base): + pass + + return Sub + """, + # The root class's own base runs __init_subclass__ for it. + "made5": """ + from microcosm.walkcases import t_root_base + + + class Base: + def __init_subclass__(cls, **kw): + return t_root_base.X + """, + "r_root_base": """ + from microcosm.walkcases.made5 import Base + + + class Kernel(Base): + def run(self): + return 1 + """, + # A dataclass field's default factory. Importing made2 also runs Base's + # body, which the walk doesn't reach from here (see BLIND). + "r_factory": """ + from dataclasses import dataclass, field + + from microcosm.walkcases.made2 import factory + + + @dataclass + class Holder: + items: list = field(default_factory=factory) + + + def run(): + return Holder() + """, + "r_lambda": """ + from microcosm.walkcases import t_lambda + + + def run(): + f = lambda: t_lambda.X # noqa: E731 + return f() + """, + # -- Import-time code ------------------------------------------------------- + # Headers of compound statements at module level, and in a class body. + "r_headers": """ + from contextlib import nullcontext + + from microcosm.walkcases import ( + t_header_for, + t_header_if, + t_header_match, + t_header_try, + t_header_while, + t_header_with, + ) + + if t_header_if.X: + VALUE = 1 + else: + VALUE = 2 + while not t_header_while.X: + pass + for _row in t_header_for.ROWS: + pass + with nullcontext(t_header_with.X): + pass + match t_header_match.ROWS: + case {"key": found}: + FOUND = found + case _: + FOUND = None + try: + raise ValueError + except ValueError if t_header_try.X else TypeError: + pass + + + def run(): + return VALUE + """, + "r_class_header": """ + from microcosm.walkcases import t_class_header + + + class Kernel: + if t_class_header.X: + LEVEL = 1 + else: + LEVEL = 2 + + def run(self): + return self.LEVEL + """, + # Calls at module level that fill a global or a table. + "r_fill_global": """ + from microcosm.walkcases import t_fill_global + + _CACHE = None + + + def _fill(): + global _CACHE + _CACHE = t_fill_global.X + + + _fill() + + + def run(): + return _CACHE + """, + "r_setup_call": """ + from microcosm.walkcases import t_setup_call + + _TABLE = {} + + + def _setup(): + _TABLE.update({"key": t_setup_call.X}) + + + _setup() + + + def run(): + return _TABLE + """, + "r_fill_call": """ + from microcosm.walkcases import t_fill_call + + _TABLE = {} + + + def _fill(table): + table["key"] = t_fill_call.X + + + _fill(_TABLE) + + + def run(): + return _TABLE + """, + # A del from a table run() reads, and a store through an alias of it. + # (Changes by name reaches these as well.) + "r_del_item": """ + from microcosm.walkcases import t_del_item + + _TABLE = {"a": 1, "b": 2} + del _TABLE[t_del_item.X and "a"] + + + def run(): + return _TABLE + """, + "r_mutation_alias": """ + from microcosm.walkcases import t_mutation_alias + + _TABLE = {} + _VIEW = _TABLE + _VIEW["key"] = t_mutation_alias.X + + + def run(): + return _TABLE + """, + # The same through a call's result, which no name links to the table, + # so only the scan of import-time code reaches the targets. + "r_del_hidden": """ + from microcosm.walkcases import t_del_hidden + + _TABLE = {"a": 1, "b": 2} + + + def _table(): + return _TABLE + + + _VIEW = _table() + del _VIEW[t_del_hidden.X and "a"] + + + def run(): + return _TABLE + """, + "r_store_hidden": """ + from microcosm.walkcases import t_store_hidden + + _TABLE = {} + + + def _table(): + return _TABLE + + + _VIEW = _table() + _VIEW["key"] = t_store_hidden.X + + + def run(): + return _TABLE + """, + # Patches to the root class. + "patch_helper": """ + from microcosm.walkcases import t_setattr_patch + + + def helper(): + return t_setattr_patch.X + """, + "r_setattr_patch": """ + from microcosm.walkcases.patch_helper import helper + - _first, _second = mod_a, mod_b + class Kernel: + def run(self): + return self.helper() - def run(): - return _second.fn() + setattr(Kernel, "helper", staticmethod(helper)) """, - "r_conditional": """ - from microcosm.walkcases import mod_a, mod_c + "r_delattr_patch": """ + from microcosm.walkcases import t_delattr_patch - FLAG = True - _impl = mod_c if FLAG else mod_a + class Base: + def pick(self): + return 0 - def run(): - return _impl.fn() - """, - # A class base held in a constant: run() is the base's. - "made3": """ - from microcosm.walkcases import t_alias_base + class Kernel(Base): + def pick(self): + return 1 - class Base: def run(self): - return t_alias_base.X + return self.pick() + + + delattr(Kernel, "pick" if t_delattr_patch.X else "run") """, - "r_alias_base": """ - from microcosm.walkcases import made3 + # A reached class's body changing its own attribute. + "r_class_body_update": """ + from microcosm.walkcases import t_class_body_update - _Base = made3.Base + class Kernel: + TABLE = {} + TABLE.update({"key": t_class_body_update.X}) - class Kernel(_Base): - pass + def run(self): + return self.TABLE """, - # -- Changes in place ----------------------------------------------------- + # -- Changes by name -------------------------------------------------------- "r_update": """ from microcosm.walkcases import t_update @@ -306,6 +1188,48 @@ def run(): def run(): return _TABLE """, + # A method call inside an assignment. + "r_setdefault": """ + from microcosm.walkcases import t_setdefault + + _TABLE = {} + _ROW = _TABLE.setdefault("key", t_setdefault.X) + + + def run(): + return _TABLE + """, + # The body of a class nothing reaches still runs at import, and each of + # its statements changes a table run() reads. + "r_class_fill": """ + from microcosm.walkcases import ( + t_class_fill_call, + t_class_fill_del, + t_class_fill_item, + t_class_fill_method, + t_class_fill_view, + ) + + _TABLE = {} + _ROWS = [] + _VIEW = _TABLE + + + def _fill(table): + table["call"] = t_class_fill_call.X + + + class Filler: + _TABLE.update({"method": t_class_fill_method.X}) + _TABLE["item"] = t_class_fill_item.X + del _ROWS[t_class_fill_del.X :] + _fill(_TABLE) + _VIEW["view"] = t_class_fill_view.X + + + def run(): + return _TABLE, _ROWS + """, # Another module the walk reads fills the registry run() reads. "registry": """ REGISTRY = [] @@ -328,30 +1252,7 @@ def helper(): def run(): return REGISTRY, registrar.helper() """, - # -- Attributes of objects the walk can't type ------------------------------ - # Thing's dunders count wherever Thing does (see Implicit calls). - "made": """ - from microcosm.walkcases import t_add, t_float, t_go, t_prop - - - class Thing: - def go(self): - return t_go.X - - def __float__(self): - return float(t_float.X) - - def __add__(self, other): - return t_add.X - - @property - def prop(self): - return t_prop.X - - - def make(): - return Thing() - """, + # -- Attribute fallback ----------------------------------------------------- "r_prop": """ from microcosm.walkcases.made import make @@ -359,7 +1260,7 @@ def make(): def run(): return make().prop """, - # A module passed as an argument. + # A module passed as an argument, by name and by a dotted chain. "helper2": """ from microcosm.walkcases import t_value @@ -379,23 +1280,41 @@ def use(module): def run(): return use(helper2) """, - # The base's run() calls the root class's override. - "made6": """ - class Base: - def run(self): - return self.step() + "dot.__init__": "", + "dot.helper": """ + from microcosm.walkcases import t_dotted - def step(self): - return 0 + + def compute2(): + return t_dotted.X """, - "r_override": """ - from microcosm.walkcases import t_override - from microcosm.walkcases.made6 import Base + "r_dotted": """ + import microcosm.walkcases.dot.helper + from microcosm.walkcases.use import use - class Kernel(Base): - def step(self): - return t_override.X + def run(): + return use(microcosm.walkcases.dot.helper) + """, + # A namespace sub-package (nsdir has no __init__.py) under a module used + # as a value. The walk must still settle. + "nspkg.__init__": "", + "nspkg.nsdir.leaf": """ + from microcosm.walkcases import t_namespace + + + def fn(): + return t_namespace.X + """, + "r_namespace": """ + import importlib + + import microcosm.walkcases.nspkg.nsdir.leaf # noqa: F401 + + + def run(): + package = importlib.import_module("microcosm.walkcases.nspkg") + return package.nsdir.leaf.fn() """, # -- Literal names ---------------------------------------------------------- "r_getattr": """ @@ -427,156 +1346,192 @@ def run(): setattr(thing, "level", 1) return thing """, - "made4": """ - from microcosm.walkcases import t_attrgetter + "made8": """ + from microcosm.walkcases import t_delattr, t_hasattr - class Thing: + class Probe: @property - def named(self): - return t_attrgetter.X - """, - "r_attrgetter": """ - from operator import attrgetter + def ready(self): + return t_hasattr.X - from microcosm.walkcases.made4 import Thing + @property + def item(self): + return 0 + + @item.deleter + def item(self): + assert t_delattr.X + """, + "r_hasattr": """ + from microcosm.walkcases.made8 import Probe def run(): - return attrgetter("named")(Thing()) + return hasattr(Probe(), "ready") """, - "imp_mod": """ - from microcosm.walkcases import t_import_module + "r_delattr": """ + from microcosm.walkcases.made8 import Probe - def fn(): - return t_import_module.X + def run(): + probe = Probe() + delattr(probe, "item") + return probe """, - "r_import_module": """ - import importlib + "made12": """ + from microcosm.walkcases import t_getattr_constant - def run(): - return importlib.import_module("microcosm.walkcases.imp_mod").fn() + class Gauge: + @property + def probe(self): + return t_getattr_constant.X """, - # -- Implicit calls --------------------------------------------------------- - "r_float": """ - from microcosm.walkcases.made import make + "r_getattr_constant": """ + from microcosm.walkcases.made12 import Gauge + + _NAME = "probe" def run(): - return float(make()) + return getattr(Gauge(), _NAME) """, - "r_add": """ - from microcosm.walkcases.made import make + "made4": """ + from microcosm.walkcases import t_attrgetter - def run(): - return make() + 1 + class Thing: + @property + def named(self): + return t_attrgetter.X """, - "enum_mod": """ - import enum + "r_attrgetter": """ + from operator import attrgetter - from microcosm.walkcases import t_sunder + from microcosm.walkcases.made4 import Thing - class Kind(enum.Enum): - A = "a" + def run(): + return attrgetter("named")(Thing()) + """, + "made13": """ + from microcosm.walkcases import t_methodcaller - @classmethod - def _missing_(cls, value): - return t_sunder.X + + class Pinger: + def ping(self): + return t_methodcaller.X """, - "r_sunder": """ - from microcosm.walkcases.enum_mod import Kind + "r_methodcaller": """ + from operator import methodcaller + + from microcosm.walkcases.made13 import Pinger def run(): - return Kind("b") + return methodcaller("ping")(Pinger()) """, - # A base's __init_subclass__ and class attribute, for a class run() makes. - "made2": """ - from microcosm.walkcases import t_base_attr, t_factory, t_subclass - + # A match class pattern reads its keyword names, and the names in an + # explicit __match_args__ for its positional ones. + "made10": """ + from microcosm.walkcases import t_match_keyword, t_match_positional - def factory(): - return [t_factory.X] + class Point: + __match_args__ = ("left",) - class Base: - KIND = t_base_attr.X + @property + def depth(self): + return t_match_keyword.X - def __init_subclass__(cls, **kw): - return t_subclass.X + @property + def left(self): + return t_match_positional.X """, - "r_subclass": """ - from microcosm.walkcases.made2 import Base + "r_match_keyword": """ + from microcosm.walkcases.made10 import Point def run(): - class Sub(Base): - pass - - return Sub + match Point(): + case Point(depth=value): + return value + return None """, - # The root class's own base runs __init_subclass__ for it. - "made5": """ - from microcosm.walkcases import t_root_base + "r_match_positional": """ + from microcosm.walkcases.made10 import Point - class Base: - def __init_subclass__(cls, **kw): - return t_root_base.X + def run(): + match Point(): + case Point(value): + return value + return None """, - "r_root_base": """ - from microcosm.walkcases.made5 import Base + "imp_mod": """ + from microcosm.walkcases import t_import_module - class Kernel(Base): - def run(self): - return 1 + def fn(): + return t_import_module.X """, - "r_factory": """ - from dataclasses import dataclass, field + "r_import_module": """ + import importlib - from microcosm.walkcases.made2 import factory + + def run(): + return importlib.import_module("microcosm.walkcases.imp_mod").fn() + """, + "rel_mod": """ + from microcosm.walkcases import t_import_relative - @dataclass - class Holder: - items: list = field(default_factory=factory) + def fn(): + return t_import_relative.X + """, + "r_import_relative": """ + import importlib def run(): - return Holder() + return importlib.import_module(".rel_mod", __package__).fn() """, - "r_lambda": """ - from microcosm.walkcases import t_lambda + # __import__ with a fromlist returns the module it names. + "imp_leaf": """ + from microcosm.walkcases import t_import_from + def fn(): + return t_import_from.X + """, + "r_import_from": """ def run(): - f = lambda: t_lambda.X # noqa: E731 - return f() + module = __import__("microcosm.walkcases.imp_leaf", fromlist=["fn"]) + return module.fn() """, - # A method's decorator, held in a class attribute. - "deco_mod": """ - from microcosm.walkcases import t_decorator + # -- What the walk can't see ------------------------------------------------ + # A decorator records _entry, and run() calls whatever the registry holds. + "reg": """ + REGISTRY = [] - def wrap(function): - assert t_decorator.X + def register(function): + REGISTRY.append(function) return function """, - "r_decorator": """ - from microcosm.walkcases import deco_mod + "r_registry_decorator": """ + from microcosm.walkcases import t_registry_decorator + from microcosm.walkcases.reg import REGISTRY, register - class Kernel: - _deco = deco_mod.wrap + @register + def _entry(): + return t_registry_decorator.X - @_deco - def run(self): - return 1 + + def run(): + return [entry() for entry in REGISTRY] """, } @@ -586,49 +1541,151 @@ def run(self): #: (root module, qualname) -> the targets its walk reaches, exactly. ROOTS: dict[tuple[str, str], set[str]] = { + # Bodies + ("r_default", "run"): {"default"}, + ("r_metaclass", "Kernel.run"): {"metaclass"}, + ("r_decorator", "Kernel.run"): {"decorator"}, + # Bindings + ("r_star", "run"): {"star"}, + ("r_alternative", "run"): {"alternative"}, + ("r_alternative_first", "run"): {"alternative_first"}, + ("r_local_alternative", "run"): {"local_alternative"}, + ("r_local_alternative_first", "run"): {"local_alternative_first"}, + ("r_lazy", "run"): {"lazy"}, + ("r_lazy_assigned", "run"): {"lazy_assigned"}, + ("r_star_lazy", "run"): {"star_lazy"}, + # Scopes ("r_shadow", "run"): {"shadow"}, ("r_comprehension", "run"): {"comprehension"}, ("r_sibling", "run"): {"sibling", "local"}, ("r_class_scope", "Holder.run"): {"class_scope"}, + ("r_lambda_shadow", "Kernel.run"): {"lambda_shadow"}, + ("r_nested_scope", "run"): {"nested_scope"}, ("r_class_decorator", "Kernel.run"): set(), + ("r_body_before", "Kernel.run"): {"body_before"}, + ("r_body_conditional", "Kernel.run"): {"body_conditional"}, + ("r_body_annotation", "Kernel.run"): {"body_annotation"}, + ("r_body_decorator", "Kernel.run"): {"body_decorator"}, + ("r_body_nested", "run"): {"body_nested"}, ("r_global", "run"): {"global"}, ("r_nonlocal", "run"): {"nonlocal"}, - ("r_star", "run"): {"star"}, - ("r_alternative", "run"): {"alternative"}, - ("r_lazy", "run"): {"lazy"}, + # Aliases ("r_alias", "run"): {"alias"}, ("r_unpack", "run"): {"unpack"}, ("r_conditional", "run"): {"conditional"}, ("r_alias_base", "Kernel.run"): {"alias_base"}, + ("r_base_conditional", "Kernel.run"): {"base_if", "base_else"}, + ("r_base_tuple", "Kernel.run"): {"base_first", "base_second"}, + ("r_base_getattr", "Kernel.run"): {"base_getattr"}, + ("r_base_subscript", "Kernel.run"): {"base_subscript"}, + ("r_import_alias_base", "Kernel.run"): {"base_import"}, + ("r_getattr_alias", "run"): {"getattr_alias"}, + # Inheritance + ("r_override", "Kernel.run"): {"override"}, + ("r_subscript", "Kernel.run"): {"subscript"}, + # Self and cls + ("r_self_method", "Kernel.run"): {"self_method"}, + ("r_class_import", "Kernel.run"): {"class_import"}, + # Implicit calls + ("r_float", "run"): {"float", *_THING}, + ("r_add", "run"): {"add", *_THING}, + ("r_sunder", "run"): {"sunder"}, + ("r_subclass", "run"): {"subclass", "base_attr"}, + ("r_root_base", "Kernel.run"): {"root_base"}, + ("r_factory", "run"): {"factory"}, + ("r_lambda", "run"): {"lambda"}, + # Import-time code + ("r_headers", "run"): { + "header_for", + "header_if", + "header_match", + "header_try", + "header_while", + "header_with", + }, + ("r_class_header", "Kernel.run"): {"class_header"}, + ("r_fill_global", "run"): {"fill_global"}, + ("r_setup_call", "run"): {"setup_call"}, + ("r_fill_call", "run"): {"fill_call"}, + ("r_del_item", "run"): {"del_item"}, + ("r_mutation_alias", "run"): {"mutation_alias"}, + ("r_del_hidden", "run"): {"del_hidden"}, + ("r_store_hidden", "run"): {"store_hidden"}, + ("r_setattr_patch", "Kernel.run"): {"setattr_patch"}, + ("r_delattr_patch", "Kernel.run"): {"delattr_patch"}, + ("r_class_body_update", "Kernel.run"): {"class_body_update"}, + # Changes by name ("r_update", "run"): {"update"}, ("r_setitem", "run"): {"setitem"}, + ("r_setdefault", "run"): {"setdefault"}, + ("r_class_fill", "run"): { + "class_fill_call", + "class_fill_del", + "class_fill_item", + "class_fill_method", + "class_fill_view", + }, ("r_registry", "run"): {"registrar"}, + # Attribute fallback ("r_prop", "run"): {"prop", *_THING}, ("r_value", "run"): {"value"}, - ("r_override", "Kernel.run"): {"override"}, + ("r_dotted", "run"): {"dotted"}, + ("r_namespace", "run"): {"namespace"}, + # Literal names ("r_getattr", "run"): {"go", *_THING}, ("r_setattr", "run"): {"setattr"}, + ("r_hasattr", "run"): {"hasattr"}, + ("r_delattr", "run"): {"delattr"}, + ("r_getattr_constant", "run"): {"getattr_constant"}, ("r_attrgetter", "run"): {"attrgetter"}, + ("r_methodcaller", "run"): {"methodcaller"}, + ("r_match_keyword", "run"): {"match_keyword"}, + ("r_match_positional", "run"): {"match_positional"}, ("r_import_module", "run"): {"import_module"}, - ("r_float", "run"): {"float", *_THING}, - ("r_add", "run"): {"add", *_THING}, - ("r_sunder", "run"): {"sunder"}, - ("r_subclass", "run"): {"subclass", "base_attr"}, - ("r_root_base", "Kernel.run"): {"root_base"}, - ("r_factory", "run"): {"factory"}, - ("r_lambda", "run"): {"lambda"}, - ("r_decorator", "Kernel.run"): {"decorator"}, + ("r_import_relative", "run"): {"import_relative"}, + ("r_import_from", "run"): {"import_from"}, + # What the walk can't see + ("r_registry_decorator", "run"): set(), +} + +#: Targets a root's walk reaches that its run doesn't read. The walk reaches +#: every dunder of a class it references, and both branches of a condition. +OVER: dict[tuple[str, str], set[str]] = { + ("r_float", "run"): {"add"}, + ("r_add", "run"): {"float"}, + ("r_prop", "run"): _THING, + ("r_getattr", "run"): _THING, + ("r_base_conditional", "Kernel.run"): {"base_if"}, + ("r_base_tuple", "Kernel.run"): {"base_first"}, } +#: Targets a root's run reads that its walk doesn't reach, each one a line of +#: the walk's "What it can't see". +BLIND: dict[tuple[str, str], set[str]] = { + # Importing made2 runs Base's body, a class the walk doesn't reach from + # factory. Nothing hands that value to the run. + ("r_factory", "run"): {"base_attr"}, + # Registration: the decorator records _entry, and run() calls it. + ("r_registry_decorator", "run"): {"registry_decorator"}, +} + + +def expected_reads(root: tuple[str, str]) -> set[str]: + """The targets a root's run must read: its walk's, less OVER, plus BLIND.""" + + return (ROOTS[root] - OVER.get(root, set())) | BLIND.get(root, set()) + def write(root: Path) -> dict[str, Path]: """Write :data:`SOURCES` under ``root``; return their modules' paths.""" - base = root / "packages" / "cases" / "src" / "microcosm" / "walkcases" - for name, source in SOURCES.items(): + source = root / "packages" / "cases" / "src" + base = source.joinpath(*PACKAGE.split(".")) + for name, text in SOURCES.items(): path = base.joinpath(*name.split(".")).with_suffix(".py") path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(textwrap.dedent(source).lstrip("\n"), encoding="utf-8") + path.write_text(textwrap.dedent(text).lstrip("\n"), encoding="utf-8") + (source / "walkcase_reads.py").write_text(_READS, encoding="utf-8") return module_paths(root) @@ -639,3 +1696,96 @@ def targets(walked: Reach) -> set[str]: return { name.removeprefix(prefix) for name in walked.modules if name.startswith(prefix) } + + +# Imports each root's module afresh and calls the root; prints, for each root, +# the targets Python read, or the error the root raised. +_DRIVER = """ +import importlib, json, sys + +source, package, roots = sys.argv[1], sys.argv[2], json.loads(sys.argv[3]) +sys.path.insert(0, source) +import walkcase_reads + +found = {} +for module, qualname in roots: + for name in [name for name in sys.modules if name.split(".")[0] == "microcosm"]: + del sys.modules[name] + walkcase_reads.READS.clear() + try: + loaded = importlib.import_module(f"{package}.{module}") + owner, _, name = qualname.rpartition(".") + root = getattr(getattr(loaded, owner)(), name) if owner else getattr(loaded, name) + root() + except Exception as error: + found[f"{module}:{qualname}"] = f"{type(error).__name__}: {error}" + else: + found[f"{module}:{qualname}"] = sorted(walkcase_reads.READS) +print(json.dumps(found)) +""" + + +def compose( + root: Path, roots: Iterable[tuple[str, str]] +) -> tuple[tuple[str, str], dict[str, Path]]: + """Write a module whose ``run()`` runs each of ``roots`` in turn. + + ``root`` holds what :func:`write` wrote. Returns the new root, named for + the roots it runs, and the tree's module paths with it. + """ + + picked = sorted(roots) + digest = sha256(json.dumps(picked).encode()).hexdigest()[:12] + lines = ["def run():"] + for module, qualname in picked: + owner, _, name = qualname.rpartition(".") + lines.append(f" from {PACKAGE} import {module}") + lines.append( + f" {module}.{owner}().{name}()" if owner else f" {module}.{name}()" + ) + module = f"c_{digest}" + base = root.joinpath("packages", "cases", "src", *PACKAGE.split(".")) + (base / f"{module}.py").write_text("\n".join(lines) + "\n", encoding="utf-8") + return (module, "run"), module_paths(root) + + +def python_reads( + root: Path, roots: Iterable[tuple[str, str]] | None = None +) -> dict[tuple[str, str], set[str] | str]: + """Run each root (by default each in :data:`ROOTS`) for real; the targets + Python read. + + ``root`` holds what :func:`write` wrote. One interpreter with no site + packages and no environment (``-I -S``) imports each root's module from + scratch, then calls the root (for ``Class.method``, on a new instance). A + root that raises maps to its error's text instead. + """ + + picked = sorted(ROOTS if roots is None else roots) + + done = subprocess.run( + [ + sys.executable, + "-I", + "-S", + "-B", + "-c", + _DRIVER, + str(root / "packages" / "cases" / "src"), + PACKAGE, + json.dumps(picked), + ], + capture_output=True, + check=True, + cwd=root, + text=True, + ) + found = json.loads(done.stdout) + return { + (module, qualname): ( + set(reads) + if isinstance(reads := found[f"{module}:{qualname}"], list) + else reads + ) + for module, qualname in picked + } From 005844bd7ca3a00750ec4b10afe6333f64ca0054 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 01:39:10 -0500 Subject: [PATCH 09/14] NZ G8-x2: pin each branch of the walk's binding, alias and header rules (#1567) eb236a6f2's 71 knockouts left some branches of the walk's helpers with no knockout. Each branch now has one, and a root fails when it goes: - Bindings: a for or with target, a match capture (plain, star and mapping rest) and a walrus target, each imported from the module that binds it. That import alone leads the walk to the module. - Aliases: or, a walrus, a starred list, and __import__'s top package, each as a class base. - Import-time code: match patterns and guards, except* types, and effects inside if, for-else, finally, except and match-case blocks. The walk's docstring now names these forms; its code doesn't change. static_reach_cases.py grows to 91 roots, and each of 91 one-edit knockouts fails at least one of them, on Python 3.13 and 3.14. Co-Authored-By: Claude Opus 5.5 --- test_support/microcosm_build/static_reach.py | 22 +- .../microcosm_build/static_reach_cases.py | 228 +++++++++++++++++- 2 files changed, 241 insertions(+), 9 deletions(-) diff --git a/test_support/microcosm_build/static_reach.py b/test_support/microcosm_build/static_reach.py index b173edcf04..ac7b34af4d 100644 --- a/test_support/microcosm_build/static_reach.py +++ b/test_support/microcosm_build/static_reach.py @@ -16,7 +16,10 @@ - Bindings. A name resolves to every definition that can bind it: each alternative of a ``try`` or an ``if``, at module or function level, star imports, re-exports, and a package's lazy exports through a module-level - ``__getattr__``, defined or assigned. + ``__getattr__``, defined or assigned. Every form of assignment binds: a + ``for`` or ``with`` target stands for what the loop iterates or the + statement enters, a ``match`` capture for the subject, and a walrus + target for its value. - Scopes. Each scope binds only its own names, as Python scopes them. A comprehension, lambda or nested function keeps its names to itself, a class body's names stay out of its methods, and ``global`` and ``nonlocal`` send a @@ -26,8 +29,10 @@ module's binding, so the walk follows that one too. - Aliases. A module-level constant stands for each module, class or function its value names: ``_helper = helper_module``, ``_Base = pkg.Base``, - ``pkg.Base[int]``, either branch of a conditional, each element of a tuple, - and ``getattr`` or ``import_module`` with a literal name. + ``pkg.Base[int]``, either branch of a conditional, each operand of ``and`` + or ``or``, a walrus's value, each element of a tuple, list or set (starred + ones too), and ``getattr``, ``import_module`` or ``__import__`` with a + literal name (for ``__import__``, the top package as well). - Inheritance. A member that a class doesn't define resolves on each first-party base, through aliases and subscripts (``Base[int]``). - Self and cls. In a method, a chain from ``self`` or ``cls`` continues @@ -40,11 +45,12 @@ - Import-time code. Importing a module runs its module-level code, and creating a class runs its body. In each module that holds an item the walk reaches, and in each class it reaches, the walk follows every statement - that runs only for its effect, whatever it names: an expression statement - (``_setup()``, ``setattr(Kernel, ...)``), a ``del``, and a store to an item - or an attribute. It also follows the header of each compound statement - there (``if``, ``while``, ``for``, ``with``, ``match``, and an ``except`` - clause's types). + that runs only for its effect, in any block and whatever it names: an + expression statement (``_setup()``, ``setattr(Kernel, ...)``), a ``del``, + and a store to an item or an attribute. It also follows each compound + statement's header there: an ``if`` or ``while`` test, what a ``for`` + iterates or a ``with`` enters, a ``match`` statement's subject, patterns + and guards, and each ``except`` or ``except*`` clause's types. - Changes by name. In any module the walk has read, a statement at module level or in any class body that changes a reached name in place runs with it: a method call on the name (``_TABLE.update(...)``, also inside an diff --git a/test_support/microcosm_build/static_reach_cases.py b/test_support/microcosm_build/static_reach_cases.py index 6e09b2f3e1..2ff68ebcbf 100644 --- a/test_support/microcosm_build/static_reach_cases.py +++ b/test_support/microcosm_build/static_reach_cases.py @@ -36,13 +36,22 @@ "alternative_first", "attrgetter", "base_attr", + "base_dunder_import", "base_else", "base_first", "base_getattr", "base_if", "base_import", + "base_or", "base_second", + "base_starred", "base_subscript", + "base_walrus", + "block_body", + "block_case", + "block_else", + "block_finally", + "block_handler", "body_annotation", "body_before", "body_conditional", @@ -71,15 +80,19 @@ "fill_call", "fill_global", "float", + "for_target", "getattr_alias", "getattr_constant", "global", "go", "hasattr", "header_for", + "header_guard", "header_if", "header_match", + "header_pattern", "header_try", + "header_trystar", "header_while", "header_with", "import_from", @@ -92,8 +105,11 @@ "local", "local_alternative", "local_alternative_first", + "match_capture", "match_keyword", "match_positional", + "match_rest", + "match_star", "metaclass", "methodcaller", "mutation_alias", @@ -123,6 +139,8 @@ "unpack", "update", "value", + "walrus", + "with_target", ) #: Targets a case binds so that a wrong walk would reach them. No walk @@ -286,6 +304,91 @@ def run(): from microcosm.walkcases import absent as backend return backend.impl() """, + # A name that a for loop, a with statement, a match capture or a walrus + # binds at module level, imported from that module. Nothing else leads + # the walk to the module. + "bind_for": """ + from microcosm.walkcases import t_for_target + + for VALUE in (t_for_target.X,): + pass + """, + "r_bind_for": """ + from microcosm.walkcases.bind_for import VALUE + + + def run(): + return VALUE + """, + "bind_with": """ + from contextlib import nullcontext + + from microcosm.walkcases import t_with_target + + with nullcontext(t_with_target.X) as VALUE: + pass + """, + "r_bind_with": """ + from microcosm.walkcases.bind_with import VALUE + + + def run(): + return VALUE + """, + "bind_match": """ + from microcosm.walkcases import t_match_capture + + match t_match_capture.X: + case VALUE: + pass + """, + "r_bind_match": """ + from microcosm.walkcases.bind_match import VALUE + + + def run(): + return VALUE + """, + "bind_star": """ + from microcosm.walkcases import t_match_star + + match (t_match_star.X,): + case [*VALUE]: + pass + """, + "r_bind_star": """ + from microcosm.walkcases.bind_star import VALUE + + + def run(): + return VALUE + """, + "bind_rest": """ + from microcosm.walkcases import t_match_rest + + match t_match_rest.ROWS: + case {**VALUE}: + pass + """, + "r_bind_rest": """ + from microcosm.walkcases.bind_rest import VALUE + + + def run(): + return VALUE + """, + "bind_walrus": """ + from microcosm.walkcases import t_walrus + + _FIRST = (VALUE := t_walrus.X) + """, + "r_bind_walrus": """ + from microcosm.walkcases.bind_walrus import VALUE + + + def run(): + return VALUE + """, # A package's lazy export, through its module-level __getattr__. "lazy.__init__": """ def __getattr__(name): @@ -652,16 +755,21 @@ class Kernel(_Base): pass """, # ... through each branch of a conditional (the walk follows both), each - # element of a tuple, getattr, a subscript and import_module. + # operand of or, a walrus, each element of a tuple, a starred list, + # getattr, a subscript, import_module, and __import__'s top package. "bases": """ from microcosm.walkcases import ( + t_base_dunder_import, t_base_else, t_base_first, t_base_getattr, t_base_if, t_base_import, + t_base_or, t_base_second, + t_base_starred, t_base_subscript, + t_base_walrus, ) @@ -701,6 +809,26 @@ def run(self): class ImportBase: def run(self): return t_base_import.X + + + class OrBase: + def run(self): + return t_base_or.X + + + class WalrusBase: + def run(self): + return t_base_walrus.X + + + class StarredBase: + def run(self): + return t_base_starred.X + + + class DunderImportBase: + def run(self): + return t_base_dunder_import.X """, "r_base_conditional": """ from microcosm.walkcases import bases @@ -739,6 +867,40 @@ class Kernel(_Base): class Kernel(_Base): pass """, + "r_base_or": """ + from microcosm.walkcases import bases + + _Base = None or bases.OrBase + + + class Kernel(_Base): + pass + """, + "r_base_walrus": """ + from microcosm.walkcases import bases + + _Base = (_Found := bases.WalrusBase) + + + class Kernel(_Base): + pass + """, + "r_base_starred": """ + from microcosm.walkcases import bases + + _Base = [*[bases.StarredBase]][0] + + + class Kernel(_Base): + pass + """, + "r_base_dunder_import": """ + _top = __import__("microcosm.walkcases.bases") + + + class Kernel(_top.walkcases.bases.DunderImportBase): + pass + """, "r_import_alias_base": """ import importlib @@ -964,9 +1126,12 @@ def run(): from microcosm.walkcases import ( t_header_for, + t_header_guard, t_header_if, t_header_match, + t_header_pattern, t_header_try, + t_header_trystar, t_header_while, t_header_with, ) @@ -986,15 +1151,56 @@ def run(): FOUND = found case _: FOUND = None + match 0: + case t_header_pattern.X: + PATTERN = 1 + case _ if t_header_guard.X: + PATTERN = 2 try: raise ValueError except ValueError if t_header_try.X else TypeError: pass + try: + raise ExceptionGroup("group", [ValueError()]) + except* (ValueError if t_header_trystar.X else TypeError): + pass def run(): return VALUE """, + # Statements that run for their effect inside each kind of block. + "r_blocks": """ + from microcosm.walkcases import ( + t_block_body, + t_block_case, + t_block_else, + t_block_finally, + t_block_handler, + ) + + if True: + t_block_body.X + for _row in (): + pass + else: + t_block_else.X + try: + pass + finally: + t_block_finally.X + try: + raise ValueError + except ValueError: + t_block_handler.X + match 1: + case _: + t_block_case.X + + + def run(): + return 1 + """, "r_class_header": """ from microcosm.walkcases import t_class_header @@ -1551,6 +1757,12 @@ def run(): ("r_alternative_first", "run"): {"alternative_first"}, ("r_local_alternative", "run"): {"local_alternative"}, ("r_local_alternative_first", "run"): {"local_alternative_first"}, + ("r_bind_for", "run"): {"for_target"}, + ("r_bind_with", "run"): {"with_target"}, + ("r_bind_match", "run"): {"match_capture"}, + ("r_bind_star", "run"): {"match_star"}, + ("r_bind_rest", "run"): {"match_rest"}, + ("r_bind_walrus", "run"): {"walrus"}, ("r_lazy", "run"): {"lazy"}, ("r_lazy_assigned", "run"): {"lazy_assigned"}, ("r_star_lazy", "run"): {"star_lazy"}, @@ -1578,6 +1790,10 @@ def run(): ("r_base_tuple", "Kernel.run"): {"base_first", "base_second"}, ("r_base_getattr", "Kernel.run"): {"base_getattr"}, ("r_base_subscript", "Kernel.run"): {"base_subscript"}, + ("r_base_or", "Kernel.run"): {"base_or"}, + ("r_base_walrus", "Kernel.run"): {"base_walrus"}, + ("r_base_starred", "Kernel.run"): {"base_starred"}, + ("r_base_dunder_import", "Kernel.run"): {"base_dunder_import"}, ("r_import_alias_base", "Kernel.run"): {"base_import"}, ("r_getattr_alias", "run"): {"getattr_alias"}, # Inheritance @@ -1597,12 +1813,22 @@ def run(): # Import-time code ("r_headers", "run"): { "header_for", + "header_guard", "header_if", "header_match", + "header_pattern", "header_try", + "header_trystar", "header_while", "header_with", }, + ("r_blocks", "run"): { + "block_body", + "block_case", + "block_else", + "block_finally", + "block_handler", + }, ("r_class_header", "Kernel.run"): {"class_header"}, ("r_fill_global", "run"): {"fill_global"}, ("r_setup_call", "run"): {"setup_call"}, From c622fc22f5f88369536b10a51d6a0f4a0e413efc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 04:31:16 -0500 Subject: [PATCH 10/14] NZ G8-x2: run each walk lookup once, and give every walk branch a root (#1567) The delta review of 005844bd7 asked for one change and offered four more. This commit takes all five. - Class-body globals (required). An assignment in a class body to a name that the body declares global rebinds the module's name. The walk now follows its value whether or not it reaches the class (r_class_global, r_class_nested_fill). - Misses that both walks shared. The walk now follows: - a class-level import that the class body, a default value or a decorator reads (r_class_import_scope); - a nested class body's changes in place (r_class_nested_fill); - a for or with target that stores into a reached name's item (r_for_store); - a change inside a comprehension (r_comprehension_fill); - an inherited __match_args__ (r_match_inherited); - __import__ with a level (r_import_level); - import_module relative to __spec__.parent (r_import_spec); - these callables under another module-level name (r_import_renamed); - += on a name (r_augassign). - Branches that no root pinned. 005844bd7's message said "Each branch now has one, and a root fails when it goes". That was wrong. The review found seven branches whose knockout failed no root, and this commit's own sweep found an eighth. Each live one now has a root: - del in a class body (r_body_del); - __import__'s top package (r_dunder_import_top); - a class member's change (r_class_member_change); - the global filter for a nested function (r_global); - changes in every module the walk has read: r_registry's registrar now holds no reached item, so only that pass reaches its change; - comprehension targets as locals (r_comprehension_decoy); - import_module relative to __name__ (r_import_name), from a plain module, whose __name__ differs from its __package__. The redundant pop of global names in _own_names goes, since _visits already drops them. - Cost. Each lookup now runs once per walk (_Walk._settled, which follows Tarjan's strongly connected components). That covers what a name binds in a module, what a constant stands for, a package's lazy exports, and a class's inherited __match_args__. Lookups in a cycle settle on the same items, whichever the walk met first. - Nine Cost roots pin it: - diamonds 22 levels deep of star imports, re-exports, constants, bases and import alternatives; - a clique of 8 modules that star-import each other; - a cycle of 3 lazy packages; - two cycles of constants. - On the same shapes (CPython 3.14.7), the old lookups ran past a 20 s limit on the 8-module clique, and took 11.8 s, 11.7 s and 19.2 s on the 22-level diamonds of constants, re-exports and bases. With this commit each Cost root takes at most 4.1 ms on 3.14.7 and 4.5 ms on 3.13.15 (the fastest of 5 runs). - Two new tests check that each lookup runs once and returns each item once, and that each lookup the walk settles holds what a fresh walk's same lookup finds. Checks outside pytest, on CPython 3.14.7 and 3.13.15: - The walk as committed passes every check on all 115 roots: the targets, the run-once check and the fresh-walk check. - Each of 125 one-edit knockouts of the walk fails at least one root by one of those checks. Both versions agree on every knockout. - A differential against the same walk with the old lookups found no difference in items, sites or cuts (3.14.7). It covered NZ's 8 kernels, with and without the cut (16 walks), the 106 roots before the Cost roots, and 200 seeded composites of them. - On NZ's 8 kernels, with and without the cut, the module sets equal d403cce28's (3.14.7). Local run of the three test files that exercise the walk and the income kernels (test_transport_income_evidence_kernels.py, test_nz_transport_toy_graph.py, test_nz_spec_package.py) on this tree, CPython 3.14.7: 1,564 passed in 130.4 s. Co-Authored-By: Claude Opus 5.5 --- .../test_transport_income_evidence_kernels.py | 56 +- test_support/microcosm_build/static_reach.py | 513 ++++++++++---- .../microcosm_build/static_reach_cases.py | 651 +++++++++++++++++- 3 files changed, 1067 insertions(+), 153 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py index eb4af4a8e3..e2bf5bbe8b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py @@ -16,6 +16,7 @@ import os import random import sys +from collections import Counter from decimal import Decimal from fractions import Fraction from hashlib import sha256 @@ -87,7 +88,7 @@ from microcosm.graph.canonical import canonical_json from microcosm.graph.executor import Population, _project_context from microcosm.graph.kernel import ArtifactValue, NumericScope -from test_support.microcosm_build import static_reach_cases +from test_support.microcosm_build import static_reach, static_reach_cases from test_support.microcosm_build.chronicle_feed import write_pinned_artifact from test_support.microcosm_build.first_party_imports import hashed_module_names from test_support.microcosm_build.static_reach import Cut, reach @@ -2059,6 +2060,59 @@ def test_property_composing_roots_loses_no_target(walk_case_tree, picked) -> Non ) +def test_the_walk_runs_each_lookup_once(walk_case_tree, monkeypatch) -> None: + # Each lookup runs once per walk and hands back each item once. A walk + # that ran a lookup afresh wherever it met it would take time exponential + # in a diamond's depth, or factorial in a clique's size, on the Cost roots. + _, paths = walk_case_tree + settled = static_reach._Walk._settled + runs: Counter[tuple[object, ...]] = Counter() + + def counted(self, key, compute, cut=()): + def counting(): + runs[key] += 1 + assert runs[key] == 1, key + return compute() + + found = settled(self, key, counting, cut) + assert len(found) == len(set(found)), key + return found + + monkeypatch.setattr(static_reach._Walk, "_settled", counted) + for module, qualname in _WALK_CASE_ROOTS: + runs.clear() + reach(f"{static_reach_cases.PACKAGE}.{module}", qualname, paths=paths) + assert runs, (module, qualname) + + +def _fresh_lookup(walk, key): + kind, *rest = key + if kind == "resolve": + return walk.resolve(*rest) + if kind == "alias": + return walk.aliased(*rest) + if kind == "match": + return walk.match_args(*rest) + # A package's lazy exports, which exported() looks up. + walk.exported(*rest) + return list(walk._done[key]) + + +def test_a_settled_lookup_holds_what_a_fresh_walk_finds(walk_case_tree) -> None: + # Lookups in a cycle settle on the same items, whichever the walk met + # first (r_star_clique, r_lazy_cycle, r_alias_cycle). So each lookup a + # root's walk settles holds what the same lookup finds first in a fresh + # walk. + _, paths = walk_case_tree + for module, qualname in _WALK_CASE_ROOTS: + walk = static_reach._Walk(paths, ()) + walk.run(f"{static_reach_cases.PACKAGE}.{module}", qualname) + for key, items in walk._done.items(): + fresh = static_reach._Walk(paths, ()) + fresh.modules = walk.modules + assert set(_fresh_lookup(fresh, key)) == set(items), (module, key) + + def test_the_module_stays_outside_the_batterys_binding_closure() -> None: # The registry and the composer import this module, and no battery # binding's identity covers it or them, so adding the kernels re-keys no diff --git a/test_support/microcosm_build/static_reach.py b/test_support/microcosm_build/static_reach.py index ac7b34af4d..c0f4dbf0ec 100644 --- a/test_support/microcosm_build/static_reach.py +++ b/test_support/microcosm_build/static_reach.py @@ -19,14 +19,16 @@ ``__getattr__``, defined or assigned. Every form of assignment binds: a ``for`` or ``with`` target stands for what the loop iterates or the statement enters, a ``match`` capture for the subject, and a walrus - target for its value. + target for its value. A name that a class body declares ``global`` binds + at module level, whether or not the walk reaches the class. - Scopes. Each scope binds only its own names, as Python scopes them. A comprehension, lambda or nested function keeps its names to itself, a class body's names stay out of its methods, and ``global`` and ``nonlocal`` send a name outward. A class body reads a name it binds from its own namespace - only once a statement at the body's top level has bound it. Before that, or - when only a branch, a loop or a bare annotation binds it, Python reads the - module's binding, so the walk follows that one too. + only once a statement at the body's top level has bound it. Before that, + when only a branch, a loop or a bare annotation binds it, or when a ``del`` + can undo it, Python reads the module's binding, so the walk follows that + one too. - Aliases. A module-level constant stands for each module, class or function its value names: ``_helper = helper_module``, ``_Base = pkg.Base``, ``pkg.Base[int]``, either branch of a conditional, each operand of ``and`` @@ -37,7 +39,9 @@ first-party base, through aliases and subscripts (``Base[int]``). - Self and cls. In a method, a chain from ``self`` or ``cls`` continues through a module the class body imports (``self.mod.fn()``). The - attribute fallback reaches the member itself. + attribute fallback reaches the member itself. Where the class body reads + a name it imports, in the body itself, a default value or a decorator, + the name stands for that module too. - Implicit calls. A referenced class reaches every dunder and sunder method it defines, since Python calls them without naming them: operators, conversions, containers, descriptors, copying, pickling and Enum's @@ -49,14 +53,18 @@ expression statement (``_setup()``, ``setattr(Kernel, ...)``), a ``del``, and a store to an item or an attribute. It also follows each compound statement's header there: an ``if`` or ``while`` test, what a ``for`` - iterates or a ``with`` enters, a ``match`` statement's subject, patterns - and guards, and each ``except`` or ``except*`` clause's types. + iterates or a ``with`` enters and the target each stores to, a ``match`` + statement's subject, patterns and guards, and each ``except`` or + ``except*`` clause's types. - Changes by name. In any module the walk has read, a statement at module - level or in any class body that changes a reached name in place runs with - it: a method call on the name (``_TABLE.update(...)``, also inside an - assignment), a store to or a ``del`` of its item or attribute, or a call - statement that takes it as an argument (``_fill(_TABLE)``), also through a - constant that aliases it (``_VIEW = _TABLE``). + level or in any class body, a nested class's too, that changes a reached + name in place runs with it: a method call on the name + (``_TABLE.update(...)``, also inside an assignment or a comprehension), a + store to or a ``del`` of its item or attribute (a ``for`` or ``with`` + target too, with what it stores), ``+=`` on the name, or a call statement + that takes it as an argument (``_fill(_TABLE)``), also through a constant + that aliases it (``_VIEW = _TABLE``). A nested class runs whole, so that + its own names resolve. - Attribute fallback. An attribute of an object whose type the walk can't tell reaches every same-named member of each class the walk has referenced, and every same-named definition in each module it has seen @@ -65,16 +73,31 @@ - Literal names. ``getattr``, ``hasattr``, ``setattr`` and ``delattr`` read the attribute a literal names, and so do ``attrgetter``, ``methodcaller`` and a ``match`` class pattern (its keyword names, and an explicit - ``__match_args__`` for positional patterns). ``import_module`` and - ``__import__`` reach the module a literal names as a value: - ``import_module`` relative to a literal package, ``__package__`` or - ``__name__``, and ``__import__`` with its top package as well. A - module-level constant that holds only strings counts as a literal. + ``__match_args__`` for positional patterns, the class's own or a base's). + ``import_module`` and ``__import__`` reach the module a literal names as a + value: ``import_module`` relative to a literal package, ``__package__``, + ``__spec__.parent`` or ``__name__``, and ``__import__`` with its top + package as well, counting a ``level`` above zero from the calling + module's package. Each of these callables counts under a module-level + name for it too (``load = import_module``). A module-level constant that + holds only strings counts as a literal. + +Each lookup runs once per walk: what a name binds in a module, what a +constant stands for, a package's lazy exports, and a class's inherited +``__match_args__``. Lookups that wait on each other in a cycle settle on the +same items, whichever the walk met first. A walk that ran each lookup afresh +wherever it met it would take time exponential in the depth of a diamond of +star imports, re-exports, constants or bases, and factorial in the size of a +clique of star imports: the Cost cases in ``static_reach_cases``. A round +counts only what it adds: an item, a module the walk uses as a value, or a +statement the walk hasn't run. The walk ends at the first round that adds +nothing. What it can't see: - a name built at run time (``getattr`` or ``import_module`` with a computed - name, ``globals()``, ``exec``); + name, ``globals()``, ``exec``), and those callables under a name that a + function binds; - code reached by registration rather than by name: a function that a decorator records for a framework or a registry to call (``singledispatch``, pydantic's validators); @@ -104,8 +127,9 @@ from __future__ import annotations import ast +import math from collections import defaultdict -from collections.abc import Iterable, Iterator, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping from dataclasses import dataclass from pathlib import Path @@ -118,6 +142,8 @@ ATTRIBUTE_NAMERS = frozenset({"attrgetter", "methodcaller"}) #: Callables whose literal first argument names a module to import. IMPORTERS = frozenset({"import_module", "__import__"}) +#: The callables above, which a module-level name can stand for. +_CALLABLES = ATTRIBUTE_BUILTINS | ATTRIBUTE_NAMERS | IMPORTERS #: The most rounds a walk may take to settle. Each round reaches a new item or #: value module, or runs a new statement, so a walk that needs more has a bug. MAX_ROUNDS = 10_000 @@ -208,9 +234,15 @@ def _header(node: ast.stmt) -> list[ast.AST]: if isinstance(node, ast.If | ast.While): return [node.test] if isinstance(node, ast.For | ast.AsyncFor): - return [node.iter] + # Each pass stores to the target: ``for _TABLE[key] in rows``. + return [node.iter, node.target] if isinstance(node, ast.With | ast.AsyncWith): - return [item.context_expr for item in node.items] + return [ + part + for item in node.items + for part in (item.context_expr, item.optional_vars) + if part is not None + ] if isinstance(node, ast.Match): parts: list[ast.AST] = [node.subject] for case in node.cases: @@ -324,15 +356,18 @@ def _mutations(part: ast.AST) -> Iterator[str]: """The names one statement (or one header) can change in place. ``_TABLE`` for ``_TABLE.update(...)``, ``_ROW = _TABLE.setdefault(...)``, - ``_TABLE[key] = value``, ``_TABLE.field += 1``, ``del _TABLE[key]``, and - for a call statement each name its arguments start from: - ``_fill(_TABLE)`` or ``setattr(_TABLE, ...)``. + ``_TABLE[key] = value``, ``_TABLE.field += 1``, ``_ROWS += rows`` (a + list extends in place), ``del _TABLE[key]``, any of these inside a + comprehension, and for a call statement each name its arguments start + from: ``_fill(_TABLE)`` or ``setattr(_TABLE, ...)``. """ calls = [] if isinstance(part, ast.Expr) and isinstance(part.value, ast.Call): calls.append(part.value) - for node in _same_scope([part]): + if isinstance(part, ast.AugAssign) and isinstance(part.target, ast.Name): + yield part.target.id + for node in _runs_with([part]): if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): name = _root_name(node.func.value) if name is not None: @@ -368,6 +403,9 @@ def __init__(self, name: str, path: Path) -> None: self.mutations: list[tuple[str, ast.AST, ast.ClassDef | None]] = [] #: What the module runs at import beyond its bindings. self.effects: list[ast.AST] = list(_effects(tree.body)) + #: A value that a class body binds to a name it declares global -> + #: that class body. + self.class_values: dict[int, ast.ClassDef] = {} for node in _statements(tree.body): if isinstance(node, ast.Import | ast.ImportFrom): for bound, binding in _bindings(self, node).items(): @@ -379,9 +417,33 @@ def __init__(self, name: str, path: Path) -> None: functions[node.name].append(node) elif isinstance(node, ast.ClassDef): classes[node.name].append(node) - for child in _statements(node.body): - if not isinstance(child, (*_DEFS, ast.Import, ast.ImportFrom)): - self._changes(child, node) + for body in _class_bodies(node): + declared = _declared_globals(body) + for child in _statements(body.body): + # A name the body declares global binds at module level. + if isinstance(child, ast.Import | ast.ImportFrom): + for bound, binding in _bindings(self, child).items(): + if bound in declared: + imports[bound].append(binding) + continue + if isinstance(child, _DEFS): + if child.name in declared: + table = ( + classes + if isinstance(child, ast.ClassDef) + else functions + ) + table[child.name].append(child) + continue + for target, value in _assignments(child): + for bound in _stored(target): + if bound in declared: + _add(constants[bound], value) + self.class_values[id(value)] = body + if body is node: + self._changes(child, node) + else: + self._changes(child, None, whole=body) else: for target, value in _assignments(node): for bound in _stored(target): @@ -391,10 +453,35 @@ def __init__(self, name: str, path: Path) -> None: self.functions = dict(functions) self.classes = dict(classes) self.constants = dict(constants) - - def _changes(self, node: ast.stmt, body: ast.ClassDef | None) -> None: - for part in _parts(node): - for bound in dict.fromkeys(_mutations(part)): + #: Module-level names for the callables in ``_CALLABLES``: + #: ``from importlib import import_module as load``, ``_get = getattr``. + self.callees: dict[str, str] = {} + for bound, bindings in self.imports.items(): + for _, attribute in bindings: + if attribute in _CALLABLES: + self.callees[bound] = attribute + for bound, values in self.constants.items(): + for value in values: + if (callee := _callee(value)) in _CALLABLES: + self.callees[bound] = callee + + def _changes( + self, + node: ast.stmt, + body: ast.ClassDef | None, + *, + whole: ast.ClassDef | None = None, + ) -> None: + # A compound statement's header runs as one: ``for _TABLE[key] in + # rows`` stores what ``rows`` holds. + parts = _parts(node) + for bound in dict.fromkeys(name for part in parts for name in _mutations(part)): + if whole is not None: + # A statement in a nested class: the walk runs that class + # whole, so that the names it binds resolve. + self.mutations.append((bound, whole, None)) + continue + for part in parts: self.mutations.append((bound, part, body)) def absolute(self, level: int, module: str | None) -> str: @@ -454,13 +541,31 @@ def _members(module: _Module, node: ast.ClassDef) -> dict[str, list[ast.AST]]: for target, _ in _assignments(child): for bound in _stored(target): _add(out[bound], child) - declared = { + declared = _declared_globals(node) + return {name: nodes for name, nodes in out.items() if name not in declared} + + +def _declared_globals(node: ast.AST) -> set[str]: + """The names a scope declares ``global``.""" + + return { name for child in _same_scope(_inner_parts(node)) if isinstance(child, ast.Global) for name in child.names } - return {name: nodes for name, nodes in out.items() if name not in declared} + + +def _class_bodies(node: ast.ClassDef) -> Iterator[ast.ClassDef]: + """``node``, and each class nested in it, whose body runs as ``node``'s does. + + A class inside a method runs only when the method does, so it stays out. + """ + + yield node + for child in _statements(node.body): + if isinstance(child, ast.ClassDef): + yield from _class_bodies(child) def _binds(node: ast.stmt) -> set[str]: @@ -584,6 +689,23 @@ def _same_scope(nodes: Iterable[ast.AST]) -> Iterator[ast.AST]: stack.extend(ast.iter_child_nodes(node)) +def _runs_with(nodes: Iterable[ast.AST]) -> Iterator[ast.AST]: + """Every node under ``nodes`` that can run when they run. + + That takes in each comprehension there (a generator expression runs as + something consumes it), but no function, lambda or class body. + """ + + stack = list(nodes) + while stack: + node = stack.pop() + yield node + if isinstance(node, _SCOPES) and not isinstance(node, _COMPREHENSIONS): + stack.extend(_outer_parts(node)) + else: + stack.extend(ast.iter_child_nodes(node)) + + def _walrus_targets(node: ast.AST) -> Iterator[str]: """A comprehension's walrus targets, which bind in its enclosing scope.""" @@ -597,10 +719,12 @@ def _walrus_targets(node: ast.AST) -> Iterator[str]: def _own_names( module: _Module, scope: ast.AST ) -> tuple[dict[str, tuple[Binding, ...]], set[str]]: - """The names ``scope`` binds for itself, and the names it declares global. + """The names ``scope`` binds, and the names it declares global. Each name maps to every import that binds it there: a name that a ``try`` - or an ``if`` imports two ways keeps both. + or an ``if`` imports two ways keeps both. A ``nonlocal`` name stays out, + so that it keeps the enclosing function's binding; :func:`_visits` drops + each global one. """ names: dict[str, tuple[Binding, ...]] = {} @@ -638,7 +762,7 @@ def local(name: str) -> None: elif isinstance(node, _COMPREHENSIONS): for target in _walrus_targets(node): local(target) - for name in declared_global | declared_nonlocal: + for name in declared_nonlocal: names.pop(name, None) return names, declared_global @@ -690,31 +814,32 @@ def _visits( stack.extend((child, here, closure) for child in reversed(children)) -def _named(value: ast.AST) -> Iterator[ast.AST]: +def _named(value: ast.AST, callees: Mapping[str, str]) -> Iterator[ast.AST]: """The names and dotted names an expression's value can be. ``getattr(pkg, "mod")`` counts as ``pkg.mod``, and a literal that ``import_module`` or ``__import__`` takes comes back as that constant. + ``callees`` holds the module's names for those callables. """ if isinstance(value, ast.Name | ast.Attribute): yield value elif isinstance(value, ast.Subscript): - yield from _named(value.value) + yield from _named(value.value, callees) elif isinstance(value, ast.IfExp): - yield from _named(value.body) - yield from _named(value.orelse) + yield from _named(value.body, callees) + yield from _named(value.orelse, callees) elif isinstance(value, ast.BoolOp): for part in value.values: - yield from _named(part) + yield from _named(part, callees) elif isinstance(value, ast.NamedExpr | ast.Starred): - yield from _named(value.value) + yield from _named(value.value, callees) elif isinstance(value, ast.Tuple | ast.List | ast.Set): # ``a, b = first, second``: over-approximate by element. for part in value.elts: - yield from _named(part) + yield from _named(part, callees) elif isinstance(value, ast.Call): - callee = _callee(value.func) + callee = _callee(value.func, callees) texts = [_literal(argument) for argument in value.args] if callee == "getattr" and len(texts) >= 2 and texts[1]: read = ast.Attribute(value.args[0], texts[1], ast.Load()) @@ -725,14 +850,24 @@ def _named(value: ast.AST) -> Iterator[ast.AST]: yield ast.Constant(texts[0].partition(".")[0]) -def _callee(func: ast.AST) -> str | None: +def _callee(func: ast.AST, callees: Mapping[str, str] | None = None) -> str | None: + """The name a call calls, or the callable a module-level name stands for.""" + if isinstance(func, ast.Name): - return func.id + return (callees or {}).get(func.id, func.id) if isinstance(func, ast.Attribute): return func.attr return None +def _argument(call: ast.Call, position: int, keyword: str) -> ast.AST | None: + """A call's argument, by its position or its keyword.""" + + if len(call.args) > position: + return call.args[position] + return next((item.value for item in call.keywords if item.arg == keyword), None) + + def _literal(node: ast.AST) -> str | None: if isinstance(node, ast.Constant) and isinstance(node.value, str): return node.value @@ -765,11 +900,17 @@ def __init__(self, paths: Mapping[str, Path], cuts: Iterable[Cut]) -> None: #: A top-level class member's binding statement -> its class body. self._body_of: dict[int, ast.ClassDef] = {} self._inside: dict[str, list[str]] = {} - # The lookups in progress. A cycle of re-exports or aliases adds - # nothing the outer lookup doesn't already see. - self._resolving: set[tuple[str, str]] = set() - self._lazy: set[tuple[str, str]] = set() - self._aliasing: set[Item] = set() + # Lookups, each computed once (see _settled): what each settled one + # holds; when each unsettled one began, and its cut; the unsettled + # ones in that order; and those another lookup met unsettled. + self._done: dict[tuple[object, ...], tuple[object, ...]] = {} + self._began: dict[tuple[object, ...], int] = {} + self._cuts: dict[tuple[object, ...], tuple[object, ...]] = {} + self._open: list[tuple[object, ...]] = [] + self._met: set[tuple[object, ...]] = set() + self._begun = 0 + #: When the earliest unsettled lookup that the one under way met began. + self._low: float = math.inf def module(self, name: str) -> _Module | None: if name not in self.paths: @@ -821,31 +962,80 @@ def members(self, module: str, name: str) -> dict[str, list[ast.AST]]: self._members[key] = dict(out) return self._members[key] + def _settled[T]( + self, + key: tuple[object, ...], + compute: Callable[[], Iterable[T]], + cut: Iterable[T] = (), + ) -> list[T]: + """``compute()``'s items for ``key``, once each, computing ``key`` once. + + Lookups can wait on each other in a cycle: modules that import a name + from each other, star imports, a lazy package's modules, constants + that name each other, or bases. A lookup that meets one that hasn't + settled gets that one's ``cut``, and every lookup in the cycle + settles on the same items: what the cycle's first lookup ends with, + and each one's cut (Tarjan's strongly connected components). In a + cycle, each lookup's caller keeps what it found, back to the first + lookup, and the lookups reach each other, so each can reach every + item the first one ends with, whichever the walk met first. Each + lookup then runs once, where running it afresh at each visit costs + time exponential in a diamond's depth. + """ + + if key in self._done: + return list(self._done[key]) + if key in self._began: + self._low = min(self._low, self._began[key]) + self._met.add(key) + return list(self._cuts[key]) + began = self._began[key] = self._begun + self._begun += 1 + self._cuts[key] = tuple(cut) + self._open.append(key) + outer, self._low = self._low, began + # One copy of each: through a diamond, every path adds its own. + items = list(dict.fromkeys(compute())) + low = self._low + if low < began: + # It waits on a lookup that began earlier and hasn't settled. + self._low = min(outer, low) + return list(items) + self._low = outer + start = self._open.index(key) + members = self._open[start:] + del self._open[start:] + if len(members) > 1 or key in self._met: + items = list( + dict.fromkeys([*items, *(c for m in members for c in self._cuts[m])]) + ) + for member in members: + del self._began[member], self._cuts[member] + self._met.discard(member) + self._done[member] = tuple(items) + return list(items) + def resolve(self, module: str, name: str) -> list[Item]: """Every definition ``name`` can bind in ``module``'s top-level scope.""" found = self.module(module) - key = (module, name) - if found is None or key in self._resolving: + if found is None: return [] - self._resolving.add(key) - try: - items: list[Item] = [ - (module, kind, name) - for kind, table in ( - ("function", found.functions), - ("class", found.classes), - ("constant", found.constants), - ) - if name in table - ] + + def compute() -> Iterator[Item]: + for kind, table in ( + ("function", found.functions), + ("class", found.classes), + ("constant", found.constants), + ): + if name in table: + yield (module, kind, name) for binding in found.imports.get(name, ()): - items.extend(self.resolve_import(*binding)) + yield from self.resolve_import(*binding) for star in found.star_imports: - items.extend(self.exported(star, name)) - finally: - self._resolving.discard(key) - return items + yield from self.exported(star, name) + + return self._settled(("resolve", module, name), compute) def resolve_import(self, target: str, attribute: str | None) -> list[Item]: """What ``import target`` or ``from target import attribute`` binds.""" @@ -862,7 +1052,8 @@ def exported(self, module: str, name: str) -> list[Item]: items.append((f"{module}.{name}", "module", None)) items.extend(self.resolve(module, name)) found = self.module(module) - if items or found is None: + if items or found is None or name == "__getattr__": + # Python finds a module's __getattr__ in its namespace only. return items # A lazy export: whatever binds the module's __getattr__ and, for a # package, the name wherever the package's own modules bind it. @@ -870,15 +1061,18 @@ def exported(self, module: str, name: str) -> list[Item]: if not getters: return items items.extend(getters) - key = (module, name) - if found.is_package and key not in self._lazy: - self._lazy.add(key) - try: - for inner in self.inside(module): - items.extend(self.resolve(inner, name)) - finally: - self._lazy.discard(key) - return items + if found.is_package: + items.extend( + self._settled( + ("lazy", module, name), + lambda: ( + item + for inner in self.inside(module) + for item in self.resolve(inner, name) + ), + ) + ) + return list(dict.fromkeys(items)) def resolve_expression(self, module: str, node: ast.AST) -> list[Item]: """What a name or dotted name in ``module``'s top-level scope names.""" @@ -907,22 +1101,24 @@ def aliased(self, item: Item) -> list[Item]: A constant whose values name nothing stands for itself. """ - if item[1] != "constant" or item in self._aliasing: + if item[1] != "constant": return [item] - self._aliasing.add(item) - found: list[Item] = [] - try: - for value in self.module(item[0]).constants[item[2]]: - for named in _named(value): + + def compute() -> list[Item]: + module = self.module(item[0]) + found: list[Item] = [] + for value in module.constants[item[2]]: + for named in _named(value, module.callees): if isinstance(named, ast.Constant): if self.is_module(named.value): found.append((named.value, "module", None)) continue for resolved in self.resolve_expression(item[0], named): found.extend(self.aliased(resolved)) - finally: - self._aliasing.discard(item) - return found or [item] + return found or [item] + + # In a cycle of constants, each stands for every one in it as well. + return self._settled(("alias", item), compute, cut=[item]) def member_imports(self, module: str, owner: str, member: str) -> list[Item]: """What a top-level class member is where the class body imports it. @@ -994,6 +1190,8 @@ def lookup(name: str, here: Scope, read: ast.AST) -> list[Item]: items.extend(self.resolve(module, name)) else: items.append((module, "member", f"{binding[1]}.{name}")) + # Where the class body imports the name, the module too. + items.extend(self.member_imports(module, binding[1], name)) if body is None or not _bound_before(body, name, read): items.extend(self.resolve(module, name)) return items @@ -1075,7 +1273,7 @@ def attribute(node: ast.Attribute, here: Scope, at: str) -> None: self.values.update(start[0] for start in starts) def call(node: ast.Call, here: Scope, at: str) -> None: - callee = _callee(node.func) + callee = _callee(node.func, found.callees) if callee in ATTRIBUTE_BUILTINS and len(node.args) >= 2: for text in literal_names(node.args[1], here): read = ast.Attribute(node.args[0], text, ast.Load()) @@ -1086,13 +1284,14 @@ def call(node: ast.Call, here: Scope, at: str) -> None: self.attributes.update(text.split(".")) elif callee in IMPORTERS and node.args: for text in literal_names(node.args[0], here): - for name in self.absolute(module, text, node, here): + for name in self.absolute(module, callee, text, node): # import_module returns the module it names; - # __import__ returns the top package, or with a - # fromlist the module itself. + # __import__ returns the top package (counted from + # its level), or with a fromlist the module itself. named = [name] if callee == "__import__": - named.append(name.partition(".")[0]) + top = text.partition(".")[0] + named.append(name[: len(name) - len(text)] + top) for value in named: if self.is_module(value): self.push((value, "module", None), at) @@ -1127,35 +1326,28 @@ def match_class(node: ast.MatchClass, here: Scope) -> None: match_class(child, here) def absolute( - self, module: str, text: str, node: ast.Call, here: Scope + self, module: str, callee: str, text: str, node: ast.Call ) -> list[str]: """The module names an import call's literal can mean.""" - if not text.startswith("."): - return [text] - level = len(text) - len(text.lstrip(".")) - rest = text[level:] - if len(node.args) > 1: - package_node = node.args[1] - else: - package_node = next( - ( - keyword.value - for keyword in node.keywords - if keyword.arg == "package" - ), - None, - ) - if package_node is None: - return [] found = self.module(module) - if isinstance(package_node, ast.Name) and package_node.id == "__package__": + if callee == "__import__": + # __import__(name, globals, locals, fromlist, level) counts a level + # above zero from the calling module's package. + level_node = _argument(node, 4, "level") + level = 0 + if isinstance(level_node, ast.Constant) and type(level_node.value) is int: + level = level_node.value + if level <= 0: + return [text] + rest = text packages = [found.package] - elif isinstance(package_node, ast.Name) and package_node.id == "__name__": - packages = [module] else: - text_node = _literal(package_node) - packages = [text_node] if text_node is not None else [] + if not text.startswith("."): + return [text] + level = len(text) - len(text.lstrip(".")) + rest = text[level:] + packages = self.packages(module, _argument(node, 1, "package")) names = [] for package in packages: parts = package.split(".") @@ -1165,17 +1357,66 @@ def absolute( names.append(f"{base}.{rest}" if rest else base) return names + def packages(self, module: str, node: ast.AST | None) -> list[str]: + """The packages an ``import_module`` package argument can name.""" + + found = self.module(module) + if isinstance(node, ast.Name) and node.id == "__package__": + return [found.package] + if isinstance(node, ast.Name) and node.id == "__name__": + return [module] + if ( + isinstance(node, ast.Attribute) + and node.attr == "parent" + and isinstance(node.value, ast.Name) + and node.value.id == "__spec__" + ): + # A module's spec names its package as its parent. + return [found.package] + text = None if node is None else _literal(node) + return [] if text is None else [text] + + def bases(self, module: str, name: str) -> list[Item]: + """Each first-party class a top-level class names as a base. + + The walk reads a base through aliases and subscripts (``Base[int]``). + """ + + found = self.module(module) + out: list[Item] = [] + for node in found.classes.get(name, ()): + for base in node.bases: + while isinstance(base, ast.Subscript): + base = base.value + for resolved in self.resolve_expression(module, base): + for start in self.aliased(resolved): + if start[1] == "class": + out.append(start) + return out + def match_args(self, item: Item) -> list[str]: """The attributes a class's explicit ``__match_args__`` names. - A positional ``match`` class pattern reads them. A dataclass's own - ``__match_args__`` names its fields, which hold data, not code. + A positional ``match`` class pattern reads them. A class that doesn't + set ``__match_args__`` takes each base's, as attribute lookup does. A + dataclass's own ``__match_args__`` names its fields, which hold data, + not code. """ module, _, name = item + nodes = self.members(module, name).get("__match_args__") + if nodes is None: + return self._settled( + ("match", item), + lambda: ( + text + for base in self.bases(module, name) + for text in self.match_args(base) + ), + ) return [ text - for node in self.members(module, name).get("__match_args__", ()) + for node in nodes for _, value in _assignments(node) for element in getattr(value, "elts", ()) if (text := _literal(element)) is not None @@ -1191,7 +1432,21 @@ def process(self, item: Item) -> None: self.scan(module, node, f"{module}.{name}", qualname=name) elif kind == "constant": for value in found.constants.get(name, ()): - self.scan(module, value, f"{module}.{name}=") + # A class body that declares the name global binds it there, + # in the class's scope. + body = found.class_values.get(id(value)) + owner = None + if body is not None and any( + node is body for node in found.classes.get(body.name, ()) + ): + owner = body.name + self.scan( + module, + value, + f"{module}.{name}=", + owner=owner, + body=None if owner is None else body, + ) elif kind == "class": self.classes.add((module, name)) for node in found.classes.get(name, ()): @@ -1234,17 +1489,11 @@ def process(self, item: Item) -> None: if nodes: return # Inherited: the same name on each first-party base class. - for node in found.classes[owner]: - for base in node.bases: - while isinstance(base, ast.Subscript): - base = base.value - for resolved in self.resolve_expression(module, base): - for start in self.aliased(resolved): - if start[1] == "class": - self.push( - (start[0], "member", f"{start[2]}.{member}"), - f"{module}.{name}(base)", - ) + for base in self.bases(module, owner): + self.push( + (base[0], "member", f"{base[2]}.{member}"), + f"{module}.{name}(base)", + ) def is_reached(self, item: Item) -> bool: if item[1] == "module": diff --git a/test_support/microcosm_build/static_reach_cases.py b/test_support/microcosm_build/static_reach_cases.py index 2ff68ebcbf..3758d40533 100644 --- a/test_support/microcosm_build/static_reach_cases.py +++ b/test_support/microcosm_build/static_reach_cases.py @@ -32,9 +32,13 @@ "add", "alias", "alias_base", + "alias_cycle", + "alias_diamond", + "alias_modules", "alternative", "alternative_first", "attrgetter", + "augassign", "base_attr", "base_dunder_import", "base_else", @@ -56,6 +60,7 @@ "body_before", "body_conditional", "body_decorator", + "body_del", "body_hidden", "body_nested", "class_body_update", @@ -64,10 +69,23 @@ "class_fill_item", "class_fill_method", "class_fill_view", + "class_global", + "class_global_alias", + "class_global_class", + "class_global_def", + "class_global_hidden", + "class_global_import", "class_header", "class_import", + "class_import_body", + "class_import_decorator", + "class_import_default", + "class_member_change", + "class_nested_fill", "class_scope", "comprehension", + "comprehension_fill", + "comprehension_iter", "conditional", "decorator", "default", @@ -76,10 +94,12 @@ "delattr", "delattr_patch", "dotted", + "dunder_top", "factory", "fill_call", "fill_global", "float", + "for_store", "for_target", "getattr_alias", "getattr_constant", @@ -87,6 +107,7 @@ "go", "hasattr", "header_for", + "header_for_target", "header_guard", "header_if", "header_match", @@ -95,17 +116,27 @@ "header_trystar", "header_while", "header_with", + "header_with_target", + "import_diamond", "import_from", + "import_level", "import_module", + "import_name", "import_relative", + "import_renamed", + "import_renamed_alias", + "import_spec", "lambda", "lambda_shadow", "lazy", "lazy_assigned", + "lazy_cycle", "local", "local_alternative", "local_alternative_first", "match_capture", + "match_diamond", + "match_inherited", "match_keyword", "match_positional", "match_rest", @@ -131,6 +162,8 @@ "shadowed", "sibling", "star", + "star_clique", + "star_diamond", "star_lazy", "store_hidden", "subclass", @@ -140,13 +173,15 @@ "update", "value", "walrus", + "with_store", "with_target", ) #: Targets a case binds so that a wrong walk would reach them. No walk #: reaches them and no run reads them: t_shadowed behind a name the class -#: body rebinds (r_class_decorator), and t_body_hidden behind the function's -#: import of nest_p, which the class in r_body_nested never reads. +#: body rebinds (r_class_decorator) and behind a comprehension's target +#: (r_comprehension_decoy), and t_body_hidden behind the function's import of +#: nest_p, which the class in r_body_nested never reads. DECOYS = frozenset({"body_hidden", "shadowed"}) #: Each target's source: reads of X or ROWS are recorded, anything else fails. @@ -389,6 +424,56 @@ def run(): def run(): return VALUE """, + # A name a class body declares global binds at module level: through an + # assignment, an alias of a module, an import, a def and a class there. + # In _Setup, a class nothing reaches, the assignment reads a name the + # class body bound. + "cglob_alias": """ + from microcosm.walkcases import t_class_global_alias + + + def fn(): + return t_class_global_alias.X + """, + "cglob_import": """ + from microcosm.walkcases import t_class_global_import + + + def fn(): + return t_class_global_import.X + """, + "r_class_global": """ + from microcosm.walkcases import ( + cglob_alias, + t_class_global, + t_class_global_class, + t_class_global_def, + t_class_global_hidden, + ) + + + class Kernel: + global _CACHE, _IMPL, _probe, Probe, imported + _CACHE = t_class_global.X + _IMPL = cglob_alias + import microcosm.walkcases.cglob_import as imported + + def _probe(): + return t_class_global_def.X + + class Probe: + VALUE = t_class_global_class.X + + def run(self): + found = _CACHE, _IMPL.fn(), imported.fn(), _probe() + return found, Probe.VALUE, _ROWS + + + class _Setup: + global _ROWS + _source = t_class_global_hidden + _ROWS = _source.ROWS + """, # A package's lazy export, through its module-level __getattr__. "lazy.__init__": """ def __getattr__(name): @@ -483,6 +568,15 @@ def run(): values = [target for target in range(3)] return target.X, values """, + # ... even where the module binds the same name to another module. + "r_comprehension_decoy": """ + from microcosm.walkcases import t_comprehension_iter + from microcosm.walkcases import t_shadowed as row + + + def run(): + return [row.X for row in (t_comprehension_iter,)] + """, # A local import binds only in its own function, not in a sibling. "r_sibling": """ from microcosm.walkcases import t_sibling as target @@ -575,7 +669,8 @@ def run(self): """, # Before the body binds a name, Python reads the module's binding: a read # above the binding, a binding under an if that doesn't run, a bare - # annotation (which binds nothing), and a decorator above the binding. + # annotation (which binds nothing), a decorator above the binding, and a + # read after a del of the binding. "r_body_before": """ from microcosm.walkcases import t_body_before as target @@ -631,6 +726,18 @@ def run(self): target = None """, + "r_body_del": """ + from microcosm.walkcases import t_body_del as target + + + class Kernel: + target = None + del target + seen = target.X + + def run(self): + return self.seen + """, # A class inside a function reads such a name from the module too, never # from the function, so nest_p stays unreached. "r_body_nested": """ @@ -967,8 +1074,8 @@ class Kernel(Base[int]): pass """, # -- Self and cls ----------------------------------------------------------- - # A method called through self. (The class-attribute fallback reaches - # it as well.) + # A method called through self, which the class-attribute fallback + # reaches. "r_self_method": """ from microcosm.walkcases import t_self_method @@ -995,6 +1102,42 @@ class Kernel: def run(self): return self.mod.fn() """, + # ... and read where the class body runs: in the body itself, in a + # decorator and in a default value. + "cimp_body": """ + from microcosm.walkcases import t_class_import_body + + + def fn(): + return t_class_import_body.X + """, + "cimp_decorator": """ + from microcosm.walkcases import t_class_import_decorator + + + def mark(function): + function.mark = t_class_import_decorator.X + return function + """, + "cimp_default": """ + from microcosm.walkcases import t_class_import_default + + + def fn(): + return t_class_import_default.X + """, + "r_class_import_scope": """ + class Kernel: + import microcosm.walkcases.cimp_body as body_mod + import microcosm.walkcases.cimp_decorator as decorator_mod + import microcosm.walkcases.cimp_default as default_mod + + seen = body_mod.fn() + + @decorator_mod.mark + def run(self, value=default_mod.fn()): + return self.seen, value + """, # -- Implicit calls --------------------------------------------------------- # Thing's dunders count wherever Thing does. "made": """ @@ -1126,6 +1269,7 @@ def run(): from microcosm.walkcases import ( t_header_for, + t_header_for_target, t_header_guard, t_header_if, t_header_match, @@ -1134,6 +1278,7 @@ def run(): t_header_trystar, t_header_while, t_header_with, + t_header_with_target, ) if t_header_if.X: @@ -1146,6 +1291,11 @@ def run(): pass with nullcontext(t_header_with.X): pass + _SLOTS = {} + for _SLOTS[t_header_for_target.X] in (1,): + pass + with nullcontext(1) as _SLOTS[t_header_with_target.X]: + pass match t_header_match.ROWS: case {"key": found}: FOUND = found @@ -1374,25 +1524,40 @@ def run(self): return self.TABLE """, # -- Changes by name -------------------------------------------------------- - "r_update": """ + # Another module re-exports the table run() reads and changes it in place. + # The walk reads that module but reaches nothing it defines, so only the + # change by name leads to the target. + "update_table": """ + TABLE = {} + """, + "update_fill": """ from microcosm.walkcases import t_update + from microcosm.walkcases.update_table import TABLE - _TABLE = {} - _TABLE.update(t_update.ROWS) + TABLE.update(t_update.ROWS) + """, + "r_update": """ + from microcosm.walkcases.update_fill import TABLE def run(): - return _TABLE + return TABLE """, - "r_setitem": """ + "setitem_table": """ + TABLE = {} + """, + "setitem_fill": """ from microcosm.walkcases import t_setitem + from microcosm.walkcases.setitem_table import TABLE - _TABLE = {} - _TABLE["key"] = t_setitem.X + TABLE["key"] = t_setitem.X + """, + "r_setitem": """ + from microcosm.walkcases.setitem_fill import TABLE def run(): - return _TABLE + return TABLE """, # A method call inside an assignment. "r_setdefault": """ @@ -1436,7 +1601,8 @@ class Filler: def run(): return _TABLE, _ROWS """, - # Another module the walk reads fills the registry run() reads. + # Another module re-exports the registry run() reads, and a call statement + # there takes the registry as an argument. "registry": """ REGISTRY = [] """, @@ -1444,19 +1610,123 @@ def run(): from microcosm.walkcases import t_registrar from microcosm.walkcases.registry import REGISTRY - REGISTRY.append(t_registrar.X) + + def _register(registry, value): + registry.append(value) + + + _register(REGISTRY, t_registrar.X) + """, + "r_registry": """ + from microcosm.walkcases.registrar import REGISTRY + + + def run(): + return REGISTRY + """, + # ... with a for and a with statement there that store to an item of the + # table, and what each stores. + "store_table": """ + TABLE = {} + """, + "store_fill": """ + from microcosm.walkcases import t_for_store, t_with_store + from microcosm.walkcases.store_table import TABLE + + + class _Entered: + def __init__(self, value): + self.value = value + + def __enter__(self): + return self.value + + def __exit__(self, *exc): + return False + + + for TABLE["for"] in (t_for_store.X,): + pass + with _Entered(t_with_store.X) as TABLE["with"]: + pass + """, + "r_for_store": """ + from microcosm.walkcases.store_fill import TABLE + + + def run(): + return TABLE + """, + # ... with a comprehension there that changes the table. + "comp_table": """ + TABLE = {} + """, + "comp_fill": """ + from microcosm.walkcases import t_comprehension_fill + from microcosm.walkcases.comp_table import TABLE + + [TABLE.update({key: t_comprehension_fill.X}) for key in ("key",)] + """, + "r_comprehension_fill": """ + from microcosm.walkcases.comp_fill import TABLE + + + def run(): + return TABLE + """, + # += on an imported list extends that list in place. The walk reaches + # aug_fill for helper(), but the statement only binds a name there that + # nothing reads, so it doesn't run for its effect. + "aug_table": """ + ROWS = [] + """, + "aug_fill": """ + from microcosm.walkcases import t_augassign + from microcosm.walkcases.aug_table import ROWS + + ROWS += [t_augassign.X] def helper(): return 1 """, - "r_registry": """ - from microcosm.walkcases import registrar - from microcosm.walkcases.registry import REGISTRY + "r_augassign": """ + from microcosm.walkcases import aug_fill + from microcosm.walkcases.aug_table import ROWS def run(): - return REGISTRY, registrar.helper() + return ROWS, aug_fill.helper() + """, + # A class attribute that a statement in the class body changes, where the + # statement neither runs for its effect nor binds a class attribute. + "r_class_member_change": """ + from microcosm.walkcases import t_class_member_change + + + class Kernel: + TABLE = {} + assert TABLE.setdefault("key", t_class_member_change.X) + + def run(self): + return self.TABLE + """, + # A class nested in one nothing reaches runs at import too, and the names + # it binds resolve in its own body. + "r_class_nested_fill": """ + from microcosm.walkcases import t_class_nested_fill + + _TABLE = {} + + + class Outer: + class Inner: + row = t_class_nested_fill.X + _TABLE.update({"row": row}) + + + def run(): + return _TABLE """, # -- Attribute fallback ----------------------------------------------------- "r_prop": """ @@ -1669,6 +1939,30 @@ def run(): from microcosm.walkcases.made10 import Point + def run(): + match Point(): + case Point(value): + return value + return None + """, + # A class that doesn't set __match_args__ takes its base's. + "made14": """ + from microcosm.walkcases import t_match_inherited + + + class Base: + __match_args__ = ("left",) + + + class Point(Base): + @property + def left(self): + return t_match_inherited.X + """, + "r_match_inherited": """ + from microcosm.walkcases.made14 import Point + + def run(): match Point(): case Point(value): @@ -1716,6 +2010,88 @@ def run(): module = __import__("microcosm.walkcases.imp_leaf", fromlist=["fn"]) return module.fn() """, + # Without a fromlist, __import__ returns the top package, and a chain from + # there reaches a module that the package holds. + "dunder_named": """ + VALUE = 1 + """, + "r_dunder_import_top": """ + import microcosm.walkcases.t_dunder_top # so that the attribute exists + + + def run(): + package = __import__("microcosm.walkcases.dunder_named") + return package.walkcases.t_dunder_top.X + """, + # __import__ with a level above zero counts from the caller's package. + "level_mod": """ + from microcosm.walkcases import t_import_level + + + def fn(): + return t_import_level.X + """, + "r_import_level": """ + def run(): + return __import__("level_mod", globals(), None, ["fn"], 1).fn() + """, + # import_module relative to the package that __spec__ names. + "spec_mod": """ + from microcosm.walkcases import t_import_spec + + + def fn(): + return t_import_spec.X + """, + "r_import_spec": """ + import importlib + + + def run(): + return importlib.import_module(".spec_mod", package=__spec__.parent).fn() + """, + # import_module relative to the module's own name: two dots climb from + # the module to its package, one level short of where __package__ climbs. + "name_mod": """ + from microcosm.walkcases import t_import_name + + + def fn(): + return t_import_name.X + """, + "r_import_name": """ + import importlib + + + def run(): + return importlib.import_module("..name_mod", __name__).fn() + """, + # An importer under a module-level name: an import's and an assignment's. + "renamed_mod": """ + from microcosm.walkcases import t_import_renamed + + + def fn(): + return t_import_renamed.X + """, + "renamed_alias_mod": """ + from microcosm.walkcases import t_import_renamed_alias + + + def fn(): + return t_import_renamed_alias.X + """, + "r_import_renamed": """ + import importlib + from importlib import import_module as load + + _import = importlib.import_module + + + def run(): + first = load("microcosm.walkcases.renamed_mod").fn() + return first, _import("microcosm.walkcases.renamed_alias_mod").fn() + """, # -- What the walk can't see ------------------------------------------------ # A decorator records _entry, and run() calls whatever the registry holds. "reg": """ @@ -1741,6 +2117,201 @@ def run(): """, } +# -- Cost ---------------------------------------------------------------------- +# Shapes in which a walk that ran each lookup afresh wherever it met it would +# take time exponential in a diamond's depth or factorial in a clique's size. +# The walk runs each lookup once, so each of these takes milliseconds. + +#: How deep each diamond goes, and how many modules the clique holds. +_DEPTH = 22 +_CLIQUE = 8 +#: The lazy packages in the cycle. +_LAZY = 3 + +# Each level's two modules star-import both modules of the level below, and +# the bottom level imports the target. run() also calls len, which no module +# binds, so the walk looks len up through every module. +SOURCES.update( + { + f"diamond_{level}_{side}": ( + f"from {PACKAGE} import t_star_diamond # noqa: F401\n" + if level == _DEPTH - 1 + else f"from {PACKAGE}.diamond_{level + 1}_a import *\n" + f"from {PACKAGE}.diamond_{level + 1}_b import *\n" + ) + for level in range(_DEPTH) + for side in "ab" + } +) +SOURCES["r_star_diamond"] = """ + from microcosm.walkcases.diamond_0_a import * + + + def run(): + return len([t_star_diamond.X]) + """ +# Each module of a clique star-imports all the others; the first imports the +# target. +SOURCES.update( + { + f"clique_{index}": ( + f"from {PACKAGE} import t_star_clique # noqa: F401\n" if index == 0 else "" + ) + + "".join( + f"from {PACKAGE}.clique_{other} import *\n" + for other in range(_CLIQUE) + if other != index + ) + for index in range(_CLIQUE) + } +) +SOURCES["r_star_clique"] = """ + from microcosm.walkcases.clique_1 import * + + + def run(): + return len([t_star_clique.X]) + """ +# Constants in a diamond: each of a level's two names both of the next. +SOURCES["r_alias_diamond"] = ( + f"from {PACKAGE} import t_alias_diamond\n\n" + f"_V{_DEPTH}a = t_alias_diamond\n" + f"_V{_DEPTH}b = t_alias_diamond\n" + + "".join( + f"_V{level}{side} = _V{level + 1}a or _V{level + 1}b\n" + for level in range(_DEPTH - 1, -1, -1) + for side in "ab" + ) + + "\n\ndef run():\n return len([_V0a.X])\n" +) +# Modules in a diamond, each binding V to either module's V below it. +SOURCES.update( + { + f"aliases_{level}_{side}": ( + f"from {PACKAGE} import t_alias_modules\n\nV = t_alias_modules\n" + if level == _DEPTH - 1 + else f"from {PACKAGE}.aliases_{level + 1}_a import V as _a\n" + f"from {PACKAGE}.aliases_{level + 1}_b import V as _b\n\n" + "V = _a or _b\n" + ) + for level in range(_DEPTH) + for side in "ab" + } +) +SOURCES["r_alias_modules"] = """ + from microcosm.walkcases.aliases_0_a import V + + + def run(): + return len([V.X]) + """ +# Classes in a diamond, and a positional class pattern on the top one: it +# takes the bottom class's __match_args__, through each path. +SOURCES["r_match_diamond"] = ( + f"from {PACKAGE} import t_match_diamond\n\n\n" + f"class K{_DEPTH}a:\n" + ' __match_args__ = ("go",)\n\n' + " def go(self):\n" + " return t_match_diamond.X\n\n\n" + f"class K{_DEPTH}b:\n" + " pass\n" + + "".join( + f"\n\nclass K{level}{side}(K{level + 1}a, K{level + 1}b):\n pass\n" + for level in range(_DEPTH - 1, -1, -1) + for side in "ab" + ) + + "\n\ndef run():\n" + " match K0a():\n" + " case K0a(go):\n" + " return go()\n" +) +# Modules in a diamond of alternatives: each imports the target from either +# module below it. +SOURCES.update( + { + f"alts_{level}_{side}": ( + f"from {PACKAGE} import t_import_diamond # noqa: F401\n" + if level == _DEPTH - 1 + else "try:\n" + f" from {PACKAGE}.alts_{level + 1}_a import t_import_diamond\n" + "except ImportError:\n" + f" from {PACKAGE}.alts_{level + 1}_b import t_import_diamond\n" + ) + for level in range(_DEPTH) + for side in "ab" + } +) +SOURCES["r_import_diamond"] = """ + from microcosm.walkcases.alts_0_a import t_import_diamond + + + def run(): + return len([t_import_diamond.X]) + """ +# Lazy packages in a cycle: each one's __getattr__ reads its inner module, +# which imports the name from the next package. The first package's inner +# module imports the target, or, failing that, the name from the second. +SOURCES.update( + { + f"lazyc_{index}.__init__": """ + def __getattr__(name): + from importlib import import_module + + return getattr(import_module(f"{__name__}.inner"), name) + """ + for index in range(_LAZY) + } +) +SOURCES.update( + { + f"lazyc_{index}.inner": ( + f"from {PACKAGE}.lazyc_{(index + 1) % _LAZY} import t_lazy_cycle\n" + ) + for index in range(1, _LAZY) + } +) +SOURCES["lazyc_0.inner"] = """ + try: + from microcosm.walkcases import t_lazy_cycle + except ImportError: + from microcosm.walkcases.lazyc_1 import t_lazy_cycle + """ +SOURCES["r_lazy_cycle"] = """ + from microcosm.walkcases.lazyc_1 import t_lazy_cycle + + + def run(): + return len([t_lazy_cycle.X]) + """ +# Constants that name each other: _B names _A and the target, and _A, bound +# again, names _B. Python binds both to Base. +SOURCES["r_alias_cycle"] = """ + from microcosm.walkcases import t_alias_cycle + + + class Base: + X = 1 + + + _A = Base + _B = _A or t_alias_cycle + _A = _B + + + def run(): + return _A.X + """ +# Constants that name only each other: each stands for both. +SOURCES["r_alias_pure_cycle"] = """ + _A = None + _B = _A + _A = _B + + + def run(): + return getattr(_B, "X", None) + """ + # A root that makes a Thing reaches every dunder Thing defines, so it reaches # t_add and t_float as well: the walk over-approximates there by design. _THING = {"add", "float"} @@ -1766,9 +2337,18 @@ def run(): ("r_lazy", "run"): {"lazy"}, ("r_lazy_assigned", "run"): {"lazy_assigned"}, ("r_star_lazy", "run"): {"star_lazy"}, + ("r_class_global", "Kernel.run"): { + "class_global", + "class_global_alias", + "class_global_class", + "class_global_def", + "class_global_hidden", + "class_global_import", + }, # Scopes ("r_shadow", "run"): {"shadow"}, ("r_comprehension", "run"): {"comprehension"}, + ("r_comprehension_decoy", "run"): {"comprehension_iter"}, ("r_sibling", "run"): {"sibling", "local"}, ("r_class_scope", "Holder.run"): {"class_scope"}, ("r_lambda_shadow", "Kernel.run"): {"lambda_shadow"}, @@ -1778,6 +2358,7 @@ def run(): ("r_body_conditional", "Kernel.run"): {"body_conditional"}, ("r_body_annotation", "Kernel.run"): {"body_annotation"}, ("r_body_decorator", "Kernel.run"): {"body_decorator"}, + ("r_body_del", "Kernel.run"): {"body_del"}, ("r_body_nested", "run"): {"body_nested"}, ("r_global", "run"): {"global"}, ("r_nonlocal", "run"): {"nonlocal"}, @@ -1802,6 +2383,11 @@ def run(): # Self and cls ("r_self_method", "Kernel.run"): {"self_method"}, ("r_class_import", "Kernel.run"): {"class_import"}, + ("r_class_import_scope", "Kernel.run"): { + "class_import_body", + "class_import_decorator", + "class_import_default", + }, # Implicit calls ("r_float", "run"): {"float", *_THING}, ("r_add", "run"): {"add", *_THING}, @@ -1813,6 +2399,7 @@ def run(): # Import-time code ("r_headers", "run"): { "header_for", + "header_for_target", "header_guard", "header_if", "header_match", @@ -1821,6 +2408,7 @@ def run(): "header_trystar", "header_while", "header_with", + "header_with_target", }, ("r_blocks", "run"): { "block_body", @@ -1852,6 +2440,11 @@ def run(): "class_fill_view", }, ("r_registry", "run"): {"registrar"}, + ("r_for_store", "run"): {"for_store", "with_store"}, + ("r_comprehension_fill", "run"): {"comprehension_fill"}, + ("r_augassign", "run"): {"augassign"}, + ("r_class_member_change", "Kernel.run"): {"class_member_change"}, + ("r_class_nested_fill", "run"): {"class_nested_fill"}, # Attribute fallback ("r_prop", "run"): {"prop", *_THING}, ("r_value", "run"): {"value"}, @@ -1867,16 +2460,34 @@ def run(): ("r_methodcaller", "run"): {"methodcaller"}, ("r_match_keyword", "run"): {"match_keyword"}, ("r_match_positional", "run"): {"match_positional"}, + ("r_match_inherited", "run"): {"match_inherited"}, ("r_import_module", "run"): {"import_module"}, ("r_import_relative", "run"): {"import_relative"}, ("r_import_from", "run"): {"import_from"}, + ("r_dunder_import_top", "run"): {"dunder_top"}, + ("r_import_level", "run"): {"import_level"}, + ("r_import_spec", "run"): {"import_spec"}, + ("r_import_name", "run"): {"import_name"}, + ("r_import_renamed", "run"): {"import_renamed", "import_renamed_alias"}, + # Cost + ("r_star_diamond", "run"): {"star_diamond"}, + ("r_star_clique", "run"): {"star_clique"}, + ("r_alias_diamond", "run"): {"alias_diamond"}, + ("r_alias_modules", "run"): {"alias_modules"}, + ("r_match_diamond", "run"): {"match_diamond"}, + ("r_import_diamond", "run"): {"import_diamond"}, + ("r_lazy_cycle", "run"): {"lazy_cycle"}, + ("r_alias_cycle", "run"): {"alias_cycle"}, + ("r_alias_pure_cycle", "run"): set(), # What the walk can't see ("r_registry_decorator", "run"): set(), } #: Targets a root's walk reaches that its run doesn't read. The walk reaches -#: every dunder of a class it references, and both branches of a condition. +#: every dunder of a class it references, both branches of a condition, and +#: both operands of ``or``. OVER: dict[tuple[str, str], set[str]] = { + ("r_alias_cycle", "run"): {"alias_cycle"}, ("r_float", "run"): {"add"}, ("r_add", "run"): {"float"}, ("r_prop", "run"): _THING, From e6e1f4c4eea902df0e3a49aac3c53d926a0f4134 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 11:20:42 -0500 Subject: [PATCH 11/14] NZ G8-x2: settle lookup cycles to a fixpoint, and root each review gap (#1567) The delta review of c622fc22f asked for one change and offered six more. Belgium's look at c622fc22f asked for two changes and four cases, and its OK at 005844bd7 asked for one open form and twelve roots. This commit takes all of them. - Lookup cycles (required). c622fc22f gave every lookup in a cycle the items its first lookup ended with. A constant that maps another's items through an attribute (X = Y.sub, in a cycle with Y) then settled on less than a fresh lookup finds, and what it lost depended on which lookup the walk met first. 005844bd7 reached what it lost. Now, when a cycle's first lookup ends, the cycle's lookups run again, each meeting what the others hold so far, until a pass adds nothing (_Walk._cycle). So each settles on what it finds once the others are complete, whichever the walk met first. New roots: - r_alias_cycle_dotted and r_alias_cycle_dotted_first, the review's shape met from each end; - r_alias_self_dotted, a lookup that waits on itself; - r_alias_cycle_join, a lookup that a pass begins and that joins the cycle. - A module's __getattr__ in a cycle (Belgium). exported() added a module's __getattr__ only when the name's lookup had found nothing. Inside a cycle that lookup hasn't settled, and Python can read the module part-built, before it binds the name. So what the walk reached depended on which lookup it met first. exported() now leaves the getter out only when the name's lookup has settled with items (r_getattr_cycle, r_getattr_cycle_first). OVER holds the getter that Python's import order doesn't pick, since the walk doesn't track that order. r_getattr_self pins the older rule that Python finds a module's __getattr__ in its namespace only. - Belgium's roots. - Form 1, the last of the three forms Belgium found at 005844bd7. A class body's X += 1 reads X before it binds it, so it reads the module's X until the body has bound it (r_aug_class). - Belgium's four probes become roots: r_aug_class, r_nested_change, r_global_setter and r_global_read. So do its eight roots for rules that no NZ root pinned: r_del_class, r_self_ref, r_with_class, r_ann_class, r_aug_store, r_kw_fill, r_comp_first_iter and r_walrus_comp. OVER takes the target that the walk reaches and Python doesn't read in r_del_class, r_global_read and r_global_setter. - Belgium's two controls at c622fc22f join too (r_class_iadd, r_class_comp_fill), and r_base_set passes an alias through a set display, which no root did. - Class-body globals at import. In a class the walk reaches, an assignment to a name the body declares global binds outside the class, but it runs as Python creates the class. The walk now follows it there (r_class_global_aug: X += 1 calls GlobalBox's +). - Knockouts that failed no root (review finding 5). Each now fails one: - _CALLABLES narrowed to the importers (r_getattr_renamed: getattr under a module-level name); - the class table for a class that a class body declares global (r_class_global_inherit); - __getattr__ as a lazy export of itself (r_getattr_self); - _members' filter of global names (r_global_read); - body= for a class-body global's value (r_class_global, whose module now binds _source to a decoy); - _met.add (r_alias_self_dotted), and the cut that a lookup under way gives back (seven roots, r_lazy_cycle and r_star_clique among them). The _own_names revert stays equivalent, as c622fc22f's message said. - Recursion depth (finding 2). The docstring now says how deep a chain of re-exports, star imports, constants or inherited __match_args__ can go before the walk raises RecursionError (about 200, 250 and 330), and that the kernels' lookups nest at most four deep. - Misses the docstring didn't list (findings 3, 4 and 7). "What it can't see" now lists each, and a BLIND root pins each: - a nested class's global value, and the bases, keywords and decorators of a nested class that the walk runs whole (r_nested_global_value, r_nested_base_scope); - a match class pattern through a class-body global bound to the body's own import (r_class_global_match), and a name that a function binds through global (r_function_global, r_function_global_import); - a change inside a lambda that a statement calls (r_lambda_change); - import_module with another package (r_spec_name), and importers that partial wraps or a call returns (r_partial_import, r_getattr_importer); - an explicit __match_args__ that names a constant (r_match_args_constant); - an operator's side effect inside a statement that binds a name (r_aug_module_effect). The literal-constant sentence now says that each binding of the constant must set it to a string, and the annotations sentence names class bodies and modules as well as def statements. - Tests (finding 6). - test_the_walk_runs_each_lookup_once becomes test_the_walk_runs_each_lookup_once_outside_a_cycle. A lookup in a cycle runs once more for each pass that runs the cycle again (the walk's own count), and every lookup a walk begins settles. - test_the_walk_makes_few_lookups_on_the_cost_roots counts every call of the four lookup methods, the memo's answers too, and fails at the first call past 10,000. So a lookup that skips the memo, or keys it more finely than what it looks up, fails there instead of hanging. With the memo, the most any of the six roots makes is 1,257; with each lookup run afresh, each passes 300,000. - The fresh-walk test now runs on the new cycle shapes too. c622fc22f's message said "r_registry's registrar now holds no reached item". The walk does reach registrar._register in the end, through the change pass itself: registrar holds no reached item until that pass finds its change, so only that pass reaches it. Checks outside pytest, on CPython 3.14.7 and 3.13.15, which agree: - The walk passes every check on all 151 roots: the targets, the runtime reads, the run-count check, the fresh-walk check and the lookup budget. - Each of the hub's 139 one-edit knockouts of the walk fails at least one root, and so does each of Belgium's 51. Of the c622fc22f review's 51, 50 fail a root, and the 51st, the _own_names revert, is equivalent. Both Pythons fail the same roots. - Against c622fc22f's walk, the walk loses no item on NZ's 16 kernel walks, the 151 roots or 200 seeded composites of them. Twelve roots differ, each by a change above: ten gain items and two gain sites only. On three cycle roots, a site that settle() recorded for a module used as a value goes, since the walk now reaches the item first through the root's own lines. No kernel walk changes. - On NZ's 8 kernels, with and without the hierarchy cut, the module sets equal d403cce28's, and no kernel walk has a lookup cycle. CI's engine-free jobs run the three test files; this commit had no local pytest run. Co-Authored-By: Claude Opus 5.5 --- .../test_transport_income_evidence_kernels.py | 76 +- test_support/microcosm_build/static_reach.py | 192 +++- .../microcosm_build/static_reach_cases.py | 879 +++++++++++++++++- 3 files changed, 1081 insertions(+), 66 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py index e2bf5bbe8b..bf0543744f 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py @@ -2060,10 +2060,15 @@ def test_property_composing_roots_loses_no_target(walk_case_tree, picked) -> Non ) -def test_the_walk_runs_each_lookup_once(walk_case_tree, monkeypatch) -> None: - # Each lookup runs once per walk and hands back each item once. A walk - # that ran a lookup afresh wherever it met it would take time exponential - # in a diamond's depth, or factorial in a clique's size, on the Cost roots. +def test_the_walk_runs_each_lookup_once_outside_a_cycle( + walk_case_tree, monkeypatch +) -> None: + # A lookup outside a cycle runs once per walk, and a lookup in a cycle + # once more for each pass that runs the cycle again. Each hands back each + # item once, and every lookup a walk begins settles, a lookup that joins a + # cycle during a pass too (r_alias_cycle_join). A walk that ran a lookup + # afresh wherever it met it would take time exponential in a diamond's + # depth, or factorial in a clique's size, on the Cost roots. _, paths = walk_case_tree settled = static_reach._Walk._settled runs: Counter[tuple[object, ...]] = Counter() @@ -2071,7 +2076,6 @@ def test_the_walk_runs_each_lookup_once(walk_case_tree, monkeypatch) -> None: def counted(self, key, compute, cut=()): def counting(): runs[key] += 1 - assert runs[key] == 1, key return compute() found = settled(self, key, counting, cut) @@ -2081,8 +2085,12 @@ def counting(): monkeypatch.setattr(static_reach._Walk, "_settled", counted) for module, qualname in _WALK_CASE_ROOTS: runs.clear() - reach(f"{static_reach_cases.PACKAGE}.{module}", qualname, paths=paths) + walk = static_reach._Walk(paths, ()) + walk.run(f"{static_reach_cases.PACKAGE}.{module}", qualname) assert runs, (module, qualname) + assert not walk._began and not walk._open, (module, qualname) + for key, count in runs.items(): + assert count == 1 + walk.reruns.get(key, 0), (module, key) def _fresh_lookup(walk, key): @@ -2099,10 +2107,14 @@ def _fresh_lookup(walk, key): def test_a_settled_lookup_holds_what_a_fresh_walk_finds(walk_case_tree) -> None: - # Lookups in a cycle settle on the same items, whichever the walk met - # first (r_star_clique, r_lazy_cycle, r_alias_cycle). So each lookup a - # root's walk settles holds what the same lookup finds first in a fresh - # walk. + # Each lookup in a cycle ends with what it finds once the others are + # complete, whichever the walk met first: r_star_clique, r_lazy_cycle and + # r_alias_cycle; r_alias_cycle_dotted and r_alias_cycle_dotted_first, + # where one lookup maps another's items through an attribute; + # r_alias_self_dotted, where a lookup waits on itself; and r_getattr_cycle + # and r_getattr_cycle_first, where a module's __getattr__ answers inside a + # cycle of imports. So each lookup a root's walk settles holds what the + # same lookup finds first in a fresh walk. _, paths = walk_case_tree for module, qualname in _WALK_CASE_ROOTS: walk = static_reach._Walk(paths, ()) @@ -2113,6 +2125,50 @@ def test_a_settled_lookup_holds_what_a_fresh_walk_finds(walk_case_tree) -> None: assert set(_fresh_lookup(fresh, key)) == set(items), (module, key) +#: Cost roots on which a walk that ran each lookup afresh wherever it met it +#: would make more than 300,000 lookups: each diamond doubles its paths at +#: each of its 22 levels, and the clique's grow factorially with its size. +_COST_ROOTS = [ + ("r_alias_diamond", "run"), + ("r_alias_modules", "run"), + ("r_import_diamond", "run"), + ("r_match_diamond", "run"), + ("r_star_clique", "run"), + ("r_star_diamond", "run"), +] +_LOOKUP_BUDGET = 10_000 + + +@pytest.mark.parametrize( + ("module", "qualname"), _COST_ROOTS, ids=[f"{m}:{q}" for m, q in _COST_ROOTS] +) +def test_the_walk_makes_few_lookups_on_the_cost_roots( + walk_case_tree, monkeypatch, module, qualname +) -> None: + # Each lookup method counts every call, the memo's answers too, so a + # lookup that skips _settled, or keys it more finely than what it looks + # up, fails here at the budget instead of hanging. With the memo, the + # most any of these walks makes is about an eighth of the budget. + _, paths = walk_case_tree + calls = 0 + + def counting(method): + def counted(self, *args, **kwargs): + nonlocal calls + calls += 1 + assert calls <= _LOOKUP_BUDGET, (module, method.__name__) + return method(self, *args, **kwargs) + + return counted + + for name in ("resolve", "exported", "aliased", "match_args"): + method = getattr(static_reach._Walk, name) + monkeypatch.setattr(static_reach._Walk, name, counting(method)) + walk = static_reach._Walk(paths, ()) + walk.run(f"{static_reach_cases.PACKAGE}.{module}", qualname) + assert calls, module + + def test_the_module_stays_outside_the_batterys_binding_closure() -> None: # The registry and the composer import this module, and no battery # binding's identity covers it or them, so adding the kernels re-keys no diff --git a/test_support/microcosm_build/static_reach.py b/test_support/microcosm_build/static_reach.py index c0f4dbf0ec..6a5bf681d5 100644 --- a/test_support/microcosm_build/static_reach.py +++ b/test_support/microcosm_build/static_reach.py @@ -28,7 +28,9 @@ only once a statement at the body's top level has bound it. Before that, when only a branch, a loop or a bare annotation binds it, or when a ``del`` can undo it, Python reads the module's binding, so the walk follows that - one too. + one too. An augmented assignment (``X += 1``) reads its target before it + binds it, so a class body's ``X += 1`` reads the module's ``X`` until the + body has bound it. - Aliases. A module-level constant stands for each module, class or function its value names: ``_helper = helper_module``, ``_Base = pkg.Base``, ``pkg.Base[int]``, either branch of a conditional, each operand of ``and`` @@ -55,7 +57,9 @@ statement's header there: an ``if`` or ``while`` test, what a ``for`` iterates or a ``with`` enters and the target each stores to, a ``match`` statement's subject, patterns and guards, and each ``except`` or - ``except*`` clause's types. + ``except*`` clause's types. In a class it reaches, it also follows each + assignment to a name the body declares ``global``: the name binds outside + the class, but the assignment runs as Python creates the class. - Changes by name. In any module the walk has read, a statement at module level or in any class body, a nested class's too, that changes a reached name in place runs with it: a method call on the name @@ -79,46 +83,76 @@ ``__spec__.parent`` or ``__name__``, and ``__import__`` with its top package as well, counting a ``level`` above zero from the calling module's package. Each of these callables counts under a module-level - name for it too (``load = import_module``). A module-level constant that - holds only strings counts as a literal. - -Each lookup runs once per walk: what a name binds in a module, what a -constant stands for, a package's lazy exports, and a class's inherited -``__match_args__``. Lookups that wait on each other in a cycle settle on the -same items, whichever the walk met first. A walk that ran each lookup afresh -wherever it met it would take time exponential in the depth of a diamond of -star imports, re-exports, constants or bases, and factorial in the size of a -clique of star imports: the Cost cases in ``static_reach_cases``. A round -counts only what it adds: an item, a module the walk uses as a value, or a -statement the walk hasn't run. The walk ends at the first round that adds -nothing. + name that an import or an assignment binds to it + (``load = import_module``, ``_get = getattr``). A module-level constant + that each of its bindings sets to a string counts as a literal + (``_NAME = "go"``). + +Each lookup outside a cycle runs once per walk: what a name binds in a +module, what a constant stands for, a package's lazy exports, and a class's +inherited ``__match_args__``. Lookups that wait on each other in a cycle run +again, each meeting what the others hold so far, until a pass over the cycle +adds nothing. So each settles on what it finds once the others are complete, +whichever the walk met first. Within a cycle, a module's ``__getattr__`` +answers for a name whose lookup hasn't settled, since Python can read the +module part-built, before it binds the name. A walk that ran each lookup +afresh wherever it met it would take time exponential in the depth of a +diamond of star imports, re-exports, constants or bases, and factorial in +the size of a clique of star imports: the Cost cases in +``static_reach_cases``. A lookup nests inside the one that needs it, three +to five Python frames a level. So on a chain of about 200 re-exports, 250 +star imports, or 330 constants or inherited ``__match_args__``, each naming +the next, the walk passes Python's default recursion limit of 1,000 and +raises ``RecursionError``; the kernels' lookups nest at most four deep. A +round counts only what it adds: an item, a module the walk uses as a value, +or a statement the walk hasn't run. The walk ends at the first round that +adds nothing. What it can't see: - a name built at run time (``getattr`` or ``import_module`` with a computed - name, ``globals()``, ``exec``), and those callables under a name that a - function binds; + name, ``import_module`` with a package other than those "Literal names" + lists, ``globals()``, ``exec``), and those callables under a name that a + function binds or that a call returns (``partial(import_module, ...)``, + ``getattr(importlib, "import_module")``); - code reached by registration rather than by name: a function that a decorator records for a framework or a registry to call (``singledispatch``, pydantic's validators); - the import-time code of a module that holds no item the walk reaches, such as the ``__init__`` of a package above a reached module, or a module imported only for its effects, unless a statement there changes a reached - name in place in a module the walk has read; -- the side effects of a call inside a statement that binds a name or tests a - value (``_X = _setup()``, ``assert _setup()``), unless the walk reaches that - statement another way; + name in place, in a way "Changes by name" lists, in a module the walk has + read. A change inside a function or a lambda that the statement calls + doesn't count (``(lambda: _TABLE.update(...))()``); +- the side effects of a call or an operator inside a statement that binds a + name or tests a value (``_X = _setup()``, ``_X += 1``, ``assert _setup()``), + unless the walk reaches that statement another way; - the import-time code of a class or function the walk doesn't reach: its decorators, default values, class bases and class attributes, unless a statement in the class body changes a reached name in place; +- what a name stands for where code in a function binds it through + ``global``. The walk follows the function's code where it reaches the + function, but reads what the name stands for from the module's own + bindings only. So a ``match`` class pattern through the name finds no + class, and an attribute chain through a module that the function imports + under the name (``global _MOD; import p.m as _MOD``) reaches nothing; +- a ``match`` class pattern through a name that a class body declares + ``global`` and binds to one of the body's own names: the walk reads the + body's names at module level there and finds no class; +- in a class nested in a class body, the value that it binds to a name it + declares ``global``, and the bases, keywords and decorators of a nested + class that the walk runs whole: the walk reads their names at module level; +- an explicit ``__match_args__`` other than a tuple of string literals + (``__match_args__ = _FIELDS``); - a member inherited from a base that the ``class`` statement computes with a call; - code that an argument carries in from outside the walk (the executor's context, for one); -- a module that only an annotation names. Before Python 3.14 a ``def`` - statement evaluates its annotations as it runs; from 3.14 they wait until - something reads them (``typing.get_type_hints``); and - ``from __future__ import annotations`` keeps them as strings. +- what only an annotation names or calls. Before Python 3.14, a ``def`` + statement, a class body and a module evaluate their annotations as they + run; from 3.14 they wait until something reads them + (``typing.get_type_hints``); and ``from __future__ import annotations`` + keeps them as strings. A :class:`Cut` leaves out one guarded branch, so a test names each exception it relies on rather than skipping a module. @@ -352,6 +386,25 @@ def _effects(body: Iterable[ast.stmt]) -> Iterator[ast.AST]: yield node +def _global_stores(node: ast.ClassDef) -> Iterator[ast.stmt]: + """Each assignment in a class body to a name the body declares ``global``. + + The name binds at module level, so it isn't one of the class's members, + but creating the class runs the assignment. + """ + + declared = _declared_globals(node) + if not declared: + return + for child in _statements(node.body): + if isinstance(child, ast.Assign | ast.AugAssign) and any( + bound in declared + for target, _ in _assignments(child) + for bound in _stored(target) + ): + yield child + + def _mutations(part: ast.AST) -> Iterator[str]: """The names one statement (or one header) can change in place. @@ -900,14 +953,17 @@ def __init__(self, paths: Mapping[str, Path], cuts: Iterable[Cut]) -> None: #: A top-level class member's binding statement -> its class body. self._body_of: dict[int, ast.ClassDef] = {} self._inside: dict[str, list[str]] = {} - # Lookups, each computed once (see _settled): what each settled one - # holds; when each unsettled one began, and its cut; the unsettled - # ones in that order; and those another lookup met unsettled. + # Lookups (see _settled): what each settled one holds; when each + # unsettled one began, what it holds so far and how to run it again; + # the unsettled ones in that order; and those another lookup met + # unsettled. How many times each lookup in a cycle ran again. self._done: dict[tuple[object, ...], tuple[object, ...]] = {} self._began: dict[tuple[object, ...], int] = {} self._cuts: dict[tuple[object, ...], tuple[object, ...]] = {} + self._computes: dict[tuple[object, ...], Callable[[], Iterable[object]]] = {} self._open: list[tuple[object, ...]] = [] self._met: set[tuple[object, ...]] = set() + self.reruns: dict[tuple[object, ...], int] = defaultdict(int) self._begun = 0 #: When the earliest unsettled lookup that the one under way met began. self._low: float = math.inf @@ -968,19 +1024,21 @@ def _settled[T]( compute: Callable[[], Iterable[T]], cut: Iterable[T] = (), ) -> list[T]: - """``compute()``'s items for ``key``, once each, computing ``key`` once. + """``compute()``'s items for ``key``, once each. Lookups can wait on each other in a cycle: modules that import a name from each other, star imports, a lazy package's modules, constants that name each other, or bases. A lookup that meets one that hasn't - settled gets that one's ``cut``, and every lookup in the cycle - settles on the same items: what the cycle's first lookup ends with, - and each one's cut (Tarjan's strongly connected components). In a - cycle, each lookup's caller keeps what it found, back to the first - lookup, and the lookups reach each other, so each can reach every - item the first one ends with, whichever the walk met first. Each - lookup then runs once, where running it afresh at each visit costs - time exponential in a diamond's depth. + settled gets what that one holds so far, starting from its ``cut``, + and Tarjan's strongly connected components find the cycle. When the + cycle's first lookup ends, the cycle's lookups run again, each meeting + what the others hold so far, until a pass adds nothing (see + :meth:`_cycle`). So each ends with what it finds once the others are + complete, whichever the walk met first. That matters where a lookup + maps another's items rather than adding them: ``X = Y.sub`` in a + cycle with ``Y`` holds what ``Y.sub`` names, not what ``Y`` names. A + lookup outside a cycle runs once, where running each lookup afresh at + each visit costs time exponential in a diamond's depth. """ if key in self._done: @@ -992,6 +1050,7 @@ def _settled[T]( began = self._began[key] = self._begun self._begun += 1 self._cuts[key] = tuple(cut) + self._computes[key] = compute self._open.append(key) outer, self._low = self._low, began # One copy of each: through a diamond, every path adds its own. @@ -1001,20 +1060,48 @@ def _settled[T]( # It waits on a lookup that began earlier and hasn't settled. self._low = min(outer, low) return list(items) - self._low = outer start = self._open.index(key) members = self._open[start:] - del self._open[start:] if len(members) > 1 or key in self._met: - items = list( - dict.fromkeys([*items, *(c for m in members for c in self._cuts[m])]) - ) + self._cycle(members, start) + items = list(self._cuts[key]) + else: + self._cuts[key] = tuple(items) + self._low = outer + del self._open[start:] for member in members: - del self._began[member], self._cuts[member] + self._done[member] = self._cuts[member] + del self._began[member], self._cuts[member], self._computes[member] self._met.discard(member) - self._done[member] = tuple(items) return list(items) + def _cycle(self, members: list[tuple[object, ...]], start: int) -> None: + """Run a cycle's lookups again until a pass adds nothing. + + ``members`` are the cycle's lookups, ``self._open[start:]``, each + holding in ``_cuts`` its cut. Each pass runs every member again, the + last to begin first: each run meets what the others hold so far, and + its lookup then holds what it held and what the run found. A lookup + that a run begins and that waits on the cycle joins it. Items and + members only grow, and both are finite, so the passes end. + """ + + grew = True + while grew: + grew = False + for member in reversed(list(members)): + self.reruns[member] += 1 + found = list(self._computes[member]()) + joined = self._open[start + len(members) :] + if joined: + members.extend(joined) + grew = True + held = self._cuts[member] + now = tuple(dict.fromkeys([*held, *found])) + if len(now) > len(held): + self._cuts[member] = now + grew = True + def resolve(self, module: str, name: str) -> list[Item]: """Every definition ``name`` can bind in ``module``'s top-level scope.""" @@ -1052,9 +1139,14 @@ def exported(self, module: str, name: str) -> list[Item]: items.append((f"{module}.{name}", "module", None)) items.extend(self.resolve(module, name)) found = self.module(module) - if items or found is None or name == "__getattr__": + if found is None or name == "__getattr__": # Python finds a module's __getattr__ in its namespace only. return items + if items and ("resolve", module, name) in self._done: + # Bound for sure. A lookup that hasn't settled waits on a cycle, + # where Python can read the module part-built, before it binds + # the name, so its __getattr__ counts too. + return items # A lazy export: whatever binds the module's __getattr__ and, for a # package, the name wherever the package's own modules bind it. getters = self.resolve(module, "__getattr__") @@ -1308,7 +1400,13 @@ def match_class(node: ast.MatchClass, here: Scope) -> None: for child, here in _visits(found, node, skipped, start, {}): at = f"{site}:{getattr(child, 'lineno', '?')}" - if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Load): + if isinstance(child, ast.AugAssign) and isinstance(child.target, ast.Name): + # ``NAME += value`` reads NAME before it binds it. + for item in lookup(child.target.id, here, child.target): + self.push(item, at) + if item[1] == "module": + self.values.add(item[0]) + elif isinstance(child, ast.Name) and isinstance(child.ctx, ast.Load): for item in lookup(child.id, here, child): self.push(item, at) if item[1] == "module" and id(child) not in rooted: @@ -1453,7 +1551,7 @@ def process(self, item: Item) -> None: for part in (*node.decorator_list, *node.bases, *node.keywords): self.scan(module, part, f"{module}.{name}(class)") # Creating the class runs its body. - for part in _effects(node.body): + for part in (*_effects(node.body), *_global_stores(node)): if id(part) not in self.ran: self.ran.add(id(part)) self.scan( diff --git a/test_support/microcosm_build/static_reach_cases.py b/test_support/microcosm_build/static_reach_cases.py index 3758d40533..54b400b3d7 100644 --- a/test_support/microcosm_build/static_reach_cases.py +++ b/test_support/microcosm_build/static_reach_cases.py @@ -33,11 +33,20 @@ "alias", "alias_base", "alias_cycle", + "alias_cycle_dotted", + "alias_cycle_dotted_first", + "alias_cycle_join", "alias_diamond", "alias_modules", + "alias_self_dotted", "alternative", "alternative_first", + "ann_class", + "ann_module", "attrgetter", + "aug_class", + "aug_module_effect", + "aug_store", "augassign", "base_attr", "base_dunder_import", @@ -48,6 +57,7 @@ "base_import", "base_or", "base_second", + "base_set", "base_starred", "base_subscript", "base_walrus", @@ -64,6 +74,7 @@ "body_hidden", "body_nested", "class_body_update", + "class_comp_fill", "class_fill_call", "class_fill_del", "class_fill_item", @@ -71,11 +82,15 @@ "class_fill_view", "class_global", "class_global_alias", + "class_global_aug", "class_global_class", "class_global_def", "class_global_hidden", "class_global_import", + "class_global_inherit", + "class_global_match", "class_header", + "class_iadd", "class_import", "class_import_body", "class_import_decorator", @@ -83,14 +98,17 @@ "class_member_change", "class_nested_fill", "class_scope", + "comp_first_iter", "comprehension", "comprehension_fill", "comprehension_iter", "conditional", "decorator", "default", + "del_class", "del_hidden", "del_item", + "del_module", "delattr", "delattr_patch", "dotted", @@ -101,9 +119,20 @@ "float", "for_store", "for_target", + "function_global", + "function_global_import", "getattr_alias", "getattr_constant", + "getattr_cycle_a", + "getattr_cycle_b", + "getattr_importer", + "getattr_renamed", + "getattr_self", "global", + "global_read_class", + "global_read_first", + "global_setter", + "global_setter_first", "go", "hasattr", "header_for", @@ -126,7 +155,9 @@ "import_renamed", "import_renamed_alias", "import_spec", + "kw_fill", "lambda", + "lambda_change", "lambda_shadow", "lazy", "lazy_assigned", @@ -134,6 +165,7 @@ "local", "local_alternative", "local_alternative_first", + "match_args_constant", "match_capture", "match_diamond", "match_inherited", @@ -145,14 +177,19 @@ "methodcaller", "mutation_alias", "namespace", + "nested_base_scope", + "nested_change", + "nested_global_value", "nested_scope", "nonlocal", "override", + "partial_import", "prop", "registrar", "registry_decorator", "root_base", "self_method", + "self_ref", "setattr", "setattr_patch", "setdefault", @@ -161,6 +198,7 @@ "shadow", "shadowed", "sibling", + "spec_name", "star", "star_clique", "star_diamond", @@ -173,16 +211,27 @@ "update", "value", "walrus", + "walrus_comp", + "with_class", + "with_module", "with_store", "with_target", ) #: Targets a case binds so that a wrong walk would reach them. No walk #: reaches them and no run reads them: t_shadowed behind a name the class -#: body rebinds (r_class_decorator) and behind a comprehension's target -#: (r_comprehension_decoy), and t_body_hidden behind the function's import of -#: nest_p, which the class in r_body_nested never reads. -DECOYS = frozenset({"body_hidden", "shadowed"}) +#: body rebinds (r_class_decorator), behind a comprehension's target +#: (r_comprehension_decoy) and behind a module binding of a name that a +#: class body binds before its global's value reads it (r_class_global), +#: t_body_hidden behind the function's import of +#: nest_p, which the class in r_body_nested never reads, and t_ann_module and +#: t_with_module behind a module binding that the class body has surely +#: replaced before it reads the name (r_ann_class, r_with_class), and +#: t_getattr_self behind a package module's ``__getattr__``, which the +#: package's own ``__getattr__`` never stands for (r_getattr_self). +DECOYS = frozenset( + {"ann_module", "body_hidden", "getattr_self", "shadowed", "with_module"} +) #: Each target's source: reads of X or ROWS are recorded, anything else fails. _TARGET = """\ @@ -427,7 +476,7 @@ def run(): # A name a class body declares global binds at module level: through an # assignment, an alias of a module, an import, a def and a class there. # In _Setup, a class nothing reaches, the assignment reads a name the - # class body bound. + # class body bound, not the module's _source. "cglob_alias": """ from microcosm.walkcases import t_class_global_alias @@ -449,8 +498,11 @@ def fn(): t_class_global_class, t_class_global_def, t_class_global_hidden, + t_shadowed, ) + _source = t_shadowed + class Kernel: global _CACHE, _IMPL, _probe, Probe, imported @@ -474,6 +526,34 @@ class _Setup: _source = t_class_global_hidden _ROWS = _source.ROWS """, + # A class that a class body declares global and defines takes its base's + # __match_args__, so a positional pattern reads the base's property. + "cglob_shapes": """ + from microcosm.walkcases import t_class_global_inherit + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_class_global_inherit.X + """, + "r_class_global_inherit": """ + from microcosm.walkcases.cglob_shapes import Base + + + class Kernel: + global Probe + + class Probe(Base): + pass + + def run(self): + match Probe(): + case Probe(v): + return v + """, # A package's lazy export, through its module-level __getattr__. "lazy.__init__": """ def __getattr__(name): @@ -546,6 +626,48 @@ def go(self): def run(): return Lazy3().go() # noqa: F405 """, + # A walrus in a comprehension binds in the enclosing function, to what the + # comprehension's first iterable yields. + "r_walrus_comp": """ + from microcosm.walkcases import t_walrus_comp as other + + + def run(): + values = [(last := item) for item in (other,)] + return last.X, values + """, + # A class the walk doesn't reach binds a name it declares global, and a + # module function reads the name. Every binding of it counts, so the walk + # reaches the module's first binding as well. + "r_global_setter": """ + from microcosm.walkcases import t_global_setter, t_global_setter_first + + SOURCE = t_global_setter_first + + + class _Setter: + global SOURCE + SOURCE = t_global_setter + + + def run(): + return SOURCE.X + """, + # The root's own class binds a name it declares global, then reads it. + "r_global_read": """ + from microcosm.walkcases import t_global_read_class, t_global_read_first + + SOURCE = t_global_read_first + + + class Kernel: + global SOURCE + SOURCE = t_global_read_class + seen = SOURCE + + def run(self): + return self.seen.X + """, # -- Scopes ----------------------------------------------------------------- # A nested function's local leaves the enclosing scope's import alone. "r_shadow": """ @@ -785,6 +907,124 @@ def inner(): return inner() """, + # A comprehension's first iterable runs in the enclosing scope, so it reads + # the enclosing binding of a name the comprehension's target rebinds. + "r_comp_first_iter": """ + from microcosm.walkcases import t_comp_first_iter as target + + + def run(): + return [target for target in target.ROWS] + """, + # A class body's SOURCE = SOURCE.X reads the module's SOURCE: a statement + # binds its name only once it ends. + "r_self_ref": """ + from microcosm.walkcases import t_self_ref + + SOURCE = t_self_ref + + + class Kernel: + SOURCE = SOURCE.X + + def run(self): + return self.SOURCE + """, + # Classes whose + reads a target: for X += 1 in a class body (Box), in a + # class body that declares X global (GlobalBox), and at module level + # where nothing reads X (BlindBox). + "aug_boxes": """ + from microcosm.walkcases import ( + t_aug_class, + t_aug_module_effect, + t_class_global_aug, + ) + + + class Box: + def __add__(self, other): + return t_aug_class.X + other + + + class GlobalBox: + def __add__(self, other): + return t_class_global_aug.X + other + + + class BlindBox: + def __add__(self, other): + return t_aug_module_effect.X + other + """, + # X += 1 in a class body reads X before it binds it: here the module's X. + "r_aug_class": """ + from microcosm.walkcases.aug_boxes import Box + + X = Box() + + + class Kernel: + X += 1 + + def run(self): + return self.X + """, + # A del can undo a class body's binding, so a read after it reads the + # module's binding. The walk follows the class's own binding as well. + "r_del_class": """ + from microcosm.walkcases import t_del_class, t_del_module + + SOURCE = t_del_module + + + class Kernel: + SOURCE = t_del_class + del SOURCE + source = SOURCE + + def run(self): + return self.source.X + """, + # A with statement and an annotated assignment at the body's top level + # bind for sure, so a read after either reads only the class's binding. + "with_ctx": """ + class Ctx: + def __init__(self, value): + self.value = value + + def __enter__(self): + return self.value + + def __exit__(self, *exc): + return False + """, + "r_with_class": """ + from microcosm.walkcases import t_with_class, t_with_module + from microcosm.walkcases.with_ctx import Ctx + + SOURCE = t_with_module + + + class Kernel: + with Ctx(t_with_class) as SOURCE: + pass + source = SOURCE + + def run(self): + return self.source.X + """, + "r_ann_class": """ + from microcosm.walkcases import t_ann_class, t_ann_module + + SOURCE = t_ann_module + + + class Kernel: + SOURCE: object = t_ann_class + source = SOURCE + + def run(self): + return self.source.X + """, # -- Aliases ---------------------------------------------------------------- # A constant that holds a module stands for it, through tuple unpacking, # and through either branch of a condition. (A module used as a value @@ -862,7 +1102,7 @@ class Kernel(_Base): pass """, # ... through each branch of a conditional (the walk follows both), each - # operand of or, a walrus, each element of a tuple, a starred list, + # operand of or, a walrus, each element of a tuple, a starred list, a set, # getattr, a subscript, import_module, and __import__'s top package. "bases": """ from microcosm.walkcases import ( @@ -874,6 +1114,7 @@ class Kernel(_Base): t_base_import, t_base_or, t_base_second, + t_base_set, t_base_starred, t_base_subscript, t_base_walrus, @@ -933,6 +1174,11 @@ def run(self): return t_base_starred.X + class SetBase: + def run(self): + return t_base_set.X + + class DunderImportBase: def run(self): return t_base_dunder_import.X @@ -998,6 +1244,15 @@ class Kernel(_Base): _Base = [*[bases.StarredBase]][0] + class Kernel(_Base): + pass + """, + "r_base_set": """ + from microcosm.walkcases import bases + + _Base = [*{bases.SetBase}][0] + + class Kernel(_Base): pass """, @@ -1523,6 +1778,44 @@ class Kernel: def run(self): return self.TABLE """, + # A class body's assignment to a name it declares global runs as the class + # is created, though nothing reads the name: X += 1 calls GlobalBox's +. + "r_class_global_aug": """ + from microcosm.walkcases.aug_boxes import GlobalBox + + X = GlobalBox() + + + class Kernel: + global X + X += 1 + + def run(self): + return None + """, + # An augmented store to an attribute runs for its effect, and its value + # changes the table. + "r_aug_store": """ + from microcosm.walkcases import t_aug_store + + _TABLE = {} + + + class _Counter: + n = 0 + + + def _fill(table): + table.update(t_aug_store.ROWS) + return 1 + + + _Counter.n += _fill(_TABLE) + + + def run(): + return _TABLE + """, # -- Changes by name -------------------------------------------------------- # Another module re-exports the table run() reads and changes it in place. # The walk reads that module but reaches nothing it defines, so only the @@ -1725,6 +2018,68 @@ class Inner: _TABLE.update({"row": row}) + def run(): + return _TABLE + """, + # A call statement in a class nothing reaches passes the table by keyword. + "r_kw_fill": """ + from microcosm.walkcases import t_kw_fill + + _TABLE = {} + + + def _fill(*, table): + table.update(t_kw_fill.ROWS) + + + class _Registrar: + _fill(table=_TABLE) + + + def run(): + return _TABLE + """, + # A class nested in one nothing reaches changes the table directly. + "r_nested_change": """ + from microcosm.walkcases import t_nested_change + + _TABLE = {} + + + class _Registrar: + class Inner: + _TABLE.update(t_nested_change.ROWS) + + + def run(): + return _TABLE + """, + # += in a class body nothing reaches reads the module's list and extends + # it in place. + "r_class_iadd": """ + from microcosm.walkcases import t_class_iadd + + _ROWS = [] + + + class _Registrar: + _ROWS += list(t_class_iadd.ROWS) + + + def run(): + return _ROWS + """, + # A comprehension in a class body nothing reaches changes the table. + "r_class_comp_fill": """ + from microcosm.walkcases import t_class_comp_fill + + _TABLE = {} + + + class _Registrar: + [_TABLE.update(rows) for rows in (t_class_comp_fill.ROWS,)] + + def run(): return _TABLE """, @@ -2092,6 +2447,23 @@ def run(): first = load("microcosm.walkcases.renamed_mod").fn() return first, _import("microcosm.walkcases.renamed_alias_mod").fn() """, + # getattr under a module-level name. + "renamed_get_mod": """ + from microcosm.walkcases import t_getattr_renamed + + + def fn(): + return t_getattr_renamed.X + """, + "r_getattr_renamed": """ + from microcosm.walkcases import renamed_get_mod + + _get = getattr + + + def run(): + return _get(renamed_get_mod, "fn")() + """, # -- What the walk can't see ------------------------------------------------ # A decorator records _entry, and run() calls whatever the registry holds. "reg": """ @@ -2115,12 +2487,238 @@ def _entry(): def run(): return [entry() for entry in REGISTRY] """, + # An operator inside a statement that binds a name: at import, X += 1 + # calls BlindBox's +, and nothing reads X. + "r_aug_module_effect": """ + from microcosm.walkcases.aug_boxes import BlindBox + + X = BlindBox() + X += 1 + + + def run(): + return None + """, + # A class nested in a class body binds SOURCE, which it declares global, + # to one of its own names. The walk reads that name at module level. + "r_nested_global_value": """ + from microcosm.walkcases import t_nested_global_value + + SOURCE = None + + + class _Outer: + class _Inner: + _SRC = t_nested_global_value + global SOURCE + SOURCE = _SRC + + + def run(): + return SOURCE.X + """, + # A nested class that changes the table runs whole, and Python evaluates + # its base in _Outer's body, where _SRC binds the target. The walk reads + # _SRC at module level. + "r_nested_base_scope": """ + from microcosm.walkcases import t_nested_base_scope + + _TABLE = {} + + + class _Outer: + _SRC = t_nested_base_scope + + class _Inner(_SRC.ROWS.__class__): + _TABLE.update(()) + + + def run(): + return _TABLE + """, + # A function binds _PAT through global. The walk reaches the function and + # Base, but the match class pattern reads what _PAT stands for from the + # module's own bindings only. + "fglob_shapes": """ + from microcosm.walkcases import t_function_global + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_function_global.X + """, + "r_function_global": """ + from microcosm.walkcases import fglob_shapes + + _PAT = None + + + def _setup(): + global _PAT + _PAT = fglob_shapes.Base + + + _setup() + + + def run(): + match fglob_shapes.Base(): + case _PAT(v): + return v + """, + # ... and an attribute chain through a module that the function imports + # under the global name. + "fglob_mod": """ + from microcosm.walkcases import t_function_global_import + + + def fn(): + return t_function_global_import.X + """, + "r_function_global_import": """ + _MOD = None + + + def _setup(): + global _MOD + import microcosm.walkcases.fglob_mod as _MOD + + + _setup() + + + def run(): + return _MOD.fn() + """, + # A class body binds _PAT, which it declares global, to its own import. + # The match class pattern reads base_cls at module level and finds no + # class. + "cglobm_shapes": """ + from microcosm.walkcases import t_class_global_match + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_class_global_match.X + """, + "r_class_global_match": """ + from microcosm.walkcases import cglobm_shapes + + _PAT = None + + + class _Holder: + global _PAT + from microcosm.walkcases.cglobm_shapes import Base as base_cls + _PAT = base_cls + + + def run(): + match cglobm_shapes.Base(): + case _PAT(v): + return v + """, + # A module that holds no reached item changes the table inside a lambda + # that it calls in place. + "lam_table": """ + TABLE = {} + """, + "lam_fill": """ + from microcosm.walkcases import t_lambda_change + from microcosm.walkcases.lam_table import TABLE + + (lambda: TABLE.update(key=t_lambda_change.X))() + """, + "r_lambda_change": """ + from microcosm.walkcases.lam_fill import TABLE + + + def run(): + return TABLE + """, + # import_module relative to __spec__.name, which in a package's __init__ + # names the package. + "r_spec_name.__init__": """ + from importlib import import_module + + + def run(): + return import_module(".inner", package=__spec__.name).fn() + """, + "r_spec_name.inner": """ + from microcosm.walkcases import t_spec_name + + + def fn(): + return t_spec_name.X + """, + # An importer that functools.partial wraps, and one that a call returns. + "partial_mod": """ + from microcosm.walkcases import t_partial_import + + + def fn(): + return t_partial_import.X + """, + "r_partial_import": """ + from functools import partial + from importlib import import_module + + load = partial(import_module, package=__package__) + + + def run(): + return load(".partial_mod").fn() + """, + "getimp_mod": """ + from microcosm.walkcases import t_getattr_importer + + + def fn(): + return t_getattr_importer.X + """, + "r_getattr_importer": """ + import importlib + + load = getattr(importlib, "import_module") + + + def run(): + return load("microcosm.walkcases.getimp_mod").fn() + """, + # An explicit __match_args__ that names a constant. + "r_match_args_constant": """ + from microcosm.walkcases import t_match_args_constant + + _FIELDS = ("left",) + + + class Point: + __match_args__ = _FIELDS + + @property + def left(self): + return t_match_args_constant.X + + + def run(): + match Point(): + case Point(v): + return v + """, } # -- Cost ---------------------------------------------------------------------- # Shapes in which a walk that ran each lookup afresh wherever it met it would # take time exponential in a diamond's depth or factorial in a clique's size. -# The walk runs each lookup once, so each of these takes milliseconds. +# The walk runs each lookup outside a cycle once, and a cycle's lookups again +# only until a pass adds nothing, so each of these takes milliseconds. #: How deep each diamond goes, and how many modules the clique holds. _DEPTH = 22 @@ -2311,6 +2909,203 @@ def run(): def run(): return getattr(_B, "X", None) """ +# A cycle through a dotted name: X stands for Y.sub, and Y, bound again in a +# branch that never runs, for X. Python binds Y to the package and X to its +# sub module. run() meets Y first, and X still stands for what Y.sub names: +# the positional pattern reads the property that Base's __match_args__ names. +SOURCES["cycd.__init__"] = "" +SOURCES["cycd.sub"] = """ + from microcosm.walkcases import t_alias_cycle_dotted + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_alias_cycle_dotted.X + """ +SOURCES["r_alias_cycle_dotted"] = """ + import microcosm.walkcases.cycd.sub # noqa: F401 + from microcosm.walkcases import cycd + + Y = cycd + X = Y.sub + if False: + Y = X + + + def run(): + seen = Y.sub + match X.Base(): + case X.Base(v): + return v, seen + """ +# The same cycle met from X: Y still stands for the package. +SOURCES["cycf.__init__"] = """ + from microcosm.walkcases import t_alias_cycle_dotted_first + + + class Thing: + __match_args__ = ("left",) + + @property + def left(self): + return t_alias_cycle_dotted_first.X + """ +SOURCES["cycf.sub"] = """ + class Base: + pass + """ +SOURCES["r_alias_cycle_dotted_first"] = """ + import microcosm.walkcases.cycf.sub # noqa: F401 + from microcosm.walkcases import cycf + + Y = cycf + X = Y.sub + if False: + Y = X + + + def run(): + seen = X.Base + match Y.Thing(): + case Y.Thing(v): + return v, seen + """ +# A lookup that a pass over the cycle begins, and that waits on the cycle, +# joins it: X stands for Y.sub.A, and A, bound again in a branch that never +# runs, for X. +SOURCES["cycj.__init__"] = "" +SOURCES["cycj.sub"] = """ + from microcosm.walkcases import t_alias_cycle_join + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_alias_cycle_join.X + + + A = Base + if False: + from microcosm.walkcases.r_alias_cycle_join import X + + A = X + """ +SOURCES["r_alias_cycle_join"] = """ + import microcosm.walkcases.cycj.sub # noqa: F401 + from microcosm.walkcases import cycj + + Y = cycj + X = Y.sub.A + if False: + Y = X + + + def run(): + seen = Y.sub + match X(): + case X(v): + return v, seen + """ +# A name bound again to its own attribute: X stands for cycs, then for its +# sub module. Its lookup waits on itself, and a second pass reads cycs.sub. +SOURCES["cycs.__init__"] = "" +SOURCES["cycs.sub"] = """ + from microcosm.walkcases import t_alias_self_dotted + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_alias_self_dotted.X + """ +SOURCES["r_alias_self_dotted"] = """ + import microcosm.walkcases.cycs.sub # noqa: F401 + from microcosm.walkcases import cycs + + X = cycs + X = X.sub + + + def run(): + match X.Base(): + case X.Base(v): + return v + """ +# Two modules that each define __getattr__ and import x from the other. +# Whichever Python imports first, the other's ``from ... import x`` finds it +# part-built and calls its __getattr__. Each __getattr__ answers x only: a +# from-import asks its source for __path__ first. In r_getattr_cycle Python +# imports gcyc_b first, so it reads t_getattr_cycle_b; the walk meets gcyc_a's +# x first. In r_getattr_cycle_first both meet gcyc_a first. Either way the +# walk reaches both targets, since it doesn't track import order. +SOURCES.update( + { + f"gcyc_{first}": f""" + from microcosm.walkcases import t_getattr_cycle_{first} + + + def __getattr__(name): + if name == "x": + return t_getattr_cycle_{first}.X + raise AttributeError(name) + + + from microcosm.walkcases.gcyc_{second} import x + """ + for first, second in (("a", "b"), ("b", "a")) + } +) +SOURCES["r_getattr_cycle"] = """ + import microcosm.walkcases.gcyc_b # noqa: F401 + from microcosm.walkcases.gcyc_a import x + + + def run(): + return x + """ +SOURCES["r_getattr_cycle_first"] = """ + from microcosm.walkcases.gcyc_a import x + + + def run(): + return x + """ +# A package's own __getattr__, whose lookup waits on a cycle: gself_a binds +# it, and in a branch that never runs imports it from gself_b, which imports +# it from gself_a. Python finds a module's __getattr__ in its namespace +# only, so gself_a.inner's __getattr__ never stands for it. +SOURCES["gself_a.__init__"] = """ + def __getattr__(name): + raise AttributeError(name) + + + if False: + from microcosm.walkcases.gself_b import __getattr__ # noqa: F811 + """ +SOURCES["gself_a.inner"] = """ + from microcosm.walkcases import t_getattr_self + + + def __getattr__(name): + return t_getattr_self.X + """ +SOURCES["gself_b"] = """ + from microcosm.walkcases.gself_a import __getattr__ # noqa: F401 + """ +SOURCES["r_getattr_self"] = """ + from microcosm.walkcases.gself_a import __getattr__ as hook + + + def run(): + return hook + """ # A root that makes a Thing reaches every dunder Thing defines, so it reaches # t_add and t_float as well: the walk over-approximates there by design. @@ -2345,6 +3140,10 @@ def run(): "class_global_hidden", "class_global_import", }, + ("r_class_global_inherit", "Kernel.run"): {"class_global_inherit"}, + ("r_walrus_comp", "run"): {"walrus_comp"}, + ("r_global_setter", "run"): {"global_setter", "global_setter_first"}, + ("r_global_read", "Kernel.run"): {"global_read_class", "global_read_first"}, # Scopes ("r_shadow", "run"): {"shadow"}, ("r_comprehension", "run"): {"comprehension"}, @@ -2362,6 +3161,12 @@ def run(): ("r_body_nested", "run"): {"body_nested"}, ("r_global", "run"): {"global"}, ("r_nonlocal", "run"): {"nonlocal"}, + ("r_comp_first_iter", "run"): {"comp_first_iter"}, + ("r_self_ref", "Kernel.run"): {"self_ref"}, + ("r_aug_class", "Kernel.run"): {"aug_class"}, + ("r_del_class", "Kernel.run"): {"del_class", "del_module"}, + ("r_with_class", "Kernel.run"): {"with_class"}, + ("r_ann_class", "Kernel.run"): {"ann_class"}, # Aliases ("r_alias", "run"): {"alias"}, ("r_unpack", "run"): {"unpack"}, @@ -2374,6 +3179,7 @@ def run(): ("r_base_or", "Kernel.run"): {"base_or"}, ("r_base_walrus", "Kernel.run"): {"base_walrus"}, ("r_base_starred", "Kernel.run"): {"base_starred"}, + ("r_base_set", "Kernel.run"): {"base_set"}, ("r_base_dunder_import", "Kernel.run"): {"base_dunder_import"}, ("r_import_alias_base", "Kernel.run"): {"base_import"}, ("r_getattr_alias", "run"): {"getattr_alias"}, @@ -2428,6 +3234,8 @@ def run(): ("r_setattr_patch", "Kernel.run"): {"setattr_patch"}, ("r_delattr_patch", "Kernel.run"): {"delattr_patch"}, ("r_class_body_update", "Kernel.run"): {"class_body_update"}, + ("r_class_global_aug", "Kernel.run"): {"class_global_aug"}, + ("r_aug_store", "run"): {"aug_store"}, # Changes by name ("r_update", "run"): {"update"}, ("r_setitem", "run"): {"setitem"}, @@ -2445,6 +3253,10 @@ def run(): ("r_augassign", "run"): {"augassign"}, ("r_class_member_change", "Kernel.run"): {"class_member_change"}, ("r_class_nested_fill", "run"): {"class_nested_fill"}, + ("r_kw_fill", "run"): {"kw_fill"}, + ("r_nested_change", "run"): {"nested_change"}, + ("r_class_iadd", "run"): {"class_iadd"}, + ("r_class_comp_fill", "run"): {"class_comp_fill"}, # Attribute fallback ("r_prop", "run"): {"prop", *_THING}, ("r_value", "run"): {"value"}, @@ -2469,6 +3281,7 @@ def run(): ("r_import_spec", "run"): {"import_spec"}, ("r_import_name", "run"): {"import_name"}, ("r_import_renamed", "run"): {"import_renamed", "import_renamed_alias"}, + ("r_getattr_renamed", "run"): {"getattr_renamed"}, # Cost ("r_star_diamond", "run"): {"star_diamond"}, ("r_star_clique", "run"): {"star_clique"}, @@ -2479,15 +3292,41 @@ def run(): ("r_lazy_cycle", "run"): {"lazy_cycle"}, ("r_alias_cycle", "run"): {"alias_cycle"}, ("r_alias_pure_cycle", "run"): set(), + ("r_alias_cycle_dotted", "run"): {"alias_cycle_dotted"}, + ("r_alias_cycle_dotted_first", "run"): {"alias_cycle_dotted_first"}, + ("r_alias_cycle_join", "run"): {"alias_cycle_join"}, + ("r_alias_self_dotted", "run"): {"alias_self_dotted"}, + ("r_getattr_cycle", "run"): {"getattr_cycle_a", "getattr_cycle_b"}, + ("r_getattr_cycle_first", "run"): {"getattr_cycle_a", "getattr_cycle_b"}, + ("r_getattr_self", "run"): set(), # What the walk can't see ("r_registry_decorator", "run"): set(), + ("r_aug_module_effect", "run"): set(), + ("r_nested_global_value", "run"): set(), + ("r_nested_base_scope", "run"): set(), + ("r_function_global", "run"): set(), + ("r_function_global_import", "run"): set(), + ("r_class_global_match", "run"): set(), + ("r_lambda_change", "run"): set(), + ("r_spec_name", "run"): set(), + ("r_partial_import", "run"): set(), + ("r_getattr_importer", "run"): set(), + ("r_match_args_constant", "run"): set(), } #: Targets a root's walk reaches that its run doesn't read. The walk reaches -#: every dunder of a class it references, both branches of a condition, and -#: both operands of ``or``. +#: every dunder of a class it references, both branches of a condition, both +#: operands of ``or``, every binding of a module-level name (a class body's +#: ``global`` one too), what each statement in a class body it reaches +#: names, even a binding that a ``del`` undoes, and in a cycle of imports, +#: each module's ``__getattr__``, whichever module Python imports first. OVER: dict[tuple[str, str], set[str]] = { ("r_alias_cycle", "run"): {"alias_cycle"}, + ("r_getattr_cycle", "run"): {"getattr_cycle_a"}, + ("r_getattr_cycle_first", "run"): {"getattr_cycle_b"}, + ("r_global_setter", "run"): {"global_setter_first"}, + ("r_global_read", "Kernel.run"): {"global_read_first"}, + ("r_del_class", "Kernel.run"): {"del_class"}, ("r_float", "run"): {"add"}, ("r_add", "run"): {"float"}, ("r_prop", "run"): _THING, @@ -2504,6 +3343,28 @@ def run(): ("r_factory", "run"): {"base_attr"}, # Registration: the decorator records _entry, and run() calls it. ("r_registry_decorator", "run"): {"registry_decorator"}, + # An operator inside a statement that binds a name: X += 1 runs at import. + ("r_aug_module_effect", "run"): {"aug_module_effect"}, + # In a class nested in a class body, the value bound to a global name, and + # the bases of a nested class the walk runs whole: it reads their names at + # module level. + ("r_nested_global_value", "run"): {"nested_global_value"}, + ("r_nested_base_scope", "run"): {"nested_base_scope"}, + # A binding that a function makes through global: a match class pattern + # and an attribute chain through the name read the module's bindings only. + ("r_function_global", "run"): {"function_global"}, + ("r_function_global_import", "run"): {"function_global_import"}, + # A match class pattern through a class body's global name, which the + # body binds to its own import. + ("r_class_global_match", "run"): {"class_global_match"}, + # A change in place inside a lambda that a module-level statement calls. + ("r_lambda_change", "run"): {"lambda_change"}, + # import_module with another package, and importers that a call returns. + ("r_spec_name", "run"): {"spec_name"}, + ("r_partial_import", "run"): {"partial_import"}, + ("r_getattr_importer", "run"): {"getattr_importer"}, + # An explicit __match_args__ other than a tuple of string literals. + ("r_match_args_constant", "run"): {"match_args_constant"}, } From 6d0a1a040f286f7ce433e9ad0ad4ae3329735ca9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 17:37:33 -0500 Subject: [PATCH 12/14] NZ G8-x2: settle a cycle with the earlier lookup it meets (#1567) The delta review of e6e1f4c4e asked for two changes, offered three more and listed four nits. Belgium's review of e6e1f4c4e asked for the same two changes and two roots. This commit takes all of them. - A cycle that meets an earlier lookup (required; Belgium's first change). e6e1f4c4e settled a cycle when its first lookup ended, even when a pass had met a lookup that began before the cycle and hadn't settled. The cycle's lookups then froze with that lookup's partial items, so what the walk reached still depended on which lookup it met first. Now such a cycle stays open, as a single lookup that meets an earlier one already did, and joins the earlier lookup's cycle, which runs it again with the rest (_settled, as both reviews proposed). New roots: - r_alias_cycle_outer, the review's shape: a pass meets a lookup under way; - r_alias_cycle_two_up, with that lookup two lookups up; - r_alias_cycle_sibling, with a lookup that waits rather than runs; - Belgium's r_outer_cycle_first, where only a later pass meets the earlier lookup, and r_outer_cycle, which meets the same lookups in the other order. Their modules take the prefix ocyc, since cyco already names r_alias_cycle_outer's package. On e6e1f4c4e's walk the first four miss their target and fail the fresh-walk check, and r_outer_cycle fails the fresh-walk check. - The composing property (required; Belgium's second change). r_getattr_cycle and r_getattr_cycle_first shared one pair of modules, whose reads depend on which root imports them first, and Python imports a module once. So a composite that held both read only the first part's target, and the property failed in CI on both Pythons. r_getattr_cycle_first now imports its own copy of the pair (gcyf_a and gcyf_b), so no two roots share a module whose reads depend on import order. Belgium proposed skipping the Python check on such draws; the copy keeps that check on every draw. - Lines that no root pinned (finding 3). Each of the review's four knockouts now fails a root: - the += branch's use of the name's value (r_aug_value: X += value hands X's module to Adder.__radd__, which reads an attribute that the module binds); - plain assignments among a class body's global stores (r_class_global_store); - exported()'s settled check (r_getattr_bound, whose module binds the name that its __getattr__ would answer, and r_getattr_settled, where the name's lookup settles before the getter counts). Each holds a decoy target that the walk reaches without the check; - the pass that a join forces (r_join_no_growth). The review's three survivors from c622fc22f fail a root too: getattr under another name in a constant's value (r_getattr_named), a reached class's decorators (r_class_decorated) and keyword-only defaults (r_kw_default). - Module __getattr__ calls outside a cycle of lookups (finding 4). "What it can't see" now lists the three other times Python calls a module's __getattr__: for a name the module binds, when an import cycle reads the module part-built; for a sub module's name, which a from-import asks the package for before it imports the sub module; and for __path__, which a from-import asks a module that isn't a package for. BLIND roots pin each: r_getattr_import_cycle, r_getattr_import_cycle_from and r_getattr_init_cycle (the review's e1 to e3), r_getattr_submodule (e4) and r_getattr_path (e5). No kernel meets them. Of the 565 first-party modules, two define a module-level __getattr__, and the kernels' walks parse one, the package microcosm.graph. Its getter answers only its 48 lazy exports, and the package binds none of them and holds no sub module of the same name. A package has __path__, so Python never asks its getter for it. - The pass log and two tests (finding 5). - _Walk.cycles records each cycle's passes: the lookups each pass ran and how many items and lookups it added. reruns, which tools read, now counts from that log. - test_the_walk_runs_each_lookup_once_outside_a_cycle counts each lookup's runs itself rather than reading the walk's count. Each lookup must run once, and once more for each pass that runs it, and in each cycle every pass but the last must add something and the last nothing. Six knockouts of the pass log each fail it: one more pass after the fixpoint, each member running twice in a pass, a pass that logs that it added nothing, no log, and reruns counting the first pass only (18 cycle roots each), and a log that names the members after the pass rather than the lookups it ran (4 roots). - test_each_lookup_in_a_random_tree_settles_on_what_a_fresh_walk_finds walks 300 seeded random trees (static_reach_cases.random_tree): four modules, a package pk with two sub modules and a getter, and a root whose run() and run2() make the same reads in opposite orders. Each walk passes the once test's checks, each lookup it settles holds what a fresh walk's same lookup finds, and the two walks reach the same items. More than half of the 600 walks must run a cycle again. e6e1f4c4e's walk fails 7 of the 300 trees, and 31 of the first 1,000. - Long cycles (the review's note). The budget test's comment now says that its 10,000 lookups hold for the six Cost roots, not for every cycle: a cycle that gathers one module a pass, over N modules, takes N + 3 passes, and 18,023 lookups at N = 100. - Corrections and nits. - e6e1f4c4e's message said that the walk passed every check on all 151 roots. Each root passed alone, but a composite of r_getattr_cycle and r_getattr_cycle_first failed the composing property, which only CI ran. - It also said that a site goes on each of three cycle roots. Four (value) sites go: one on r_alias_cycle_dotted, two on r_alias_cycle_dotted_first and one on r_alias_self_dotted. - The __match_args__ entry in "What it can't see" stays. The walk could resolve a constant bound once to a tuple of literals, as it resolves a string constant, but no first-party module assigns __match_args__, so the rule would serve no kernel. Checks outside pytest, on CPython 3.14.7 and 3.13.15, which agree: - Each of the 169 roots passes every check: the targets, the runtime reads, the once check, the fresh-walk check and the lookup budget. - The composing property's two checks pass on every pair of the 169 roots (14,196), on 2,000 seeded draws of 3 to 6 roots and on the composite of all 169. - On seeds 0 to 2,999 of random_tree, each walk passes the once checks, each of the 301,826 lookups it settles holds what a fresh walk's same lookup finds, and both orders reach the same items. On the review's 3,000 random 7-module trees, e6e1f4c4e's walk fails the fresh-walk check on 73, and this walk on none. - One-edit knockouts of the walk, run outside the repository: each of the hub's 148 fails a root but two. So do 50 of the c622fc22f review's 51, 57 of the e6e1f4c4e review's 61, each of that review's ports of Belgium's 51, and each of Belgium's 51 under Belgium's own runner. The seven that fail none change nothing a test can see, and the PR body says why for each. - Against e6e1f4c4e's walk, no item goes on NZ's 16 kernel walks, the 169 roots or 200 seeded composites of them. 24 differ (four roots and 20 composites), each holding one of the four roots that miss their target on e6e1f4c4e's walk: 20 gain items, and the other four trade a (value) site for the root's own lines. No kernel walk changes, and none has a lookup cycle. Local run: pytest -n 0 passes the three test files with this commit's text on CPython 3.14.7 (1,679 passed in 611 s on a loaded host). Co-Authored-By: Claude Opus 5.5 --- .../test_transport_income_evidence_kernels.py | 103 ++- test_support/microcosm_build/static_reach.py | 71 +- .../microcosm_build/static_reach_cases.py | 700 +++++++++++++++++- 3 files changed, 831 insertions(+), 43 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py index bf0543744f..899f1c9264 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py @@ -2060,16 +2060,9 @@ def test_property_composing_roots_loses_no_target(walk_case_tree, picked) -> Non ) -def test_the_walk_runs_each_lookup_once_outside_a_cycle( - walk_case_tree, monkeypatch -) -> None: - # A lookup outside a cycle runs once per walk, and a lookup in a cycle - # once more for each pass that runs the cycle again. Each hands back each - # item once, and every lookup a walk begins settles, a lookup that joins a - # cycle during a pass too (r_alias_cycle_join). A walk that ran a lookup - # afresh wherever it met it would take time exponential in a diamond's - # depth, or factorial in a clique's size, on the Cost roots. - _, paths = walk_case_tree +def _counted(monkeypatch) -> Counter[tuple[object, ...]]: + # Counts each lookup's runs, its first and each pass's, in the Counter it + # returns, and checks that each lookup hands back each item once. settled = static_reach._Walk._settled runs: Counter[tuple[object, ...]] = Counter() @@ -2083,14 +2076,42 @@ def counting(): return found monkeypatch.setattr(static_reach._Walk, "_settled", counted) + return runs + + +def _check_runs(walk, runs, label) -> None: + # Every lookup the walk began settled. Each pass over a cycle added items + # or lookups but the last, which added nothing. And each lookup ran once, + # and once more for each pass that ran it. + assert runs, label + assert not walk._began and not walk._open, label + for passes in walk.cycles: + assert all(added for _, added in passes[:-1]), (label, passes) + assert passes[-1][1] == 0, (label, passes) + reruns = walk.reruns + assert set(reruns) <= set(runs), label + for key, count in runs.items(): + assert count == 1 + reruns[key], (label, key) + + +def test_the_walk_runs_each_lookup_once_outside_a_cycle( + walk_case_tree, monkeypatch +) -> None: + # A lookup outside a cycle runs once per walk, and a lookup in a cycle + # once more for each pass that runs the cycle again. Each pass adds items + # or lookups but the last, which adds nothing, so the passes stop at the + # fixpoint. Each lookup hands back each item once, and every lookup a walk + # begins settles, a lookup that joins a cycle during a pass too + # (r_alias_cycle_join and r_join_no_growth). A walk that ran a lookup + # afresh wherever it met it would take time exponential in a diamond's + # depth, or factorial in a clique's size, on the Cost roots. + _, paths = walk_case_tree + runs = _counted(monkeypatch) for module, qualname in _WALK_CASE_ROOTS: runs.clear() walk = static_reach._Walk(paths, ()) walk.run(f"{static_reach_cases.PACKAGE}.{module}", qualname) - assert runs, (module, qualname) - assert not walk._began and not walk._open, (module, qualname) - for key, count in runs.items(): - assert count == 1 + walk.reruns.get(key, 0), (module, key) + _check_runs(walk, runs, (module, qualname)) def _fresh_lookup(walk, key): @@ -2111,10 +2132,14 @@ def test_a_settled_lookup_holds_what_a_fresh_walk_finds(walk_case_tree) -> None: # complete, whichever the walk met first: r_star_clique, r_lazy_cycle and # r_alias_cycle; r_alias_cycle_dotted and r_alias_cycle_dotted_first, # where one lookup maps another's items through an attribute; - # r_alias_self_dotted, where a lookup waits on itself; and r_getattr_cycle + # r_alias_self_dotted, where a lookup waits on itself; r_getattr_cycle # and r_getattr_cycle_first, where a module's __getattr__ answers inside a - # cycle of imports. So each lookup a root's walk settles holds what the - # same lookup finds first in a fresh walk. + # cycle of imports; r_alias_cycle_outer, r_alias_cycle_two_up, + # r_alias_cycle_sibling and r_outer_cycle_first, where a pass meets a + # lookup that began before the cycle and hasn't settled; and + # r_outer_cycle, which meets r_outer_cycle_first's lookups in the other + # order. So each lookup a root's walk settles holds what the same lookup + # finds first in a fresh walk. _, paths = walk_case_tree for module, qualname in _WALK_CASE_ROOTS: walk = static_reach._Walk(paths, ()) @@ -2125,9 +2150,53 @@ def test_a_settled_lookup_holds_what_a_fresh_walk_finds(walk_case_tree) -> None: assert set(_fresh_lookup(fresh, key)) == set(items), (module, key) +#: Seeds of the random trees (static_reach_cases.random_tree) that the +#: random-tree test walks. +_RANDOM_TREE_SEEDS = range(300) + + +def test_each_lookup_in_a_random_tree_settles_on_what_a_fresh_walk_finds( + tmp_path_factory, monkeypatch +) -> None: + # Random trees of modules that import each other's names, star-import, + # bind names to modules, names and attributes, in branches that run and + # in ones that never do, and define __getattr__, so their lookups wait on + # each other in cycles of many shapes. In each tree, run() and run2() + # make the same reads in opposite orders. Each walk runs its lookups as + # the once test checks and settles each lookup on what the same lookup + # finds first in a fresh walk, and both walks reach the same items. + root = tmp_path_factory.mktemp("walk-random") + runs = _counted(monkeypatch) + package = static_reach_cases.RANDOM_PACKAGE + own = {(f"{package}.r", "function", name) for name in ("run", "run2")} + cyclic = 0 + for seed in _RANDOM_TREE_SEEDS: + sources = static_reach_cases.random_tree(random.Random(seed)) + paths = static_reach_cases.write_tree(root / str(seed), sources) + reached = [] + for qualname in ("run", "run2"): + runs.clear() + walk = static_reach._Walk(paths, ()) + walk.run(f"{package}.r", qualname) + _check_runs(walk, runs, (seed, qualname)) + cyclic += bool(walk.cycles) + for key, items in walk._done.items(): + fresh = static_reach._Walk(paths, ()) + fresh.modules = walk.modules + found = set(_fresh_lookup(fresh, key)) + assert found == set(items), (seed, qualname, key) + reached.append(set(walk.reached) - own) + assert reached[0] == reached[1], seed + # More than half the walks run a cycle again, so the trees test the passes. + assert cyclic > len(_RANDOM_TREE_SEEDS) + + #: Cost roots on which a walk that ran each lookup afresh wherever it met it #: would make more than 300,000 lookups: each diamond doubles its paths at #: each of its 22 levels, and the clique's grow factorially with its size. +#: The budget holds for these roots, not for every cycle: a cycle that +#: gathers one module a pass, X = Y.sub and Y = X over N modules that each +#: bind sub to the next, takes N + 3 passes, and 18,023 lookups at N = 100. _COST_ROOTS = [ ("r_alias_diamond", "run"), ("r_alias_modules", "run"), diff --git a/test_support/microcosm_build/static_reach.py b/test_support/microcosm_build/static_reach.py index 6a5bf681d5..10c7764393 100644 --- a/test_support/microcosm_build/static_reach.py +++ b/test_support/microcosm_build/static_reach.py @@ -92,7 +92,9 @@ module, what a constant stands for, a package's lazy exports, and a class's inherited ``__match_args__``. Lookups that wait on each other in a cycle run again, each meeting what the others hold so far, until a pass over the cycle -adds nothing. So each settles on what it finds once the others are complete, +adds nothing. A pass that meets a lookup that began before the cycle and +hasn't settled joins the cycle to that lookup's, and the joined cycle runs +again as one. So each settles on what it finds once the others are complete, whichever the walk met first. Within a cycle, a module's ``__getattr__`` answers for a name whose lookup hasn't settled, since Python can read the module part-built, before it binds the name. A walk that ran each lookup @@ -118,6 +120,14 @@ - code reached by registration rather than by name: a function that a decorator records for a framework or a registry to call (``singledispatch``, pydantic's validators); +- a module's ``__getattr__`` where Python calls it outside a cycle of + lookups: for a name the module binds, when an import cycle reads the + module part-built (the module imports another, which runs + ``from module import name`` before the module binds the name); for a sub + module's name, which a ``from`` import asks the package for before it + imports the sub module, and whose answer takes the sub module's place; + and for ``__path__``, which a ``from`` import asks a module that isn't a + package for; - the import-time code of a module that holds no item the walk reaches, such as the ``__init__`` of a package above a reached module, or a module imported only for its effects, unless a statement there changes a reached @@ -162,7 +172,7 @@ class that the walk runs whole: the walk reads their names at module level; import ast import math -from collections import defaultdict +from collections import Counter, defaultdict from collections.abc import Callable, Iterable, Iterator, Mapping from dataclasses import dataclass from pathlib import Path @@ -956,18 +966,28 @@ def __init__(self, paths: Mapping[str, Path], cuts: Iterable[Cut]) -> None: # Lookups (see _settled): what each settled one holds; when each # unsettled one began, what it holds so far and how to run it again; # the unsettled ones in that order; and those another lookup met - # unsettled. How many times each lookup in a cycle ran again. + # unsettled. self._done: dict[tuple[object, ...], tuple[object, ...]] = {} self._began: dict[tuple[object, ...], int] = {} self._cuts: dict[tuple[object, ...], tuple[object, ...]] = {} self._computes: dict[tuple[object, ...], Callable[[], Iterable[object]]] = {} self._open: list[tuple[object, ...]] = [] self._met: set[tuple[object, ...]] = set() - self.reruns: dict[tuple[object, ...], int] = defaultdict(int) + #: Each time :meth:`_cycle` ran a cycle again, in order: its passes, + #: each the lookups it ran and how many items and lookups it added. + self.cycles: list[list[tuple[tuple[tuple[object, ...], ...], int]]] = [] self._begun = 0 #: When the earliest unsettled lookup that the one under way met began. self._low: float = math.inf + @property + def reruns(self) -> Counter[tuple[object, ...]]: + """How many times each lookup in a cycle ran again (:attr:`cycles`).""" + + return Counter( + key for passes in self.cycles for ran, _ in passes for key in ran + ) + def module(self, name: str) -> _Module | None: if name not in self.paths: return None @@ -1033,7 +1053,10 @@ def _settled[T]( and Tarjan's strongly connected components find the cycle. When the cycle's first lookup ends, the cycle's lookups run again, each meeting what the others hold so far, until a pass adds nothing (see - :meth:`_cycle`). So each ends with what it finds once the others are + :meth:`_cycle`). A pass can meet a lookup that began before the + cycle's first and hasn't settled; the cycle then stays unsettled as + part of that lookup's cycle, whose first lookup runs them all again + when it ends. So each ends with what it finds once the others are complete, whichever the walk met first. That matters where a lookup maps another's items rather than adding them: ``X = Y.sub`` in a cycle with ``Y`` holds what ``Y.sub`` names, not what ``Y`` names. A @@ -1065,6 +1088,11 @@ def _settled[T]( if len(members) > 1 or key in self._met: self._cycle(members, start) items = list(self._cuts[key]) + if self._low < began: + # A pass met a lookup that began earlier and hasn't + # settled, so the cycle is part of that lookup's. + self._low = min(outer, self._low) + return items else: self._cuts[key] = tuple(items) self._low = outer @@ -1082,25 +1110,30 @@ def _cycle(self, members: list[tuple[object, ...]], start: int) -> None: holding in ``_cuts`` its cut. Each pass runs every member again, the last to begin first: each run meets what the others hold so far, and its lookup then holds what it held and what the run found. A lookup - that a run begins and that waits on the cycle joins it. Items and - members only grow, and both are finite, so the passes end. + that a run begins and that waits on the cycle joins it, and runs from + the next pass on. Items and members only grow, and both are finite, + so the passes end. :attr:`cycles` records each pass: the lookups it + ran and how many items and lookups it added. A run that meets a + lookup that began before the first member and hasn't settled lowers + ``_low`` below the first member's start, and :meth:`_settled` then + leaves the members unsettled. """ - grew = True - while grew: - grew = False - for member in reversed(list(members)): - self.reruns[member] += 1 + passes: list[tuple[tuple[tuple[object, ...], ...], int]] = [] + self.cycles.append(passes) + while True: + ran = tuple(reversed(members)) + added = 0 + for member in ran: found = list(self._computes[member]()) joined = self._open[start + len(members) :] - if joined: - members.extend(joined) - grew = True + members.extend(joined) held = self._cuts[member] - now = tuple(dict.fromkeys([*held, *found])) - if len(now) > len(held): - self._cuts[member] = now - grew = True + self._cuts[member] = tuple(dict.fromkeys([*held, *found])) + added += len(joined) + len(self._cuts[member]) - len(held) + passes.append((ran, added)) + if not added: + return def resolve(self, module: str, name: str) -> list[Item]: """Every definition ``name`` can bind in ``module``'s top-level scope.""" diff --git a/test_support/microcosm_build/static_reach_cases.py b/test_support/microcosm_build/static_reach_cases.py index 54b400b3d7..643e61ebaa 100644 --- a/test_support/microcosm_build/static_reach_cases.py +++ b/test_support/microcosm_build/static_reach_cases.py @@ -16,6 +16,7 @@ from __future__ import annotations import json +import random import subprocess import sys import textwrap @@ -36,6 +37,9 @@ "alias_cycle_dotted", "alias_cycle_dotted_first", "alias_cycle_join", + "alias_cycle_outer", + "alias_cycle_sibling", + "alias_cycle_two_up", "alias_diamond", "alias_modules", "alias_self_dotted", @@ -47,6 +51,7 @@ "aug_class", "aug_module_effect", "aug_store", + "aug_value", "augassign", "base_attr", "base_dunder_import", @@ -75,6 +80,7 @@ "body_nested", "class_body_update", "class_comp_fill", + "class_decorated", "class_fill_call", "class_fill_del", "class_fill_item", @@ -89,6 +95,7 @@ "class_global_import", "class_global_inherit", "class_global_match", + "class_global_store", "class_header", "class_iadd", "class_import", @@ -122,12 +129,20 @@ "function_global", "function_global_import", "getattr_alias", + "getattr_bound", "getattr_constant", "getattr_cycle_a", "getattr_cycle_b", + "getattr_import_cycle", + "getattr_import_cycle_from", "getattr_importer", + "getattr_init_cycle", + "getattr_named", + "getattr_path", "getattr_renamed", "getattr_self", + "getattr_settled", + "getattr_submodule", "global", "global_read_class", "global_read_first", @@ -155,6 +170,8 @@ "import_renamed", "import_renamed_alias", "import_spec", + "join_no_growth", + "kw_default", "kw_fill", "lambda", "lambda_change", @@ -182,6 +199,7 @@ "nested_global_value", "nested_scope", "nonlocal", + "outer_cycle", "override", "partial_import", "prop", @@ -228,9 +246,22 @@ #: t_with_module behind a module binding that the class body has surely #: replaced before it reads the name (r_ann_class, r_with_class), and #: t_getattr_self behind a package module's ``__getattr__``, which the -#: package's own ``__getattr__`` never stands for (r_getattr_self). +#: package's own ``__getattr__`` never stands for (r_getattr_self), and +#: t_getattr_bound behind a module's ``__getattr__``, which Python never +#: calls for a name the module binds that code reads as an attribute +#: (r_getattr_bound), and t_getattr_settled behind one that the walk counts +#: only while the lookup of the name waits on a cycle that hasn't settled +#: (r_getattr_settled). DECOYS = frozenset( - {"ann_module", "body_hidden", "getattr_self", "shadowed", "with_module"} + { + "ann_module", + "body_hidden", + "getattr_bound", + "getattr_self", + "getattr_settled", + "shadowed", + "with_module", + } ) #: Each target's source: reads of X or ROWS are recorded, anything else fails. @@ -3038,16 +3069,207 @@ def run(): case X.Base(v): return v """ +# A pass over a cycle can meet a lookup that began before the cycle and +# hasn't settled. run() meets W first (W.nothing, in a branch that never +# runs), and W's binding in another such branch begins X's lookup. X and Y +# form a cycle, and only its second pass reaches cyco.sub, whose sub is W. +# Python binds X to cyco2, whose Base names a property in __match_args__. +SOURCES["cyco.__init__"] = "" +SOURCES["cyco.sub"] = """ + from microcosm.walkcases.r_alias_cycle_outer import W as sub # noqa: F401 + """ +SOURCES["cyco2"] = """ + from microcosm.walkcases import t_alias_cycle_outer + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_alias_cycle_outer.X + """ +SOURCES["r_alias_cycle_outer"] = """ + from microcosm.walkcases import cyco2 + + W = cyco2 + if False: + W = X + import microcosm.walkcases.cyco.sub # noqa: E402,F401 + from microcosm.walkcases import cyco # noqa: E402 + + Y = cyco + X = Y.sub + Y = X + X = Y.sub + + + def run(): + if False: + W.nothing + match X.Base(): + case X.Base(v): + return v + """ +# The same, with the lookup under way two lookups up: V's binding begins +# W's, W's begins the X and Y cycle, and cyct.sub's sub is V. +SOURCES["cyct.__init__"] = "" +SOURCES["cyct.sub"] = """ + from microcosm.walkcases.r_alias_cycle_two_up import V as sub # noqa: F401 + """ +SOURCES["cyct2"] = """ + from microcosm.walkcases import t_alias_cycle_two_up + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_alias_cycle_two_up.X + """ +SOURCES["r_alias_cycle_two_up"] = """ + from microcosm.walkcases import cyct2 + + V = cyct2 + if False: + V = W + W = cyct2 + if False: + W = X + import microcosm.walkcases.cyct.sub # noqa: E402,F401 + from microcosm.walkcases import cyct # noqa: E402 + + Y = cyct + X = Y.sub + Y = X + X = Y.sub + + + def run(): + if False: + V.nothing + match X.Base(): + case X.Base(v): + return v + """ +# The same, with a lookup that waits rather than one under way: U's +# bindings begin W's, which waits on U, then the X and Y cycle, whose +# second pass meets W through cycb.sub. +SOURCES["cycb.__init__"] = "" +SOURCES["cycb.sub"] = """ + from microcosm.walkcases.r_alias_cycle_sibling import W as sub # noqa: F401 + """ +SOURCES["cycb2"] = """ + from microcosm.walkcases import t_alias_cycle_sibling + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_alias_cycle_sibling.X + """ +SOURCES["r_alias_cycle_sibling"] = """ + from microcosm.walkcases import cycb2 + + U = cycb2 + if False: + U = W + U = X + W = cycb2 + if False: + W = U + import microcosm.walkcases.cycb.sub # noqa: E402,F401 + from microcosm.walkcases import cycb # noqa: E402 + + Y = cycb + X = Y.sub + Y = X + X = Y.sub + + + def run(): + if False: + U.nothing + match X.Base(): + case X.Base(v): + return v + """ +# A cycle whose passes find, only on a later pass, a lookup that began before +# the cycle's first one. X = X.sub twice reads ocyc.sub.sub on its third pass, +# which is ocyc_e.Z, and Z names X again in a branch that never runs. Python +# binds X to ocyc_t. r_outer_cycle_first reads ocyc_e.Z first, so Z's lookup +# is under way when X's cycle begins; r_outer_cycle meets X first. +SOURCES["ocyc.__init__"] = "" +SOURCES["ocyc.sub"] = """ + from microcosm.walkcases import ocyc_e as _e + + sub = _e.Z + """ +SOURCES["ocyc_t"] = """ + from microcosm.walkcases import t_outer_cycle + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_outer_cycle.X + """ +SOURCES["ocyc_e"] = """ + from microcosm.walkcases import ocyc_t + + Z = ocyc_t + if False: + from microcosm.walkcases.ocyc_x import X + + Z = X + """ +SOURCES["ocyc_x"] = """ + import microcosm.walkcases.ocyc.sub # noqa: F401 + from microcosm.walkcases import ocyc + + X = ocyc + X = X.sub + X = X.sub + """ +SOURCES["r_outer_cycle_first"] = """ + from microcosm.walkcases import ocyc_e + from microcosm.walkcases.ocyc_x import X + + + def run(): + seen = ocyc_e.Z.Base + match X.Base(): + case X.Base(v): + return v, seen + """ +SOURCES["r_outer_cycle"] = """ + from microcosm.walkcases import ocyc_e + from microcosm.walkcases.ocyc_x import X + + + def run(): + match X.Base(): + case X.Base(v): + seen = ocyc_e.Z.Base + return v, seen + """ # Two modules that each define __getattr__ and import x from the other. # Whichever Python imports first, the other's ``from ... import x`` finds it # part-built and calls its __getattr__. Each __getattr__ answers x only: a # from-import asks its source for __path__ first. In r_getattr_cycle Python # imports gcyc_b first, so it reads t_getattr_cycle_b; the walk meets gcyc_a's -# x first. In r_getattr_cycle_first both meet gcyc_a first. Either way the -# walk reaches both targets, since it doesn't track import order. +# x first. In r_getattr_cycle_first both meet gcyf_a first. Either way the +# walk reaches both targets, since it doesn't track import order. gcyf_a and +# gcyf_b copy the pair: Python imports a module once, so a run of both roots +# that shared one pair would read only the target of whichever imports first. SOURCES.update( { - f"gcyc_{first}": f""" + f"{pair}_{first}": f""" from microcosm.walkcases import t_getattr_cycle_{first} @@ -3057,8 +3279,9 @@ def __getattr__(name): raise AttributeError(name) - from microcosm.walkcases.gcyc_{second} import x + from microcosm.walkcases.{pair}_{second} import x """ + for pair in ("gcyc", "gcyf") for first, second in (("a", "b"), ("b", "a")) } ) @@ -3071,7 +3294,7 @@ def run(): return x """ SOURCES["r_getattr_cycle_first"] = """ - from microcosm.walkcases.gcyc_a import x + from microcosm.walkcases.gcyf_a import x def run(): @@ -3107,6 +3330,332 @@ def run(): return hook """ +# Python calls a module's __getattr__ for a name the module binds, outside +# a cycle of lookups. gic_m imports gic_n before it binds name, and gic_n's +# from-import finds gic_m part-built; gicf_m does the same and then binds +# name with a from-import; ginit's __init__ does the same with its own sub +# module. +SOURCES["gic_m"] = """ + def __getattr__(attr): + if attr == "name": + from microcosm.walkcases import t_getattr_import_cycle + + return t_getattr_import_cycle.X + raise AttributeError(attr) + + + from microcosm.walkcases.gic_n import other # noqa: E402,F401 + + name = 1 + """ +SOURCES["gic_n"] = """ + from microcosm.walkcases.gic_m import name + + other = 2 + + + def run(): + return name + """ +SOURCES["r_getattr_import_cycle"] = """ + from microcosm.walkcases import gic_m # noqa: F401 + from microcosm.walkcases.gic_n import run as inner + + + def run(): + return inner() + """ +SOURCES["gicf_m"] = """ + def __getattr__(attr): + if attr == "name": + from microcosm.walkcases import t_getattr_import_cycle_from + + return t_getattr_import_cycle_from.X + raise AttributeError(attr) + + + from microcosm.walkcases.gicf_n import other # noqa: E402,F401 + from microcosm.walkcases.gicf_q import name # noqa: E402,F401 + """ +SOURCES["gicf_n"] = """ + from microcosm.walkcases.gicf_m import name + + other = 2 + + + def run(): + return name + """ +SOURCES["gicf_q"] = """ + name = 1 + """ +SOURCES["r_getattr_import_cycle_from"] = """ + from microcosm.walkcases import gicf_m # noqa: F401 + from microcosm.walkcases.gicf_n import run as inner + + + def run(): + return inner() + """ +SOURCES["ginit.__init__"] = """ + def __getattr__(attr): + if attr == "name": + from microcosm.walkcases import t_getattr_init_cycle + + return t_getattr_init_cycle.X + raise AttributeError(attr) + + + from microcosm.walkcases.ginit.sub import other # noqa: E402,F401 + + name = 1 + """ +SOURCES["ginit.sub"] = """ + from microcosm.walkcases.ginit import name + + other = 2 + + + def run(): + return name + """ +SOURCES["r_getattr_init_cycle"] = """ + from microcosm.walkcases.ginit.sub import run as inner + + + def run(): + return inner() + """ +# A from-import asks a package for a sub module's name before it imports +# the sub module, and a getter that answers takes the sub module's place. +SOURCES["gsub.__init__"] = """ + def __getattr__(attr): + if attr == "sub": + from microcosm.walkcases import t_getattr_submodule + + return t_getattr_submodule.X + raise AttributeError(attr) + """ +SOURCES["gsub.sub"] = """ + VALUE = 1 + """ +SOURCES["r_getattr_submodule"] = """ + from microcosm.walkcases.gsub import sub + + + def run(): + return sub + """ +# Every from-import asks a module that isn't a package for __path__. +SOURCES["gpath"] = """ + from microcosm.walkcases import t_getattr_path + + + def __getattr__(attr): + t_getattr_path.X + raise AttributeError(attr) + + + x = 1 + """ +SOURCES["r_getattr_path"] = """ + from microcosm.walkcases.gpath import x + + + def run(): + return x + """ +# A keyword-only default runs where Python defines the function. +SOURCES["r_kw_default"] = """ + from microcosm.walkcases import t_kw_default + + + def run(*, value=t_kw_default.X): + return value + """ +# A reached class's decorators run as Python creates the class. +SOURCES["cdeco"] = """ + from microcosm.walkcases import t_class_decorated + + + def mark(cls): + t_class_decorated.X + return cls + """ +SOURCES["r_class_decorated"] = """ + from microcosm.walkcases.cdeco import mark + + + @mark + class Kernel: + def run(self): + return 1 + """ +# A module's __getattr__ answers only for a name the module hasn't bound: +# Python never calls gbound's for x, which run() reads as an attribute. +SOURCES["gbound"] = """ + from microcosm.walkcases import t_getattr_bound + + + def __getattr__(attr): + t_getattr_bound.X + raise AttributeError(attr) + + + x = 1 + """ +SOURCES["r_getattr_bound"] = """ + from microcosm.walkcases import gbound + + + def run(): + return gbound.x + """ +# ``NAME += value`` uses NAME's module as a value. A module has no +# __iadd__ or __add__, so Python calls Adder.__radd__ with it, which reads +# an attribute that the module binds. +SOURCES["augv"] = """ + from microcosm.walkcases import t_aug_value + + + def helper(): + return t_aug_value.X + """ +SOURCES["r_aug_value"] = """ + from microcosm.walkcases import augv as mod + + + class Adder: + def __radd__(self, other): + return other.helper() + + + def run(): + global mod + mod += Adder() + """ +# A reached class's assignment to a name its body declares global runs as +# Python creates the class. +SOURCES["gstore_setup"] = """ + from microcosm.walkcases import t_class_global_store + + + def make(): + return t_class_global_store.X + """ +SOURCES["r_class_global_store"] = """ + from microcosm.walkcases.gstore_setup import make + + + class Kernel: + global SEEN + SEEN = make() + + def run(self): + return 1 + """ +# getattr under another name in a constant's value: Shape stands for +# named_get_mod.Base, so the positional pattern reads the property that +# Base's __match_args__ names. +SOURCES["named_get_mod"] = """ + from microcosm.walkcases import t_getattr_named + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_getattr_named.X + """ +SOURCES["r_getattr_named"] = """ + from microcosm.walkcases import named_get_mod + + _get = getattr + Shape = _get(named_get_mod, "Base") + + + def run(): + match Shape(): + case Shape(v): + return v + """ +# A lookup that a pass begins and that waits on the cycle, in a pass where +# no lookup in the cycle grows: the cycle runs again for it. a stands for +# jnp_b, and jnp_a's a, b and c name each other in branches that never run. +SOURCES["jnp_a"] = """ + from microcosm.walkcases import jnp_b + + a = jnp_b + if False: + a = c.b + if False: + b = a.c + if False: + c = b.b + """ +SOURCES["jnp_b"] = """ + from microcosm.walkcases import t_join_no_growth + + + class Base: + __match_args__ = ("left",) + + @property + def left(self): + return t_join_no_growth.X + + + if False: + from microcosm.walkcases import jnp_a + + a = jnp_a + c = a.b + """ +SOURCES["r_join_no_growth"] = """ + from microcosm.walkcases.jnp_a import a + + + def run(): + match a.Base(): + case a.Base(v): + return v + """ +# lsg_b and lsg_c import name from each other, in a branch that never runs, +# and lsg_a takes it from lsg_b. That cycle waits on no lookup outside it, so +# it settles when its first lookup ends, and the lookup of lsg_a's name then +# settles too: lsg_a binds name for sure, and its __getattr__ doesn't count. +SOURCES["lsg_a"] = """ + from microcosm.walkcases import t_getattr_settled + + + def __getattr__(attr): + if attr == "name": + return t_getattr_settled.X + raise AttributeError(attr) + + + from microcosm.walkcases.lsg_b import * # noqa: E402,F403 + """ +SOURCES["lsg_b"] = """ + from microcosm.walkcases.lsg_c import * # noqa: F403 + """ +SOURCES["lsg_c"] = """ + if False: + from microcosm.walkcases.lsg_b import name # noqa: F401 + from microcosm.walkcases.lsg_d import * # noqa: E402,F403 + """ +SOURCES["lsg_d"] = """ + name = 1 + """ +SOURCES["r_getattr_settled"] = """ + from microcosm.walkcases.lsg_a import name + + + def run(): + return name + """ + # A root that makes a Thing reaches every dunder Thing defines, so it reaches # t_add and t_float as well: the walk over-approximates there by design. _THING = {"add", "float"} @@ -3117,6 +3666,8 @@ def run(): ("r_default", "run"): {"default"}, ("r_metaclass", "Kernel.run"): {"metaclass"}, ("r_decorator", "Kernel.run"): {"decorator"}, + ("r_kw_default", "run"): {"kw_default"}, + ("r_class_decorated", "Kernel.run"): {"class_decorated"}, # Bindings ("r_star", "run"): {"star"}, ("r_alternative", "run"): {"alternative"}, @@ -3132,6 +3683,7 @@ def run(): ("r_lazy", "run"): {"lazy"}, ("r_lazy_assigned", "run"): {"lazy_assigned"}, ("r_star_lazy", "run"): {"star_lazy"}, + ("r_getattr_bound", "run"): set(), ("r_class_global", "Kernel.run"): { "class_global", "class_global_alias", @@ -3164,6 +3716,7 @@ def run(): ("r_comp_first_iter", "run"): {"comp_first_iter"}, ("r_self_ref", "Kernel.run"): {"self_ref"}, ("r_aug_class", "Kernel.run"): {"aug_class"}, + ("r_aug_value", "run"): {"aug_value"}, ("r_del_class", "Kernel.run"): {"del_class", "del_module"}, ("r_with_class", "Kernel.run"): {"with_class"}, ("r_ann_class", "Kernel.run"): {"ann_class"}, @@ -3235,6 +3788,7 @@ def run(): ("r_delattr_patch", "Kernel.run"): {"delattr_patch"}, ("r_class_body_update", "Kernel.run"): {"class_body_update"}, ("r_class_global_aug", "Kernel.run"): {"class_global_aug"}, + ("r_class_global_store", "Kernel.run"): {"class_global_store"}, ("r_aug_store", "run"): {"aug_store"}, # Changes by name ("r_update", "run"): {"update"}, @@ -3282,6 +3836,7 @@ def run(): ("r_import_name", "run"): {"import_name"}, ("r_import_renamed", "run"): {"import_renamed", "import_renamed_alias"}, ("r_getattr_renamed", "run"): {"getattr_renamed"}, + ("r_getattr_named", "run"): {"getattr_named"}, # Cost ("r_star_diamond", "run"): {"star_diamond"}, ("r_star_clique", "run"): {"star_clique"}, @@ -3296,6 +3851,13 @@ def run(): ("r_alias_cycle_dotted_first", "run"): {"alias_cycle_dotted_first"}, ("r_alias_cycle_join", "run"): {"alias_cycle_join"}, ("r_alias_self_dotted", "run"): {"alias_self_dotted"}, + ("r_alias_cycle_outer", "run"): {"alias_cycle_outer"}, + ("r_alias_cycle_two_up", "run"): {"alias_cycle_two_up"}, + ("r_alias_cycle_sibling", "run"): {"alias_cycle_sibling"}, + ("r_outer_cycle", "run"): {"outer_cycle"}, + ("r_outer_cycle_first", "run"): {"outer_cycle"}, + ("r_join_no_growth", "run"): {"join_no_growth"}, + ("r_getattr_settled", "run"): set(), ("r_getattr_cycle", "run"): {"getattr_cycle_a", "getattr_cycle_b"}, ("r_getattr_cycle_first", "run"): {"getattr_cycle_a", "getattr_cycle_b"}, ("r_getattr_self", "run"): set(), @@ -3312,6 +3874,11 @@ def run(): ("r_partial_import", "run"): set(), ("r_getattr_importer", "run"): set(), ("r_match_args_constant", "run"): set(), + ("r_getattr_import_cycle", "run"): set(), + ("r_getattr_import_cycle_from", "run"): set(), + ("r_getattr_init_cycle", "run"): set(), + ("r_getattr_submodule", "run"): set(), + ("r_getattr_path", "run"): set(), } #: Targets a root's walk reaches that its run doesn't read. The walk reaches @@ -3365,6 +3932,16 @@ def run(): ("r_getattr_importer", "run"): {"getattr_importer"}, # An explicit __match_args__ other than a tuple of string literals. ("r_match_args_constant", "run"): {"match_args_constant"}, + # A module's __getattr__ that Python calls outside a cycle of lookups: for + # a name the module binds, when an import cycle reads the module + # part-built; for a sub module's name, which a from-import asks the + # package for before it imports the sub module; and for __path__, which a + # from-import asks a module that isn't a package for. + ("r_getattr_import_cycle", "run"): {"getattr_import_cycle"}, + ("r_getattr_import_cycle_from", "run"): {"getattr_import_cycle_from"}, + ("r_getattr_init_cycle", "run"): {"getattr_init_cycle"}, + ("r_getattr_submodule", "run"): {"getattr_submodule"}, + ("r_getattr_path", "run"): {"getattr_path"}, } @@ -3487,3 +4064,112 @@ def python_reads( ) for module, qualname in picked } + + +#: The package that holds a random tree's modules (see :func:`random_tree`). +RANDOM_PACKAGE = "microcosm.walkrandom" +_RANDOM_MODULES = ("m0", "m1", "m2", "m3") +_RANDOM_SUBMODULES = ("pk.s0", "pk.s1") +_RANDOM_NAMES = ("a", "b", "c") +_RANDOM_TARGETS = ("t_0", "t_1", "t_2", "t_3") + + +def _random_module(rng: random.Random, own: str) -> str: + """Two to six statements in :func:`random_tree`'s forms, for module ``own``.""" + + others = [module for module in _RANDOM_MODULES if module != own] + sources = [m for m in (*_RANDOM_MODULES, *_RANDOM_SUBMODULES) if m != own] + imported: dict[str, None] = {} + lines = [] + for _ in range(rng.randint(2, 6)): + form = rng.randrange(10) + name, other = rng.choice(_RANDOM_NAMES), rng.choice(_RANDOM_NAMES) + if form == 0: + lines.append(f"from {RANDOM_PACKAGE}.{rng.choice(sources)} import {name}") + elif form == 1: + lines.append(f"from {RANDOM_PACKAGE}.{rng.choice(sources)} import *") + elif form in (2, 6): + module = rng.choice(others if form == 2 else _RANDOM_TARGETS) + imported[module] = None + lines.append(f"{name} = {module}") + elif form == 3: + lines.append(f"{name} = {other}") + elif form == 4: + lines.append(f"{name} = {other}.{rng.choice(_RANDOM_NAMES)}") + elif form == 5: + lines.append(f"if False:\n {name} = {other}") + elif form == 7: + attribute = rng.choice((*_RANDOM_NAMES, "s0", "s1")) + lines.append(f"from {RANDOM_PACKAGE} import pk\n{name} = pk.{attribute}") + elif form == 8: + target = rng.choice(_RANDOM_TARGETS) + imported[target] = None + lines.append(f"def __getattr__(attr):\n return {target}.X") + else: + attribute = rng.choice(_RANDOM_NAMES) + lines.append(f"if False:\n {name} = {other}.{attribute}") + header = [f"from {RANDOM_PACKAGE} import {module}" for module in imported] + return "\n".join([*header, *lines]) + "\n" + + +def random_tree(rng: random.Random) -> dict[str, str]: + """A random tree of modules, each name's source, as :func:`write_tree` takes it. + + ``m0`` to ``m3``, and ``s0`` and ``s1`` in the package ``pk``, each run + two to six statements, each in one of ten forms: a from-import of a name, + or a star import, from one of the other five; a name bound to one of + ``m0`` to ``m3``, to a target ``t_*``, to an attribute of ``pk`` (``s0`` + and ``s1`` among them), to another name or to another name's attribute; + the last two again in a branch that never runs; and a module + ``__getattr__`` that reads a target. ``pk`` defines a ``__getattr__`` + too. ``r``'s ``run()`` reads ``X`` through two to four of the six + modules' names, each through up to two attributes, and ``run2()`` makes + the same reads in the opposite order; both then match a class pattern + through the first name. So the names can wait on each other in cycles of + any shape these forms make. + """ + + sources = {"__init__": "", **{target: "X = 1\n" for target in _RANDOM_TARGETS}} + for module in _RANDOM_MODULES: + sources[module] = _random_module(rng, module) + target = rng.choice(_RANDOM_TARGETS) + sources["pk.__init__"] = ( + f"from {RANDOM_PACKAGE} import {target}\n\n\n" + f"def __getattr__(attr):\n return {target}.X\n" + ) + for module in _RANDOM_SUBMODULES: + sources[module] = _random_module(rng, module) + reads = [] + for _ in range(rng.randint(2, 4)): + module = rng.choice((*_RANDOM_MODULES, *_RANDOM_SUBMODULES)) + name = rng.choice(_RANDOM_NAMES) + chain = "".join( + f".{rng.choice(_RANDOM_NAMES)}" for _ in range(rng.randint(0, 2)) + ) + reads.append((f"{module.replace('.', '_')}_{name}", module, name, chain)) + imports = sorted( + { + f"from {RANDOM_PACKAGE}.{module} import {name} as {local}" + for local, module, name, _ in reads + } + ) + body = [f" {local}{chain}.X" for local, _, _, chain in reads] + first = reads[0][0] + match = ( + f" match {first}.Base():\n case {first}.Base(v):\n pass" + ) + run = "\n".join(["def run():", *body, match]) + run2 = "\n".join(["def run2():", *reversed(body), match]) + sources["r"] = "\n".join(imports) + f"\n\n\n{run}\n\n\n{run2}\n" + return sources + + +def write_tree(root: Path, sources: dict[str, str]) -> dict[str, Path]: + """Write a :func:`random_tree` under ``root``; return its modules' paths.""" + + base = root.joinpath("packages", "random", "src", *RANDOM_PACKAGE.split(".")) + for name, text in sources.items(): + path = base.joinpath(*name.split(".")).with_suffix(".py") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return module_paths(root) From 2278fb3d7e6a593689e6f2c8ff5d81dc4c5ae403 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 20:49:10 -0500 Subject: [PATCH 13/14] NZ G8-x2: count a module's getter unless a name binds for sure (#1567) The Belgium session's review of 6d0a1a040 found a gap in exported(). It skipped a module's __getattr__ whenever the name's resolve lookup had settled with any items, and its comment called that "bound for sure", so any binding counted: - one in a branch, in a try, or under if TYPE_CHECKING; - one that a del or the end of an except clause undoes; - one a star import brings in from a module whose __all__ leaves it out; - a walrus, or a comprehension's name, inside a subscript in an assignment's target. A package's sub module that nothing has imported, read as pkg.sub, counted too. Python calls the getter in each of those cases, so the walk missed what the getter reads. The class-body rule in _bound_before shared two of the gaps: the end of an except clause, and a walrus or a comprehension's name inside a subscript target. This commit applies that session's suggested-getter-v5.diff verbatim: git diff 6d0a1a040 against this commit matches the file byte for byte (sha256 fb084e9e0488f01b...). - _undoes() returns the name a del target or an except clause unbinds, and _stored_here() returns only the names an assignment target spells out. _binds() uses _stored_here(), and _bound_before() uses _undoes(). - _Module gains deleted, sure, sure_stars and names_all, and the new _Walk._binds_for_sure() reads them. exported() skips the getter only when the name's lookup has settled with items and the module's top level binds the name for sure: directly, or through a star import of a public name from a module that binds it for sure and nowhere names __all__. - The docstring's Bindings, Scopes and "What it can't see" entries and the OVER comment follow. "What it can't see" drops the sub module's name, which the walk now sees. It adds a function the module calls while it runs, a name another module deletes or the module removes without del, and an __all__ the walk can't see. - The cases gain ten getter roots for a name a module binds only maybe, and r_getattr_sub_attribute; each reaches its own target. r_getattr_submodule moves from BLIND to ROOTS. r_except_class and r_walrus_class pin the class-body rule, and r_getattr_call_cycle, r_getattr_outside_del and r_getattr_hidden_all pin the new blind spots. The cases hold 185 roots, 222 targets, 14 in OVER, 20 in BLIND and seven decoys. The next commit adds o8's changes on top, and its message gives the checks of both. Co-Authored-By: Claude Opus 5.5 --- test_support/microcosm_build/static_reach.py | 169 ++++++-- .../microcosm_build/static_reach_cases.py | 377 +++++++++++++++++- 2 files changed, 513 insertions(+), 33 deletions(-) diff --git a/test_support/microcosm_build/static_reach.py b/test_support/microcosm_build/static_reach.py index 10c7764393..c1be10ca57 100644 --- a/test_support/microcosm_build/static_reach.py +++ b/test_support/microcosm_build/static_reach.py @@ -16,21 +16,34 @@ - Bindings. A name resolves to every definition that can bind it: each alternative of a ``try`` or an ``if``, at module or function level, star imports, re-exports, and a package's lazy exports through a module-level - ``__getattr__``, defined or assigned. Every form of assignment binds: a - ``for`` or ``with`` target stands for what the loop iterates or the - statement enters, a ``match`` capture for the subject, and a walrus - target for its value. A name that a class body declares ``global`` binds - at module level, whether or not the walk reaches the class. + ``__getattr__``, defined or assigned. A module's ``__getattr__`` answers + for a name unless a statement at the module's top level binds the name + for sure. A branch, a loop or a ``try`` can skip a binding (an import + under ``if TYPE_CHECKING:``, an optional import). A ``del``, or the end + of an ``except`` clause that names it, can undo one, in the module's + scope or through ``global``. The walk counts a star import as binding a + public name for sure only when the other module binds it for sure and + nowhere names ``__all__``. It counts an assignment as binding for sure + only the names its target spells out: a walrus inside a subscript or an + attribute can sit in a branch that doesn't run, and a comprehension's + name there binds in the comprehension. A package holds a sub module's + name only once something imports the sub module. Every form of + assignment binds: a ``for`` or ``with`` target stands for what the loop + iterates or the statement enters, a ``match`` capture for the subject, + and a walrus target for its value. A name that a class body declares + ``global`` binds at module level, whether or not the walk reaches the + class. - Scopes. Each scope binds only its own names, as Python scopes them. A comprehension, lambda or nested function keeps its names to itself, a class body's names stay out of its methods, and ``global`` and ``nonlocal`` send a name outward. A class body reads a name it binds from its own namespace only once a statement at the body's top level has bound it. Before that, - when only a branch, a loop or a bare annotation binds it, or when a ``del`` - can undo it, Python reads the module's binding, so the walk follows that - one too. An augmented assignment (``X += 1``) reads its target before it - binds it, so a class body's ``X += 1`` reads the module's ``X`` until the - body has bound it. + when only a branch, a loop, a bare annotation or a walrus inside a + subscript or an attribute binds it, or when a ``del`` or the end of an + ``except`` clause can undo it, Python can read the module's binding, so + the walk follows that one too. An augmented assignment (``X += 1``) + reads its target before it binds it, so a class body's ``X += 1`` reads + the module's ``X`` until the body has bound it. - Aliases. A module-level constant stands for each module, class or function its value names: ``_helper = helper_module``, ``_Base = pkg.Base``, ``pkg.Base[int]``, either branch of a conditional, each operand of ``and`` @@ -121,13 +134,16 @@ decorator records for a framework or a registry to call (``singledispatch``, pydantic's validators); - a module's ``__getattr__`` where Python calls it outside a cycle of - lookups: for a name the module binds, when an import cycle reads the - module part-built (the module imports another, which runs - ``from module import name`` before the module binds the name); for a sub - module's name, which a ``from`` import asks the package for before it - imports the sub module, and whose answer takes the sub module's place; - and for ``__path__``, which a ``from`` import asks a module that isn't a - package for; + lookups: for a name the module's top level binds for sure, when Python + reads the module part-built (the module imports another, or calls a + function while it runs, that runs ``from module import name`` before the + module binds the name); for a name that another module deletes + (``del module.name``) or that the module removes without ``del`` + (``delattr``, ``globals()``); for a name that a star import leaves out + through an ``__all__`` the walk can't see (one that another module sets, + one set through ``globals()``, or one a star import brings in); and for + ``__path__``, which a ``from`` import asks a module that isn't a package + for; - the import-time code of a module that holds no item the walk reaches, such as the ``__init__`` of a package above a reached module, or a module imported only for its effects, unless a statement there changes a reached @@ -512,6 +528,43 @@ def __init__(self, name: str, path: Path) -> None: for bound in _stored(target): _add(constants[bound], value) self._changes(node, None) + declared = { + name + for node in ast.walk(tree) + if isinstance(node, ast.Global) + for name in node.names + } + #: Names that a ``del`` or the end of an ``except ... as`` clause can + #: unbind at module level: in the module's scope, or anywhere for a + #: name that some scope declares ``global``. + self.deleted = frozenset( + {unbound for node in _same_scope(tree.body) if (unbound := _undoes(node))} + | { + unbound + for node in ast.walk(tree) + if (unbound := _undoes(node)) and unbound in declared + } + ) + #: Names a statement at the module's top level binds whenever it runs + #: to its end (:func:`_binds`), less those in :attr:`deleted`. + self.sure = ( + frozenset(name for statement in tree.body for name in _binds(statement)) + - self.deleted + ) + #: The modules that the module's top-level star imports read. + self.sure_stars = tuple( + binding[0] + for statement in tree.body + if isinstance(statement, ast.ImportFrom) + for bound, binding in _bindings(self, statement).items() + if bound == "*" + ) + #: Whether the module names ``__all__`` anywhere, so that it may + #: define one: a name in any scope, or a ``global`` declaration. + self.names_all = "__all__" in declared or any( + isinstance(node, ast.Name) and node.id == "__all__" + for node in ast.walk(tree) + ) self.imports = dict(imports) self.functions = dict(functions) self.classes = dict(classes) @@ -631,6 +684,39 @@ def _class_bodies(node: ast.ClassDef) -> Iterator[ast.ClassDef]: yield from _class_bodies(child) +def _undoes(node: ast.AST) -> str | None: + """The name a ``del`` target or an ``except ... as`` clause unbinds. + + Python deletes an ``except`` clause's name when the clause ends. + """ + + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Del): + return node.id + if isinstance(node, ast.ExceptHandler): + return node.name + return None + + +def _stored_here(target: ast.AST) -> list[str]: + """The names an assignment target binds whenever the assignment runs to + its end. + + Only the names the target spells out count: a name, or one inside a + tuple, a list or a starred target. Unlike :func:`_stored`, it leaves out + names inside a subscript or an attribute: a walrus there can be skipped + (``d[(x := 1) if c else 0] = 2``), and a comprehension's names bind in + the comprehension. + """ + + if isinstance(target, ast.Name): + return [target.id] + if isinstance(target, ast.Tuple | ast.List): + return [name for element in target.elts for name in _stored_here(element)] + if isinstance(target, ast.Starred): + return _stored_here(target.value) + return [] + + def _binds(node: ast.stmt) -> set[str]: """The names a statement binds whenever it runs to its end.""" @@ -643,15 +729,15 @@ def _binds(node: ast.stmt) -> set[str]: if alias.name != "*" } if isinstance(node, ast.Assign): - return {name for target in node.targets for name in _stored(target)} + return {name for target in node.targets for name in _stored_here(target)} if isinstance(node, ast.AnnAssign | ast.AugAssign) and node.value is not None: - return set(_stored(node.target)) + return set(_stored_here(node.target)) if isinstance(node, ast.With | ast.AsyncWith): return { name for item in node.items if item.optional_vars is not None - for name in _stored(item.optional_vars) + for name in _stored_here(item.optional_vars) } return set() @@ -661,15 +747,15 @@ def _bound_before(body: ast.ClassDef, name: str, node: ast.AST) -> bool: Only a statement at the body's top level that ends before ``node`` starts binds for sure: a branch, a loop or a ``try`` can skip its bindings, a bare - annotation binds nothing, and a ``del`` can undo a binding. + annotation binds nothing, and a ``del`` or the end of an ``except`` clause + that names it can undo a binding. """ if not hasattr(node, "lineno"): return False for child in _same_scope(_inner_parts(body)): - if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Del): - if child.id == name: - return False + if _undoes(child) == name: + return False start = (node.lineno, node.col_offset) for statement in body.body: if (statement.end_lineno, statement.end_col_offset) > start: @@ -1164,6 +1250,35 @@ def resolve_import(self, target: str, attribute: str | None) -> list[Item]: return [(target, "module", None)] if self.is_module(target) else [] return self.exported(target, attribute) + def _binds_for_sure(self, module: str, name: str, seen: tuple[str, ...]) -> bool: + """Whether ``module``'s top level binds ``name`` whenever it runs. + + A top-level statement that binds the name whenever it runs to its end + (:func:`_binds`) binds it for sure, unless a ``del`` or an ``except`` + clause can unbind it (:attr:`_Module.deleted`). So does a top-level + star import of a public name from a module that binds it for sure and + nowhere names ``__all__``, unless the importer can unbind the name. + ``seen`` holds the modules whose star imports led here. + """ + + found = self.module(module) + if found is None or module in seen: + return False + if seen and ( + found.names_all + or "__all__" in found.constants + or "__all__" in found.imports + ): + return False + if name in found.sure: + return True + if name.startswith("_") or name in found.deleted: + return False + return any( + self._binds_for_sure(star, name, (*seen, module)) + for star in found.sure_stars + ) + def exported(self, module: str, name: str) -> list[Item]: """What ``module.name`` can be, read from outside ``module``.""" @@ -1175,7 +1290,11 @@ def exported(self, module: str, name: str) -> list[Item]: if found is None or name == "__getattr__": # Python finds a module's __getattr__ in its namespace only. return items - if items and ("resolve", module, name) in self._done: + if ( + items + and ("resolve", module, name) in self._done + and self._binds_for_sure(module, name, ()) + ): # Bound for sure. A lookup that hasn't settled waits on a cycle, # where Python can read the module part-built, before it binds # the name, so its __getattr__ counts too. diff --git a/test_support/microcosm_build/static_reach_cases.py b/test_support/microcosm_build/static_reach_cases.py index 643e61ebaa..c844d44e3a 100644 --- a/test_support/microcosm_build/static_reach_cases.py +++ b/test_support/microcosm_build/static_reach_cases.py @@ -120,6 +120,8 @@ "delattr_patch", "dotted", "dunder_top", + "except_class", + "except_module", "factory", "fill_call", "fill_global", @@ -130,18 +132,32 @@ "function_global_import", "getattr_alias", "getattr_bound", + "getattr_call_cycle", "getattr_constant", "getattr_cycle_a", "getattr_cycle_b", + "getattr_hidden_all", "getattr_import_cycle", "getattr_import_cycle_from", "getattr_importer", "getattr_init_cycle", + "getattr_maybe_branch", + "getattr_maybe_deleted", + "getattr_maybe_except", + "getattr_maybe_global_del", + "getattr_maybe_optional", + "getattr_maybe_star_all", + "getattr_maybe_star_del", + "getattr_maybe_target", + "getattr_maybe_typing", + "getattr_maybe_walrus", "getattr_named", + "getattr_outside_del", "getattr_path", "getattr_renamed", "getattr_self", "getattr_settled", + "getattr_sub_attribute", "getattr_submodule", "global", "global_read_class", @@ -229,7 +245,9 @@ "update", "value", "walrus", + "walrus_class", "walrus_comp", + "walrus_module", "with_class", "with_module", "with_store", @@ -1012,6 +1030,41 @@ class Kernel: del SOURCE source = SOURCE + def run(self): + return self.source.X + """, + # The end of an except clause that names a class body's binding undoes + # it too, so a read after it reads the module's binding. + "r_except_class": """ + from microcosm.walkcases import t_except_class, t_except_module + + SOURCE = t_except_module + + + class Kernel: + SOURCE = t_except_class + try: + raise ValueError + except ValueError as SOURCE: + pass + source = SOURCE + + def run(self): + return self.source.X + """, + # A walrus inside a subscript of an assignment's target can be skipped, + # so a read after it reads the module's binding too. + "r_walrus_class": """ + from microcosm.walkcases import t_walrus_class, t_walrus_module + + SOURCE = t_walrus_module + + + class Kernel: + rows = {} + rows[(SOURCE := t_walrus_class) if False else 0] = 1 + source = SOURCE + def run(self): return self.source.X """, @@ -3465,6 +3518,292 @@ def __getattr__(attr): def run(): return x """ +# A module's __getattr__ answers for a name that the module binds only +# maybe: in a branch that never runs, in an import for type checkers, in a try +# whose import fails, before a del (at module level, or in a function that +# declares the name global), before an except clause that names it, or through +# a star import that a del or an __all__ can undo. A walrus inside a +# subscript of an assignment's target can be skipped, and a comprehension's +# name there binds in the comprehension, not the module. A package's +# __getattr__ answers for a sub module that nothing has imported, read as an +# attribute. +SOURCES["gmb_branch"] = """ + from microcosm.walkcases import t_getattr_maybe_branch + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_branch.X + raise AttributeError(attr) + + + if False: + name = 1 + """ +SOURCES["gmb_typing"] = """ + from typing import TYPE_CHECKING + + from microcosm.walkcases import t_getattr_maybe_typing + + if TYPE_CHECKING: + from microcosm.walkcases.gmb_typed import name + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_typing.X + raise AttributeError(attr) + """ +SOURCES["gmb_typed"] = """ + name = 1 + """ +SOURCES["gmb_optional"] = """ + from microcosm.walkcases import t_getattr_maybe_optional + + try: + from microcosm.walkcases.gmb_missing import name + except ImportError: + pass + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_optional.X + raise AttributeError(attr) + """ +SOURCES["gmb_missing"] = """ + raise ImportError("an optional dependency that isn't installed") + + name = 1 + """ +SOURCES["gmb_deleted"] = """ + from microcosm.walkcases import t_getattr_maybe_deleted + + name = 1 + del name + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_deleted.X + raise AttributeError(attr) + """ +SOURCES["gmb_global_del"] = """ + from microcosm.walkcases import t_getattr_maybe_global_del + + name = 1 + + + def _drop(): + global name + del name + + + _drop() + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_global_del.X + raise AttributeError(attr) + """ +SOURCES["gmb_except"] = """ + from microcosm.walkcases import t_getattr_maybe_except + + name = 1 + try: + raise ValueError + except ValueError as name: + pass + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_except.X + raise AttributeError(attr) + """ +SOURCES["gmb_star_src"] = """ + name = 1 + """ +SOURCES["gmb_star_del"] = """ + from microcosm.walkcases import t_getattr_maybe_star_del + from microcosm.walkcases.gmb_star_src import * + + del name + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_star_del.X + raise AttributeError(attr) + """ +SOURCES["gmb_star_all_src"] = """ + name = 1 + + + def _hide(): + global __all__ + __all__ = [] + + + _hide() + """ +SOURCES["gmb_star_all"] = """ + from microcosm.walkcases import t_getattr_maybe_star_all + from microcosm.walkcases.gmb_star_all_src import * + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_star_all.X + raise AttributeError(attr) + """ +SOURCES["gmb_target"] = """ + from microcosm.walkcases import t_getattr_maybe_target + + rows = {} + rows[[name for name in "a"][0]] = 1 + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_target.X + raise AttributeError(attr) + """ +SOURCES["gmb_walrus"] = """ + from microcosm.walkcases import t_getattr_maybe_walrus + + rows = {} + rows[(name := 1) if False else 0] = 1 + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_walrus.X + raise AttributeError(attr) + """ +SOURCES.update( + { + f"r_getattr_maybe_{form}": f""" + from microcosm.walkcases.gmb_{form} import name + + + def run(): + return name + """ + for form in ( + "branch", + "typing", + "optional", + "deleted", + "global_del", + "except", + "star_del", + "star_all", + "target", + "walrus", + ) + } +) +SOURCES["gsa.__init__"] = """ + def __getattr__(attr): + if attr == "sub": + from microcosm.walkcases import t_getattr_sub_attribute + + return t_getattr_sub_attribute.X + raise AttributeError(attr) + """ +SOURCES["gsa.sub"] = """ + VALUE = 1 + """ +SOURCES["r_getattr_sub_attribute"] = """ + from microcosm.walkcases import gsa + + + def run(): + return gsa.sub + """ +# Python reads a module part-built when the module calls a function while it +# runs that imports from it; the walk counts the module's binding only. +SOURCES["gcc_m"] = """ + from microcosm.walkcases import gcc_h, t_getattr_call_cycle + + + def __getattr__(attr): + if attr == "name": + return t_getattr_call_cycle.X + raise AttributeError(attr) + + + early = gcc_h.go() + name = 1 + """ +SOURCES["gcc_h"] = """ + def go(): + from microcosm.walkcases.gcc_m import name + + return name + """ +SOURCES["r_getattr_call_cycle"] = """ + from microcosm.walkcases.gcc_m import early + + + def run(): + return early + """ +# Another module deletes a name the module binds for sure. +SOURCES["god_m"] = """ + from microcosm.walkcases import t_getattr_outside_del + + name = 1 + + + def __getattr__(attr): + if attr == "name": + return t_getattr_outside_del.X + raise AttributeError(attr) + """ +SOURCES["god_drop"] = """ + from microcosm.walkcases import god_m + + del god_m.name + """ +SOURCES["r_getattr_outside_del"] = """ + from microcosm.walkcases import god_drop # noqa: F401 + from microcosm.walkcases.god_m import name + + + def run(): + return name + """ +# A star import leaves a name out through an __all__ that another module +# sets, which the walk doesn't see. +SOURCES["gha_src"] = """ + name = 1 + """ +SOURCES["gha_set"] = """ + from microcosm.walkcases import gha_src + + gha_src.__all__ = [] + """ +SOURCES["gha_m"] = """ + from microcosm.walkcases import gha_set # noqa: F401 + from microcosm.walkcases import t_getattr_hidden_all + from microcosm.walkcases.gha_src import * + + + def __getattr__(attr): + if attr == "name": + return t_getattr_hidden_all.X + raise AttributeError(attr) + """ +SOURCES["r_getattr_hidden_all"] = """ + from microcosm.walkcases.gha_m import name + + + def run(): + return name + """ # A keyword-only default runs where Python defines the function. SOURCES["r_kw_default"] = """ from microcosm.walkcases import t_kw_default @@ -3684,6 +4023,18 @@ def run(): ("r_lazy_assigned", "run"): {"lazy_assigned"}, ("r_star_lazy", "run"): {"star_lazy"}, ("r_getattr_bound", "run"): set(), + ("r_getattr_maybe_branch", "run"): {"getattr_maybe_branch"}, + ("r_getattr_maybe_typing", "run"): {"getattr_maybe_typing"}, + ("r_getattr_maybe_optional", "run"): {"getattr_maybe_optional"}, + ("r_getattr_maybe_deleted", "run"): {"getattr_maybe_deleted"}, + ("r_getattr_maybe_global_del", "run"): {"getattr_maybe_global_del"}, + ("r_getattr_maybe_except", "run"): {"getattr_maybe_except"}, + ("r_getattr_maybe_star_del", "run"): {"getattr_maybe_star_del"}, + ("r_getattr_maybe_star_all", "run"): {"getattr_maybe_star_all"}, + ("r_getattr_maybe_target", "run"): {"getattr_maybe_target"}, + ("r_getattr_maybe_walrus", "run"): {"getattr_maybe_walrus"}, + ("r_getattr_sub_attribute", "run"): {"getattr_sub_attribute"}, + ("r_getattr_submodule", "run"): {"getattr_submodule"}, ("r_class_global", "Kernel.run"): { "class_global", "class_global_alias", @@ -3718,6 +4069,8 @@ def run(): ("r_aug_class", "Kernel.run"): {"aug_class"}, ("r_aug_value", "run"): {"aug_value"}, ("r_del_class", "Kernel.run"): {"del_class", "del_module"}, + ("r_except_class", "Kernel.run"): {"except_class", "except_module"}, + ("r_walrus_class", "Kernel.run"): {"walrus_class", "walrus_module"}, ("r_with_class", "Kernel.run"): {"with_class"}, ("r_ann_class", "Kernel.run"): {"ann_class"}, # Aliases @@ -3877,7 +4230,9 @@ def run(): ("r_getattr_import_cycle", "run"): set(), ("r_getattr_import_cycle_from", "run"): set(), ("r_getattr_init_cycle", "run"): set(), - ("r_getattr_submodule", "run"): set(), + ("r_getattr_call_cycle", "run"): set(), + ("r_getattr_outside_del", "run"): set(), + ("r_getattr_hidden_all", "run"): set(), ("r_getattr_path", "run"): set(), } @@ -3885,8 +4240,9 @@ def run(): #: every dunder of a class it references, both branches of a condition, both #: operands of ``or``, every binding of a module-level name (a class body's #: ``global`` one too), what each statement in a class body it reaches -#: names, even a binding that a ``del`` undoes, and in a cycle of imports, -#: each module's ``__getattr__``, whichever module Python imports first. +#: names, even a binding that a ``del`` or an ``except`` clause undoes, and +#: in a cycle of imports, each module's ``__getattr__``, whichever module +#: Python imports first. OVER: dict[tuple[str, str], set[str]] = { ("r_alias_cycle", "run"): {"alias_cycle"}, ("r_getattr_cycle", "run"): {"getattr_cycle_a"}, @@ -3894,6 +4250,8 @@ def run(): ("r_global_setter", "run"): {"global_setter_first"}, ("r_global_read", "Kernel.run"): {"global_read_first"}, ("r_del_class", "Kernel.run"): {"del_class"}, + ("r_except_class", "Kernel.run"): {"except_class"}, + ("r_walrus_class", "Kernel.run"): {"walrus_class"}, ("r_float", "run"): {"add"}, ("r_add", "run"): {"float"}, ("r_prop", "run"): _THING, @@ -3933,14 +4291,17 @@ def run(): # An explicit __match_args__ other than a tuple of string literals. ("r_match_args_constant", "run"): {"match_args_constant"}, # A module's __getattr__ that Python calls outside a cycle of lookups: for - # a name the module binds, when an import cycle reads the module - # part-built; for a sub module's name, which a from-import asks the - # package for before it imports the sub module; and for __path__, which a - # from-import asks a module that isn't a package for. + # a name the module's top level binds for sure, when Python reads the + # module part-built (through an import cycle, or a function the module + # calls while it runs); for a name another module deletes; for a name a + # star import leaves out through an __all__ that another module sets; and + # for __path__, which a from-import asks a module that isn't a package for. ("r_getattr_import_cycle", "run"): {"getattr_import_cycle"}, ("r_getattr_import_cycle_from", "run"): {"getattr_import_cycle_from"}, ("r_getattr_init_cycle", "run"): {"getattr_init_cycle"}, - ("r_getattr_submodule", "run"): {"getattr_submodule"}, + ("r_getattr_call_cycle", "run"): {"getattr_call_cycle"}, + ("r_getattr_outside_del", "run"): {"getattr_outside_del"}, + ("r_getattr_hidden_all", "run"): {"getattr_hidden_all"}, ("r_getattr_path", "run"): {"getattr_path"}, } From 65e3e7931345105e5071f599e8ff25146e0e01f8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 21:46:55 -0500 Subject: [PATCH 14/14] NZ G8-x2: check what each lookup meets; pin the getter change (#1567) The delta review of 6d0a1a040 approved it with no required change and offered four optional findings and two nits. This commit takes the four findings and adds four roots that pin the getter change in 2278fb3d7. The nits correct the PR body. - The once test checks what each lookup meets (finding 1). The test took the walk's own pass log as its allowance for extra runs, so a walk that did extra work and logged it still passed. _counted now also records, from the calls themselves, each lookup that a call met before the lookup settled: one that had begun and that the memo didn't hold yet. _check_runs adds three checks: each pass runs distinct lookups; each logged cycle's first pass runs more than one lookup, or its one lookup met itself; and no two logged cycles run the same lookups, since a cycle runs its passes once and then settles or joins a larger one. The review's five edits that passed 6d0a1a040's test now fail it: - _cycle called twice, and each member run twice in a pass (each on the 18 cycle roots); - True in place of the cycle condition, and one more logged pass for a lookup that settled outside a cycle (each on all 191 roots); - an open cycle that runs its passes again before it joins (4 roots). They join the hub's knockouts. - Two more getter calls (finding 2). The review named three more times Python calls a module's __getattr__ outside a cycle of lookups. 2278fb3d7 makes the walk see the first: reading a package's attribute asks it for a sub module's name until something imports the sub module (r_getattr_sub_attribute). "What it can't see" now lists the other two: a star import asks a module that doesn't define __all__ for it, and a star import asks for each name that __all__ lists and the module doesn't bind. BLIND roots pin each: r_getattr_star_noall and r_getattr_star_lazy. Python reads each target, and the walk reaches neither. No kernel meets them: no first-party module star-imports. - Nested cycles (finding 3). The budget test's comment names a second shape past its 10,000 lookups: nested cycles, each of which a pass of the next meets. Each then stays open and runs again in every pass of each cycle around it, and K of them make 9,916 lookups at K = 10 and 43,026 at K = 20. The comment's staircase figure moves to 18,125 lookups at N = 100, since 2278fb3d7's walk also looks up the getter of a module that binds a name only maybe. No kernel walk has a lookup cycle. - Random trees (finding 4). random_tree gains an eleventh form, a class Base whose base is another name's Base or the Base of another name's attribute, with a __match_args__ that names one of a, b and c or with none, so that it takes its base's. Before, no tree settled a match lookup. Now, on the test's 300 seeds, 33 trees settle one and 14 run one in a cycle. The random test now requires a walk that runs a match lookup in a cycle (28 of the 600 do), and its floor counts walks that run a cycle through two passes or more (384 do; it needs more than 300). e6e1f4c4e's walk fails 7 of the 300 trees, and 22 of the first 1,000. - Four getter roots for 2278fb3d7: a star import's private name (r_getattr_maybe_star_private), and a walrus inside a subscript of a with target, an annotated target and an augmented target (r_getattr_maybe_with, r_getattr_maybe_annotated and r_getattr_maybe_augmented). Python reads each target through the module's getter, and the walk reaches it; 6d0a1a040's walk doesn't. Without them, two of the hub's one-edit knockouts of 2278fb3d7 failed no root. The cases hold 191 roots, 228 targets, 14 in OVER, 22 in BLIND and seven decoys. Checks outside pytest, on CPython 3.14.7 and 3.13.15, which agree: - The walk's code is 2278fb3d7's: only its docstring changes. - Each of the 191 roots passes every check: the targets, the runtime reads, the once check, the fresh-walk check and the lookup budget. - The composing property's two checks pass on every pair of the 191 roots (18,145), on 2,000 seeded draws of 3 to 6 roots and on the composite of all 191. - On seeds 0 to 2,999 of random_tree, each walk passes the once checks, each of the 287,246 lookups it settles holds what a fresh walk's same lookup finds, and both orders reach the same items. - The test's own _counted and _check_runs, run outside pytest, pass on every root and on seeds 0 to 2,999, and fail each of the five edits. Belgium's four mutants that 6d0a1a040's tests didn't catch (M2, M3, M8 and M9) pass them too; Belgium's review reports that none changes an item or a settled lookup. - One-edit knockouts of the walk, run outside the repository: each of the hub's 164 fails a root or a tree but two that only cost a pass. So do 50 of the c622fc22f review's 51, 57 of the e6e1f4c4e review's 61, each of that review's ports of Belgium's 51, and each of Belgium's 51 under Belgium's own runner, with two of its anchors re-pointed to 2278fb3d7's text. The PR body says why each of the rest changes nothing a test can see. - Against 6d0a1a040's walk, no item goes on NZ's 16 kernel walks, the 191 roots or 200 seeded composites of them; the 90 that differ each hold one of the 18 roots it misses and gain items. Against e6e1f4c4e's walk, no item goes either. 104 walks differ there: the 18, the four roots that miss their target on e6e1f4c4e's walk, and 82 composites that hold one of the 22. 101 gain items, and the other 3 trade a (value) site for the root's own lines. Local run: pytest -n 0 passes the three test files with this commit's text on CPython 3.14.7 (1,723 passed in 138 s). Co-Authored-By: Claude Opus 5.5 --- .../test_transport_income_evidence_kernels.py | 99 +++++++--- test_support/microcosm_build/static_reach.py | 8 +- .../microcosm_build/static_reach_cases.py | 169 ++++++++++++++++-- 3 files changed, 227 insertions(+), 49 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py index 899f1c9264..344258a57b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_income_evidence_kernels.py @@ -2060,13 +2060,23 @@ def test_property_composing_roots_loses_no_target(walk_case_tree, picked) -> Non ) -def _counted(monkeypatch) -> Counter[tuple[object, ...]]: - # Counts each lookup's runs, its first and each pass's, in the Counter it - # returns, and checks that each lookup hands back each item once. +_Lookup = tuple[object, ...] + + +def _counted(monkeypatch) -> tuple[Counter[_Lookup], set[_Lookup]]: + # Counts each lookup's runs, its first and each pass's, and records each + # lookup that a lookup met before it settled: one that had begun to run + # and that the memo didn't hold yet. Both come from the calls, not from + # the walk's own pass log, and each walk starts with both cleared. It + # also checks that each lookup hands back each item once. settled = static_reach._Walk._settled - runs: Counter[tuple[object, ...]] = Counter() + runs: Counter[_Lookup] = Counter() + met: set[_Lookup] = set() def counted(self, key, compute, cut=()): + if runs[key] and key not in self._done: + met.add(key) + def counting(): runs[key] += 1 return compute() @@ -2076,18 +2086,31 @@ def counting(): return found monkeypatch.setattr(static_reach._Walk, "_settled", counted) - return runs - - -def _check_runs(walk, runs, label) -> None: - # Every lookup the walk began settled. Each pass over a cycle added items - # or lookups but the last, which added nothing. And each lookup ran once, - # and once more for each pass that ran it. + return runs, met + + +def _check_runs(walk, counted, label) -> None: + # Every lookup the walk began settled. Each pass over a cycle ran each of + # its lookups once and added items or lookups but the last, which added + # nothing. Each logged cycle's lookups met each other before they + # settled: there were more than one, or the one met itself. No two logged + # cycles ran the same lookups, since a cycle runs its passes once and + # then settles or joins a larger one. And each lookup ran once, and once + # more for each pass that ran it, so the log accounts for every rerun and + # logs only what a cycle needs. + runs, met = counted assert runs, label assert not walk._began and not walk._open, label + logged = set() for passes in walk.cycles: assert all(added for _, added in passes[:-1]), (label, passes) assert passes[-1][1] == 0, (label, passes) + assert all(len(set(ran)) == len(ran) for ran, _ in passes), (label, passes) + first = passes[0][0] + assert len(first) > 1 or first[0] in met, (label, first) + lookups = frozenset(key for ran, _ in passes for key in ran) + assert lookups not in logged, (label, passes) + logged.add(lookups) reruns = walk.reruns assert set(reruns) <= set(runs), label for key, count in runs.items(): @@ -2102,16 +2125,19 @@ def test_the_walk_runs_each_lookup_once_outside_a_cycle( # or lookups but the last, which adds nothing, so the passes stop at the # fixpoint. Each lookup hands back each item once, and every lookup a walk # begins settles, a lookup that joins a cycle during a pass too - # (r_alias_cycle_join and r_join_no_growth). A walk that ran a lookup - # afresh wherever it met it would take time exponential in a diamond's - # depth, or factorial in a clique's size, on the Cost roots. + # (r_alias_cycle_join and r_join_no_growth). The test counts the runs and + # the lookups met under way itself, so a walk that runs a lookup again + # outside a cycle fails here even when it logs the run as a pass. A walk + # that ran a lookup afresh wherever it met it would take time exponential + # in a diamond's depth, or factorial in a clique's size, on the Cost roots. _, paths = walk_case_tree - runs = _counted(monkeypatch) + counted = _counted(monkeypatch) for module, qualname in _WALK_CASE_ROOTS: - runs.clear() + for record in counted: + record.clear() walk = static_reach._Walk(paths, ()) walk.run(f"{static_reach_cases.PACKAGE}.{module}", qualname) - _check_runs(walk, runs, (module, qualname)) + _check_runs(walk, counted, (module, qualname)) def _fresh_lookup(walk, key): @@ -2160,26 +2186,35 @@ def test_each_lookup_in_a_random_tree_settles_on_what_a_fresh_walk_finds( ) -> None: # Random trees of modules that import each other's names, star-import, # bind names to modules, names and attributes, in branches that run and - # in ones that never do, and define __getattr__, so their lookups wait on - # each other in cycles of many shapes. In each tree, run() and run2() - # make the same reads in opposite orders. Each walk runs its lookups as - # the once test checks and settles each lookup on what the same lookup - # finds first in a fresh walk, and both walks reach the same items. + # in ones that never do, define __getattr__, and define classes whose + # __match_args__ a class pattern reads, their own or a base's, so their + # lookups wait on each other in cycles of many shapes. In each tree, + # run() and run2() make the same reads in opposite orders. Each walk runs + # its lookups as the once test checks and settles each lookup on what the + # same lookup finds first in a fresh walk, and both walks reach the same + # items. root = tmp_path_factory.mktemp("walk-random") - runs = _counted(monkeypatch) + counted = _counted(monkeypatch) package = static_reach_cases.RANDOM_PACKAGE own = {(f"{package}.r", "function", name) for name in ("run", "run2")} - cyclic = 0 + cyclic = matched = 0 for seed in _RANDOM_TREE_SEEDS: sources = static_reach_cases.random_tree(random.Random(seed)) paths = static_reach_cases.write_tree(root / str(seed), sources) reached = [] for qualname in ("run", "run2"): - runs.clear() + for record in counted: + record.clear() walk = static_reach._Walk(paths, ()) walk.run(f"{package}.r", qualname) - _check_runs(walk, runs, (seed, qualname)) - cyclic += bool(walk.cycles) + _check_runs(walk, counted, (seed, qualname)) + cyclic += any(len(passes) > 1 for passes in walk.cycles) + matched += any( + key[0] == "match" + for passes in walk.cycles + for ran, _ in passes + for key in ran + ) for key, items in walk._done.items(): fresh = static_reach._Walk(paths, ()) fresh.modules = walk.modules @@ -2187,8 +2222,11 @@ def test_each_lookup_in_a_random_tree_settles_on_what_a_fresh_walk_finds( assert found == set(items), (seed, qualname, key) reached.append(set(walk.reached) - own) assert reached[0] == reached[1], seed - # More than half the walks run a cycle again, so the trees test the passes. + # More than half the walks run a cycle through two passes or more, and + # some run a class's __match_args__ lookup in a cycle, so the trees test + # the passes and those lookups. assert cyclic > len(_RANDOM_TREE_SEEDS) + assert matched #: Cost roots on which a walk that ran each lookup afresh wherever it met it @@ -2196,7 +2234,10 @@ def test_each_lookup_in_a_random_tree_settles_on_what_a_fresh_walk_finds( #: each of its 22 levels, and the clique's grow factorially with its size. #: The budget holds for these roots, not for every cycle: a cycle that #: gathers one module a pass, X = Y.sub and Y = X over N modules that each -#: bind sub to the next, takes N + 3 passes, and 18,023 lookups at N = 100. +#: bind sub to the next, takes N + 3 passes, and 18,125 lookups at N = 100. +#: Nested cycles cost more where a pass of each meets the one around it: +#: each then stays open and runs again in every pass of each cycle around +#: it, and K such cycles make 9,916 lookups at K = 10 and 43,026 at K = 20. _COST_ROOTS = [ ("r_alias_diamond", "run"), ("r_alias_modules", "run"), diff --git a/test_support/microcosm_build/static_reach.py b/test_support/microcosm_build/static_reach.py index c1be10ca57..63a9a485d6 100644 --- a/test_support/microcosm_build/static_reach.py +++ b/test_support/microcosm_build/static_reach.py @@ -141,9 +141,11 @@ (``del module.name``) or that the module removes without ``del`` (``delattr``, ``globals()``); for a name that a star import leaves out through an ``__all__`` the walk can't see (one that another module sets, - one set through ``globals()``, or one a star import brings in); and for - ``__path__``, which a ``from`` import asks a module that isn't a package - for; + one set through ``globals()``, or one a star import brings in); for + ``__all__``, which a star import asks a module that doesn't define it + for; for each name that a star import's ``__all__`` lists and the module + doesn't bind; and for ``__path__``, which a ``from`` import asks a module + that isn't a package for; - the import-time code of a module that holds no item the walk reaches, such as the ``__init__`` of a package above a reached module, or a module imported only for its effects, unless a statement there changes a reached diff --git a/test_support/microcosm_build/static_reach_cases.py b/test_support/microcosm_build/static_reach_cases.py index c844d44e3a..4d5e8ae81c 100644 --- a/test_support/microcosm_build/static_reach_cases.py +++ b/test_support/microcosm_build/static_reach_cases.py @@ -141,6 +141,8 @@ "getattr_import_cycle_from", "getattr_importer", "getattr_init_cycle", + "getattr_maybe_annotated", + "getattr_maybe_augmented", "getattr_maybe_branch", "getattr_maybe_deleted", "getattr_maybe_except", @@ -148,15 +150,19 @@ "getattr_maybe_optional", "getattr_maybe_star_all", "getattr_maybe_star_del", + "getattr_maybe_star_private", "getattr_maybe_target", "getattr_maybe_typing", "getattr_maybe_walrus", + "getattr_maybe_with", "getattr_named", "getattr_outside_del", "getattr_path", "getattr_renamed", "getattr_self", "getattr_settled", + "getattr_star_lazy", + "getattr_star_noall", "getattr_sub_attribute", "getattr_submodule", "global", @@ -3518,13 +3524,58 @@ def __getattr__(attr): def run(): return x """ +# A star import asks a module that doesn't define __all__ for it. The run +# reads a name that the star import binds, so the walk has a lookup. +SOURCES["gsnoall.__init__"] = """ + from microcosm.walkcases import t_getattr_star_noall + + + def __getattr__(attr): + t_getattr_star_noall.X + raise AttributeError(attr) + + + y = 1 + """ +SOURCES["r_getattr_star_noall"] = """ + from microcosm.walkcases.gsnoall import * # noqa: F403 + + + def run(): + return y # noqa: F405 + """ +# A star import asks for each name that __all__ lists and the module doesn't +# bind. The run reads one that the module binds. +SOURCES["gslazy.__init__"] = """ + __all__ = ["lazy", "y"] + + + def __getattr__(attr): + if attr == "lazy": + from microcosm.walkcases import t_getattr_star_lazy + + return t_getattr_star_lazy.X + raise AttributeError(attr) + + + y = 1 + """ +SOURCES["r_getattr_star_lazy"] = """ + from microcosm.walkcases.gslazy import * # noqa: F403 + + + def run(): + return y # noqa: F405 + """ # A module's __getattr__ answers for a name that the module binds only # maybe: in a branch that never runs, in an import for type checkers, in a try # whose import fails, before a del (at module level, or in a function that # declares the name global), before an except clause that names it, or through -# a star import that a del or an __all__ can undo. A walrus inside a -# subscript of an assignment's target can be skipped, and a comprehension's -# name there binds in the comprehension, not the module. A package's +# a star import that a del or an __all__ can undo. A star import brings in +# no name that starts with an underscore from a module without __all__. A +# walrus inside a subscript of a target can be skipped, in a plain, annotated +# or augmented assignment or a with statement, and a comprehension's name +# there binds in the comprehension, not the module. A package's # __getattr__ answers for a sub module that nothing has imported, read as an # attribute. SOURCES["gmb_branch"] = """ @@ -3682,6 +3733,65 @@ def __getattr__(attr): return t_getattr_maybe_walrus.X raise AttributeError(attr) """ +SOURCES["gmb_star_private_src"] = """ + _name = 1 + """ +SOURCES["gmb_star_private"] = """ + from microcosm.walkcases import t_getattr_maybe_star_private + from microcosm.walkcases.gmb_star_private_src import * + + + def __getattr__(attr): + if attr == "_name": + return t_getattr_maybe_star_private.X + raise AttributeError(attr) + """ +SOURCES["r_getattr_maybe_star_private"] = """ + from microcosm.walkcases.gmb_star_private import _name + + + def run(): + return _name + """ +SOURCES["gmb_with"] = """ + from contextlib import nullcontext + + from microcosm.walkcases import t_getattr_maybe_with + + rows = {} + with nullcontext(1) as rows[(name := 1) if False else 0]: + pass + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_with.X + raise AttributeError(attr) + """ +SOURCES["gmb_annotated"] = """ + from microcosm.walkcases import t_getattr_maybe_annotated + + rows = {} + rows[(name := 1) if False else 0]: int = 1 + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_annotated.X + raise AttributeError(attr) + """ +SOURCES["gmb_augmented"] = """ + from microcosm.walkcases import t_getattr_maybe_augmented + + rows = {0: 0} + rows[(name := 1) if False else 0] += 1 + + + def __getattr__(attr): + if attr == "name": + return t_getattr_maybe_augmented.X + raise AttributeError(attr) + """ SOURCES.update( { f"r_getattr_maybe_{form}": f""" @@ -3702,6 +3812,9 @@ def run(): "star_all", "target", "walrus", + "with", + "annotated", + "augmented", ) } ) @@ -4033,6 +4146,10 @@ def run(): ("r_getattr_maybe_star_all", "run"): {"getattr_maybe_star_all"}, ("r_getattr_maybe_target", "run"): {"getattr_maybe_target"}, ("r_getattr_maybe_walrus", "run"): {"getattr_maybe_walrus"}, + ("r_getattr_maybe_with", "run"): {"getattr_maybe_with"}, + ("r_getattr_maybe_annotated", "run"): {"getattr_maybe_annotated"}, + ("r_getattr_maybe_augmented", "run"): {"getattr_maybe_augmented"}, + ("r_getattr_maybe_star_private", "run"): {"getattr_maybe_star_private"}, ("r_getattr_sub_attribute", "run"): {"getattr_sub_attribute"}, ("r_getattr_submodule", "run"): {"getattr_submodule"}, ("r_class_global", "Kernel.run"): { @@ -4234,6 +4351,8 @@ def run(): ("r_getattr_outside_del", "run"): set(), ("r_getattr_hidden_all", "run"): set(), ("r_getattr_path", "run"): set(), + ("r_getattr_star_noall", "run"): set(), + ("r_getattr_star_lazy", "run"): set(), } #: Targets a root's walk reaches that its run doesn't read. The walk reaches @@ -4294,8 +4413,11 @@ def run(): # a name the module's top level binds for sure, when Python reads the # module part-built (through an import cycle, or a function the module # calls while it runs); for a name another module deletes; for a name a - # star import leaves out through an __all__ that another module sets; and - # for __path__, which a from-import asks a module that isn't a package for. + # star import leaves out through an __all__ that another module sets; for + # __all__, which a star import asks a module that doesn't define it for; + # for each name that a star import's __all__ lists and the module doesn't + # bind; and for __path__, which a from-import asks a module that isn't a + # package for. ("r_getattr_import_cycle", "run"): {"getattr_import_cycle"}, ("r_getattr_import_cycle_from", "run"): {"getattr_import_cycle_from"}, ("r_getattr_init_cycle", "run"): {"getattr_init_cycle"}, @@ -4303,6 +4425,8 @@ def run(): ("r_getattr_outside_del", "run"): {"getattr_outside_del"}, ("r_getattr_hidden_all", "run"): {"getattr_hidden_all"}, ("r_getattr_path", "run"): {"getattr_path"}, + ("r_getattr_star_noall", "run"): {"getattr_star_noall"}, + ("r_getattr_star_lazy", "run"): {"getattr_star_lazy"}, } @@ -4443,7 +4567,7 @@ def _random_module(rng: random.Random, own: str) -> str: imported: dict[str, None] = {} lines = [] for _ in range(rng.randint(2, 6)): - form = rng.randrange(10) + form = rng.randrange(11) name, other = rng.choice(_RANDOM_NAMES), rng.choice(_RANDOM_NAMES) if form == 0: lines.append(f"from {RANDOM_PACKAGE}.{rng.choice(sources)} import {name}") @@ -4466,6 +4590,14 @@ def _random_module(rng: random.Random, own: str) -> str: target = rng.choice(_RANDOM_TARGETS) imported[target] = None lines.append(f"def __getattr__(attr):\n return {target}.X") + elif form == 10: + base = rng.choice((other, f"{other}.{rng.choice(_RANDOM_NAMES)}")) + body = ( + f" __match_args__ = ({rng.choice(_RANDOM_NAMES)!r},)" + if rng.randrange(2) + else " pass" + ) + lines.append(f"class Base({base}.Base):\n{body}") else: attribute = rng.choice(_RANDOM_NAMES) lines.append(f"if False:\n {name} = {other}.{attribute}") @@ -4477,17 +4609,20 @@ def random_tree(rng: random.Random) -> dict[str, str]: """A random tree of modules, each name's source, as :func:`write_tree` takes it. ``m0`` to ``m3``, and ``s0`` and ``s1`` in the package ``pk``, each run - two to six statements, each in one of ten forms: a from-import of a name, - or a star import, from one of the other five; a name bound to one of - ``m0`` to ``m3``, to a target ``t_*``, to an attribute of ``pk`` (``s0`` - and ``s1`` among them), to another name or to another name's attribute; - the last two again in a branch that never runs; and a module - ``__getattr__`` that reads a target. ``pk`` defines a ``__getattr__`` - too. ``r``'s ``run()`` reads ``X`` through two to four of the six - modules' names, each through up to two attributes, and ``run2()`` makes - the same reads in the opposite order; both then match a class pattern - through the first name. So the names can wait on each other in cycles of - any shape these forms make. + two to six statements, each in one of eleven forms: a from-import of a + name, or a star import, from one of the other five; a name bound to one + of ``m0`` to ``m3``, to a target ``t_*``, to an attribute of ``pk`` + (``s0`` and ``s1`` among them), to another name or to another name's + attribute; the last two again in a branch that never runs; a module + ``__getattr__`` that reads a target; and a class ``Base`` whose base is + the ``Base`` of another name or of another name's attribute, with a + ``__match_args__`` that names one of ``a``, ``b`` and ``c`` or with none, + so that it takes its base's. ``pk`` defines a ``__getattr__`` too. + ``r``'s ``run()`` reads ``X`` through two to four of the six modules' + names, each through up to two attributes, and ``run2()`` makes the same + reads in the opposite order; both then match a class pattern through the + first name's ``Base``. So the names and the classes' ``__match_args__`` + can wait on each other in cycles of any shape these forms make. """ sources = {"__init__": "", **{target: "X = 1\n" for target in _RANDOM_TARGETS}}