From ea589311d93a8fa2045ad16ebea0aedcd21713f1 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:01:18 +0400 Subject: [PATCH 1/7] Test that target compilation keys cover geography_constants (red) ledger_targets takes a target hierarchy's geography label from microcosm.calibrate.geography_constants, and the UK target loss weights take UK_WIDE_GEOGRAPHY_IDS from it, through `from ... import`. source_hash does not follow imports, and no target-compilation kernel lists the module, so an edit to a label keeps the old node keys (microcosm#1492). New tests, failing on main: - every transport kernel that hashes target_kernels._COMPILE_MODULES (targets.compile@1, takeup.compare@1, takeup.gap@1, transport.create@1/@2, transport.quantile_map@1/@2) changes its implementation hash when geography_constants' file changes; - the same for the UK full target, selection, measures and problem kernels and the national target and problem kernels; - the UK problem kernel's loss-weight hash, the one covering target_weights, includes geography_constants itself. Import-closure checks: test_support/microcosm_build/first_party_imports.py reads a module's AST and returns every first-party (microcosm.*) module it imports, including imports inside functions, and resolves a name imported from a package to the module that defines it (a constant through the package __init__'s own imports). The transport test requires every such import of a _COMPILE_MODULES module to be listed or named, with a reason, in COMPILE_EXCLUDED_IMPORTS, which lives in the test so that editing a reason does not change target_kernels.py's hash. The UK check now uses the same scan instead of one limited to microcosm.build, which is why it never saw a microcosm.calibrate module. On main both report the missing modules. Red run (new and changed tests only, by node ID): 16 failed, 3 passed. The 3 that pass are the two exclusion-liveness checks and the scan's control test. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_transport_compile_modules.py | 159 ++++++++++++++++++ .../uk/test_uk_target_compile_modules.py | 109 ++++++------ .../microcosm_build/first_party_imports.py | 112 ++++++++++++ 3 files changed, 332 insertions(+), 48 deletions(-) create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py create mode 100644 test_support/microcosm_build/first_party_imports.py diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py new file mode 100644 index 0000000000..c7114f25e2 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py @@ -0,0 +1,159 @@ +"""The transport compile kernels hash every module target compilation reads. + +``targets.compile@1`` and every other transport kernel built on target +compilation hash ``target_kernels._COMPILE_MODULES``. ``source_hash`` hashes +the files it is given and does not follow imports, so a module that a listed +module imports, and that nobody lists, can change a compiled surface without +changing a node key. ``ledger_targets`` takes a target's geography label from +``microcosm.calibrate.geography_constants`` that way; before microcosm#1492 no +compile kernel hashed it, so a resumed store published stored surfaces with +the old label. + +``COMPILE_EXCLUDED_IMPORTS`` names every first-party module a listed module +imports but the tuple leaves out, with the reason. It lives here rather than +next to the tuple because ``target_kernels.py`` is hashed: editing a reason +there would give every kernel that hashes the file a new key. +""" + +from __future__ import annotations + +from pathlib import Path +from types import MappingProxyType + +import pytest + +import microcosm.calibrate.geography_constants as geography_constants_module +from microcosm.build.transport import target_kernels +from microcosm.build.transport.population_kernels import ( + TRANSPORT_CREATE, + TRANSPORT_CREATE_DECLARED, + TRANSPORT_QUANTILE_MAP, +) +from microcosm.build.transport.quantile_kernels import TRANSPORT_QUANTILE_MAP_V2 +from microcosm.build.transport.takeup_kernels import TAKEUP_GAP +from microcosm.build.transport.target_kernels import TAKEUP_COMPARE, TARGETS_COMPILE +from test_support.microcosm_build.first_party_imports import ( + first_party_imports, + uncovered_imports, +) + +#: Every kernel whose implementation hash includes ``_COMPILE_MODULES``. The +#: first four compile references with ``compile_target_surface``; the two +#: quantile maps decode the surface an upstream compile stored. +KERNELS_HASHING_THE_COMPILE_MODULES = ( + TARGETS_COMPILE, + TAKEUP_COMPARE, + TAKEUP_GAP, + TRANSPORT_CREATE, + TRANSPORT_CREATE_DECLARED, + TRANSPORT_QUANTILE_MAP, + TRANSPORT_QUANTILE_MAP_V2, +) + +_COUNTRY_SPEC_LOADER = ( + "country_spec is listed for the reference validators target compilation " + "calls, _validate_target_references and _carried_value_keys; it imports " + "this module for its country-spec loader and stage plans, which those " + "validators do not use" +) +_GRAPH_RUNTIME = ( + "the graph runtime (kernel protocol, declarations and the canonical JSON " + "encoder node keys are computed with); no kernel in the repository " + "hashes a microcosm.graph module" +) + +COMPILE_EXCLUDED_IMPORTS = MappingProxyType( + { + "microcosm.build.plan": _COUNTRY_SPEC_LOADER, + "microcosm.build.source_manifest": _COUNTRY_SPEC_LOADER, + "microcosm.build.trace": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.canonical": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.engine_abi": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.loader": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.model": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.schemas": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.take_up_semantics": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.yaml12": _COUNTRY_SPEC_LOADER, + "microcosm.build.reference_intervals": ( + "reference_profiles reaches it only when given an interval " + "document; compile_target_surface calls validate_reference_resolution " + "without one. calibration.prepare@1 and targets.problem@1, which " + "apply interval tables, hash it" + ), + "microcosm.calibrate.artifacts": ( + "artifact_types imports PROBLEM_TYPE to declare targets.problem@1's " + "output; target compilation checks only its own surface output" + ), + "microcosm.diagnostics.schema": ( + "artifact_types imports the diagnostics schema version to declare " + "the diagnostics output type; target compilation emits no diagnostics" + ), + "microcosm.frame.bundle": ( + "the population container: graph_inputs.context_frame builds one, " + "calibrate.target measures a target on one and country_spec's stage " + "plans transform one; target compilation reads facts, not a population" + ), + "microcosm.frame.schema": ( + "graph_inputs.context_frame's entity schema; target compilation " + "reads no population" + ), + "microcosm.graph.canonical": _GRAPH_RUNTIME, + "microcosm.graph.decl": _GRAPH_RUNTIME, + "microcosm.graph.kernel": _GRAPH_RUNTIME, + } +) + + +def _compile_module_names() -> tuple[str, ...]: + return tuple(module.__name__ for module in target_kernels._COMPILE_MODULES) + + +@pytest.mark.parametrize( + "kernel", KERNELS_HASHING_THE_COMPILE_MODULES, ids=lambda kernel: kernel.ref +) +def test_an_edit_to_the_geography_labels_rekeys_the_kernel( + tmp_path, monkeypatch, kernel +): + """ledger_targets labels a target's geography from UK_GEOGRAPHY_ID_TO_LABEL + when the facts give no name, so an edit there must give a new key.""" + before = kernel.implementation_hash() + source = Path(geography_constants_module.__file__) + edited = tmp_path / source.name + edited.write_bytes(source.read_bytes() + b"\n# an edited geography label\n") + monkeypatch.setattr(geography_constants_module, "__file__", str(edited)) + assert kernel.implementation_hash() != before + + +def test_every_first_party_import_of_a_compile_module_is_listed_or_excluded(): + uncovered = uncovered_imports( + _compile_module_names(), frozenset(COMPILE_EXCLUDED_IMPORTS) + ) + assert not uncovered, sorted(uncovered) + + +def test_compile_exclusions_are_live_and_never_listed(): + imported = set().union(*map(first_party_imports, _compile_module_names())) + assert set(COMPILE_EXCLUDED_IMPORTS) <= imported, sorted( + set(COMPILE_EXCLUDED_IMPORTS) - imported + ) + assert not set(COMPILE_EXCLUDED_IMPORTS) & set(_compile_module_names()) + + +def test_the_import_scan_sees_each_import_form(): + """Control: the three forms that hide a module from a plain scan.""" + # A constant imported from its own module. + assert "microcosm.calibrate.geography_constants" in first_party_imports( + "microcosm.build.ledger_targets" + ) + # A class imported from a package resolves to the module defining it. + assert "microcosm.calibrate.registry" in first_party_imports( + "microcosm.build.ledger_targets" + ) + # A constant the package __init__ re-exports resolves through the __init__. + assert "microcosm.diagnostics.schema" in first_party_imports( + "microcosm.build.transport.artifact_types" + ) + # An import inside a function body. + assert "microcosm.build.reference_intervals" in first_party_imports( + "microcosm.build.reference_profiles" + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index 8abc5978f7..0579fe3b4c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -2,10 +2,11 @@ from __future__ import annotations -import ast -import importlib from pathlib import Path +import pytest + +import microcosm.calibrate.geography_constants as geography_constants_module from microcosm.build import ledger_artifact from microcosm.build import ledger_targets as shared_ledger_targets from microcosm.build.uk_runtime import ( @@ -13,65 +14,32 @@ graph_targets, hmrc_uprating, measure_simulation, + target_weights, ) from microcosm.build.uk_runtime.target_compile_modules import ( EXCLUDED_DIRECT_IMPORTS, UK_TARGET_COMPILE_MODULE_NAMES, uk_target_compile_modules, ) - - -def _direct_build_imports(module_name: str) -> set[str]: - """Every ``microcosm.build`` module the named module imports, by AST.""" - - module = importlib.import_module(module_name) - tree = ast.parse(Path(module.__file__).read_text(encoding="utf-8")) - package = module_name.rsplit(".", 1)[0] - candidates: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - if node.level: - base = package.rsplit(".", node.level - 1)[0] - stem = f"{base}.{node.module}" if node.module else base - else: - stem = node.module or "" - if not stem.startswith("microcosm.build"): - continue - candidates.add(stem) - candidates.update(f"{stem}.{alias.name}" for alias in node.names) - elif isinstance(node, ast.Import): - candidates.update( - alias.name - for alias in node.names - if alias.name.startswith("microcosm.build") - ) - modules: set[str] = set() - for name in candidates: - try: - imported = importlib.import_module(name) - except ImportError: - continue - source = getattr(imported, "__file__", None) - if source is not None and not source.endswith("__init__.py"): - modules.add(name) - return modules +from test_support.microcosm_build.first_party_imports import ( + first_party_imports, + uncovered_imports, +) def test_every_direct_import_of_a_listed_module_is_listed_or_excluded(): - listed = set(UK_TARGET_COMPILE_MODULE_NAMES) - uncovered = { - f"{name} -> {imported}" - for name in UK_TARGET_COMPILE_MODULE_NAMES - for imported in _direct_build_imports(name) - if imported not in listed and imported not in EXCLUDED_DIRECT_IMPORTS - } + """Every first-party import, not only microcosm.build: the target + hierarchies take their geography labels from + microcosm.calibrate.geography_constants, which a scan of microcosm.build + imports never saw (microcosm#1492).""" + uncovered = uncovered_imports( + UK_TARGET_COMPILE_MODULE_NAMES, frozenset(EXCLUDED_DIRECT_IMPORTS) + ) assert not uncovered, sorted(uncovered) def test_exclusions_are_live_and_never_listed(): - imported = set().union( - *(_direct_build_imports(name) for name in UK_TARGET_COMPILE_MODULE_NAMES) - ) + imported = set().union(*map(first_party_imports, UK_TARGET_COMPILE_MODULE_NAMES)) assert set(EXCLUDED_DIRECT_IMPORTS) <= imported assert not set(EXCLUDED_DIRECT_IMPORTS) & set(UK_TARGET_COMPILE_MODULE_NAMES) @@ -149,3 +117,48 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): graph_national.UKNationalProblemKernel(resolver_factory=None), ): assert ledger_artifact in _hashed_by(kernel, monkeypatch, graph_national) + + +#: The kernels whose hash includes the UK target compile modules: the full +#: graph's target, selection, measures and problem kernels and the national +#: graph's target and problem kernels. +_TARGET_AND_PROBLEM_KERNELS = ( + graph_targets.UKFullTargetCompilationKernel(), + graph_targets.UKFullTargetSelectionKernel(), + graph_targets.UKFullMeasureKernel(), + graph_targets.UKFullProblemKernel(), + graph_national.UKNationalTargetKernel(), + graph_national.UKNationalProblemKernel(resolver_factory=None), +) + + +@pytest.mark.parametrize( + "kernel", _TARGET_AND_PROBLEM_KERNELS, ids=lambda kernel: kernel.ref +) +def test_an_edit_to_the_geography_labels_rekeys_the_kernel( + tmp_path, monkeypatch, kernel +): + """The shared and UK ledger targets label a target's geography from + UK_GEOGRAPHY_ID_TO_LABEL, so an edit there must give these kernels new + keys instead of reusing a surface with the old label (microcosm#1492).""" + before = kernel.implementation_hash() + source = Path(geography_constants_module.__file__) + edited = tmp_path / source.name + edited.write_bytes(source.read_bytes() + b"\n# an edited geography label\n") + monkeypatch.setattr(geography_constants_module, "__file__", str(edited)) + assert kernel.implementation_hash() != before + + +def test_full_problem_loss_weights_hash_the_uk_wide_geographies(monkeypatch): + """target_weights takes UK_WIDE_GEOGRAPHY_IDS from geography_constants for + the loss weights the problem binds, so the problem's weight hash covers + that module itself, not only through the target compile modules.""" + calls: list[tuple] = [] + monkeypatch.setattr( + graph_targets, + "source_hash", + lambda *objects, **_dependencies: calls.append(objects) or "test", + ) + graph_targets.UKFullProblemKernel().implementation_hash() + (weights,) = [objects for objects in calls if target_weights in objects] + assert geography_constants_module in weights diff --git a/test_support/microcosm_build/first_party_imports.py b/test_support/microcosm_build/first_party_imports.py new file mode 100644 index 0000000000..17bb13c54d --- /dev/null +++ b/test_support/microcosm_build/first_party_imports.py @@ -0,0 +1,112 @@ +"""The first-party modules a module imports, read from its source. + +A kernel's implementation hash covers the files of the modules it lists, and +``source_hash`` does not follow imports. A module list that stands for "the +code this kernel runs" is therefore complete only if every first-party module +a listed module imports is listed too, or excluded with a reason. These +helpers find those imports, so a test can pin the list's import closure. +""" + +from __future__ import annotations + +import ast +import importlib +import inspect +from pathlib import Path +from types import ModuleType + +#: Every workspace package imports as ``microcosm.``. +FIRST_PARTY_PREFIX = "microcosm." + + +def _is_package(module: ModuleType) -> bool: + return Path(module.__file__).name == "__init__.py" + + +def _import_stem(module_name: str, level: int, stem: str | None) -> str: + """The absolute module name an ``import`` statement's ``from`` part names.""" + + if not level: + return stem or "" + module = importlib.import_module(module_name) + package = module_name if _is_package(module) else module_name.rsplit(".", 1)[0] + base = package.rsplit(".", level - 1)[0] + return f"{base}.{stem}" if stem else base + + +def defining_module(package_name: str, attribute: str) -> str: + """The module that defines ``attribute`` as the package re-exports it. + + A submodule, class or function names its own module. A constant is traced + through the package ``__init__``'s ``from … import`` statements. A name + the ``__init__`` defines itself resolves to the package. + """ + + package = importlib.import_module(package_name) + value = getattr(package, attribute, None) + if isinstance(value, ModuleType): + return value.__name__ + if inspect.isclass(value) or inspect.isfunction(value): + return inspect.getmodule(value).__name__ + tree = ast.parse(Path(package.__file__).read_text(encoding="utf-8")) + for statement in tree.body: + if not isinstance(statement, ast.ImportFrom): + continue + for alias in statement.names: + if (alias.asname or alias.name) != attribute: + continue + stem = _import_stem(package_name, statement.level, statement.module) + if _is_package(importlib.import_module(stem)): + return defining_module(stem, alias.name) + return stem + return package_name + + +def first_party_imports(module_name: str) -> set[str]: + """Every first-party module ``module_name`` imports, at any depth of its AST. + + Function-level imports count: they run when the function does. A name + imported from a package resolves to the module that defines it, so + ``from microcosm.calibrate import TargetRegistry`` names + ``microcosm.calibrate.registry``. + """ + + module = importlib.import_module(module_name) + tree = ast.parse(Path(module.__file__).read_text(encoding="utf-8")) + found: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + stem = _import_stem(module_name, node.level, node.module) + if not stem.startswith(FIRST_PARTY_PREFIX): + continue + if not _is_package(importlib.import_module(stem)): + found.add(stem) + continue + found.update(defining_module(stem, alias.name) for alias in node.names) + elif isinstance(node, ast.Import): + found.update( + alias.name + for alias in node.names + if alias.name.startswith(FIRST_PARTY_PREFIX) + and not _is_package(importlib.import_module(alias.name)) + ) + found.discard(module_name) + return found + + +def uncovered_imports( + listed: tuple[str, ...], excluded: set[str] | frozenset[str] +) -> set[str]: + """``listed -> imported`` for each import that is neither listed nor excluded. + + Excluded modules are not followed, so an exclusion stops the walk; every + listed module is checked, so the closure holds transitively. + """ + + covered = set(listed) | set(excluded) + return { + f"{name} -> {imported}" + for name in listed + for imported in first_party_imports(name) + if imported not in covered + } From 8cfac3c55ec0c4c9a43b518dc66569bf68deae51 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:10:32 +0400 Subject: [PATCH 2/7] Hash geography_constants in the target-compilation kernels (green) ledger_targets labels a target's geography from UK_GEOGRAPHY_ID_TO_LABEL when the Chronicle facts give no name, and uk_runtime/target_weights reads UK_WIDE_GEOGRAPHY_IDS for the UK loss weights. Both import the module with `from ... import`, which source_hash does not follow (microcosm#1492). - transport/target_kernels.py: geography_constants joins _COMPILE_MODULES, which targets.compile@1, takeup.compare@1, takeup.gap@1, transport.create@1/@2 and transport.quantile_map@1/@2 hash. - uk_runtime/target_compile_modules.py: the UK list gains microcosm.calibrate.geography_constants, and also calibrate.registry, calibrate.hierarchy and calibrate.target. The widened import check found ledger_targets and seven UK compile modules building the surface from TargetRegistry, TargetSpec and the hierarchy classes, which no UK target kernel hashed; the transport list already hashes all three. Listing them costs no extra recompute here, since the same UK kernels get new keys from geography_constants. microcosm.frame.bundle, frame.schema and frame.weights are excluded with their reasons (the spine-side CGT calibration frame, the national calibration's frame adapter, and calibrate.target's measurement). - uk_runtime/graph_targets.py: the UK problem kernel's loss-weight hash lists geography_constants beside target_weights, so the weights do not depend on the target modules' list for it. - uk_runtime/graph_terminal.py: uk_terminal_only_code_hash's docstring said an edit to its modules never re-solves; geography_constants now does, so the docstring says so. Comment only, but the file is hashed (see below). - test_uk_solve_scope._module_path mapped every name into microcosm.build; it now maps a module of any workspace package, as the UK list now holds microcosm.calibrate modules. test_uk_graph_terminal's comment on the same point is corrected. Kernels whose implementation hash changes (27 of the 96 importable ones, compared against main by computing every hash): - they now hash geography_constants: the 7 transport kernels above, and uk.full.target_compilation@1, target_selection@1, measures@1, problem@1, national_targets@1 and national_problem@1; - they hash the file transport/target_kernels.py, which this change edits: calibration.prepare@1, targets.problem@1, diagnostics.calibration@1, gates.battery@1, geography.support_from_facts@1, takeup.assign@1; - they hash graph_targets.py or graph_terminal.py: uk.full.rotated-holdout@1, uk.full-gates.preflight@1, uk.full-gates@1, uk.full-export.prepare@1, uk.full-export.package@1, uk.full-export.readback@1, uk.full-export.prepare-inspection@1, uk.full.certification-readiness@1. All of these read the UK target surface or the solve's results, so their node keys move anyway. Pins: none changed. No tracked file contains any of the 27 old hashes, and the parity pins (calibrate, fit.qrf, simulate) cover kernels whose hashes are unchanged. Green run (new and changed tests only, by node ID): 21 passed. Lint and format checked on the changed files only. Co-Authored-By: Claude Opus 5.5 --- ...-compile-keys-geography-constants.fixed.md | 1 + .../build/transport/target_kernels.py | 6 ++++ .../build/uk_runtime/graph_targets.py | 8 ++++- .../build/uk_runtime/graph_terminal.py | 5 ++- .../uk_runtime/target_compile_modules.py | 35 ++++++++++++++++--- .../engine_free/uk/test_uk_graph_terminal.py | 6 ++-- .../engine_free/uk/test_uk_solve_scope.py | 8 +++-- 7 files changed, 59 insertions(+), 10 deletions(-) create mode 100644 changelog.d/1492-target-compile-keys-geography-constants.fixed.md diff --git a/changelog.d/1492-target-compile-keys-geography-constants.fixed.md b/changelog.d/1492-target-compile-keys-geography-constants.fixed.md new file mode 100644 index 0000000000..3375e839d5 --- /dev/null +++ b/changelog.d/1492-target-compile-keys-geography-constants.fixed.md @@ -0,0 +1 @@ +The graph kernels that compile targets now include `microcosm.calibrate.geography_constants` in their implementation hashes, so an edit to a geography label recompiles the targets instead of reusing a stored surface that carries the old label (microcosm#1492). `ledger_targets` labels a target's geography from `UK_GEOGRAPHY_ID_TO_LABEL` when the Chronicle facts give no name, and `uk_runtime/target_weights` reads `UK_WIDE_GEOGRAPHY_IDS` for the UK loss weights, both through `from … import`, which `source_hash` does not follow. The module joins the transport `_COMPILE_MODULES` (hashed by `targets.compile@1`, `takeup.compare@1`, `takeup.gap@1`, `transport.create@1`/`@2` and `transport.quantile_map@1`/`@2`), the UK target compile modules (hashed by the full graph's target, selection, measures and problem kernels and the national graph's target and problem kernels), and the UK problem kernel's loss-weight hash. The UK list also gains `microcosm.calibrate.registry`, `target` and `hierarchy`, which the transport list already hashed: the UK import check read only `microcosm.build` imports and now reads every `microcosm.*` import, and a new transport check does the same for `_COMPILE_MODULES`, so a module a compile module imports must be hashed or excluded with a reason. Node keys move once: the target compilation of every transport country and everything after it, and the UK target, selection, measures, problem and national target and problem nodes and everything after them, including the UK solve. Because `transport/target_kernels.py` itself changes, `calibration.prepare@1` and `gates.battery@1` nodes that do not read a compiled surface also get new keys once. diff --git a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py index 7b3d926540..1afcc5a110 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py @@ -58,6 +58,7 @@ import microcosm.build.reference_intervals as reference_intervals_module import microcosm.build.reference_profiles as reference_profiles_module import microcosm.calibrate.artifacts as calibrate_artifacts_module +import microcosm.calibrate.geography_constants as geography_constants_module import microcosm.calibrate.hierarchy as hierarchy_module import microcosm.calibrate.matrix as matrix_module import microcosm.calibrate.monetary_binding as monetary_binding_module @@ -565,6 +566,10 @@ def _optional_float(value: float) -> float | None: return value if math.isfinite(value) else None +#: The modules target compilation reads. ``source_hash`` does not follow +#: imports, so a module a listed one imports is listed too, or excluded with a +#: reason in ``test_transport_compile_modules``. ``ledger_targets`` labels a +#: target's geography from ``geography_constants`` (microcosm#1492). _COMPILE_MODULES = ( graph_inputs, artifact_types, @@ -577,6 +582,7 @@ def _optional_float(value: float) -> float | None: registry_module, target_module, hierarchy_module, + geography_constants_module, reference_selection_module, ) _MATRIX_MODULES = ( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index b43a1a4569..e8d41a4286 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -15,6 +15,7 @@ import numpy as np import pandas as pd +import microcosm.calibrate.geography_constants as geography_constants from microcosm.build import cross_grain from microcosm.calibrate import CalibrationHierarchy, TargetRegistry, TargetSpec from microcosm.calibrate.artifacts import ( @@ -649,12 +650,17 @@ def implementation_hash(self) -> str: # ``local_rowwise.prepare_uk_full_solve`` from the rules in # ``target_weights`` and ``national_doctrine`` (microcosm#1124), so an # edit there re-keys the problem instead of replaying a stale vector. + # ``target_weights`` reads the UK-wide geography codes from + # ``geography_constants`` (microcosm#1492). return hashlib.sha256( canonical_json( { "targets": super().implementation_hash(), "weights": source_hash( - local_rowwise, national_doctrine, target_weights + local_rowwise, + national_doctrine, + target_weights, + geography_constants, ), } ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 27d8f5c1da..6fad53387c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -726,7 +726,10 @@ def uk_terminal_only_code_hash() -> str: ``catalog_calibration_diagnostics`` with the UK label choices (which read ``UK_GEOGRAPHY_ID_TO_LABEL``), dumped from ``CalibrationDiagnosticsV9``. Those modules decide the published labels and bytes, so an edit to them - re-keys the calibrated node the same way, never the solve. + re-keys the calibrated node the same way. ``geography_constants``, like + ``high_value_homes``, also feeds the target kernels, which hash it + (microcosm#1492), so an edit to it still re-solves; an edit to the other + three never does. Hashed once per process, as the code was imported: an edit to the files while a build runs cannot fail the calibrated gates after the solve. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py index 642a1e912a..586965f443 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py @@ -9,10 +9,16 @@ appliers, so an edit there could reuse a cached surface computed by the old code. -``EXCLUDED_DIRECT_IMPORTS`` names every ``microcosm.build`` module a listed -module imports but the tuple leaves out, with the reason. A test pins that -the two together cover every direct import of every listed module, so the -coverage holds transitively: a new import must be listed or excluded. +``EXCLUDED_DIRECT_IMPORTS`` names every first-party (``microcosm.*``) module +a listed module imports but the tuple leaves out, with the reason. A test +pins that the two together cover every direct import of every listed module, +so the coverage holds transitively: a new import must be listed or excluded. +Until microcosm#1492 that test read only ``microcosm.build`` imports, so it +never saw ``microcosm.calibrate.geography_constants``, the geography labels +of the target hierarchies, nor the ``microcosm.calibrate`` registry and +hierarchy classes the compiled surface is built from (with +``microcosm.calibrate.target``, which the registry imports). The transport +compile modules already hashed those three. """ from __future__ import annotations @@ -52,6 +58,10 @@ "microcosm.build.uk_runtime.uc_source_periods", "microcosm.build.uk_runtime.tenure_drift", "microcosm.build.uk_runtime.uprating_holds", + "microcosm.calibrate.geography_constants", + "microcosm.calibrate.hierarchy", + "microcosm.calibrate.registry", + "microcosm.calibrate.target", ) EXCLUDED_DIRECT_IMPORTS: Mapping[str, str] = MappingProxyType( @@ -111,6 +121,23 @@ "energy_pricing imports the identity-keyed uniforms for the " "spine's gas connection walk, which no target value reads" ), + "microcosm.frame.bundle": ( + "the population container: cgt_calibration builds the spine-side " + "CGT calibration frame with it, ledger_targets' UKFrameTargetAdapter " + "reads and rebuilds the national calibration's population with it, " + "and calibrate.target measures a target on one; no target value " + "comes from it" + ), + "microcosm.frame.schema": ( + "cgt_calibration imports EntitySchema for the spine-side CGT " + "calibration frame; the compile uses only uk_cgt_annual_exempt_amount " + "and uk_cgt_dated_measure_year" + ), + "microcosm.frame.weights": ( + "cgt_calibration imports Weights for the spine-side CGT calibration " + "frame; the compile uses only uk_cgt_annual_exempt_amount and " + "uk_cgt_dated_measure_year" + ), } ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index ccc785feaf..6f2e20b815 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -1517,8 +1517,10 @@ def test_the_calibrated_node_keys_on_the_schema_9_publication_code_not_the_solve # changes the output's bytes, so it must re-key the calibrated node (through # its terminal_code parameter); otherwise a store run with resume="auto" # serves the old labels. Like the high-value homes code, it stays out of - # the two kernels' shared code hash, so it never re-keys the preflight node - # or the solve. + # the two kernels' shared code hash. An edit to the catalogs, the schema + # or the label choices therefore never re-keys the preflight node or the + # solve; an edit to geography_constants still does, because the target + # kernels the preflight reads hash it too (microcosm#1492). import microcosm.calibrate.geography_constants as geography_constants_module import microcosm.calibrate.hierarchy_labels as hierarchy_labels_module import microcosm.diagnostics.catalogs as catalogs_module diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_solve_scope.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_solve_scope.py index cf82c461cb..f6f596959f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_solve_scope.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_solve_scope.py @@ -140,8 +140,12 @@ def _module_path(name: str) -> str: - relative = Path(*name.split(".")[2:]).with_suffix(".py") - return str((_BUILD_SOURCE / relative).relative_to(_REPOSITORY)) + """The repository path of a first-party module in any workspace package: + the compile modules include ``microcosm.calibrate`` ones (microcosm#1492).""" + shard, *rest = name.split(".")[1:] + relative = Path(*rest).with_suffix(".py") + source = _REPOSITORY / "packages" / f"microcosm-{shard}" / "src" / "microcosm" + return str((source / shard / relative).relative_to(_REPOSITORY)) def _docstrings(tree: ast.Module) -> set[int]: From 849bc69451d7cdcb0765769d5c954bea7ecd5842 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:43:11 +0400 Subject: [PATCH 3/7] Check every module each compile kernel hashes for unhashed imports (red) Review round 1 of #1493 found the UK target, selection, measures and problem kernels running first-party code they do not hash, which the shared-list check could not see: it scanned only the two shared module lists, not each kernel's own module or the modules hashed beside the lists. New and changed tests, failing on this branch's current code: - test_uk_target_compile_modules::test_an_edit_to_code_the_kernels_run_rekeys_them: an edit to calibrate.target_selection, calibrate.matrix, calibrate.artifacts, calibrate.solve, uk_runtime.weighted_integrity or the new uk_runtime.ledger_surface must re-key the kernels that run it (31 cases). - test_full_problem_loss_weights_hash_what_target_weights_reads: the problem's loss-weight hash also covers calibrate.solve (default_target_loss_scales). - test_every_import_a_target_kernel_reaches_is_hashed_or_classified, and the transport test_every_import_a_compile_kernel_reaches_is_hashed_or_classified: walk every module each compile kernel passes to source_hash, its own module included, and require each first-party import met to be hashed by that kernel, one of the shared list's exclusions (from a listed importer only), or recorded in UNCLASSIFIED_IMPORTS (UK 66 modules and 111 import edges, transport 35 and 74). Those entries are recorded as found; #1497 classifies them. The test fails on an import missing from the mapping and on an entry the walk no longer meets, so the mapping can only shrink. - The transport list checks now read the list and its 18 exclusions from transport.compile_modules, a module no kernel hashes (the review's point 3), and a new test requires that no compile kernel hashes it. - test_uk_calibration_run's two tests of _ledger_provenance and _validate_band_edge_registry call them in uk_runtime.ledger_surface, where they move. Scan helper (test_support/microcosm_build/first_party_imports.py): - reads import statements through the attribute chains the module reads from the bound name; - resolves a submodule the same way whether or not it has been imported yet; - walks through an excluded module's own imports; - documents what it cannot see: modules named by a string, and data files; - shares the edit-the-file hash check (assert_an_edit_rekeys) and the recording of a kernel's hashed modules between the two test modules; - uses Sequence and Set type hints, and refuses an object with no module. The scan's controls move to test_first_party_imports.py, with new cases for package imports, the submodule resolution and the walk. Red run (these tests only, by node ID): 39 failed, 20 passed. The 20 that pass are the geography-label cases already fixed on this branch, the UK list checks and the scan controls. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_first_party_imports.py | 112 ++++++ .../shared/test_transport_compile_modules.py | 332 +++++++++++----- .../engine_free/uk/test_uk_calibration_run.py | 19 +- .../uk/test_uk_target_compile_modules.py | 373 ++++++++++++++++-- .../microcosm_build/first_party_imports.py | 229 +++++++++-- 5 files changed, 899 insertions(+), 166 deletions(-) create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py diff --git a/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py b/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py new file mode 100644 index 0000000000..716c3b96b2 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py @@ -0,0 +1,112 @@ +"""Controls for the import scan behind the compile kernels' hash checks. + +``test_support.microcosm_build.first_party_imports`` reads a module's source +for the first-party modules it imports, and walks a kernel's hashed modules +through them. These tests pin each import form it resolves, that a +submodule resolves the same way before and after it is imported, and how the +walk treats a shared list's exclusions. +""" + +from __future__ import annotations + +import sys +import uuid + +import pytest + +import microcosm.calibrate +from test_support.microcosm_build.first_party_imports import ( + defining_module, + first_party_imports, + unhashed_imports, +) + + +@pytest.fixture +def scratch_package(tmp_path, monkeypatch): + """A throwaway ``microcosm.`` package written under ``tmp_path``. + + ``microcosm`` is a namespace package, so a directory on ``sys.path`` + adds to it. Returns a writer: ``write(module, source)`` creates + ``microcosm..`` and returns its dotted name. + """ + + name = f"scan_control_{uuid.uuid4().hex[:8]}" + package = tmp_path / "microcosm" / name + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + monkeypatch.syspath_prepend(str(tmp_path)) + + def write(module: str, source: str) -> str: + (package / f"{module}.py").write_text(source) + return f"microcosm.{name}.{module}" + + yield write + for loaded in [key for key in sys.modules if key.startswith(f"microcosm.{name}")]: + del sys.modules[loaded] + + +def test_the_scan_sees_each_import_form_the_compile_modules_use(): + # A constant imported from its own module. + assert "microcosm.calibrate.geography_constants" in first_party_imports( + "microcosm.build.ledger_targets" + ) + # A class imported from a package resolves to the module defining it. + assert "microcosm.calibrate.registry" in first_party_imports( + "microcosm.build.ledger_targets" + ) + # A constant the package __init__ re-exports resolves through the __init__. + assert "microcosm.diagnostics.schema" in first_party_imports( + "microcosm.build.transport.artifact_types" + ) + # An import inside a function body. + assert "microcosm.build.reference_intervals" in first_party_imports( + "microcosm.build.reference_profiles" + ) + + +def test_a_package_import_resolves_through_the_names_read_from_it(scratch_package): + module = scratch_package( + "package_imports", + "import microcosm.calibrate\n" + "import microcosm.diagnostics as diagnostics_package\n" + "def read():\n" + " registry = microcosm.calibrate.TargetRegistry\n" + " return registry, diagnostics_package.CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION\n", + ) + assert first_party_imports(module) == { + "microcosm.calibrate.registry", + "microcosm.diagnostics.schema", + } + + +def test_a_package_import_with_no_name_read_reads_the_package(scratch_package): + module = scratch_package("bare_import", "import microcosm.calibrate\n") + assert first_party_imports(module) == {"microcosm.calibrate"} + + +def test_a_submodule_resolves_the_same_before_it_is_imported(monkeypatch): + """A function-level ``from package import submodule`` names the submodule + whether or not an earlier test imported it.""" + name = "microcosm.calibrate.target_selection" + assert defining_module("microcosm.calibrate", "target_selection") == name + monkeypatch.delitem(sys.modules, name, raising=False) + monkeypatch.delattr(microcosm.calibrate, "target_selection", raising=False) + assert defining_module("microcosm.calibrate", "target_selection") == name + + +def test_the_walk_follows_an_exclusion_but_not_its_imports(scratch_package): + """A list's exclusion covers only the listed module's import of it; the + excluded module's own imports must be hashed.""" + leaf = scratch_package("leaf", "VALUE = 1\n") + middle = scratch_package("middle", f"from {leaf} import VALUE\n") + top = scratch_package("top", f"from {middle} import VALUE\n") + excluded = frozenset({middle}) + assert unhashed_imports({top}, listed=(top,), excluded=excluded) == { + f"{middle} -> {leaf}" + } + # From a module outside the list, the exclusion does not apply. + assert unhashed_imports({top}, listed=(), excluded=excluded) == { + f"{top} -> {middle}" + } + assert unhashed_imports({top, middle, leaf}, listed=(), excluded=excluded) == set() diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py index c7114f25e2..c85f08032f 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py @@ -1,159 +1,275 @@ """The transport compile kernels hash every module target compilation reads. ``targets.compile@1`` and every other transport kernel built on target -compilation hash ``target_kernels._COMPILE_MODULES``. ``source_hash`` hashes -the files it is given and does not follow imports, so a module that a listed -module imports, and that nobody lists, can change a compiled surface without -changing a node key. ``ledger_targets`` takes a target's geography label from -``microcosm.calibrate.geography_constants`` that way; before microcosm#1492 no -compile kernel hashed it, so a resumed store published stored surfaces with -the old label. - -``COMPILE_EXCLUDED_IMPORTS`` names every first-party module a listed module -imports but the tuple leaves out, with the reason. It lives here rather than -next to the tuple because ``target_kernels.py`` is hashed: editing a reason -there would give every kernel that hashes the file a new key. +compilation hash ``target_kernels._COMPILE_MODULES``, the modules named in +``transport.compile_modules``. ``source_hash`` hashes the files it is given +and does not follow imports, so a module that a hashed module imports, and +that the kernel does not hash, can change a kernel's output without changing +a node key. ``ledger_targets`` takes a target's geography label from +``microcosm.calibrate.geography_constants`` that way; before microcosm#1492 +no compile kernel hashed it. + +Two checks pin the closure: + +- the shared list's own: every first-party import of a listed module is + listed, or excluded with a reason in ``compile_modules``; +- each kernel's: the walk from every module the kernel hashes, its own + module included, meets only modules the kernel hashes, the list's + exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS``, which no + review has classified yet (microcosm#1497). That mapping may only shrink. """ from __future__ import annotations -from pathlib import Path -from types import MappingProxyType +import importlib +from types import MappingProxyType, ModuleType import pytest import microcosm.calibrate.geography_constants as geography_constants_module -from microcosm.build.transport import target_kernels -from microcosm.build.transport.population_kernels import ( - TRANSPORT_CREATE, - TRANSPORT_CREATE_DECLARED, - TRANSPORT_QUANTILE_MAP, +from microcosm.build.transport import ( + population_kernels, + quantile_kernels, + takeup_kernels, + target_kernels, ) -from microcosm.build.transport.quantile_kernels import TRANSPORT_QUANTILE_MAP_V2 -from microcosm.build.transport.takeup_kernels import TAKEUP_GAP -from microcosm.build.transport.target_kernels import TAKEUP_COMPARE, TARGETS_COMPILE from test_support.microcosm_build.first_party_imports import ( + assert_an_edit_rekeys, + edges, first_party_imports, - uncovered_imports, + hashed_module_names, + unhashed_imports, + unlisted_imports, ) -#: Every kernel whose implementation hash includes ``_COMPILE_MODULES``. The -#: first four compile references with ``compile_target_surface``; the two -#: quantile maps decode the surface an upstream compile stored. +#: Every kernel whose implementation hash includes ``_COMPILE_MODULES``, with +#: the module whose ``source_hash`` it calls. The first five compile +#: references with ``compile_target_surface``; the two quantile maps decode +#: the surface an upstream compile stored. KERNELS_HASHING_THE_COMPILE_MODULES = ( - TARGETS_COMPILE, - TAKEUP_COMPARE, - TAKEUP_GAP, - TRANSPORT_CREATE, - TRANSPORT_CREATE_DECLARED, - TRANSPORT_QUANTILE_MAP, - TRANSPORT_QUANTILE_MAP_V2, + (target_kernels.TARGETS_COMPILE, target_kernels), + (target_kernels.TAKEUP_COMPARE, target_kernels), + (takeup_kernels.TAKEUP_GAP, takeup_kernels), + (population_kernels.TRANSPORT_CREATE, population_kernels), + (population_kernels.TRANSPORT_CREATE_DECLARED, population_kernels), + (population_kernels.TRANSPORT_QUANTILE_MAP, population_kernels), + (quantile_kernels.TRANSPORT_QUANTILE_MAP_V2, quantile_kernels), ) +_KERNEL_IDS = [kernel.ref for kernel, _ in KERNELS_HASHING_THE_COMPILE_MODULES] -_COUNTRY_SPEC_LOADER = ( - "country_spec is listed for the reference validators target compilation " - "calls, _validate_target_references and _carried_value_keys; it imports " - "this module for its country-spec loader and stage plans, which those " - "validators do not use" -) -_GRAPH_RUNTIME = ( - "the graph runtime (kernel protocol, declarations and the canonical JSON " - "encoder node keys are computed with); no kernel in the repository " - "hashes a microcosm.graph module" +#: Imports the kernels' modules make only to build the hash. They stay +#: unhashed on purpose, so an edit to the list re-keys only the kernels whose +#: hashed modules it adds or removes. +HASH_INPUTS = MappingProxyType( + { + "microcosm.build.transport.compile_modules": ( + "names the modules the compile kernels hash; target_kernels reads " + "it only to build _COMPILE_MODULES" + ), + } ) -COMPILE_EXCLUDED_IMPORTS = MappingProxyType( +#: Every first-party module the walk from a compile kernel's hashed modules +#: meets that the kernel does not hash and no review has classified, with the +#: hashed (or excluded) modules that import it. Recorded as found when the +#: per-kernel check was added; microcosm#1497 classifies them. Hash an entry +#: in the kernels that run it, move what they run into a module they hash, or +#: record why they need not hash it, then delete it here. The check fails on +#: an import missing from this mapping and on an entry the walk no longer +#: meets, so the mapping can only shrink. +UNCLASSIFIED_IMPORTS = MappingProxyType( { - "microcosm.build.plan": _COUNTRY_SPEC_LOADER, - "microcosm.build.source_manifest": _COUNTRY_SPEC_LOADER, - "microcosm.build.trace": _COUNTRY_SPEC_LOADER, - "microcosm.build.spec_engine.canonical": _COUNTRY_SPEC_LOADER, - "microcosm.build.spec_engine.engine_abi": _COUNTRY_SPEC_LOADER, - "microcosm.build.spec_engine.loader": _COUNTRY_SPEC_LOADER, - "microcosm.build.spec_engine.model": _COUNTRY_SPEC_LOADER, - "microcosm.build.spec_engine.schemas": _COUNTRY_SPEC_LOADER, - "microcosm.build.spec_engine.take_up_semantics": _COUNTRY_SPEC_LOADER, - "microcosm.build.spec_engine.yaml12": _COUNTRY_SPEC_LOADER, + "microcosm.build": ("microcosm.build.trace",), + "microcosm.build.calibration_columns": ( + "microcosm.build.transport.target_kernels", + ), + "microcosm.build.calibration_predicates": ( + "microcosm.build.transport.target_kernels", + ), + "microcosm.build.observation": ("microcosm.build.plan",), "microcosm.build.reference_intervals": ( - "reference_profiles reaches it only when given an interval " - "document; compile_target_surface calls validate_reference_resolution " - "without one. calibration.prepare@1 and targets.problem@1, which " - "apply interval tables, hash it" + "microcosm.build.transport.target_kernels", + ), + "microcosm.build.spec_engine.canonical": ( + "microcosm.build.spec_engine.engine_abi", + "microcosm.build.spec_engine.loader", + "microcosm.build.spec_engine.take_up_semantics", + ), + "microcosm.build.spec_engine.engine_abi": ( + "microcosm.build.spec_engine.loader", + ), + "microcosm.build.spec_engine.errors": ( + "microcosm.build.spec_engine.engine_abi", + "microcosm.build.spec_engine.loader", + "microcosm.build.spec_engine.schemas", + "microcosm.build.spec_engine.take_up_semantics", + "microcosm.build.spec_engine.yaml12", + ), + "microcosm.build.spec_engine.model": ( + "microcosm.build.spec_engine.canonical", + "microcosm.build.spec_engine.engine_abi", + "microcosm.build.spec_engine.loader", + ), + "microcosm.build.spec_engine.resolver": ("microcosm.build.spec_engine.loader",), + "microcosm.build.spec_engine.schemas": ( + "microcosm.build.spec_engine.engine_abi", + "microcosm.build.spec_engine.loader", + ), + "microcosm.build.spec_engine.seeds": ("microcosm.build.spec_engine.model",), + "microcosm.build.spec_engine.take_up_semantics": ( + "microcosm.build.spec_engine.engine_abi", + ), + "microcosm.build.spec_engine.yaml12": ("microcosm.build.spec_engine.loader",), + "microcosm.build.transport.column_kernels": ( + "microcosm.build.transport.target_kernels", + ), + "microcosm.build.transport.unit_declarations": ( + "microcosm.build.transport.population_kernels", + "microcosm.build.transport.target_kernels", + ), + "microcosm.build.us_runtime.multispine_pool": ( + "microcosm.build.spec_engine.engine_abi", + ), + "microcosm.calibrate.artifacts": ("microcosm.build.transport.target_kernels",), + "microcosm.calibrate.matrix": ( + "microcosm.build.transport.target_kernels", + "microcosm.calibrate.artifacts", ), - "microcosm.calibrate.artifacts": ( - "artifact_types imports PROBLEM_TYPE to declare targets.problem@1's " - "output; target compilation checks only its own surface output" + "microcosm.calibrate.monetary_binding": ( + "microcosm.build.transport.target_kernels", ), - "microcosm.diagnostics.schema": ( - "artifact_types imports the diagnostics schema version to declare " - "the diagnostics output type; target compilation emits no diagnostics" + "microcosm.calibrate.solve": ("microcosm.calibrate.artifacts",), + "microcosm.frame.adapters.policyengine_us": ( + "microcosm.build.spec_engine.engine_abi", + ), + "microcosm.frame.adapters.policyengine_us_concepts": ( + "microcosm.build.transport.population_kernels", + "microcosm.frame.transport", ), "microcosm.frame.bundle": ( - "the population container: graph_inputs.context_frame builds one, " - "calibrate.target measures a target on one and country_spec's stage " - "plans transform one; target compilation reads facts, not a population" + "microcosm.build.plan", + "microcosm.build.transport.population_kernels", + "microcosm.build.transport.target_kernels", + "microcosm.calibrate.artifacts", + "microcosm.graph.kernel", + ), + "microcosm.frame.concept_mapping": ( + "microcosm.build.transport.population_kernels", + ), + "microcosm.frame.concepts": ( + "microcosm.build.reference_intervals", + "microcosm.build.transport.population_kernels", + "microcosm.frame.transport", + ), + "microcosm.frame.materialize": ( + "microcosm.build.transport.population_kernels", + "microcosm.frame.transport", + ), + "microcosm.frame.scaling": ( + "microcosm.build.transport.population_kernels", + "microcosm.frame.transport", ), "microcosm.frame.schema": ( - "graph_inputs.context_frame's entity schema; target compilation " - "reads no population" + "microcosm.build.transport.population_kernels", + "microcosm.build.transport.target_kernels", + "microcosm.frame.bundle", + ), + "microcosm.frame.unit_construction": ( + "microcosm.build.transport.population_kernels", + "microcosm.build.transport.target_kernels", + ), + "microcosm.frame.weights": ( + "microcosm.build.transport.population_kernels", + "microcosm.build.transport.target_kernels", + "microcosm.calibrate.artifacts", + "microcosm.frame.bundle", + "microcosm.graph.kernel", + ), + "microcosm.graph.canonical": ( + "microcosm.build.reference_intervals", + "microcosm.build.transport.quantile_kernels", + "microcosm.build.transport.takeup_kernels", + "microcosm.build.transport.target_kernels", + "microcosm.calibrate.artifacts", + ), + "microcosm.graph.decl": ( + "microcosm.build.transport.population_kernels", + "microcosm.build.transport.takeup_kernels", + "microcosm.build.transport.target_kernels", + "microcosm.calibrate.artifacts", + "microcosm.graph.canonical", + "microcosm.graph.kernel", ), - "microcosm.graph.canonical": _GRAPH_RUNTIME, - "microcosm.graph.decl": _GRAPH_RUNTIME, - "microcosm.graph.kernel": _GRAPH_RUNTIME, + "microcosm.graph.kernel": ( + "microcosm.build.transport.population_kernels", + "microcosm.build.transport.quantile_kernels", + "microcosm.build.transport.takeup_kernels", + "microcosm.build.transport.target_kernels", + ), + "microcosm.graph.population": ("microcosm.build.transport.population_kernels",), } ) -def _compile_module_names() -> tuple[str, ...]: - return tuple(module.__name__ for module in target_kernels._COMPILE_MODULES) +def _compile_list() -> ModuleType: + """The shared list and its exclusions (``compile_modules``, microcosm#1492).""" + return importlib.import_module("microcosm.build.transport.compile_modules") @pytest.mark.parametrize( - "kernel", KERNELS_HASHING_THE_COMPILE_MODULES, ids=lambda kernel: kernel.ref + ("kernel", "module"), KERNELS_HASHING_THE_COMPILE_MODULES, ids=_KERNEL_IDS ) def test_an_edit_to_the_geography_labels_rekeys_the_kernel( - tmp_path, monkeypatch, kernel + tmp_path, monkeypatch, kernel, module ): """ledger_targets labels a target's geography from UK_GEOGRAPHY_ID_TO_LABEL when the facts give no name, so an edit there must give a new key.""" - before = kernel.implementation_hash() - source = Path(geography_constants_module.__file__) - edited = tmp_path / source.name - edited.write_bytes(source.read_bytes() + b"\n# an edited geography label\n") - monkeypatch.setattr(geography_constants_module, "__file__", str(edited)) - assert kernel.implementation_hash() != before + assert_an_edit_rekeys(kernel, geography_constants_module, tmp_path, monkeypatch) + + +def test_the_compile_list_lives_in_a_module_no_compile_kernel_hashes(): + """target_kernels is hashed by 13 kernels; the list is not, so an edit to + a name or a reason no longer re-keys all of them.""" + compile_list = _compile_list() + assert target_kernels._COMPILE_MODULES == compile_list.transport_compile_modules() + for kernel, module in KERNELS_HASHING_THE_COMPILE_MODULES: + assert compile_list.__name__ not in hashed_module_names(kernel, module) -def test_every_first_party_import_of_a_compile_module_is_listed_or_excluded(): - uncovered = uncovered_imports( - _compile_module_names(), frozenset(COMPILE_EXCLUDED_IMPORTS) +def test_every_direct_import_of_a_compile_module_is_listed_or_excluded(): + compile_list = _compile_list() + uncovered = unlisted_imports( + compile_list.TRANSPORT_COMPILE_MODULE_NAMES, + frozenset(compile_list.EXCLUDED_DIRECT_IMPORTS), ) assert not uncovered, sorted(uncovered) def test_compile_exclusions_are_live_and_never_listed(): - imported = set().union(*map(first_party_imports, _compile_module_names())) - assert set(COMPILE_EXCLUDED_IMPORTS) <= imported, sorted( - set(COMPILE_EXCLUDED_IMPORTS) - imported - ) - assert not set(COMPILE_EXCLUDED_IMPORTS) & set(_compile_module_names()) + compile_list = _compile_list() + listed = compile_list.TRANSPORT_COMPILE_MODULE_NAMES + excluded = set(compile_list.EXCLUDED_DIRECT_IMPORTS) + imported = set().union(*map(first_party_imports, listed)) + assert excluded <= imported, sorted(excluded - imported) + assert not excluded & set(listed) -def test_the_import_scan_sees_each_import_form(): - """Control: the three forms that hide a module from a plain scan.""" - # A constant imported from its own module. - assert "microcosm.calibrate.geography_constants" in first_party_imports( - "microcosm.build.ledger_targets" - ) - # A class imported from a package resolves to the module defining it. - assert "microcosm.calibrate.registry" in first_party_imports( - "microcosm.build.ledger_targets" - ) - # A constant the package __init__ re-exports resolves through the __init__. - assert "microcosm.diagnostics.schema" in first_party_imports( - "microcosm.build.transport.artifact_types" - ) - # An import inside a function body. - assert "microcosm.build.reference_intervals" in first_party_imports( - "microcosm.build.reference_profiles" - ) +def test_every_import_a_compile_kernel_reaches_is_hashed_or_classified(): + """The walk covers each kernel's own module and every module hashed + beside the shared list, not only the list (microcosm#1492 review).""" + compile_list = _compile_list() + met: set[str] = set() + for kernel, module in KERNELS_HASHING_THE_COMPILE_MODULES: + hashed = hashed_module_names(kernel, module) + assert not set(HASH_INPUTS) & hashed, kernel.ref + met |= unhashed_imports( + hashed, + listed=compile_list.TRANSPORT_COMPILE_MODULE_NAMES, + excluded=frozenset(compile_list.EXCLUDED_DIRECT_IMPORTS), + ) + met = {edge for edge in met if edge.split(" -> ")[1] not in HASH_INPUTS} + recorded = edges(UNCLASSIFIED_IMPORTS) + assert met == recorded, { + "unhashed and not classified": sorted(met - recorded), + "recorded but no longer met (delete them)": sorted(recorded - met), + } diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py index fc9cf36b98..47e112e777 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py @@ -185,10 +185,14 @@ def test_the_uk_block_delegates_rather_than_reassembling_the_shared_one(tmp_path field the loader learns and this seam silently drops, and a test that enumerated today's fields would not catch tomorrow's. """ + # The block moved from calibration_run to ledger_surface, which the UK + # target kernels hash (microcosm#1492). + from microcosm.build.uk_runtime import ledger_surface + artifact = load_ledger_consumer_artifact(_mixed_epoch_artifact_dir(tmp_path)) shared = artifact.provenance() - block = calibration_run._ledger_provenance(artifact) + block = ledger_surface._ledger_provenance(artifact) for field, value in shared.items(): if field in {"path_name", "schema_version", "profiles"}: @@ -218,23 +222,26 @@ def test_band_edge_register_must_reconstitute_the_compiled_roster(): """The reconciliation always runs: an empty receipt claims nothing was pruned, so the two rosters must then be name-identical; a receipt that disagrees with the pruned names refuses. The graph's national target - node and the dense target compiler both stand on this check.""" + node and the dense target compiler both stand on this check, now in + ledger_surface, which the UK target kernels hash (microcosm#1492).""" + from microcosm.build.uk_runtime import ledger_surface + compiled = TargetRegistry([_spec("a"), _spec("b")], country="uk") pruned = TargetRegistry([_spec("a")], country="uk") - calibration_run._validate_band_edge_registry( + ledger_surface._validate_band_edge_registry( register_registry=pruned, band_edge_registry=compiled, exclusion_receipt={"b": {"reason": "test"}}, ) - calibration_run._validate_band_edge_registry( + ledger_surface._validate_band_edge_registry( register_registry=compiled, band_edge_registry=compiled, exclusion_receipt={} ) with pytest.raises(ValueError, match="must match the measure exclusion receipt"): - calibration_run._validate_band_edge_registry( + ledger_surface._validate_band_edge_registry( register_registry=pruned, band_edge_registry=compiled, exclusion_receipt={} ) with pytest.raises(ValueError, match="must include every materialized"): - calibration_run._validate_band_edge_registry( + ledger_surface._validate_band_edge_registry( register_registry=compiled, band_edge_registry=pruned, exclusion_receipt={}, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index 0579fe3b4c..47626f3cd1 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -1,12 +1,25 @@ -"""Both UK target kernels hash every module that decides a target value.""" +"""Both UK target kernels hash every module that decides a target value. + +Two checks pin the closure, as for the transport compile kernels: + +- the shared list's own: every first-party import of a module in + ``UK_TARGET_COMPILE_MODULE_NAMES`` is listed, or excluded with a reason; +- each kernel's: the walk from every module a target or problem kernel + hashes, its own module included, meets only modules the kernel hashes, the + list's exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS``, + which no review has classified yet (microcosm#1497). That mapping may only + shrink. +""" from __future__ import annotations -from pathlib import Path +import importlib +from types import MappingProxyType import pytest import microcosm.calibrate.geography_constants as geography_constants_module +import microcosm.calibrate.solve as calibrate_solve from microcosm.build import ledger_artifact from microcosm.build import ledger_targets as shared_ledger_targets from microcosm.build.uk_runtime import ( @@ -22,8 +35,12 @@ uk_target_compile_modules, ) from test_support.microcosm_build.first_party_imports import ( + assert_an_edit_rekeys, + edges, first_party_imports, - uncovered_imports, + hashed_module_names, + unhashed_imports, + unlisted_imports, ) @@ -32,7 +49,7 @@ def test_every_direct_import_of_a_listed_module_is_listed_or_excluded(): hierarchies take their geography labels from microcosm.calibrate.geography_constants, which a scan of microcosm.build imports never saw (microcosm#1492).""" - uncovered = uncovered_imports( + uncovered = unlisted_imports( UK_TARGET_COMPILE_MODULE_NAMES, frozenset(EXCLUDED_DIRECT_IMPORTS) ) assert not uncovered, sorted(uncovered) @@ -119,40 +136,321 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): assert ledger_artifact in _hashed_by(kernel, monkeypatch, graph_national) -#: The kernels whose hash includes the UK target compile modules: the full -#: graph's target, selection, measures and problem kernels and the national -#: graph's target and problem kernels. +#: The kernels whose hash includes the UK target compile modules, with the +#: module whose ``source_hash`` each calls: the full graph's target, +#: selection, measures and problem kernels and the national graph's target +#: and problem kernels. _TARGET_AND_PROBLEM_KERNELS = ( - graph_targets.UKFullTargetCompilationKernel(), - graph_targets.UKFullTargetSelectionKernel(), - graph_targets.UKFullMeasureKernel(), - graph_targets.UKFullProblemKernel(), - graph_national.UKNationalTargetKernel(), - graph_national.UKNationalProblemKernel(resolver_factory=None), + (graph_targets.UKFullTargetCompilationKernel(), graph_targets), + (graph_targets.UKFullTargetSelectionKernel(), graph_targets), + (graph_targets.UKFullMeasureKernel(), graph_targets), + (graph_targets.UKFullProblemKernel(), graph_targets), + (graph_national.UKNationalTargetKernel(), graph_national), + (graph_national.UKNationalProblemKernel(resolver_factory=None), graph_national), +) +_KERNELS_BY_REF = { + kernel.ref: (kernel, module) for kernel, module in _TARGET_AND_PROBLEM_KERNELS +} +_FULL_GRAPH = tuple( + ref for ref in _KERNELS_BY_REF if not ref.startswith("uk.full.national") +) +_NATIONAL_GRAPH = tuple( + ref for ref in _KERNELS_BY_REF if ref.startswith("uk.full.national") +) + +#: Code the target kernels ran without hashing it, found by the review of +#: microcosm#1492, and the kernels that now hash it. All four full-graph +#: kernels share one hash, so what one of them runs re-keys all four. +_RUN_MODULES = { + # uk.full.target_selection@1 calls select_targets. + "microcosm.calibrate.target_selection": _FULL_GRAPH, + # The measures and problem kernels build and encode constraint matrices. + "microcosm.calibrate.matrix": _FULL_GRAPH + _NATIONAL_GRAPH, + "microcosm.calibrate.artifacts": _FULL_GRAPH + _NATIONAL_GRAPH, + # The full problem's loss scales (default_target_loss_scales) and the + # national problem's effective sample size. + "microcosm.calibrate.solve": ("uk.full.problem@1", *_NATIONAL_GRAPH), + # full_targets' exclusion clock (exclusion_evaluation_date). + "microcosm.build.uk_runtime.weighted_integrity": _FULL_GRAPH + _NATIONAL_GRAPH, + # full_targets' surface provenance and band-edge roster check, moved out + # of calibration_run, which no target kernel hashes. + "microcosm.build.uk_runtime.ledger_surface": _FULL_GRAPH + _NATIONAL_GRAPH, +} + +#: Imports the kernels' modules make only to build the hash. They stay +#: unhashed on purpose, so an edit to the list re-keys only the kernels whose +#: hashed modules it adds or removes. +HASH_INPUTS = MappingProxyType( + { + "microcosm.build.uk_runtime.target_compile_modules": ( + "names the modules the target kernels hash; graph_targets and " + "graph_national read it only to build their hashes" + ), + "microcosm.build.uk_runtime.solve_scope": ( + "computes uk_solve_fingerprint, which graph_targets puts in the " + "full graph's target hashes and in the graph's contract-identity " + "parameter; no kernel's run reads it" + ), + } +) + +#: Every first-party module the walk from a target or problem kernel's +#: hashed modules meets that the kernel does not hash and no review has +#: classified, with the hashed (or excluded) modules that import it. Recorded +#: as found when the per-kernel check was added; microcosm#1497 classifies +#: them. Hash an entry in the kernels that run it, move what they run into a +#: module they hash, or record why they need not hash it, then delete it +#: here. The check fails on an import missing from this mapping and on an +#: entry the walk no longer meets, so the mapping can only shrink. +UNCLASSIFIED_IMPORTS = MappingProxyType( + { + "microcosm.build.artifact_files": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.country_spec": ( + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.measure_simulation", + ), + "microcosm.build.gate_battery": ("microcosm.build.uk_runtime.graph_national",), + "microcosm.build.gates": ( + "microcosm.build.uk_runtime.fiscal_targets", + "microcosm.build.uk_runtime.weighted_integrity", + ), + "microcosm.build.holdout": ("microcosm.build.uk_runtime.local_rowwise",), + "microcosm.build.input_mass": ( + "microcosm.build.uk_runtime.weighted_integrity", + ), + "microcosm.build.logbook_adoption": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.plan": ("microcosm.build.country_spec",), + "microcosm.build.source_manifest": ( + "microcosm.build.country_spec", + "microcosm.build.uk_runtime.frs_relationships", + ), + "microcosm.build.spec_engine.canonical": ("microcosm.build.country_spec",), + "microcosm.build.spec_engine.engine_abi": ("microcosm.build.country_spec",), + "microcosm.build.spec_engine.loader": ("microcosm.build.country_spec",), + "microcosm.build.spec_engine.model": ("microcosm.build.country_spec",), + "microcosm.build.spec_engine.schemas": ("microcosm.build.country_spec",), + "microcosm.build.spec_engine.take_up_semantics": ( + "microcosm.build.country_spec", + ), + "microcosm.build.spec_engine.yaml12": ("microcosm.build.country_spec",), + "microcosm.build.staging_v2": ("microcosm.build.uk_runtime.graph_national",), + "microcosm.build.trace": ("microcosm.build.country_spec",), + "microcosm.build.uk_runtime.atomic_area_support": ( + "microcosm.build.uk_runtime.measure_simulation", + ), + "microcosm.build.uk_runtime.battery_bindings": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.calibration_run": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.cgt_projection": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.content_identity": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.dataset_size": ( + "microcosm.build.uk_runtime.local_rowwise", + ), + "microcosm.build.uk_runtime.dfe_spring_term": ( + "microcosm.build.uk_runtime.measure_simulation", + ), + "microcosm.build.uk_runtime.diagnostics": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.frs_annual_income": ( + "microcosm.build.uk_runtime.measure_simulation", + ), + "microcosm.build.uk_runtime.frs_spine": ( + "microcosm.build.uk_runtime.frs_relationships", + "microcosm.build.uk_runtime.measure_simulation", + ), + "microcosm.build.uk_runtime.full_gates": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.geography_sources": ( + "microcosm.build.uk_runtime.local_authority_input", + ), + "microcosm.build.uk_runtime.graph_build": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.graph_calibration": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.graph_population": ( + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_targets", + ), + "microcosm.build.uk_runtime.graph_targets": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.graph_terminal": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.hmrc_income": ( + "microcosm.build.uk_runtime.hmrc_replay", + "microcosm.build.uk_runtime.measure_simulation", + ), + "microcosm.build.uk_runtime.hmrc_source_contract": ( + "microcosm.build.uk_runtime.weighted_integrity", + ), + "microcosm.build.uk_runtime.income_tax_rate_classes": ( + "microcosm.build.uk_runtime.measure_simulation", + ), + "microcosm.build.uk_runtime.lcfs_consumption": ( + "microcosm.build.uk_runtime.measure_simulation", + ), + "microcosm.build.uk_runtime.local_geography": ( + "microcosm.build.uk_runtime.local_rowwise", + ), + "microcosm.build.uk_runtime.local_rowwise": ( + "microcosm.build.uk_runtime.full_measure", + "microcosm.build.uk_runtime.full_problem", + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_targets", + ), + "microcosm.build.uk_runtime.national_calibration": ( + "microcosm.build.uk_runtime.full_measure", + ), + "microcosm.build.uk_runtime.national_doctrine": ( + "microcosm.build.uk_runtime.graph_targets", + "microcosm.build.uk_runtime.local_doctrine", + ), + "microcosm.build.uk_runtime.national_frame": ( + "microcosm.build.uk_runtime.frs_relationships", + "microcosm.build.uk_runtime.full_measure", + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.local_rowwise", + "microcosm.build.uk_runtime.measure_simulation", + "microcosm.build.uk_runtime.weighted_integrity", + ), + "microcosm.build.uk_runtime.rowwise_dataset": ( + "microcosm.build.uk_runtime.full_measure", + ), + "microcosm.build.uk_runtime.rowwise_posture": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.uk_runtime.size_checkpoint": ( + "microcosm.build.uk_runtime.local_rowwise", + ), + "microcosm.build.uk_runtime.solve_progress": ( + "microcosm.build.uk_runtime.local_rowwise", + ), + "microcosm.build.uk_runtime.target_weights": ( + "microcosm.build.uk_runtime.graph_targets", + ), + "microcosm.build.uk_runtime.uc_target_measurements": ( + "microcosm.build.uk_runtime.measure_simulation", + ), + "microcosm.build.uk_runtime.weight_year": ( + "microcosm.build.uk_runtime.graph_national", + ), + "microcosm.build.us_runtime.stacked_spine": ("microcosm.build.gates",), + "microcosm.calibrate.exact_k": ("microcosm.calibrate.solve",), + "microcosm.calibrate.gates": ("microcosm.calibrate.solve",), + "microcosm.calibrate.initialization": ("microcosm.calibrate.solve",), + "microcosm.calibrate.monetary_binding": ("microcosm.calibrate.matrix",), + "microcosm.calibrate.solve": ( + "microcosm.build.gates", + "microcosm.build.uk_runtime.graph_targets", + "microcosm.calibrate.artifacts", + ), + "microcosm.frame.adapters.policyengine_uk": ( + "microcosm.build.uk_runtime.measure_simulation", + "microcosm.build.uk_runtime.national_frame", + ), + "microcosm.frame.bundle": ( + "microcosm.build.country_spec", + "microcosm.build.uk_runtime.frs_relationships", + "microcosm.build.uk_runtime.full_measure", + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.local_rowwise", + "microcosm.build.uk_runtime.measure_simulation", + "microcosm.build.uk_runtime.national_calibration", + "microcosm.build.uk_runtime.national_frame", + "microcosm.build.uk_runtime.weighted_integrity", + "microcosm.calibrate.artifacts", + "microcosm.calibrate.matrix", + "microcosm.calibrate.solve", + ), + "microcosm.frame.materialize": ("microcosm.build.uk_runtime.national_frame",), + "microcosm.frame.schema": ( + "microcosm.build.uk_runtime.national_frame", + "microcosm.frame.bundle", + ), + "microcosm.frame.weights": ( + "microcosm.build.gates", + "microcosm.build.uk_runtime.full_measure", + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.local_rowwise", + "microcosm.build.uk_runtime.measure_simulation", + "microcosm.build.uk_runtime.national_calibration", + "microcosm.build.uk_runtime.national_frame", + "microcosm.calibrate.artifacts", + "microcosm.calibrate.matrix", + "microcosm.calibrate.solve", + "microcosm.frame.bundle", + ), + "microcosm.graph.canonical": ( + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_targets", + "microcosm.calibrate.artifacts", + ), + "microcosm.graph.codecs": ("microcosm.build.uk_runtime.graph_targets",), + "microcosm.graph.decl": ( + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_targets", + "microcosm.calibrate.artifacts", + ), + "microcosm.graph.kernel": ( + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_targets", + ), + } ) @pytest.mark.parametrize( - "kernel", _TARGET_AND_PROBLEM_KERNELS, ids=lambda kernel: kernel.ref + ("kernel", "module"), + _TARGET_AND_PROBLEM_KERNELS, + ids=[kernel.ref for kernel, _ in _TARGET_AND_PROBLEM_KERNELS], ) def test_an_edit_to_the_geography_labels_rekeys_the_kernel( - tmp_path, monkeypatch, kernel + tmp_path, monkeypatch, kernel, module ): """The shared and UK ledger targets label a target's geography from UK_GEOGRAPHY_ID_TO_LABEL, so an edit there must give these kernels new keys instead of reusing a surface with the old label (microcosm#1492).""" - before = kernel.implementation_hash() - source = Path(geography_constants_module.__file__) - edited = tmp_path / source.name - edited.write_bytes(source.read_bytes() + b"\n# an edited geography label\n") - monkeypatch.setattr(geography_constants_module, "__file__", str(edited)) - assert kernel.implementation_hash() != before - - -def test_full_problem_loss_weights_hash_the_uk_wide_geographies(monkeypatch): - """target_weights takes UK_WIDE_GEOGRAPHY_IDS from geography_constants for - the loss weights the problem binds, so the problem's weight hash covers - that module itself, not only through the target compile modules.""" + assert_an_edit_rekeys(kernel, geography_constants_module, tmp_path, monkeypatch) + + +@pytest.mark.parametrize( + ("module_name", "ref"), + [(name, ref) for name, refs in _RUN_MODULES.items() for ref in refs], + ids=[ + f"{name.rsplit('.', 1)[1]}-{ref}" + for name, refs in _RUN_MODULES.items() + for ref in refs + ], +) +def test_an_edit_to_code_the_kernels_run_rekeys_them( + tmp_path, monkeypatch, module_name, ref +): + kernel, _ = _KERNELS_BY_REF[ref] + module = importlib.import_module(module_name) + assert_an_edit_rekeys(kernel, module, tmp_path, monkeypatch) + + +def test_full_problem_loss_weights_hash_what_target_weights_reads(monkeypatch): + """target_weights takes UK_WIDE_GEOGRAPHY_IDS from geography_constants + and the loss scales from calibrate.solve for the weights the problem + binds, so the problem's weight hash covers both modules itself, not only + through the target modules' hash.""" calls: list[tuple] = [] monkeypatch.setattr( graph_targets, @@ -162,3 +460,24 @@ def test_full_problem_loss_weights_hash_the_uk_wide_geographies(monkeypatch): graph_targets.UKFullProblemKernel().implementation_hash() (weights,) = [objects for objects in calls if target_weights in objects] assert geography_constants_module in weights + assert calibrate_solve in weights + + +def test_every_import_a_target_kernel_reaches_is_hashed_or_classified(): + """The walk covers each kernel's own module and every module hashed + beside the shared list, not only the list (microcosm#1492 review).""" + met: set[str] = set() + for kernel, module in _TARGET_AND_PROBLEM_KERNELS: + hashed = hashed_module_names(kernel, module) + assert not set(HASH_INPUTS) & hashed, kernel.ref + met |= unhashed_imports( + hashed, + listed=UK_TARGET_COMPILE_MODULE_NAMES, + excluded=frozenset(EXCLUDED_DIRECT_IMPORTS), + ) + met = {edge for edge in met if edge.split(" -> ")[1] not in HASH_INPUTS} + recorded = edges(UNCLASSIFIED_IMPORTS) + assert met == recorded, { + "unhashed and not classified": sorted(met - recorded), + "recorded but no longer met (delete them)": sorted(recorded - met), + } diff --git a/test_support/microcosm_build/first_party_imports.py b/test_support/microcosm_build/first_party_imports.py index 17bb13c54d..13295cadbc 100644 --- a/test_support/microcosm_build/first_party_imports.py +++ b/test_support/microcosm_build/first_party_imports.py @@ -1,17 +1,33 @@ """The first-party modules a module imports, read from its source. -A kernel's implementation hash covers the files of the modules it lists, and -``source_hash`` does not follow imports. A module list that stands for "the -code this kernel runs" is therefore complete only if every first-party module -a listed module imports is listed too, or excluded with a reason. These -helpers find those imports, so a test can pin the list's import closure. +A kernel's implementation hash covers the files of the modules it passes to +``source_hash``, and ``source_hash`` does not follow imports. A kernel's +hashed modules therefore cover the code it runs only if every first-party +module they import is hashed too, or excluded with a reason. These helpers +find those imports, so a test can pin that closure. + +What the scan sees: ``import a.b`` and ``import a.b as c`` (for a package, +through the attribute chains the module reads from the name it binds), and +``from a.b import c``, absolute or relative, at any depth of the module, +function bodies included. A name imported from a package resolves to the +module that defines it. + +What it cannot see: a module named by a string (``importlib.import_module``, +``__import__``, ``getattr`` with a computed name), a module object passed in +as a value, and data files read by path (``importlib.resources``, ``open``). +Data files are not source, so a kernel that reads one must cover it in its +hash some other way, as the UK target kernels do with the country spec's +resource fingerprint. """ from __future__ import annotations import ast import importlib +import importlib.util import inspect +from collections.abc import Mapping, Sequence +from collections.abc import Set as AbstractSet from pathlib import Path from types import ModuleType @@ -23,6 +39,13 @@ def _is_package(module: ModuleType) -> bool: return Path(module.__file__).name == "__init__.py" +def _module_exists(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except (ImportError, ValueError): + return False + + def _import_stem(module_name: str, level: int, stem: str | None) -> str: """The absolute module name an ``import`` statement's ``from`` part names.""" @@ -34,20 +57,38 @@ def _import_stem(module_name: str, level: int, stem: str | None) -> str: return f"{base}.{stem}" if stem else base +def _defining_module_of(value: object) -> str | None: + """The module that defines a class or function, through any wrapper.""" + + if inspect.isclass(value): + return value.__module__ + unwrapped = inspect.unwrap(value) if callable(value) else value + if inspect.isroutine(unwrapped): + module = inspect.getmodule(unwrapped) + return module.__name__ if module is not None else None + return None + + def defining_module(package_name: str, attribute: str) -> str: - """The module that defines ``attribute`` as the package re-exports it. + """The module that defines ``attribute`` as the package exposes it. - A submodule, class or function names its own module. A constant is traced - through the package ``__init__``'s ``from … import`` statements. A name - the ``__init__`` defines itself resolves to the package. + A submodule resolves to itself whether or not it has been imported yet, + so the answer does not depend on which tests ran first. A class or + function names its own module. A constant is traced through the package + ``__init__``'s ``from … import`` statements. A name the ``__init__`` + defines itself resolves to the package. """ package = importlib.import_module(package_name) value = getattr(package, attribute, None) if isinstance(value, ModuleType): return value.__name__ - if inspect.isclass(value) or inspect.isfunction(value): - return inspect.getmodule(value).__name__ + submodule = f"{package_name}.{attribute}" + if value is None and _module_exists(submodule): + return submodule + defined_in = _defining_module_of(value) + if defined_in is not None: + return defined_in tree = ast.parse(Path(package.__file__).read_text(encoding="utf-8")) for statement in tree.body: if not isinstance(statement, ast.ImportFrom): @@ -62,6 +103,58 @@ def defining_module(package_name: str, attribute: str) -> str: return package_name +def _resolve_dotted(name: str) -> str: + """The module a dotted attribute chain such as ``a.b.C`` reads from.""" + + parts = name.split(".") + for end in range(len(parts), 0, -1): + prefix = ".".join(parts[:end]) + if not _module_exists(prefix): + continue + module = importlib.import_module(prefix) + if _is_package(module) and end < len(parts): + return defining_module(prefix, parts[end]) + return prefix + return name + + +def _attribute_chains(tree: ast.AST) -> list[str]: + """Every outermost dotted chain rooted at a plain name (``a.b.c``).""" + + inner = { + id(node.value) for node in ast.walk(tree) if isinstance(node, ast.Attribute) + } + chains = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Attribute) or id(node) in inner: + continue + parts = [node.attr] + value = node.value + while isinstance(value, ast.Attribute): + parts.append(value.attr) + value = value.value + if isinstance(value, ast.Name): + chains.append(".".join([value.id, *reversed(parts)])) + return chains + + +def _package_import_reads(binding: str, target: str, chains: list[str]) -> set[str]: + """The modules read through ``import target`` bound to the name ``binding``. + + ``import a.b`` binds ``a`` and ``import a.b as c`` binds ``c``; either way + each chain read from the bound name resolves to the module defining what + it names. With no chain into ``target``, the package itself is read. + """ + + prefix = target if binding == target.split(".")[0] else binding + reads = { + _resolve_dotted(target + chain[len(prefix) :]) + for chain in chains + if chain == prefix or chain.startswith(prefix + ".") + } + return {name for name in reads if name.startswith(target)} or {target} + + def first_party_imports(module_name: str) -> set[str]: """Every first-party module ``module_name`` imports, at any depth of its AST. @@ -73,6 +166,7 @@ def first_party_imports(module_name: str) -> set[str]: module = importlib.import_module(module_name) tree = ast.parse(Path(module.__file__).read_text(encoding="utf-8")) + chains: list[str] | None = None found: set[str] = set() for node in ast.walk(tree): if isinstance(node, ast.ImportFrom): @@ -84,24 +178,23 @@ def first_party_imports(module_name: str) -> set[str]: continue found.update(defining_module(stem, alias.name) for alias in node.names) elif isinstance(node, ast.Import): - found.update( - alias.name - for alias in node.names - if alias.name.startswith(FIRST_PARTY_PREFIX) - and not _is_package(importlib.import_module(alias.name)) - ) + for alias in node.names: + if not alias.name.startswith(FIRST_PARTY_PREFIX): + continue + if not _is_package(importlib.import_module(alias.name)): + found.add(alias.name) + continue + if chains is None: + chains = _attribute_chains(tree) + binding = alias.asname or alias.name.split(".")[0] + found |= _package_import_reads(binding, alias.name, chains) found.discard(module_name) return found -def uncovered_imports( - listed: tuple[str, ...], excluded: set[str] | frozenset[str] -) -> set[str]: - """``listed -> imported`` for each import that is neither listed nor excluded. - - Excluded modules are not followed, so an exclusion stops the walk; every - listed module is checked, so the closure holds transitively. - """ +def unlisted_imports(listed: Sequence[str], excluded: AbstractSet[str]) -> set[str]: + """``listed -> imported`` for each import of a listed module neither + listed nor excluded. The check of a shared module list.""" covered = set(listed) | set(excluded) return { @@ -110,3 +203,89 @@ def uncovered_imports( for imported in first_party_imports(name) if imported not in covered } + + +def unhashed_imports( + hashed: AbstractSet[str], *, listed: Sequence[str], excluded: AbstractSet[str] +) -> set[str]: + """``importer -> imported`` for each import a kernel's hash leaves uncovered. + + The walk starts at every module the kernel hashes. An import is covered + when the kernel hashes it, or when the importer is in the shared list + ``listed`` and the import is one of that list's reviewed exclusions + ``excluded``. The walk goes on through every covered import, so an + excluded module's own imports must be covered too, and its imports are + not covered by the list's exclusions, whose reasons were written for the + listed importers alone. An uncovered import is reported and not followed. + """ + + listed_names = set(listed) + uncovered: set[str] = set() + pending = sorted(hashed) + seen: set[str] = set() + while pending: + name = pending.pop() + if name in seen: + continue + seen.add(name) + for imported in sorted(first_party_imports(name)): + if imported in hashed or (name in listed_names and imported in excluded): + pending.append(imported) + else: + uncovered.add(f"{name} -> {imported}") + return uncovered + + +def edges(unclassified: Mapping[str, Sequence[str]]) -> set[str]: + """``importer -> imported`` for a mapping of each import to its importers.""" + + return { + f"{importer} -> {imported}" + for imported, importers in unclassified.items() + for importer in importers + } + + +def hashed_module_names(kernel: object, module: ModuleType) -> frozenset[str]: + """Every module ``kernel.implementation_hash()`` passes to ``source_hash``. + + ``module`` is the module whose ``source_hash`` name the kernel calls; the + recorder stands in for it during one call, then the original is restored. + """ + + original = module.source_hash + recorded: list[object] = [] + + def record(*objects: object, **dependencies: object) -> str: + recorded.extend(objects) + return original(*objects, **dependencies) + + module.source_hash = record + try: + kernel.implementation_hash() + finally: + module.source_hash = original + names = set() + for item in recorded: + defining = item if isinstance(item, ModuleType) else inspect.getmodule(item) + if defining is None: + raise ValueError(f"{item!r} has no module whose file can be hashed.") + names.add(defining.__name__) + return frozenset(names) + + +def assert_an_edit_rekeys( + kernel: object, module: ModuleType, tmp_path: Path, monkeypatch +) -> None: + """``kernel``'s implementation hash changes when ``module``'s file does. + + The file is copied with one more line and ``module.__file__`` points at + the copy, which ``source_hash`` reads. + """ + + before = kernel.implementation_hash() + source = Path(module.__file__) + edited = tmp_path / source.name + edited.write_bytes(source.read_bytes() + b"\n# an edited line\n") + monkeypatch.setattr(module, "__file__", str(edited)) + assert kernel.implementation_hash() != before, (kernel.ref, module.__name__) From 7ef6ffee9634e47d7a78144bc6b9510a454e3f03 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:46:57 +0400 Subject: [PATCH 4/7] Expect calibration_run hashed by the UK target kernels, not split (red) The previous commit expected _ledger_provenance and _validate_band_edge_registry to move out of uk_runtime.calibration_run into a small module. That move would edit calibration_run.py, which uk.full.bound_spine@1 hashes, and so re-key the bound spine checkpoint node and the population nodes after it: kernels this PR does not otherwise re-key. bound_spine already hashes calibration_run ahead of the target nodes (in every national build and every full build from a spine checkpoint), so the UK target kernels hash the whole module instead. - test_an_edit_to_code_the_kernels_run_rekeys_them: calibration_run takes the moved module's place (six cases). - UNCLASSIFIED_IMPORTS: calibration_run's own imports join the walk; the UK entries are now 67 modules and 118 import edges. - test_uk_calibration_run is restored to main's version. Red run (by node ID): the two changed test functions, 32 cases, all fail: the 31 module cases on an unchanged hash, and the walk on the six named modules' imports. Co-Authored-By: Claude Opus 5.5 --- .../engine_free/uk/test_uk_calibration_run.py | 19 +++++--------- .../uk/test_uk_target_compile_modules.py | 25 +++++++++++++------ 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py index 47e112e777..fc9cf36b98 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_calibration_run.py @@ -185,14 +185,10 @@ def test_the_uk_block_delegates_rather_than_reassembling_the_shared_one(tmp_path field the loader learns and this seam silently drops, and a test that enumerated today's fields would not catch tomorrow's. """ - # The block moved from calibration_run to ledger_surface, which the UK - # target kernels hash (microcosm#1492). - from microcosm.build.uk_runtime import ledger_surface - artifact = load_ledger_consumer_artifact(_mixed_epoch_artifact_dir(tmp_path)) shared = artifact.provenance() - block = ledger_surface._ledger_provenance(artifact) + block = calibration_run._ledger_provenance(artifact) for field, value in shared.items(): if field in {"path_name", "schema_version", "profiles"}: @@ -222,26 +218,23 @@ def test_band_edge_register_must_reconstitute_the_compiled_roster(): """The reconciliation always runs: an empty receipt claims nothing was pruned, so the two rosters must then be name-identical; a receipt that disagrees with the pruned names refuses. The graph's national target - node and the dense target compiler both stand on this check, now in - ledger_surface, which the UK target kernels hash (microcosm#1492).""" - from microcosm.build.uk_runtime import ledger_surface - + node and the dense target compiler both stand on this check.""" compiled = TargetRegistry([_spec("a"), _spec("b")], country="uk") pruned = TargetRegistry([_spec("a")], country="uk") - ledger_surface._validate_band_edge_registry( + calibration_run._validate_band_edge_registry( register_registry=pruned, band_edge_registry=compiled, exclusion_receipt={"b": {"reason": "test"}}, ) - ledger_surface._validate_band_edge_registry( + calibration_run._validate_band_edge_registry( register_registry=compiled, band_edge_registry=compiled, exclusion_receipt={} ) with pytest.raises(ValueError, match="must match the measure exclusion receipt"): - ledger_surface._validate_band_edge_registry( + calibration_run._validate_band_edge_registry( register_registry=pruned, band_edge_registry=compiled, exclusion_receipt={} ) with pytest.raises(ValueError, match="must include every materialized"): - ledger_surface._validate_band_edge_registry( + calibration_run._validate_band_edge_registry( register_registry=compiled, band_edge_registry=pruned, exclusion_receipt={}, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index 47626f3cd1..089dcf7b40 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -172,9 +172,9 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "microcosm.calibrate.solve": ("uk.full.problem@1", *_NATIONAL_GRAPH), # full_targets' exclusion clock (exclusion_evaluation_date). "microcosm.build.uk_runtime.weighted_integrity": _FULL_GRAPH + _NATIONAL_GRAPH, - # full_targets' surface provenance and band-edge roster check, moved out - # of calibration_run, which no target kernel hashes. - "microcosm.build.uk_runtime.ledger_surface": _FULL_GRAPH + _NATIONAL_GRAPH, + # full_targets' surface provenance (_ledger_provenance) and band-edge + # roster check (_validate_band_edge_registry). + "microcosm.build.uk_runtime.calibration_run": _FULL_GRAPH + _NATIONAL_GRAPH, } #: Imports the kernels' modules make only to build the hash. They stay @@ -208,10 +208,14 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "microcosm.build.uk_runtime.graph_national", ), "microcosm.build.country_spec": ( + "microcosm.build.uk_runtime.calibration_run", "microcosm.build.uk_runtime.graph_national", "microcosm.build.uk_runtime.measure_simulation", ), - "microcosm.build.gate_battery": ("microcosm.build.uk_runtime.graph_national",), + "microcosm.build.gate_battery": ( + "microcosm.build.uk_runtime.calibration_run", + "microcosm.build.uk_runtime.graph_national", + ), "microcosm.build.gates": ( "microcosm.build.uk_runtime.fiscal_targets", "microcosm.build.uk_runtime.weighted_integrity", @@ -245,13 +249,12 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "microcosm.build.uk_runtime.battery_bindings": ( "microcosm.build.uk_runtime.graph_national", ), - "microcosm.build.uk_runtime.calibration_run": ( - "microcosm.build.uk_runtime.graph_national", - ), "microcosm.build.uk_runtime.cgt_projection": ( + "microcosm.build.uk_runtime.calibration_run", "microcosm.build.uk_runtime.graph_national", ), "microcosm.build.uk_runtime.content_identity": ( + "microcosm.build.uk_runtime.calibration_run", "microcosm.build.uk_runtime.graph_national", ), "microcosm.build.uk_runtime.dataset_size": ( @@ -263,6 +266,9 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "microcosm.build.uk_runtime.diagnostics": ( "microcosm.build.uk_runtime.graph_national", ), + "microcosm.build.uk_runtime.etb_services": ( + "microcosm.build.uk_runtime.calibration_run", + ), "microcosm.build.uk_runtime.frs_annual_income": ( "microcosm.build.uk_runtime.measure_simulation", ), @@ -322,6 +328,7 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "microcosm.build.uk_runtime.local_doctrine", ), "microcosm.build.uk_runtime.national_frame": ( + "microcosm.build.uk_runtime.calibration_run", "microcosm.build.uk_runtime.frs_relationships", "microcosm.build.uk_runtime.full_measure", "microcosm.build.uk_runtime.graph_national", @@ -329,6 +336,9 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "microcosm.build.uk_runtime.measure_simulation", "microcosm.build.uk_runtime.weighted_integrity", ), + "microcosm.build.uk_runtime.release_certification": ( + "microcosm.build.uk_runtime.calibration_run", + ), "microcosm.build.uk_runtime.rowwise_dataset": ( "microcosm.build.uk_runtime.full_measure", ), @@ -366,6 +376,7 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): ), "microcosm.frame.bundle": ( "microcosm.build.country_spec", + "microcosm.build.uk_runtime.calibration_run", "microcosm.build.uk_runtime.frs_relationships", "microcosm.build.uk_runtime.full_measure", "microcosm.build.uk_runtime.graph_national", From 6a2e35d7816fc1165221e828e524a4846fc743db Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:48:56 +0400 Subject: [PATCH 5/7] Hash the code the UK target kernels run, and move the transport list out of target_kernels (green) Review round 1 of #1493. UK target and problem kernels (uk_runtime/graph_targets.py, graph_national.py): - calibrate.target_selection in _TargetKernel: uk.full.target_selection@1 calls select_targets. - calibrate.matrix and calibrate.artifacts in _TargetKernel and _NationalKernel: the measures and problem kernels build and encode constraint matrices, as the transport kernels already hash them. - calibrate.solve, the whole module, in the full problem's loss-weight hash and in _NationalKernel: the full problem stores default_target_loss_scales' output, and the national problem calls effective_sample_size. calibrate.adam@1 already hashes solve.py and the UK calibration kernels embed adam's hash, so an edit there already re-solves and now also re-keys these nodes. Moving the function out would have edited solve.py, changing adam's hash, pinned in microcosm-graph's calibrate/pins.json, and re-keying every calibration of every country. - uk_runtime.weighted_integrity in both base classes, directly rather than through UK_TARGET_COMPILE_MODULE_NAMES: full_targets takes exclusion_evaluation_date from it, and the module also loads the registers only the validation checks read, which test_uk_solve_scope forbids for a listed module. The UK preflight validation kernel already hashes it. - uk_runtime.calibration_run in both base classes, for _ledger_provenance and _validate_band_edge_registry. uk.full.bound_spine@1 already hashes it, ahead of the target nodes in every national build and every full build from a spine checkpoint. Moving the two functions into a small module would have edited calibration_run.py and so re-keyed the bound spine node and the population nodes after it. Transport: TRANSPORT_COMPILE_MODULE_NAMES and its 18 exclusions move to transport/compile_modules.py, which no kernel hashes, as the UK list's module is not hashed. target_kernels._COMPILE_MODULES is built from it, so a later edit to a name re-keys only what it adds or removes, and an edit to a reason re-keys nothing. Three imports target_kernels kept only for the old tuple are removed. The module docstrings of both lists describe the second, per-kernel check and microcosm#1497. The changelog fragment now covers these modules and names the takeup.assign@1 nodes given no compiled surface. Implementation hashes: of the 96 kernel classes, the 89 that build without an engine object were computed on main and here. 29 change: the 27 of 8cfac3c55, plus uk.full.national-gates@1 and uk.full.national-readback@1, which hash graph_national.py. Both run on the calibrated national population, so their node keys move anyway. No tracked file contains any of the 29 old hashes, so no pin changes. Green run (by node ID): the 17 new and changed test functions, 58 cases, all pass. Lint and format checked on the changed files only. Co-Authored-By: Claude Opus 5.5 --- ...-compile-keys-geography-constants.fixed.md | 2 +- .../build/transport/compile_modules.py | 106 ++++++++++++++++++ .../build/transport/target_kernels.py | 27 +---- .../build/uk_runtime/graph_national.py | 17 +++ .../build/uk_runtime/graph_targets.py | 33 +++++- .../uk_runtime/target_compile_modules.py | 6 + 6 files changed, 165 insertions(+), 26 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/transport/compile_modules.py diff --git a/changelog.d/1492-target-compile-keys-geography-constants.fixed.md b/changelog.d/1492-target-compile-keys-geography-constants.fixed.md index 3375e839d5..e98a87adc7 100644 --- a/changelog.d/1492-target-compile-keys-geography-constants.fixed.md +++ b/changelog.d/1492-target-compile-keys-geography-constants.fixed.md @@ -1 +1 @@ -The graph kernels that compile targets now include `microcosm.calibrate.geography_constants` in their implementation hashes, so an edit to a geography label recompiles the targets instead of reusing a stored surface that carries the old label (microcosm#1492). `ledger_targets` labels a target's geography from `UK_GEOGRAPHY_ID_TO_LABEL` when the Chronicle facts give no name, and `uk_runtime/target_weights` reads `UK_WIDE_GEOGRAPHY_IDS` for the UK loss weights, both through `from … import`, which `source_hash` does not follow. The module joins the transport `_COMPILE_MODULES` (hashed by `targets.compile@1`, `takeup.compare@1`, `takeup.gap@1`, `transport.create@1`/`@2` and `transport.quantile_map@1`/`@2`), the UK target compile modules (hashed by the full graph's target, selection, measures and problem kernels and the national graph's target and problem kernels), and the UK problem kernel's loss-weight hash. The UK list also gains `microcosm.calibrate.registry`, `target` and `hierarchy`, which the transport list already hashed: the UK import check read only `microcosm.build` imports and now reads every `microcosm.*` import, and a new transport check does the same for `_COMPILE_MODULES`, so a module a compile module imports must be hashed or excluded with a reason. Node keys move once: the target compilation of every transport country and everything after it, and the UK target, selection, measures, problem and national target and problem nodes and everything after them, including the UK solve. Because `transport/target_kernels.py` itself changes, `calibration.prepare@1` and `gates.battery@1` nodes that do not read a compiled surface also get new keys once. +The graph kernels that compile targets now include `microcosm.calibrate.geography_constants` in their implementation hashes, so an edit to a geography label recompiles the targets instead of reusing a stored surface that carries the old label (microcosm#1492). `ledger_targets` labels a target's geography from `UK_GEOGRAPHY_ID_TO_LABEL` when the Chronicle facts give no name, and `uk_runtime/target_weights` reads `UK_WIDE_GEOGRAPHY_IDS` for the UK loss weights, both through `from … import`, which `source_hash` does not follow. The module joins the transport compile modules (hashed by `targets.compile@1`, `takeup.compare@1`, `takeup.gap@1`, `transport.create@1`/`@2` and `transport.quantile_map@1`/`@2`), the UK target compile modules (hashed by the full graph's target, selection, measures and problem kernels and the national graph's target and problem kernels) and the UK problem kernel's loss-weight hash. The UK list also gains `microcosm.calibrate.registry`, `target` and `hierarchy`, which the transport list already hashed. The UK target and problem kernels also now hash code they ran unhashed: `calibrate.target_selection` (full graph), `calibrate.matrix` and `calibrate.artifacts`, `calibrate.solve` (the full problem's loss scales and the national problem's effective sample size; `calibrate.adam@1`, whose hash the UK calibration kernels embed, already hashed it), and `uk_runtime.weighted_integrity` and `uk_runtime.calibration_run`, whose exclusion clock, surface provenance and band-edge check `full_targets` runs. The transport compile list and its 18 exclusions move from `transport/target_kernels.py`, which 13 kernels hash, to `transport/compile_modules.py`, which none does, as the UK list already sat apart. Two checks now pin the hashes: a shared list's imports must be listed or excluded, reading every `microcosm.*` import rather than only `microcosm.build` ones, and the walk from every module a compile kernel hashes, its own module included, must meet only hashed modules, the list's exclusions or the imports no review has classified yet, recorded in the tests and tracked by microcosm#1497 (67 modules for the UK, 35 for transport), a record that can only shrink. Node keys move once: the target compilation of every transport country and everything after it; the UK target, selection, measures, problem and national target and problem nodes and everything after them, including the UK solve; and, because `transport/target_kernels.py` itself changes, `calibration.prepare@1` nodes, `gates.battery@1` nodes at a phase before compilation, and `takeup.assign@1` nodes given no compiled surface. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py new file mode 100644 index 0000000000..e38932c372 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py @@ -0,0 +1,106 @@ +"""The modules whose source decides a transport target surface. + +``targets.compile@1`` and every other transport kernel built on target +compilation hash :func:`transport_compile_modules` (through +``target_kernels._COMPILE_MODULES``). ``source_hash`` hashes the files it is +given and does not follow imports, so the tuple has to name every module the +compile reads; ``ledger_targets`` labels a target's geography from +``microcosm.calibrate.geography_constants``, which no compile kernel hashed +before microcosm#1492. + +The list lives here, in a module no kernel hashes, as the UK's lives in +``uk_runtime.target_compile_modules``: editing a name or a reason re-keys +only the kernels whose hashed modules the edit adds or removes, not every +kernel that hashes ``target_kernels``. + +``EXCLUDED_DIRECT_IMPORTS`` names every first-party (``microcosm.*``) module +a listed module imports but the tuple leaves out, with the reason. A test +pins that the two together cover every direct import of every listed module, +so a new import must be listed or excluded. The exclusions cover only the +listed modules' own imports; a second test walks every module each compile +kernel hashes, its own module included, and records the imports no review +has classified yet (microcosm#1497). +""" + +from __future__ import annotations + +import importlib +from collections.abc import Mapping +from types import MappingProxyType, ModuleType + +TRANSPORT_COMPILE_MODULE_NAMES: tuple[str, ...] = ( + "microcosm.build.transport.graph_inputs", + "microcosm.build.transport.artifact_types", + "microcosm.build.country_spec", + "microcosm.build.ledger_targets", + "microcosm.build.ledger_artifact", + "microcosm.build.chronicle_epoch", + "microcosm.build.reference_factors", + "microcosm.build.reference_profiles", + "microcosm.calibrate.registry", + "microcosm.calibrate.target", + "microcosm.calibrate.hierarchy", + "microcosm.calibrate.geography_constants", + "microcosm.build.transport.reference_selection", +) + +_COUNTRY_SPEC_LOADER = ( + "country_spec is listed for the reference validators target compilation " + "calls, _validate_target_references and _carried_value_keys; it imports " + "this module for its country-spec loader and stage plans, which those " + "validators do not use" +) +_GRAPH_RUNTIME = ( + "the graph runtime (kernel protocol, declarations and the canonical JSON " + "encoder node keys are computed with); no kernel in the repository " + "hashes a microcosm.graph module" +) + +EXCLUDED_DIRECT_IMPORTS: Mapping[str, str] = MappingProxyType( + { + "microcosm.build.plan": _COUNTRY_SPEC_LOADER, + "microcosm.build.source_manifest": _COUNTRY_SPEC_LOADER, + "microcosm.build.trace": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.canonical": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.engine_abi": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.loader": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.model": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.schemas": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.take_up_semantics": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.yaml12": _COUNTRY_SPEC_LOADER, + "microcosm.build.reference_intervals": ( + "reference_profiles reaches it only when given an interval " + "document; compile_target_surface calls validate_reference_resolution " + "without one. calibration.prepare@1 and targets.problem@1, which " + "apply interval tables, hash it" + ), + "microcosm.calibrate.artifacts": ( + "artifact_types imports PROBLEM_TYPE to declare targets.problem@1's " + "output; target compilation checks only its own surface output" + ), + "microcosm.diagnostics.schema": ( + "artifact_types imports the diagnostics schema version to declare " + "the diagnostics output type; target compilation emits no diagnostics" + ), + "microcosm.frame.bundle": ( + "the population container: graph_inputs.context_frame builds one, " + "calibrate.target measures a target on one and country_spec's stage " + "plans transform one; target compilation reads facts, not a population" + ), + "microcosm.frame.schema": ( + "graph_inputs.context_frame's entity schema; target compilation " + "reads no population" + ), + "microcosm.graph.canonical": _GRAPH_RUNTIME, + "microcosm.graph.decl": _GRAPH_RUNTIME, + "microcosm.graph.kernel": _GRAPH_RUNTIME, + } +) + + +def transport_compile_modules() -> tuple[ModuleType, ...]: + """The modules of :data:`TRANSPORT_COMPILE_MODULE_NAMES`, imported.""" + + return tuple( + importlib.import_module(name) for name in TRANSPORT_COMPILE_MODULE_NAMES + ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py index 1afcc5a110..cdea94844e 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py @@ -50,15 +50,11 @@ import microcosm.build.calibration_columns as calibration_columns_module import microcosm.build.calibration_predicates as calibration_predicates_module -import microcosm.build.chronicle_epoch as chronicle_epoch_module -import microcosm.build.country_spec as country_spec_module import microcosm.build.ledger_artifact as ledger_artifact_module -import microcosm.build.ledger_targets as ledger_targets_module import microcosm.build.reference_factors as reference_factors_module import microcosm.build.reference_intervals as reference_intervals_module import microcosm.build.reference_profiles as reference_profiles_module import microcosm.calibrate.artifacts as calibrate_artifacts_module -import microcosm.calibrate.geography_constants as geography_constants_module import microcosm.calibrate.hierarchy as hierarchy_module import microcosm.calibrate.matrix as matrix_module import microcosm.calibrate.monetary_binding as monetary_binding_module @@ -106,6 +102,7 @@ COMPARISON_TYPE, TARGET_SURFACE_TYPE, ) +from .compile_modules import transport_compile_modules from .graph_inputs import ( canonical_document_param, context_frame, @@ -566,25 +563,9 @@ def _optional_float(value: float) -> float | None: return value if math.isfinite(value) else None -#: The modules target compilation reads. ``source_hash`` does not follow -#: imports, so a module a listed one imports is listed too, or excluded with a -#: reason in ``test_transport_compile_modules``. ``ledger_targets`` labels a -#: target's geography from ``geography_constants`` (microcosm#1492). -_COMPILE_MODULES = ( - graph_inputs, - artifact_types, - country_spec_module, - ledger_targets_module, - ledger_artifact_module, - chronicle_epoch_module, - reference_factors_module, - reference_profiles_module, - registry_module, - target_module, - hierarchy_module, - geography_constants_module, - reference_selection_module, -) +#: The modules target compilation reads, named in ``compile_modules``, which +#: no kernel hashes, so an edit to the list re-keys only what it changes. +_COMPILE_MODULES = transport_compile_modules() _MATRIX_MODULES = ( matrix_module, monetary_binding_module, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index 94e8f55483..c7ae40f03e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -472,6 +472,11 @@ class _NationalKernel(KernelBase): ) def implementation_hash(self) -> str: + import microcosm.calibrate.artifacts as calibrate_artifacts + import microcosm.calibrate.matrix as calibrate_matrix + import microcosm.calibrate.solve as calibrate_solve + + from . import weighted_integrity from .target_compile_modules import uk_target_compile_modules implementation = source_hash( @@ -489,6 +494,18 @@ def implementation_hash(self) -> str: national_calibration, national_doctrine, support_floor_rules, + # The constraint matrices and their encoding, and the problem's + # effective sample size, as the dense kernels hash them + # (microcosm#1492). calibrate.adam@1, whose hash the UK calibration + # kernels of this graph embed, already hashes calibrate.solve. + calibrate_matrix, + calibrate_artifacts, + calibrate_solve, + # full_targets takes its exclusion clock from weighted_integrity, + # and its surface provenance and band-edge check from + # calibration_run; hashed directly, as the dense target kernels do. + weighted_integrity, + calibration_run, *uk_target_compile_modules(), ) return hashlib.sha256( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index e8d41a4286..39bbe4308f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -15,7 +15,11 @@ import numpy as np import pandas as pd +import microcosm.calibrate.artifacts as calibrate_artifacts import microcosm.calibrate.geography_constants as geography_constants +import microcosm.calibrate.matrix as calibrate_matrix +import microcosm.calibrate.solve as calibrate_solve +import microcosm.calibrate.target_selection as calibrate_target_selection from microcosm.build import cross_grain from microcosm.calibrate import CalibrationHierarchy, TargetRegistry, TargetSpec from microcosm.calibrate.artifacts import ( @@ -218,7 +222,13 @@ class _TargetKernel(KernelBase): ) def implementation_hash(self) -> str: - from . import full_targets, high_value_homes, measure_simulation + from . import ( + calibration_run, + full_targets, + high_value_homes, + measure_simulation, + weighted_integrity, + ) from .target_compile_modules import uk_target_compile_modules implementation = source_hash( @@ -236,6 +246,20 @@ def implementation_hash(self) -> str: # alone must re-key the measures and everything they feed. measure_simulation, support_floor, + # The selection, the constraint matrices and their encoding, as + # the transport kernels hash them (microcosm#1492). + calibrate_target_selection, + calibrate_matrix, + calibrate_artifacts, + # full_targets takes its exclusion clock from here. Hashed + # directly, not listed: the module also loads the registers only + # the validation checks read, which no listed module may. + weighted_integrity, + # full_targets writes _ledger_provenance into the surface and runs + # _validate_band_edge_registry from here. uk.full.bound_spine@1, + # which loads a spine checkpoint ahead of the target nodes, + # already hashes it. + calibration_run, *uk_target_compile_modules(), ) # The solve fingerprint: every packaged resource but the gate-only @@ -651,7 +675,11 @@ def implementation_hash(self) -> str: # ``target_weights`` and ``national_doctrine`` (microcosm#1124), so an # edit there re-keys the problem instead of replaying a stale vector. # ``target_weights`` reads the UK-wide geography codes from - # ``geography_constants`` (microcosm#1492). + # ``geography_constants``, and the loss scales the problem stores from + # ``calibrate.solve.default_target_loss_scales`` (microcosm#1492). All + # of ``calibrate.solve`` is hashed: ``calibrate.adam@1``, whose hash + # every UK solve kernel embeds, already hashes it, so an edit there + # re-solves anyway and re-keys only this node besides. return hashlib.sha256( canonical_json( { @@ -661,6 +689,7 @@ def implementation_hash(self) -> str: national_doctrine, target_weights, geography_constants, + calibrate_solve, ), } ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py index 586965f443..340ae23100 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py @@ -19,6 +19,12 @@ hierarchy classes the compiled surface is built from (with ``microcosm.calibrate.target``, which the registry imports). The transport compile modules already hashed those three. + +The exclusions cover only the listed modules' own imports. A second test +walks every module each target and problem kernel hashes, its own module and +the modules hashed beside this tuple included, and records the imports no +review has classified yet (microcosm#1497). This module is not hashed, so an +edit to a reason re-keys nothing; an edit to the tuple re-keys the kernels. """ from __future__ import annotations From 56593e3f86b262c0835b861c836ab13d312dff5d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:32:57 +0400 Subject: [PATCH 6/7] Key the recorded imports per kernel group, and expect the round 2 hash changes (red) Review round 2 of #1493, and Anthony's decision to move the two calibration_run functions. - The per-kernel check keeps UNCLASSIFIED_IMPORTS per group of kernels sharing one hash: each importer -> imported edge names the groups that leave it unhashed. UK groups: the full graph's target, selection and measures kernels; the full problem; the two national kernels. Transport: each kernel alone, and transport.create@1 with @2. Removing a module from one group's hash now fails even where another group records the same edge, which closes the three cases the review named (local_rowwise and national_doctrine in uk.full.problem@1, _MATRIX_MODULES in takeup.compare@1, the frame modules in transport.create@1 and @2). Records: UK 112 edges, transport 74 edges. - HASH_INPUTS now names the importers its reasons cover, and only those edges are skipped; each must still be met by some group. - The shared check is recorded_import_mismatches in the scan helper, with a control test that a record for one group does not cover another. - test_the_national_kernels_leave_calibrate_solve_unhashed: neither the national target nor the problem path calls calibrate.solve. Its one caller in graph_national, national_calibration_manifest, runs in uk.full.national-gates@1. The national cases for solve leave test_an_edit_to_code_the_kernels_run_rekeys_them. - test_an_edit_to_code_the_kernels_run_rekeys_them expects uk_runtime.ledger_surface in place of calibration_run (six cases), and test_an_edit_to_calibration_run_leaves_the_kernels_keys requires an edit elsewhere in calibration_run to leave all six kernels' hashes unchanged. Red run (by node ID): 9 test functions, 42 cases: 14 failed, 28 passed. The six ledger_surface cases fail on a missing module, the six calibration_run cases and the national solve test on the current hashes, and the UK record check on the national solve edge and calibration_run's imports. The transport record check, the control test and the other cases pass. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_first_party_imports.py | 44 + .../shared/test_transport_compile_modules.py | 676 +++++++++++---- .../uk/test_uk_target_compile_modules.py | 778 ++++++++++++------ .../microcosm_build/first_party_imports.py | 73 +- 4 files changed, 1165 insertions(+), 406 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py b/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py index 716c3b96b2..eb5b6a9953 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py @@ -9,8 +9,10 @@ from __future__ import annotations +import importlib import sys import uuid +from types import SimpleNamespace import pytest @@ -18,6 +20,7 @@ from test_support.microcosm_build.first_party_imports import ( defining_module, first_party_imports, + recorded_import_mismatches, unhashed_imports, ) @@ -110,3 +113,44 @@ def test_the_walk_follows_an_exclusion_but_not_its_imports(scratch_package): f"{top} -> {middle}" } assert unhashed_imports({top, middle, leaf}, listed=(), excluded=excluded) == set() + + +class _RecordingKernel: + """A stand-in kernel whose hash names ``modules`` through ``owner``.""" + + def __init__(self, ref, owner, modules): + self.ref, self.owner, self.modules = ref, owner, modules + + def implementation_hash(self): + return self.owner.source_hash(*self.modules) + + +def test_a_group_record_does_not_cover_another_group(scratch_package): + """Removing a module from one group's hash fails the check even when + another group records the same edge (round 2 of #1493).""" + middle = scratch_package("group_middle", "VALUE = 1\n") + top = scratch_package("group_top", f"from {middle} import VALUE\n") + owner = SimpleNamespace(source_hash=lambda *objects, **dependencies: "x") + top_module, middle_module = ( + importlib.import_module(name) for name in (top, middle) + ) + hashes_both = _RecordingKernel("hashes.both@1", owner, (top_module, middle_module)) + hashes_top = _RecordingKernel("hashes.top@1", owner, (top_module,)) + recorded = {f"{top} -> {middle}": ("top only",)} + + def mismatches(groups): + return recorded_import_mismatches( + groups, listed=(), excluded=frozenset(), hash_inputs={}, recorded=recorded + ) + + assert ( + mismatches( + {"both": ((hashes_both, owner),), "top only": ((hashes_top, owner),)} + ) + == {} + ) + # The first group drops the middle module: the edge is recorded for the + # second group alone, so the first group's walk reports it. + assert mismatches( + {"both": ((hashes_top, owner),), "top only": ((hashes_top, owner),)} + ) == {"both: unhashed and not recorded": [f"{top} -> {middle}"]} diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py index c85f08032f..2aa1ba903a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py @@ -15,8 +15,9 @@ listed, or excluded with a reason in ``compile_modules``; - each kernel's: the walk from every module the kernel hashes, its own module included, meets only modules the kernel hashes, the list's - exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS``, which no - review has classified yet (microcosm#1497). That mapping may only shrink. + exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS`` for that + kernel's group, which no review has classified yet (microcosm#1497). That + mapping may only shrink. """ from __future__ import annotations @@ -34,11 +35,11 @@ target_kernels, ) from test_support.microcosm_build.first_party_imports import ( + HashInput, assert_an_edit_rekeys, - edges, first_party_imports, hashed_module_names, - unhashed_imports, + recorded_import_mismatches, unlisted_imports, ) @@ -57,156 +58,516 @@ ) _KERNEL_IDS = [kernel.ref for kernel, _ in KERNELS_HASHING_THE_COMPILE_MODULES] -#: Imports the kernels' modules make only to build the hash. They stay -#: unhashed on purpose, so an edit to the list re-keys only the kernels whose -#: hashed modules it adds or removes. +#: The kernels grouped by shared hash: the two CREATE versions share one +#: class body, so one hashed set; every other kernel is its own group. +COMPILE = "targets.compile@1" +COMPARE = "takeup.compare@1" +GAP = "takeup.gap@1" +CREATE = "transport.create@1 and @2" +QUANTILE_MAP_1 = "transport.quantile_map@1" +QUANTILE_MAP_2 = "transport.quantile_map@2" +KERNEL_GROUPS = MappingProxyType( + { + COMPILE: ((target_kernels.TARGETS_COMPILE, target_kernels),), + COMPARE: ((target_kernels.TAKEUP_COMPARE, target_kernels),), + GAP: ((takeup_kernels.TAKEUP_GAP, takeup_kernels),), + CREATE: ( + (population_kernels.TRANSPORT_CREATE, population_kernels), + (population_kernels.TRANSPORT_CREATE_DECLARED, population_kernels), + ), + QUANTILE_MAP_1: ( + (population_kernels.TRANSPORT_QUANTILE_MAP, population_kernels), + ), + QUANTILE_MAP_2: ( + (quantile_kernels.TRANSPORT_QUANTILE_MAP_V2, quantile_kernels), + ), + } +) + +#: Imports a kernel's module makes only to build the hash, and the importers +#: the reason covers. They stay unhashed on purpose, so an edit to the list +#: re-keys only the kernels whose hashed modules it adds or removes. HASH_INPUTS = MappingProxyType( { - "microcosm.build.transport.compile_modules": ( - "names the modules the compile kernels hash; target_kernels reads " - "it only to build _COMPILE_MODULES" + "microcosm.build.transport.compile_modules": HashInput( + importers=("microcosm.build.transport.target_kernels",), + reason=( + "names the modules the compile kernels hash; target_kernels " + "reads it only to build _COMPILE_MODULES" + ), ), } ) -#: Every first-party module the walk from a compile kernel's hashed modules -#: meets that the kernel does not hash and no review has classified, with the -#: hashed (or excluded) modules that import it. Recorded as found when the +#: Every ``importer -> imported`` edge the walk from a kernel group's hashed +#: modules meets that the group does not hash and no review has classified, +#: with the groups that leave it unhashed. Recorded as found when the #: per-kernel check was added; microcosm#1497 classifies them. Hash an entry #: in the kernels that run it, move what they run into a module they hash, or #: record why they need not hash it, then delete it here. The check fails on -#: an import missing from this mapping and on an entry the walk no longer -#: meets, so the mapping can only shrink. +#: an edge a group meets that is not recorded for it and on an edge recorded +#: for a group that no longer meets it, so the mapping can only shrink, and +#: removing a module from one group's hash fails even where another group +#: records the same edge. UNCLASSIFIED_IMPORTS = MappingProxyType( { - "microcosm.build": ("microcosm.build.trace",), - "microcosm.build.calibration_columns": ( - "microcosm.build.transport.target_kernels", - ), - "microcosm.build.calibration_predicates": ( - "microcosm.build.transport.target_kernels", - ), - "microcosm.build.observation": ("microcosm.build.plan",), - "microcosm.build.reference_intervals": ( - "microcosm.build.transport.target_kernels", - ), - "microcosm.build.spec_engine.canonical": ( - "microcosm.build.spec_engine.engine_abi", - "microcosm.build.spec_engine.loader", - "microcosm.build.spec_engine.take_up_semantics", - ), - "microcosm.build.spec_engine.engine_abi": ( - "microcosm.build.spec_engine.loader", - ), - "microcosm.build.spec_engine.errors": ( - "microcosm.build.spec_engine.engine_abi", - "microcosm.build.spec_engine.loader", - "microcosm.build.spec_engine.schemas", - "microcosm.build.spec_engine.take_up_semantics", - "microcosm.build.spec_engine.yaml12", - ), - "microcosm.build.spec_engine.model": ( - "microcosm.build.spec_engine.canonical", - "microcosm.build.spec_engine.engine_abi", - "microcosm.build.spec_engine.loader", - ), - "microcosm.build.spec_engine.resolver": ("microcosm.build.spec_engine.loader",), - "microcosm.build.spec_engine.schemas": ( - "microcosm.build.spec_engine.engine_abi", - "microcosm.build.spec_engine.loader", - ), - "microcosm.build.spec_engine.seeds": ("microcosm.build.spec_engine.model",), - "microcosm.build.spec_engine.take_up_semantics": ( - "microcosm.build.spec_engine.engine_abi", - ), - "microcosm.build.spec_engine.yaml12": ("microcosm.build.spec_engine.loader",), - "microcosm.build.transport.column_kernels": ( - "microcosm.build.transport.target_kernels", - ), - "microcosm.build.transport.unit_declarations": ( - "microcosm.build.transport.population_kernels", - "microcosm.build.transport.target_kernels", - ), - "microcosm.build.us_runtime.multispine_pool": ( - "microcosm.build.spec_engine.engine_abi", - ), - "microcosm.calibrate.artifacts": ("microcosm.build.transport.target_kernels",), - "microcosm.calibrate.matrix": ( - "microcosm.build.transport.target_kernels", - "microcosm.calibrate.artifacts", - ), - "microcosm.calibrate.monetary_binding": ( - "microcosm.build.transport.target_kernels", - ), - "microcosm.calibrate.solve": ("microcosm.calibrate.artifacts",), - "microcosm.frame.adapters.policyengine_us": ( - "microcosm.build.spec_engine.engine_abi", - ), - "microcosm.frame.adapters.policyengine_us_concepts": ( - "microcosm.build.transport.population_kernels", - "microcosm.frame.transport", - ), - "microcosm.frame.bundle": ( - "microcosm.build.plan", - "microcosm.build.transport.population_kernels", - "microcosm.build.transport.target_kernels", - "microcosm.calibrate.artifacts", - "microcosm.graph.kernel", - ), - "microcosm.frame.concept_mapping": ( - "microcosm.build.transport.population_kernels", - ), - "microcosm.frame.concepts": ( - "microcosm.build.reference_intervals", - "microcosm.build.transport.population_kernels", - "microcosm.frame.transport", - ), - "microcosm.frame.materialize": ( - "microcosm.build.transport.population_kernels", - "microcosm.frame.transport", - ), - "microcosm.frame.scaling": ( - "microcosm.build.transport.population_kernels", - "microcosm.frame.transport", - ), - "microcosm.frame.schema": ( - "microcosm.build.transport.population_kernels", - "microcosm.build.transport.target_kernels", - "microcosm.frame.bundle", - ), - "microcosm.frame.unit_construction": ( - "microcosm.build.transport.population_kernels", - "microcosm.build.transport.target_kernels", - ), - "microcosm.frame.weights": ( - "microcosm.build.transport.population_kernels", - "microcosm.build.transport.target_kernels", - "microcosm.calibrate.artifacts", - "microcosm.frame.bundle", - "microcosm.graph.kernel", - ), - "microcosm.graph.canonical": ( - "microcosm.build.reference_intervals", - "microcosm.build.transport.quantile_kernels", - "microcosm.build.transport.takeup_kernels", - "microcosm.build.transport.target_kernels", - "microcosm.calibrate.artifacts", - ), - "microcosm.graph.decl": ( - "microcosm.build.transport.population_kernels", - "microcosm.build.transport.takeup_kernels", - "microcosm.build.transport.target_kernels", - "microcosm.calibrate.artifacts", - "microcosm.graph.canonical", - "microcosm.graph.kernel", - ), - "microcosm.graph.kernel": ( - "microcosm.build.transport.population_kernels", - "microcosm.build.transport.quantile_kernels", - "microcosm.build.transport.takeup_kernels", - "microcosm.build.transport.target_kernels", - ), - "microcosm.graph.population": ("microcosm.build.transport.population_kernels",), + "microcosm.build.trace -> microcosm.build": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.calibration_columns": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.calibration_predicates": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.plan -> microcosm.build.observation": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.reference_intervals": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.take_up_semantics -> microcosm.build.spec_engine.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.engine_abi": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.schemas -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.take_up_semantics -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.yaml12 -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.canonical -> microcosm.build.spec_engine.model": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.model": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.model": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.resolver": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.schemas": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.schemas": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.model -> microcosm.build.spec_engine.seeds": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.take_up_semantics": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.yaml12": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.transport.column_kernels": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.build.transport.unit_declarations": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.transport.unit_declarations": ( + COMPILE, + COMPARE, + GAP, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.us_runtime.multispine_pool": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.calibrate.artifacts": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.calibrate.matrix": ( + COMPILE, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.calibrate.matrix": ( + COMPILE, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.calibrate.monetary_binding": ( + COMPILE, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.calibrate.solve": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.frame.adapters.policyengine_us": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.adapters.policyengine_us_concepts": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.transport -> microcosm.frame.adapters.policyengine_us_concepts": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.plan -> microcosm.frame.bundle": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.bundle": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.frame.bundle": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.frame.bundle": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.graph.kernel -> microcosm.frame.bundle": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.concept_mapping": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.reference_intervals -> microcosm.frame.concepts": ( + COMPILE, + COMPARE, + GAP, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.concepts": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.transport -> microcosm.frame.concepts": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.materialize": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.transport -> microcosm.frame.materialize": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.scaling": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.transport -> microcosm.frame.scaling": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.schema": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.frame.schema": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.bundle -> microcosm.frame.schema": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.unit_construction": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.frame.unit_construction": ( + COMPILE, + COMPARE, + GAP, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.weights": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.frame.weights": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.frame.weights": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.bundle -> microcosm.frame.weights": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.graph.kernel -> microcosm.frame.weights": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.reference_intervals -> microcosm.graph.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.quantile_kernels -> microcosm.graph.canonical": ( + QUANTILE_MAP_2, + ), + "microcosm.build.transport.takeup_kernels -> microcosm.graph.canonical": (GAP,), + "microcosm.build.transport.target_kernels -> microcosm.graph.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.graph.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.graph.decl": ( + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.takeup_kernels -> microcosm.graph.decl": (GAP,), + "microcosm.build.transport.target_kernels -> microcosm.graph.decl": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.graph.decl": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.graph.canonical -> microcosm.graph.decl": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.graph.kernel -> microcosm.graph.decl": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.graph.kernel": ( + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.quantile_kernels -> microcosm.graph.kernel": ( + QUANTILE_MAP_2, + ), + "microcosm.build.transport.takeup_kernels -> microcosm.graph.kernel": (GAP,), + "microcosm.build.transport.target_kernels -> microcosm.graph.kernel": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.graph.population": ( + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), } ) @@ -254,22 +615,19 @@ def test_compile_exclusions_are_live_and_never_listed(): assert not excluded & set(listed) -def test_every_import_a_compile_kernel_reaches_is_hashed_or_classified(): +def test_every_import_a_compile_kernel_reaches_is_hashed_or_recorded(): """The walk covers each kernel's own module and every module hashed - beside the shared list, not only the list (microcosm#1492 review).""" + beside the shared list, not only the list (microcosm#1492 review), and + the record is kept per group of kernels sharing one hash.""" compile_list = _compile_list() - met: set[str] = set() - for kernel, module in KERNELS_HASHING_THE_COMPILE_MODULES: - hashed = hashed_module_names(kernel, module) - assert not set(HASH_INPUTS) & hashed, kernel.ref - met |= unhashed_imports( - hashed, - listed=compile_list.TRANSPORT_COMPILE_MODULE_NAMES, - excluded=frozenset(compile_list.EXCLUDED_DIRECT_IMPORTS), - ) - met = {edge for edge in met if edge.split(" -> ")[1] not in HASH_INPUTS} - recorded = edges(UNCLASSIFIED_IMPORTS) - assert met == recorded, { - "unhashed and not classified": sorted(met - recorded), - "recorded but no longer met (delete them)": sorted(recorded - met), + assert {kernel for group in KERNEL_GROUPS.values() for kernel, _ in group} == { + kernel for kernel, _ in KERNELS_HASHING_THE_COMPILE_MODULES } + mismatches = recorded_import_mismatches( + KERNEL_GROUPS, + listed=compile_list.TRANSPORT_COMPILE_MODULE_NAMES, + excluded=frozenset(compile_list.EXCLUDED_DIRECT_IMPORTS), + hash_inputs=HASH_INPUTS, + recorded=UNCLASSIFIED_IMPORTS, + ) + assert not mismatches, mismatches diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index 089dcf7b40..145c7e4790 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -6,14 +6,15 @@ ``UK_TARGET_COMPILE_MODULE_NAMES`` is listed, or excluded with a reason; - each kernel's: the walk from every module a target or problem kernel hashes, its own module included, meets only modules the kernel hashes, the - list's exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS``, - which no review has classified yet (microcosm#1497). That mapping may only - shrink. + list's exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS`` for + that kernel's group, which no review has classified yet (microcosm#1497). + That mapping may only shrink. """ from __future__ import annotations import importlib +from pathlib import Path from types import MappingProxyType import pytest @@ -35,11 +36,11 @@ uk_target_compile_modules, ) from test_support.microcosm_build.first_party_imports import ( + HashInput, assert_an_edit_rekeys, - edges, first_party_imports, hashed_module_names, - unhashed_imports, + recorded_import_mismatches, unlisted_imports, ) @@ -167,260 +168,523 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): # The measures and problem kernels build and encode constraint matrices. "microcosm.calibrate.matrix": _FULL_GRAPH + _NATIONAL_GRAPH, "microcosm.calibrate.artifacts": _FULL_GRAPH + _NATIONAL_GRAPH, - # The full problem's loss scales (default_target_loss_scales) and the - # national problem's effective sample size. - "microcosm.calibrate.solve": ("uk.full.problem@1", *_NATIONAL_GRAPH), + # The full problem's loss scales (default_target_loss_scales). The + # national kernels call nothing in it (see the test below). + "microcosm.calibrate.solve": ("uk.full.problem@1",), # full_targets' exclusion clock (exclusion_evaluation_date). "microcosm.build.uk_runtime.weighted_integrity": _FULL_GRAPH + _NATIONAL_GRAPH, # full_targets' surface provenance (_ledger_provenance) and band-edge - # roster check (_validate_band_edge_registry). - "microcosm.build.uk_runtime.calibration_run": _FULL_GRAPH + _NATIONAL_GRAPH, + # roster check (_validate_band_edge_registry), moved out of + # calibration_run so that the runner itself stays unhashed. + "microcosm.build.uk_runtime.ledger_surface": _FULL_GRAPH + _NATIONAL_GRAPH, } -#: Imports the kernels' modules make only to build the hash. They stay -#: unhashed on purpose, so an edit to the list re-keys only the kernels whose -#: hashed modules it adds or removes. +#: The kernels grouped by shared hash: the full graph's target, selection and +#: measures kernels share _TargetKernel's; the problem adds its loss-weight +#: hash; the two national kernels share _NationalKernel's. +FULL_TARGETS = "uk.full.target_compilation@1, target_selection@1 and measures@1" +FULL_PROBLEM = "uk.full.problem@1" +NATIONAL = "uk.full.national_targets@1 and national_problem@1" +KERNEL_GROUPS = MappingProxyType( + { + FULL_TARGETS: tuple( + _KERNELS_BY_REF[ref] + for ref in ( + "uk.full.target_compilation@1", + "uk.full.target_selection@1", + "uk.full.measures@1", + ) + ), + FULL_PROBLEM: (_KERNELS_BY_REF["uk.full.problem@1"],), + NATIONAL: tuple(_KERNELS_BY_REF[ref] for ref in _NATIONAL_GRAPH), + } +) + +#: Imports a kernel's module makes only to build the hash, and the importers +#: the reason covers. They stay unhashed on purpose, so an edit to the list +#: re-keys only the kernels whose hashed modules it adds or removes. HASH_INPUTS = MappingProxyType( { - "microcosm.build.uk_runtime.target_compile_modules": ( - "names the modules the target kernels hash; graph_targets and " - "graph_national read it only to build their hashes" - ), - "microcosm.build.uk_runtime.solve_scope": ( - "computes uk_solve_fingerprint, which graph_targets puts in the " - "full graph's target hashes and in the graph's contract-identity " - "parameter; no kernel's run reads it" + "microcosm.build.uk_runtime.target_compile_modules": HashInput( + importers=( + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_targets", + ), + reason=( + "names the modules the target kernels hash; graph_targets and " + "graph_national read it only to build their hashes" + ), + ), + "microcosm.build.uk_runtime.solve_scope": HashInput( + importers=("microcosm.build.uk_runtime.graph_targets",), + reason=( + "computes uk_solve_fingerprint, which graph_targets puts in the " + "full graph's target hashes and in the graph's contract-identity " + "parameter; no kernel's run reads it" + ), ), } ) -#: Every first-party module the walk from a target or problem kernel's -#: hashed modules meets that the kernel does not hash and no review has -#: classified, with the hashed (or excluded) modules that import it. Recorded -#: as found when the per-kernel check was added; microcosm#1497 classifies -#: them. Hash an entry in the kernels that run it, move what they run into a -#: module they hash, or record why they need not hash it, then delete it -#: here. The check fails on an import missing from this mapping and on an -#: entry the walk no longer meets, so the mapping can only shrink. +#: Every ``importer -> imported`` edge the walk from a kernel group's hashed +#: modules meets that the group does not hash and no review has classified, +#: with the groups that leave it unhashed. Recorded as found when the +#: per-kernel check was added; microcosm#1497 classifies them. Hash an entry +#: in the kernels that run it, move what they run into a module they hash, or +#: record why they need not hash it, then delete it here. The check fails on +#: an edge a group meets that is not recorded for it and on an edge recorded +#: for a group that no longer meets it, so the mapping can only shrink, and +#: removing a module from one group's hash fails even where another group +#: records the same edge. UNCLASSIFIED_IMPORTS = MappingProxyType( { - "microcosm.build.artifact_files": ( - "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_national -> microcosm.build.artifact_files": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.country_spec": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.country_spec": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.gate_battery": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.fiscal_targets -> microcosm.build.gates": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.gates": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.holdout": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.input_mass": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.logbook_adoption": ( + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.plan": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.source_manifest": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.frs_relationships -> microcosm.build.source_manifest": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.canonical": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.engine_abi": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.loader": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.model": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.schemas": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.take_up_semantics": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.yaml12": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.staging_v2": ( + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.trace": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.atomic_area_support": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.battery_bindings": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.calibration_run": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.cgt_projection": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.content_identity": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.dataset_size": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.dfe_spring_term": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.diagnostics": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.frs_annual_income": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.frs_relationships -> microcosm.build.uk_runtime.frs_spine": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, ), - "microcosm.build.country_spec": ( - "microcosm.build.uk_runtime.calibration_run", - "microcosm.build.uk_runtime.graph_national", - "microcosm.build.uk_runtime.measure_simulation", + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.frs_spine": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.full_gates": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_authority_input -> microcosm.build.uk_runtime.geography_sources": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_build": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_calibration": ( + NATIONAL, ), - "microcosm.build.gate_battery": ( - "microcosm.build.uk_runtime.calibration_run", - "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_population": ( + NATIONAL, ), - "microcosm.build.gates": ( - "microcosm.build.uk_runtime.fiscal_targets", - "microcosm.build.uk_runtime.weighted_integrity", + "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.graph_population": ( + FULL_TARGETS, + FULL_PROBLEM, ), - "microcosm.build.holdout": ("microcosm.build.uk_runtime.local_rowwise",), - "microcosm.build.input_mass": ( - "microcosm.build.uk_runtime.weighted_integrity", + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_targets": ( + NATIONAL, ), - "microcosm.build.logbook_adoption": ( - "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_terminal": ( + NATIONAL, ), - "microcosm.build.plan": ("microcosm.build.country_spec",), - "microcosm.build.source_manifest": ( - "microcosm.build.country_spec", - "microcosm.build.uk_runtime.frs_relationships", + "microcosm.build.uk_runtime.hmrc_replay -> microcosm.build.uk_runtime.hmrc_income": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, ), - "microcosm.build.spec_engine.canonical": ("microcosm.build.country_spec",), - "microcosm.build.spec_engine.engine_abi": ("microcosm.build.country_spec",), - "microcosm.build.spec_engine.loader": ("microcosm.build.country_spec",), - "microcosm.build.spec_engine.model": ("microcosm.build.country_spec",), - "microcosm.build.spec_engine.schemas": ("microcosm.build.country_spec",), - "microcosm.build.spec_engine.take_up_semantics": ( - "microcosm.build.country_spec", + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.hmrc_income": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, ), - "microcosm.build.spec_engine.yaml12": ("microcosm.build.country_spec",), - "microcosm.build.staging_v2": ("microcosm.build.uk_runtime.graph_national",), - "microcosm.build.trace": ("microcosm.build.country_spec",), - "microcosm.build.uk_runtime.atomic_area_support": ( - "microcosm.build.uk_runtime.measure_simulation", + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.uk_runtime.hmrc_source_contract": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, ), - "microcosm.build.uk_runtime.battery_bindings": ( - "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.income_tax_rate_classes": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, ), - "microcosm.build.uk_runtime.cgt_projection": ( - "microcosm.build.uk_runtime.calibration_run", - "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.lcfs_consumption": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, ), - "microcosm.build.uk_runtime.content_identity": ( - "microcosm.build.uk_runtime.calibration_run", - "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.local_geography": ( + FULL_PROBLEM, ), - "microcosm.build.uk_runtime.dataset_size": ( - "microcosm.build.uk_runtime.local_rowwise", + "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.local_rowwise": ( + FULL_TARGETS, + NATIONAL, ), - "microcosm.build.uk_runtime.dfe_spring_term": ( - "microcosm.build.uk_runtime.measure_simulation", + "microcosm.build.uk_runtime.full_problem -> microcosm.build.uk_runtime.local_rowwise": ( + FULL_TARGETS, ), - "microcosm.build.uk_runtime.diagnostics": ( - "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.local_rowwise": ( + NATIONAL, ), - "microcosm.build.uk_runtime.etb_services": ( - "microcosm.build.uk_runtime.calibration_run", + "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.local_rowwise": ( + FULL_TARGETS, ), - "microcosm.build.uk_runtime.frs_annual_income": ( - "microcosm.build.uk_runtime.measure_simulation", + "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.national_calibration": ( + FULL_TARGETS, + FULL_PROBLEM, ), - "microcosm.build.uk_runtime.frs_spine": ( - "microcosm.build.uk_runtime.frs_relationships", - "microcosm.build.uk_runtime.measure_simulation", + "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.national_doctrine": ( + FULL_TARGETS, + ), + "microcosm.build.uk_runtime.local_doctrine -> microcosm.build.uk_runtime.national_doctrine": ( + FULL_TARGETS, + ), + "microcosm.build.uk_runtime.frs_relationships -> microcosm.build.uk_runtime.national_frame": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.national_frame": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, ), - "microcosm.build.uk_runtime.full_gates": ( - "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.national_frame": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.national_frame": ( + FULL_PROBLEM, ), - "microcosm.build.uk_runtime.geography_sources": ( - "microcosm.build.uk_runtime.local_authority_input", + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.national_frame": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, ), - "microcosm.build.uk_runtime.graph_build": ( - "microcosm.build.uk_runtime.graph_national", - ), - "microcosm.build.uk_runtime.graph_calibration": ( - "microcosm.build.uk_runtime.graph_national", - ), - "microcosm.build.uk_runtime.graph_population": ( - "microcosm.build.uk_runtime.graph_national", - "microcosm.build.uk_runtime.graph_targets", - ), - "microcosm.build.uk_runtime.graph_targets": ( - "microcosm.build.uk_runtime.graph_national", - ), - "microcosm.build.uk_runtime.graph_terminal": ( - "microcosm.build.uk_runtime.graph_national", - ), - "microcosm.build.uk_runtime.hmrc_income": ( - "microcosm.build.uk_runtime.hmrc_replay", - "microcosm.build.uk_runtime.measure_simulation", - ), - "microcosm.build.uk_runtime.hmrc_source_contract": ( - "microcosm.build.uk_runtime.weighted_integrity", - ), - "microcosm.build.uk_runtime.income_tax_rate_classes": ( - "microcosm.build.uk_runtime.measure_simulation", - ), - "microcosm.build.uk_runtime.lcfs_consumption": ( - "microcosm.build.uk_runtime.measure_simulation", - ), - "microcosm.build.uk_runtime.local_geography": ( - "microcosm.build.uk_runtime.local_rowwise", - ), - "microcosm.build.uk_runtime.local_rowwise": ( - "microcosm.build.uk_runtime.full_measure", - "microcosm.build.uk_runtime.full_problem", - "microcosm.build.uk_runtime.graph_national", - "microcosm.build.uk_runtime.graph_targets", - ), - "microcosm.build.uk_runtime.national_calibration": ( - "microcosm.build.uk_runtime.full_measure", - ), - "microcosm.build.uk_runtime.national_doctrine": ( - "microcosm.build.uk_runtime.graph_targets", - "microcosm.build.uk_runtime.local_doctrine", - ), - "microcosm.build.uk_runtime.national_frame": ( - "microcosm.build.uk_runtime.calibration_run", - "microcosm.build.uk_runtime.frs_relationships", - "microcosm.build.uk_runtime.full_measure", - "microcosm.build.uk_runtime.graph_national", - "microcosm.build.uk_runtime.local_rowwise", - "microcosm.build.uk_runtime.measure_simulation", - "microcosm.build.uk_runtime.weighted_integrity", - ), - "microcosm.build.uk_runtime.release_certification": ( - "microcosm.build.uk_runtime.calibration_run", - ), - "microcosm.build.uk_runtime.rowwise_dataset": ( - "microcosm.build.uk_runtime.full_measure", - ), - "microcosm.build.uk_runtime.rowwise_posture": ( - "microcosm.build.uk_runtime.graph_national", - ), - "microcosm.build.uk_runtime.size_checkpoint": ( - "microcosm.build.uk_runtime.local_rowwise", - ), - "microcosm.build.uk_runtime.solve_progress": ( - "microcosm.build.uk_runtime.local_rowwise", - ), - "microcosm.build.uk_runtime.target_weights": ( - "microcosm.build.uk_runtime.graph_targets", - ), - "microcosm.build.uk_runtime.uc_target_measurements": ( - "microcosm.build.uk_runtime.measure_simulation", - ), - "microcosm.build.uk_runtime.weight_year": ( - "microcosm.build.uk_runtime.graph_national", - ), - "microcosm.build.us_runtime.stacked_spine": ("microcosm.build.gates",), - "microcosm.calibrate.exact_k": ("microcosm.calibrate.solve",), - "microcosm.calibrate.gates": ("microcosm.calibrate.solve",), - "microcosm.calibrate.initialization": ("microcosm.calibrate.solve",), - "microcosm.calibrate.monetary_binding": ("microcosm.calibrate.matrix",), - "microcosm.calibrate.solve": ( - "microcosm.build.gates", - "microcosm.build.uk_runtime.graph_targets", - "microcosm.calibrate.artifacts", - ), - "microcosm.frame.adapters.policyengine_uk": ( - "microcosm.build.uk_runtime.measure_simulation", - "microcosm.build.uk_runtime.national_frame", - ), - "microcosm.frame.bundle": ( - "microcosm.build.country_spec", - "microcosm.build.uk_runtime.calibration_run", - "microcosm.build.uk_runtime.frs_relationships", - "microcosm.build.uk_runtime.full_measure", - "microcosm.build.uk_runtime.graph_national", - "microcosm.build.uk_runtime.local_rowwise", - "microcosm.build.uk_runtime.measure_simulation", - "microcosm.build.uk_runtime.national_calibration", - "microcosm.build.uk_runtime.national_frame", - "microcosm.build.uk_runtime.weighted_integrity", - "microcosm.calibrate.artifacts", - "microcosm.calibrate.matrix", - "microcosm.calibrate.solve", - ), - "microcosm.frame.materialize": ("microcosm.build.uk_runtime.national_frame",), - "microcosm.frame.schema": ( - "microcosm.build.uk_runtime.national_frame", - "microcosm.frame.bundle", - ), - "microcosm.frame.weights": ( - "microcosm.build.gates", - "microcosm.build.uk_runtime.full_measure", - "microcosm.build.uk_runtime.graph_national", - "microcosm.build.uk_runtime.local_rowwise", - "microcosm.build.uk_runtime.measure_simulation", - "microcosm.build.uk_runtime.national_calibration", - "microcosm.build.uk_runtime.national_frame", - "microcosm.calibrate.artifacts", - "microcosm.calibrate.matrix", - "microcosm.calibrate.solve", - "microcosm.frame.bundle", - ), - "microcosm.graph.canonical": ( - "microcosm.build.uk_runtime.graph_national", - "microcosm.build.uk_runtime.graph_targets", - "microcosm.calibrate.artifacts", - ), - "microcosm.graph.codecs": ("microcosm.build.uk_runtime.graph_targets",), - "microcosm.graph.decl": ( - "microcosm.build.uk_runtime.graph_national", - "microcosm.build.uk_runtime.graph_targets", - "microcosm.calibrate.artifacts", - ), - "microcosm.graph.kernel": ( - "microcosm.build.uk_runtime.graph_national", - "microcosm.build.uk_runtime.graph_targets", + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.uk_runtime.national_frame": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.rowwise_dataset": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.rowwise_posture": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.size_checkpoint": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.solve_progress": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.target_weights": ( + FULL_TARGETS, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.uc_target_measurements": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.weight_year": ( + NATIONAL, + ), + "microcosm.build.gates -> microcosm.build.us_runtime.stacked_spine": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.solve -> microcosm.calibrate.exact_k": (FULL_PROBLEM,), + "microcosm.calibrate.solve -> microcosm.calibrate.gates": (FULL_PROBLEM,), + "microcosm.calibrate.solve -> microcosm.calibrate.initialization": ( + FULL_PROBLEM, + ), + "microcosm.calibrate.matrix -> microcosm.calibrate.monetary_binding": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.gates -> microcosm.calibrate.solve": (FULL_TARGETS, NATIONAL), + # Reviewed (round 2 of #1493): only national_calibration_manifest calls + # it, in uk.full.national-gates@1, which reads the solve's result. + "microcosm.build.uk_runtime.graph_national -> microcosm.calibrate.solve": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.calibrate.solve": ( + FULL_TARGETS, + ), + "microcosm.calibrate.artifacts -> microcosm.calibrate.solve": ( + FULL_TARGETS, + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.frame.adapters.policyengine_uk": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.adapters.policyengine_uk": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.frs_relationships -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.frame.bundle": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.frame.bundle": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.national_calibration -> microcosm.frame.bundle": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.artifacts -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.matrix -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.solve -> microcosm.frame.bundle": (FULL_PROBLEM,), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.materialize": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.schema": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.frame.bundle -> microcosm.frame.schema": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.gates -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.frame.weights": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.frame.weights": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.national_calibration -> microcosm.frame.weights": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.artifacts -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.matrix -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.solve -> microcosm.frame.weights": (FULL_PROBLEM,), + "microcosm.frame.bundle -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.graph.canonical": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.canonical": ( + FULL_TARGETS, + FULL_PROBLEM, + ), + "microcosm.calibrate.artifacts -> microcosm.graph.canonical": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.codecs": ( + FULL_TARGETS, + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.graph.decl": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.decl": ( + FULL_TARGETS, + FULL_PROBLEM, + ), + "microcosm.calibrate.artifacts -> microcosm.graph.decl": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.graph.kernel": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.kernel": ( + FULL_TARGETS, + FULL_PROBLEM, ), } ) @@ -457,6 +721,28 @@ def test_an_edit_to_code_the_kernels_run_rekeys_them( assert_an_edit_rekeys(kernel, module, tmp_path, monkeypatch) +@pytest.mark.parametrize( + ("kernel", "module"), + _TARGET_AND_PROBLEM_KERNELS, + ids=[kernel.ref for kernel, _ in _TARGET_AND_PROBLEM_KERNELS], +) +def test_an_edit_to_calibration_run_leaves_the_kernels_keys( + tmp_path, monkeypatch, kernel, module +): + """The target kernels run only the two functions now in ledger_surface. + An edit elsewhere in calibration_run, the national calibration runner, + re-runs no target compilation and, in builds that construct the spine in + the graph, no solve (microcosm#1492).""" + from microcosm.build.uk_runtime import calibration_run + + before = kernel.implementation_hash() + source = Path(calibration_run.__file__) + edited = tmp_path / source.name + edited.write_bytes(source.read_bytes() + b"\n# an edited line\n") + monkeypatch.setattr(calibration_run, "__file__", str(edited)) + assert kernel.implementation_hash() == before + + def test_full_problem_loss_weights_hash_what_target_weights_reads(monkeypatch): """target_weights takes UK_WIDE_GEOGRAPHY_IDS from geography_constants and the loss scales from calibrate.solve for the weights the problem @@ -474,21 +760,29 @@ def test_full_problem_loss_weights_hash_what_target_weights_reads(monkeypatch): assert calibrate_solve in weights -def test_every_import_a_target_kernel_reaches_is_hashed_or_classified(): +def test_the_national_kernels_leave_calibrate_solve_unhashed(): + """Neither the national target nor the problem path calls calibrate.solve: + its one caller in graph_national, national_calibration_manifest, runs in + uk.full.national-gates@1, whose node reads the solve's result. Hashing it + here would re-run national target compilation and the problem's engine + measures on every solver edit, with no coverage gain (round 2 of #1493).""" + for ref in _NATIONAL_GRAPH: + kernel, module = _KERNELS_BY_REF[ref] + assert calibrate_solve.__name__ not in hashed_module_names(kernel, module) + + +def test_every_import_a_target_kernel_reaches_is_hashed_or_recorded(): """The walk covers each kernel's own module and every module hashed - beside the shared list, not only the list (microcosm#1492 review).""" - met: set[str] = set() - for kernel, module in _TARGET_AND_PROBLEM_KERNELS: - hashed = hashed_module_names(kernel, module) - assert not set(HASH_INPUTS) & hashed, kernel.ref - met |= unhashed_imports( - hashed, - listed=UK_TARGET_COMPILE_MODULE_NAMES, - excluded=frozenset(EXCLUDED_DIRECT_IMPORTS), - ) - met = {edge for edge in met if edge.split(" -> ")[1] not in HASH_INPUTS} - recorded = edges(UNCLASSIFIED_IMPORTS) - assert met == recorded, { - "unhashed and not classified": sorted(met - recorded), - "recorded but no longer met (delete them)": sorted(recorded - met), + beside the shared list, not only the list (microcosm#1492 review), and + the record is kept per group of kernels sharing one hash.""" + assert {kernel for group in KERNEL_GROUPS.values() for kernel, _ in group} == { + kernel for kernel, _ in _TARGET_AND_PROBLEM_KERNELS } + mismatches = recorded_import_mismatches( + KERNEL_GROUPS, + listed=UK_TARGET_COMPILE_MODULE_NAMES, + excluded=frozenset(EXCLUDED_DIRECT_IMPORTS), + hash_inputs=HASH_INPUTS, + recorded=UNCLASSIFIED_IMPORTS, + ) + assert not mismatches, mismatches diff --git a/test_support/microcosm_build/first_party_imports.py b/test_support/microcosm_build/first_party_imports.py index 13295cadbc..105b79b360 100644 --- a/test_support/microcosm_build/first_party_imports.py +++ b/test_support/microcosm_build/first_party_imports.py @@ -30,6 +30,7 @@ from collections.abc import Set as AbstractSet from pathlib import Path from types import ModuleType +from typing import NamedTuple #: Every workspace package imports as ``microcosm.``. FIRST_PARTY_PREFIX = "microcosm." @@ -236,14 +237,76 @@ def unhashed_imports( return uncovered -def edges(unclassified: Mapping[str, Sequence[str]]) -> set[str]: - """``importer -> imported`` for a mapping of each import to its importers.""" +class HashInput(NamedTuple): + """A module a kernel's file imports only to compute the hash itself. - return { + The exemption holds for the named importers alone, so a hashed module + that starts calling it at run time is still reported. + """ + + importers: tuple[str, ...] + reason: str + + +def recorded_import_mismatches( + kernel_groups: Mapping[str, Sequence[tuple[object, ModuleType]]], + *, + listed: Sequence[str], + excluded: AbstractSet[str], + hash_inputs: Mapping[str, HashInput], + recorded: Mapping[str, Sequence[str]], +) -> dict[str, object]: + """Where the walk of each kernel group disagrees with the record. + + ``kernel_groups`` maps a group name to its ``(kernel, module)`` pairs; + every kernel in a group must reach the same unhashed imports, because + the group shares one hash. ``recorded`` maps each ``importer -> + imported`` edge to the groups that leave it unhashed, so removing a + module from one group's hash fails even when another group records the + same edge. Every ``hash_inputs`` edge must be met by some group, and no + kernel may hash a hash input. An empty result means the record holds. + """ + + allowed = { f"{importer} -> {imported}" - for imported, importers in unclassified.items() - for importer in importers + for imported, hash_input in hash_inputs.items() + for importer in hash_input.importers + } + mismatches: dict[str, object] = {} + unknown = { + edge: sorted(set(groups) - set(kernel_groups)) + for edge, groups in recorded.items() + if set(groups) - set(kernel_groups) } + if unknown: + mismatches["recorded for an unknown group"] = unknown + allowed_met: set[str] = set() + for group, kernels in kernel_groups.items(): + walks = [] + for kernel, module in kernels: + hashed = hashed_module_names(kernel, module) + if set(hash_inputs) & hashed: + mismatches[f"{kernel.ref} hashes a hash input"] = sorted( + set(hash_inputs) & hashed + ) + walks.append(unhashed_imports(hashed, listed=listed, excluded=excluded)) + if any(walk != walks[0] for walk in walks): + mismatches[f"{group}: its kernels reach different imports"] = sorted( + set().union(*walks) - set.intersection(*walks) + ) + met = set().union(*walks) + allowed_met |= met & allowed + met -= allowed + expected = {edge for edge, groups in recorded.items() if group in groups} + if met - expected: + mismatches[f"{group}: unhashed and not recorded"] = sorted(met - expected) + if expected - met: + mismatches[f"{group}: recorded but no longer met (delete them)"] = sorted( + expected - met + ) + if allowed - allowed_met: + mismatches["hash inputs no importer reaches"] = sorted(allowed - allowed_met) + return mismatches def hashed_module_names(kernel: object, module: ModuleType) -> frozenset[str]: From 42759b7ec0cf75b78bfd32a22aa4986a5668862f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:35:02 +0400 Subject: [PATCH 7/7] Move the surface helpers out of calibration_run, and drop calibrate.solve from the national hash (green) Review round 2 of #1493, and Anthony's decision on calibration_run ("Move the functions"). - uk_runtime/ledger_surface.py (new): _ledger_provenance, _validate_band_edge_registry and the constants they read, moved from calibration_run. full_targets imports them from here. calibration_run re-exports both, so its other callers, the tests among them, see no change. The UK list names the new module, so both base classes hash it. - calibration_run leaves _TargetKernel's and _NationalKernel's hashes. In builds that construct the spine in the graph, an edit to the rest of it no longer re-runs target compilation, selection, the measures node, the problem, the preflight node, the solve or the holdout. In builds from a spine checkpoint, uk.full.bound_spine@1 still hashes it. Editing calibration_run.py gives that kernel, and so the bound spine node and the population nodes after it, new keys once; Anthony accepted that. - calibrate.solve leaves _NationalKernel. Neither the national target nor the problem path calls it; its one caller in graph_national, national_calibration_manifest, runs in uk.full.national-gates@1, which reads the solve's result. The full problem's loss-weight hash keeps it, for default_target_loss_scales. This corrects the effective-sample-size reason given for the national kernels in 6a2e35d78. - The changelog fragment states the bound spine re-key, the calibration_run consequence, weighted_integrity in national builds (no validation node runs before the national solve, so an edit there now re-runs the national targets, problem and solve), and the per-group record. Implementation hashes: of the 96 kernel classes, the 89 that build without an engine object, compared with main: 30 change, the 29 of 6a2e35d78 plus uk.full.bound_spine@1. No tracked file contains any of the 30 old hashes, so no pin changes. Green run (by node ID): the same 9 test functions, 42 cases, all pass. As a check that the per-group record closes the pooled cases, removing local_rowwise from the full problem's loss-weight hash, and _MATRIX_MODULES from takeup.compare@1, each made the record check fail; both edits were then reverted. Lint and format checked on the changed files only. Co-Authored-By: Claude Opus 5.5 --- ...-compile-keys-geography-constants.fixed.md | 2 +- .../build/uk_runtime/calibration_run.py | 112 ++-------------- .../build/uk_runtime/full_targets.py | 8 +- .../build/uk_runtime/graph_national.py | 18 ++- .../build/uk_runtime/graph_targets.py | 6 - .../build/uk_runtime/ledger_surface.py | 120 ++++++++++++++++++ .../uk_runtime/target_compile_modules.py | 1 + 7 files changed, 142 insertions(+), 125 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_surface.py diff --git a/changelog.d/1492-target-compile-keys-geography-constants.fixed.md b/changelog.d/1492-target-compile-keys-geography-constants.fixed.md index e98a87adc7..41e2cd2c2b 100644 --- a/changelog.d/1492-target-compile-keys-geography-constants.fixed.md +++ b/changelog.d/1492-target-compile-keys-geography-constants.fixed.md @@ -1 +1 @@ -The graph kernels that compile targets now include `microcosm.calibrate.geography_constants` in their implementation hashes, so an edit to a geography label recompiles the targets instead of reusing a stored surface that carries the old label (microcosm#1492). `ledger_targets` labels a target's geography from `UK_GEOGRAPHY_ID_TO_LABEL` when the Chronicle facts give no name, and `uk_runtime/target_weights` reads `UK_WIDE_GEOGRAPHY_IDS` for the UK loss weights, both through `from … import`, which `source_hash` does not follow. The module joins the transport compile modules (hashed by `targets.compile@1`, `takeup.compare@1`, `takeup.gap@1`, `transport.create@1`/`@2` and `transport.quantile_map@1`/`@2`), the UK target compile modules (hashed by the full graph's target, selection, measures and problem kernels and the national graph's target and problem kernels) and the UK problem kernel's loss-weight hash. The UK list also gains `microcosm.calibrate.registry`, `target` and `hierarchy`, which the transport list already hashed. The UK target and problem kernels also now hash code they ran unhashed: `calibrate.target_selection` (full graph), `calibrate.matrix` and `calibrate.artifacts`, `calibrate.solve` (the full problem's loss scales and the national problem's effective sample size; `calibrate.adam@1`, whose hash the UK calibration kernels embed, already hashed it), and `uk_runtime.weighted_integrity` and `uk_runtime.calibration_run`, whose exclusion clock, surface provenance and band-edge check `full_targets` runs. The transport compile list and its 18 exclusions move from `transport/target_kernels.py`, which 13 kernels hash, to `transport/compile_modules.py`, which none does, as the UK list already sat apart. Two checks now pin the hashes: a shared list's imports must be listed or excluded, reading every `microcosm.*` import rather than only `microcosm.build` ones, and the walk from every module a compile kernel hashes, its own module included, must meet only hashed modules, the list's exclusions or the imports no review has classified yet, recorded in the tests and tracked by microcosm#1497 (67 modules for the UK, 35 for transport), a record that can only shrink. Node keys move once: the target compilation of every transport country and everything after it; the UK target, selection, measures, problem and national target and problem nodes and everything after them, including the UK solve; and, because `transport/target_kernels.py` itself changes, `calibration.prepare@1` nodes, `gates.battery@1` nodes at a phase before compilation, and `takeup.assign@1` nodes given no compiled surface. +The graph kernels that compile targets now include `microcosm.calibrate.geography_constants` in their implementation hashes, so an edit to a geography label recompiles the targets instead of reusing a stored surface that carries the old label (microcosm#1492). `ledger_targets` labels a target's geography from `UK_GEOGRAPHY_ID_TO_LABEL` when the Chronicle facts give no name, and `uk_runtime/target_weights` reads `UK_WIDE_GEOGRAPHY_IDS` for the UK loss weights, both through `from … import`, which `source_hash` does not follow. The module joins the transport compile modules (hashed by `targets.compile@1`, `takeup.compare@1`, `takeup.gap@1`, `transport.create@1`/`@2` and `transport.quantile_map@1`/`@2`), the UK target compile modules (hashed by the full graph's target, selection, measures and problem kernels and the national graph's target and problem kernels) and the UK problem kernel's loss-weight hash. The UK list also gains `microcosm.calibrate.registry`, `target` and `hierarchy`, which the transport list already hashed. The UK target and problem kernels also now hash code they ran unhashed: `calibrate.target_selection` (full graph), `calibrate.matrix` and `calibrate.artifacts`, `calibrate.solve` (the full problem's loss scales only; `calibrate.adam@1`, whose hash the UK calibration kernels embed, already hashed it), and `uk_runtime.weighted_integrity`, whose exclusion clock `full_targets` reads. In national builds an edit to `weighted_integrity` now re-runs the national targets, problem and solve; before, only the national validation node after the solve hashed it. `_ledger_provenance` and `_validate_band_edge_registry`, which `full_targets` runs, move from `uk_runtime.calibration_run` into a small `uk_runtime.ledger_surface` that the UK list names, and `calibration_run` re-exports them. The rest of `calibration_run` stays outside the target kernels' hashes: in builds that construct the spine in the graph, an edit to it re-runs no target compilation and no solve, and in builds from a spine checkpoint it re-keys the bound spine node and everything after it, as before. The transport compile list and its 18 exclusions move from `transport/target_kernels.py`, which 13 kernels hash, to `transport/compile_modules.py`, which none does, as the UK list already sat apart. Two checks now pin the hashes: a shared list's imports must be listed or excluded, reading every `microcosm.*` import rather than only `microcosm.build` ones, and the walk from every module a compile kernel hashes, its own module included, must meet only hashed modules, the list's exclusions or the imports no review has classified yet, recorded per group of kernels sharing one hash and tracked by microcosm#1497 (66 modules for the UK, 35 for transport), a record that can only shrink. Node keys move once: the target compilation of every transport country and everything after it; the UK target, selection, measures, problem and national target and problem nodes and everything after them, including the UK solve; because `transport/target_kernels.py` itself changes, `calibration.prepare@1` nodes, `gates.battery@1` nodes at a phase before compilation, and `takeup.assign@1` nodes given no compiled surface; and, because `calibration_run.py` changes, the bound spine checkpoint node (`uk.full.bound_spine@1`) and the population nodes after it. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 0a88fab22d..580b2a6ff4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -46,6 +46,14 @@ from microcosm.build.uk_runtime.etb_services import ( UK_NHS_SPENDING_COMPONENT_COLUMNS, ) + +# Moved to ledger_surface, which the UK target kernels hash, so an edit +# elsewhere in this module no longer re-keys them (microcosm#1492). Kept +# importable from here for this module's other callers. +from microcosm.build.uk_runtime.ledger_surface import ( # noqa: F401 + _ledger_provenance, + _validate_band_edge_registry, +) from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_time_period, @@ -307,45 +315,6 @@ def new_uk_calibration_attempt_id(*, timestamp: datetime) -> str: _new_calibration_attempt_id = new_uk_calibration_attempt_id -def _validate_band_edge_registry( - *, - register_registry: TargetRegistry, - band_edge_registry: TargetRegistry, - exclusion_receipt: Mapping[str, Mapping[str, str]], -) -> None: - """Require the band-edge register to reconstitute the compiled roster. - - The reconciliation always runs — an empty receipt is a claim that nothing - was pruned, so the two rosters must then be name-identical; it is never - permission to skip the check. Receipt keys are spec names by the - applier's construction: ``apply_uk_calibration_measure_exclusions`` - builds the receipt from the matched ``spec.name`` set and raises on any - exclusion matching zero registry specs, so ``pruned + receipt keys == - compiled`` is an exact identity, not a heuristic. - """ - - register_names = _registry_spec_names(register_registry) - edge_names = _registry_spec_names(band_edge_registry) - excluded_names = {str(name) for name in exclusion_receipt} - if not register_names <= edge_names: - missing = sorted(register_names - edge_names) - raise ValueError( - "band-edge register must include every materialized registry spec; " - f"missing={missing}." - ) - extra = edge_names - register_names - if extra != excluded_names: - raise ValueError( - "band-edge register extra spec names must match the measure " - f"exclusion receipt; extra={sorted(extra)}, " - f"receipt={sorted(excluded_names)}." - ) - - -def _registry_spec_names(registry: TargetRegistry) -> set[str]: - return {str(spec.name) for spec in registry.specs} - - def load_bound_spine_sidecar(path: Path, frame: Frame) -> dict[str, object]: if not path.is_file(): raise ValueError(f"input H5 build sidecar absent: {path}") @@ -834,71 +803,6 @@ def uk_aggregate_admin_totals( return totals, receipt -#: The manifest fields the UK run seals into ``run_config``. Narrower than -#: the artifact's whole manifest on purpose: the identity digest should name -#: *which* published artifact was compiled, not re-hash its manifest. -_LEDGER_MANIFEST_IDENTITY_FIELDS = ( - "artifact_id", - "profile", - "schema_version", - "generated_at", -) - -#: Epoch witnesses :meth:`LedgerConsumerArtifact.provenance` supplies, split -#: by shape. Named here so the delegation below cannot quietly drop one: a -#: run that compiled a chronicle-era or mixed-epoch feed has to say so in its -#: own evidence, not only in the loader's return value -#: (PolicyEngine/chronicle#143). -_LEDGER_EPOCH_WITNESS_SCALARS = ("schema_epoch",) -_LEDGER_EPOCH_WITNESS_LISTS = ( - "fact_key_epochs", - "undeclared_fact_key_domains", - "fact_schema_versions", -) - - -def _ledger_provenance(artifact: Any) -> dict[str, object]: - """The verified identity of the Chronicle consumer feed this run compiled. - - Delegates to :meth:`microcosm.build.ledger_artifact.LedgerConsumerArtifact.provenance` - rather than rebuilding the block field by field. Rebuilding it is how the - epoch witnesses went missing here in the first place: the loader learned - which era resolved the targets and the UK run kept reporting only the - hashes. Anything the shared block gains, this block gains. - - The whole block is sealed into ``run_config`` and so into the run's - identity digest: the epoch witnesses enter it beside the hashes, which is - why this delegation is a change to the digest of an otherwise identical - configuration, once. Only the manifest sub-block is UK-shaped, and it - stays narrow so the digest names the artifact rather than re-hashing its - whole manifest. - - A bare ``consumer_facts.jsonl`` feed carries no manifest, so its - Chronicle-side provenance is recorded as absent rather than invented. So - is a stand-in that predates the shared block: the fields it cannot supply - are recorded ``None``/empty rather than fabricated. - """ - - shared = getattr(artifact, "provenance", None) - block: Mapping[str, Any] = shared() if callable(shared) else {} - provenance: dict[str, object] = { - field: block.get(field, getattr(artifact, field, None)) - for field in ("facts_sha256", "fact_row_count", "manifest_sha256") - } - for field in _LEDGER_EPOCH_WITNESS_SCALARS: - provenance[field] = block.get(field) - for field in _LEDGER_EPOCH_WITNESS_LISTS: - provenance[field] = list(block.get(field) or ()) - manifest = getattr(artifact, "manifest", None) - if isinstance(manifest, Mapping): - provenance["manifest"] = { - key: manifest.get(key) - for key in _LEDGER_MANIFEST_IDENTITY_FIELDS - if manifest.get(key) is not None - } - return provenance - - #: Packages whose versions the seam pins into the signed diagnostics build #: block. The release assembler treats these as the only legitimate source #: for manifest runtime/compatibility pins. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py index 60f3fb3465..2a9e1fd30c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py @@ -14,10 +14,6 @@ from typing import Any from microcosm.build.ledger_artifact import load_ledger_consumer_artifact -from microcosm.build.uk_runtime.calibration_run import ( - _ledger_provenance, - _validate_band_edge_registry, -) from microcosm.build.uk_runtime.chronicle_feed import ( load_uk_chronicle_feed, require_committed_uk_chronicle_feed_pin, @@ -27,6 +23,10 @@ uk_derived_fact_rows, ) from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.ledger_surface import ( + _ledger_provenance, + _validate_band_edge_registry, +) from microcosm.build.uk_runtime.ledger_targets import ( _uk_contract_targets, assert_uk_local_deferrals_in_force, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index c7ae40f03e..54bc522b9f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -474,7 +474,6 @@ class _NationalKernel(KernelBase): def implementation_hash(self) -> str: import microcosm.calibrate.artifacts as calibrate_artifacts import microcosm.calibrate.matrix as calibrate_matrix - import microcosm.calibrate.solve as calibrate_solve from . import weighted_integrity from .target_compile_modules import uk_target_compile_modules @@ -494,18 +493,17 @@ def implementation_hash(self) -> str: national_calibration, national_doctrine, support_floor_rules, - # The constraint matrices and their encoding, and the problem's - # effective sample size, as the dense kernels hash them - # (microcosm#1492). calibrate.adam@1, whose hash the UK calibration - # kernels of this graph embed, already hashes calibrate.solve. + # The constraint matrices and their encoding, as the dense kernels + # hash them (microcosm#1492). Not calibrate.solve: neither the + # national target nor the problem path calls it. Its one caller + # here, national_calibration_manifest, runs in + # uk.full.national-gates@1, which reads the solve's result, whose + # key moves with calibrate.solve through calibrate.adam@1. calibrate_matrix, calibrate_artifacts, - calibrate_solve, - # full_targets takes its exclusion clock from weighted_integrity, - # and its surface provenance and band-edge check from - # calibration_run; hashed directly, as the dense target kernels do. + # full_targets takes its exclusion clock from here; hashed + # directly, as the dense target kernels do. weighted_integrity, - calibration_run, *uk_target_compile_modules(), ) return hashlib.sha256( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index 39bbe4308f..2dddec06d7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -223,7 +223,6 @@ class _TargetKernel(KernelBase): def implementation_hash(self) -> str: from . import ( - calibration_run, full_targets, high_value_homes, measure_simulation, @@ -255,11 +254,6 @@ def implementation_hash(self) -> str: # directly, not listed: the module also loads the registers only # the validation checks read, which no listed module may. weighted_integrity, - # full_targets writes _ledger_provenance into the surface and runs - # _validate_band_edge_registry from here. uk.full.bound_spine@1, - # which loads a spine checkpoint ahead of the target nodes, - # already hashes it. - calibration_run, *uk_target_compile_modules(), ) # The solve fingerprint: every packaged resource but the gate-only diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_surface.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_surface.py new file mode 100644 index 0000000000..76d53024c8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_surface.py @@ -0,0 +1,120 @@ +"""What UK target compilation records about its Chronicle feed, and a roster check. + +``full_targets`` writes :func:`_ledger_provenance` into every compiled surface +and runs :func:`_validate_band_edge_registry` on the compiled registers. Both +lived in ``calibration_run``, the national calibration runner. The UK target +kernels hash this small module through ``UK_TARGET_COMPILE_MODULE_NAMES`` +instead of all of ``calibration_run`` (microcosm#1492), so an edit to either +function re-keys the compiled surface and an edit to the runner does not. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from microcosm.calibrate import TargetRegistry + + +def _validate_band_edge_registry( + *, + register_registry: TargetRegistry, + band_edge_registry: TargetRegistry, + exclusion_receipt: Mapping[str, Mapping[str, str]], +) -> None: + """Require the band-edge register to reconstitute the compiled roster. + + The reconciliation always runs — an empty receipt is a claim that nothing + was pruned, so the two rosters must then be name-identical; it is never + permission to skip the check. Receipt keys are spec names by the + applier's construction: ``apply_uk_calibration_measure_exclusions`` + builds the receipt from the matched ``spec.name`` set and raises on any + exclusion matching zero registry specs, so ``pruned + receipt keys == + compiled`` is an exact identity, not a heuristic. + """ + + register_names = _registry_spec_names(register_registry) + edge_names = _registry_spec_names(band_edge_registry) + excluded_names = {str(name) for name in exclusion_receipt} + if not register_names <= edge_names: + missing = sorted(register_names - edge_names) + raise ValueError( + "band-edge register must include every materialized registry spec; " + f"missing={missing}." + ) + extra = edge_names - register_names + if extra != excluded_names: + raise ValueError( + "band-edge register extra spec names must match the measure " + f"exclusion receipt; extra={sorted(extra)}, " + f"receipt={sorted(excluded_names)}." + ) + + +def _registry_spec_names(registry: TargetRegistry) -> set[str]: + return {str(spec.name) for spec in registry.specs} + + +#: The manifest fields the UK run seals into ``run_config``. Narrower than +#: the artifact's whole manifest on purpose: the identity digest should name +#: *which* published artifact was compiled, not re-hash its manifest. +_LEDGER_MANIFEST_IDENTITY_FIELDS = ( + "artifact_id", + "profile", + "schema_version", + "generated_at", +) + +#: Epoch witnesses :meth:`LedgerConsumerArtifact.provenance` supplies, split +#: by shape. Named here so the delegation below cannot quietly drop one: a +#: run that compiled a chronicle-era or mixed-epoch feed has to say so in its +#: own evidence, not only in the loader's return value +#: (PolicyEngine/chronicle#143). +_LEDGER_EPOCH_WITNESS_SCALARS = ("schema_epoch",) +_LEDGER_EPOCH_WITNESS_LISTS = ( + "fact_key_epochs", + "undeclared_fact_key_domains", + "fact_schema_versions", +) + + +def _ledger_provenance(artifact: Any) -> dict[str, object]: + """The verified identity of the Chronicle consumer feed this run compiled. + + Delegates to :meth:`microcosm.build.ledger_artifact.LedgerConsumerArtifact.provenance` + rather than rebuilding the block field by field. Rebuilding it is how the + epoch witnesses went missing here in the first place: the loader learned + which era resolved the targets and the UK run kept reporting only the + hashes. Anything the shared block gains, this block gains. + + The whole block is sealed into ``run_config`` and so into the run's + identity digest: the epoch witnesses enter it beside the hashes, which is + why this delegation is a change to the digest of an otherwise identical + configuration, once. Only the manifest sub-block is UK-shaped, and it + stays narrow so the digest names the artifact rather than re-hashing its + whole manifest. + + A bare ``consumer_facts.jsonl`` feed carries no manifest, so its + Chronicle-side provenance is recorded as absent rather than invented. So + is a stand-in that predates the shared block: the fields it cannot supply + are recorded ``None``/empty rather than fabricated. + """ + + shared = getattr(artifact, "provenance", None) + block: Mapping[str, Any] = shared() if callable(shared) else {} + provenance: dict[str, object] = { + field: block.get(field, getattr(artifact, field, None)) + for field in ("facts_sha256", "fact_row_count", "manifest_sha256") + } + for field in _LEDGER_EPOCH_WITNESS_SCALARS: + provenance[field] = block.get(field) + for field in _LEDGER_EPOCH_WITNESS_LISTS: + provenance[field] = list(block.get(field) or ()) + manifest = getattr(artifact, "manifest", None) + if isinstance(manifest, Mapping): + provenance["manifest"] = { + key: manifest.get(key) + for key in _LEDGER_MANIFEST_IDENTITY_FIELDS + if manifest.get(key) is not None + } + return provenance diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py index 340ae23100..911869d8fc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py @@ -53,6 +53,7 @@ "microcosm.build.uk_runtime.high_value_homes", "microcosm.build.uk_runtime.hmrc_uprating", "microcosm.build.uk_runtime.ledger_fact_vendoring", + "microcosm.build.uk_runtime.ledger_surface", "microcosm.build.uk_runtime.ledger_targets", "microcosm.build.uk_runtime.local_target_census", "microcosm.build.uk_runtime.local_targets",