diff --git a/changelog.d/1492-target-compile-keys-geography-constants.fixed.md b/changelog.d/1492-target-compile-keys-geography-constants.fixed.md new file mode 100644 index 0000000000..41e2cd2c2b --- /dev/null +++ b/changelog.d/1492-target-compile-keys-geography-constants.fixed.md @@ -0,0 +1 @@ +The graph kernels that compile targets now include `microcosm.calibrate.geography_constants` in their implementation hashes, so an edit to a geography label recompiles the targets instead of reusing a stored surface that carries the old label (microcosm#1492). `ledger_targets` labels a target's geography from `UK_GEOGRAPHY_ID_TO_LABEL` when the Chronicle facts give no name, and `uk_runtime/target_weights` reads `UK_WIDE_GEOGRAPHY_IDS` for the UK loss weights, both through `from … import`, which `source_hash` does not follow. The module joins the transport compile modules (hashed by `targets.compile@1`, `takeup.compare@1`, `takeup.gap@1`, `transport.create@1`/`@2` and `transport.quantile_map@1`/`@2`), the UK target compile modules (hashed by the full graph's target, selection, measures and problem kernels and the national graph's target and problem kernels) and the UK problem kernel's loss-weight hash. The UK list also gains `microcosm.calibrate.registry`, `target` and `hierarchy`, which the transport list already hashed. The UK target and problem kernels also now hash code they ran unhashed: `calibrate.target_selection` (full graph), `calibrate.matrix` and `calibrate.artifacts`, `calibrate.solve` (the full problem's loss scales only; `calibrate.adam@1`, whose hash the UK calibration kernels embed, already hashed it), and `uk_runtime.weighted_integrity`, whose exclusion clock `full_targets` reads. In national builds an edit to `weighted_integrity` now re-runs the national targets, problem and solve; before, only the national validation node after the solve hashed it. `_ledger_provenance` and `_validate_band_edge_registry`, which `full_targets` runs, move from `uk_runtime.calibration_run` into a small `uk_runtime.ledger_surface` that the UK list names, and `calibration_run` re-exports them. The rest of `calibration_run` stays outside the target kernels' hashes: in builds that construct the spine in the graph, an edit to it re-runs no target compilation and no solve, and in builds from a spine checkpoint it re-keys the bound spine node and everything after it, as before. The transport compile list and its 18 exclusions move from `transport/target_kernels.py`, which 13 kernels hash, to `transport/compile_modules.py`, which none does, as the UK list already sat apart. Two checks now pin the hashes: a shared list's imports must be listed or excluded, reading every `microcosm.*` import rather than only `microcosm.build` ones, and the walk from every module a compile kernel hashes, its own module included, must meet only hashed modules, the list's exclusions or the imports no review has classified yet, recorded per group of kernels sharing one hash and tracked by microcosm#1497 (66 modules for the UK, 35 for transport), a record that can only shrink. Node keys move once: the target compilation of every transport country and everything after it; the UK target, selection, measures, problem and national target and problem nodes and everything after them, including the UK solve; because `transport/target_kernels.py` itself changes, `calibration.prepare@1` nodes, `gates.battery@1` nodes at a phase before compilation, and `takeup.assign@1` nodes given no compiled surface; and, because `calibration_run.py` changes, the bound spine checkpoint node (`uk.full.bound_spine@1`) and the population nodes after it. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py new file mode 100644 index 0000000000..e38932c372 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py @@ -0,0 +1,106 @@ +"""The modules whose source decides a transport target surface. + +``targets.compile@1`` and every other transport kernel built on target +compilation hash :func:`transport_compile_modules` (through +``target_kernels._COMPILE_MODULES``). ``source_hash`` hashes the files it is +given and does not follow imports, so the tuple has to name every module the +compile reads; ``ledger_targets`` labels a target's geography from +``microcosm.calibrate.geography_constants``, which no compile kernel hashed +before microcosm#1492. + +The list lives here, in a module no kernel hashes, as the UK's lives in +``uk_runtime.target_compile_modules``: editing a name or a reason re-keys +only the kernels whose hashed modules the edit adds or removes, not every +kernel that hashes ``target_kernels``. + +``EXCLUDED_DIRECT_IMPORTS`` names every first-party (``microcosm.*``) module +a listed module imports but the tuple leaves out, with the reason. A test +pins that the two together cover every direct import of every listed module, +so a new import must be listed or excluded. The exclusions cover only the +listed modules' own imports; a second test walks every module each compile +kernel hashes, its own module included, and records the imports no review +has classified yet (microcosm#1497). +""" + +from __future__ import annotations + +import importlib +from collections.abc import Mapping +from types import MappingProxyType, ModuleType + +TRANSPORT_COMPILE_MODULE_NAMES: tuple[str, ...] = ( + "microcosm.build.transport.graph_inputs", + "microcosm.build.transport.artifact_types", + "microcosm.build.country_spec", + "microcosm.build.ledger_targets", + "microcosm.build.ledger_artifact", + "microcosm.build.chronicle_epoch", + "microcosm.build.reference_factors", + "microcosm.build.reference_profiles", + "microcosm.calibrate.registry", + "microcosm.calibrate.target", + "microcosm.calibrate.hierarchy", + "microcosm.calibrate.geography_constants", + "microcosm.build.transport.reference_selection", +) + +_COUNTRY_SPEC_LOADER = ( + "country_spec is listed for the reference validators target compilation " + "calls, _validate_target_references and _carried_value_keys; it imports " + "this module for its country-spec loader and stage plans, which those " + "validators do not use" +) +_GRAPH_RUNTIME = ( + "the graph runtime (kernel protocol, declarations and the canonical JSON " + "encoder node keys are computed with); no kernel in the repository " + "hashes a microcosm.graph module" +) + +EXCLUDED_DIRECT_IMPORTS: Mapping[str, str] = MappingProxyType( + { + "microcosm.build.plan": _COUNTRY_SPEC_LOADER, + "microcosm.build.source_manifest": _COUNTRY_SPEC_LOADER, + "microcosm.build.trace": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.canonical": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.engine_abi": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.loader": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.model": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.schemas": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.take_up_semantics": _COUNTRY_SPEC_LOADER, + "microcosm.build.spec_engine.yaml12": _COUNTRY_SPEC_LOADER, + "microcosm.build.reference_intervals": ( + "reference_profiles reaches it only when given an interval " + "document; compile_target_surface calls validate_reference_resolution " + "without one. calibration.prepare@1 and targets.problem@1, which " + "apply interval tables, hash it" + ), + "microcosm.calibrate.artifacts": ( + "artifact_types imports PROBLEM_TYPE to declare targets.problem@1's " + "output; target compilation checks only its own surface output" + ), + "microcosm.diagnostics.schema": ( + "artifact_types imports the diagnostics schema version to declare " + "the diagnostics output type; target compilation emits no diagnostics" + ), + "microcosm.frame.bundle": ( + "the population container: graph_inputs.context_frame builds one, " + "calibrate.target measures a target on one and country_spec's stage " + "plans transform one; target compilation reads facts, not a population" + ), + "microcosm.frame.schema": ( + "graph_inputs.context_frame's entity schema; target compilation " + "reads no population" + ), + "microcosm.graph.canonical": _GRAPH_RUNTIME, + "microcosm.graph.decl": _GRAPH_RUNTIME, + "microcosm.graph.kernel": _GRAPH_RUNTIME, + } +) + + +def transport_compile_modules() -> tuple[ModuleType, ...]: + """The modules of :data:`TRANSPORT_COMPILE_MODULE_NAMES`, imported.""" + + return tuple( + importlib.import_module(name) for name in TRANSPORT_COMPILE_MODULE_NAMES + ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py index 7b3d926540..cdea94844e 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py @@ -50,10 +50,7 @@ import microcosm.build.calibration_columns as calibration_columns_module import microcosm.build.calibration_predicates as calibration_predicates_module -import microcosm.build.chronicle_epoch as chronicle_epoch_module -import microcosm.build.country_spec as country_spec_module import microcosm.build.ledger_artifact as ledger_artifact_module -import microcosm.build.ledger_targets as ledger_targets_module import microcosm.build.reference_factors as reference_factors_module import microcosm.build.reference_intervals as reference_intervals_module import microcosm.build.reference_profiles as reference_profiles_module @@ -105,6 +102,7 @@ COMPARISON_TYPE, TARGET_SURFACE_TYPE, ) +from .compile_modules import transport_compile_modules from .graph_inputs import ( canonical_document_param, context_frame, @@ -565,20 +563,9 @@ def _optional_float(value: float) -> float | None: return value if math.isfinite(value) else None -_COMPILE_MODULES = ( - graph_inputs, - artifact_types, - country_spec_module, - ledger_targets_module, - ledger_artifact_module, - chronicle_epoch_module, - reference_factors_module, - reference_profiles_module, - registry_module, - target_module, - hierarchy_module, - reference_selection_module, -) +#: The modules target compilation reads, named in ``compile_modules``, which +#: no kernel hashes, so an edit to the list re-keys only what it changes. +_COMPILE_MODULES = transport_compile_modules() _MATRIX_MODULES = ( matrix_module, monetary_binding_module, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 0a88fab22d..580b2a6ff4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -46,6 +46,14 @@ from microcosm.build.uk_runtime.etb_services import ( UK_NHS_SPENDING_COMPONENT_COLUMNS, ) + +# Moved to ledger_surface, which the UK target kernels hash, so an edit +# elsewhere in this module no longer re-keys them (microcosm#1492). Kept +# importable from here for this module's other callers. +from microcosm.build.uk_runtime.ledger_surface import ( # noqa: F401 + _ledger_provenance, + _validate_band_edge_registry, +) from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_time_period, @@ -307,45 +315,6 @@ def new_uk_calibration_attempt_id(*, timestamp: datetime) -> str: _new_calibration_attempt_id = new_uk_calibration_attempt_id -def _validate_band_edge_registry( - *, - register_registry: TargetRegistry, - band_edge_registry: TargetRegistry, - exclusion_receipt: Mapping[str, Mapping[str, str]], -) -> None: - """Require the band-edge register to reconstitute the compiled roster. - - The reconciliation always runs — an empty receipt is a claim that nothing - was pruned, so the two rosters must then be name-identical; it is never - permission to skip the check. Receipt keys are spec names by the - applier's construction: ``apply_uk_calibration_measure_exclusions`` - builds the receipt from the matched ``spec.name`` set and raises on any - exclusion matching zero registry specs, so ``pruned + receipt keys == - compiled`` is an exact identity, not a heuristic. - """ - - register_names = _registry_spec_names(register_registry) - edge_names = _registry_spec_names(band_edge_registry) - excluded_names = {str(name) for name in exclusion_receipt} - if not register_names <= edge_names: - missing = sorted(register_names - edge_names) - raise ValueError( - "band-edge register must include every materialized registry spec; " - f"missing={missing}." - ) - extra = edge_names - register_names - if extra != excluded_names: - raise ValueError( - "band-edge register extra spec names must match the measure " - f"exclusion receipt; extra={sorted(extra)}, " - f"receipt={sorted(excluded_names)}." - ) - - -def _registry_spec_names(registry: TargetRegistry) -> set[str]: - return {str(spec.name) for spec in registry.specs} - - def load_bound_spine_sidecar(path: Path, frame: Frame) -> dict[str, object]: if not path.is_file(): raise ValueError(f"input H5 build sidecar absent: {path}") @@ -834,71 +803,6 @@ def uk_aggregate_admin_totals( return totals, receipt -#: The manifest fields the UK run seals into ``run_config``. Narrower than -#: the artifact's whole manifest on purpose: the identity digest should name -#: *which* published artifact was compiled, not re-hash its manifest. -_LEDGER_MANIFEST_IDENTITY_FIELDS = ( - "artifact_id", - "profile", - "schema_version", - "generated_at", -) - -#: Epoch witnesses :meth:`LedgerConsumerArtifact.provenance` supplies, split -#: by shape. Named here so the delegation below cannot quietly drop one: a -#: run that compiled a chronicle-era or mixed-epoch feed has to say so in its -#: own evidence, not only in the loader's return value -#: (PolicyEngine/chronicle#143). -_LEDGER_EPOCH_WITNESS_SCALARS = ("schema_epoch",) -_LEDGER_EPOCH_WITNESS_LISTS = ( - "fact_key_epochs", - "undeclared_fact_key_domains", - "fact_schema_versions", -) - - -def _ledger_provenance(artifact: Any) -> dict[str, object]: - """The verified identity of the Chronicle consumer feed this run compiled. - - Delegates to :meth:`microcosm.build.ledger_artifact.LedgerConsumerArtifact.provenance` - rather than rebuilding the block field by field. Rebuilding it is how the - epoch witnesses went missing here in the first place: the loader learned - which era resolved the targets and the UK run kept reporting only the - hashes. Anything the shared block gains, this block gains. - - The whole block is sealed into ``run_config`` and so into the run's - identity digest: the epoch witnesses enter it beside the hashes, which is - why this delegation is a change to the digest of an otherwise identical - configuration, once. Only the manifest sub-block is UK-shaped, and it - stays narrow so the digest names the artifact rather than re-hashing its - whole manifest. - - A bare ``consumer_facts.jsonl`` feed carries no manifest, so its - Chronicle-side provenance is recorded as absent rather than invented. So - is a stand-in that predates the shared block: the fields it cannot supply - are recorded ``None``/empty rather than fabricated. - """ - - shared = getattr(artifact, "provenance", None) - block: Mapping[str, Any] = shared() if callable(shared) else {} - provenance: dict[str, object] = { - field: block.get(field, getattr(artifact, field, None)) - for field in ("facts_sha256", "fact_row_count", "manifest_sha256") - } - for field in _LEDGER_EPOCH_WITNESS_SCALARS: - provenance[field] = block.get(field) - for field in _LEDGER_EPOCH_WITNESS_LISTS: - provenance[field] = list(block.get(field) or ()) - manifest = getattr(artifact, "manifest", None) - if isinstance(manifest, Mapping): - provenance["manifest"] = { - key: manifest.get(key) - for key in _LEDGER_MANIFEST_IDENTITY_FIELDS - if manifest.get(key) is not None - } - return provenance - - #: Packages whose versions the seam pins into the signed diagnostics build #: block. The release assembler treats these as the only legitimate source #: for manifest runtime/compatibility pins. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py index 60f3fb3465..2a9e1fd30c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py @@ -14,10 +14,6 @@ from typing import Any from microcosm.build.ledger_artifact import load_ledger_consumer_artifact -from microcosm.build.uk_runtime.calibration_run import ( - _ledger_provenance, - _validate_band_edge_registry, -) from microcosm.build.uk_runtime.chronicle_feed import ( load_uk_chronicle_feed, require_committed_uk_chronicle_feed_pin, @@ -27,6 +23,10 @@ uk_derived_fact_rows, ) from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.ledger_surface import ( + _ledger_provenance, + _validate_band_edge_registry, +) from microcosm.build.uk_runtime.ledger_targets import ( _uk_contract_targets, assert_uk_local_deferrals_in_force, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index 94e8f55483..54bc522b9f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -472,6 +472,10 @@ class _NationalKernel(KernelBase): ) def implementation_hash(self) -> str: + import microcosm.calibrate.artifacts as calibrate_artifacts + import microcosm.calibrate.matrix as calibrate_matrix + + from . import weighted_integrity from .target_compile_modules import uk_target_compile_modules implementation = source_hash( @@ -489,6 +493,17 @@ def implementation_hash(self) -> str: national_calibration, national_doctrine, support_floor_rules, + # The constraint matrices and their encoding, as the dense kernels + # hash them (microcosm#1492). Not calibrate.solve: neither the + # national target nor the problem path calls it. Its one caller + # here, national_calibration_manifest, runs in + # uk.full.national-gates@1, which reads the solve's result, whose + # key moves with calibrate.solve through calibrate.adam@1. + calibrate_matrix, + calibrate_artifacts, + # full_targets takes its exclusion clock from here; hashed + # directly, as the dense target kernels do. + weighted_integrity, *uk_target_compile_modules(), ) return hashlib.sha256( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index b43a1a4569..2dddec06d7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -15,6 +15,11 @@ import numpy as np import pandas as pd +import microcosm.calibrate.artifacts as calibrate_artifacts +import microcosm.calibrate.geography_constants as geography_constants +import microcosm.calibrate.matrix as calibrate_matrix +import microcosm.calibrate.solve as calibrate_solve +import microcosm.calibrate.target_selection as calibrate_target_selection from microcosm.build import cross_grain from microcosm.calibrate import CalibrationHierarchy, TargetRegistry, TargetSpec from microcosm.calibrate.artifacts import ( @@ -217,7 +222,12 @@ class _TargetKernel(KernelBase): ) def implementation_hash(self) -> str: - from . import full_targets, high_value_homes, measure_simulation + from . import ( + full_targets, + high_value_homes, + measure_simulation, + weighted_integrity, + ) from .target_compile_modules import uk_target_compile_modules implementation = source_hash( @@ -235,6 +245,15 @@ def implementation_hash(self) -> str: # alone must re-key the measures and everything they feed. measure_simulation, support_floor, + # The selection, the constraint matrices and their encoding, as + # the transport kernels hash them (microcosm#1492). + calibrate_target_selection, + calibrate_matrix, + calibrate_artifacts, + # full_targets takes its exclusion clock from here. Hashed + # directly, not listed: the module also loads the registers only + # the validation checks read, which no listed module may. + weighted_integrity, *uk_target_compile_modules(), ) # The solve fingerprint: every packaged resource but the gate-only @@ -649,12 +668,22 @@ def implementation_hash(self) -> str: # ``local_rowwise.prepare_uk_full_solve`` from the rules in # ``target_weights`` and ``national_doctrine`` (microcosm#1124), so an # edit there re-keys the problem instead of replaying a stale vector. + # ``target_weights`` reads the UK-wide geography codes from + # ``geography_constants``, and the loss scales the problem stores from + # ``calibrate.solve.default_target_loss_scales`` (microcosm#1492). All + # of ``calibrate.solve`` is hashed: ``calibrate.adam@1``, whose hash + # every UK solve kernel embeds, already hashes it, so an edit there + # re-solves anyway and re-keys only this node besides. return hashlib.sha256( canonical_json( { "targets": super().implementation_hash(), "weights": source_hash( - local_rowwise, national_doctrine, target_weights + local_rowwise, + national_doctrine, + target_weights, + geography_constants, + calibrate_solve, ), } ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 27d8f5c1da..6fad53387c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -726,7 +726,10 @@ def uk_terminal_only_code_hash() -> str: ``catalog_calibration_diagnostics`` with the UK label choices (which read ``UK_GEOGRAPHY_ID_TO_LABEL``), dumped from ``CalibrationDiagnosticsV9``. Those modules decide the published labels and bytes, so an edit to them - re-keys the calibrated node the same way, never the solve. + re-keys the calibrated node the same way. ``geography_constants``, like + ``high_value_homes``, also feeds the target kernels, which hash it + (microcosm#1492), so an edit to it still re-solves; an edit to the other + three never does. Hashed once per process, as the code was imported: an edit to the files while a build runs cannot fail the calibrated gates after the solve. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_surface.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_surface.py new file mode 100644 index 0000000000..76d53024c8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_surface.py @@ -0,0 +1,120 @@ +"""What UK target compilation records about its Chronicle feed, and a roster check. + +``full_targets`` writes :func:`_ledger_provenance` into every compiled surface +and runs :func:`_validate_band_edge_registry` on the compiled registers. Both +lived in ``calibration_run``, the national calibration runner. The UK target +kernels hash this small module through ``UK_TARGET_COMPILE_MODULE_NAMES`` +instead of all of ``calibration_run`` (microcosm#1492), so an edit to either +function re-keys the compiled surface and an edit to the runner does not. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from microcosm.calibrate import TargetRegistry + + +def _validate_band_edge_registry( + *, + register_registry: TargetRegistry, + band_edge_registry: TargetRegistry, + exclusion_receipt: Mapping[str, Mapping[str, str]], +) -> None: + """Require the band-edge register to reconstitute the compiled roster. + + The reconciliation always runs — an empty receipt is a claim that nothing + was pruned, so the two rosters must then be name-identical; it is never + permission to skip the check. Receipt keys are spec names by the + applier's construction: ``apply_uk_calibration_measure_exclusions`` + builds the receipt from the matched ``spec.name`` set and raises on any + exclusion matching zero registry specs, so ``pruned + receipt keys == + compiled`` is an exact identity, not a heuristic. + """ + + register_names = _registry_spec_names(register_registry) + edge_names = _registry_spec_names(band_edge_registry) + excluded_names = {str(name) for name in exclusion_receipt} + if not register_names <= edge_names: + missing = sorted(register_names - edge_names) + raise ValueError( + "band-edge register must include every materialized registry spec; " + f"missing={missing}." + ) + extra = edge_names - register_names + if extra != excluded_names: + raise ValueError( + "band-edge register extra spec names must match the measure " + f"exclusion receipt; extra={sorted(extra)}, " + f"receipt={sorted(excluded_names)}." + ) + + +def _registry_spec_names(registry: TargetRegistry) -> set[str]: + return {str(spec.name) for spec in registry.specs} + + +#: The manifest fields the UK run seals into ``run_config``. Narrower than +#: the artifact's whole manifest on purpose: the identity digest should name +#: *which* published artifact was compiled, not re-hash its manifest. +_LEDGER_MANIFEST_IDENTITY_FIELDS = ( + "artifact_id", + "profile", + "schema_version", + "generated_at", +) + +#: Epoch witnesses :meth:`LedgerConsumerArtifact.provenance` supplies, split +#: by shape. Named here so the delegation below cannot quietly drop one: a +#: run that compiled a chronicle-era or mixed-epoch feed has to say so in its +#: own evidence, not only in the loader's return value +#: (PolicyEngine/chronicle#143). +_LEDGER_EPOCH_WITNESS_SCALARS = ("schema_epoch",) +_LEDGER_EPOCH_WITNESS_LISTS = ( + "fact_key_epochs", + "undeclared_fact_key_domains", + "fact_schema_versions", +) + + +def _ledger_provenance(artifact: Any) -> dict[str, object]: + """The verified identity of the Chronicle consumer feed this run compiled. + + Delegates to :meth:`microcosm.build.ledger_artifact.LedgerConsumerArtifact.provenance` + rather than rebuilding the block field by field. Rebuilding it is how the + epoch witnesses went missing here in the first place: the loader learned + which era resolved the targets and the UK run kept reporting only the + hashes. Anything the shared block gains, this block gains. + + The whole block is sealed into ``run_config`` and so into the run's + identity digest: the epoch witnesses enter it beside the hashes, which is + why this delegation is a change to the digest of an otherwise identical + configuration, once. Only the manifest sub-block is UK-shaped, and it + stays narrow so the digest names the artifact rather than re-hashing its + whole manifest. + + A bare ``consumer_facts.jsonl`` feed carries no manifest, so its + Chronicle-side provenance is recorded as absent rather than invented. So + is a stand-in that predates the shared block: the fields it cannot supply + are recorded ``None``/empty rather than fabricated. + """ + + shared = getattr(artifact, "provenance", None) + block: Mapping[str, Any] = shared() if callable(shared) else {} + provenance: dict[str, object] = { + field: block.get(field, getattr(artifact, field, None)) + for field in ("facts_sha256", "fact_row_count", "manifest_sha256") + } + for field in _LEDGER_EPOCH_WITNESS_SCALARS: + provenance[field] = block.get(field) + for field in _LEDGER_EPOCH_WITNESS_LISTS: + provenance[field] = list(block.get(field) or ()) + manifest = getattr(artifact, "manifest", None) + if isinstance(manifest, Mapping): + provenance["manifest"] = { + key: manifest.get(key) + for key in _LEDGER_MANIFEST_IDENTITY_FIELDS + if manifest.get(key) is not None + } + return provenance diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py index 642a1e912a..911869d8fc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py @@ -9,10 +9,22 @@ appliers, so an edit there could reuse a cached surface computed by the old code. -``EXCLUDED_DIRECT_IMPORTS`` names every ``microcosm.build`` module a listed -module imports but the tuple leaves out, with the reason. A test pins that -the two together cover every direct import of every listed module, so the -coverage holds transitively: a new import must be listed or excluded. +``EXCLUDED_DIRECT_IMPORTS`` names every first-party (``microcosm.*``) module +a listed module imports but the tuple leaves out, with the reason. A test +pins that the two together cover every direct import of every listed module, +so the coverage holds transitively: a new import must be listed or excluded. +Until microcosm#1492 that test read only ``microcosm.build`` imports, so it +never saw ``microcosm.calibrate.geography_constants``, the geography labels +of the target hierarchies, nor the ``microcosm.calibrate`` registry and +hierarchy classes the compiled surface is built from (with +``microcosm.calibrate.target``, which the registry imports). The transport +compile modules already hashed those three. + +The exclusions cover only the listed modules' own imports. A second test +walks every module each target and problem kernel hashes, its own module and +the modules hashed beside this tuple included, and records the imports no +review has classified yet (microcosm#1497). This module is not hashed, so an +edit to a reason re-keys nothing; an edit to the tuple re-keys the kernels. """ from __future__ import annotations @@ -41,6 +53,7 @@ "microcosm.build.uk_runtime.high_value_homes", "microcosm.build.uk_runtime.hmrc_uprating", "microcosm.build.uk_runtime.ledger_fact_vendoring", + "microcosm.build.uk_runtime.ledger_surface", "microcosm.build.uk_runtime.ledger_targets", "microcosm.build.uk_runtime.local_target_census", "microcosm.build.uk_runtime.local_targets", @@ -52,6 +65,10 @@ "microcosm.build.uk_runtime.uc_source_periods", "microcosm.build.uk_runtime.tenure_drift", "microcosm.build.uk_runtime.uprating_holds", + "microcosm.calibrate.geography_constants", + "microcosm.calibrate.hierarchy", + "microcosm.calibrate.registry", + "microcosm.calibrate.target", ) EXCLUDED_DIRECT_IMPORTS: Mapping[str, str] = MappingProxyType( @@ -111,6 +128,23 @@ "energy_pricing imports the identity-keyed uniforms for the " "spine's gas connection walk, which no target value reads" ), + "microcosm.frame.bundle": ( + "the population container: cgt_calibration builds the spine-side " + "CGT calibration frame with it, ledger_targets' UKFrameTargetAdapter " + "reads and rebuilds the national calibration's population with it, " + "and calibrate.target measures a target on one; no target value " + "comes from it" + ), + "microcosm.frame.schema": ( + "cgt_calibration imports EntitySchema for the spine-side CGT " + "calibration frame; the compile uses only uk_cgt_annual_exempt_amount " + "and uk_cgt_dated_measure_year" + ), + "microcosm.frame.weights": ( + "cgt_calibration imports Weights for the spine-side CGT calibration " + "frame; the compile uses only uk_cgt_annual_exempt_amount and " + "uk_cgt_dated_measure_year" + ), } ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py b/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py new file mode 100644 index 0000000000..eb5b6a9953 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py @@ -0,0 +1,156 @@ +"""Controls for the import scan behind the compile kernels' hash checks. + +``test_support.microcosm_build.first_party_imports`` reads a module's source +for the first-party modules it imports, and walks a kernel's hashed modules +through them. These tests pin each import form it resolves, that a +submodule resolves the same way before and after it is imported, and how the +walk treats a shared list's exclusions. +""" + +from __future__ import annotations + +import importlib +import sys +import uuid +from types import SimpleNamespace + +import pytest + +import microcosm.calibrate +from test_support.microcosm_build.first_party_imports import ( + defining_module, + first_party_imports, + recorded_import_mismatches, + unhashed_imports, +) + + +@pytest.fixture +def scratch_package(tmp_path, monkeypatch): + """A throwaway ``microcosm.`` package written under ``tmp_path``. + + ``microcosm`` is a namespace package, so a directory on ``sys.path`` + adds to it. Returns a writer: ``write(module, source)`` creates + ``microcosm..`` and returns its dotted name. + """ + + name = f"scan_control_{uuid.uuid4().hex[:8]}" + package = tmp_path / "microcosm" / name + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + monkeypatch.syspath_prepend(str(tmp_path)) + + def write(module: str, source: str) -> str: + (package / f"{module}.py").write_text(source) + return f"microcosm.{name}.{module}" + + yield write + for loaded in [key for key in sys.modules if key.startswith(f"microcosm.{name}")]: + del sys.modules[loaded] + + +def test_the_scan_sees_each_import_form_the_compile_modules_use(): + # A constant imported from its own module. + assert "microcosm.calibrate.geography_constants" in first_party_imports( + "microcosm.build.ledger_targets" + ) + # A class imported from a package resolves to the module defining it. + assert "microcosm.calibrate.registry" in first_party_imports( + "microcosm.build.ledger_targets" + ) + # A constant the package __init__ re-exports resolves through the __init__. + assert "microcosm.diagnostics.schema" in first_party_imports( + "microcosm.build.transport.artifact_types" + ) + # An import inside a function body. + assert "microcosm.build.reference_intervals" in first_party_imports( + "microcosm.build.reference_profiles" + ) + + +def test_a_package_import_resolves_through_the_names_read_from_it(scratch_package): + module = scratch_package( + "package_imports", + "import microcosm.calibrate\n" + "import microcosm.diagnostics as diagnostics_package\n" + "def read():\n" + " registry = microcosm.calibrate.TargetRegistry\n" + " return registry, diagnostics_package.CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION\n", + ) + assert first_party_imports(module) == { + "microcosm.calibrate.registry", + "microcosm.diagnostics.schema", + } + + +def test_a_package_import_with_no_name_read_reads_the_package(scratch_package): + module = scratch_package("bare_import", "import microcosm.calibrate\n") + assert first_party_imports(module) == {"microcosm.calibrate"} + + +def test_a_submodule_resolves_the_same_before_it_is_imported(monkeypatch): + """A function-level ``from package import submodule`` names the submodule + whether or not an earlier test imported it.""" + name = "microcosm.calibrate.target_selection" + assert defining_module("microcosm.calibrate", "target_selection") == name + monkeypatch.delitem(sys.modules, name, raising=False) + monkeypatch.delattr(microcosm.calibrate, "target_selection", raising=False) + assert defining_module("microcosm.calibrate", "target_selection") == name + + +def test_the_walk_follows_an_exclusion_but_not_its_imports(scratch_package): + """A list's exclusion covers only the listed module's import of it; the + excluded module's own imports must be hashed.""" + leaf = scratch_package("leaf", "VALUE = 1\n") + middle = scratch_package("middle", f"from {leaf} import VALUE\n") + top = scratch_package("top", f"from {middle} import VALUE\n") + excluded = frozenset({middle}) + assert unhashed_imports({top}, listed=(top,), excluded=excluded) == { + f"{middle} -> {leaf}" + } + # From a module outside the list, the exclusion does not apply. + assert unhashed_imports({top}, listed=(), excluded=excluded) == { + f"{top} -> {middle}" + } + assert unhashed_imports({top, middle, leaf}, listed=(), excluded=excluded) == set() + + +class _RecordingKernel: + """A stand-in kernel whose hash names ``modules`` through ``owner``.""" + + def __init__(self, ref, owner, modules): + self.ref, self.owner, self.modules = ref, owner, modules + + def implementation_hash(self): + return self.owner.source_hash(*self.modules) + + +def test_a_group_record_does_not_cover_another_group(scratch_package): + """Removing a module from one group's hash fails the check even when + another group records the same edge (round 2 of #1493).""" + middle = scratch_package("group_middle", "VALUE = 1\n") + top = scratch_package("group_top", f"from {middle} import VALUE\n") + owner = SimpleNamespace(source_hash=lambda *objects, **dependencies: "x") + top_module, middle_module = ( + importlib.import_module(name) for name in (top, middle) + ) + hashes_both = _RecordingKernel("hashes.both@1", owner, (top_module, middle_module)) + hashes_top = _RecordingKernel("hashes.top@1", owner, (top_module,)) + recorded = {f"{top} -> {middle}": ("top only",)} + + def mismatches(groups): + return recorded_import_mismatches( + groups, listed=(), excluded=frozenset(), hash_inputs={}, recorded=recorded + ) + + assert ( + mismatches( + {"both": ((hashes_both, owner),), "top only": ((hashes_top, owner),)} + ) + == {} + ) + # The first group drops the middle module: the edge is recorded for the + # second group alone, so the first group's walk reports it. + assert mismatches( + {"both": ((hashes_top, owner),), "top only": ((hashes_top, owner),)} + ) == {"both: unhashed and not recorded": [f"{top} -> {middle}"]} diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py new file mode 100644 index 0000000000..2aa1ba903a --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py @@ -0,0 +1,633 @@ +"""The transport compile kernels hash every module target compilation reads. + +``targets.compile@1`` and every other transport kernel built on target +compilation hash ``target_kernels._COMPILE_MODULES``, the modules named in +``transport.compile_modules``. ``source_hash`` hashes the files it is given +and does not follow imports, so a module that a hashed module imports, and +that the kernel does not hash, can change a kernel's output without changing +a node key. ``ledger_targets`` takes a target's geography label from +``microcosm.calibrate.geography_constants`` that way; before microcosm#1492 +no compile kernel hashed it. + +Two checks pin the closure: + +- the shared list's own: every first-party import of a listed module is + listed, or excluded with a reason in ``compile_modules``; +- each kernel's: the walk from every module the kernel hashes, its own + module included, meets only modules the kernel hashes, the list's + exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS`` for that + kernel's group, which no review has classified yet (microcosm#1497). That + mapping may only shrink. +""" + +from __future__ import annotations + +import importlib +from types import MappingProxyType, ModuleType + +import pytest + +import microcosm.calibrate.geography_constants as geography_constants_module +from microcosm.build.transport import ( + population_kernels, + quantile_kernels, + takeup_kernels, + target_kernels, +) +from test_support.microcosm_build.first_party_imports import ( + HashInput, + assert_an_edit_rekeys, + first_party_imports, + hashed_module_names, + recorded_import_mismatches, + unlisted_imports, +) + +#: Every kernel whose implementation hash includes ``_COMPILE_MODULES``, with +#: the module whose ``source_hash`` it calls. The first five compile +#: references with ``compile_target_surface``; the two quantile maps decode +#: the surface an upstream compile stored. +KERNELS_HASHING_THE_COMPILE_MODULES = ( + (target_kernels.TARGETS_COMPILE, target_kernels), + (target_kernels.TAKEUP_COMPARE, target_kernels), + (takeup_kernels.TAKEUP_GAP, takeup_kernels), + (population_kernels.TRANSPORT_CREATE, population_kernels), + (population_kernels.TRANSPORT_CREATE_DECLARED, population_kernels), + (population_kernels.TRANSPORT_QUANTILE_MAP, population_kernels), + (quantile_kernels.TRANSPORT_QUANTILE_MAP_V2, quantile_kernels), +) +_KERNEL_IDS = [kernel.ref for kernel, _ in KERNELS_HASHING_THE_COMPILE_MODULES] + +#: The kernels grouped by shared hash: the two CREATE versions share one +#: class body, so one hashed set; every other kernel is its own group. +COMPILE = "targets.compile@1" +COMPARE = "takeup.compare@1" +GAP = "takeup.gap@1" +CREATE = "transport.create@1 and @2" +QUANTILE_MAP_1 = "transport.quantile_map@1" +QUANTILE_MAP_2 = "transport.quantile_map@2" +KERNEL_GROUPS = MappingProxyType( + { + COMPILE: ((target_kernels.TARGETS_COMPILE, target_kernels),), + COMPARE: ((target_kernels.TAKEUP_COMPARE, target_kernels),), + GAP: ((takeup_kernels.TAKEUP_GAP, takeup_kernels),), + CREATE: ( + (population_kernels.TRANSPORT_CREATE, population_kernels), + (population_kernels.TRANSPORT_CREATE_DECLARED, population_kernels), + ), + QUANTILE_MAP_1: ( + (population_kernels.TRANSPORT_QUANTILE_MAP, population_kernels), + ), + QUANTILE_MAP_2: ( + (quantile_kernels.TRANSPORT_QUANTILE_MAP_V2, quantile_kernels), + ), + } +) + +#: Imports a kernel's module makes only to build the hash, and the importers +#: the reason covers. They stay unhashed on purpose, so an edit to the list +#: re-keys only the kernels whose hashed modules it adds or removes. +HASH_INPUTS = MappingProxyType( + { + "microcosm.build.transport.compile_modules": HashInput( + importers=("microcosm.build.transport.target_kernels",), + reason=( + "names the modules the compile kernels hash; target_kernels " + "reads it only to build _COMPILE_MODULES" + ), + ), + } +) + +#: Every ``importer -> imported`` edge the walk from a kernel group's hashed +#: modules meets that the group does not hash and no review has classified, +#: with the groups that leave it unhashed. Recorded as found when the +#: per-kernel check was added; microcosm#1497 classifies them. Hash an entry +#: in the kernels that run it, move what they run into a module they hash, or +#: record why they need not hash it, then delete it here. The check fails on +#: an edge a group meets that is not recorded for it and on an edge recorded +#: for a group that no longer meets it, so the mapping can only shrink, and +#: removing a module from one group's hash fails even where another group +#: records the same edge. +UNCLASSIFIED_IMPORTS = MappingProxyType( + { + "microcosm.build.trace -> microcosm.build": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.calibration_columns": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.calibration_predicates": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.plan -> microcosm.build.observation": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.reference_intervals": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.take_up_semantics -> microcosm.build.spec_engine.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.engine_abi": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.schemas -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.take_up_semantics -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.yaml12 -> microcosm.build.spec_engine.errors": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.canonical -> microcosm.build.spec_engine.model": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.model": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.model": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.resolver": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.schemas": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.schemas": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.model -> microcosm.build.spec_engine.seeds": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.take_up_semantics": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.yaml12": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.transport.column_kernels": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.build.transport.unit_declarations": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.build.transport.unit_declarations": ( + COMPILE, + COMPARE, + GAP, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.build.us_runtime.multispine_pool": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.calibrate.artifacts": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.calibrate.matrix": ( + COMPILE, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.calibrate.matrix": ( + COMPILE, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.calibrate.monetary_binding": ( + COMPILE, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.calibrate.solve": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.spec_engine.engine_abi -> microcosm.frame.adapters.policyengine_us": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.adapters.policyengine_us_concepts": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.transport -> microcosm.frame.adapters.policyengine_us_concepts": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.plan -> microcosm.frame.bundle": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.bundle": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.frame.bundle": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.frame.bundle": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.graph.kernel -> microcosm.frame.bundle": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.concept_mapping": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.reference_intervals -> microcosm.frame.concepts": ( + COMPILE, + COMPARE, + GAP, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.concepts": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.transport -> microcosm.frame.concepts": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.materialize": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.transport -> microcosm.frame.materialize": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.scaling": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.transport -> microcosm.frame.scaling": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.schema": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.frame.schema": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.bundle -> microcosm.frame.schema": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.unit_construction": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.frame.unit_construction": ( + COMPILE, + COMPARE, + GAP, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.frame.weights": ( + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.target_kernels -> microcosm.frame.weights": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.frame.weights": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.frame.bundle -> microcosm.frame.weights": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.graph.kernel -> microcosm.frame.weights": ( + COMPILE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.reference_intervals -> microcosm.graph.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.quantile_kernels -> microcosm.graph.canonical": ( + QUANTILE_MAP_2, + ), + "microcosm.build.transport.takeup_kernels -> microcosm.graph.canonical": (GAP,), + "microcosm.build.transport.target_kernels -> microcosm.graph.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.graph.canonical": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.graph.decl": ( + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.takeup_kernels -> microcosm.graph.decl": (GAP,), + "microcosm.build.transport.target_kernels -> microcosm.graph.decl": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.calibrate.artifacts -> microcosm.graph.decl": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.graph.canonical -> microcosm.graph.decl": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.graph.kernel -> microcosm.graph.decl": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.graph.kernel": ( + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.quantile_kernels -> microcosm.graph.kernel": ( + QUANTILE_MAP_2, + ), + "microcosm.build.transport.takeup_kernels -> microcosm.graph.kernel": (GAP,), + "microcosm.build.transport.target_kernels -> microcosm.graph.kernel": ( + COMPILE, + COMPARE, + GAP, + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + "microcosm.build.transport.population_kernels -> microcosm.graph.population": ( + CREATE, + QUANTILE_MAP_1, + QUANTILE_MAP_2, + ), + } +) + + +def _compile_list() -> ModuleType: + """The shared list and its exclusions (``compile_modules``, microcosm#1492).""" + return importlib.import_module("microcosm.build.transport.compile_modules") + + +@pytest.mark.parametrize( + ("kernel", "module"), KERNELS_HASHING_THE_COMPILE_MODULES, ids=_KERNEL_IDS +) +def test_an_edit_to_the_geography_labels_rekeys_the_kernel( + tmp_path, monkeypatch, kernel, module +): + """ledger_targets labels a target's geography from UK_GEOGRAPHY_ID_TO_LABEL + when the facts give no name, so an edit there must give a new key.""" + assert_an_edit_rekeys(kernel, geography_constants_module, tmp_path, monkeypatch) + + +def test_the_compile_list_lives_in_a_module_no_compile_kernel_hashes(): + """target_kernels is hashed by 13 kernels; the list is not, so an edit to + a name or a reason no longer re-keys all of them.""" + compile_list = _compile_list() + assert target_kernels._COMPILE_MODULES == compile_list.transport_compile_modules() + for kernel, module in KERNELS_HASHING_THE_COMPILE_MODULES: + assert compile_list.__name__ not in hashed_module_names(kernel, module) + + +def test_every_direct_import_of_a_compile_module_is_listed_or_excluded(): + compile_list = _compile_list() + uncovered = unlisted_imports( + compile_list.TRANSPORT_COMPILE_MODULE_NAMES, + frozenset(compile_list.EXCLUDED_DIRECT_IMPORTS), + ) + assert not uncovered, sorted(uncovered) + + +def test_compile_exclusions_are_live_and_never_listed(): + compile_list = _compile_list() + listed = compile_list.TRANSPORT_COMPILE_MODULE_NAMES + excluded = set(compile_list.EXCLUDED_DIRECT_IMPORTS) + imported = set().union(*map(first_party_imports, listed)) + assert excluded <= imported, sorted(excluded - imported) + assert not excluded & set(listed) + + +def test_every_import_a_compile_kernel_reaches_is_hashed_or_recorded(): + """The walk covers each kernel's own module and every module hashed + beside the shared list, not only the list (microcosm#1492 review), and + the record is kept per group of kernels sharing one hash.""" + compile_list = _compile_list() + assert {kernel for group in KERNEL_GROUPS.values() for kernel, _ in group} == { + kernel for kernel, _ in KERNELS_HASHING_THE_COMPILE_MODULES + } + mismatches = recorded_import_mismatches( + KERNEL_GROUPS, + listed=compile_list.TRANSPORT_COMPILE_MODULE_NAMES, + excluded=frozenset(compile_list.EXCLUDED_DIRECT_IMPORTS), + hash_inputs=HASH_INPUTS, + recorded=UNCLASSIFIED_IMPORTS, + ) + assert not mismatches, mismatches diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index ccc785feaf..6f2e20b815 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -1517,8 +1517,10 @@ def test_the_calibrated_node_keys_on_the_schema_9_publication_code_not_the_solve # changes the output's bytes, so it must re-key the calibrated node (through # its terminal_code parameter); otherwise a store run with resume="auto" # serves the old labels. Like the high-value homes code, it stays out of - # the two kernels' shared code hash, so it never re-keys the preflight node - # or the solve. + # the two kernels' shared code hash. An edit to the catalogs, the schema + # or the label choices therefore never re-keys the preflight node or the + # solve; an edit to geography_constants still does, because the target + # kernels the preflight reads hash it too (microcosm#1492). import microcosm.calibrate.geography_constants as geography_constants_module import microcosm.calibrate.hierarchy_labels as hierarchy_labels_module import microcosm.diagnostics.catalogs as catalogs_module diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_solve_scope.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_solve_scope.py index cf82c461cb..f6f596959f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_solve_scope.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_solve_scope.py @@ -140,8 +140,12 @@ def _module_path(name: str) -> str: - relative = Path(*name.split(".")[2:]).with_suffix(".py") - return str((_BUILD_SOURCE / relative).relative_to(_REPOSITORY)) + """The repository path of a first-party module in any workspace package: + the compile modules include ``microcosm.calibrate`` ones (microcosm#1492).""" + shard, *rest = name.split(".")[1:] + relative = Path(*rest).with_suffix(".py") + source = _REPOSITORY / "packages" / f"microcosm-{shard}" / "src" / "microcosm" + return str((source / shard / relative).relative_to(_REPOSITORY)) def _docstrings(tree: ast.Module) -> set[int]: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index 8abc5978f7..145c7e4790 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -1,11 +1,26 @@ -"""Both UK target kernels hash every module that decides a target value.""" +"""Both UK target kernels hash every module that decides a target value. + +Two checks pin the closure, as for the transport compile kernels: + +- the shared list's own: every first-party import of a module in + ``UK_TARGET_COMPILE_MODULE_NAMES`` is listed, or excluded with a reason; +- each kernel's: the walk from every module a target or problem kernel + hashes, its own module included, meets only modules the kernel hashes, the + list's exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS`` for + that kernel's group, which no review has classified yet (microcosm#1497). + That mapping may only shrink. +""" from __future__ import annotations -import ast import importlib from pathlib import Path +from types import MappingProxyType +import pytest + +import microcosm.calibrate.geography_constants as geography_constants_module +import microcosm.calibrate.solve as calibrate_solve from microcosm.build import ledger_artifact from microcosm.build import ledger_targets as shared_ledger_targets from microcosm.build.uk_runtime import ( @@ -13,65 +28,36 @@ graph_targets, hmrc_uprating, measure_simulation, + target_weights, ) from microcosm.build.uk_runtime.target_compile_modules import ( EXCLUDED_DIRECT_IMPORTS, UK_TARGET_COMPILE_MODULE_NAMES, uk_target_compile_modules, ) - - -def _direct_build_imports(module_name: str) -> set[str]: - """Every ``microcosm.build`` module the named module imports, by AST.""" - - module = importlib.import_module(module_name) - tree = ast.parse(Path(module.__file__).read_text(encoding="utf-8")) - package = module_name.rsplit(".", 1)[0] - candidates: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - if node.level: - base = package.rsplit(".", node.level - 1)[0] - stem = f"{base}.{node.module}" if node.module else base - else: - stem = node.module or "" - if not stem.startswith("microcosm.build"): - continue - candidates.add(stem) - candidates.update(f"{stem}.{alias.name}" for alias in node.names) - elif isinstance(node, ast.Import): - candidates.update( - alias.name - for alias in node.names - if alias.name.startswith("microcosm.build") - ) - modules: set[str] = set() - for name in candidates: - try: - imported = importlib.import_module(name) - except ImportError: - continue - source = getattr(imported, "__file__", None) - if source is not None and not source.endswith("__init__.py"): - modules.add(name) - return modules +from test_support.microcosm_build.first_party_imports import ( + HashInput, + assert_an_edit_rekeys, + first_party_imports, + hashed_module_names, + recorded_import_mismatches, + unlisted_imports, +) def test_every_direct_import_of_a_listed_module_is_listed_or_excluded(): - listed = set(UK_TARGET_COMPILE_MODULE_NAMES) - uncovered = { - f"{name} -> {imported}" - for name in UK_TARGET_COMPILE_MODULE_NAMES - for imported in _direct_build_imports(name) - if imported not in listed and imported not in EXCLUDED_DIRECT_IMPORTS - } + """Every first-party import, not only microcosm.build: the target + hierarchies take their geography labels from + microcosm.calibrate.geography_constants, which a scan of microcosm.build + imports never saw (microcosm#1492).""" + uncovered = unlisted_imports( + UK_TARGET_COMPILE_MODULE_NAMES, frozenset(EXCLUDED_DIRECT_IMPORTS) + ) assert not uncovered, sorted(uncovered) def test_exclusions_are_live_and_never_listed(): - imported = set().union( - *(_direct_build_imports(name) for name in UK_TARGET_COMPILE_MODULE_NAMES) - ) + imported = set().union(*map(first_party_imports, UK_TARGET_COMPILE_MODULE_NAMES)) assert set(EXCLUDED_DIRECT_IMPORTS) <= imported assert not set(EXCLUDED_DIRECT_IMPORTS) & set(UK_TARGET_COMPILE_MODULE_NAMES) @@ -149,3 +135,654 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): graph_national.UKNationalProblemKernel(resolver_factory=None), ): assert ledger_artifact in _hashed_by(kernel, monkeypatch, graph_national) + + +#: The kernels whose hash includes the UK target compile modules, with the +#: module whose ``source_hash`` each calls: the full graph's target, +#: selection, measures and problem kernels and the national graph's target +#: and problem kernels. +_TARGET_AND_PROBLEM_KERNELS = ( + (graph_targets.UKFullTargetCompilationKernel(), graph_targets), + (graph_targets.UKFullTargetSelectionKernel(), graph_targets), + (graph_targets.UKFullMeasureKernel(), graph_targets), + (graph_targets.UKFullProblemKernel(), graph_targets), + (graph_national.UKNationalTargetKernel(), graph_national), + (graph_national.UKNationalProblemKernel(resolver_factory=None), graph_national), +) +_KERNELS_BY_REF = { + kernel.ref: (kernel, module) for kernel, module in _TARGET_AND_PROBLEM_KERNELS +} +_FULL_GRAPH = tuple( + ref for ref in _KERNELS_BY_REF if not ref.startswith("uk.full.national") +) +_NATIONAL_GRAPH = tuple( + ref for ref in _KERNELS_BY_REF if ref.startswith("uk.full.national") +) + +#: Code the target kernels ran without hashing it, found by the review of +#: microcosm#1492, and the kernels that now hash it. All four full-graph +#: kernels share one hash, so what one of them runs re-keys all four. +_RUN_MODULES = { + # uk.full.target_selection@1 calls select_targets. + "microcosm.calibrate.target_selection": _FULL_GRAPH, + # The measures and problem kernels build and encode constraint matrices. + "microcosm.calibrate.matrix": _FULL_GRAPH + _NATIONAL_GRAPH, + "microcosm.calibrate.artifacts": _FULL_GRAPH + _NATIONAL_GRAPH, + # The full problem's loss scales (default_target_loss_scales). The + # national kernels call nothing in it (see the test below). + "microcosm.calibrate.solve": ("uk.full.problem@1",), + # full_targets' exclusion clock (exclusion_evaluation_date). + "microcosm.build.uk_runtime.weighted_integrity": _FULL_GRAPH + _NATIONAL_GRAPH, + # full_targets' surface provenance (_ledger_provenance) and band-edge + # roster check (_validate_band_edge_registry), moved out of + # calibration_run so that the runner itself stays unhashed. + "microcosm.build.uk_runtime.ledger_surface": _FULL_GRAPH + _NATIONAL_GRAPH, +} + +#: The kernels grouped by shared hash: the full graph's target, selection and +#: measures kernels share _TargetKernel's; the problem adds its loss-weight +#: hash; the two national kernels share _NationalKernel's. +FULL_TARGETS = "uk.full.target_compilation@1, target_selection@1 and measures@1" +FULL_PROBLEM = "uk.full.problem@1" +NATIONAL = "uk.full.national_targets@1 and national_problem@1" +KERNEL_GROUPS = MappingProxyType( + { + FULL_TARGETS: tuple( + _KERNELS_BY_REF[ref] + for ref in ( + "uk.full.target_compilation@1", + "uk.full.target_selection@1", + "uk.full.measures@1", + ) + ), + FULL_PROBLEM: (_KERNELS_BY_REF["uk.full.problem@1"],), + NATIONAL: tuple(_KERNELS_BY_REF[ref] for ref in _NATIONAL_GRAPH), + } +) + +#: Imports a kernel's module makes only to build the hash, and the importers +#: the reason covers. They stay unhashed on purpose, so an edit to the list +#: re-keys only the kernels whose hashed modules it adds or removes. +HASH_INPUTS = MappingProxyType( + { + "microcosm.build.uk_runtime.target_compile_modules": HashInput( + importers=( + "microcosm.build.uk_runtime.graph_national", + "microcosm.build.uk_runtime.graph_targets", + ), + reason=( + "names the modules the target kernels hash; graph_targets and " + "graph_national read it only to build their hashes" + ), + ), + "microcosm.build.uk_runtime.solve_scope": HashInput( + importers=("microcosm.build.uk_runtime.graph_targets",), + reason=( + "computes uk_solve_fingerprint, which graph_targets puts in the " + "full graph's target hashes and in the graph's contract-identity " + "parameter; no kernel's run reads it" + ), + ), + } +) + +#: Every ``importer -> imported`` edge the walk from a kernel group's hashed +#: modules meets that the group does not hash and no review has classified, +#: with the groups that leave it unhashed. Recorded as found when the +#: per-kernel check was added; microcosm#1497 classifies them. Hash an entry +#: in the kernels that run it, move what they run into a module they hash, or +#: record why they need not hash it, then delete it here. The check fails on +#: an edge a group meets that is not recorded for it and on an edge recorded +#: for a group that no longer meets it, so the mapping can only shrink, and +#: removing a module from one group's hash fails even where another group +#: records the same edge. +UNCLASSIFIED_IMPORTS = MappingProxyType( + { + "microcosm.build.uk_runtime.graph_national -> microcosm.build.artifact_files": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.country_spec": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.country_spec": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.gate_battery": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.fiscal_targets -> microcosm.build.gates": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.gates": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.holdout": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.input_mass": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.logbook_adoption": ( + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.plan": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.source_manifest": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.frs_relationships -> microcosm.build.source_manifest": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.canonical": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.engine_abi": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.loader": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.model": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.schemas": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.take_up_semantics": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.spec_engine.yaml12": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.staging_v2": ( + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.build.trace": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.atomic_area_support": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.battery_bindings": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.calibration_run": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.cgt_projection": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.content_identity": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.dataset_size": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.dfe_spring_term": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.diagnostics": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.frs_annual_income": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.frs_relationships -> microcosm.build.uk_runtime.frs_spine": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.frs_spine": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.full_gates": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_authority_input -> microcosm.build.uk_runtime.geography_sources": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_build": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_calibration": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_population": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.graph_population": ( + FULL_TARGETS, + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_targets": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_terminal": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.hmrc_replay -> microcosm.build.uk_runtime.hmrc_income": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.hmrc_income": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.uk_runtime.hmrc_source_contract": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.income_tax_rate_classes": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.lcfs_consumption": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.local_geography": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.local_rowwise": ( + FULL_TARGETS, + NATIONAL, + ), + "microcosm.build.uk_runtime.full_problem -> microcosm.build.uk_runtime.local_rowwise": ( + FULL_TARGETS, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.local_rowwise": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.local_rowwise": ( + FULL_TARGETS, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.national_calibration": ( + FULL_TARGETS, + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.national_doctrine": ( + FULL_TARGETS, + ), + "microcosm.build.uk_runtime.local_doctrine -> microcosm.build.uk_runtime.national_doctrine": ( + FULL_TARGETS, + ), + "microcosm.build.uk_runtime.frs_relationships -> microcosm.build.uk_runtime.national_frame": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.national_frame": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.national_frame": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.national_frame": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.national_frame": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.uk_runtime.national_frame": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.rowwise_dataset": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.rowwise_posture": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.size_checkpoint": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.solve_progress": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.target_weights": ( + FULL_TARGETS, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.uc_target_measurements": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.weight_year": ( + NATIONAL, + ), + "microcosm.build.gates -> microcosm.build.us_runtime.stacked_spine": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.solve -> microcosm.calibrate.exact_k": (FULL_PROBLEM,), + "microcosm.calibrate.solve -> microcosm.calibrate.gates": (FULL_PROBLEM,), + "microcosm.calibrate.solve -> microcosm.calibrate.initialization": ( + FULL_PROBLEM, + ), + "microcosm.calibrate.matrix -> microcosm.calibrate.monetary_binding": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.gates -> microcosm.calibrate.solve": (FULL_TARGETS, NATIONAL), + # Reviewed (round 2 of #1493): only national_calibration_manifest calls + # it, in uk.full.national-gates@1, which reads the solve's result. + "microcosm.build.uk_runtime.graph_national -> microcosm.calibrate.solve": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.calibrate.solve": ( + FULL_TARGETS, + ), + "microcosm.calibrate.artifacts -> microcosm.calibrate.solve": ( + FULL_TARGETS, + NATIONAL, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.frame.adapters.policyengine_uk": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.adapters.policyengine_uk": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.country_spec -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.frs_relationships -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.frame.bundle": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.frame.bundle": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.national_calibration -> microcosm.frame.bundle": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.weighted_integrity -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.artifacts -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.matrix -> microcosm.frame.bundle": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.solve -> microcosm.frame.bundle": (FULL_PROBLEM,), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.materialize": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.schema": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.frame.bundle -> microcosm.frame.schema": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.gates -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.full_measure -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.frame.weights": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.local_rowwise -> microcosm.frame.weights": ( + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.measure_simulation -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.national_calibration -> microcosm.frame.weights": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.national_frame -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.artifacts -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.matrix -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.calibrate.solve -> microcosm.frame.weights": (FULL_PROBLEM,), + "microcosm.frame.bundle -> microcosm.frame.weights": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.graph.canonical": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.canonical": ( + FULL_TARGETS, + FULL_PROBLEM, + ), + "microcosm.calibrate.artifacts -> microcosm.graph.canonical": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.codecs": ( + FULL_TARGETS, + FULL_PROBLEM, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.graph.decl": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.decl": ( + FULL_TARGETS, + FULL_PROBLEM, + ), + "microcosm.calibrate.artifacts -> microcosm.graph.decl": ( + FULL_TARGETS, + FULL_PROBLEM, + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_national -> microcosm.graph.kernel": ( + NATIONAL, + ), + "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.kernel": ( + FULL_TARGETS, + FULL_PROBLEM, + ), + } +) + + +@pytest.mark.parametrize( + ("kernel", "module"), + _TARGET_AND_PROBLEM_KERNELS, + ids=[kernel.ref for kernel, _ in _TARGET_AND_PROBLEM_KERNELS], +) +def test_an_edit_to_the_geography_labels_rekeys_the_kernel( + tmp_path, monkeypatch, kernel, module +): + """The shared and UK ledger targets label a target's geography from + UK_GEOGRAPHY_ID_TO_LABEL, so an edit there must give these kernels new + keys instead of reusing a surface with the old label (microcosm#1492).""" + assert_an_edit_rekeys(kernel, geography_constants_module, tmp_path, monkeypatch) + + +@pytest.mark.parametrize( + ("module_name", "ref"), + [(name, ref) for name, refs in _RUN_MODULES.items() for ref in refs], + ids=[ + f"{name.rsplit('.', 1)[1]}-{ref}" + for name, refs in _RUN_MODULES.items() + for ref in refs + ], +) +def test_an_edit_to_code_the_kernels_run_rekeys_them( + tmp_path, monkeypatch, module_name, ref +): + kernel, _ = _KERNELS_BY_REF[ref] + module = importlib.import_module(module_name) + assert_an_edit_rekeys(kernel, module, tmp_path, monkeypatch) + + +@pytest.mark.parametrize( + ("kernel", "module"), + _TARGET_AND_PROBLEM_KERNELS, + ids=[kernel.ref for kernel, _ in _TARGET_AND_PROBLEM_KERNELS], +) +def test_an_edit_to_calibration_run_leaves_the_kernels_keys( + tmp_path, monkeypatch, kernel, module +): + """The target kernels run only the two functions now in ledger_surface. + An edit elsewhere in calibration_run, the national calibration runner, + re-runs no target compilation and, in builds that construct the spine in + the graph, no solve (microcosm#1492).""" + from microcosm.build.uk_runtime import calibration_run + + before = kernel.implementation_hash() + source = Path(calibration_run.__file__) + edited = tmp_path / source.name + edited.write_bytes(source.read_bytes() + b"\n# an edited line\n") + monkeypatch.setattr(calibration_run, "__file__", str(edited)) + assert kernel.implementation_hash() == before + + +def test_full_problem_loss_weights_hash_what_target_weights_reads(monkeypatch): + """target_weights takes UK_WIDE_GEOGRAPHY_IDS from geography_constants + and the loss scales from calibrate.solve for the weights the problem + binds, so the problem's weight hash covers both modules itself, not only + through the target modules' hash.""" + calls: list[tuple] = [] + monkeypatch.setattr( + graph_targets, + "source_hash", + lambda *objects, **_dependencies: calls.append(objects) or "test", + ) + graph_targets.UKFullProblemKernel().implementation_hash() + (weights,) = [objects for objects in calls if target_weights in objects] + assert geography_constants_module in weights + assert calibrate_solve in weights + + +def test_the_national_kernels_leave_calibrate_solve_unhashed(): + """Neither the national target nor the problem path calls calibrate.solve: + its one caller in graph_national, national_calibration_manifest, runs in + uk.full.national-gates@1, whose node reads the solve's result. Hashing it + here would re-run national target compilation and the problem's engine + measures on every solver edit, with no coverage gain (round 2 of #1493).""" + for ref in _NATIONAL_GRAPH: + kernel, module = _KERNELS_BY_REF[ref] + assert calibrate_solve.__name__ not in hashed_module_names(kernel, module) + + +def test_every_import_a_target_kernel_reaches_is_hashed_or_recorded(): + """The walk covers each kernel's own module and every module hashed + beside the shared list, not only the list (microcosm#1492 review), and + the record is kept per group of kernels sharing one hash.""" + assert {kernel for group in KERNEL_GROUPS.values() for kernel, _ in group} == { + kernel for kernel, _ in _TARGET_AND_PROBLEM_KERNELS + } + mismatches = recorded_import_mismatches( + KERNEL_GROUPS, + listed=UK_TARGET_COMPILE_MODULE_NAMES, + excluded=frozenset(EXCLUDED_DIRECT_IMPORTS), + hash_inputs=HASH_INPUTS, + recorded=UNCLASSIFIED_IMPORTS, + ) + assert not mismatches, mismatches diff --git a/test_support/microcosm_build/first_party_imports.py b/test_support/microcosm_build/first_party_imports.py new file mode 100644 index 0000000000..105b79b360 --- /dev/null +++ b/test_support/microcosm_build/first_party_imports.py @@ -0,0 +1,354 @@ +"""The first-party modules a module imports, read from its source. + +A kernel's implementation hash covers the files of the modules it passes to +``source_hash``, and ``source_hash`` does not follow imports. A kernel's +hashed modules therefore cover the code it runs only if every first-party +module they import is hashed too, or excluded with a reason. These helpers +find those imports, so a test can pin that closure. + +What the scan sees: ``import a.b`` and ``import a.b as c`` (for a package, +through the attribute chains the module reads from the name it binds), and +``from a.b import c``, absolute or relative, at any depth of the module, +function bodies included. A name imported from a package resolves to the +module that defines it. + +What it cannot see: a module named by a string (``importlib.import_module``, +``__import__``, ``getattr`` with a computed name), a module object passed in +as a value, and data files read by path (``importlib.resources``, ``open``). +Data files are not source, so a kernel that reads one must cover it in its +hash some other way, as the UK target kernels do with the country spec's +resource fingerprint. +""" + +from __future__ import annotations + +import ast +import importlib +import importlib.util +import inspect +from collections.abc import Mapping, Sequence +from collections.abc import Set as AbstractSet +from pathlib import Path +from types import ModuleType +from typing import NamedTuple + +#: Every workspace package imports as ``microcosm.``. +FIRST_PARTY_PREFIX = "microcosm." + + +def _is_package(module: ModuleType) -> bool: + return Path(module.__file__).name == "__init__.py" + + +def _module_exists(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except (ImportError, ValueError): + return False + + +def _import_stem(module_name: str, level: int, stem: str | None) -> str: + """The absolute module name an ``import`` statement's ``from`` part names.""" + + if not level: + return stem or "" + module = importlib.import_module(module_name) + package = module_name if _is_package(module) else module_name.rsplit(".", 1)[0] + base = package.rsplit(".", level - 1)[0] + return f"{base}.{stem}" if stem else base + + +def _defining_module_of(value: object) -> str | None: + """The module that defines a class or function, through any wrapper.""" + + if inspect.isclass(value): + return value.__module__ + unwrapped = inspect.unwrap(value) if callable(value) else value + if inspect.isroutine(unwrapped): + module = inspect.getmodule(unwrapped) + return module.__name__ if module is not None else None + return None + + +def defining_module(package_name: str, attribute: str) -> str: + """The module that defines ``attribute`` as the package exposes it. + + A submodule resolves to itself whether or not it has been imported yet, + so the answer does not depend on which tests ran first. A class or + function names its own module. A constant is traced through the package + ``__init__``'s ``from … import`` statements. A name the ``__init__`` + defines itself resolves to the package. + """ + + package = importlib.import_module(package_name) + value = getattr(package, attribute, None) + if isinstance(value, ModuleType): + return value.__name__ + submodule = f"{package_name}.{attribute}" + if value is None and _module_exists(submodule): + return submodule + defined_in = _defining_module_of(value) + if defined_in is not None: + return defined_in + tree = ast.parse(Path(package.__file__).read_text(encoding="utf-8")) + for statement in tree.body: + if not isinstance(statement, ast.ImportFrom): + continue + for alias in statement.names: + if (alias.asname or alias.name) != attribute: + continue + stem = _import_stem(package_name, statement.level, statement.module) + if _is_package(importlib.import_module(stem)): + return defining_module(stem, alias.name) + return stem + return package_name + + +def _resolve_dotted(name: str) -> str: + """The module a dotted attribute chain such as ``a.b.C`` reads from.""" + + parts = name.split(".") + for end in range(len(parts), 0, -1): + prefix = ".".join(parts[:end]) + if not _module_exists(prefix): + continue + module = importlib.import_module(prefix) + if _is_package(module) and end < len(parts): + return defining_module(prefix, parts[end]) + return prefix + return name + + +def _attribute_chains(tree: ast.AST) -> list[str]: + """Every outermost dotted chain rooted at a plain name (``a.b.c``).""" + + inner = { + id(node.value) for node in ast.walk(tree) if isinstance(node, ast.Attribute) + } + chains = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Attribute) or id(node) in inner: + continue + parts = [node.attr] + value = node.value + while isinstance(value, ast.Attribute): + parts.append(value.attr) + value = value.value + if isinstance(value, ast.Name): + chains.append(".".join([value.id, *reversed(parts)])) + return chains + + +def _package_import_reads(binding: str, target: str, chains: list[str]) -> set[str]: + """The modules read through ``import target`` bound to the name ``binding``. + + ``import a.b`` binds ``a`` and ``import a.b as c`` binds ``c``; either way + each chain read from the bound name resolves to the module defining what + it names. With no chain into ``target``, the package itself is read. + """ + + prefix = target if binding == target.split(".")[0] else binding + reads = { + _resolve_dotted(target + chain[len(prefix) :]) + for chain in chains + if chain == prefix or chain.startswith(prefix + ".") + } + return {name for name in reads if name.startswith(target)} or {target} + + +def first_party_imports(module_name: str) -> set[str]: + """Every first-party module ``module_name`` imports, at any depth of its AST. + + Function-level imports count: they run when the function does. A name + imported from a package resolves to the module that defines it, so + ``from microcosm.calibrate import TargetRegistry`` names + ``microcosm.calibrate.registry``. + """ + + module = importlib.import_module(module_name) + tree = ast.parse(Path(module.__file__).read_text(encoding="utf-8")) + chains: list[str] | None = None + found: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + stem = _import_stem(module_name, node.level, node.module) + if not stem.startswith(FIRST_PARTY_PREFIX): + continue + if not _is_package(importlib.import_module(stem)): + found.add(stem) + continue + found.update(defining_module(stem, alias.name) for alias in node.names) + elif isinstance(node, ast.Import): + for alias in node.names: + if not alias.name.startswith(FIRST_PARTY_PREFIX): + continue + if not _is_package(importlib.import_module(alias.name)): + found.add(alias.name) + continue + if chains is None: + chains = _attribute_chains(tree) + binding = alias.asname or alias.name.split(".")[0] + found |= _package_import_reads(binding, alias.name, chains) + found.discard(module_name) + return found + + +def unlisted_imports(listed: Sequence[str], excluded: AbstractSet[str]) -> set[str]: + """``listed -> imported`` for each import of a listed module neither + listed nor excluded. The check of a shared module list.""" + + covered = set(listed) | set(excluded) + return { + f"{name} -> {imported}" + for name in listed + for imported in first_party_imports(name) + if imported not in covered + } + + +def unhashed_imports( + hashed: AbstractSet[str], *, listed: Sequence[str], excluded: AbstractSet[str] +) -> set[str]: + """``importer -> imported`` for each import a kernel's hash leaves uncovered. + + The walk starts at every module the kernel hashes. An import is covered + when the kernel hashes it, or when the importer is in the shared list + ``listed`` and the import is one of that list's reviewed exclusions + ``excluded``. The walk goes on through every covered import, so an + excluded module's own imports must be covered too, and its imports are + not covered by the list's exclusions, whose reasons were written for the + listed importers alone. An uncovered import is reported and not followed. + """ + + listed_names = set(listed) + uncovered: set[str] = set() + pending = sorted(hashed) + seen: set[str] = set() + while pending: + name = pending.pop() + if name in seen: + continue + seen.add(name) + for imported in sorted(first_party_imports(name)): + if imported in hashed or (name in listed_names and imported in excluded): + pending.append(imported) + else: + uncovered.add(f"{name} -> {imported}") + return uncovered + + +class HashInput(NamedTuple): + """A module a kernel's file imports only to compute the hash itself. + + The exemption holds for the named importers alone, so a hashed module + that starts calling it at run time is still reported. + """ + + importers: tuple[str, ...] + reason: str + + +def recorded_import_mismatches( + kernel_groups: Mapping[str, Sequence[tuple[object, ModuleType]]], + *, + listed: Sequence[str], + excluded: AbstractSet[str], + hash_inputs: Mapping[str, HashInput], + recorded: Mapping[str, Sequence[str]], +) -> dict[str, object]: + """Where the walk of each kernel group disagrees with the record. + + ``kernel_groups`` maps a group name to its ``(kernel, module)`` pairs; + every kernel in a group must reach the same unhashed imports, because + the group shares one hash. ``recorded`` maps each ``importer -> + imported`` edge to the groups that leave it unhashed, so removing a + module from one group's hash fails even when another group records the + same edge. Every ``hash_inputs`` edge must be met by some group, and no + kernel may hash a hash input. An empty result means the record holds. + """ + + allowed = { + f"{importer} -> {imported}" + for imported, hash_input in hash_inputs.items() + for importer in hash_input.importers + } + mismatches: dict[str, object] = {} + unknown = { + edge: sorted(set(groups) - set(kernel_groups)) + for edge, groups in recorded.items() + if set(groups) - set(kernel_groups) + } + if unknown: + mismatches["recorded for an unknown group"] = unknown + allowed_met: set[str] = set() + for group, kernels in kernel_groups.items(): + walks = [] + for kernel, module in kernels: + hashed = hashed_module_names(kernel, module) + if set(hash_inputs) & hashed: + mismatches[f"{kernel.ref} hashes a hash input"] = sorted( + set(hash_inputs) & hashed + ) + walks.append(unhashed_imports(hashed, listed=listed, excluded=excluded)) + if any(walk != walks[0] for walk in walks): + mismatches[f"{group}: its kernels reach different imports"] = sorted( + set().union(*walks) - set.intersection(*walks) + ) + met = set().union(*walks) + allowed_met |= met & allowed + met -= allowed + expected = {edge for edge, groups in recorded.items() if group in groups} + if met - expected: + mismatches[f"{group}: unhashed and not recorded"] = sorted(met - expected) + if expected - met: + mismatches[f"{group}: recorded but no longer met (delete them)"] = sorted( + expected - met + ) + if allowed - allowed_met: + mismatches["hash inputs no importer reaches"] = sorted(allowed - allowed_met) + return mismatches + + +def hashed_module_names(kernel: object, module: ModuleType) -> frozenset[str]: + """Every module ``kernel.implementation_hash()`` passes to ``source_hash``. + + ``module`` is the module whose ``source_hash`` name the kernel calls; the + recorder stands in for it during one call, then the original is restored. + """ + + original = module.source_hash + recorded: list[object] = [] + + def record(*objects: object, **dependencies: object) -> str: + recorded.extend(objects) + return original(*objects, **dependencies) + + module.source_hash = record + try: + kernel.implementation_hash() + finally: + module.source_hash = original + names = set() + for item in recorded: + defining = item if isinstance(item, ModuleType) else inspect.getmodule(item) + if defining is None: + raise ValueError(f"{item!r} has no module whose file can be hashed.") + names.add(defining.__name__) + return frozenset(names) + + +def assert_an_edit_rekeys( + kernel: object, module: ModuleType, tmp_path: Path, monkeypatch +) -> None: + """``kernel``'s implementation hash changes when ``module``'s file does. + + The file is copied with one more line and ``module.__file__`` points at + the copy, which ``source_hash`` reads. + """ + + before = kernel.implementation_hash() + source = Path(module.__file__) + edited = tmp_path / source.name + edited.write_bytes(source.read_bytes() + b"\n# an edited line\n") + monkeypatch.setattr(module, "__file__", str(edited)) + assert kernel.implementation_hash() != before, (kernel.ref, module.__name__)