From 3ca0435a1630bb1e0a83885b757c0e510e92fb5e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:01:26 +0400 Subject: [PATCH 1/8] Test that Microcosm accepts Chronicle consumer_fact.v5 facts Chronicle main stamps every consumer fact chronicle.consumer_fact.v5 since PolicyEngine/chronicle#377 (44eebe037): the v4 row plus an optional package_dimension_value_labels field holding a package's own wording of a dimension value. These tests run before the change, and 11 of the 20 fail on assertions: - the declared row contracts include v5, in the chronicle epoch, declared as v4 is (test_chronicle_epoch.py); - a v5 artifact built from Chronicle's sample rows loads under its full feed pin and its per-row id is accepted, and the v4 restatement of the same rows still loads (test_ledger_artifact.py); - a target compiled from the v5 row takes Chronicle's label "All returns, total", never the package's "All returns", and the v4 row compiles the same; a malformed package_dimension_value_labels on a selected fact is refused, in each of eight shapes the v5 schema refuses (test_ledger_targets.py). The fixture directory holds Chronicle's v5 schema (sha256 acd33a09...) and sample rows from 44eebe037, unchanged at Chronicle main 28064d0; a test validates the rows and the malformed shapes against that schema. The shared artifact helper gains a schema-digest argument. Refs #1509 Co-Authored-By: Claude Opus 5.5 --- .../shared/test_chronicle_epoch.py | 31 ++ .../shared/test_ledger_artifact.py | 97 +++- .../engine_free/shared/test_ledger_targets.py | 90 ++++ .../chronicle_consumer_fact_v5/SOURCE.md | 22 + .../consumer_fact.v5.schema.json | 502 ++++++++++++++++++ .../consumer_facts.jsonl | 3 + .../microcosm_build/chronicle_feed.py | 75 ++- 7 files changed, 816 insertions(+), 4 deletions(-) create mode 100644 packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/SOURCE.md create mode 100644 packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_fact.v5.schema.json create mode 100644 packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_facts.jsonl diff --git a/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py b/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py index 877cd42542..2a07cbcb66 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py @@ -46,6 +46,7 @@ consumer_artifact_schema_epoch, consumer_fact_schema_epoch, declared_identity, + describe_accepted_consumer_fact_schema_versions, fact_key_epoch, fact_key_epoch_label, feed_fact_key_epochs, @@ -57,6 +58,7 @@ row_fact_key_epochs, ) from microcosm.build.ledger_artifact import load_ledger_consumer_artifact +from test_support.microcosm_build.chronicle_feed import chronicle_v5_schema_version from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -66,6 +68,11 @@ _CHRONICLE_AGGREGATE_KEY = "chronicle.aggregate_fact.v3:abc123" _LEDGER_AGGREGATE_KEY = "ledger.aggregate_fact.v2:abc123" +#: The per-row id Chronicle ``main`` stamps on every fact since chronicle#377 +#: (44eebe037), read from Chronicle's own v5 schema so the spelling is +#: Chronicle's, not this module's. +_CHRONICLE_V5_ROW_SCHEMA_ID = chronicle_v5_schema_version() + #: Every consumer-artifact manifest id a real producer stamps. Microcosm's #: own minted v1 and Chronicle's published v2 are both ledger-era; v3 is the #: chronicle-era successor. @@ -402,6 +409,7 @@ def test_schema_id_membership_covers_every_real_producer_id() -> None: CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION, CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V3, CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V4, + _CHRONICLE_V5_ROW_SCHEMA_ID, } for schema_id, expected_epoch in _ARTIFACT_SCHEMA_IDS_BY_EPOCH: assert is_accepted_consumer_artifact_schema_version(schema_id), schema_id @@ -418,6 +426,7 @@ def test_schema_id_membership_covers_every_real_producer_id() -> None: assert consumer_fact_schema_epoch(CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V4) == ( CHRONICLE_EPOCH ) + assert consumer_fact_schema_epoch(_CHRONICLE_V5_ROW_SCHEMA_ID) == CHRONICLE_EPOCH assert not is_accepted_consumer_artifact_schema_version( "policyengine_chronicle.other.v9" ) @@ -431,6 +440,27 @@ def test_schema_id_membership_covers_every_real_producer_id() -> None: assert consumer_artifact_schema_epoch(LEDGER_CONSUMER_FACT_SCHEMA_VERSION) is None +def test_v5_row_contract_is_declared_like_v4() -> None: + """chronicle#377's v5 row is a declared chronicle-era row contract. + + v5 is the v4 row plus the optional ``package_dimension_value_labels``, + so it is declared exactly as v4 is: the same identity kind, the + chronicle epoch, and named in the accepted-versions message a refusal + prints. Chronicle's own table (``CONSUMER_FACT_ROW_CONTRACTS`` in + ``chronicle/epoch.py``) ends at v5 from 44eebe037. + """ + v4 = declared_identity(CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V4) + v5 = declared_identity(_CHRONICLE_V5_ROW_SCHEMA_ID) + + assert v4 is not None + assert v5 is not None + assert (v5.kind, v5.epoch, v5.family) == (v4.kind, CHRONICLE_EPOCH, v4.family) + assert is_accepted_consumer_fact_schema_version(_CHRONICLE_V5_ROW_SCHEMA_ID) + assert _CHRONICLE_V5_ROW_SCHEMA_ID in ( + describe_accepted_consumer_fact_schema_versions() + ) + + def test_chronicle_main_manifest_loads_exactly_as_published(tmp_path) -> None: """The manifest chronicle ``main`` writes today, field for field. @@ -603,6 +633,7 @@ def test_malformed_schema_version_types_raise_the_documented_error( CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION, CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V3, CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V4, + _CHRONICLE_V5_ROW_SCHEMA_ID, ], ) def test_per_row_schema_id_is_accepted_in_either_era( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py b/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py index 79a1caafb4..e2f3c1a3f3 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py @@ -16,7 +16,13 @@ import pytest from hypothesis import given, settings from hypothesis import strategies as st +from jsonschema import Draft202012Validator +from microcosm.build.chronicle_epoch import ( + CHRONICLE_EPOCH, + consumer_fact_schema_epoch, + is_accepted_consumer_fact_schema_version, +) from microcosm.build.ledger_artifact import ( CONSUMER_ARTIFACT_SCHEMA_VERSION, ChronicleFeedPinMismatchError, @@ -28,7 +34,16 @@ validate_chronicle_feed_pin, ) from microcosm.graph.canonical import canonical_json -from test_support.microcosm_build.chronicle_feed import write_pinned_artifact +from test_support.microcosm_build.chronicle_feed import ( + MALFORMED_PACKAGE_DIMENSION_VALUE_LABELS, + PACKAGE_DIMENSION_VALUE_LABELS, + as_v4_row, + chronicle_v5_fact_rows, + chronicle_v5_schema, + chronicle_v5_schema_sha256, + chronicle_v5_schema_version, + write_pinned_artifact, +) from test_support.paths import paths_for @@ -368,6 +383,86 @@ def test_complete_feed_pin_accepts_exact_match(tmp_path): assert artifact.fact_row_count == pin["fact_row_count"] +#: sha256 of Chronicle's ``consumer_fact.v5.schema.json`` at 44eebe037, the +#: digest chronicle#377 publishes for the v5 contract. +_CHRONICLE_V5_SCHEMA_SHA256 = ( + "acd33a091cd8d192be5d264e3e94c13e00afb6fd1972e57d4576ac55a54f6970" +) +#: The v4 schema digest the UK, NZ and BE feed pins carry today. +_CHRONICLE_V4_SCHEMA_SHA256 = ( + "d953306d6f01fbc64db4372d0262d3012ee590269f8a25b276b18dca0626c6a0" +) + + +def test_chronicle_v5_fixture_is_valid_under_chronicle_v5_schema(): + """The v5 fixture is Chronicle's, checked against Chronicle's own schema. + + The malformed ``package_dimension_value_labels`` values the target + compiler refuses are the ones this schema refuses, so Microcosm's check + and Chronicle's contract agree. + """ + schema = chronicle_v5_schema() + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema) + rows = chronicle_v5_fact_rows() + + assert chronicle_v5_schema_sha256() == _CHRONICLE_V5_SCHEMA_SHA256 + assert len(rows) == 3 + for row in rows: + assert row["schema_version"] == chronicle_v5_schema_version() + assert row[PACKAGE_DIMENSION_VALUE_LABELS] + validator.validate(row) + for case, malformed in MALFORMED_PACKAGE_DIMENSION_VALUE_LABELS: + row = deepcopy(rows[0]) + row[PACKAGE_DIMENSION_VALUE_LABELS] = malformed + assert not validator.is_valid(row), case + + +def test_chronicle_v5_artifact_loads_under_its_feed_pin(tmp_path): + """A v5 artifact loads under a full feed pin, and its rows stay as published. + + The loader keeps ``package_dimension_value_labels`` on the parsed rows, + because rows are carried exactly as the hashed file holds them; no + Microcosm reader takes a label from it. + """ + rows = chronicle_v5_fact_rows() + artifact_dir, pin = write_pinned_artifact( + tmp_path / "artifact", + rows, + consumer_fact_schema_sha256=chronicle_v5_schema_sha256(), + ) + v5 = chronicle_v5_schema_version() + + artifact = load_ledger_consumer_artifact(artifact_dir, feed_pin=pin) + + validate_chronicle_feed_pin(artifact, pin) + assert pin["consumer_fact_schema_versions"] == [v5] + assert pin["consumer_fact_schema_sha256"] == _CHRONICLE_V5_SCHEMA_SHA256 + assert artifact.fact_schema_versions == (v5,) + assert artifact.provenance()["fact_schema_versions"] == [v5] + assert is_accepted_consumer_fact_schema_version(v5) + assert consumer_fact_schema_epoch(v5) == CHRONICLE_EPOCH + assert list(artifact.facts) == rows + assert all(row[PACKAGE_DIMENSION_VALUE_LABELS] for row in artifact.facts) + + +def test_chronicle_v4_artifact_still_loads_under_its_feed_pin(tmp_path): + rows = [as_v4_row(row) for row in chronicle_v5_fact_rows()] + artifact_dir, pin = write_pinned_artifact( + tmp_path / "artifact", + rows, + consumer_fact_schema_sha256=_CHRONICLE_V4_SCHEMA_SHA256, + ) + + artifact = load_ledger_consumer_artifact(artifact_dir, feed_pin=pin) + + validate_chronicle_feed_pin(artifact, pin) + assert artifact.fact_schema_versions == ("chronicle.consumer_fact.v4",) + assert is_accepted_consumer_fact_schema_version("chronicle.consumer_fact.v4") + assert list(artifact.facts) == rows + assert not any(PACKAGE_DIMENSION_VALUE_LABELS in row for row in artifact.facts) + + def test_feed_pin_rejects_changed_facts_with_self_consistent_manifest(tmp_path): artifact_dir, pin = write_pinned_artifact(tmp_path / "artifact", [_fact_row()]) facts_path = artifact_dir / "consumer_facts.jsonl" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ledger_targets.py b/packages/microcosm-build/tests/engine_free/shared/test_ledger_targets.py index 4fa61d63ae..8852ae342b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ledger_targets.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ledger_targets.py @@ -1,4 +1,5 @@ import json +from copy import deepcopy from dataclasses import asdict, replace import pytest @@ -12,6 +13,7 @@ load_ledger_consumer_artifact, ) from microcosm.build.ledger_targets import ( + LedgerHierarchyMetadataError, LedgerTargetMapping, LedgerTargetReference, _shift_period_years, @@ -37,6 +39,12 @@ TargetRegistry, TargetSpec, ) +from test_support.microcosm_build.chronicle_feed import ( + MALFORMED_PACKAGE_DIMENSION_VALUE_LABELS, + PACKAGE_DIMENSION_VALUE_LABELS, + as_v4_row, + chronicle_v5_fact_rows, +) _FISCAL_MONTHS = [f"2025-{month:02d}" for month in range(4, 13)] + [ f"2026-{month:02d}" for month in range(1, 4) @@ -3903,6 +3911,88 @@ def test_hierarchy_rejects_conflicting_chronicle_dimension_value_labels() -> Non ) +_AGI_DIMENSION = "us:statutes/26/62#adjusted_gross_income" + + +def _chronicle_v5_agi_row(rows): + (row,) = ( + row + for row in rows + if row["observed_measure"]["source_measure_id"] == "adjusted_gross_income" + ) + return row + + +def _strings(value): + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for key, child in value.items(): + yield from _strings(key) + yield from _strings(child) + elif isinstance(value, (list, tuple)): + for child in value: + yield from _strings(child) + + +@pytest.mark.parametrize("row_contract", ["v5", "v4"]) +def test_chronicle_fact_hierarchy_takes_chronicle_label_not_package_wording( + row_contract, +) -> None: + """Each value keeps Chronicle's one label (chronicle#377). + + Chronicle's v5 sample row labels the AGI value ``all`` "All returns, + total" and keeps the IRS package's "All returns" in + ``package_dimension_value_labels``. The compiled target carries only + Chronicle's label, and the v4 restatement of the same row compiles to the + same target. + """ + rows = chronicle_v5_fact_rows() + v5_row = _chronicle_v5_agi_row(rows) + package_wording = v5_row[PACKAGE_DIMENSION_VALUE_LABELS][_AGI_DIMENSION]["all"] + chronicle_label = v5_row["dimension_value_labels"][_AGI_DIMENSION]["all"] + assert (package_wording, chronicle_label) == ("All returns", "All returns, total") + if row_contract == "v4": + rows = [as_v4_row(row) for row in rows] + + (spec,) = compile_ledger_target_references( + rows, [_exact_agi_reference()], country="us" + ).specs + + assert spec.hierarchy is not None + groupby = spec.hierarchy.dimensions[0] + assert (groupby.id, groupby.value_id, groupby.value_label) == ( + _AGI_DIMENSION, + "all", + chronicle_label, + ) + assert spec.metadata["ledger_layout_groupby_value_label"] == chronicle_label + assert package_wording not in set(_strings(asdict(spec))) + + +@pytest.mark.parametrize( + "malformed", + [value for _case, value in MALFORMED_PACKAGE_DIMENSION_VALUE_LABELS], + ids=[case for case, _value in MALFORMED_PACKAGE_DIMENSION_VALUE_LABELS], +) +def test_hierarchy_refuses_malformed_package_dimension_value_labels( + malformed, +) -> None: + """A selected v5 fact's package wording must be shaped as the v5 schema says. + + ``package_dimension_value_labels`` maps dimension id, then value id, to + non-empty text, like ``dimension_value_labels``; compilation refuses any + other shape, as it refuses a missing or blank Chronicle label. + """ + rows = chronicle_v5_fact_rows() + _chronicle_v5_agi_row(rows)[PACKAGE_DIMENSION_VALUE_LABELS] = deepcopy(malformed) + + with pytest.raises( + LedgerHierarchyMetadataError, match="package_dimension_value_labels" + ): + compile_ledger_target_references(rows, [_exact_agi_reference()], country="us") + + def test_multi_fact_hierarchy_requires_explicit_microcosm_target_label() -> None: first = _consumer_fact_row(value=10.0) second = _consumer_fact_row( diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/SOURCE.md b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/SOURCE.md new file mode 100644 index 0000000000..aaa2d8659e --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/SOURCE.md @@ -0,0 +1,22 @@ +# Chronicle consumer_fact.v5 fixtures + +Both files come from PolicyEngine/chronicle at `44eebe0378cd4aa708e283837e5a4ed1f7b18ef0`, +the merge of PolicyEngine/chronicle#377, which introduced `chronicle.consumer_fact.v5`. +Neither file differs at Chronicle `main` `28064d037562b47377193a5b75f4de120d3768a0` +(2026-11-04). + +- `consumer_fact.v5.schema.json`: `policyengine_chronicle/schemas/consumer_fact.v5.schema.json`, + byte for byte (sha256 + `acd33a091cd8d192be5d264e3e94c13e00afb6fd1972e57d4576ac55a54f6970`). Chronicle ships a + byte-identical copy at `docs/schemas/consumer_fact.v5.schema.json`. This digest is the + `consumer_fact_schema_sha256` a Chronicle consumer artifact built from v5 rows declares. +- `consumer_facts.jsonl`: Chronicle's sample consumer rows, `chronicle/fixtures/consumer_facts.jsonl` + (sha256 `450a22c689b7a086cfc03522aa4492d491efc6875a4661dbd851a2015506897a`): IRS SOI + Publication 1304 Table 1.1, tax year 2023, all returns, for the return count, adjusted gross + income and total income tax. Each row carries Chronicle's label for the adjusted gross income + value `all` ("All returns, total") in `dimension_value_labels` and + `layout.groupby_value_label`, and the package's own wording ("All returns") in + `package_dimension_value_labels`. Two changes from Chronicle's file: the raw-storage bucket + name in `source.raw_r2_bucket` and `source.raw_r2_uri` is replaced by the placeholder + `example-raw-bucket`, and each row is written with sorted keys. Every row still validates + against the schema above. diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_fact.v5.schema.json b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_fact.v5.schema.json new file mode 100644 index 0000000000..59ff1389e9 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_fact.v5.schema.json @@ -0,0 +1,502 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://policyengine.org/chronicle/schemas/consumer_fact.v5.schema.json", + "title": "Chronicle consumer fact contract row", + "description": "The v4 contract row with one label per dimension value (chronicle#376). Where Chronicle's register of dimension value labels carries a value, dimension_value_labels and layout.groupby_value_label carry the register's label, and package_dimension_value_labels carries the source package's own wording of that value when it differs, which is not checked against the publisher's file and may be text Chronicle wrote; elsewhere the labels are the package's, as in v4. Both stay display metadata: neither enters a fact key, and a v4 row restamped with this schema_version is a valid v5 row.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "aggregate_fact_key", + "semantic_fact_key", + "legacy_fact_key", + "source_release_key", + "source_series_key", + "observed_measure_key", + "dimension_set_key", + "universe_constraint_set_key", + "value", + "value_type", + "assertion", + "provenance_class", + "period", + "geography", + "entity", + "aggregation", + "observed_measure", + "dimensions", + "universe_constraints", + "source", + "lineage" + ], + "allOf": [ + { + "if": { + "properties": { + "provenance_class": { + "const": "survey_aggregate" + } + }, + "required": [ + "provenance_class" + ] + }, + "then": { + "required": [ + "survey_instrument" + ] + }, + "else": { + "not": { + "required": [ + "survey_instrument" + ] + } + } + } + ], + "properties": { + "schema_version": { + "const": "chronicle.consumer_fact.v5" + }, + "aggregate_fact_key": { + "type": "string", + "pattern": "^ledger\\.aggregate_fact\\.v2:[0-9a-f]{24}$" + }, + "semantic_fact_key": { + "type": "string", + "pattern": "^ledger\\.semantic_fact\\.v2:[0-9a-f]{24}$" + }, + "legacy_fact_key": { + "type": "string", + "pattern": "^ledger\\.fact\\.v1:[0-9a-f]{24}$" + }, + "source_release_key": { + "type": "string", + "pattern": "^ledger\\.source_release\\.v2:[0-9a-f]{24}$" + }, + "source_series_key": { + "type": "string", + "pattern": "^ledger\\.source_series\\.v2:[0-9a-f]{24}$" + }, + "observed_measure_key": { + "type": "string", + "pattern": "^ledger\\.observed_measure\\.v2:[0-9a-f]{24}$" + }, + "dimension_set_key": { + "type": "string", + "pattern": "^ledger\\.dimension_set\\.v2:[0-9a-f]{24}$" + }, + "universe_constraint_set_key": { + "type": "string", + "pattern": "^ledger\\.universe_constraint_set\\.v2:[0-9a-f]{24}$" + }, + "value": { + "type": [ + "integer", + "number", + "string", + "boolean" + ] + }, + "value_type": { + "enum": [ + "integer", + "number", + "decimal", + "string", + "boolean" + ] + }, + "period": { + "type": "object", + "additionalProperties": false, + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "Publisher-asserted period kind; the ledger.core.ALLOWED_PERIOD_TYPES vocabulary: calendar_year, tax_year, fiscal_year, academic_year, gas_year, month, quarter, week. Split-label years store the opening year of the publisher's label (UK FY2024-25 -> fiscal_year 2024; AY 2024/25 -> academic_year 2024). gas_year keeps DESNZ's year label: the opening year from 2016 (mid-May 2024 to mid-May 2025 -> gas_year 2024) and the closing year of the October-to-September years to 2015. Publishers labelling a fiscal year with a single year keep that label year (US federal FY2024 -> 2024). Quarter values use YYYY-QN and ISO week values use YYYY-Www, with exact dates in period_coverage." + }, + "value": { + "type": [ + "integer", + "string" + ] + } + } + }, + "geography": { + "type": "object", + "additionalProperties": false, + "required": [ + "level", + "id" + ], + "properties": { + "level": { + "type": "string" + }, + "id": { + "type": "string" + }, + "vintage": { + "type": "string" + }, + "name": { + "$ref": "#/$defs/label", + "description": "The name for this geography: Chronicle's register name where it carries the identifier, otherwise the publisher's own text; absent when neither states a name." + }, + "publisher_name": { + "$ref": "#/$defs/label", + "description": "The publisher's own name for this geography, present only where it differs from the register name carried in 'name'." + } + } + }, + "entity": { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "role": { + "type": "string" + } + } + }, + "aggregation": { + "type": "object", + "additionalProperties": false, + "required": [ + "method" + ], + "properties": { + "method": { + "type": "string" + }, + "denominator": { + "type": "string" + } + } + }, + "observed_measure": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_name", + "source_table", + "source_measure_id", + "source_concept", + "unit" + ], + "properties": { + "source_name": { + "type": "string" + }, + "source_table": { + "type": "string" + }, + "source_measure_id": { + "type": "string" + }, + "source_concept": { + "type": "string" + }, + "unit": { + "type": "string" + } + } + }, + "dimensions": { + "type": "object", + "additionalProperties": { + "type": [ + "integer", + "number", + "string", + "boolean" + ] + } + }, + "dimension_labels": { + "type": "object", + "description": "Label for each dimension id: every key of dimensions and, when layout.groupby_dimension is set, that id.", + "additionalProperties": { + "$ref": "#/$defs/label" + } + }, + "dimension_value_labels": { + "type": "object", + "description": "Label for each dimension value, keyed dimension id then value id: the label in Chronicle's register of dimension value labels when it carries the value, otherwise the source package's label. A value id is the value's string form: booleans lower-cased (true, false), integers, numbers and strings as Python str() writes them, lists and mappings JSON-serialised with sorted keys and no spaces. Covers every dimensions entry and, when layout.groupby_dimension is set, layout.groupby_value_id, whose label layout.groupby_value_label repeats.", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/label" + } + } + }, + "package_dimension_value_labels": { + "type": "object", + "description": "The source package's own wording of a dimension value, present only where it differs from the label this row carries for that value: the package row's label, else the package's dimension_value_labels declaration, else the text in the matching column of the parsed source row. This text is not checked against the publisher's file. It may be the publisher's wording (for example a row label that the row's guard compares with a header cell) or text Chronicle wrote (a parser's constant, a value id in an extract Chronicle prepared, or a package author's phrase); the publisher's own cells are reached through lineage.source_cell_keys. Keyed dimension id then value id like dimension_value_labels; the groupby value of a groupby axis that is also a filter with a different value is keyed by layout.groupby_value_id and differs from layout.groupby_value_label.", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/label" + } + } + }, + "universe_constraints": { + "type": "object", + "additionalProperties": false, + "required": [ + "domain" + ], + "properties": { + "domain": { + "type": "string" + }, + "constraints": { + "type": "array", + "items": { + "$ref": "#/$defs/constraint" + } + } + } + }, + "source": { + "type": "object", + "additionalProperties": true, + "required": [ + "source_name", + "source_table", + "source_file", + "vintage", + "extracted_at", + "extraction_method", + "source_sha256", + "source_size_bytes", + "raw_r2_uri" + ], + "properties": { + "source_name": { + "type": "string" + }, + "source_table": { + "type": "string" + }, + "source_file": { + "type": "string" + }, + "url": { + "type": "string" + }, + "vintage": { + "type": "string" + }, + "extracted_at": { + "type": "string" + }, + "extraction_method": { + "type": "string" + }, + "method_notes": { + "type": "string" + }, + "source_sha256": { + "type": "string" + }, + "source_size_bytes": { + "type": "integer" + }, + "raw_r2_bucket": { + "type": "string" + }, + "raw_r2_key": { + "type": "string" + }, + "raw_r2_uri": { + "type": "string" + } + } + }, + "lineage": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_record_id", + "source_cell_keys" + ], + "properties": { + "source_record_id": { + "type": "string" + }, + "source_cell_keys": { + "type": "array", + "items": { + "type": "string" + } + }, + "source_row_keys": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "layout": { + "type": "object", + "additionalProperties": true, + "properties": { + "groupby_dimension_label": { + "$ref": "#/$defs/label", + "description": "Label of layout.groupby_dimension; equals dimension_labels for that id." + } + } + }, + "label": { + "type": "string" + }, + "concept_alignment": { + "type": "object", + "additionalProperties": false, + "required": [ + "concept_alignment_key", + "source_concept", + "canonical_concept", + "relation", + "authority" + ], + "properties": { + "concept_alignment_key": { + "type": "string", + "pattern": "^ledger\\.concept_alignment\\.v2:[0-9a-f]{24}$" + }, + "source_concept": { + "type": "string" + }, + "canonical_concept": { + "type": "string" + }, + "relation": { + "type": "string" + }, + "authority": { + "type": "string" + }, + "evidence_url": { + "type": "string" + }, + "evidence_notes": { + "type": "string" + }, + "legal_vintage": { + "type": "string" + } + } + }, + "assertion": { + "type": "string", + "enum": [ + "observation", + "source_projection" + ], + "description": "Who asserted the value: a publisher-measured outcome (observation) or the publisher's own forward-looking estimate (source_projection). PolicyEngine-computed values are never consumer facts." + }, + "provenance_class": { + "type": "string", + "enum": [ + "administrative", + "census", + "survey_aggregate", + "model_output" + ], + "description": "Publisher measurement basis. Consumers can gate calibration inputs on this required, closed classification." + }, + "survey_instrument": { + "type": "string", + "pattern": ".*\\S.*", + "description": "Survey name required exactly when provenance_class is survey_aggregate." + }, + "period_coverage": { + "type": "object", + "additionalProperties": false, + "description": "Non-identity provenance for the fact's reference period: coverage dates, basis, the publisher's period label, and accounting basis.", + "properties": { + "start_date": { + "type": "string" + }, + "end_date": { + "type": "string" + }, + "basis": { + "type": "string", + "enum": [ + "calendar", + "tax", + "fiscal", + "survey_reference", + "projection_horizon" + ] + }, + "source_period_label": { + "type": "string" + }, + "accounting_basis": { + "type": "string", + "enum": [ + "cash", + "accrual" + ] + }, + "notes": { + "type": "string" + } + } + } + }, + "$defs": { + "label": { + "type": "string", + "pattern": ".*\\S.*" + }, + "constraint": { + "type": "object", + "additionalProperties": false, + "required": [ + "variable", + "operator", + "value", + "role" + ], + "properties": { + "variable": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": [ + "integer", + "number", + "string", + "boolean" + ] + }, + "unit": { + "type": "string" + }, + "role": { + "type": "string" + } + } + } + } +} diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_facts.jsonl b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_facts.jsonl new file mode 100644 index 0000000000..80043defef --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_facts.jsonl @@ -0,0 +1,3 @@ +{"aggregate_fact_key": "ledger.aggregate_fact.v2:13153fb8dfd73fa12d5715e9", "aggregation": {"method": "sum"}, "assertion": "observation", "dimension_labels": {"filing_status": "Filing status", "income_range": "Income range", "us:statutes/26/62#adjusted_gross_income": "Adjusted gross income"}, "dimension_set_key": "ledger.dimension_set.v2:a55266d8b1d53a8a7e49927d", "dimension_value_labels": {"filing_status": {"all": "All filing statuses"}, "income_range": {"all": "All income ranges"}, "us:statutes/26/62#adjusted_gross_income": {"all": "All returns, total"}}, "dimensions": {"filing_status": "all", "income_range": "all"}, "entity": {"name": "tax_unit", "role": "filing_unit"}, "geography": {"id": "0100000US", "level": "country", "name": "United States", "vintage": "2020_census"}, "label": "United States 2023 tax year count irs soi individual income tax returns for tax unit (filing status=all, income range=all) [irs_soi Publication 1304 Table 1.1 23in11si.xls tax_year_2023]", "layout": {"groupby_dimension": "us:statutes/26/62#adjusted_gross_income", "groupby_dimension_label": "Adjusted gross income", "groupby_ordinal": 0, "groupby_value_id": "all", "groupby_value_label": "All returns, total", "measure_id": "return_count", "measure_label": "Number of returns", "measure_ordinal": 0, "record_set_id": "irs_soi.ty2023.table_1_1", "record_set_spec_hash": "a1ecc4d67a3d281548a4660e", "record_set_spec_id": "irs_soi.table_1_1.v1", "source_column_id": "return_count", "source_row_id": "all", "table_record_kind": "total"}, "legacy_fact_key": "ledger.fact.v1:079f0697e3b090abf422781b", "lineage": {"source_cell_keys": ["ledger.source_cell.v1:82f39b2a5aeab53c1b717928"], "source_record_id": "irs_soi.ty2023.table_1_1.all.return_count", "source_row_keys": []}, "observed_measure": {"source_concept": "irs_soi.individual_income_tax_returns", "source_measure_id": "return_count", "source_name": "irs_soi", "source_table": "Publication 1304 Table 1.1", "unit": "count"}, "observed_measure_key": "ledger.observed_measure.v2:bc96277bae6ef31da4a5794c", "package_dimension_value_labels": {"us:statutes/26/62#adjusted_gross_income": {"all": "All returns"}}, "period": {"type": "tax_year", "value": 2023}, "provenance_class": "administrative", "schema_version": "chronicle.consumer_fact.v5", "semantic_fact_key": "ledger.semantic_fact.v2:1ae4c4f93f554ec7ca14d2ad", "source": {"extracted_at": "2026-05-04", "extraction_method": "CellSelectorSpec and SourceRecordSpec resolved from cells", "method_notes": "Each fact carries source_record_id, source_cell_keys, and source_row_keys when available.", "raw_r2_bucket": "example-raw-bucket", "raw_r2_key": "raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "raw_r2_uri": "r2://example-raw-bucket/raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "source_file": "23in11si.xls", "source_name": "irs_soi", "source_sha256": "842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17", "source_size_bytes": 54272, "source_table": "Publication 1304 Table 1.1", "url": "https://www.irs.gov/pub/irs-soi/23in11si.xls", "vintage": "tax_year_2023"}, "source_release_key": "ledger.source_release.v2:e418de50f65dc3f88fbcdd6c", "source_series_key": "ledger.source_series.v2:ac7ef40c4f9adbd84a82f24c", "universe_constraint_set_key": "ledger.universe_constraint_set.v2:18df86d027cde55a3f217b77", "universe_constraints": {"domain": "all_individual_income_tax_returns"}, "value": 160602107, "value_type": "integer"} +{"aggregate_fact_key": "ledger.aggregate_fact.v2:36ce90001adb50e8e9758e4f", "aggregation": {"method": "sum"}, "assertion": "observation", "concept_alignment": {"authority": "ledger-us", "canonical_concept": "us:statutes/26/62#adjusted_gross_income", "concept_alignment_key": "ledger.concept_alignment.v2:2e0ac323f1b261710e4397a1", "evidence_notes": "IRS SOI Table 1.1 reports adjusted gross income for individual income tax returns; IRC section 62 defines adjusted gross income. This Ledger assertion treats the SOI AGI column as exactly adopting that legal concept for the tax-year source record.", "evidence_url": "https://uscode.house.gov/view.xhtml?req=(title:26%20section:62%20edition:prelim)", "legal_vintage": "tax_year_2023", "relation": "exact", "source_concept": "irs_soi.adjusted_gross_income"}, "dimension_labels": {"filing_status": "Filing status", "income_range": "Income range", "us:statutes/26/62#adjusted_gross_income": "Adjusted gross income"}, "dimension_set_key": "ledger.dimension_set.v2:a55266d8b1d53a8a7e49927d", "dimension_value_labels": {"filing_status": {"all": "All filing statuses"}, "income_range": {"all": "All income ranges"}, "us:statutes/26/62#adjusted_gross_income": {"all": "All returns, total"}}, "dimensions": {"filing_status": "all", "income_range": "all"}, "entity": {"name": "tax_unit", "role": "filing_unit"}, "geography": {"id": "0100000US", "level": "country", "name": "United States", "vintage": "2020_census"}, "label": "United States 2023 tax year sum us:statutes/26/62#adjusted gross income for tax unit (filing status=all, income range=all) [irs_soi Publication 1304 Table 1.1 23in11si.xls tax_year_2023]", "layout": {"groupby_dimension": "us:statutes/26/62#adjusted_gross_income", "groupby_dimension_label": "Adjusted gross income", "groupby_ordinal": 0, "groupby_value_id": "all", "groupby_value_label": "All returns, total", "measure_id": "adjusted_gross_income", "measure_label": "Adjusted gross income", "measure_ordinal": 1, "record_set_id": "irs_soi.ty2023.table_1_1", "record_set_spec_hash": "a1ecc4d67a3d281548a4660e", "record_set_spec_id": "irs_soi.table_1_1.v1", "source_column_id": "adjusted_gross_income", "source_row_id": "all", "table_record_kind": "total"}, "legacy_fact_key": "ledger.fact.v1:7f8e9d3c64fec2670b571c3b", "lineage": {"source_cell_keys": ["ledger.source_cell.v1:c3d349e26856fb6224303675"], "source_record_id": "irs_soi.ty2023.table_1_1.all.adjusted_gross_income", "source_row_keys": []}, "observed_measure": {"source_concept": "irs_soi.adjusted_gross_income", "source_measure_id": "adjusted_gross_income", "source_name": "irs_soi", "source_table": "Publication 1304 Table 1.1", "unit": "usd"}, "observed_measure_key": "ledger.observed_measure.v2:ac9c3cef252ef7ead0066cb1", "package_dimension_value_labels": {"us:statutes/26/62#adjusted_gross_income": {"all": "All returns"}}, "period": {"type": "tax_year", "value": 2023}, "provenance_class": "administrative", "schema_version": "chronicle.consumer_fact.v5", "semantic_fact_key": "ledger.semantic_fact.v2:b83082b8e69c1ddc6fdebed2", "source": {"extracted_at": "2026-05-04", "extraction_method": "CellSelectorSpec and SourceRecordSpec resolved from cells", "method_notes": "Each fact carries source_record_id, source_cell_keys, and source_row_keys when available.", "raw_r2_bucket": "example-raw-bucket", "raw_r2_key": "raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "raw_r2_uri": "r2://example-raw-bucket/raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "source_file": "23in11si.xls", "source_name": "irs_soi", "source_sha256": "842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17", "source_size_bytes": 54272, "source_table": "Publication 1304 Table 1.1", "url": "https://www.irs.gov/pub/irs-soi/23in11si.xls", "vintage": "tax_year_2023"}, "source_release_key": "ledger.source_release.v2:e418de50f65dc3f88fbcdd6c", "source_series_key": "ledger.source_series.v2:ac7ef40c4f9adbd84a82f24c", "universe_constraint_set_key": "ledger.universe_constraint_set.v2:18df86d027cde55a3f217b77", "universe_constraints": {"domain": "all_individual_income_tax_returns"}, "value": 15286017359000, "value_type": "integer"} +{"aggregate_fact_key": "ledger.aggregate_fact.v2:0fa0e8c17cfe0ebd844c4f95", "aggregation": {"method": "sum"}, "assertion": "observation", "dimension_labels": {"filing_status": "Filing status", "income_range": "Income range", "us:statutes/26/62#adjusted_gross_income": "Adjusted gross income"}, "dimension_set_key": "ledger.dimension_set.v2:a55266d8b1d53a8a7e49927d", "dimension_value_labels": {"filing_status": {"all": "All filing statuses"}, "income_range": {"all": "All income ranges"}, "us:statutes/26/62#adjusted_gross_income": {"all": "All returns, total"}}, "dimensions": {"filing_status": "all", "income_range": "all"}, "entity": {"name": "tax_unit", "role": "filing_unit"}, "geography": {"id": "0100000US", "level": "country", "name": "United States", "vintage": "2020_census"}, "label": "United States 2023 tax year sum irs soi total income tax for tax unit (filing status=all, income range=all) [irs_soi Publication 1304 Table 1.1 23in11si.xls tax_year_2023]", "layout": {"groupby_dimension": "us:statutes/26/62#adjusted_gross_income", "groupby_dimension_label": "Adjusted gross income", "groupby_ordinal": 0, "groupby_value_id": "all", "groupby_value_label": "All returns, total", "measure_id": "total_income_tax", "measure_label": "Total income tax", "measure_ordinal": 2, "record_set_id": "irs_soi.ty2023.table_1_1", "record_set_spec_hash": "a1ecc4d67a3d281548a4660e", "record_set_spec_id": "irs_soi.table_1_1.v1", "source_column_id": "total_income_tax", "source_row_id": "all", "table_record_kind": "total"}, "legacy_fact_key": "ledger.fact.v1:ec48b42ac43674a2a2206b8b", "lineage": {"source_cell_keys": ["ledger.source_cell.v1:f38c48e6e2f23204eb997143"], "source_record_id": "irs_soi.ty2023.table_1_1.all.total_income_tax", "source_row_keys": []}, "observed_measure": {"source_concept": "irs_soi.total_income_tax", "source_measure_id": "total_income_tax", "source_name": "irs_soi", "source_table": "Publication 1304 Table 1.1", "unit": "usd"}, "observed_measure_key": "ledger.observed_measure.v2:c11ff13c2dfe8af0795a1194", "package_dimension_value_labels": {"us:statutes/26/62#adjusted_gross_income": {"all": "All returns"}}, "period": {"type": "tax_year", "value": 2023}, "provenance_class": "administrative", "schema_version": "chronicle.consumer_fact.v5", "semantic_fact_key": "ledger.semantic_fact.v2:f8b2801687c0ba560054ed36", "source": {"extracted_at": "2026-05-04", "extraction_method": "CellSelectorSpec and SourceRecordSpec resolved from cells", "method_notes": "Each fact carries source_record_id, source_cell_keys, and source_row_keys when available.", "raw_r2_bucket": "example-raw-bucket", "raw_r2_key": "raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "raw_r2_uri": "r2://example-raw-bucket/raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "source_file": "23in11si.xls", "source_name": "irs_soi", "source_sha256": "842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17", "source_size_bytes": 54272, "source_table": "Publication 1304 Table 1.1", "url": "https://www.irs.gov/pub/irs-soi/23in11si.xls", "vintage": "tax_year_2023"}, "source_release_key": "ledger.source_release.v2:e418de50f65dc3f88fbcdd6c", "source_series_key": "ledger.source_series.v2:ac7ef40c4f9adbd84a82f24c", "universe_constraint_set_key": "ledger.universe_constraint_set.v2:18df86d027cde55a3f217b77", "universe_constraints": {"domain": "all_individual_income_tax_returns"}, "value": 2147909818000, "value_type": "integer"} diff --git a/test_support/microcosm_build/chronicle_feed.py b/test_support/microcosm_build/chronicle_feed.py index 7c59a11d84..a2982c59c6 100644 --- a/test_support/microcosm_build/chronicle_feed.py +++ b/test_support/microcosm_build/chronicle_feed.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import hashlib import json from collections.abc import Mapping, Sequence @@ -9,12 +10,80 @@ from typing import Any from microcosm.build.ledger_artifact import CONSUMER_ARTIFACT_SCHEMA_VERSION +from test_support.paths import paths_for + +#: Chronicle's v5 consumer-fact schema and sample rows, taken from +#: PolicyEngine/chronicle at 44eebe037 (the merge of chronicle#377); the +#: directory's SOURCE.md records the paths and digests. +CHRONICLE_V5_FIXTURES = ( + paths_for("microcosm-build").tests / "fixtures/chronicle_consumer_fact_v5" +) +CHRONICLE_V5_SCHEMA_PATH = CHRONICLE_V5_FIXTURES / "consumer_fact.v5.schema.json" +CHRONICLE_V5_FACTS_PATH = CHRONICLE_V5_FIXTURES / "consumer_facts.jsonl" + +#: The field chronicle#377 added to the v5 row: a package's own wording of a +#: dimension value where it differs from the label the row exports. +PACKAGE_DIMENSION_VALUE_LABELS = "package_dimension_value_labels" + +_AGI_DIMENSION = "us:statutes/26/62#adjusted_gross_income" + +#: Values of ``package_dimension_value_labels`` that the v5 schema refuses: the +#: field must map each dimension id to an object of value id -> non-blank text. +MALFORMED_PACKAGE_DIMENSION_VALUE_LABELS: tuple[tuple[str, object], ...] = ( + ("not_an_object", ["All returns"]), + ("null", None), + ("dimension_not_an_object", {_AGI_DIMENSION: "All returns"}), + ("dimension_null", {_AGI_DIMENSION: None}), + ("label_empty", {_AGI_DIMENSION: {"all": ""}}), + ("label_blank", {_AGI_DIMENSION: {"all": " "}}), + ("label_not_text", {_AGI_DIMENSION: {"all": 1}}), + ("label_null", {_AGI_DIMENSION: {"all": None}}), +) + + +def chronicle_v5_schema() -> dict[str, Any]: + """Chronicle's v5 consumer-fact JSON Schema, parsed.""" + return json.loads(CHRONICLE_V5_SCHEMA_PATH.read_bytes()) + + +def chronicle_v5_schema_sha256() -> str: + """The digest a v5 artifact manifest declares as its fact schema digest.""" + return hashlib.sha256(CHRONICLE_V5_SCHEMA_PATH.read_bytes()).hexdigest() + + +def chronicle_v5_schema_version() -> str: + """The per-row ``schema_version`` the v5 schema requires, read from it.""" + return chronicle_v5_schema()["properties"]["schema_version"]["const"] + + +def chronicle_v5_fact_rows() -> list[dict[str, Any]]: + """Fresh copies of Chronicle's sample v5 rows.""" + return [ + json.loads(line) + for line in CHRONICLE_V5_FACTS_PATH.read_text().splitlines() + if line.strip() + ] + + +def as_v4_row(row: Mapping[str, Any]) -> dict[str, Any]: + """The v4 row a v5 row restates: the v4 stamp, without the v5-only field.""" + v4_row = copy.deepcopy(dict(row)) + v4_row.pop(PACKAGE_DIMENSION_VALUE_LABELS, None) + v4_row["schema_version"] = "chronicle.consumer_fact.v4" + return v4_row def write_pinned_artifact( - path: Path, rows: Sequence[Mapping[str, Any]] + path: Path, + rows: Sequence[Mapping[str, Any]], + *, + consumer_fact_schema_sha256: str = "ab" * 32, ) -> tuple[Path, dict[str, Any]]: - """Write a small artifact and return its complete six-field feed pin.""" + """Write a small artifact and return its complete six-field feed pin. + + Rows without a ``schema_version`` are stamped v4; a row's own stamp is + kept, so v5 rows write a v5 artifact. + """ path.mkdir(parents=True, exist_ok=True) facts = [{"schema_version": "chronicle.consumer_fact.v4", **row} for row in rows] facts_bytes = "".join( @@ -27,7 +96,7 @@ def write_pinned_artifact( "facts_sha256": hashlib.sha256(facts_bytes).hexdigest(), "fact_row_count": len(facts), "consumer_fact_schema_versions": versions, - "consumer_fact_schema_sha256": "ab" * 32, + "consumer_fact_schema_sha256": consumer_fact_schema_sha256, } manifest_bytes = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode() (path / "manifest.json").write_bytes(manifest_bytes) From 5dd751a40bda60d47e5aca835d42ec846ca217f9 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:04:01 +0400 Subject: [PATCH 2/8] Accept Chronicle consumer_fact.v5 facts Chronicle main writes every consumer fact as chronicle.consumer_fact.v5 since PolicyEngine/chronicle#377 (44eebe037). Microcosm declared row contracts only up to v4, so the next re-pin of any country's feed onto Chronicle main needed this first. - chronicle_epoch declares CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V5 in the chronicle epoch, so v5 joins ACCEPTED_CONSUMER_FACT_SCHEMA_VERSIONS beside v1 to v4, matching Chronicle's CONSUMER_FACT_ROW_CONTRACTS. - ledger_targets checks the optional package_dimension_value_labels field where it checks dimension_value_labels: on each fact a hierarchy target selects, the field, when present, must map each dimension id to an object of value id to non-empty text, the v5 schema's shape; any other shape raises LedgerHierarchyMetadataError. No code takes a label from the field, so each value keeps Chronicle's one label from dimension_value_labels and layout.groupby_value_label. - The loader is unchanged: it keeps every published field on the parsed rows, the new one included. - docs/calibration-target-hierarchy.md and docs/uk-chronicle-feed-repin.md say so; a changelog fragment records it. No feed pin moves. Closes #1509 Co-Authored-By: Claude Opus 5.5 --- .../1509-chronicle-consumer-fact-v5.added.md | 1 + docs/calibration-target-hierarchy.md | 8 +++ docs/uk-chronicle-feed-repin.md | 6 ++- .../src/microcosm/build/chronicle_epoch.py | 28 +++++++--- .../src/microcosm/build/ledger_targets.py | 53 +++++++++++++++++++ 5 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 changelog.d/1509-chronicle-consumer-fact-v5.added.md diff --git a/changelog.d/1509-chronicle-consumer-fact-v5.added.md b/changelog.d/1509-chronicle-consumer-fact-v5.added.md new file mode 100644 index 0000000000..da000f5a0a --- /dev/null +++ b/changelog.d/1509-chronicle-consumer-fact-v5.added.md @@ -0,0 +1 @@ +Accept Chronicle's `chronicle.consumer_fact.v5` facts (microcosm#1509), the row contract Chronicle `main` writes since PolicyEngine/chronicle#377 (`44eebe037`). `microcosm.build.chronicle_epoch` declares v5 beside v1 to v4, so it is in `ACCEPTED_CONSUMER_FACT_SCHEMA_VERSIONS` and belongs to the chronicle epoch; v2 to v4 keep loading. v5 adds an optional `package_dimension_value_labels` field, a source package's own wording of a dimension value where it differs from the label the row exports. The loader keeps it on the parsed rows, as it keeps every published field, and Microcosm never takes a label from it: each value keeps the one label Chronicle gives it in `dimension_value_labels` and `layout.groupby_value_label`. Target compilation refuses a selected fact whose field is not an object of dimension id to an object of value id to non-empty text, the shape the v5 schema requires. No feed pin moves: the UK, US, NZ and Belgian feeds stay on their current Chronicle commits until their own re-pins. diff --git a/docs/calibration-target-hierarchy.md b/docs/calibration-target-hierarchy.md index 163c677e3f..5641eb1ac2 100644 --- a/docs/calibration-target-hierarchy.md +++ b/docs/calibration-target-hierarchy.md @@ -89,6 +89,14 @@ metadata, not hints for constructing display text. exactly one non-empty `dimension_labels` entry and one non-empty `dimension_value_labels` entry for each selected dimension and value. Missing or conflicting labels fail compilation. +- A `chronicle.consumer_fact.v5` row (PolicyEngine/chronicle#377) may also + carry `package_dimension_value_labels`: a source package's own wording of a + value, present only where it differs from the label the row exports. + Microcosm never takes a label from it, so each value keeps the one label + Chronicle gives it in `dimension_value_labels` and + `layout.groupby_value_label`. When the field is present on a selected fact + it must map each dimension id to an object of value id to non-empty text, + as the v5 schema requires; any other shape fails compilation. - A layout dimension and a fact dimension merge only when their identifiers are exactly equal. Similar labels do not establish identity. - A multi-fact aggregate inherits only dimensions present with the same value diff --git a/docs/uk-chronicle-feed-repin.md b/docs/uk-chronicle-feed-repin.md index f5958ea924..24ef2fbe74 100644 --- a/docs/uk-chronicle-feed-repin.md +++ b/docs/uk-chronicle-feed-repin.md @@ -24,7 +24,11 @@ restate it: `tests/engine_free/uk/test_uk_chronicle_feed.py` (the pin and the consumer-fact schema digest) and `tests/engine_free/uk/test_uk_full_targets.py` (`test_current_national_and_local_pins_share_one_reviewed_identity`). A schema digest moves for every country whose feed is built after the Chronicle change, -so the NZ pin (`nz/chronicle_feed.json`) moves at its own next re-pin. Then +so the NZ pin (`nz/chronicle_feed.json`) moves at its own next re-pin. A feed +whose rows carry a newer per-row `schema_version` also needs that row contract +declared in `microcosm.build.chronicle_epoch` first. Microcosm declares +`ledger.consumer_fact.v1` and `chronicle.consumer_fact.v2` to `v5` +(microcosm#1509); the v5 schema's digest is `acd33a09…`. Then regenerate. Vendor the per-concern fact resources (step 5) before the national generator: the generator refuses to compile against a vendored resource whose feed identity diff --git a/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py b/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py index 2bba59d1e5..e5eac8f85e 100644 --- a/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py +++ b/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py @@ -67,6 +67,7 @@ "CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION", "CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V3", "CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V4", + "CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V5", "CHRONICLE_EPOCH", "CHRONICLE_FACT_KEY_DOMAINS", "CHRONICLE_NAMESPACES", @@ -206,17 +207,25 @@ def _declare(identity: str, *, epoch: str, kind: str) -> DeclaredIdentity: #: The per-row consumer-fact ``schema_version`` Chronicle stamped before #: chronicle#261, its chronicle-era successor (the v2 row adds the dimension #: and value labels), the v3 row chronicle#266 (PR #267) added (the v2 row -#: plus the publisher's ``geography.name``), and the v4 row chronicle#281 -#: (PR #282) emits today: the v3 row with ``geography.name`` resolved to one -#: spelling per identifier where Chronicle's register carries it, the -#: publisher's own text moving to an optional ``geography.publisher_name``. -#: Every declared row contract keeps loading; the name and the labels are -#: display metadata and never enter a fact key, so the keys of a v4 row are -#: the v2 bytes. +#: plus the publisher's ``geography.name``), the v4 row chronicle#281 (PR +#: #282) added (the v3 row with ``geography.name`` resolved to one spelling +#: per identifier where Chronicle's register carries it, the publisher's own +#: text moving to an optional ``geography.publisher_name``), and the v5 row +#: chronicle#376 (PR #377, merged as 44eebe037) emits today: the v4 row with +#: ``dimension_value_labels`` and ``layout.groupby_value_label`` carrying the +#: one label Chronicle's register gives each dimension value, and an optional +#: ``package_dimension_value_labels`` keeping a source package's own wording +#: where it differs. Microcosm never takes a label from that field +#: (:mod:`microcosm.build.ledger_targets` checks only its shape). Every +#: declared row contract keeps loading; the names and the labels are display +#: metadata and never enter a fact key, so the keys of a v5 row are the v2 +#: bytes. Chronicle's own list is ``CONSUMER_FACT_ROW_CONTRACTS`` in +#: ``chronicle/epoch.py``. LEDGER_CONSUMER_FACT_SCHEMA_VERSION = "ledger.consumer_fact.v1" CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION = "chronicle.consumer_fact.v2" CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V3 = "chronicle.consumer_fact.v3" CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V4 = "chronicle.consumer_fact.v4" +CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V5 = "chronicle.consumer_fact.v5" #: Fact-key domains, ledger-era spelling beside its chronicle-era successor. #: The ledger column is what Chronicle's ``main`` hashes with today; the @@ -282,6 +291,11 @@ def _declare(identity: str, *, epoch: str, kind: str) -> DeclaredIdentity: epoch=CHRONICLE_EPOCH, kind=CONSUMER_FACT_SCHEMA_IDENTITY_KIND, ), + _declare( + CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V5, + epoch=CHRONICLE_EPOCH, + kind=CONSUMER_FACT_SCHEMA_IDENTITY_KIND, + ), *( _declare(identity, epoch=epoch, kind=FACT_KEY_IDENTITY_KIND) for ledger_domain, chronicle_domain in _FACT_KEY_DOMAIN_DECLARATIONS diff --git a/packages/microcosm-build/src/microcosm/build/ledger_targets.py b/packages/microcosm-build/src/microcosm/build/ledger_targets.py index c7ed78167a..509b06de73 100644 --- a/packages/microcosm-build/src/microcosm/build/ledger_targets.py +++ b/packages/microcosm-build/src/microcosm/build/ledger_targets.py @@ -175,6 +175,17 @@ class LedgerHierarchyMetadataError(ValueError): """A selected Chronicle fact cannot populate the required hierarchy.""" +#: The optional ``chronicle.consumer_fact.v5`` field (chronicle#376, PR #377) +#: that keeps a source package's own wording of a dimension value where it +#: differs from the label the row exports. It is keyed like +#: ``dimension_value_labels`` (dimension id, then value id, then text). +#: Microcosm never takes a label from it: ``dimension_value_labels`` and +#: ``layout.groupby_value_label`` carry the one label Chronicle gives each +#: value, so the hierarchy check validates only this field's shape. +PACKAGE_DIMENSION_VALUE_LABELS_FIELD = "package_dimension_value_labels" +_ABSENT = object() + + def hierarchy_seed_from_catalog( hierarchy: Mapping[str, object], category_id: str, @@ -1481,6 +1492,9 @@ def _validate_chronicle_hierarchy_labels( f"Ledger target reference {reference_name!r}: every selected " f"Chronicle fact requires a non-empty label; missing for {fact_id!r}." ) + _validate_package_dimension_value_labels( + fact, fact_id=fact_id, reference_name=reference_name + ) raw_dimensions = _dimensions(fact) dimensions = {str(key): value for key, value in raw_dimensions.items()} @@ -1554,6 +1568,45 @@ def _validate_chronicle_hierarchy_labels( ) +def _validate_package_dimension_value_labels( + fact: object, *, fact_id: str, reference_name: str +) -> None: + """Refuse a malformed v5 package-wording map; never read it as a label. + + The field is optional. When present it must have the v5 schema's shape: + an object mapping each dimension id to an object of value id to + non-empty text, the shape ``dimension_value_labels`` has. + """ + + if isinstance(fact, Mapping): + package_labels = fact.get(PACKAGE_DIMENSION_VALUE_LABELS_FIELD, _ABSENT) + else: + package_labels = getattr(fact, PACKAGE_DIMENSION_VALUE_LABELS_FIELD, _ABSENT) + if package_labels is _ABSENT: + return + prefix = ( + f"Ledger target reference {reference_name!r}: Chronicle fact {fact_id!r} " + f"{PACKAGE_DIMENSION_VALUE_LABELS_FIELD}" + ) + if not isinstance(package_labels, Mapping): + raise LedgerHierarchyMetadataError( + f"{prefix} must be an object keyed by dimension id, got " + f"{type(package_labels).__name__}." + ) + for dimension_id, value_labels in package_labels.items(): + if not isinstance(value_labels, Mapping): + raise LedgerHierarchyMetadataError( + f"{prefix} dimension {dimension_id!r} must be an object keyed by " + f"value id, got {type(value_labels).__name__}." + ) + for value_id, label in value_labels.items(): + if not isinstance(label, str) or not label.strip(): + raise LedgerHierarchyMetadataError( + f"{prefix} dimension {dimension_id!r} value {value_id!r} " + f"requires non-empty text, got {label!r}." + ) + + def _requires_microcosm_target_label( facts: tuple[object, ...], *, From e70084b1222dca4d6481f62a08c976ef28296b1c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:35:24 +0400 Subject: [PATCH 3/8] Test that feed-pin checks refuse an undeclared Chronicle row contract Review round 1 of #1510 found that nothing outside the tests read the accepted row-contract list, so a v5 feed already loaded and the declaration refused nothing. Anthony decided to enforce it. These tests run before that change; 12 of the 36 cases fail, each on a pytest.raises that does not raise: - validate_chronicle_feed_pin, directly and through the loader, refuses an artifact whose rows and pin agree on chronicle.consumer_fact.v6, or mix it in beside v4; the message names the undeclared version, says to declare it in microcosm.build.chronicle_epoch, and lists the declared ones (test_ledger_artifact.py); - chronicle_feed_identity, which reference factors, money normalization and the transport composer use, refuses a v6, a v5-and-v6 or an arch.consumer_fact.v1 list, and a list that is a string; - the targets.compile and takeup.compare kernels refuse a v6 feed under its own pin (test_transport_target_kernels.py); - the UK and US pin loaders refuse a committed pin naming v6 (test_uk_chronicle_feed.py, test_us_chronicle_feed.py). The other 24 cases pass before the change and guard what must keep working or already holds: ledger.consumer_fact.v1 and v2 to v5 pass a full feed pin; every committed pin (UK, US, NZ, BE) and every resource restating one (the NZ and BE reference inventories and ageing files, the NZ MSD reconciliation) names declared versions and passes chronicle_feed_identity; empty package wording objects compile; a malformed entry under a dimension the target does not select is refused. The fixtures move to tests/fixtures/chronicle_consumer_fact and gain Chronicle's v4 schema (d953306d...) and its v4 sample rows from 1a3f2c4, main just before chronicle#377, which label the AGI value "All returns". The v4 label and load tests use those rows in place of v5 rows restamped as v4. Refs #1509 Co-Authored-By: Claude Opus 5.5 --- .../shared/test_chronicle_epoch.py | 10 +- .../shared/test_ledger_artifact.py | 201 ++++++- .../engine_free/shared/test_ledger_targets.py | 103 +++- .../shared/test_transport_target_kernels.py | 27 + .../engine_free/uk/test_uk_chronicle_feed.py | 28 + .../engine_free/us/test_us_chronicle_feed.py | 26 + .../chronicle_consumer_fact/SOURCE.md | 41 ++ .../consumer_fact.v4.schema.json | 492 ++++++++++++++++++ .../consumer_fact.v5.schema.json | 0 .../consumer_facts.v4.jsonl | 3 + .../consumer_facts.v5.jsonl} | 0 .../chronicle_consumer_fact_v5/SOURCE.md | 22 - .../microcosm_build/chronicle_feed.py | 51 +- 13 files changed, 935 insertions(+), 69 deletions(-) create mode 100644 packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/SOURCE.md create mode 100644 packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_fact.v4.schema.json rename packages/microcosm-build/tests/fixtures/{chronicle_consumer_fact_v5 => chronicle_consumer_fact}/consumer_fact.v5.schema.json (100%) create mode 100644 packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_facts.v4.jsonl rename packages/microcosm-build/tests/fixtures/{chronicle_consumer_fact_v5/consumer_facts.jsonl => chronicle_consumer_fact/consumer_facts.v5.jsonl} (100%) delete mode 100644 packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/SOURCE.md diff --git a/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py b/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py index 2a07cbcb66..1859697a64 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py @@ -444,10 +444,12 @@ def test_v5_row_contract_is_declared_like_v4() -> None: """chronicle#377's v5 row is a declared chronicle-era row contract. v5 is the v4 row plus the optional ``package_dimension_value_labels``, - so it is declared exactly as v4 is: the same identity kind, the - chronicle epoch, and named in the accepted-versions message a refusal - prints. Chronicle's own table (``CONSUMER_FACT_ROW_CONTRACTS`` in - ``chronicle/epoch.py``) ends at v5 from 44eebe037. + so it is declared exactly as v4 is: the same identity kind, the chronicle + epoch, and a place in the accepted-versions list that the feed-pin checks + print when they refuse an undeclared row contract + (``test_ledger_artifact.py``). Chronicle's own table + (``CONSUMER_FACT_ROW_CONTRACTS`` in ``chronicle/epoch.py``) ends at v5 + from 44eebe037. """ v4 = declared_identity(CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION_V4) v5 = declared_identity(_CHRONICLE_V5_ROW_SCHEMA_ID) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py b/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py index e2f3c1a3f3..ca0ecc6404 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py @@ -29,6 +29,7 @@ LedgerConsumerArtifact, LexicalFloat, add_ledger_artifact_args, + chronicle_feed_identity, load_ledger_consumer_artifact, resolve_ledger_artifact, validate_chronicle_feed_pin, @@ -37,7 +38,9 @@ from test_support.microcosm_build.chronicle_feed import ( MALFORMED_PACKAGE_DIMENSION_VALUE_LABELS, PACKAGE_DIMENSION_VALUE_LABELS, - as_v4_row, + chronicle_v4_fact_rows, + chronicle_v4_schema, + chronicle_v4_schema_sha256, chronicle_v5_fact_rows, chronicle_v5_schema, chronicle_v5_schema_sha256, @@ -392,6 +395,17 @@ def test_complete_feed_pin_accepts_exact_match(tmp_path): _CHRONICLE_V4_SCHEMA_SHA256 = ( "d953306d6f01fbc64db4372d0262d3012ee590269f8a25b276b18dca0626c6a0" ) +#: Every per-row contract Chronicle has issued, oldest first, as Chronicle's +#: ``CONSUMER_FACT_ROW_CONTRACTS`` lists them at 44eebe037. +_DECLARED_ROW_CONTRACTS = ( + "ledger.consumer_fact.v1", + "chronicle.consumer_fact.v2", + "chronicle.consumer_fact.v3", + "chronicle.consumer_fact.v4", + "chronicle.consumer_fact.v5", +) +#: A row contract nobody has declared: the next version Chronicle might issue. +_UNDECLARED_ROW_CONTRACT = "chronicle.consumer_fact.v6" def test_chronicle_v5_fixture_is_valid_under_chronicle_v5_schema(): @@ -418,6 +432,32 @@ def test_chronicle_v5_fixture_is_valid_under_chronicle_v5_schema(): assert not validator.is_valid(row), case +def test_chronicle_v4_fixture_is_valid_under_chronicle_v4_schema(): + """The v4 fixture is Chronicle's own v4 rows, from just before chronicle#377. + + They carry v4's label for the AGI value ``all``, the package's "All + returns", and v4's schema refuses the v5-only field. + """ + schema = chronicle_v4_schema() + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema) + rows = chronicle_v4_fact_rows() + + assert chronicle_v4_schema_sha256() == _CHRONICLE_V4_SCHEMA_SHA256 + assert len(rows) == 3 + for row in rows: + assert row["schema_version"] == "chronicle.consumer_fact.v4" + assert PACKAGE_DIMENSION_VALUE_LABELS not in row + validator.validate(row) + with_v5_field = { + **rows[0], + PACKAGE_DIMENSION_VALUE_LABELS: chronicle_v5_fact_rows()[0][ + PACKAGE_DIMENSION_VALUE_LABELS + ], + } + assert not validator.is_valid(with_v5_field) + + def test_chronicle_v5_artifact_loads_under_its_feed_pin(tmp_path): """A v5 artifact loads under a full feed pin, and its rows stay as published. @@ -447,11 +487,11 @@ def test_chronicle_v5_artifact_loads_under_its_feed_pin(tmp_path): def test_chronicle_v4_artifact_still_loads_under_its_feed_pin(tmp_path): - rows = [as_v4_row(row) for row in chronicle_v5_fact_rows()] + rows = chronicle_v4_fact_rows() artifact_dir, pin = write_pinned_artifact( tmp_path / "artifact", rows, - consumer_fact_schema_sha256=_CHRONICLE_V4_SCHEMA_SHA256, + consumer_fact_schema_sha256=chronicle_v4_schema_sha256(), ) artifact = load_ledger_consumer_artifact(artifact_dir, feed_pin=pin) @@ -463,6 +503,161 @@ def test_chronicle_v4_artifact_still_loads_under_its_feed_pin(tmp_path): assert not any(PACKAGE_DIMENSION_VALUE_LABELS in row for row in artifact.facts) +@pytest.mark.parametrize("row_contract", _DECLARED_ROW_CONTRACTS) +def test_feed_pin_accepts_each_declared_row_contract(tmp_path, row_contract): + artifact_dir, pin = write_pinned_artifact( + tmp_path / "artifact", [_fact_row(schema_version=row_contract)] + ) + + artifact = load_ledger_consumer_artifact(artifact_dir, feed_pin=pin) + + validate_chronicle_feed_pin(artifact, pin) + assert artifact.fact_schema_versions == (row_contract,) + + +def _assert_undeclared_refusal(message: str, undeclared: list[str]) -> None: + assert repr(undeclared) in message + assert "microcosm.build.chronicle_epoch" in message + for declared in _DECLARED_ROW_CONTRACTS: + assert declared in message + + +def test_feed_pin_refuses_an_undeclared_row_contract(tmp_path): + """A pin and an artifact that agree on an undeclared row contract are refused. + + The exact comparison passes, because both name the same version; the + accepted-contract check then refuses it, naming the version and where to + declare it. + """ + artifact_dir, pin = write_pinned_artifact( + tmp_path / "artifact", [_fact_row(schema_version=_UNDECLARED_ROW_CONTRACT)] + ) + assert pin["consumer_fact_schema_versions"] == [_UNDECLARED_ROW_CONTRACT] + artifact = load_ledger_consumer_artifact(artifact_dir) + + with pytest.raises(ValueError) as direct: + validate_chronicle_feed_pin(artifact, pin) + with pytest.raises(ValueError) as loaded: + load_ledger_consumer_artifact(artifact_dir, feed_pin=pin) + + for refusal in (direct.value, loaded.value): + assert not isinstance(refusal, ChronicleFeedPinMismatchError) + _assert_undeclared_refusal(str(refusal), [_UNDECLARED_ROW_CONTRACT]) + + +def test_feed_pin_refuses_rows_that_mix_in_an_undeclared_row_contract(tmp_path): + artifact_dir, pin = write_pinned_artifact( + tmp_path / "artifact", + [ + _fact_row(schema_version="chronicle.consumer_fact.v4"), + _fact_row( + aggregate_fact_key="ledger.aggregate_fact.v2:def456", + schema_version=_UNDECLARED_ROW_CONTRACT, + ), + ], + ) + assert pin["consumer_fact_schema_versions"] == [ + "chronicle.consumer_fact.v4", + _UNDECLARED_ROW_CONTRACT, + ] + + with pytest.raises(ValueError) as refusal: + load_ledger_consumer_artifact(artifact_dir, feed_pin=pin) + + _assert_undeclared_refusal(str(refusal.value), [_UNDECLARED_ROW_CONTRACT]) + + +@pytest.mark.parametrize( + ("versions", "undeclared"), + [ + ([_UNDECLARED_ROW_CONTRACT], [_UNDECLARED_ROW_CONTRACT]), + ( + ["chronicle.consumer_fact.v5", _UNDECLARED_ROW_CONTRACT], + [_UNDECLARED_ROW_CONTRACT], + ), + (["arch.consumer_fact.v1"], ["arch.consumer_fact.v1"]), + ], + ids=["undeclared_only", "declared_and_undeclared", "foreign_namespace"], +) +def test_feed_identity_refuses_an_undeclared_row_contract( + tmp_path, versions, undeclared +): + """Factor declarations and restated identities go through the same check. + + ``chronicle_feed_identity`` selects the reviewed fields of a feed pin for + reference factors, money normalization and the transport composer. + """ + _, pin = write_pinned_artifact(tmp_path / "artifact", [_fact_row()]) + pin["source_commit"] = "c" * 40 + assert chronicle_feed_identity(pin)["consumer_fact_schema_versions"] == [ + "chronicle.consumer_fact.v4" + ] + + with pytest.raises(ValueError) as refusal: + chronicle_feed_identity({**pin, "consumer_fact_schema_versions": versions}) + + _assert_undeclared_refusal(str(refusal.value), undeclared) + + +def test_feed_identity_refuses_a_row_contract_list_that_is_not_a_list(tmp_path): + _, pin = write_pinned_artifact(tmp_path / "artifact", [_fact_row()]) + pin["source_commit"] = "c" * 40 + + with pytest.raises( + ValueError, match="consumer_fact_schema_versions must be a list" + ): + chronicle_feed_identity( + {**pin, "consumer_fact_schema_versions": "chronicle.consumer_fact.v4"} + ) + + +def _committed_feed_identities(): + """``(resource, path, object)`` for each committed object naming row contracts.""" + build_root = paths_for("microcosm-build").package / "src/microcosm/build" + found = [] + + def walk(node, resource, location): + if isinstance(node, dict): + if "consumer_fact_schema_versions" in node: + found.append((resource, location, node)) + for key, child in node.items(): + walk(child, resource, f"{location}/{key}") + elif isinstance(node, list): + for index, child in enumerate(node): + walk(child, resource, f"{location}[{index}]") + + for path in sorted(build_root.glob("*/*.json")): + resource = path.relative_to(build_root).as_posix() + walk(json.loads(path.read_text()), resource, "") + return found + + +def test_committed_feed_pins_and_restated_identities_name_declared_row_contracts(): + """The UK, US, NZ and BE pins, and every resource restating one, still pass.""" + identities = _committed_feed_identities() + resources = {resource for resource, _location, _identity in identities} + + assert { + "uk/chronicle_feed.json", + "us/chronicle_feed.json", + "nz/chronicle_feed.json", + "be/chronicle_feed.json", + "be/reference_inventory.json", + "be/reference_aging.json", + "nz/reference_inventory.json", + "nz/reference_aging.json", + "nz/msd_pre_activation_reconciliation.json", + } <= resources + for resource, location, identity in identities: + versions = identity["consumer_fact_schema_versions"] + assert all( + is_accepted_consumer_fact_schema_version(version) for version in versions + ), (resource, location) + assert chronicle_feed_identity(identity)["consumer_fact_schema_versions"] == ( + versions + ), (resource, location) + + def test_feed_pin_rejects_changed_facts_with_self_consistent_manifest(tmp_path): artifact_dir, pin = write_pinned_artifact(tmp_path / "artifact", [_fact_row()]) facts_path = artifact_dir / "consumer_facts.jsonl" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ledger_targets.py b/packages/microcosm-build/tests/engine_free/shared/test_ledger_targets.py index 8852ae342b..dab0257325 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ledger_targets.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ledger_targets.py @@ -42,7 +42,7 @@ from test_support.microcosm_build.chronicle_feed import ( MALFORMED_PACKAGE_DIMENSION_VALUE_LABELS, PACKAGE_DIMENSION_VALUE_LABELS, - as_v4_row, + chronicle_v4_fact_rows, chronicle_v5_fact_rows, ) @@ -3914,7 +3914,7 @@ def test_hierarchy_rejects_conflicting_chronicle_dimension_value_labels() -> Non _AGI_DIMENSION = "us:statutes/26/62#adjusted_gross_income" -def _chronicle_v5_agi_row(rows): +def _chronicle_agi_row(rows): (row,) = ( row for row in rows @@ -3935,31 +3935,30 @@ def _strings(value): yield from _strings(child) -@pytest.mark.parametrize("row_contract", ["v5", "v4"]) -def test_chronicle_fact_hierarchy_takes_chronicle_label_not_package_wording( - row_contract, -) -> None: +def _compiled_agi_target(rows): + (spec,) = compile_ledger_target_references( + rows, [_exact_agi_reference()], country="us" + ).specs + assert spec.hierarchy is not None + return spec + + +def test_chronicle_v5_fact_hierarchy_takes_chronicle_label_not_package_wording(): """Each value keeps Chronicle's one label (chronicle#377). Chronicle's v5 sample row labels the AGI value ``all`` "All returns, total" and keeps the IRS package's "All returns" in ``package_dimension_value_labels``. The compiled target carries only - Chronicle's label, and the v4 restatement of the same row compiles to the - same target. + Chronicle's label. """ rows = chronicle_v5_fact_rows() - v5_row = _chronicle_v5_agi_row(rows) + v5_row = _chronicle_agi_row(rows) package_wording = v5_row[PACKAGE_DIMENSION_VALUE_LABELS][_AGI_DIMENSION]["all"] chronicle_label = v5_row["dimension_value_labels"][_AGI_DIMENSION]["all"] assert (package_wording, chronicle_label) == ("All returns", "All returns, total") - if row_contract == "v4": - rows = [as_v4_row(row) for row in rows] - (spec,) = compile_ledger_target_references( - rows, [_exact_agi_reference()], country="us" - ).specs + spec = _compiled_agi_target(rows) - assert spec.hierarchy is not None groupby = spec.hierarchy.dimensions[0] assert (groupby.id, groupby.value_id, groupby.value_label) == ( _AGI_DIMENSION, @@ -3970,6 +3969,35 @@ def test_chronicle_fact_hierarchy_takes_chronicle_label_not_package_wording( assert package_wording not in set(_strings(asdict(spec))) +def test_chronicle_v4_fact_hierarchy_keeps_v4_label(): + """A real v4 row still compiles, with the label its own v4 row carries. + + Chronicle's sample row from just before chronicle#377 labels the AGI value + ``all`` "All returns" in ``dimension_value_labels`` and + ``layout.groupby_value_label``. The same fact as a v5 row compiles to the + same target except that label. + """ + v4_rows = chronicle_v4_fact_rows() + v4_label = _chronicle_agi_row(v4_rows)["dimension_value_labels"][_AGI_DIMENSION][ + "all" + ] + assert v4_label == "All returns" + + v4_spec = _compiled_agi_target(v4_rows) + v5_spec = _compiled_agi_target(chronicle_v5_fact_rows()) + + groupby = v4_spec.hierarchy.dimensions[0] + assert (groupby.id, groupby.value_id, groupby.value_label) == ( + _AGI_DIMENSION, + "all", + v4_label, + ) + assert v4_spec.metadata["ledger_layout_groupby_value_label"] == v4_label + assert v4_spec.value == v5_spec.value + assert v4_spec.metadata["ledger_fact_key"] == v5_spec.metadata["ledger_fact_key"] + assert v5_spec.hierarchy.dimensions[0].value_label == "All returns, total" + + @pytest.mark.parametrize( "malformed", [value for _case, value in MALFORMED_PACKAGE_DIMENSION_VALUE_LABELS], @@ -3980,12 +4008,12 @@ def test_hierarchy_refuses_malformed_package_dimension_value_labels( ) -> None: """A selected v5 fact's package wording must be shaped as the v5 schema says. - ``package_dimension_value_labels`` maps dimension id, then value id, to - non-empty text, like ``dimension_value_labels``; compilation refuses any - other shape, as it refuses a missing or blank Chronicle label. + The v5 schema gives ``package_dimension_value_labels`` the shape it gives + ``dimension_value_labels``: dimension id, then value id, then non-empty + text. Compilation refuses any other shape. """ rows = chronicle_v5_fact_rows() - _chronicle_v5_agi_row(rows)[PACKAGE_DIMENSION_VALUE_LABELS] = deepcopy(malformed) + _chronicle_agi_row(rows)[PACKAGE_DIMENSION_VALUE_LABELS] = deepcopy(malformed) with pytest.raises( LedgerHierarchyMetadataError, match="package_dimension_value_labels" @@ -3993,6 +4021,43 @@ def test_hierarchy_refuses_malformed_package_dimension_value_labels( compile_ledger_target_references(rows, [_exact_agi_reference()], country="us") +def test_hierarchy_refuses_malformed_package_wording_under_an_unselected_dimension(): + """Every entry is checked, not only the selected dimension's. + + Microcosm reads ``dimension_value_labels`` only at the dimensions and + values a target selects; the package-wording check walks the whole field, + because nothing reads it at any one entry. + """ + rows = chronicle_v5_fact_rows() + agi_row = _chronicle_agi_row(rows) + agi_row[PACKAGE_DIMENSION_VALUE_LABELS] = { + **agi_row[PACKAGE_DIMENSION_VALUE_LABELS], + "unselected.dimension": {"some_value": " "}, + } + assert "unselected.dimension" not in agi_row["dimensions"] + + with pytest.raises( + LedgerHierarchyMetadataError, + match="dimension 'unselected.dimension' value 'some_value'", + ): + compile_ledger_target_references(rows, [_exact_agi_reference()], country="us") + + +@pytest.mark.parametrize( + "empty", + [{}, {_AGI_DIMENSION: {}}], + ids=["no_dimensions", "dimension_without_values"], +) +def test_hierarchy_accepts_empty_package_dimension_value_labels(empty) -> None: + """The v5 schema accepts an empty object at either level, and so does Microcosm.""" + rows = chronicle_v5_fact_rows() + _chronicle_agi_row(rows)[PACKAGE_DIMENSION_VALUE_LABELS] = deepcopy(empty) + + spec = _compiled_agi_target(rows) + + assert spec.hierarchy.dimensions[0].value_label == "All returns, total" + + def test_multi_fact_hierarchy_requires_explicit_microcosm_target_label() -> None: first = _consumer_fact_row(value=10.0) second = _consumer_fact_row( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_target_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_target_kernels.py index 7c74b41200..4f539850d0 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_target_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_target_kernels.py @@ -808,6 +808,33 @@ def test_target_kernels_enforce_full_feed_pin(tmp_path, kernel) -> None: kernel.run(_context(node, facts=facts, frame=toy_frame())) +@pytest.mark.parametrize( + "kernel", [TARGETS_COMPILE, TAKEUP_COMPARE], ids=lambda k: k.ref +) +def test_target_kernels_refuse_an_undeclared_row_contract(tmp_path, kernel) -> None: + """A feed whose rows and pin agree on an undeclared row contract is refused.""" + rows = [ + {**row, "schema_version": "chronicle.consumer_fact.v6"} + for row in toy_facts(ALL_TARGETS) + ] + facts, pin = write_pinned_artifact(tmp_path / "artifact", rows) + original = ( + _compile_node(references_for(CALIBRATION_TARGETS)) + if kernel is TARGETS_COMPILE + else _compare_node() + ) + node = replace( + original, params={**original.params, "chronicle_feed_pin": canonical_text(pin)} + ) + + with pytest.raises(ValueError) as refusal: + kernel.run(_context(node, facts=facts, frame=toy_frame())) + + assert not isinstance(refusal.value, ChronicleFeedPinMismatchError) + assert "['chronicle.consumer_fact.v6']" in str(refusal.value) + assert "microcosm.build.chronicle_epoch" in str(refusal.value) + + def test_compile_refuses_unmatched_duplicate_and_noncanonical_input(tmp_path) -> None: facts = write_facts(tmp_path / "facts.jsonl", toy_facts(CALIBRATION_TARGETS[:1])) with pytest.raises(ValueError, match="did not match a Ledger fact"): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py index 7fb808521d..e1482e7c0a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py @@ -60,6 +60,34 @@ def test_national_feed_rejects_malformed_identity( chronicle_feed.load_uk_chronicle_feed() +@pytest.mark.parametrize( + "versions", + [ + ["chronicle.consumer_fact.v6"], + ["chronicle.consumer_fact.v4", "chronicle.consumer_fact.v6"], + ], + ids=["undeclared_only", "declared_and_undeclared"], +) +def test_national_feed_rejects_an_undeclared_row_contract( + monkeypatch, tmp_path, versions +): + """A pin naming a row contract chronicle_epoch does not declare is refused.""" + from microcosm.build.uk_runtime import chronicle_feed + + raw = json.loads(chronicle_feed._feed_path().read_text()) + raw["consumer_fact_schema_versions"] = versions + path = tmp_path / "pin.json" + path.write_text(json.dumps(raw)) + monkeypatch.setattr(chronicle_feed, "_feed_path", lambda: path) + + with pytest.raises(ValueError) as refusal: + chronicle_feed.load_uk_chronicle_feed() + + message = str(refusal.value) + assert "['chronicle.consumer_fact.v6']" in message + assert "microcosm.build.chronicle_epoch" in message + + def test_committed_feed_pin_check_refuses_a_foreign_artifact(): from microcosm.build.uk_runtime.chronicle_feed import ( UKChronicleFeedPinError, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py index f9bdf52743..d7b0eb6be9 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py @@ -104,6 +104,32 @@ def test_pin_rejects_malformed_identity( chronicle_feed.load_us_chronicle_feed() +@pytest.mark.parametrize( + "versions", + [ + ["chronicle.consumer_fact.v6"], + ["chronicle.consumer_fact.v3", "chronicle.consumer_fact.v6"], + ], + ids=["undeclared_only", "declared_and_undeclared"], +) +def test_pin_rejects_an_undeclared_row_contract( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, versions: list[str] +) -> None: + """A pin naming a row contract chronicle_epoch does not declare is refused.""" + raw = json.loads(chronicle_feed._feed_path().read_text()) + raw["consumer_fact_schema_versions"] = versions + path = tmp_path / "pin.json" + path.write_text(json.dumps(raw)) + monkeypatch.setattr(chronicle_feed, "_feed_path", lambda: path) + + with pytest.raises(ValueError) as refusal: + chronicle_feed.load_us_chronicle_feed() + + message = str(refusal.value) + assert "['chronicle.consumer_fact.v6']" in message + assert "microcosm.build.chronicle_epoch" in message + + def test_pin_rejects_an_unsupported_artifact_schema( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/SOURCE.md b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/SOURCE.md new file mode 100644 index 0000000000..41bb37a85b --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/SOURCE.md @@ -0,0 +1,41 @@ +# Chronicle consumer fact fixtures + +Chronicle's own consumer fact schemas and sample rows, at the two row contracts on either +side of PolicyEngine/chronicle#377, which introduced `chronicle.consumer_fact.v5`. + +## v5, from Chronicle `44eebe0378cd4aa708e283837e5a4ed1f7b18ef0` (the merge of #377) + +Neither file differs at Chronicle `main` `28064d037562b47377193a5b75f4de120d3768a0` +(2026-11-04). + +- `consumer_fact.v5.schema.json`: `policyengine_chronicle/schemas/consumer_fact.v5.schema.json`, + byte for byte (sha256 + `acd33a091cd8d192be5d264e3e94c13e00afb6fd1972e57d4576ac55a54f6970`). Chronicle ships a + byte-identical copy at `docs/schemas/consumer_fact.v5.schema.json`. This digest is the + `consumer_fact_schema_sha256` a Chronicle consumer artifact built from v5 rows declares. +- `consumer_facts.v5.jsonl`: Chronicle's sample consumer rows, `chronicle/fixtures/consumer_facts.jsonl` + (sha256 `450a22c689b7a086cfc03522aa4492d491efc6875a4661dbd851a2015506897a`): IRS SOI + Publication 1304 Table 1.1, tax year 2023, all returns, for the return count, adjusted gross + income and total income tax. Each row carries Chronicle's label for the adjusted gross income + value `all` ("All returns, total") in `dimension_value_labels` and + `layout.groupby_value_label`, and the package's own wording ("All returns") in + `package_dimension_value_labels`. + +## v4, from Chronicle `1a3f2c41005ed0138fd82fe08a9d87201061e198` (`main` just before #377 merged) + +- `consumer_fact.v4.schema.json`: `policyengine_chronicle/schemas/consumer_fact.v4.schema.json`, + byte for byte (sha256 + `d953306d6f01fbc64db4372d0262d3012ee590269f8a25b276b18dca0626c6a0`, the digest the UK, NZ + and Belgian feed pins carry). It is unchanged at `44eebe037`. +- `consumer_facts.v4.jsonl`: the same sample file at that commit (sha256 + `d02308d049edcd0e60b29d590211456c6427a505e1acf50f1ae71bbcd2962a8c`): the same three facts + as v4 rows, labelling the adjusted gross income value `all` "All returns" in + `dimension_value_labels` and `layout.groupby_value_label`, with no + `package_dimension_value_labels`. They differ from the v5 rows only in `schema_version`, + that label and the v5 field. + +## Changes from Chronicle's files + +In both row files, the raw-storage bucket name in `source.raw_r2_bucket` and +`source.raw_r2_uri` is replaced by the placeholder `example-raw-bucket`, and each row is +written with sorted keys. Every row still validates against the schema of its row contract. diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_fact.v4.schema.json b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_fact.v4.schema.json new file mode 100644 index 0000000000..756673681c --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_fact.v4.schema.json @@ -0,0 +1,492 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://policyengine.org/chronicle/schemas/consumer_fact.v4.schema.json", + "title": "Chronicle consumer fact contract row", + "description": "The v3 contract row with one name per geography identifier (chronicle#281). Where Chronicle's register carries the identifier, 'name' is the register's name and 'publisher_name' is the source's own text when it differs; elsewhere 'name' is the source's text, as in v3. Both stay display metadata: neither enters a fact key, and a v3 row restamped with this schema_version is a valid v4 row.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "aggregate_fact_key", + "semantic_fact_key", + "legacy_fact_key", + "source_release_key", + "source_series_key", + "observed_measure_key", + "dimension_set_key", + "universe_constraint_set_key", + "value", + "value_type", + "assertion", + "provenance_class", + "period", + "geography", + "entity", + "aggregation", + "observed_measure", + "dimensions", + "universe_constraints", + "source", + "lineage" + ], + "allOf": [ + { + "if": { + "properties": { + "provenance_class": { + "const": "survey_aggregate" + } + }, + "required": [ + "provenance_class" + ] + }, + "then": { + "required": [ + "survey_instrument" + ] + }, + "else": { + "not": { + "required": [ + "survey_instrument" + ] + } + } + } + ], + "properties": { + "schema_version": { + "const": "chronicle.consumer_fact.v4" + }, + "aggregate_fact_key": { + "type": "string", + "pattern": "^ledger\\.aggregate_fact\\.v2:[0-9a-f]{24}$" + }, + "semantic_fact_key": { + "type": "string", + "pattern": "^ledger\\.semantic_fact\\.v2:[0-9a-f]{24}$" + }, + "legacy_fact_key": { + "type": "string", + "pattern": "^ledger\\.fact\\.v1:[0-9a-f]{24}$" + }, + "source_release_key": { + "type": "string", + "pattern": "^ledger\\.source_release\\.v2:[0-9a-f]{24}$" + }, + "source_series_key": { + "type": "string", + "pattern": "^ledger\\.source_series\\.v2:[0-9a-f]{24}$" + }, + "observed_measure_key": { + "type": "string", + "pattern": "^ledger\\.observed_measure\\.v2:[0-9a-f]{24}$" + }, + "dimension_set_key": { + "type": "string", + "pattern": "^ledger\\.dimension_set\\.v2:[0-9a-f]{24}$" + }, + "universe_constraint_set_key": { + "type": "string", + "pattern": "^ledger\\.universe_constraint_set\\.v2:[0-9a-f]{24}$" + }, + "value": { + "type": [ + "integer", + "number", + "string", + "boolean" + ] + }, + "value_type": { + "enum": [ + "integer", + "number", + "decimal", + "string", + "boolean" + ] + }, + "period": { + "type": "object", + "additionalProperties": false, + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "Publisher-asserted period kind; the ledger.core.ALLOWED_PERIOD_TYPES vocabulary: calendar_year, tax_year, fiscal_year, academic_year, gas_year, month, quarter, week. Split-label years store the opening year of the publisher's label (UK FY2024-25 -> fiscal_year 2024; AY 2024/25 -> academic_year 2024). gas_year keeps DESNZ's year label: the opening year from 2016 (mid-May 2024 to mid-May 2025 -> gas_year 2024) and the closing year of the October-to-September years to 2015. Publishers labelling a fiscal year with a single year keep that label year (US federal FY2024 -> 2024). Quarter values use YYYY-QN and ISO week values use YYYY-Www, with exact dates in period_coverage." + }, + "value": { + "type": [ + "integer", + "string" + ] + } + } + }, + "geography": { + "type": "object", + "additionalProperties": false, + "required": [ + "level", + "id" + ], + "properties": { + "level": { + "type": "string" + }, + "id": { + "type": "string" + }, + "vintage": { + "type": "string" + }, + "name": { + "$ref": "#/$defs/label", + "description": "The name for this geography: Chronicle's register name where it carries the identifier, otherwise the publisher's own text; absent when neither states a name." + }, + "publisher_name": { + "$ref": "#/$defs/label", + "description": "The publisher's own name for this geography, present only where it differs from the register name carried in 'name'." + } + } + }, + "entity": { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "role": { + "type": "string" + } + } + }, + "aggregation": { + "type": "object", + "additionalProperties": false, + "required": [ + "method" + ], + "properties": { + "method": { + "type": "string" + }, + "denominator": { + "type": "string" + } + } + }, + "observed_measure": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_name", + "source_table", + "source_measure_id", + "source_concept", + "unit" + ], + "properties": { + "source_name": { + "type": "string" + }, + "source_table": { + "type": "string" + }, + "source_measure_id": { + "type": "string" + }, + "source_concept": { + "type": "string" + }, + "unit": { + "type": "string" + } + } + }, + "dimensions": { + "type": "object", + "additionalProperties": { + "type": [ + "integer", + "number", + "string", + "boolean" + ] + } + }, + "dimension_labels": { + "type": "object", + "description": "Label for each dimension id: every key of dimensions and, when layout.groupby_dimension is set, that id.", + "additionalProperties": { + "$ref": "#/$defs/label" + } + }, + "dimension_value_labels": { + "type": "object", + "description": "Label for each dimension value, keyed dimension id then value id. A value id is the value's string form: booleans lower-cased (true, false), integers, numbers and strings as Python str() writes them, lists and mappings JSON-serialised with sorted keys and no spaces. Covers every dimensions entry and, when layout.groupby_dimension is set, layout.groupby_value_id.", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/label" + } + } + }, + "universe_constraints": { + "type": "object", + "additionalProperties": false, + "required": [ + "domain" + ], + "properties": { + "domain": { + "type": "string" + }, + "constraints": { + "type": "array", + "items": { + "$ref": "#/$defs/constraint" + } + } + } + }, + "source": { + "type": "object", + "additionalProperties": true, + "required": [ + "source_name", + "source_table", + "source_file", + "vintage", + "extracted_at", + "extraction_method", + "source_sha256", + "source_size_bytes", + "raw_r2_uri" + ], + "properties": { + "source_name": { + "type": "string" + }, + "source_table": { + "type": "string" + }, + "source_file": { + "type": "string" + }, + "url": { + "type": "string" + }, + "vintage": { + "type": "string" + }, + "extracted_at": { + "type": "string" + }, + "extraction_method": { + "type": "string" + }, + "method_notes": { + "type": "string" + }, + "source_sha256": { + "type": "string" + }, + "source_size_bytes": { + "type": "integer" + }, + "raw_r2_bucket": { + "type": "string" + }, + "raw_r2_key": { + "type": "string" + }, + "raw_r2_uri": { + "type": "string" + } + } + }, + "lineage": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_record_id", + "source_cell_keys" + ], + "properties": { + "source_record_id": { + "type": "string" + }, + "source_cell_keys": { + "type": "array", + "items": { + "type": "string" + } + }, + "source_row_keys": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "layout": { + "type": "object", + "additionalProperties": true, + "properties": { + "groupby_dimension_label": { + "$ref": "#/$defs/label", + "description": "Label of layout.groupby_dimension; equals dimension_labels for that id." + } + } + }, + "label": { + "type": "string" + }, + "concept_alignment": { + "type": "object", + "additionalProperties": false, + "required": [ + "concept_alignment_key", + "source_concept", + "canonical_concept", + "relation", + "authority" + ], + "properties": { + "concept_alignment_key": { + "type": "string", + "pattern": "^ledger\\.concept_alignment\\.v2:[0-9a-f]{24}$" + }, + "source_concept": { + "type": "string" + }, + "canonical_concept": { + "type": "string" + }, + "relation": { + "type": "string" + }, + "authority": { + "type": "string" + }, + "evidence_url": { + "type": "string" + }, + "evidence_notes": { + "type": "string" + }, + "legal_vintage": { + "type": "string" + } + } + }, + "assertion": { + "type": "string", + "enum": [ + "observation", + "source_projection" + ], + "description": "Who asserted the value: a publisher-measured outcome (observation) or the publisher's own forward-looking estimate (source_projection). PolicyEngine-computed values are never consumer facts." + }, + "provenance_class": { + "type": "string", + "enum": [ + "administrative", + "census", + "survey_aggregate", + "model_output" + ], + "description": "Publisher measurement basis. Consumers can gate calibration inputs on this required, closed classification." + }, + "survey_instrument": { + "type": "string", + "pattern": ".*\\S.*", + "description": "Survey name required exactly when provenance_class is survey_aggregate." + }, + "period_coverage": { + "type": "object", + "additionalProperties": false, + "description": "Non-identity provenance for the fact's reference period: coverage dates, basis, the publisher's period label, and accounting basis.", + "properties": { + "start_date": { + "type": "string" + }, + "end_date": { + "type": "string" + }, + "basis": { + "type": "string", + "enum": [ + "calendar", + "tax", + "fiscal", + "survey_reference", + "projection_horizon" + ] + }, + "source_period_label": { + "type": "string" + }, + "accounting_basis": { + "type": "string", + "enum": [ + "cash", + "accrual" + ] + }, + "notes": { + "type": "string" + } + } + } + }, + "$defs": { + "label": { + "type": "string", + "pattern": ".*\\S.*" + }, + "constraint": { + "type": "object", + "additionalProperties": false, + "required": [ + "variable", + "operator", + "value", + "role" + ], + "properties": { + "variable": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": [ + "integer", + "number", + "string", + "boolean" + ] + }, + "unit": { + "type": "string" + }, + "role": { + "type": "string" + } + } + } + } +} diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_fact.v5.schema.json b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_fact.v5.schema.json similarity index 100% rename from packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_fact.v5.schema.json rename to packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_fact.v5.schema.json diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_facts.v4.jsonl b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_facts.v4.jsonl new file mode 100644 index 0000000000..b0068a1ce5 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_facts.v4.jsonl @@ -0,0 +1,3 @@ +{"aggregate_fact_key": "ledger.aggregate_fact.v2:13153fb8dfd73fa12d5715e9", "aggregation": {"method": "sum"}, "assertion": "observation", "dimension_labels": {"filing_status": "Filing status", "income_range": "Income range", "us:statutes/26/62#adjusted_gross_income": "Adjusted gross income"}, "dimension_set_key": "ledger.dimension_set.v2:a55266d8b1d53a8a7e49927d", "dimension_value_labels": {"filing_status": {"all": "All filing statuses"}, "income_range": {"all": "All income ranges"}, "us:statutes/26/62#adjusted_gross_income": {"all": "All returns"}}, "dimensions": {"filing_status": "all", "income_range": "all"}, "entity": {"name": "tax_unit", "role": "filing_unit"}, "geography": {"id": "0100000US", "level": "country", "name": "United States", "vintage": "2020_census"}, "label": "United States 2023 tax year count irs soi individual income tax returns for tax unit (filing status=all, income range=all) [irs_soi Publication 1304 Table 1.1 23in11si.xls tax_year_2023]", "layout": {"groupby_dimension": "us:statutes/26/62#adjusted_gross_income", "groupby_dimension_label": "Adjusted gross income", "groupby_ordinal": 0, "groupby_value_id": "all", "groupby_value_label": "All returns", "measure_id": "return_count", "measure_label": "Number of returns", "measure_ordinal": 0, "record_set_id": "irs_soi.ty2023.table_1_1", "record_set_spec_hash": "a1ecc4d67a3d281548a4660e", "record_set_spec_id": "irs_soi.table_1_1.v1", "source_column_id": "return_count", "source_row_id": "all", "table_record_kind": "total"}, "legacy_fact_key": "ledger.fact.v1:079f0697e3b090abf422781b", "lineage": {"source_cell_keys": ["ledger.source_cell.v1:82f39b2a5aeab53c1b717928"], "source_record_id": "irs_soi.ty2023.table_1_1.all.return_count", "source_row_keys": []}, "observed_measure": {"source_concept": "irs_soi.individual_income_tax_returns", "source_measure_id": "return_count", "source_name": "irs_soi", "source_table": "Publication 1304 Table 1.1", "unit": "count"}, "observed_measure_key": "ledger.observed_measure.v2:bc96277bae6ef31da4a5794c", "period": {"type": "tax_year", "value": 2023}, "provenance_class": "administrative", "schema_version": "chronicle.consumer_fact.v4", "semantic_fact_key": "ledger.semantic_fact.v2:1ae4c4f93f554ec7ca14d2ad", "source": {"extracted_at": "2026-05-04", "extraction_method": "CellSelectorSpec and SourceRecordSpec resolved from cells", "method_notes": "Each fact carries source_record_id, source_cell_keys, and source_row_keys when available.", "raw_r2_bucket": "example-raw-bucket", "raw_r2_key": "raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "raw_r2_uri": "r2://example-raw-bucket/raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "source_file": "23in11si.xls", "source_name": "irs_soi", "source_sha256": "842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17", "source_size_bytes": 54272, "source_table": "Publication 1304 Table 1.1", "url": "https://www.irs.gov/pub/irs-soi/23in11si.xls", "vintage": "tax_year_2023"}, "source_release_key": "ledger.source_release.v2:e418de50f65dc3f88fbcdd6c", "source_series_key": "ledger.source_series.v2:ac7ef40c4f9adbd84a82f24c", "universe_constraint_set_key": "ledger.universe_constraint_set.v2:18df86d027cde55a3f217b77", "universe_constraints": {"domain": "all_individual_income_tax_returns"}, "value": 160602107, "value_type": "integer"} +{"aggregate_fact_key": "ledger.aggregate_fact.v2:36ce90001adb50e8e9758e4f", "aggregation": {"method": "sum"}, "assertion": "observation", "concept_alignment": {"authority": "ledger-us", "canonical_concept": "us:statutes/26/62#adjusted_gross_income", "concept_alignment_key": "ledger.concept_alignment.v2:2e0ac323f1b261710e4397a1", "evidence_notes": "IRS SOI Table 1.1 reports adjusted gross income for individual income tax returns; IRC section 62 defines adjusted gross income. This Ledger assertion treats the SOI AGI column as exactly adopting that legal concept for the tax-year source record.", "evidence_url": "https://uscode.house.gov/view.xhtml?req=(title:26%20section:62%20edition:prelim)", "legal_vintage": "tax_year_2023", "relation": "exact", "source_concept": "irs_soi.adjusted_gross_income"}, "dimension_labels": {"filing_status": "Filing status", "income_range": "Income range", "us:statutes/26/62#adjusted_gross_income": "Adjusted gross income"}, "dimension_set_key": "ledger.dimension_set.v2:a55266d8b1d53a8a7e49927d", "dimension_value_labels": {"filing_status": {"all": "All filing statuses"}, "income_range": {"all": "All income ranges"}, "us:statutes/26/62#adjusted_gross_income": {"all": "All returns"}}, "dimensions": {"filing_status": "all", "income_range": "all"}, "entity": {"name": "tax_unit", "role": "filing_unit"}, "geography": {"id": "0100000US", "level": "country", "name": "United States", "vintage": "2020_census"}, "label": "United States 2023 tax year sum us:statutes/26/62#adjusted gross income for tax unit (filing status=all, income range=all) [irs_soi Publication 1304 Table 1.1 23in11si.xls tax_year_2023]", "layout": {"groupby_dimension": "us:statutes/26/62#adjusted_gross_income", "groupby_dimension_label": "Adjusted gross income", "groupby_ordinal": 0, "groupby_value_id": "all", "groupby_value_label": "All returns", "measure_id": "adjusted_gross_income", "measure_label": "Adjusted gross income", "measure_ordinal": 1, "record_set_id": "irs_soi.ty2023.table_1_1", "record_set_spec_hash": "a1ecc4d67a3d281548a4660e", "record_set_spec_id": "irs_soi.table_1_1.v1", "source_column_id": "adjusted_gross_income", "source_row_id": "all", "table_record_kind": "total"}, "legacy_fact_key": "ledger.fact.v1:7f8e9d3c64fec2670b571c3b", "lineage": {"source_cell_keys": ["ledger.source_cell.v1:c3d349e26856fb6224303675"], "source_record_id": "irs_soi.ty2023.table_1_1.all.adjusted_gross_income", "source_row_keys": []}, "observed_measure": {"source_concept": "irs_soi.adjusted_gross_income", "source_measure_id": "adjusted_gross_income", "source_name": "irs_soi", "source_table": "Publication 1304 Table 1.1", "unit": "usd"}, "observed_measure_key": "ledger.observed_measure.v2:ac9c3cef252ef7ead0066cb1", "period": {"type": "tax_year", "value": 2023}, "provenance_class": "administrative", "schema_version": "chronicle.consumer_fact.v4", "semantic_fact_key": "ledger.semantic_fact.v2:b83082b8e69c1ddc6fdebed2", "source": {"extracted_at": "2026-05-04", "extraction_method": "CellSelectorSpec and SourceRecordSpec resolved from cells", "method_notes": "Each fact carries source_record_id, source_cell_keys, and source_row_keys when available.", "raw_r2_bucket": "example-raw-bucket", "raw_r2_key": "raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "raw_r2_uri": "r2://example-raw-bucket/raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "source_file": "23in11si.xls", "source_name": "irs_soi", "source_sha256": "842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17", "source_size_bytes": 54272, "source_table": "Publication 1304 Table 1.1", "url": "https://www.irs.gov/pub/irs-soi/23in11si.xls", "vintage": "tax_year_2023"}, "source_release_key": "ledger.source_release.v2:e418de50f65dc3f88fbcdd6c", "source_series_key": "ledger.source_series.v2:ac7ef40c4f9adbd84a82f24c", "universe_constraint_set_key": "ledger.universe_constraint_set.v2:18df86d027cde55a3f217b77", "universe_constraints": {"domain": "all_individual_income_tax_returns"}, "value": 15286017359000, "value_type": "integer"} +{"aggregate_fact_key": "ledger.aggregate_fact.v2:0fa0e8c17cfe0ebd844c4f95", "aggregation": {"method": "sum"}, "assertion": "observation", "dimension_labels": {"filing_status": "Filing status", "income_range": "Income range", "us:statutes/26/62#adjusted_gross_income": "Adjusted gross income"}, "dimension_set_key": "ledger.dimension_set.v2:a55266d8b1d53a8a7e49927d", "dimension_value_labels": {"filing_status": {"all": "All filing statuses"}, "income_range": {"all": "All income ranges"}, "us:statutes/26/62#adjusted_gross_income": {"all": "All returns"}}, "dimensions": {"filing_status": "all", "income_range": "all"}, "entity": {"name": "tax_unit", "role": "filing_unit"}, "geography": {"id": "0100000US", "level": "country", "name": "United States", "vintage": "2020_census"}, "label": "United States 2023 tax year sum irs soi total income tax for tax unit (filing status=all, income range=all) [irs_soi Publication 1304 Table 1.1 23in11si.xls tax_year_2023]", "layout": {"groupby_dimension": "us:statutes/26/62#adjusted_gross_income", "groupby_dimension_label": "Adjusted gross income", "groupby_ordinal": 0, "groupby_value_id": "all", "groupby_value_label": "All returns", "measure_id": "total_income_tax", "measure_label": "Total income tax", "measure_ordinal": 2, "record_set_id": "irs_soi.ty2023.table_1_1", "record_set_spec_hash": "a1ecc4d67a3d281548a4660e", "record_set_spec_id": "irs_soi.table_1_1.v1", "source_column_id": "total_income_tax", "source_row_id": "all", "table_record_kind": "total"}, "legacy_fact_key": "ledger.fact.v1:ec48b42ac43674a2a2206b8b", "lineage": {"source_cell_keys": ["ledger.source_cell.v1:f38c48e6e2f23204eb997143"], "source_record_id": "irs_soi.ty2023.table_1_1.all.total_income_tax", "source_row_keys": []}, "observed_measure": {"source_concept": "irs_soi.total_income_tax", "source_measure_id": "total_income_tax", "source_name": "irs_soi", "source_table": "Publication 1304 Table 1.1", "unit": "usd"}, "observed_measure_key": "ledger.observed_measure.v2:c11ff13c2dfe8af0795a1194", "period": {"type": "tax_year", "value": 2023}, "provenance_class": "administrative", "schema_version": "chronicle.consumer_fact.v4", "semantic_fact_key": "ledger.semantic_fact.v2:f8b2801687c0ba560054ed36", "source": {"extracted_at": "2026-05-04", "extraction_method": "CellSelectorSpec and SourceRecordSpec resolved from cells", "method_notes": "Each fact carries source_record_id, source_cell_keys, and source_row_keys when available.", "raw_r2_bucket": "example-raw-bucket", "raw_r2_key": "raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "raw_r2_uri": "r2://example-raw-bucket/raw/irs_soi/soi-table-1-1/2023/842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17/23in11si.xls", "source_file": "23in11si.xls", "source_name": "irs_soi", "source_sha256": "842da11fe09123796c861e919d16011ef161338de1c172d39064bdfa666aca17", "source_size_bytes": 54272, "source_table": "Publication 1304 Table 1.1", "url": "https://www.irs.gov/pub/irs-soi/23in11si.xls", "vintage": "tax_year_2023"}, "source_release_key": "ledger.source_release.v2:e418de50f65dc3f88fbcdd6c", "source_series_key": "ledger.source_series.v2:ac7ef40c4f9adbd84a82f24c", "universe_constraint_set_key": "ledger.universe_constraint_set.v2:18df86d027cde55a3f217b77", "universe_constraints": {"domain": "all_individual_income_tax_returns"}, "value": 2147909818000, "value_type": "integer"} diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_facts.jsonl b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_facts.v5.jsonl similarity index 100% rename from packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/consumer_facts.jsonl rename to packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/consumer_facts.v5.jsonl diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/SOURCE.md b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/SOURCE.md deleted file mode 100644 index aaa2d8659e..0000000000 --- a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact_v5/SOURCE.md +++ /dev/null @@ -1,22 +0,0 @@ -# Chronicle consumer_fact.v5 fixtures - -Both files come from PolicyEngine/chronicle at `44eebe0378cd4aa708e283837e5a4ed1f7b18ef0`, -the merge of PolicyEngine/chronicle#377, which introduced `chronicle.consumer_fact.v5`. -Neither file differs at Chronicle `main` `28064d037562b47377193a5b75f4de120d3768a0` -(2026-11-04). - -- `consumer_fact.v5.schema.json`: `policyengine_chronicle/schemas/consumer_fact.v5.schema.json`, - byte for byte (sha256 - `acd33a091cd8d192be5d264e3e94c13e00afb6fd1972e57d4576ac55a54f6970`). Chronicle ships a - byte-identical copy at `docs/schemas/consumer_fact.v5.schema.json`. This digest is the - `consumer_fact_schema_sha256` a Chronicle consumer artifact built from v5 rows declares. -- `consumer_facts.jsonl`: Chronicle's sample consumer rows, `chronicle/fixtures/consumer_facts.jsonl` - (sha256 `450a22c689b7a086cfc03522aa4492d491efc6875a4661dbd851a2015506897a`): IRS SOI - Publication 1304 Table 1.1, tax year 2023, all returns, for the return count, adjusted gross - income and total income tax. Each row carries Chronicle's label for the adjusted gross income - value `all` ("All returns, total") in `dimension_value_labels` and - `layout.groupby_value_label`, and the package's own wording ("All returns") in - `package_dimension_value_labels`. Two changes from Chronicle's file: the raw-storage bucket - name in `source.raw_r2_bucket` and `source.raw_r2_uri` is replaced by the placeholder - `example-raw-bucket`, and each row is written with sorted keys. Every row still validates - against the schema above. diff --git a/test_support/microcosm_build/chronicle_feed.py b/test_support/microcosm_build/chronicle_feed.py index a2982c59c6..4acc27d277 100644 --- a/test_support/microcosm_build/chronicle_feed.py +++ b/test_support/microcosm_build/chronicle_feed.py @@ -2,7 +2,6 @@ from __future__ import annotations -import copy import hashlib import json from collections.abc import Mapping, Sequence @@ -12,14 +11,17 @@ from microcosm.build.ledger_artifact import CONSUMER_ARTIFACT_SCHEMA_VERSION from test_support.paths import paths_for -#: Chronicle's v5 consumer-fact schema and sample rows, taken from -#: PolicyEngine/chronicle at 44eebe037 (the merge of chronicle#377); the -#: directory's SOURCE.md records the paths and digests. -CHRONICLE_V5_FIXTURES = ( - paths_for("microcosm-build").tests / "fixtures/chronicle_consumer_fact_v5" +#: Chronicle's v4 and v5 consumer-fact schemas and sample rows: v5 from +#: PolicyEngine/chronicle at 44eebe037 (the merge of chronicle#377), v4 from +#: 1a3f2c4 (Chronicle main just before it). The directory's SOURCE.md records +#: the paths and digests. +CHRONICLE_FIXTURES = ( + paths_for("microcosm-build").tests / "fixtures/chronicle_consumer_fact" ) -CHRONICLE_V5_SCHEMA_PATH = CHRONICLE_V5_FIXTURES / "consumer_fact.v5.schema.json" -CHRONICLE_V5_FACTS_PATH = CHRONICLE_V5_FIXTURES / "consumer_facts.jsonl" +CHRONICLE_V4_SCHEMA_PATH = CHRONICLE_FIXTURES / "consumer_fact.v4.schema.json" +CHRONICLE_V4_FACTS_PATH = CHRONICLE_FIXTURES / "consumer_facts.v4.jsonl" +CHRONICLE_V5_SCHEMA_PATH = CHRONICLE_FIXTURES / "consumer_fact.v5.schema.json" +CHRONICLE_V5_FACTS_PATH = CHRONICLE_FIXTURES / "consumer_facts.v5.jsonl" #: The field chronicle#377 added to the v5 row: a package's own wording of a #: dimension value where it differs from the label the row exports. @@ -41,6 +43,10 @@ ) +def _jsonl_rows(path: Path) -> list[dict[str, Any]]: + return [json.loads(row) for row in path.read_text().splitlines() if row.strip()] + + def chronicle_v5_schema() -> dict[str, Any]: """Chronicle's v5 consumer-fact JSON Schema, parsed.""" return json.loads(CHRONICLE_V5_SCHEMA_PATH.read_bytes()) @@ -58,19 +64,22 @@ def chronicle_v5_schema_version() -> str: def chronicle_v5_fact_rows() -> list[dict[str, Any]]: """Fresh copies of Chronicle's sample v5 rows.""" - return [ - json.loads(line) - for line in CHRONICLE_V5_FACTS_PATH.read_text().splitlines() - if line.strip() - ] - - -def as_v4_row(row: Mapping[str, Any]) -> dict[str, Any]: - """The v4 row a v5 row restates: the v4 stamp, without the v5-only field.""" - v4_row = copy.deepcopy(dict(row)) - v4_row.pop(PACKAGE_DIMENSION_VALUE_LABELS, None) - v4_row["schema_version"] = "chronicle.consumer_fact.v4" - return v4_row + return _jsonl_rows(CHRONICLE_V5_FACTS_PATH) + + +def chronicle_v4_schema() -> dict[str, Any]: + """Chronicle's v4 consumer-fact JSON Schema, parsed.""" + return json.loads(CHRONICLE_V4_SCHEMA_PATH.read_bytes()) + + +def chronicle_v4_schema_sha256() -> str: + """The digest the UK, NZ and Belgian feed pins carry for v4.""" + return hashlib.sha256(CHRONICLE_V4_SCHEMA_PATH.read_bytes()).hexdigest() + + +def chronicle_v4_fact_rows() -> list[dict[str, Any]]: + """Fresh copies of Chronicle's sample v4 rows, from just before chronicle#377.""" + return _jsonl_rows(CHRONICLE_V4_FACTS_PATH) def write_pinned_artifact( From deadfb7f914be1ee9c16f53c82e5016ed4648c39 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:38:57 +0400 Subject: [PATCH 4/8] Refuse a Chronicle feed pinned on an undeclared row contract Review round 1 of #1510 found that declaring v5 changed nothing at run time: no code outside the tests read the accepted row-contract list. Anthony decided to enforce it. - chronicle_epoch adds require_accepted_consumer_fact_schema_versions and UndeclaredConsumerFactSchemaVersionError (a ValueError). It refuses a row-contract list that is not a list, or that names any version the declaration table lacks; the message names the undeclared versions, says to declare them in microcosm.build.chronicle_epoch first, and lists the declared ones. - ledger_artifact calls it in validate_chronicle_feed_pin after the exact comparisons, so a pin that differs from its artifact still reports the differing field, and in chronicle_feed_identity, which reference factors, money normalization and the transport composer use. The transport kernels reach both. A feed loaded without a full pin still carries its per-row ids unchecked, as the US fiscal-refresh feed needs. - The UK and US pin loaders call it on the committed pin. - ledger_targets: the shape check's docstring says it is stricter than the dimension_value_labels check, which reads only selected entries. - congressional_district_vintage: comments in _translated_fact_shell and _state_total_proxy_source_district_fact record that a v5 row's package wording is copied keyed by the source row's ids, and that nothing reads it. - The UK and US feed re-pin guides describe the refusal; the US guide adds the district note. The changelog fragment now says the shape check runs for hierarchy targets and how callers treat its error. Refs #1509 Co-Authored-By: Claude Opus 5.5 --- .../1509-chronicle-consumer-fact-v5.added.md | 8 +++- docs/uk-chronicle-feed-repin.md | 11 +++-- docs/us-chronicle-feed-repin.md | 20 ++++++++ .../src/microcosm/build/chronicle_epoch.py | 39 ++++++++++++++++ .../src/microcosm/build/ledger_artifact.py | 46 ++++++++++++++----- .../src/microcosm/build/ledger_targets.py | 9 ++-- .../build/uk_runtime/chronicle_feed.py | 4 ++ .../build/us_runtime/chronicle_feed.py | 4 ++ .../congressional_district_vintage.py | 9 ++++ 9 files changed, 130 insertions(+), 20 deletions(-) diff --git a/changelog.d/1509-chronicle-consumer-fact-v5.added.md b/changelog.d/1509-chronicle-consumer-fact-v5.added.md index da000f5a0a..3328a83d25 100644 --- a/changelog.d/1509-chronicle-consumer-fact-v5.added.md +++ b/changelog.d/1509-chronicle-consumer-fact-v5.added.md @@ -1 +1,7 @@ -Accept Chronicle's `chronicle.consumer_fact.v5` facts (microcosm#1509), the row contract Chronicle `main` writes since PolicyEngine/chronicle#377 (`44eebe037`). `microcosm.build.chronicle_epoch` declares v5 beside v1 to v4, so it is in `ACCEPTED_CONSUMER_FACT_SCHEMA_VERSIONS` and belongs to the chronicle epoch; v2 to v4 keep loading. v5 adds an optional `package_dimension_value_labels` field, a source package's own wording of a dimension value where it differs from the label the row exports. The loader keeps it on the parsed rows, as it keeps every published field, and Microcosm never takes a label from it: each value keeps the one label Chronicle gives it in `dimension_value_labels` and `layout.groupby_value_label`. Target compilation refuses a selected fact whose field is not an object of dimension id to an object of value id to non-empty text, the shape the v5 schema requires. No feed pin moves: the UK, US, NZ and Belgian feeds stay on their current Chronicle commits until their own re-pins. +Declare Chronicle's `chronicle.consumer_fact.v5` row contract, the one Chronicle `main` writes since PolicyEngine/chronicle#377 (`44eebe037`), and refuse a Chronicle feed pinned on a row contract Microcosm does not declare (microcosm#1509). + +- **The declaration.** `microcosm.build.chronicle_epoch` declares v5 in the chronicle epoch beside `ledger.consumer_fact.v1` and `chronicle.consumer_fact.v2` to v4, matching Chronicle's own `CONSUMER_FACT_ROW_CONTRACTS`. Before this change nothing outside the tests read that list, so a v5 feed would already have loaded. +- **The refusal.** A pin whose `consumer_fact_schema_versions` names an undeclared row contract is refused, with a message that names it and says to declare it in `chronicle_epoch` first. The checks are the UK and US pin loaders, `chronicle_feed_identity` (used by reference factors, money normalization and the transport composer), and `validate_chronicle_feed_pin` once a pin, its artifact's rows and its manifest agree. The transport kernels reach the last two. Every committed pin (UK, NZ and Belgium on v4, the US on v3) and every resource restating one names declared contracts. A feed loaded without a full pin, such as the US fiscal-refresh feed with its `arch.consumer_fact.v1` rows, still carries its per-row ids unchecked. +- **The new field.** v5 adds an optional `package_dimension_value_labels`: a source package's own wording of a dimension value where it differs from the label the row exports. The loader keeps it on the parsed rows, as it keeps every published field, and Microcosm never takes a label from it: each value keeps the one label Chronicle gives it in `dimension_value_labels` and `layout.groupby_value_label`. +- **Its shape.** When a target with a calibration hierarchy (a provider and category seed) is compiled, each selected fact's field must be an object of dimension id to an object of value id to non-empty text, the shape the v5 schema requires. Any other shape raises `LedgerHierarchyMetadataError`, the error a missing Chronicle label raises, and callers treat it the same way: the UK compilers report the reference as unsupported, which stops the build year and is recorded for validation periods. This check walks every entry of the field, where the check of `dimension_value_labels` reads only the selected entries. +- **No feed pin moves.** The UK, US, NZ and Belgian feeds stay on their current Chronicle commits until their own re-pins. diff --git a/docs/uk-chronicle-feed-repin.md b/docs/uk-chronicle-feed-repin.md index 24ef2fbe74..067f3601fd 100644 --- a/docs/uk-chronicle-feed-repin.md +++ b/docs/uk-chronicle-feed-repin.md @@ -25,10 +25,13 @@ consumer-fact schema digest) and `tests/engine_free/uk/test_uk_full_targets.py` (`test_current_national_and_local_pins_share_one_reviewed_identity`). A schema digest moves for every country whose feed is built after the Chronicle change, so the NZ pin (`nz/chronicle_feed.json`) moves at its own next re-pin. A feed -whose rows carry a newer per-row `schema_version` also needs that row contract -declared in `microcosm.build.chronicle_epoch` first. Microcosm declares -`ledger.consumer_fact.v1` and `chronicle.consumer_fact.v2` to `v5` -(microcosm#1509); the v5 schema's digest is `acd33a09…`. Then +whose rows carry a newer per-row `schema_version` needs that row contract +declared in `microcosm.build.chronicle_epoch` first: the UK pin loader, the +identity check the factor and transport code use (`chronicle_feed_identity`) +and the feed-pin check against a loaded artifact (`validate_chronicle_feed_pin`) +refuse a pin naming an undeclared one (microcosm#1509). Microcosm declares +`ledger.consumer_fact.v1` and `chronicle.consumer_fact.v2` to `v5`; the v5 +schema's digest is `acd33a09…`. Then regenerate. Vendor the per-concern fact resources (step 5) before the national generator: the generator refuses to compile against a vendored resource whose feed identity diff --git a/docs/us-chronicle-feed-repin.md b/docs/us-chronicle-feed-repin.md index 5083a0488b..95416f3889 100644 --- a/docs/us-chronicle-feed-repin.md +++ b/docs/us-chronicle-feed-repin.md @@ -13,6 +13,26 @@ any other feed, and `test_us_chronicle_feed.py` fails if the parity resources, the generator and the pin disagree, or if the scope file changes without a new pin. +## A re-pin onto a newer row contract + +The US feed is on `chronicle.consumer_fact.v3`; Chronicle `main` writes +`chronicle.consumer_fact.v5` since PolicyEngine/chronicle#377. The US pin +loader, like the UK one and the feed-pin check against a loaded artifact, +refuses a pin whose `consumer_fact_schema_versions` names a row contract that +`microcosm.build.chronicle_epoch` does not declare (microcosm#1509). v5 is +declared, and its schema digest is `acd33a09…`. + +v5 rows may carry `package_dimension_value_labels`, a package's own wording of +a value where it differs from Chronicle's label. The congressional-district +vintage translation (`us_runtime/congressional_district_vintage.py`: +`_translated_fact_shell` and `_state_total_proxy_source_district_fact`) copies +each source row whole and rewrites only the geography, the layout and the +group-by entry of `dimension_value_labels`. So after a re-pin onto v5, a +derived district row keeps any package wording keyed by the source row's value +ids (the 118th-Congress district, or the state row's value), not the derived +row's. No code reads the field, so no label, value or refusal changes; v4's +`geography.publisher_name` is copied the same way. + ## The 2026-10-27 re-pin: one label per dimension value (microcosm#1343) The `cf9ece2` feed labelled three dimension values two ways, so compiled diff --git a/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py b/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py index e5eac8f85e..176a3aa289 100644 --- a/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py +++ b/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py @@ -88,6 +88,7 @@ "MICROCOSM_CONSUMER_ARTIFACT_SCHEMA_VERSION", "PUBLISHED_CONSUMER_ARTIFACT_SCHEMA_VERSION", "UNDECLARED", + "UndeclaredConsumerFactSchemaVersionError", "consumer_artifact_schema_epoch", "consumer_fact_schema_epoch", "declared_identity", @@ -101,6 +102,7 @@ "is_accepted_consumer_fact_schema_version", "is_chronicle_fact_key", "parse_fact_key", + "require_accepted_consumer_fact_schema_versions", "row_fact_key_epochs", ] @@ -583,6 +585,43 @@ def describe_accepted_consumer_fact_schema_versions() -> str: return _describe(CONSUMER_FACT_SCHEMA_IDENTITY_KIND) +class UndeclaredConsumerFactSchemaVersionError(ValueError): + """A Chronicle feed names a per-row row contract this module does not declare.""" + + +def require_accepted_consumer_fact_schema_versions( + versions: object, *, subject: str +) -> None: + """Refuse a feed's row-contract list unless every entry is declared here. + + ``versions`` is a reviewed feed's ``consumer_fact_schema_versions``: the + per-row ``schema_version`` values its rows carry. Feed pins, the + identities restating them, and the pin check against a loaded artifact + call this, so a feed on a row contract Microcosm has not been told about + is refused until :data:`DECLARED_IDENTITIES` declares it. A bare feed + loaded without a pin is not checked: its per-row ids are carried as + published (:mod:`microcosm.build.ledger_artifact`). + """ + if isinstance(versions, (str, bytes)) or not isinstance(versions, (list, tuple)): + raise ValueError( + f"{subject} consumer_fact_schema_versions must be a list of row " + f"contracts, got {type(versions).__name__}." + ) + undeclared = [ + version + for version in versions + if not is_accepted_consumer_fact_schema_version(version) + ] + if undeclared: + raise UndeclaredConsumerFactSchemaVersionError( + f"{subject} names Chronicle consumer-fact row contract(s) " + f"{undeclared!r} that Microcosm does not declare. Declare each in " + "microcosm.build.chronicle_epoch (DECLARED_IDENTITIES) before " + "pinning a feed on it; declared: " + f"{describe_accepted_consumer_fact_schema_versions()}." + ) + + def _describe(kind: str) -> str: return ", ".join(repr(identity) for identity in _identities_of(kind)) diff --git a/packages/microcosm-build/src/microcosm/build/ledger_artifact.py b/packages/microcosm-build/src/microcosm/build/ledger_artifact.py index c181d68e9a..868ef164aa 100644 --- a/packages/microcosm-build/src/microcosm/build/ledger_artifact.py +++ b/packages/microcosm-build/src/microcosm/build/ledger_artifact.py @@ -29,11 +29,14 @@ :meth:`LedgerConsumerArtifact.provenance`, so a release manifest witnesses which era it actually consumed rather than which era it assumed. -Acceptance widened here; nothing narrowed. Only the *manifest* schema id is -gated, and only against the declared ids. Row-level ids and fact keys are -carried as published — real feeds mint rows in namespaces that belong to -neither era, and a consumer that rejected them would fail closed on data that -has always loaded. +Acceptance widened here, with one exception. A load checks the *manifest* +schema id against the declared ids. Row-level ids and fact keys are carried as +published, because real feeds mint rows in namespaces that belong to neither +era and a consumer that rejected them would fail closed on data that has +always loaded. The exception is a load checked against a full Chronicle feed +pin: :func:`validate_chronicle_feed_pin` also requires every row contract the +pin and its rows name to be declared, so a re-pin onto a new Chronicle row +contract has to declare it first. """ from __future__ import annotations @@ -54,6 +57,7 @@ feed_fact_key_epochs, feed_undeclared_fact_key_domains, is_accepted_consumer_artifact_schema_version, + require_accepted_consumer_fact_schema_versions, ) __all__ = [ @@ -95,7 +99,9 @@ def chronicle_feed_identity(feed_pin: Mapping[str, Any]) -> dict[str, Any]: """Select reviewed source commit and loader-verified fields, excluding prose. The source commit is a caller declaration unless the authenticated manifest - also carries it; the artifact loader verifies the remaining six fields. + also carries it; the artifact loader verifies the remaining six fields. An + identity naming a row contract :mod:`microcosm.build.chronicle_epoch` does + not declare is refused. """ missing = sorted(set(_FEED_IDENTITY_FIELDS) - feed_pin.keys()) if missing: @@ -104,6 +110,10 @@ def chronicle_feed_identity(feed_pin: Mapping[str, Any]) -> dict[str, Any]: ) if not isinstance(feed_pin["source_commit"], str) or not feed_pin["source_commit"]: raise ValueError("Chronicle factor feed identity requires a source commit.") + require_accepted_consumer_fact_schema_versions( + feed_pin["consumer_fact_schema_versions"], + subject="Chronicle feed identity", + ) return {field: feed_pin[field] for field in _FEED_IDENTITY_FIELDS} @@ -264,6 +274,12 @@ def validate_chronicle_feed_pin( must agree with both the loaded rows and the manifest, rather than trusting manifest metadata alone. A full artifact pin cannot authenticate a bare facts file's missing manifest metadata. + + Once the pin, the rows and the manifest agree, every row contract they + name must be one :mod:`microcosm.build.chronicle_epoch` declares, so a + feed on a newer Chronicle row contract is refused until it is declared. + The exact comparison runs first, so a pin that differs from its artifact + reports the differing field. """ if not isinstance(feed_pin, Mapping): @@ -289,6 +305,10 @@ def validate_chronicle_feed_pin( expected = feed_pin[field] if type(actual) is not type(expected) or actual != expected: raise ChronicleFeedPinMismatchError(field, expected=expected, actual=actual) + require_accepted_consumer_fact_schema_versions( + feed_pin["consumer_fact_schema_versions"], + subject="Chronicle feed pin and its artifact's rows", + ) def load_ledger_consumer_artifact( @@ -461,14 +481,16 @@ def _load_fact_rows( f"Invalid Chronicle facts JSONL row {line_number}: expected " f"object, got {type(row).__name__}." ) - # A per-row ``schema_version`` is carried, never gated on. Real + # A per-row ``schema_version`` is carried here, not checked. Real # feeds stamp ids from namespaces that are neither era: the pinned # US fiscal-refresh feed (consumer_facts_buildn_v9_4.jsonl) mixes - # 'arch.consumer_fact.v1' with 'ledger.consumer_fact.v1'. Rejecting - # an unrecognized id would fail the build closed on its own pinned - # input, and rejecting is not what dual acceptance asks for. The - # observed ids are reported through - # :attr:`LedgerConsumerArtifact.fact_schema_versions` instead. + # 'arch.consumer_fact.v1' with 'ledger.consumer_fact.v1' and loads + # without a full feed pin. Rejecting an unrecognized id here would + # fail the build closed on its own pinned input. The observed ids + # are reported through + # :attr:`LedgerConsumerArtifact.fact_schema_versions`, and a full + # Chronicle feed pin requires each to be declared + # (:func:`validate_chronicle_feed_pin`). assertion = row.get("assertion", DEFAULT_LEDGER_ASSERTION) if assertion not in ALLOWED_LEDGER_ASSERTIONS: raise ValueError( diff --git a/packages/microcosm-build/src/microcosm/build/ledger_targets.py b/packages/microcosm-build/src/microcosm/build/ledger_targets.py index 509b06de73..7e4d7dd99c 100644 --- a/packages/microcosm-build/src/microcosm/build/ledger_targets.py +++ b/packages/microcosm-build/src/microcosm/build/ledger_targets.py @@ -1573,9 +1573,12 @@ def _validate_package_dimension_value_labels( ) -> None: """Refuse a malformed v5 package-wording map; never read it as a label. - The field is optional. When present it must have the v5 schema's shape: - an object mapping each dimension id to an object of value id to - non-empty text, the shape ``dimension_value_labels`` has. + The field is optional. When present it must have the shape the v5 schema + gives it and ``dimension_value_labels`` alike: an object mapping each + dimension id to an object of value id to non-empty text. This check is + stricter than Microcosm's check of ``dimension_value_labels`` above, which + reads only the entries a target selects and takes any non-null value as + text: nothing reads this field at any one entry, so every entry is checked. """ if isinstance(fact, Mapping): diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py index a66791aa96..4539810b47 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py @@ -18,6 +18,7 @@ from microcosm.build.chronicle_epoch import ( is_accepted_consumer_artifact_schema_version, + require_accepted_consumer_fact_schema_versions, ) @@ -130,6 +131,9 @@ def load_uk_chronicle_feed() -> UKChronicleFeed: or len(set(versions)) != len(versions) ): raise ValueError("UK Chronicle feed consumer_fact_schema_versions is invalid.") + require_accepted_consumer_fact_schema_versions( + versions, subject="UK Chronicle feed" + ) return UKChronicleFeed( **{**raw, "consumer_fact_schema_versions": tuple(versions)}, resource_sha256=hashlib.sha256(content).hexdigest(), diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/chronicle_feed.py b/packages/microcosm-build/src/microcosm/build/us_runtime/chronicle_feed.py index 1af0747dbe..1302771f16 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/chronicle_feed.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/chronicle_feed.py @@ -25,6 +25,7 @@ from microcosm.build.chronicle_epoch import ( is_accepted_consumer_artifact_schema_version, + require_accepted_consumer_fact_schema_versions, ) __all__ = [ @@ -141,6 +142,9 @@ def load_us_chronicle_feed() -> USChronicleFeed: or len(set(versions)) != len(versions) ): raise ValueError("US Chronicle feed consumer_fact_schema_versions is invalid.") + require_accepted_consumer_fact_schema_versions( + versions, subject="US Chronicle feed" + ) expected_scope_sha256 = us_chronicle_feed_scope_sha256() if raw["scope_sha256"] != expected_scope_sha256: raise ValueError( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py index b532375abd..b0e9a1e0fc 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py @@ -374,6 +374,9 @@ def _state_total_proxy_source_district_fact( source_geography_id: str, source_prefix: str, ) -> dict[str, Any]: + # The copy keeps every field it does not rewrite. On a chronicle.consumer_fact.v5 + # row that includes package_dimension_value_labels (microcosm#1509), still + # keyed by the state row's values; no code reads it, so no label changes. proxy = copy.deepcopy(dict(fact)) source_geoid = source_geography_id.removeprefix(source_prefix) state_geography_id = str(_mapping_at(fact, "geography").get("id") or "") @@ -619,6 +622,12 @@ def _translated_fact_shell( crosswalk_basis: str, key: tuple[str, ...], ) -> dict[str, Any]: + # The copy keeps every field it does not rewrite. On a chronicle.consumer_fact.v5 + # row that includes package_dimension_value_labels (microcosm#1509): package + # wording for the group-by district stays keyed by the source district's id, + # not the target's. No code reads the field (labels come from + # dimension_value_labels and layout.groupby_value_label, rewritten below), so + # no label changes; as with v4's geography.publisher_name, it is left as is. translated = copy.deepcopy(dict(fact)) target_geoid = target_geography_id.removeprefix( CURRENT_CONGRESSIONAL_DISTRICT_PREFIX From 30213d71887f93fc0bc2077f9514e507bf428e26 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:43:17 +0400 Subject: [PATCH 5/8] Test that UK and US builds check the row contracts their rows carry Review round 2 of #1510 found that the UK and US builds hold a feed to its committed pin by digest only, and never compare the row contracts the rows carry with the pin's hand-typed list or the declared set. These tests run before the fix; 20 of 55 cases fail, each on an assertion, a pytest.raises that does not raise, or an AssertionError from a compile the check should have prevented: - UK full build (load_uk_full_target_inputs): rows on v6, rows on v4 and v5 against a v4 pin, and unstamped rows are refused before either compiler runs (test_uk_full_targets.py); - UK national role: rows on v6, rows on v5 against a v4 pin, and v6 rows of a feed admitted by --allow-unpinned-feed are refused before target compilation; - US release tool: _check_us_ledger_feed_row_contracts refuses the same cases and _main calls it between the committed-pin check and target compilation (test_us_fiscal_refresh_builder.py); - the UK national and local reference generators refuse v6 rows before writing anything (test_target_reference_authoring.py), and the UK vendoring tool, release certifier, compile-parity tool and DESNZ constituency builder call the UK row-contract checks (test_uk_chronicle_feed.py); - a foreign id such as arch.consumer_fact.v1 is refused without advice to declare it, which chronicle_epoch cannot do. The other cases pass before the fix: the full pin check already refuses rows carrying a row contract the pin leaves out; the committed-resource walk now reads every package's JSON resources at any depth; the UK full-build fixture and national-role test gain the rows' and pin's row-contract lists; the DESNZ tool test's pin lists v4. The epoch test's docstring no longer cites the old US fiscal-refresh feed as the US pin. Refs #1509 Co-Authored-By: Claude Opus 5.5 --- .../shared/test_chronicle_epoch.py | 19 ++-- .../shared/test_ledger_artifact.py | 90 +++++++++++++++--- .../shared/test_target_reference_authoring.py | 61 ++++++++++++ .../engine_free/uk/test_uk_chronicle_feed.py | 38 ++++++++ .../uk/test_uk_desnz_constituency_energy.py | 1 + .../engine_free/uk/test_uk_full_targets.py | 93 ++++++++++++++++++- .../us/test_us_fiscal_refresh_builder.py | 48 ++++++++++ 7 files changed, 324 insertions(+), 26 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py b/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py index 1859697a64..98cd7be9d2 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_chronicle_epoch.py @@ -660,15 +660,16 @@ def test_per_row_schema_id_is_optional(tmp_path) -> None: def test_per_row_schema_id_outside_both_eras_still_loads(tmp_path) -> None: - """Dual acceptance widens what loads; it must not narrow it. - - Real feeds stamp rows from namespaces that are neither era. The pinned US - fiscal-refresh feed ``consumer_facts_buildn_v9_4.jsonl`` declares - ``arch.consumer_fact.v1`` on the overwhelming majority of its rows and - ``ledger.consumer_fact.v1`` on the rest, and it loads through this loader - on the release path. A consumer that gated the per-row id against the two - ids chronicle#143 names would fail the build closed on its own pinned - input — so the id is carried and reported, never gated. + """A load without a full feed pin carries per-row ids it does not declare. + + Feeds loaded by path for diagnostics and experiments, and bare feeds with + no manifest, stamp rows from namespaces that are neither era: the July 2026 + US fiscal-refresh feed ``consumer_facts_buildn_v9_4.jsonl``, the US pin + until 2026-09-18, declares ``arch.consumer_fact.v1`` on most of its rows. + The loader carries and reports such ids. The checks that hold a reviewed + feed to its pin refuse them: ``validate_chronicle_feed_pin``, and the UK and + US builds' row-contract checks (``test_ledger_artifact.py``, + ``test_uk_full_targets.py``, ``test_us_fiscal_refresh_builder.py``). """ facts_path = tmp_path / "consumer_facts.jsonl" rows = [ diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py b/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py index ca0ecc6404..dd754b8979 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py @@ -575,9 +575,8 @@ def test_feed_pin_refuses_rows_that_mix_in_an_undeclared_row_contract(tmp_path): ["chronicle.consumer_fact.v5", _UNDECLARED_ROW_CONTRACT], [_UNDECLARED_ROW_CONTRACT], ), - (["arch.consumer_fact.v1"], ["arch.consumer_fact.v1"]), ], - ids=["undeclared_only", "declared_and_undeclared", "foreign_namespace"], + ids=["undeclared_only", "declared_and_undeclared"], ) def test_feed_identity_refuses_an_undeclared_row_contract( tmp_path, versions, undeclared @@ -599,6 +598,63 @@ def test_feed_identity_refuses_an_undeclared_row_contract( _assert_undeclared_refusal(str(refusal.value), undeclared) +def test_feed_identity_refuses_a_foreign_row_contract_without_advising_a_declaration( + tmp_path, +): + """An id outside Chronicle's namespaces is not a row contract to declare. + + ``_declare`` refuses any namespace outside ``CHRONICLE_NAMESPACES``, so the + message must not tell the reader to declare ``arch.consumer_fact.v1``. + """ + _, pin = write_pinned_artifact(tmp_path / "artifact", [_fact_row()]) + pin["source_commit"] = "c" * 40 + + with pytest.raises(ValueError) as refusal: + chronicle_feed_identity( + { + **pin, + "consumer_fact_schema_versions": [ + "arch.consumer_fact.v1", + _UNDECLARED_ROW_CONTRACT, + ], + } + ) + + message = str(refusal.value) + assert "['arch.consumer_fact.v1'] is not a Chronicle row contract" in message + assert "cannot declare it" in message + assert f"declare {[_UNDECLARED_ROW_CONTRACT]!r}" in message + for declared in _DECLARED_ROW_CONTRACTS: + assert declared in message + + +def test_feed_pin_refuses_rows_carrying_a_row_contract_the_pin_does_not_list( + tmp_path, +): + """Rows carrying a declared row contract the pin's list leaves out are refused.""" + artifact_dir, pin = write_pinned_artifact( + tmp_path / "artifact", + [ + _fact_row(schema_version="chronicle.consumer_fact.v4"), + _fact_row( + aggregate_fact_key="ledger.aggregate_fact.v2:def456", + schema_version="chronicle.consumer_fact.v5", + ), + ], + ) + pinned_v4 = {**pin, "consumer_fact_schema_versions": ["chronicle.consumer_fact.v4"]} + + with pytest.raises(ChronicleFeedPinMismatchError) as refusal: + load_ledger_consumer_artifact(artifact_dir, feed_pin=pinned_v4) + + _assert_pin_mismatch( + refusal.value, + "consumer_fact_schema_versions", + ["chronicle.consumer_fact.v4"], + ["chronicle.consumer_fact.v4", "chronicle.consumer_fact.v5"], + ) + + def test_feed_identity_refuses_a_row_contract_list_that_is_not_a_list(tmp_path): _, pin = write_pinned_artifact(tmp_path / "artifact", [_fact_row()]) pin["source_commit"] = "c" * 40 @@ -612,8 +668,11 @@ def test_feed_identity_refuses_a_row_contract_list_that_is_not_a_list(tmp_path): def _committed_feed_identities(): - """``(resource, path, object)`` for each committed object naming row contracts.""" - build_root = paths_for("microcosm-build").package / "src/microcosm/build" + """``(resource, path, object)`` for each committed object naming row contracts. + + Every JSON resource of every workspace package, at any depth. + """ + packages = paths_for("microcosm-build").repository / "packages" found = [] def walk(node, resource, location): @@ -626,8 +685,8 @@ def walk(node, resource, location): for index, child in enumerate(node): walk(child, resource, f"{location}[{index}]") - for path in sorted(build_root.glob("*/*.json")): - resource = path.relative_to(build_root).as_posix() + for path in sorted(packages.glob("*/src/**/*.json")): + resource = path.relative_to(packages).as_posix() walk(json.loads(path.read_text()), resource, "") return found @@ -637,16 +696,17 @@ def test_committed_feed_pins_and_restated_identities_name_declared_row_contracts identities = _committed_feed_identities() resources = {resource for resource, _location, _identity in identities} + build = "microcosm-build/src/microcosm/build" assert { - "uk/chronicle_feed.json", - "us/chronicle_feed.json", - "nz/chronicle_feed.json", - "be/chronicle_feed.json", - "be/reference_inventory.json", - "be/reference_aging.json", - "nz/reference_inventory.json", - "nz/reference_aging.json", - "nz/msd_pre_activation_reconciliation.json", + f"{build}/uk/chronicle_feed.json", + f"{build}/us/chronicle_feed.json", + f"{build}/nz/chronicle_feed.json", + f"{build}/be/chronicle_feed.json", + f"{build}/be/reference_inventory.json", + f"{build}/be/reference_aging.json", + f"{build}/nz/reference_inventory.json", + f"{build}/nz/reference_aging.json", + f"{build}/nz/msd_pre_activation_reconciliation.json", } <= resources for resource, location, identity in identities: versions = identity["consumer_fact_schema_versions"] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_target_reference_authoring.py b/packages/microcosm-build/tests/engine_free/shared/test_target_reference_authoring.py index 388846d664..78b8848bf1 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_target_reference_authoring.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_target_reference_authoring.py @@ -426,6 +426,67 @@ def test_generate_uk_local_target_references_cli_refuses_incomplete_contract( assert not membership_path.exists() +@pytest.mark.parametrize( + "tool", + [ + "tools/generate_uk_local_target_references.py", + "tools/generate_uk_target_references.py", + ], +) +def test_uk_reference_generators_refuse_rows_on_an_undeclared_row_contract( + tmp_path: Path, tool: str +) -> None: + """The generators read a subset of the pinned feed and check its rows. + + Every row's ``schema_version`` must be a row contract that + ``microcosm.build.chronicle_epoch`` declares and the committed UK pin + lists; the refusal comes before any reference is authored or written. + """ + repo = _TEST_PATHS.repository + contract_path = tmp_path / "contract.json" + facts_path = tmp_path / "facts.jsonl" + output_path = tmp_path / "references.json" + membership_path = tmp_path / "membership.json" + contract_path.write_text(json.dumps(_single_age_contract()), encoding="utf-8") + facts_path.write_text( + "\n".join( + json.dumps({**row, "schema_version": "chronicle.consumer_fact.v6"}) + for row in ( + _area_fact("ons", "population", 10.0, area_id="A1", fact_key="a1-0"), + _area_fact("ons", "population", 20.0, area_id="A1", fact_key="a1-1"), + ) + ) + + "\n", + encoding="utf-8", + ) + arguments = [ + sys.executable, + tool, + "--contract", + str(contract_path), + "--ledger-facts", + str(facts_path), + "--output", + str(output_path), + "--membership-report", + str(membership_path), + ] + if "local" in tool: + crosswalk_path = tmp_path / "crosswalk.json" + crosswalk_path.write_text(json.dumps({"levels": {}}), encoding="utf-8") + arguments += ["--crosswalk", str(crosswalk_path)] + + result = subprocess.run( + arguments, cwd=repo, text=True, capture_output=True, check=False + ) + + assert result.returncode != 0 + assert "['chronicle.consumer_fact.v6']" in result.stderr + assert "microcosm.build.chronicle_epoch" in result.stderr + assert not output_path.exists() + assert not membership_path.exists() + + def _candidate(report: dict, target_id: str, area_id: str) -> dict: candidates = report["targets"][target_id]["geography_levels"]["constituency"][ "candidates" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py index e1482e7c0a..c0bca1685d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py @@ -88,6 +88,44 @@ def test_national_feed_rejects_an_undeclared_row_contract( assert "microcosm.build.chronicle_epoch" in message +@pytest.mark.parametrize( + ("tool", "check"), + [ + ("tools/vendor_uk_ledger_facts.py", "require_uk_chronicle_feed_row_contracts("), + ("tools/certify_uk_release_cut.py", "require_uk_chronicle_feed_row_contracts("), + ( + "tools/generate_uk_target_references.py", + "require_uk_chronicle_feed_subset_row_contracts(", + ), + ( + "tools/generate_uk_local_target_references.py", + "require_uk_chronicle_feed_subset_row_contracts(", + ), + ( + "tools/build_uk_ledger_compile_parity_signed_differences.py", + "require_uk_chronicle_feed_subset_row_contracts(", + ), + ( + "tools/build_uk_desnz_constituency_energy_facts.py", + "require_uk_chronicle_feed_subset_row_contracts(", + ), + ], +) +def test_uk_feed_tools_check_the_row_contracts_of_the_rows_they_read(tool, check): + """Each UK tool that reads the pinned feed's rows holds their row contracts. + + The full-artifact tools hold the rows to the pin's list when the facts are + the committed feed; the tools that read a subset of it, or only some of + its rows, require each row's contract to be one the pin lists. Both refuse + a row contract ``microcosm.build.chronicle_epoch`` does not declare. + """ + from test_support.paths import paths_for + + source = (paths_for("microcosm-build").repository / tool).read_text() + + assert check in source + + def test_committed_feed_pin_check_refuses_a_foreign_artifact(): from microcosm.build.uk_runtime.chronicle_feed import ( UKChronicleFeedPinError, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_desnz_constituency_energy.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_desnz_constituency_energy.py index ba46ee1567..15b47bd8fe 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_desnz_constituency_energy.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_desnz_constituency_energy.py @@ -529,6 +529,7 @@ def test_the_tool_builds_a_resource_the_loader_accepts(tmp_path) -> None: "facts_sha256": facts_sha, "manifest_sha256": hashlib.sha256(manifest.read_bytes()).hexdigest(), "fact_row_count": len(lines), + "consumer_fact_schema_versions": ["chronicle.consumer_fact.v4"], } ), encoding="utf-8", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py index 34b4e6abb6..3461a25a0f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_targets.py @@ -2,6 +2,7 @@ import hashlib import json +import re from dataclasses import replace from datetime import date from types import SimpleNamespace @@ -39,12 +40,16 @@ def prepared(monkeypatch): local = _registry("local") calls = [] artifact = SimpleNamespace( - facts=({"test": True},), facts_sha256="a" * 64, manifest_sha256="b" * 64 + facts=({"test": True},), + facts_sha256="a" * 64, + manifest_sha256="b" * 64, + fact_schema_versions=("chronicle.consumer_fact.v4",), ) pin = SimpleNamespace( facts_sha256=artifact.facts_sha256, manifest_sha256=artifact.manifest_sha256, fact_row_count=1, + consumer_fact_schema_versions=("chronicle.consumer_fact.v4",), to_dict=lambda: { "facts_sha256": artifact.facts_sha256, "manifest_sha256": artifact.manifest_sha256, @@ -115,6 +120,33 @@ def _load(**kwargs): ) +@pytest.mark.parametrize( + ("rows", "message"), + [ + (("chronicle.consumer_fact.v6",), "['chronicle.consumer_fact.v6']"), + ( + ("chronicle.consumer_fact.v4", "chronicle.consumer_fact.v5"), + "consumer_fact_schema_versions", + ), + ((), "consumer_fact_schema_versions"), + ], + ids=["undeclared", "not_in_the_pin", "unstamped"], +) +def test_full_inputs_hold_the_rows_row_contracts_to_the_pin(prepared, rows, message): + """The committed feed's rows must carry exactly the pin's declared row contracts. + + The pin's list is typed by hand at a re-pin, so a feed whose digests match + the pin is also held to it by the row contracts its rows actually carry. + """ + _national, _approved, _local, artifact, calls = prepared + artifact.fact_schema_versions = rows + + with pytest.raises(ValueError, match=re.escape(message)): + _load() + + assert calls == [] + + def test_full_inputs_preserve_band_edges_and_validation_periods(prepared): national, approved, local, artifact, calls = prepared result = _load() @@ -423,12 +455,65 @@ def _with_region_rows(*names): return TargetRegistry([*plain.specs, *region.specs], country="uk"), plain +@pytest.mark.parametrize( + ("pinned_feed", "rows", "message"), + [ + (True, ("chronicle.consumer_fact.v6",), "['chronicle.consumer_fact.v6']"), + (True, ("chronicle.consumer_fact.v5",), "consumer_fact_schema_versions"), + (False, ("chronicle.consumer_fact.v6",), "['chronicle.consumer_fact.v6']"), + ], + ids=["undeclared", "not_in_the_pin", "undeclared_unpinned_feed"], +) +def test_the_national_role_holds_the_rows_row_contracts( + monkeypatch, pinned_feed, rows, message +): + """The national role refuses rows on a row contract the pin or epoch lacks. + + On the committed feed the rows must carry the pin's list. A feed admitted + by ``--allow-unpinned-feed`` is another artifact, so its rows are held only + to the declared row contracts. + """ + pin = SimpleNamespace( + facts_sha256="a" * 64, + manifest_sha256="b" * 64, + consumer_fact_schema_versions=("chronicle.consumer_fact.v4",), + to_dict=lambda: {}, + ) + artifact = SimpleNamespace( + facts=({"feed": True},), + facts_sha256=pin.facts_sha256 if pinned_feed else "c" * 64, + manifest_sha256=pin.manifest_sha256, + fact_schema_versions=rows, + provenance=lambda: {"artifact_id": "synthetic"}, + ) + monkeypatch.setattr( + runtime, "load_ledger_consumer_artifact", lambda *a, **kw: artifact + ) + monkeypatch.setattr( + runtime, "require_committed_uk_chronicle_feed_pin", lambda *a, **kw: pin + ) + + def compile_national(facts, *, target_period): + raise AssertionError("the rows reached target compilation") + + monkeypatch.setattr(runtime, "compile_uk_target_registry", compile_national) + + with pytest.raises(ValueError, match=re.escape(message)): + runtime.load_uk_national_target_inputs( + "facts.jsonl", + calibration_year=2025, + exclusions_evaluated_on=date(2026, 10, 16), + allow_unpinned_feed=not pinned_feed, + ) + + def test_the_national_line_drops_local_only_rows_before_reconciliation(monkeypatch): compiled, plain = _with_region_rows("retained", "excluded") artifact = SimpleNamespace( facts=({"feed": True},), facts_sha256="a" * 64, manifest_sha256="b" * 64, + fact_schema_versions=("chronicle.consumer_fact.v4",), provenance=lambda: {"artifact_id": "synthetic"}, ) monkeypatch.setattr( @@ -437,7 +522,11 @@ def test_the_national_line_drops_local_only_rows_before_reconciliation(monkeypat monkeypatch.setattr( runtime, "require_committed_uk_chronicle_feed_pin", - lambda facts_sha256, **kwargs: SimpleNamespace(to_dict=lambda: {}), + lambda facts_sha256, **kwargs: SimpleNamespace( + facts_sha256=facts_sha256, + consumer_fact_schema_versions=("chronicle.consumer_fact.v4",), + to_dict=lambda: {}, + ), ) compiled_facts = [] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 1b4d446838..d9bfcc58e7 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -16793,6 +16793,54 @@ def test_committed_feed_pin_is_waived_only_by_the_flag() -> None: ) +@pytest.mark.parametrize( + ("facts_sha256", "rows", "message"), + [ + ("a" * 64, ("chronicle.consumer_fact.v6",), "['chronicle.consumer_fact.v6']"), + ("a" * 64, ("chronicle.consumer_fact.v5",), "consumer_fact_schema_versions"), + ("a" * 64, (), "consumer_fact_schema_versions"), + ("f" * 64, ("chronicle.consumer_fact.v6",), "['chronicle.consumer_fact.v6']"), + ], + ids=["undeclared", "not_in_the_pin", "unstamped", "undeclared_unpinned_feed"], +) +def test_feed_row_contract_check_refuses_rows_the_pin_or_epoch_lacks( + facts_sha256, rows, message +) -> None: + """The release build holds the rows' row contracts, not only the pin's list. + + On the committed feed (its facts digest) the rows must carry exactly the + pin's list, which a re-pin types by hand; a feed admitted by + ``--allow-unpinned-feed`` is held only to the declared row contracts. + """ + builder = _load_builder_module() + check = getattr(builder, "_check_us_ledger_feed_row_contracts", None) + assert callable(check) + bare = _feed_pin(bare=True) + assert bare.consumer_fact_schema_versions == ("chronicle.consumer_fact.v3",) + check(("chronicle.consumer_fact.v3",), facts_sha256="a" * 64, pin=bare) + check(("chronicle.consumer_fact.v5",), facts_sha256="f" * 64, pin=bare) + + with pytest.raises(ValueError) as refusal: + check(rows, facts_sha256=facts_sha256, pin=bare) + + assert message in str(refusal.value) + + +def test_main_checks_the_feed_row_contracts_before_compiling_targets() -> None: + builder = _load_builder_module() + source = inspect.getsource(builder._main) + call = ( + "_check_us_ledger_feed_row_contracts(\n" + " ledger_artifact.fact_schema_versions," + ) + assert call in source + assert source.count("_check_us_ledger_feed_row_contracts(") == 1 + pinned = source.index("_check_committed_us_ledger_feed_pin(") + checked = source.index(call) + compiled = source.index("target_registry = compile_us_fiscal_target_registry(") + assert pinned < checked < compiled + + def _minimal_pin_argv() -> list[str]: return [ "--base-h5", From 049b30d1a70e0536d981c3314993932191550404 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:00:35 +0400 Subject: [PATCH 6/8] Check the row contracts UK and US builds read, not only the pin's list Review round 2 of #1510 found that the UK and US builds hold a feed to its committed pin by digest only, so a re-pin that moved the digests but left the hand-typed list naming v5 would compile rows on an undeclared row contract. A feed whose pin or rows name one is now refused. - ledger_artifact: consumer_fact_schema_versions_of(rows), which LedgerConsumerArtifact.fact_schema_versions now uses, and validate_pinned_row_contracts. It refuses rows on an undeclared row contract and, given a pin's list, rows whose contracts are not exactly that list, or (for a subset) not all on it. - uk_runtime.chronicle_feed: require_uk_chronicle_feed_row_contracts (exact on the committed feed, declared-only on a feed admitted by --allow-unpinned-feed) and require_uk_chronicle_feed_subset_row_contracts. The full and national target loaders, the vendoring tool and the release certifier call the first; the national and local reference generators (which now read their facts first), the compile-parity tool and the DESNZ constituency builder call the second. The DESNZ builder loads the pin module by path, as it does the energy module, so a first run still needs no UK country spec. - build_us_fiscal_refresh_release: _check_us_ledger_feed_row_contracts, called after the committed-pin digest check; the US pin describes a bare feed, so the full feed-pin check cannot run on it. - chronicle_epoch: the refusal names an id outside Chronicle's namespaces as not a Chronicle row contract, which chronicle_epoch cannot declare, instead of advising a declaration. The subject and message no longer need verb agreement, and the redundant str/bytes test is gone. - The loader comment and docstrings, the UK and US re-pin guides and the changelog give the reasons that hold for loads without a pin (feeds loaded by path for diagnostics and experiments, and bare feeds); the old fiscal-refresh feed is no longer called the US pin. - Test fakes standing in for a loaded artifact or a pin in the UK national graph and rowwise national role tests, and the US release tool's corridor test, carry the row-contract lists the new checks read. Refs #1509 Co-Authored-By: Claude Opus 5.5 --- .../1509-chronicle-consumer-fact-v5.added.md | 6 +- docs/uk-chronicle-feed-repin.md | 26 +++-- docs/us-chronicle-feed-repin.md | 12 +- .../src/microcosm/build/chronicle_epoch.py | 62 +++++++--- .../src/microcosm/build/ledger_artifact.py | 107 +++++++++++++----- .../build/uk_runtime/chronicle_feed.py | 59 ++++++++++ .../build/uk_runtime/full_targets.py | 7 ++ .../engine_free/uk/test_uk_national_graph.py | 5 +- .../uk/test_uk_rowwise_national_role.py | 6 +- .../us/test_us_fiscal_refresh_builder.py | 1 + ...uild_uk_desnz_constituency_energy_facts.py | 39 +++++-- ...edger_compile_parity_signed_differences.py | 9 ++ tools/build_us_fiscal_refresh_release.py | 38 ++++++- tools/certify_uk_release_cut.py | 8 ++ tools/generate_uk_local_target_references.py | 11 +- tools/generate_uk_target_references.py | 19 +++- tools/vendor_uk_ledger_facts.py | 6 + 17 files changed, 343 insertions(+), 78 deletions(-) diff --git a/changelog.d/1509-chronicle-consumer-fact-v5.added.md b/changelog.d/1509-chronicle-consumer-fact-v5.added.md index 3328a83d25..fa4206e3eb 100644 --- a/changelog.d/1509-chronicle-consumer-fact-v5.added.md +++ b/changelog.d/1509-chronicle-consumer-fact-v5.added.md @@ -1,7 +1,9 @@ -Declare Chronicle's `chronicle.consumer_fact.v5` row contract, the one Chronicle `main` writes since PolicyEngine/chronicle#377 (`44eebe037`), and refuse a Chronicle feed pinned on a row contract Microcosm does not declare (microcosm#1509). +Declare Chronicle's `chronicle.consumer_fact.v5` row contract, the one Chronicle `main` writes since PolicyEngine/chronicle#377 (`44eebe037`), and refuse a Chronicle feed whose pin or rows name a row contract Microcosm does not declare (microcosm#1509). - **The declaration.** `microcosm.build.chronicle_epoch` declares v5 in the chronicle epoch beside `ledger.consumer_fact.v1` and `chronicle.consumer_fact.v2` to v4, matching Chronicle's own `CONSUMER_FACT_ROW_CONTRACTS`. Before this change nothing outside the tests read that list, so a v5 feed would already have loaded. -- **The refusal.** A pin whose `consumer_fact_schema_versions` names an undeclared row contract is refused, with a message that names it and says to declare it in `chronicle_epoch` first. The checks are the UK and US pin loaders, `chronicle_feed_identity` (used by reference factors, money normalization and the transport composer), and `validate_chronicle_feed_pin` once a pin, its artifact's rows and its manifest agree. The transport kernels reach the last two. Every committed pin (UK, NZ and Belgium on v4, the US on v3) and every resource restating one names declared contracts. A feed loaded without a full pin, such as the US fiscal-refresh feed with its `arch.consumer_fact.v1` rows, still carries its per-row ids unchecked. +- **Pins.** The UK and US pin loaders, `chronicle_feed_identity` (used by reference factors, money normalization and the transport composer) and `validate_chronicle_feed_pin` refuse a pin whose `consumer_fact_schema_versions` names an undeclared row contract. The message names it and says to declare it in `chronicle_epoch` first; an id outside Chronicle's namespaces, such as `arch.consumer_fact.v1`, is named as not a Chronicle row contract, which `chronicle_epoch` cannot declare. +- **Rows.** `validate_chronicle_feed_pin` compares the rows' `schema_version` values with the pin before it checks them, so the transport kernels and the NZ and Belgian generators, which use it, cover the rows. The UK and US builds hold their feeds to committed pins by digest, so they now also check the rows they read (`validate_pinned_row_contracts`): rows on an undeclared row contract are refused, and rows of the committed feed must carry exactly the pin's list, which a re-pin types by hand. That covers the UK full and national builds, the UK vendoring tool and release certifier, and the US fiscal-refresh release tool. The UK reference generators, the compile-parity tool and the DESNZ constituency builder, which read a subset of the feed or only some of its rows, require each row's contract to be one the pin lists. A feed admitted by `--allow-unpinned-feed` is held to the declared row contracts only. +- **Loads without a pin.** A feed loaded by path for diagnostics and experiments, or a bare feed with no manifest that no reviewed pin covers, still carries its per-row ids unchecked; the July 2026 US fiscal-refresh feed, whose rows carry `arch.consumer_fact.v1`, loads that way. - **The new field.** v5 adds an optional `package_dimension_value_labels`: a source package's own wording of a dimension value where it differs from the label the row exports. The loader keeps it on the parsed rows, as it keeps every published field, and Microcosm never takes a label from it: each value keeps the one label Chronicle gives it in `dimension_value_labels` and `layout.groupby_value_label`. - **Its shape.** When a target with a calibration hierarchy (a provider and category seed) is compiled, each selected fact's field must be an object of dimension id to an object of value id to non-empty text, the shape the v5 schema requires. Any other shape raises `LedgerHierarchyMetadataError`, the error a missing Chronicle label raises, and callers treat it the same way: the UK compilers report the reference as unsupported, which stops the build year and is recorded for validation periods. This check walks every entry of the field, where the check of `dimension_value_labels` reads only the selected entries. - **No feed pin moves.** The UK, US, NZ and Belgian feeds stay on their current Chronicle commits until their own re-pins. diff --git a/docs/uk-chronicle-feed-repin.md b/docs/uk-chronicle-feed-repin.md index 067f3601fd..9e473f0490 100644 --- a/docs/uk-chronicle-feed-repin.md +++ b/docs/uk-chronicle-feed-repin.md @@ -24,14 +24,24 @@ restate it: `tests/engine_free/uk/test_uk_chronicle_feed.py` (the pin and the consumer-fact schema digest) and `tests/engine_free/uk/test_uk_full_targets.py` (`test_current_national_and_local_pins_share_one_reviewed_identity`). A schema digest moves for every country whose feed is built after the Chronicle change, -so the NZ pin (`nz/chronicle_feed.json`) moves at its own next re-pin. A feed -whose rows carry a newer per-row `schema_version` needs that row contract -declared in `microcosm.build.chronicle_epoch` first: the UK pin loader, the -identity check the factor and transport code use (`chronicle_feed_identity`) -and the feed-pin check against a loaded artifact (`validate_chronicle_feed_pin`) -refuse a pin naming an undeclared one (microcosm#1509). Microcosm declares -`ledger.consumer_fact.v1` and `chronicle.consumer_fact.v2` to `v5`; the v5 -schema's digest is `acd33a09…`. Then +so the NZ pin (`nz/chronicle_feed.json`) moves at its own next re-pin. Set the +pin's `consumer_fact_schema_versions` from the manifest, not by hand from the +last pin. A feed whose rows carry a newer per-row `schema_version` needs that +row contract declared in `microcosm.build.chronicle_epoch` first +(microcosm#1509): + +- the UK pin loader, and `chronicle_feed_identity` and + `validate_chronicle_feed_pin` where the factor and transport code use them, + refuse a pin naming an undeclared row contract; +- the UK code that reads the feed's rows refuses rows on an undeclared row + contract, or rows whose contracts differ from the pin's list: the full and + national builds, the vendoring tool and the release certifier on the + committed artifact (exactly the pin's list), and the reference generators, + the compile-parity tool and the DESNZ constituency builder on a subset or + on the rows they parse (each a contract the pin lists). + +Microcosm declares `ledger.consumer_fact.v1` and `chronicle.consumer_fact.v2` +to `v5`; the v5 schema's digest is `acd33a09…`. Then regenerate. Vendor the per-concern fact resources (step 5) before the national generator: the generator refuses to compile against a vendored resource whose feed identity diff --git a/docs/us-chronicle-feed-repin.md b/docs/us-chronicle-feed-repin.md index 95416f3889..10821f1997 100644 --- a/docs/us-chronicle-feed-repin.md +++ b/docs/us-chronicle-feed-repin.md @@ -17,10 +17,14 @@ new pin. The US feed is on `chronicle.consumer_fact.v3`; Chronicle `main` writes `chronicle.consumer_fact.v5` since PolicyEngine/chronicle#377. The US pin -loader, like the UK one and the feed-pin check against a loaded artifact, -refuses a pin whose `consumer_fact_schema_versions` names a row contract that -`microcosm.build.chronicle_epoch` does not declare (microcosm#1509). v5 is -declared, and its schema digest is `acd33a09…`. +loader refuses a pin whose `consumer_fact_schema_versions` names a row contract +that `microcosm.build.chronicle_epoch` does not declare (microcosm#1509). The +pin describes a bare feed, so the full feed-pin check cannot run on it; +instead the fiscal-refresh release tool, after its digest check, refuses rows +on an undeclared row contract and, on the committed feed, rows whose contracts +are not exactly the pin's list (`_check_us_ledger_feed_row_contracts`). So set +the list from the rebuilt feed's rows at a re-pin: `tools/build_us_chronicle_feed.py` +does not write it. v5 is declared, and its schema digest is `acd33a09…`. v5 rows may carry `package_dimension_value_labels`, a package's own wording of a value where it differs from Chronicle's label. The congressional-district diff --git a/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py b/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py index 176a3aa289..334b311330 100644 --- a/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py +++ b/packages/microcosm-build/src/microcosm/build/chronicle_epoch.py @@ -592,19 +592,20 @@ class UndeclaredConsumerFactSchemaVersionError(ValueError): def require_accepted_consumer_fact_schema_versions( versions: object, *, subject: str ) -> None: - """Refuse a feed's row-contract list unless every entry is declared here. - - ``versions`` is a reviewed feed's ``consumer_fact_schema_versions``: the - per-row ``schema_version`` values its rows carry. Feed pins, the - identities restating them, and the pin check against a loaded artifact - call this, so a feed on a row contract Microcosm has not been told about - is refused until :data:`DECLARED_IDENTITIES` declares it. A bare feed - loaded without a pin is not checked: its per-row ids are carried as - published (:mod:`microcosm.build.ledger_artifact`). + """Refuse a row-contract list unless every entry is declared here. + + ``versions`` is a feed's ``consumer_fact_schema_versions``, or the + ``schema_version`` values its rows carry. Feed pins, the identities + restating them, the full feed-pin check, and the UK and US builds' checks + of the rows they compile call this, so a feed on a row contract Microcosm + has not been told about is refused until :data:`DECLARED_IDENTITIES` + declares it. An id outside Chronicle's namespaces is not a Chronicle row + contract at all, and :func:`_declare` cannot declare it, so the message + says so instead of advising a declaration. """ - if isinstance(versions, (str, bytes)) or not isinstance(versions, (list, tuple)): + if not isinstance(versions, (list, tuple)): raise ValueError( - f"{subject} consumer_fact_schema_versions must be a list of row " + f"{subject}: consumer_fact_schema_versions must be a list of row " f"contracts, got {type(versions).__name__}." ) undeclared = [ @@ -612,14 +613,39 @@ def require_accepted_consumer_fact_schema_versions( for version in versions if not is_accepted_consumer_fact_schema_version(version) ] - if undeclared: - raise UndeclaredConsumerFactSchemaVersionError( - f"{subject} names Chronicle consumer-fact row contract(s) " - f"{undeclared!r} that Microcosm does not declare. Declare each in " - "microcosm.build.chronicle_epoch (DECLARED_IDENTITIES) before " - "pinning a feed on it; declared: " - f"{describe_accepted_consumer_fact_schema_versions()}." + if not undeclared: + return + declarable = [ + version + for version in undeclared + if isinstance(version, str) + and version.partition(".")[0] in CHRONICLE_NAMESPACES + ] + foreign = [version for version in undeclared if version not in declarable] + problems = [] + if declarable: + problems.append( + f"undeclared Chronicle row contract(s) {declarable!r}: declare " + f"{declarable!r} in microcosm.build.chronicle_epoch " + "(DECLARED_IDENTITIES) before pinning a feed on it" ) + if foreign: + single = len(foreign) == 1 + problems.append( + f"{foreign!r} " + + ( + "is not a Chronicle row contract: its namespace is" + if single + else "are not Chronicle row contracts: their namespaces are" + ) + + f" not among {sorted(CHRONICLE_NAMESPACES)!r}, so " + "microcosm.build.chronicle_epoch cannot declare " + + ("it" if single else "them") + ) + raise UndeclaredConsumerFactSchemaVersionError( + f"{subject}: " + "; ".join(problems) + ". Declared row contracts: " + f"{describe_accepted_consumer_fact_schema_versions()}." + ) def _describe(kind: str) -> str: diff --git a/packages/microcosm-build/src/microcosm/build/ledger_artifact.py b/packages/microcosm-build/src/microcosm/build/ledger_artifact.py index 868ef164aa..f84b4044c3 100644 --- a/packages/microcosm-build/src/microcosm/build/ledger_artifact.py +++ b/packages/microcosm-build/src/microcosm/build/ledger_artifact.py @@ -29,14 +29,16 @@ :meth:`LedgerConsumerArtifact.provenance`, so a release manifest witnesses which era it actually consumed rather than which era it assumed. -Acceptance widened here, with one exception. A load checks the *manifest* -schema id against the declared ids. Row-level ids and fact keys are carried as -published, because real feeds mint rows in namespaces that belong to neither -era and a consumer that rejected them would fail closed on data that has -always loaded. The exception is a load checked against a full Chronicle feed -pin: :func:`validate_chronicle_feed_pin` also requires every row contract the -pin and its rows name to be declared, so a re-pin onto a new Chronicle row -contract has to declare it first. +Acceptance widened here; a reviewed pin narrows it. A load checks the +*manifest* schema id against the declared ids. Row-level ids and fact keys are +carried as published, because this loader also serves feeds loaded by path +for diagnostics and experiments, and bare feeds with no manifest, some minted +in namespaces that belong to neither era. A feed held to a reviewed pin is +narrower: :func:`validate_chronicle_feed_pin` requires every row contract the +full pin and its rows name to be declared, and +:func:`validate_pinned_row_contracts` does the same for the UK and US builds, +which hold their feeds to committed pins by digest. So a re-pin onto a new +Chronicle row contract has to declare it first. """ from __future__ import annotations @@ -44,7 +46,7 @@ import hashlib import json from argparse import ArgumentParser, Namespace -from collections.abc import Mapping +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import Any @@ -71,9 +73,11 @@ "LexicalFloat", "add_ledger_artifact_args", "chronicle_feed_identity", + "consumer_fact_schema_versions_of", "load_ledger_consumer_artifact", "resolve_ledger_artifact", "validate_chronicle_feed_pin", + "validate_pinned_row_contracts", ] #: The id Microcosm's own minted artifacts still declare. Frozen at v1 by @@ -217,16 +221,13 @@ def undeclared_fact_key_domains(self) -> tuple[str, ...]: def fact_schema_versions(self) -> tuple[str, ...]: """Distinct per-row ``schema_version`` values in the feed, sorted. - Reported verbatim and never gated on — see :func:`_load_fact_rows`. - Empty when no row declares one; more than one entry when a feed mixes - producers, which the pinned US fiscal-refresh feed already does. + Reported verbatim; the loader does not check them (see + :func:`_load_fact_rows`). :func:`validate_chronicle_feed_pin` and the + UK and US builds' checks (:func:`validate_pinned_row_contracts`) hold + them to a reviewed pin and the declared row contracts. Empty when no + row declares one; more than one entry when a feed mixes producers. """ - observed = { - str(row["schema_version"]) - for row in self.facts - if isinstance(row, dict) and row.get("schema_version") is not None - } - return tuple(sorted(observed)) + return consumer_fact_schema_versions_of(self.facts) def provenance(self) -> dict[str, Any]: """Chronicle-artifact identity block for build and release manifests. @@ -311,6 +312,54 @@ def validate_chronicle_feed_pin( ) +def consumer_fact_schema_versions_of(rows: Iterable[Any]) -> tuple[str, ...]: + """Distinct per-row ``schema_version`` values of ``rows``, sorted. + + The same reading as :attr:`LedgerConsumerArtifact.fact_schema_versions`, + for tools that parse a feed's rows themselves. + """ + observed = { + str(row["schema_version"]) + for row in rows + if isinstance(row, Mapping) and row.get("schema_version") is not None + } + return tuple(sorted(observed)) + + +def validate_pinned_row_contracts( + fact_schema_versions: Iterable[str], + *, + pinned: Sequence[str] | None, + subject: str, + subset: bool = False, +) -> None: + """Hold the row contracts a consumer's rows carry to the declared set and a pin. + + For builds that hold a feed to its committed pin by digest, where + :func:`validate_chronicle_feed_pin` cannot run (the US pin describes a + bare feed) or is not used. Every row contract the rows carry must be + declared in :mod:`microcosm.build.chronicle_epoch`. With ``pinned``, the + rows must also carry exactly the pin's ``consumer_fact_schema_versions``, + which a re-pin types by hand; with ``subset``, only row contracts the pin + lists, for a reader of part of the pinned feed. ``pinned=None`` is for a + feed that is not the pinned one (a reviewed ``--allow-unpinned-feed`` + run), whose rows are held to the declared row contracts only. + """ + observed = list(fact_schema_versions) + require_accepted_consumer_fact_schema_versions(observed, subject=f"{subject} rows") + if pinned is None: + return + expected = list(pinned) + if subset: + matches = set(observed) <= set(expected) + else: + matches = set(observed) == set(expected) + if not matches: + raise ChronicleFeedPinMismatchError( + "consumer_fact_schema_versions", expected=expected, actual=observed + ) + + def load_ledger_consumer_artifact( path: str | Path, *, @@ -481,16 +530,18 @@ def _load_fact_rows( f"Invalid Chronicle facts JSONL row {line_number}: expected " f"object, got {type(row).__name__}." ) - # A per-row ``schema_version`` is carried here, not checked. Real - # feeds stamp ids from namespaces that are neither era: the pinned - # US fiscal-refresh feed (consumer_facts_buildn_v9_4.jsonl) mixes - # 'arch.consumer_fact.v1' with 'ledger.consumer_fact.v1' and loads - # without a full feed pin. Rejecting an unrecognized id here would - # fail the build closed on its own pinned input. The observed ids - # are reported through - # :attr:`LedgerConsumerArtifact.fact_schema_versions`, and a full - # Chronicle feed pin requires each to be declared - # (:func:`validate_chronicle_feed_pin`). + # A per-row ``schema_version`` is carried here, not checked. This + # loader also serves feeds loaded by path for diagnostics and + # experiments, and bare feeds with no manifest, some stamped from + # namespaces that are neither era (the July 2026 US fiscal-refresh + # feed, consumer_facts_buildn_v9_4.jsonl, mixes + # 'arch.consumer_fact.v1' with 'ledger.consumer_fact.v1'). The + # observed ids are reported through + # :attr:`LedgerConsumerArtifact.fact_schema_versions`; the checks + # that hold a reviewed feed to its pin refuse an undeclared one: + # :func:`validate_chronicle_feed_pin` for a full pin, and + # :func:`validate_pinned_row_contracts` in the UK and US builds, + # which hold their feeds to committed pins by digest. assertion = row.get("assertion", DEFAULT_LEDGER_ASSERTION) if assertion not in ALLOWED_LEDGER_ASSERTIONS: raise ValueError( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py index 4539810b47..2950f54d33 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py @@ -13,6 +13,7 @@ import hashlib import json import re +from collections.abc import Iterable, Sequence from dataclasses import asdict, dataclass from importlib.resources import files @@ -20,6 +21,7 @@ is_accepted_consumer_artifact_schema_version, require_accepted_consumer_fact_schema_versions, ) +from microcosm.build.ledger_artifact import validate_pinned_row_contracts @dataclass(frozen=True) @@ -84,6 +86,63 @@ def require_committed_uk_chronicle_feed_pin( return pin +def require_uk_chronicle_feed_row_contracts( + fact_schema_versions: Iterable[str], + *, + facts_sha256: str, + pin: UKChronicleFeed | None = None, +) -> UKChronicleFeed: + """Hold the row contracts a loaded UK feed's rows carry to the pin. + + The digest checks say the rows are the committed feed's; this says what + they are. Rows of the committed feed (``facts_sha256`` is the pin's) must + carry exactly the pin's ``consumer_fact_schema_versions``, which a re-pin + types by hand. Rows of another feed, admitted by ``--allow-unpinned-feed`` + for a reviewed diagnostic run, must carry declared row contracts only. + Returns the pin they were checked against. + """ + + pin = pin or load_uk_chronicle_feed() + validate_pinned_row_contracts( + fact_schema_versions, + pinned=( + pin.consumer_fact_schema_versions + if facts_sha256 == pin.facts_sha256 + else None + ), + subject="UK Chronicle feed", + ) + return pin + + +def require_uk_chronicle_feed_subset_row_contracts( + fact_schema_versions: Iterable[str], + *, + pinned_versions: Sequence[str] | None = None, +) -> None: + """Hold the row contracts of part of the pinned UK feed to the pin. + + For the tools that parse the feed's rows themselves: the reference + generators and the compile-parity tool, which read the guide's subsets + of the pinned artifact, and the DESNZ constituency builder, which parses + only the rows it needs. A subset need not carry every row contract the + pin lists, so each row's contract must be one the pin lists + (``pinned_versions``, by default the committed pin's) and one + :mod:`microcosm.build.chronicle_epoch` declares. + """ + + validate_pinned_row_contracts( + fact_schema_versions, + pinned=( + load_uk_chronicle_feed().consumer_fact_schema_versions + if pinned_versions is None + else pinned_versions + ), + subject="UK Chronicle feed", + subset=True, + ) + + def _feed_path(): return files("microcosm.build.uk").joinpath("chronicle_feed.json") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py index 2a9e1fd30c..c77406a135 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py @@ -17,6 +17,7 @@ from microcosm.build.uk_runtime.chronicle_feed import ( load_uk_chronicle_feed, require_committed_uk_chronicle_feed_pin, + require_uk_chronicle_feed_row_contracts, ) from microcosm.build.uk_runtime.derived_facts import ( load_uk_derived_facts, @@ -195,6 +196,9 @@ def load_uk_full_target_inputs( "UK full-build Chronicle fact row count differs from the reviewed " "national and local feed pins." ) + require_uk_chronicle_feed_row_contracts( + artifact.fact_schema_versions, facts_sha256=artifact.facts_sha256, pin=pin + ) year = ( load_uk_frs_release().calibration_year if calibration_year is None @@ -397,6 +401,9 @@ def load_uk_national_target_inputs( manifest_sha256=artifact.manifest_sha256, allow_unpinned_feed=bool(allow_unpinned_feed), ) + require_uk_chronicle_feed_row_contracts( + artifact.fact_schema_versions, facts_sha256=artifact.facts_sha256, pin=pin + ) year = ( load_uk_frs_release().calibration_year if calibration_year is None diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_graph.py index b828a47ef3..97e97b1b5e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_graph.py @@ -84,6 +84,7 @@ def _patch_target_inputs(monkeypatch, registry): facts=None, facts_sha256="1" * 64, manifest_sha256="2" * 64, + fact_schema_versions=("chronicle.consumer_fact.v4",), provenance=lambda: { "facts_sha256": "1" * 64, "manifest_sha256": "2" * 64, @@ -97,7 +98,9 @@ def _patch_target_inputs(monkeypatch, registry): full_targets, "require_committed_uk_chronicle_feed_pin", lambda facts_sha256, **kwargs: SimpleNamespace( - to_dict=lambda: {"facts_sha256": "1" * 64, "manifest_sha256": "2" * 64} + facts_sha256="1" * 64, + consumer_fact_schema_versions=("chronicle.consumer_fact.v4",), + to_dict=lambda: {"facts_sha256": "1" * 64, "manifest_sha256": "2" * 64}, ), ) monkeypatch.setattr( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py index b507e72b3c..809b28aec4 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_national_role.py @@ -138,6 +138,7 @@ def _national_inputs(monkeypatch, tmp_path: Path, *, registry=None): facts=None, facts_sha256="1" * 64, manifest_sha256="2" * 64, + fact_schema_versions=("chronicle.consumer_fact.v4",), path=ledger_dir, provenance=lambda: { "facts_sha256": "1" * 64, @@ -146,7 +147,9 @@ def _national_inputs(monkeypatch, tmp_path: Path, *, registry=None): }, ) pin = SimpleNamespace( - to_dict=lambda: {"facts_sha256": "1" * 64, "manifest_sha256": "2" * 64} + facts_sha256="1" * 64, + consumer_fact_schema_versions=("chronicle.consumer_fact.v4",), + to_dict=lambda: {"facts_sha256": "1" * 64, "manifest_sha256": "2" * 64}, ) monkeypatch.setattr( full_targets, "load_ledger_consumer_artifact", lambda path, **kwargs: artifact @@ -1218,6 +1221,7 @@ def _foreign_artifact(ledger_dir: Path, *, facts_sha256: str, manifest_sha256): facts=None, facts_sha256=facts_sha256, manifest_sha256=manifest_sha256, + fact_schema_versions=("chronicle.consumer_fact.v4",), path=ledger_dir, provenance=lambda: { "facts_sha256": facts_sha256, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index d9bfcc58e7..2f6f30eb2b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -7310,6 +7310,7 @@ def fake_written_stored_input_check(path, pre_export): facts=({"fact": 1},), facts_sha256="facts-sha", manifest_sha256=None, + fact_schema_versions=(), provenance=lambda: { "path_name": "facts.jsonl", "fact_row_count": 1, diff --git a/tools/build_uk_desnz_constituency_energy_facts.py b/tools/build_uk_desnz_constituency_energy_facts.py index bd41271fd0..5cca8d0892 100644 --- a/tools/build_uk_desnz_constituency_energy_facts.py +++ b/tools/build_uk_desnz_constituency_energy_facts.py @@ -38,22 +38,23 @@ import numpy as np +from microcosm.build.ledger_artifact import consumer_fact_schema_versions_of + REPO_ROOT = Path(__file__).resolve().parents[1] -def _load_energy_module(): - """Load the pure library module by path. +def _load_uk_runtime_module(name: str): + """Load one pure UK runtime module by path. - Importing it through ``microcosm.build`` would load the UK country spec, - which refuses a package whose declared resources are missing: the very - resource this tool writes on a first run. + Importing it through ``microcosm.build.uk_runtime`` would load the UK + country spec, which refuses a package whose declared resources are + missing: the very resource this tool writes on a first run. """ path = ( - REPO_ROOT - / "packages/microcosm-build/src/microcosm/build/uk_runtime/desnz_constituency_energy.py" + REPO_ROOT / f"packages/microcosm-build/src/microcosm/build/uk_runtime/{name}.py" ) - spec = importlib.util.spec_from_file_location("desnz_constituency_energy", path) + spec = importlib.util.spec_from_file_location(name, path) if spec is None or spec.loader is None: raise RuntimeError(f"cannot import {path}") module = importlib.util.module_from_spec(spec) @@ -62,7 +63,8 @@ def _load_energy_module(): return module -energy = _load_energy_module() +energy = _load_uk_runtime_module("desnz_constituency_energy") +chronicle_feed = _load_uk_runtime_module("chronicle_feed") UK_PACKAGE = REPO_ROOT / "packages/microcosm-build/src/microcosm/build/uk" DEFAULT_FEED_PIN = UK_PACKAGE / "chronicle_feed.json" @@ -152,6 +154,19 @@ def _feed_identity(pin_path: Path, manifest_path: Path) -> dict[str, object]: } +def _pinned_row_contracts(pin_path: Path) -> list[str]: + """The row contracts the feed pin lists, refusing a pin that lists none.""" + + versions = json.loads(pin_path.read_text(encoding="utf-8")).get( + "consumer_fact_schema_versions" + ) + if not isinstance(versions, list) or not versions: + raise ValueError( + f"{pin_path} lists no consumer_fact_schema_versions for the feed's rows." + ) + return versions + + def _read_feed( path: Path, *, markers: tuple[bytes, ...], identity: Mapping[str, object] ) -> list[dict]: @@ -263,6 +278,12 @@ def main(argv: list[str] | None = None) -> int: ) _log("Streaming the pinned feed ...") facts = _read_feed(args.ledger_facts, markers=markers, identity=identity) + # Only the rows this build reads are parsed: each one's row contract must + # be one the pin lists and chronicle_epoch declares. + chronicle_feed.require_uk_chronicle_feed_subset_row_contracts( + consumer_fact_schema_versions_of(facts), + pinned_versions=_pinned_row_contracts(args.feed_pin), + ) _log("Loading the atomic supports ...") ew, ew_sha = _load_support(args.ew_support, "uk_ew_output_area_2021", provenance) scotland, scotland_sha = _load_support( diff --git a/tools/build_uk_ledger_compile_parity_signed_differences.py b/tools/build_uk_ledger_compile_parity_signed_differences.py index bbf81c31f6..3adb39c87e 100644 --- a/tools/build_uk_ledger_compile_parity_signed_differences.py +++ b/tools/build_uk_ledger_compile_parity_signed_differences.py @@ -11,6 +11,10 @@ import numpy as np from microcosm.build.gates import ledger_compile_parity_signed_differences +from microcosm.build.ledger_artifact import consumer_fact_schema_versions_of +from microcosm.build.uk_runtime.chronicle_feed import ( + require_uk_chronicle_feed_subset_row_contracts, +) from microcosm.build.uk_runtime.derived_facts import uk_derived_fact_rows from microcosm.build.uk_runtime.ledger_targets import ( LOCAL_REGISTRY_PARITY_FIXTURE_RESOURCE, @@ -1184,6 +1188,11 @@ def _fixture_for_receipt(spec: ParityReceiptSpec) -> dict: def main() -> None: args = _parse_args() facts = _load_jsonl(args.ledger_facts) + # The rows of a subset of the pinned feed: each row contract must be one + # the committed pin lists and chronicle_epoch declares. + require_uk_chronicle_feed_subset_row_contracts( + consumer_fact_schema_versions_of(facts) + ) args.output_dir.mkdir(parents=True, exist_ok=True) for spec in RECEIPTS: if args.surface != "all" and spec.surface != args.surface: diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 296a65aa57..5ee5b1c027 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -71,7 +71,10 @@ target_fit_gate, target_profile_coverage_gate, ) -from microcosm.build.ledger_artifact import load_ledger_consumer_artifact +from microcosm.build.ledger_artifact import ( + load_ledger_consumer_artifact, + validate_pinned_row_contracts, +) from microcosm.build.source_runtime import SourceRuntimeConfig, run_source_stage from microcosm.build.staging import DEFAULT_STAGING_PREFIX, StagingRunBundleWriter from microcosm.build.telemetry_emitter import ( @@ -12551,6 +12554,35 @@ def _check_committed_us_ledger_feed_pin( ) +def _check_us_ledger_feed_row_contracts( + fact_schema_versions: Sequence[str], + *, + facts_sha256: str, + pin: USChronicleFeed | None = None, +) -> None: + """Hold the row contracts the loaded feed's rows carry to the US pin. + + :func:`_check_committed_us_ledger_feed_pin` compares digests; the US pin + describes a bare feed, so the full feed-pin check cannot run. Rows of the + committed feed (``facts_sha256`` is the pin's) must carry exactly the pin's + ``consumer_fact_schema_versions``, which a re-pin types by hand. Rows of a + feed admitted by ``--allow-unpinned-feed`` must carry declared row + contracts only. Either way a row contract + ``microcosm.build.chronicle_epoch`` does not declare is refused. + """ + + pin = pin or load_us_chronicle_feed() + validate_pinned_row_contracts( + fact_schema_versions, + pinned=( + pin.consumer_fact_schema_versions + if facts_sha256 == pin.facts_sha256 + else None + ), + subject="US Chronicle feed", + ) + + def _check_us_esi_anchor_against_feed( facts: Sequence[Mapping[str, Any]], *, @@ -12801,6 +12833,10 @@ def _main(argv: Sequence[str] | None = None) -> int | None: manifest_sha256=ledger_artifact.manifest_sha256, allow_unpinned_feed=args.allow_unpinned_feed, ) + _check_us_ledger_feed_row_contracts( + ledger_artifact.fact_schema_versions, + facts_sha256=ledger_artifact.facts_sha256, + ) esi_anchor_feed_mismatches = _check_us_esi_anchor_against_feed( ledger_artifact.facts, allow_unpinned_feed=args.allow_unpinned_feed, diff --git a/tools/certify_uk_release_cut.py b/tools/certify_uk_release_cut.py index 2f6687cfb6..8f8c18682d 100644 --- a/tools/certify_uk_release_cut.py +++ b/tools/certify_uk_release_cut.py @@ -40,6 +40,9 @@ role_pins_digest, write_error_receipt, ) +from microcosm.build.uk_runtime.chronicle_feed import ( + require_uk_chronicle_feed_row_contracts, +) from microcosm.build.uk_runtime.derived_facts import uk_derived_fact_rows from microcosm.build.uk_runtime.frs_release import load_uk_frs_release from microcosm.build.uk_runtime.hmrc_uprating import assert_installed_engine_is_pinned @@ -258,6 +261,11 @@ def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: expected_facts_sha256=args.ledger_facts_sha256, expected_manifest_sha256=args.ledger_manifest_sha256, ) + # The rows' row contracts: the committed pin's on the committed feed, and + # declared ones on any other. + require_uk_chronicle_feed_row_contracts( + artifact.fact_schema_versions, facts_sha256=artifact.facts_sha256 + ) ledger_registries = {} # The compile's own report of the references that did not resolve at each # parity period travels with the registry: the parity gate needs it to diff --git a/tools/generate_uk_local_target_references.py b/tools/generate_uk_local_target_references.py index 2196273fa4..51d638913e 100644 --- a/tools/generate_uk_local_target_references.py +++ b/tools/generate_uk_local_target_references.py @@ -11,12 +11,16 @@ from pathlib import Path from typing import Any +from microcosm.build.ledger_artifact import consumer_fact_schema_versions_of from microcosm.build.target_reference_authoring import ( AreaSignedDeferral, AreaTargetReferenceAuthoringConfig, author_area_target_references, target_references_resource, ) +from microcosm.build.uk_runtime.chronicle_feed import ( + require_uk_chronicle_feed_subset_row_contracts, +) from microcosm.build.uk_runtime.desnz_constituency_energy import ( FACTS_RESOURCE as DESNZ_CONSTITUENCY_ENERGY_RESOURCE, ) @@ -83,6 +87,12 @@ def main() -> None: args = _parser().parse_args() + facts = list(_read_jsonl(args.ledger_facts)) + # The rows of a subset of the pinned feed: each row contract must be one + # the committed pin lists and chronicle_epoch declares. + require_uk_chronicle_feed_subset_row_contracts( + consumer_fact_schema_versions_of(facts) + ) contract = _filter_contract_by_geography_levels( json.loads(args.contract.read_text(encoding="utf-8")), allowed_levels=LOCAL_GEOGRAPHY_LEVELS, @@ -113,7 +123,6 @@ def main() -> None: source_fact_feed=args.source_fact_feed or f"{args.ledger_facts.resolve().parent.name}/{args.ledger_facts.name}", ) - facts = list(_read_jsonl(args.ledger_facts)) derived_resources: list[dict[str, Any]] = [] if not args.no_derived_facts: # microcosm#1240: the high-value homes family selects Microcosm-derived diff --git a/tools/generate_uk_target_references.py b/tools/generate_uk_target_references.py index effbcb6fbe..9d665c496e 100644 --- a/tools/generate_uk_target_references.py +++ b/tools/generate_uk_target_references.py @@ -17,12 +17,16 @@ from pathlib import Path from typing import Any +from microcosm.build.ledger_artifact import consumer_fact_schema_versions_of from microcosm.build.ledger_targets import MONTHLY_WINDOW_OPERATIONS from microcosm.build.target_reference_authoring import ( TargetReferenceAuthoringConfig, author_target_references, target_references_resource, ) +from microcosm.build.uk_runtime.chronicle_feed import ( + require_uk_chronicle_feed_subset_row_contracts, +) from microcosm.build.uk_runtime.high_value_homes import ( load_uk_high_value_home_stock_facts, ) @@ -111,16 +115,21 @@ def main() -> None: args = _parser().parse_args() - contract = _filter_contract_by_geography_levels( - json.loads(args.contract.read_text()), - allowed_levels=NATIONAL_GEOGRAPHY_LEVELS, - ) - _validate_amount_entity_pins(contract) facts = [ json.loads(line) for line in args.ledger_facts.read_text().splitlines() if line.strip() ] + # The rows of a subset of the pinned feed: each row contract must be one + # the committed pin lists and chronicle_epoch declares. + require_uk_chronicle_feed_subset_row_contracts( + consumer_fact_schema_versions_of(facts) + ) + contract = _filter_contract_by_geography_levels( + json.loads(args.contract.read_text()), + allowed_levels=NATIONAL_GEOGRAPHY_LEVELS, + ) + _validate_amount_entity_pins(contract) # microcosm#1301: the high-value homes region rows select the # Microcosm-derived facts the local cells bind (microcosm#1240), carried in # the committed resource rather than the Chronicle feed. Their fan-out diff --git a/tools/vendor_uk_ledger_facts.py b/tools/vendor_uk_ledger_facts.py index 679a8f4fe6..cc8f8549fd 100644 --- a/tools/vendor_uk_ledger_facts.py +++ b/tools/vendor_uk_ledger_facts.py @@ -23,6 +23,7 @@ ) from microcosm.build.uk_runtime.chronicle_feed import ( load_uk_chronicle_feed, + require_uk_chronicle_feed_row_contracts, ) from microcosm.build.uk_runtime.ledger_fact_vendoring import ( load_vendor_selections, @@ -82,6 +83,11 @@ def main() -> int: f"error: {message} Re-pin first, or pass --allow-unpinned-feed for a diagnostic run." ) print(f"warning: {message} (--allow-unpinned-feed)", file=sys.stderr) + # The digests say these are the committed feed's rows; the rows' own + # schema_version values must also be the pin's declared row contracts. + require_uk_chronicle_feed_row_contracts( + artifact.fact_schema_versions, facts_sha256=artifact.facts_sha256, pin=pin + ) register = load_vendor_selections() output_dir = ( args.output_dir From 9cc83b49646e1cbb86d5aba239261e96bd4c55d8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 07:53:59 +0400 Subject: [PATCH 7/8] Test that the US target-parity generator checks its rows' row contracts Review round 3 of #1510 found that tools/build_us_target_parity_manifest.py holds the feed to the committed US pin by digest, compiles it and writes committed files, but checks no row contracts. Before the fix all three cases of the new test fail on the AssertionError the stubbed manifest build raises when rows on v6, rows on v4 against a v3 pin, or rows on v3 and v4 against a v3 pin reach it. A second test covers the UK subset rule directly, and passes before and after the fix: a declared row contract the pin does not list is refused, and a strict subset of the pin's list, or rows with no stamp, pass. Refs #1509 Co-Authored-By: Claude Opus 5.5 --- .../engine_free/uk/test_uk_chronicle_feed.py | 36 ++++++++++++ .../engine_free/us/test_us_chronicle_feed.py | 58 +++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py index c0bca1685d..ee5b552757 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_chronicle_feed.py @@ -126,6 +126,42 @@ def test_uk_feed_tools_check_the_row_contracts_of_the_rows_they_read(tool, check assert check in source +@pytest.mark.parametrize( + ("rows", "pinned", "refused"), + [ + ( + ("chronicle.consumer_fact.v5",), + ("chronicle.consumer_fact.v4",), + True, + ), + ( + ("chronicle.consumer_fact.v4", "chronicle.consumer_fact.v5"), + ("chronicle.consumer_fact.v4",), + True, + ), + ( + ("chronicle.consumer_fact.v4",), + ("chronicle.consumer_fact.v4", "chronicle.consumer_fact.v5"), + False, + ), + ((), ("chronicle.consumer_fact.v4",), False), + ], + ids=["declared_not_on_the_list", "one_not_on_the_list", "strict_subset", "none"], +) +def test_subset_check_requires_each_row_contract_on_the_pin(rows, pinned, refused): + """A subset of the pinned feed may carry fewer row contracts, never others.""" + from microcosm.build.ledger_artifact import ChronicleFeedPinMismatchError + from microcosm.build.uk_runtime.chronicle_feed import ( + require_uk_chronicle_feed_subset_row_contracts, + ) + + if refused: + with pytest.raises(ChronicleFeedPinMismatchError): + require_uk_chronicle_feed_subset_row_contracts(rows, pinned_versions=pinned) + else: + require_uk_chronicle_feed_subset_row_contracts(rows, pinned_versions=pinned) + + def test_committed_feed_pin_check_refuses_a_foreign_artifact(): from microcosm.build.uk_runtime.chronicle_feed import ( UKChronicleFeedPinError, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py index 03b51b4f76..d4fffa147c 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py @@ -13,11 +13,13 @@ import hashlib import importlib.util import json +import re import subprocess import sys import textwrap from importlib.resources import files from pathlib import Path +from types import SimpleNamespace import pytest @@ -143,6 +145,62 @@ def test_pin_rejects_an_unsupported_artifact_schema( chronicle_feed.load_us_chronicle_feed() +@pytest.mark.parametrize( + ("rows", "message"), + [ + (["chronicle.consumer_fact.v6"], "['chronicle.consumer_fact.v6']"), + (["chronicle.consumer_fact.v4"], "consumer_fact_schema_versions"), + ( + ["chronicle.consumer_fact.v3", "chronicle.consumer_fact.v4"], + "consumer_fact_schema_versions", + ), + ], + ids=["undeclared", "not_in_the_pin", "pin_lists_fewer"], +) +def test_parity_generator_holds_the_rows_row_contracts_to_the_pin( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + rows: list[str], + message: str, +) -> None: + """The target-parity generator checks the rows it compiles, as the release does. + + It holds the feed to the committed pin by digest and writes committed + files, so on the pinned feed the rows must carry exactly the pin's + ``consumer_fact_schema_versions``; the refusal comes before any compile + or write. + """ + generator = _load_tool("build_us_target_parity_manifest") + feed = tmp_path / "consumer_facts.jsonl" + feed.write_text( + "".join( + json.dumps({"aggregate_fact_key": f"fact-{index}", "schema_version": row}) + + "\n" + for index, row in enumerate(rows) + ) + ) + facts_sha256 = hashlib.sha256(feed.read_bytes()).hexdigest() + pin = SimpleNamespace( + facts_sha256=facts_sha256, + consumer_fact_schema_versions=("chronicle.consumer_fact.v3",), + ) + monkeypatch.setattr(generator, "EXPECTED_FEED_SHA256", facts_sha256) + monkeypatch.setattr(generator, "load_us_chronicle_feed", lambda: pin) + monkeypatch.setattr(generator, "MANIFEST_PATH", tmp_path / "manifest.json") + monkeypatch.setattr(generator, "FEED_FAMILIES_PATH", tmp_path / "families.json") + + def build_manifest(*args, **kwargs): + raise AssertionError("the rows reached the manifest build") + + monkeypatch.setattr(generator, "build_manifest", build_manifest) + monkeypatch.setattr(sys, "argv", ["generator", "--ledger-facts", str(feed)]) + + with pytest.raises(ValueError, match=re.escape(message)): + generator.main() + + assert not (tmp_path / "manifest.json").exists() + + def test_parity_resources_and_generator_restate_the_pin() -> None: pin = load_us_chronicle_feed() manifest = _us_resource("target_parity_manifest.json") From 5fdbc1ef5dd05d8ab2c423c255aca85e111190ef Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 09:17:13 +0400 Subject: [PATCH 8/8] Check the row contracts the US target-parity generator compiles Review round 3 of #1510 found that tools/build_us_target_parity_manifest.py holds the feed to the committed US pin by digest, compiles it and writes committed files, but checked no row contracts, so a US re-pin that left the hand-typed list stale passed its own rebuild steps. - us_runtime.chronicle_feed: require_us_chronicle_feed_row_contracts, the exact-or-declared check: rows of the committed feed must carry exactly the pin's list; rows of any other feed, declared row contracts only. - The release tool's _check_us_ledger_feed_row_contracts now calls it, and the target-parity generator calls it after its digest check, before compiling or writing. - Wording: validate_pinned_row_contracts and the UK row check say the release certifier holds its feed to the operator's digests, so it applies the exact rule only when given the committed feed. The changelog says the same, names the US generator, and lists the readers left unchecked because no reviewed pin holds their feed (the US preflight and ACS tools, the scoring and diagnostic tools, the UK high-value home stock, BRMA rent-quantile and Scotland BRMA-overlap builders, and the childcare take-up fit tools). - The UK and US re-pin guides name the certifier's rule and the generator's check; notes that said the NZ pin carries the v4 digest now say NZ is on v5 (#1507). Refs #1509 Co-Authored-By: Claude Opus 5.5 --- .../1509-chronicle-consumer-fact-v5.added.md | 4 +-- docs/uk-chronicle-feed-repin.md | 10 +++--- docs/us-chronicle-feed-repin.md | 12 ++++--- .../src/microcosm/build/ledger_artifact.py | 6 ++-- .../build/uk_runtime/chronicle_feed.py | 8 +++-- .../build/us_runtime/chronicle_feed.py | 33 +++++++++++++++++++ .../shared/test_ledger_artifact.py | 2 +- .../chronicle_consumer_fact/SOURCE.md | 5 +-- .../microcosm_build/chronicle_feed.py | 2 +- tools/build_us_fiscal_refresh_release.py | 17 +++------- tools/build_us_target_parity_manifest.py | 14 +++++++- 11 files changed, 80 insertions(+), 33 deletions(-) diff --git a/changelog.d/1509-chronicle-consumer-fact-v5.added.md b/changelog.d/1509-chronicle-consumer-fact-v5.added.md index fa4206e3eb..3904193402 100644 --- a/changelog.d/1509-chronicle-consumer-fact-v5.added.md +++ b/changelog.d/1509-chronicle-consumer-fact-v5.added.md @@ -2,8 +2,8 @@ Declare Chronicle's `chronicle.consumer_fact.v5` row contract, the one Chronicle - **The declaration.** `microcosm.build.chronicle_epoch` declares v5 in the chronicle epoch beside `ledger.consumer_fact.v1` and `chronicle.consumer_fact.v2` to v4, matching Chronicle's own `CONSUMER_FACT_ROW_CONTRACTS`. Before this change nothing outside the tests read that list, so a v5 feed would already have loaded. - **Pins.** The UK and US pin loaders, `chronicle_feed_identity` (used by reference factors, money normalization and the transport composer) and `validate_chronicle_feed_pin` refuse a pin whose `consumer_fact_schema_versions` names an undeclared row contract. The message names it and says to declare it in `chronicle_epoch` first; an id outside Chronicle's namespaces, such as `arch.consumer_fact.v1`, is named as not a Chronicle row contract, which `chronicle_epoch` cannot declare. -- **Rows.** `validate_chronicle_feed_pin` compares the rows' `schema_version` values with the pin before it checks them, so the transport kernels and the NZ and Belgian generators, which use it, cover the rows. The UK and US builds hold their feeds to committed pins by digest, so they now also check the rows they read (`validate_pinned_row_contracts`): rows on an undeclared row contract are refused, and rows of the committed feed must carry exactly the pin's list, which a re-pin types by hand. That covers the UK full and national builds, the UK vendoring tool and release certifier, and the US fiscal-refresh release tool. The UK reference generators, the compile-parity tool and the DESNZ constituency builder, which read a subset of the feed or only some of its rows, require each row's contract to be one the pin lists. A feed admitted by `--allow-unpinned-feed` is held to the declared row contracts only. -- **Loads without a pin.** A feed loaded by path for diagnostics and experiments, or a bare feed with no manifest that no reviewed pin covers, still carries its per-row ids unchecked; the July 2026 US fiscal-refresh feed, whose rows carry `arch.consumer_fact.v1`, loads that way. +- **Rows.** `validate_chronicle_feed_pin` compares the rows' `schema_version` values with the pin before it checks them, so the transport kernels and the NZ and Belgian generators, which use it, cover the rows. Code that holds a feed to the committed UK or US pin by digest now also checks the rows it reads (`validate_pinned_row_contracts`): rows on an undeclared row contract are refused, and rows of the committed feed must carry exactly the pin's list, which a re-pin types by hand. That covers the UK full and national builds, the UK vendoring tool, the US fiscal-refresh release tool and the US target-parity generator (`require_uk_chronicle_feed_row_contracts`, `require_us_chronicle_feed_row_contracts`). A feed admitted by `--allow-unpinned-feed` is held to the declared row contracts only. The UK release certifier holds its feed to the operator's digests rather than the pin's, so it applies the exact rule only when it is given the committed feed, and the declared-only rule otherwise. The UK reference generators, the compile-parity tool and the DESNZ constituency builder, which read a subset of the pinned feed or only some of its rows, require each row's contract to be one the pin lists. +- **Loads without a pin.** A feed loaded by path for diagnostics and experiments, or a bare feed with no manifest that no reviewed pin covers, still carries its per-row ids unchecked; the July 2026 US fiscal-refresh feed, whose rows carry `arch.consumer_fact.v1`, loads that way. So do the readers that hold their feed to no reviewed pin: the US release preflight and ACS local tools, the scoring and diagnostic tools, and the UK builders that read the feed by path into committed resources (the high-value home stock, the BRMA rent quantiles and the Scotland BRMA overlap) and the childcare take-up fit tools, which take an operator-supplied digest. Their outputs are used beside the pinned feed, whose rows the builds check. - **The new field.** v5 adds an optional `package_dimension_value_labels`: a source package's own wording of a dimension value where it differs from the label the row exports. The loader keeps it on the parsed rows, as it keeps every published field, and Microcosm never takes a label from it: each value keeps the one label Chronicle gives it in `dimension_value_labels` and `layout.groupby_value_label`. - **Its shape.** When a target with a calibration hierarchy (a provider and category seed) is compiled, each selected fact's field must be an object of dimension id to an object of value id to non-empty text, the shape the v5 schema requires. Any other shape raises `LedgerHierarchyMetadataError`, the error a missing Chronicle label raises, and callers treat it the same way: the UK compilers report the reference as unsupported, which stops the build year and is recorded for validation periods. This check walks every entry of the field, where the check of `dimension_value_labels` reads only the selected entries. - **No feed pin moves.** The UK, US, NZ and Belgian feeds stay on their current Chronicle commits until their own re-pins. diff --git a/docs/uk-chronicle-feed-repin.md b/docs/uk-chronicle-feed-repin.md index 9e473f0490..3160811520 100644 --- a/docs/uk-chronicle-feed-repin.md +++ b/docs/uk-chronicle-feed-repin.md @@ -35,10 +35,12 @@ row contract declared in `microcosm.build.chronicle_epoch` first refuse a pin naming an undeclared row contract; - the UK code that reads the feed's rows refuses rows on an undeclared row contract, or rows whose contracts differ from the pin's list: the full and - national builds, the vendoring tool and the release certifier on the - committed artifact (exactly the pin's list), and the reference generators, - the compile-parity tool and the DESNZ constituency builder on a subset or - on the rows they parse (each a contract the pin lists). + national builds and the vendoring tool on the committed artifact (exactly + the pin's list), the release certifier likewise when it is given the + committed feed (it holds the feed to the operator's digests), and the + reference generators, the compile-parity tool and the DESNZ constituency + builder on a subset or on the rows they parse (each a contract the pin + lists). Microcosm declares `ledger.consumer_fact.v1` and `chronicle.consumer_fact.v2` to `v5`; the v5 schema's digest is `acd33a09…`. Then diff --git a/docs/us-chronicle-feed-repin.md b/docs/us-chronicle-feed-repin.md index af12062a1b..db5a58df1a 100644 --- a/docs/us-chronicle-feed-repin.md +++ b/docs/us-chronicle-feed-repin.md @@ -20,11 +20,13 @@ The US feed is on `chronicle.consumer_fact.v3`; Chronicle `main` writes loader refuses a pin whose `consumer_fact_schema_versions` names a row contract that `microcosm.build.chronicle_epoch` does not declare (microcosm#1509). The pin describes a bare feed, so the full feed-pin check cannot run on it; -instead the fiscal-refresh release tool, after its digest check, refuses rows -on an undeclared row contract and, on the committed feed, rows whose contracts -are not exactly the pin's list (`_check_us_ledger_feed_row_contracts`). So set -the list from the rebuilt feed's rows at a re-pin: `tools/build_us_chronicle_feed.py` -does not write it. v5 is declared, and its schema digest is `acd33a09…`. +instead the fiscal-refresh release tool and `tools/build_us_target_parity_manifest.py`, +after their digest checks, refuse rows on an undeclared row contract and, on +the committed feed, rows whose contracts are not exactly the pin's list +(`require_us_chronicle_feed_row_contracts`). So set the list from the rebuilt +feed's rows at a re-pin: `tools/build_us_chronicle_feed.py` does not write it, +and the parity generator, which the rebuild runs, refuses a stale list. v5 is +declared, and its schema digest is `acd33a09…`. v5 rows may carry `package_dimension_value_labels`, a package's own wording of a value where it differs from Chronicle's label. The congressional-district diff --git a/packages/microcosm-build/src/microcosm/build/ledger_artifact.py b/packages/microcosm-build/src/microcosm/build/ledger_artifact.py index f84b4044c3..0571cac2e5 100644 --- a/packages/microcosm-build/src/microcosm/build/ledger_artifact.py +++ b/packages/microcosm-build/src/microcosm/build/ledger_artifact.py @@ -342,8 +342,10 @@ def validate_pinned_row_contracts( rows must also carry exactly the pin's ``consumer_fact_schema_versions``, which a re-pin types by hand; with ``subset``, only row contracts the pin lists, for a reader of part of the pinned feed. ``pinned=None`` is for a - feed that is not the pinned one (a reviewed ``--allow-unpinned-feed`` - run), whose rows are held to the declared row contracts only. + feed that is not the pinned one, whose rows are held to the declared row + contracts only: a reviewed ``--allow-unpinned-feed`` run, or the UK + release certifier given another feed than the committed one (it holds + the feed to the operator's digests, not the pin's). """ observed = list(fact_schema_versions) require_accepted_consumer_fact_schema_versions(observed, subject=f"{subject} rows") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py index 2950f54d33..b0deb22c83 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/chronicle_feed.py @@ -97,9 +97,11 @@ def require_uk_chronicle_feed_row_contracts( The digest checks say the rows are the committed feed's; this says what they are. Rows of the committed feed (``facts_sha256`` is the pin's) must carry exactly the pin's ``consumer_fact_schema_versions``, which a re-pin - types by hand. Rows of another feed, admitted by ``--allow-unpinned-feed`` - for a reviewed diagnostic run, must carry declared row contracts only. - Returns the pin they were checked against. + types by hand. Rows of another feed must carry declared row contracts + only: a feed admitted by ``--allow-unpinned-feed`` for a reviewed + diagnostic run, or one the release certifier is given (it holds the feed + to the operator's digests, not the pin's). Returns the pin they were + checked against. """ pin = pin or load_uk_chronicle_feed() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/chronicle_feed.py b/packages/microcosm-build/src/microcosm/build/us_runtime/chronicle_feed.py index 1302771f16..9fdd69426f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/chronicle_feed.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/chronicle_feed.py @@ -20,6 +20,7 @@ import hashlib import json import re +from collections.abc import Iterable from dataclasses import asdict, dataclass from importlib.resources import files @@ -27,12 +28,14 @@ is_accepted_consumer_artifact_schema_version, require_accepted_consumer_fact_schema_versions, ) +from microcosm.build.ledger_artifact import validate_pinned_row_contracts __all__ = [ "US_CHRONICLE_FEED_RESOURCE", "US_CHRONICLE_FEED_SCOPE_RESOURCE", "USChronicleFeed", "load_us_chronicle_feed", + "require_us_chronicle_feed_row_contracts", "us_chronicle_feed_scope_sha256", ] @@ -75,6 +78,36 @@ def to_dict(self) -> dict[str, object]: return result +def require_us_chronicle_feed_row_contracts( + fact_schema_versions: Iterable[str], + *, + facts_sha256: str, + pin: USChronicleFeed | None = None, +) -> USChronicleFeed: + """Hold the row contracts a loaded US feed's rows carry to the US pin. + + The US pin describes a bare feed, so the full feed-pin check cannot run + on it; the code that holds the feed to the pin by digest calls this + instead. Rows of the committed feed (``facts_sha256`` is the pin's) must + carry exactly the pin's ``consumer_fact_schema_versions``, which a re-pin + types by hand. Rows of any other feed (a reviewed + ``--allow-unpinned-feed`` run of the release tool) must carry declared + row contracts only. Returns the pin they were checked against. + """ + + pin = pin or load_us_chronicle_feed() + validate_pinned_row_contracts( + fact_schema_versions, + pinned=( + pin.consumer_fact_schema_versions + if facts_sha256 == pin.facts_sha256 + else None + ), + subject="US Chronicle feed", + ) + return pin + + def _feed_path(): return files("microcosm.build.us").joinpath(US_CHRONICLE_FEED_RESOURCE) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py b/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py index dd754b8979..17fef56b6b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ledger_artifact.py @@ -391,7 +391,7 @@ def test_complete_feed_pin_accepts_exact_match(tmp_path): _CHRONICLE_V5_SCHEMA_SHA256 = ( "acd33a091cd8d192be5d264e3e94c13e00afb6fd1972e57d4576ac55a54f6970" ) -#: The v4 schema digest the UK, NZ and BE feed pins carry today. +#: The v4 schema digest the UK and Belgian feed pins carry (NZ is on v5, #1507). _CHRONICLE_V4_SCHEMA_SHA256 = ( "d953306d6f01fbc64db4372d0262d3012ee590269f8a25b276b18dca0626c6a0" ) diff --git a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/SOURCE.md b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/SOURCE.md index 41bb37a85b..f7d48ada91 100644 --- a/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/SOURCE.md +++ b/packages/microcosm-build/tests/fixtures/chronicle_consumer_fact/SOURCE.md @@ -25,8 +25,9 @@ Neither file differs at Chronicle `main` `28064d037562b47377193a5b75f4de120d3768 - `consumer_fact.v4.schema.json`: `policyengine_chronicle/schemas/consumer_fact.v4.schema.json`, byte for byte (sha256 - `d953306d6f01fbc64db4372d0262d3012ee590269f8a25b276b18dca0626c6a0`, the digest the UK, NZ - and Belgian feed pins carry). It is unchanged at `44eebe037`. + `d953306d6f01fbc64db4372d0262d3012ee590269f8a25b276b18dca0626c6a0`, the digest the UK and + Belgian feed pins carry; the NZ pin moved to v5 in microcosm#1507). It is unchanged at + `44eebe037`. - `consumer_facts.v4.jsonl`: the same sample file at that commit (sha256 `d02308d049edcd0e60b29d590211456c6427a505e1acf50f1ae71bbcd2962a8c`): the same three facts as v4 rows, labelling the adjusted gross income value `all` "All returns" in diff --git a/test_support/microcosm_build/chronicle_feed.py b/test_support/microcosm_build/chronicle_feed.py index 4acc27d277..a59d570c37 100644 --- a/test_support/microcosm_build/chronicle_feed.py +++ b/test_support/microcosm_build/chronicle_feed.py @@ -73,7 +73,7 @@ def chronicle_v4_schema() -> dict[str, Any]: def chronicle_v4_schema_sha256() -> str: - """The digest the UK, NZ and Belgian feed pins carry for v4.""" + """The v4 schema digest the UK and Belgian feed pins carry (NZ is on v5).""" return hashlib.sha256(CHRONICLE_V4_SCHEMA_PATH.read_bytes()).hexdigest() diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 5ee5b1c027..b19006f1b5 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -71,10 +71,7 @@ target_fit_gate, target_profile_coverage_gate, ) -from microcosm.build.ledger_artifact import ( - load_ledger_consumer_artifact, - validate_pinned_row_contracts, -) +from microcosm.build.ledger_artifact import load_ledger_consumer_artifact from microcosm.build.source_runtime import SourceRuntimeConfig, run_source_stage from microcosm.build.staging import DEFAULT_STAGING_PREFIX, StagingRunBundleWriter from microcosm.build.telemetry_emitter import ( @@ -237,6 +234,7 @@ from microcosm.build.us_runtime.chronicle_feed import ( USChronicleFeed, load_us_chronicle_feed, + require_us_chronicle_feed_row_contracts, ) from microcosm.build.us_runtime.demographics import ( CENSUS_NATIONAL_AGE_BENCHMARK, @@ -12571,15 +12569,10 @@ def _check_us_ledger_feed_row_contracts( ``microcosm.build.chronicle_epoch`` does not declare is refused. """ - pin = pin or load_us_chronicle_feed() - validate_pinned_row_contracts( + require_us_chronicle_feed_row_contracts( fact_schema_versions, - pinned=( - pin.consumer_fact_schema_versions - if facts_sha256 == pin.facts_sha256 - else None - ), - subject="US Chronicle feed", + facts_sha256=facts_sha256, + pin=pin or load_us_chronicle_feed(), ) diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index 9b7605b4a7..499f34d50d 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -42,11 +42,15 @@ from collections import Counter from pathlib import Path +from microcosm.build.ledger_artifact import consumer_fact_schema_versions_of from microcosm.build.us_runtime import ( default_congressional_district_vintage_crosswalk_path, load_congressional_district_vintage_crosswalk, ) -from microcosm.build.us_runtime.chronicle_feed import load_us_chronicle_feed +from microcosm.build.us_runtime.chronicle_feed import ( + load_us_chronicle_feed, + require_us_chronicle_feed_row_contracts, +) from microcosm.build.us_runtime.fiscal_targets import compile_us_fiscal_target_registry from microcosm.build.us_runtime.medicaid_take_up import ( apply_us_medicaid_enrollment_substitutions, @@ -910,6 +914,14 @@ def main() -> None: f"{EXPECTED_FEED_SHA256}; refusing to regenerate against an " "unexpected feed." ) + # The digest says these are the committed feed's rows; their own + # schema_version values must also be exactly the pin's declared row + # contracts, as the release tool requires. + require_us_chronicle_feed_row_contracts( + consumer_fact_schema_versions_of(facts), + facts_sha256=feed_sha256, + pin=load_us_chronicle_feed(), + ) manifest, feed_families = build_manifest(facts, feed_sha256, args.feed_name)