diff --git a/README.md b/README.md index 41dd8a2ed..e38a0d0e3 100644 --- a/README.md +++ b/README.md @@ -375,6 +375,13 @@ UK release workflows stop when diagnostics are unavailable because their later release checks require that evidence; historical schema-6 and schema-7 UK artifacts remain readable through isolated compatibility validation. +A Draft 2020-12 JSON Schema for schema 8, generated from those models, is +committed at +`packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json` +for consumers that check the file without importing them, such as the +collector. Its description lists the model rules a JSON Schema cannot express; +see the [package README](packages/microcosm-diagnostics/README.md). + Standard publication uploads the locally built `releases//` artifacts to the Hugging Face dataset, tags the release, and updates `latest.json`. It runs on the build machine (it needs the freshly built H5), so it isn't a CI step: diff --git a/changelog.d/1293-calibration-diagnostics-json-schema-generator.added.md b/changelog.d/1293-calibration-diagnostics-json-schema-generator.added.md new file mode 100644 index 000000000..6db55edf7 --- /dev/null +++ b/changelog.d/1293-calibration-diagnostics-json-schema-generator.added.md @@ -0,0 +1,3 @@ +`microcosm.diagnostics.calibration_diagnostics_v8_json_schema()` generates a Draft 2020-12 JSON Schema for calibration diagnostics schema 8 from the pydantic models, so consumers such as the collector can check `calibration_diagnostics.json` without importing them (microcosm#1293). Its description lists every rule the models enforce that the schema cannot express: cross-field reconciliations, finite numbers, non-blank text, strict integers and digest anchoring under Python's `re`. It also notes the one case where the schema is stricter than the models: literal values. Tests pin that every model validator is listed, that each listed rule has a document the schema accepts and the models refuse, and that valid and representative invalid documents are classified the same way by both. + +`jsonschema`, already locked as a runtime dependency of `microcosm-build`, joins the `microcosm-diagnostics` dev group, so no new package enters the lock. The primary-QRF worker identity's approved `uv.lock` digest moves with the lock, as it does for every lock change. diff --git a/changelog.d/1293-calibration-diagnostics-json-schema.added.md b/changelog.d/1293-calibration-diagnostics-json-schema.added.md new file mode 100644 index 000000000..20eff0cd7 --- /dev/null +++ b/changelog.d/1293-calibration-diagnostics-json-schema.added.md @@ -0,0 +1 @@ +The calibration-diagnostics schema-8 JSON Schema is committed at `packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json` and ships in the `microcosm-diagnostics` wheel (microcosm#1293). `tools/generate_calibration_diagnostics_json_schema.py` regenerates it, and `--check` reports a stale file. A byte-exact drift test keeps the file equal to freshly generated output. In their engine-free tests, the US national release writer, the ACS local-area calibrate stage, the UK writer, the UK graph terminal gate kernel and the transport terminal kernel now validate the diagnostics they write against the committed file. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index a033b1684..10bc99fdd 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -413,6 +413,17 @@ label, search for an existing definition and follow AI assistants must import shared static data from its domain-specific constants module instead of copying it or reconstructing alternate views in consumers. +## Calibration diagnostics JSON Schema + +`packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json` +is generated from the schema-8 models in `microcosm.diagnostics.schema`. After +changing one, regenerate it with +`uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py`. +A new model validator also needs an entry in +`CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS` and a document in +`test_json_schema.py` that the schema accepts and the models refuse; the +diagnostics tests fail until both exist. + ## Review this guide Update this guide in the same PR whenever the workspace layout, test diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index fb9c1522b..9b7b2f4e0 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "e5d5818bdc73fd44fbb474b6f0bca4b697ee2f70a0cbb9e47eb04b20f7408d77" + "751721b231ab8cc6b3d10164f61ca429f5171661951494503fb0873e3f67b117" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_terminal_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_terminal_kernels.py index 210071681..8864f859c 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_terminal_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_terminal_kernels.py @@ -88,6 +88,10 @@ with_config, write_toy_sources, ) +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) DIAGNOSTICS = "toy.calibration.diagnostics" _KEY = "b" * 64 @@ -179,6 +183,12 @@ def test_diagnostics_validate_against_schema_8(toy_run, tmp_path) -> None: ) +def test_transport_diagnostics_match_the_published_json_schema(toy_run) -> None: + payload = toy_run.artifact(DIAGNOSTICS, "diagnostics") + + assert published_json_schema_errors(load_strict_json(payload)) == [] + + def _diagnostics_artifacts(run, **overrides) -> dict[str, ArtifactValue]: artifacts = { alias: _value(run.artifact(producer, name), kind) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_diagnostics.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_diagnostics.py index c16aefa05..a7ef17e6f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_diagnostics.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_diagnostics.py @@ -37,6 +37,10 @@ DiagnosticsWriteSuccess, ) from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) _SPI_COLUMN = "household_is_spi_synthetic" _CG_COLUMN = "household_is_capital_gains_clone" @@ -957,6 +961,37 @@ def test_writer_round_trips_strict_json(tmp_path: Path) -> None: assert list(tmp_path.glob(f".{path.name}.*.tmp")) == [] +@pytest.mark.parametrize( + ("with_skipped", "with_consumption"), + [ + pytest.param(False, False, id="scored-targets-only"), + pytest.param(True, True, id="skipped-target-and-consumption-drift"), + ], +) +def test_written_uk_diagnostics_match_the_published_json_schema( + tmp_path: Path, + with_skipped: bool, + with_consumption: bool, +) -> None: + result, frame, registry, geography = _diagnostics_case( + with_skipped=with_skipped, + with_consumption=with_consumption, + ) + path = tmp_path / "calibration_diagnostics.json" + + outcome = write_uk_calibration_diagnostics( + result, + path, + frame, + target_geography_levels=geography, + target_registry=registry, + build={"release_id": "fixture"}, + ) + + assert isinstance(outcome, DiagnosticsWriteSuccess) + assert published_json_schema_errors(load_strict_json(path.read_bytes())) == [] + + @pytest.mark.parametrize( "value", [float("nan"), float("inf"), float("-inf")], diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index 10941274d..06b1014fd 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -21,6 +21,10 @@ from test_support.microcosm_build.uk_hierarchy_fixtures import ( uk_fixture_hierarchy, ) +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) def test_export_drops_native_aliases_and_keeps_identity_keyed_assignment(tmp_path): @@ -950,6 +954,9 @@ def test_both_gate_kernels_are_registered(): def _run_final_gate_kernel(monkeypatch, *, brma_row: bool = False, solved=None): """Run the terminal gate kernel on a two-household fixture. + Rebuilding the diagnostics is forbidden, so a returned document proves + the kernel reused the decoded calibration result. + ``solved``, a list, receives the calibration result the kernel decodes. ``brma_row`` adds a Scottish BRMA rent row far from its target: its two @@ -1267,6 +1274,22 @@ def test_final_gate_kernel_owns_complete_diagnostics_and_reuses_decoded_result( assert support_csv["support_below_floor"].dtype == bool +@pytest.mark.parametrize( + "brma_row", [False, True], ids=["published_rows", "withheld_brma_row"] +) +def test_final_gate_kernel_diagnostics_match_the_published_json_schema( + monkeypatch, brma_row +): + # The withheld BRMA row exercises the parts #1305 added: the ``brma`` + # geography level and ``uk_diagnostics.model_figures_withheld``. + stored = _run_final_gate_kernel(monkeypatch, brma_row=brma_row) + document = load_strict_json(stored.artifacts["calibration_diagnostics"]) + + assert published_json_schema_errors(document) == [] + if brma_row: + assert document["uk_diagnostics"]["model_figures_withheld"]["targets"] == 1 + + def test_the_gate_kernel_keeps_a_withheld_brma_row_out_of_its_gates(monkeypatch): """The third review of #1305: the real bindings, through the kernel. diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py index 798b87e05..bdb1da7fc 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py @@ -36,6 +36,10 @@ from test_support.microcosm_build.us_batched_target_materialization import ( _nested_frame, ) +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -401,6 +405,13 @@ def test_every_calibrate_output_records_the_weights(calibrate_run) -> None: ) +def test_calibrate_diagnostics_match_the_published_json_schema(calibrate_run) -> None: + calibrate_run.run() + written = calibrate_run.path / "calibration_diagnostics.json" + + assert published_json_schema_errors(load_strict_json(written.read_bytes())) == [] + + def test_a_family_multiplier_reaches_calibrate_and_the_stamp(calibrate_run) -> None: calls = calibrate_run.run(target_family_loss_multipliers={"usda_snap": 4.0}) train = _train_specs(calibrate_run) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 9373d815e..4fa27b499 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -27,6 +27,10 @@ ) from microcosm.data.stored_inputs import register_sha256 from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -5895,10 +5899,9 @@ def fake_write_calibration_diagnostics( } -def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( - small_frame, - tmp_path, -) -> None: +def _write_release_diagnostics_with_nan_final_loss(small_frame, tmp_path) -> Path: + """Write release diagnostics for a one-target US registry with a NaN loss.""" + builder = _load_builder_module() base_h5 = tmp_path / "base.h5" base_h5.write_bytes(b"h5") @@ -5971,8 +5974,16 @@ def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( # writer; keep that call boundary explicit rather than moving the fix. default_dataset={"method": "dense_no_l0", "final_loss": None}, ) + return tmp_path / "calibration_diagnostics.json" + + +def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( + small_frame, + tmp_path, +) -> None: + written = _write_release_diagnostics_with_nan_final_loss(small_frame, tmp_path) - diagnostics = json.loads((tmp_path / "calibration_diagnostics.json").read_text()) + diagnostics = json.loads(written.read_text()) assert diagnostics["schema_version"] == 8 assert diagnostics["targets"][0]["source"] == "fixture" assert diagnostics["targets"][0]["hierarchy"] == { @@ -5997,6 +6008,15 @@ def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( assert diagnostics["build"]["default_dataset"]["final_loss"] is None +def test_release_calibration_diagnostics_match_the_published_json_schema( + small_frame, + tmp_path, +) -> None: + written = _write_release_diagnostics_with_nan_final_loss(small_frame, tmp_path) + + assert published_json_schema_errors(load_strict_json(written.read_bytes())) == [] + + #: What the stubbed stored-input gate refuses in the ``stored_input_refused`` #: harness mode, in the gate's own line format. STORED_INPUT_REFUSAL_FIXTURE = ( diff --git a/packages/microcosm-diagnostics/README.md b/packages/microcosm-diagnostics/README.md index c9e39d1f3..f7bfeec75 100644 --- a/packages/microcosm-diagnostics/README.md +++ b/packages/microcosm-diagnostics/README.md @@ -26,3 +26,24 @@ reach validation as non-finite numbers fail construction. The writer also uses strict JSON serialization with non-finite output disabled, so a value introduced after model construction returns a structured serialization failure instead of silently becoming `null`. + +`calibration_diagnostics_v8_json_schema()` generates a Draft 2020-12 JSON +Schema for schema 8 from `CalibrationDiagnosticsV8`, for consumers that check +the file without importing these models. The models remain the authority. +Their cross-field reconciliations, finite-number and non-blank-text checks, +and strict integers cannot be written in a schema generated this way. The +schema's description lists each of these rules +(`CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS`), and a test fails if a model +validator is added without being listed. Complete validation still goes +through `parse_calibration_diagnostics`. + +The generated schema is committed, and shipped in the wheel, at + +`packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json` + +`load_calibration_diagnostics_v8_json_schema()` reads it. After changing a +schema-8 model, regenerate it with +`uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py`; +`--check` reports a stale file without writing. A test fails while the +committed file differs from freshly generated output, and the US, UK and +transport build tests validate the diagnostics they write against it. diff --git a/packages/microcosm-diagnostics/pyproject.toml b/packages/microcosm-diagnostics/pyproject.toml index 1d295cb0f..ff95ecc00 100644 --- a/packages/microcosm-diagnostics/pyproject.toml +++ b/packages/microcosm-diagnostics/pyproject.toml @@ -13,7 +13,9 @@ Homepage = "https://microcosm.institute" Repository = "https://github.com/PolicyEngine/microcosm" [dependency-groups] -dev = ["pytest>=8"] +# jsonschema checks documents against the generated schema-8 JSON Schema in +# tests only; the package itself needs nothing beyond pydantic. +dev = ["pytest>=8", "jsonschema>=4.23,<5"] [build-system] requires = ["hatchling"] diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py b/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py index 04dfef83c..520a1a25e 100644 --- a/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py @@ -1,5 +1,19 @@ """Typed calibration diagnostics shared across the Microcosm stack.""" +from microcosm.diagnostics.json_schema import ( + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS, + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID, + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE, + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE, + JSON_SCHEMA_DRAFT_2020_12, + JsonSchemaGap, + assert_calibration_diagnostics_v8_json_schema_current, + calibration_diagnostics_v8_json_schema, + committed_calibration_diagnostics_v8_json_schema_path, + load_calibration_diagnostics_v8_json_schema, + render_calibration_diagnostics_v8_json_schema, + write_calibration_diagnostics_v8_json_schema, +) from microcosm.diagnostics.schema import ( CALIBRATION_DIAGNOSTICS_ADAPTER, CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, @@ -41,6 +55,11 @@ __all__ = [ "CALIBRATION_DIAGNOSTICS_ADAPTER", "CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION", + "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS", + "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID", + "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE", + "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE", + "JSON_SCHEMA_DRAFT_2020_12", "SUPPORTED_CALIBRATION_DIAGNOSTICS_SCHEMA_VERSIONS", "CalibrationDiagnostics", "CalibrationDiagnosticsV8", @@ -48,6 +67,7 @@ "DiagnosticsWriteFailure", "DiagnosticsWriteOutcome", "DiagnosticsWriteSuccess", + "JsonSchemaGap", "UK_DIAGNOSTICS_SCHEMA_VERSION", "UK_MODEL_FIGURE_FIELDS", "UK_OPTIONAL_TARGET_GEOGRAPHY_LEVELS", @@ -68,10 +88,16 @@ "UKWeightSummary", "UKWithheldModelFigures", "UKZeroWeightStratum", + "assert_calibration_diagnostics_v8_json_schema_current", "calibration_diagnostics_json_schema", + "calibration_diagnostics_v8_json_schema", + "committed_calibration_diagnostics_v8_json_schema_path", "failed_diagnostics_outcome", + "load_calibration_diagnostics_v8_json_schema", "parse_calibration_diagnostics", + "render_calibration_diagnostics_v8_json_schema", "write_calibration_diagnostics", + "write_calibration_diagnostics_v8_json_schema", ] __version__ = "0.1.0" diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py b/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py new file mode 100644 index 000000000..78cdb76d6 --- /dev/null +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py @@ -0,0 +1,347 @@ +"""A Draft 2020-12 JSON Schema for calibration diagnostics schema 8. + +The schema is generated from :class:`CalibrationDiagnosticsV8`, so it cannot +drift from the models in field names, types, bounds, enumerations, required +fields or undeclared-field refusal. The models remain the authority: some of +their rules cannot be written in a schema generated this way, and the schema's +description lists each one (:data:`CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS`). +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path +from typing import Final + +from pydantic import JsonValue + +from microcosm.diagnostics.schema import CalibrationDiagnosticsV8 + +JSON_SCHEMA_DRAFT_2020_12: Final = "https://json-schema.org/draft/2020-12/schema" +CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID: Final = ( + "calibration_diagnostics.v8.schema.json" +) +#: The committed schema, relative to the ``microcosm.diagnostics`` package. +CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE: Final = ( + f"schemas/{CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID}" +) +_REGENERATE_COMMAND: Final = ( + "uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py" +) + + +@dataclass(frozen=True) +class JsonSchemaGap: + """A rule the schema-8 models enforce that the generated schema leaves out. + + ``validators`` names the model validator functions that enforce the rule, + by qualified name; it is empty when the rule comes from model + configuration or from how validators evaluate patterns. + """ + + key: str + rule: str + validators: tuple[str, ...] = () + + +CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS: Final[tuple[JsonSchemaGap, ...]] = ( + JsonSchemaGap( + key="finite-numbers", + rule=( + "Every number is finite, at any depth, including inside options, " + "targets[].metadata and build. JSON has no NaN or Infinity, but " + "lenient parsers such as Python's json module accept those tokens, " + "and a JSON Schema number cannot exclude the values they produce, " + "so parse submissions with a parser that refuses them." + ), + validators=( + "DiagnosticsModel.reject_non_finite_numbers", + "CalibrationDiagnosticsV8.finite_optional_scalar", + ), + ), + JsonSchemaGap( + key="strict-integers", + rule=( + "Integer fields hold integers written without a fraction or " + "exponent. JSON Schema counts 2.0 and 1e3 as integers; the strict " + "models refuse them, because Python's json module decodes them as " + "floats." + ), + ), + JsonSchemaGap( + key="non-blank-text", + rule=( + "Strings the models type as non-blank contain at least one " + "character that is not whitespace. These are most names, " + "identifiers, labels, sources, entities, reasons, codes and " + "messages, and the keys of uk_diagnostics.target_observation_basis, " + "the zero-weight strata and the consumption-drift maps." + ), + validators=("_require_non_blank",), + ), + JsonSchemaGap( + key="digest-pattern-anchoring", + rule=( + "Digests match ^[0-9a-f]{64}$ with nothing after them. In the " + "ECMA-262 regular expressions JSON Schema specifies, $ matches " + "only at the end of the string, but validators that evaluate " + "pattern with Python's re module also let $ match before a final " + "newline, so they accept a digest followed by a newline." + ), + ), + JsonSchemaGap( + key="hierarchy-identity", + rule=( + "In each targets[].hierarchy, category.provider_id equals " + "provider.id and the dimensions have unique ids." + ), + validators=("CalibrationHierarchy.category_matches_provider",), + ), + JsonSchemaGap( + key="target-hierarchy", + rule="Each targets[].hierarchy.target.id equals that row's target_name.", + validators=("TargetDiagnosticV8.hierarchy_target_matches_row",), + ), + JsonSchemaGap( + key="target-surface-shape", + rule=( + "target_surface.constraint_matrix has n_targets rows and " + "n_records columns, and its nnz is at most rows times columns." + ), + validators=("TargetSurface.reconcile_matrix_shape",), + ), + JsonSchemaGap( + key="diagnostics-counts", + rule=( + "target_surface.n_targets equals the number of targets, " + "target_registry.n_specs is at least the number of targets, " + "target_surface.n_records equals n_records, and n_nonzero is at " + "most n_records." + ), + validators=("CalibrationDiagnosticsV8.reconcile_counts",), + ), + JsonSchemaGap( + key="uk-extension-presence", + rule=( + 'A UK registry (target_registry.country "uk") requires ' + "uk_diagnostics; any other registry requires it to be null or " + "absent. In a UK document, uk_diagnostics.weights.n_records, " + "effective_sample_size and top_1pct_weight_share equal the " + "top-level n_records, effective_sample_size and " + "top_1pct_weight_share. The n_targets, n_scored and n_skipped " + "values in uk_diagnostics.target_pass_rates_by_geography_level sum " + "to target_registry.n_specs, the number of targets and the number " + "of skipped targets." + ), + validators=("CalibrationDiagnosticsV8.reconcile_counts",), + ), + JsonSchemaGap( + key="uk-weight-summary", + rule=( + "In uk_diagnostics.weights, positive_weight_records plus " + "zero_weight_records equals n_records, effective_sample_size is at " + "most n_records, and ess_fraction equals effective_sample_size " + "divided by n_records. median_positive_weight and " + "max_to_median_positive_weight are null exactly when " + "positive_weight_records is 0; otherwise " + "max_to_median_positive_weight equals max_weight divided by " + "median_positive_weight." + ), + validators=("UKWeightSummary.reconcile_weight_summary",), + ), + JsonSchemaGap( + key="uk-zero-weight-strata", + rule=( + "In each uk_diagnostics.zero_weight_rows_by_stratum row, " + "positive_weight_rows plus zero_weight_rows equals rows. Across the " + "strata, rows, positive_weight_rows and zero_weight_rows sum to " + "weights.n_records, weights.positive_weight_records and " + "weights.zero_weight_records." + ), + validators=( + "UKZeroWeightStratum.reconcile_row_counts", + "UKDiagnosticsV1.reconcile_uk_diagnostics", + ), + ), + JsonSchemaGap( + key="uk-geography-pass-rates", + rule=( + "uk_diagnostics.target_pass_rates_by_geography_level has exactly " + "one row for each of national, region, country, local_authority " + "and constituency. In each row, n_scored plus n_skipped equals " + "n_targets, n_within_10pct is at most n_scored, and pass_rate " + "equals n_within_10pct divided by n_targets, or is null when " + "n_targets is 0." + ), + validators=( + "UKGeographyPassRate.reconcile_pass_rate", + "UKDiagnosticsV1.reconcile_uk_diagnostics", + ), + ), + JsonSchemaGap( + key="uk-local-fit", + rule=( + "uk_diagnostics.weakest_families and weakest_areas_by_fit are both " + "present or both absent. In weakest_families, " + "weakest_areas_by_fit.bottom_by_fit and " + "weakest_areas_by_fit.countries, n_within_10pct is at most " + "n_targets and pass_rate equals n_within_10pct divided by " + "n_targets. bottom_by_fit has at most limit rows and at most " + "n_areas_scored rows." + ), + validators=( + "UKWeakestFamily.reconcile_pass_rate", + "UKAreaFit.reconcile_pass_rate", + "UKCountryFit.reconcile_pass_rate", + "UKWeakestAreasByFit.reconcile_area_counts", + "UKDiagnosticsV1.reconcile_uk_diagnostics", + ), + ), + JsonSchemaGap( + key="uk-rotated-holdout", + rule=( + "A measured uk_diagnostics.rotated_holdout has unique grains and " + "n_folds entries in both fold_losses and folds. Its fold ids cover " + "0 to n_folds - 1, each fold's holdout_loss equals its fold_losses " + "entry, mean_holdout_loss is the mean of fold_losses and " + "worst_holdout_loss is their maximum. Each fold has " + "n_holdout_targets unique holdout_target_indices." + ), + validators=( + "UKRotatedHoldoutFold.reconcile_holdout_indices", + "UKMeasuredRotatedHoldout.reconcile_folds", + ), + ), + JsonSchemaGap( + key="uk-consumption-drift", + rule=( + "Each uk_diagnostics.consumption_drift value has change equal to " + "final minus design, and its relative_change is null exactly when " + "design is 0." + ), + validators=("UKDriftValue.reconcile_change",), + ), +) + +CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE: Final = ( + "Literal fields hold exactly their declared JSON value. The models compare " + "literals with Python equality, so their parser also accepts 1 for true " + "and true for 1, for example in rotated_holdout.skipped or " + "uk_diagnostics.schema_version. The canonical writer always emits the " + "declared value, which this schema requires." +) + +_TITLE: Final = "Microcosm calibration diagnostics, schema 8" +_COMMENT: Final = ( + "Generated from microcosm.diagnostics.CalibrationDiagnosticsV8. Do not " + f"edit it; regenerate it with `{_REGENERATE_COMMAND}`." +) + + +def _description() -> str: + gap_lines = "\n".join( + f"- {gap.rule}" for gap in CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS + ) + return ( + "The calibration_diagnostics.json document, schema 8, that Microcosm " + "builds write beside a calibrated dataset. The pydantic models it is " + "generated from remain the authority. A document that satisfies this " + "schema can still break one of the rules below, which the models " + "enforce and this schema does not express, so validate with " + "microcosm.diagnostics.parse_calibration_diagnostics for the complete " + "contract. Equalities between floating-point values hold within the " + "models' tolerances.\n\n" + "Rules the models enforce that this schema does not express:\n" + f"{gap_lines}\n\n" + "Where this schema is stricter than the models' parser:\n" + f"- {CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE}" + ) + + +def calibration_diagnostics_v8_json_schema() -> dict[str, JsonValue]: + """Return the schema-8 JSON Schema generated from the typed models.""" + + generated = CalibrationDiagnosticsV8.model_json_schema(mode="validation") + generated.pop("title", None) + generated.pop("description", None) + return { + "$schema": JSON_SCHEMA_DRAFT_2020_12, + "$id": CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID, + "$comment": _COMMENT, + "title": _TITLE, + "description": _description(), + **generated, + } + + +def render_calibration_diagnostics_v8_json_schema() -> str: + """Return the schema as the exact text committed to the repository.""" + + return ( + json.dumps( + calibration_diagnostics_v8_json_schema(), + indent=2, + ensure_ascii=False, + ) + + "\n" + ) + + +def committed_calibration_diagnostics_v8_json_schema_path() -> Path: + """Return the committed schema file, shipped inside the package.""" + + return Path( + str( + files("microcosm.diagnostics").joinpath( + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE + ) + ) + ) + + +def load_calibration_diagnostics_v8_json_schema( + path: str | Path | None = None, +) -> dict[str, JsonValue]: + """Read the committed schema, or the schema file at ``path``.""" + + source = ( + committed_calibration_diagnostics_v8_json_schema_path() + if path is None + else Path(path) + ) + return json.loads(source.read_bytes()) + + +def write_calibration_diagnostics_v8_json_schema( + path: str | Path | None = None, +) -> Path: + """Write freshly generated schema text to the committed file or ``path``.""" + + output = ( + committed_calibration_diagnostics_v8_json_schema_path() + if path is None + else Path(path) + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(render_calibration_diagnostics_v8_json_schema().encode("utf-8")) + return output + + +def assert_calibration_diagnostics_v8_json_schema_current( + path: str | Path | None = None, +) -> None: + """Raise unless the file is byte-identical to freshly generated output.""" + + source = ( + committed_calibration_diagnostics_v8_json_schema_path() + if path is None + else Path(path) + ) + expected = render_calibration_diagnostics_v8_json_schema().encode("utf-8") + if source.read_bytes() != expected: + raise ValueError( + f"{source} differs from the schema the models generate; " + f"regenerate it with `{_REGENERATE_COMMAND}`." + ) diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json b/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json new file mode 100644 index 000000000..fb3d185c5 --- /dev/null +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json @@ -0,0 +1,1790 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "calibration_diagnostics.v8.schema.json", + "$comment": "Generated from microcosm.diagnostics.CalibrationDiagnosticsV8. Do not edit it; regenerate it with `uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py`.", + "title": "Microcosm calibration diagnostics, schema 8", + "description": "The calibration_diagnostics.json document, schema 8, that Microcosm builds write beside a calibrated dataset. The pydantic models it is generated from remain the authority. A document that satisfies this schema can still break one of the rules below, which the models enforce and this schema does not express, so validate with microcosm.diagnostics.parse_calibration_diagnostics for the complete contract. Equalities between floating-point values hold within the models' tolerances.\n\nRules the models enforce that this schema does not express:\n- Every number is finite, at any depth, including inside options, targets[].metadata and build. JSON has no NaN or Infinity, but lenient parsers such as Python's json module accept those tokens, and a JSON Schema number cannot exclude the values they produce, so parse submissions with a parser that refuses them.\n- Integer fields hold integers written without a fraction or exponent. JSON Schema counts 2.0 and 1e3 as integers; the strict models refuse them, because Python's json module decodes them as floats.\n- Strings the models type as non-blank contain at least one character that is not whitespace. These are most names, identifiers, labels, sources, entities, reasons, codes and messages, and the keys of uk_diagnostics.target_observation_basis, the zero-weight strata and the consumption-drift maps.\n- Digests match ^[0-9a-f]{64}$ with nothing after them. In the ECMA-262 regular expressions JSON Schema specifies, $ matches only at the end of the string, but validators that evaluate pattern with Python's re module also let $ match before a final newline, so they accept a digest followed by a newline.\n- In each targets[].hierarchy, category.provider_id equals provider.id and the dimensions have unique ids.\n- Each targets[].hierarchy.target.id equals that row's target_name.\n- target_surface.constraint_matrix has n_targets rows and n_records columns, and its nnz is at most rows times columns.\n- target_surface.n_targets equals the number of targets, target_registry.n_specs is at least the number of targets, target_surface.n_records equals n_records, and n_nonzero is at most n_records.\n- A UK registry (target_registry.country \"uk\") requires uk_diagnostics; any other registry requires it to be null or absent. In a UK document, uk_diagnostics.weights.n_records, effective_sample_size and top_1pct_weight_share equal the top-level n_records, effective_sample_size and top_1pct_weight_share. The n_targets, n_scored and n_skipped values in uk_diagnostics.target_pass_rates_by_geography_level sum to target_registry.n_specs, the number of targets and the number of skipped targets.\n- In uk_diagnostics.weights, positive_weight_records plus zero_weight_records equals n_records, effective_sample_size is at most n_records, and ess_fraction equals effective_sample_size divided by n_records. median_positive_weight and max_to_median_positive_weight are null exactly when positive_weight_records is 0; otherwise max_to_median_positive_weight equals max_weight divided by median_positive_weight.\n- In each uk_diagnostics.zero_weight_rows_by_stratum row, positive_weight_rows plus zero_weight_rows equals rows. Across the strata, rows, positive_weight_rows and zero_weight_rows sum to weights.n_records, weights.positive_weight_records and weights.zero_weight_records.\n- uk_diagnostics.target_pass_rates_by_geography_level has exactly one row for each of national, region, country, local_authority and constituency. In each row, n_scored plus n_skipped equals n_targets, n_within_10pct is at most n_scored, and pass_rate equals n_within_10pct divided by n_targets, or is null when n_targets is 0.\n- uk_diagnostics.weakest_families and weakest_areas_by_fit are both present or both absent. In weakest_families, weakest_areas_by_fit.bottom_by_fit and weakest_areas_by_fit.countries, n_within_10pct is at most n_targets and pass_rate equals n_within_10pct divided by n_targets. bottom_by_fit has at most limit rows and at most n_areas_scored rows.\n- A measured uk_diagnostics.rotated_holdout has unique grains and n_folds entries in both fold_losses and folds. Its fold ids cover 0 to n_folds - 1, each fold's holdout_loss equals its fold_losses entry, mean_holdout_loss is the mean of fold_losses and worst_holdout_loss is their maximum. Each fold has n_holdout_targets unique holdout_target_indices.\n- Each uk_diagnostics.consumption_drift value has change equal to final minus design, and its relative_change is null exactly when design is 0.\n\nWhere this schema is stricter than the models' parser:\n- Literal fields hold exactly their declared JSON value. The models compare literals with Python equality, so their parser also accepts 1 for true and true for 1, for example in rotated_holdout.skipped or uk_diagnostics.schema_version. The canonical writer always emits the declared value, which this schema requires.", + "$defs": { + "CalibrationHierarchy": { + "additionalProperties": false, + "properties": { + "provider": { + "$ref": "#/$defs/HierarchyNode" + }, + "category": { + "$ref": "#/$defs/HierarchyCategory" + }, + "geography": { + "$ref": "#/$defs/HierarchyGeography" + }, + "dimensions": { + "items": { + "$ref": "#/$defs/HierarchyDimension" + }, + "title": "Dimensions", + "type": "array" + }, + "target": { + "$ref": "#/$defs/HierarchyNode" + } + }, + "required": [ + "provider", + "category", + "geography", + "dimensions", + "target" + ], + "title": "CalibrationHierarchy", + "type": "object" + }, + "DiagnosticsWarning": { + "additionalProperties": false, + "properties": { + "code": { + "title": "Code", + "type": "string" + }, + "severity": { + "const": "warning", + "title": "Severity", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + } + }, + "required": [ + "code", + "severity", + "message" + ], + "title": "DiagnosticsWarning", + "type": "object" + }, + "HierarchyCategory": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + }, + "provider_id": { + "title": "Provider Id", + "type": "string" + } + }, + "required": [ + "id", + "label", + "provider_id" + ], + "title": "HierarchyCategory", + "type": "object" + }, + "HierarchyDimension": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + }, + "value_id": { + "title": "Value Id", + "type": "string" + }, + "value_label": { + "title": "Value Label", + "type": "string" + } + }, + "required": [ + "id", + "label", + "value_id", + "value_label" + ], + "title": "HierarchyDimension", + "type": "object" + }, + "HierarchyGeography": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + }, + "level": { + "title": "Level", + "type": "string" + } + }, + "required": [ + "id", + "label", + "level" + ], + "title": "HierarchyGeography", + "type": "object" + }, + "HierarchyNode": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "id", + "label" + ], + "title": "HierarchyNode", + "type": "object" + }, + "JsonValue": {}, + "PastCapCensus": { + "additionalProperties": false, + "properties": { + "cap": { + "exclusiveMinimum": 0, + "title": "Cap", + "type": "number" + }, + "scale_basis": { + "title": "Scale Basis", + "type": "string" + }, + "n_targets": { + "minimum": 0, + "title": "N Targets", + "type": "integer" + }, + "initial_past_cap": { + "minimum": 0, + "title": "Initial Past Cap", + "type": "integer" + }, + "final_past_cap": { + "minimum": 0, + "title": "Final Past Cap", + "type": "integer" + }, + "escaped": { + "minimum": 0, + "title": "Escaped", + "type": "integer" + }, + "frozen": { + "minimum": 0, + "title": "Frozen", + "type": "integer" + }, + "pushed_out": { + "minimum": 0, + "title": "Pushed Out", + "type": "integer" + }, + "pushed_out_rows": { + "items": { + "$ref": "#/$defs/PushedOutTarget" + }, + "title": "Pushed Out Rows", + "type": "array" + } + }, + "required": [ + "cap", + "scale_basis", + "n_targets", + "initial_past_cap", + "final_past_cap", + "escaped", + "frozen", + "pushed_out", + "pushed_out_rows" + ], + "title": "PastCapCensus", + "type": "object" + }, + "PushedOutTarget": { + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "init_rel": { + "title": "Init Rel", + "type": "number" + }, + "final_rel": { + "title": "Final Rel", + "type": "number" + } + }, + "required": [ + "name", + "init_rel", + "final_rel" + ], + "title": "PushedOutTarget", + "type": "object" + }, + "RegistryTargetMetadata": { + "additionalProperties": false, + "properties": { + "family": { + "title": "Family", + "type": "string" + }, + "se": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Se" + }, + "signed": { + "title": "Signed", + "type": "boolean" + }, + "notes": { + "title": "Notes", + "type": "string" + } + }, + "required": [ + "family", + "se", + "signed", + "notes" + ], + "title": "RegistryTargetMetadata", + "type": "object" + }, + "Selector": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "column", + "callable" + ], + "title": "Kind", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "title": "Selector", + "type": "object" + }, + "SkippedTarget": { + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "reason": { + "title": "Reason", + "type": "string" + } + }, + "required": [ + "name", + "reason" + ], + "title": "SkippedTarget", + "type": "object" + }, + "TargetDiagnosticV8": { + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "target_name": { + "title": "Target Name", + "type": "string" + }, + "period": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "title": "Period" + }, + "entity": { + "title": "Entity", + "type": "string" + }, + "measure": { + "$ref": "#/$defs/Selector" + }, + "filter": { + "anyOf": [ + { + "$ref": "#/$defs/Selector" + }, + { + "type": "null" + } + ] + }, + "source": { + "title": "Source", + "type": "string" + }, + "metadata": { + "additionalProperties": { + "$ref": "#/$defs/JsonValue" + }, + "title": "Metadata", + "type": "object" + }, + "target": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Target" + }, + "compiled_target": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Compiled Target" + }, + "initial_estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Initial Estimate" + }, + "final_estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Final Estimate" + }, + "relative_error": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Error" + }, + "within_tolerance": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Within Tolerance" + }, + "hierarchy": { + "$ref": "#/$defs/CalibrationHierarchy" + }, + "registry": { + "anyOf": [ + { + "$ref": "#/$defs/RegistryTargetMetadata" + }, + { + "type": "null" + } + ], + "default": null + }, + "target_loss_weight": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Loss Weight" + }, + "target_loss_weight_share": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Loss Weight Share" + }, + "target_loss_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Loss Scale" + }, + "final_capped_scaled_error": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Final Capped Scaled Error" + }, + "final_loss_contribution": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Final Loss Contribution" + } + }, + "required": [ + "name", + "target_name", + "period", + "entity", + "measure", + "filter", + "source", + "metadata", + "target", + "compiled_target", + "initial_estimate", + "final_estimate", + "relative_error", + "within_tolerance", + "hierarchy" + ], + "title": "TargetDiagnosticV8", + "type": "object" + }, + "TargetLossBasis": { + "additionalProperties": false, + "properties": { + "formula": { + "title": "Formula", + "type": "string" + }, + "cap": { + "exclusiveMinimum": 0, + "title": "Cap", + "type": "number" + }, + "target_count": { + "minimum": 0, + "title": "Target Count", + "type": "integer" + }, + "total_target_weight": { + "exclusiveMinimum": 0, + "title": "Total Target Weight", + "type": "number" + }, + "weight_kind": { + "title": "Weight Kind", + "type": "string" + }, + "scale_kind": { + "title": "Scale Kind", + "type": "string" + }, + "hash_algorithm": { + "title": "Hash Algorithm", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "formula", + "cap", + "target_count", + "total_target_weight", + "weight_kind", + "scale_kind", + "hash_algorithm", + "sha256" + ], + "title": "TargetLossBasis", + "type": "object" + }, + "TargetRegistryRef": { + "additionalProperties": false, + "properties": { + "country": { + "title": "Country", + "type": "string" + }, + "version": { + "title": "Version", + "type": "string" + }, + "n_specs": { + "exclusiveMinimum": 0, + "title": "N Specs", + "type": "integer" + } + }, + "required": [ + "country", + "version", + "n_specs" + ], + "title": "TargetRegistryRef", + "type": "object" + }, + "TargetSurface": { + "additionalProperties": false, + "properties": { + "schema_version": { + "const": 1, + "title": "Schema Version", + "type": "integer" + }, + "weight_entity": { + "title": "Weight Entity", + "type": "string" + }, + "n_targets": { + "exclusiveMinimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_records": { + "exclusiveMinimum": 0, + "title": "N Records", + "type": "integer" + }, + "constraint_matrix": { + "$ref": "#/$defs/TargetSurfaceMatrix" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + }, + "names_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Names Sha256", + "type": "string" + }, + "values_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Values Sha256", + "type": "string" + } + }, + "required": [ + "schema_version", + "weight_entity", + "n_targets", + "n_records", + "constraint_matrix", + "sha256", + "names_sha256", + "values_sha256" + ], + "title": "TargetSurface", + "type": "object" + }, + "TargetSurfaceMatrix": { + "additionalProperties": false, + "properties": { + "rows": { + "minimum": 0, + "title": "Rows", + "type": "integer" + }, + "columns": { + "minimum": 0, + "title": "Columns", + "type": "integer" + }, + "nnz": { + "minimum": 0, + "title": "Nnz", + "type": "integer" + } + }, + "required": [ + "rows", + "columns", + "nnz" + ], + "title": "TargetSurfaceMatrix", + "type": "object" + }, + "UKAreaFit": { + "additionalProperties": false, + "properties": { + "geography_level": { + "enum": [ + "local_authority", + "constituency" + ], + "title": "Geography Level", + "type": "string" + }, + "area_code": { + "title": "Area Code", + "type": "string" + }, + "country": { + "enum": [ + "England", + "Northern Ireland", + "Scotland", + "Wales" + ], + "title": "Country", + "type": "string" + }, + "n_targets": { + "exclusiveMinimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_within_10pct": { + "minimum": 0, + "title": "N Within 10Pct", + "type": "integer" + }, + "pass_rate": { + "maximum": 1, + "minimum": 0, + "title": "Pass Rate", + "type": "number" + }, + "worst_target": { + "title": "Worst Target", + "type": "string" + }, + "worst_abs_relative_error": { + "minimum": 0, + "title": "Worst Abs Relative Error", + "type": "number" + }, + "loss_contribution": { + "minimum": 0, + "title": "Loss Contribution", + "type": "number" + }, + "support_below_floor": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Support Below Floor" + }, + "nonzero_households": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Nonzero Households" + }, + "nonzero_source_households": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Nonzero Source Households" + }, + "effective_sample_size": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Effective Sample Size" + } + }, + "required": [ + "geography_level", + "area_code", + "country", + "n_targets", + "n_within_10pct", + "pass_rate", + "worst_target", + "worst_abs_relative_error", + "loss_contribution" + ], + "title": "UKAreaFit", + "type": "object" + }, + "UKConsumptionDrift": { + "additionalProperties": false, + "description": "Stage-levelled spend and its composition, design against final weights.\n\nReport only (microcosm#1113): the stage sets these levels and shares, so\ncalibration should barely move them; a large move means a bound row is\npulling weight onto the columns instead.", + "properties": { + "report_only": { + "const": true, + "title": "Report Only", + "type": "boolean" + }, + "totals": { + "additionalProperties": { + "$ref": "#/$defs/UKDriftValue" + }, + "minProperties": 1, + "title": "Totals", + "type": "object" + }, + "shares": { + "additionalProperties": { + "$ref": "#/$defs/UKDriftValue" + }, + "minProperties": 1, + "title": "Shares", + "type": "object" + } + }, + "required": [ + "report_only", + "totals", + "shares" + ], + "title": "UKConsumptionDrift", + "type": "object" + }, + "UKCountryFit": { + "additionalProperties": false, + "properties": { + "country": { + "enum": [ + "England", + "Northern Ireland", + "Scotland", + "Wales" + ], + "title": "Country", + "type": "string" + }, + "geography_level": { + "enum": [ + "local_authority", + "constituency" + ], + "title": "Geography Level", + "type": "string" + }, + "n_areas": { + "exclusiveMinimum": 0, + "title": "N Areas", + "type": "integer" + }, + "n_targets": { + "exclusiveMinimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_within_10pct": { + "minimum": 0, + "title": "N Within 10Pct", + "type": "integer" + }, + "pass_rate": { + "maximum": 1, + "minimum": 0, + "title": "Pass Rate", + "type": "number" + }, + "worst_target": { + "title": "Worst Target", + "type": "string" + }, + "worst_abs_relative_error": { + "minimum": 0, + "title": "Worst Abs Relative Error", + "type": "number" + }, + "loss_contribution": { + "minimum": 0, + "title": "Loss Contribution", + "type": "number" + } + }, + "required": [ + "country", + "geography_level", + "n_areas", + "n_targets", + "n_within_10pct", + "pass_rate", + "worst_target", + "worst_abs_relative_error", + "loss_contribution" + ], + "title": "UKCountryFit", + "type": "object" + }, + "UKDiagnosticsV1": { + "additionalProperties": false, + "properties": { + "schema_version": { + "const": 1, + "title": "Schema Version", + "type": "integer" + }, + "weights": { + "$ref": "#/$defs/UKWeightSummary" + }, + "zero_weight_rows_by_stratum": { + "items": { + "$ref": "#/$defs/UKZeroWeightStratum" + }, + "minItems": 1, + "title": "Zero Weight Rows By Stratum", + "type": "array" + }, + "target_pass_rates_by_geography_level": { + "items": { + "$ref": "#/$defs/UKGeographyPassRate" + }, + "title": "Target Pass Rates By Geography Level", + "type": "array" + }, + "target_observation_basis": { + "additionalProperties": { + "type": "string" + }, + "title": "Target Observation Basis", + "type": "object" + }, + "weakest_families": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/UKWeakestFamily" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Weakest Families" + }, + "weakest_areas_by_fit": { + "anyOf": [ + { + "$ref": "#/$defs/UKWeakestAreasByFit" + }, + { + "type": "null" + } + ], + "default": null + }, + "rotated_holdout": { + "anyOf": [ + { + "$ref": "#/$defs/UKMeasuredRotatedHoldout" + }, + { + "$ref": "#/$defs/UKSkippedRotatedHoldout" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rotated Holdout" + }, + "consumption_drift": { + "anyOf": [ + { + "$ref": "#/$defs/UKConsumptionDrift" + }, + { + "type": "null" + } + ], + "default": null + }, + "model_figures_withheld": { + "anyOf": [ + { + "$ref": "#/$defs/UKWithheldModelFigures" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "schema_version", + "weights", + "zero_weight_rows_by_stratum", + "target_pass_rates_by_geography_level", + "target_observation_basis" + ], + "title": "UKDiagnosticsV1", + "type": "object" + }, + "UKDriftValue": { + "additionalProperties": false, + "description": "One quantity at the design weights and at the final weights.", + "properties": { + "design": { + "title": "Design", + "type": "number" + }, + "final": { + "title": "Final", + "type": "number" + }, + "change": { + "title": "Change", + "type": "number" + }, + "relative_change": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Change" + } + }, + "required": [ + "design", + "final", + "change", + "relative_change" + ], + "title": "UKDriftValue", + "type": "object" + }, + "UKGeographyPassRate": { + "additionalProperties": false, + "properties": { + "geography_level": { + "enum": [ + "national", + "region", + "country", + "local_authority", + "constituency", + "brma" + ], + "title": "Geography Level", + "type": "string" + }, + "n_targets": { + "minimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_scored": { + "minimum": 0, + "title": "N Scored", + "type": "integer" + }, + "n_skipped": { + "minimum": 0, + "title": "N Skipped", + "type": "integer" + }, + "n_within_10pct": { + "minimum": 0, + "title": "N Within 10Pct", + "type": "integer" + }, + "pass_rate": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Pass Rate" + } + }, + "required": [ + "geography_level", + "n_targets", + "n_scored", + "n_skipped", + "n_within_10pct", + "pass_rate" + ], + "title": "UKGeographyPassRate", + "type": "object" + }, + "UKMeasuredRotatedHoldout": { + "additionalProperties": false, + "properties": { + "report_only": { + "const": true, + "title": "Report Only", + "type": "boolean" + }, + "method": { + "const": "rotated_folds", + "title": "Method", + "type": "string" + }, + "target_loss_cap": { + "exclusiveMinimum": 0, + "title": "Target Loss Cap", + "type": "number" + }, + "loss_weight_scale": { + "const": "held_local_grains_only", + "title": "Loss Weight Scale", + "type": "string" + }, + "target_weight_rule": { + "enum": [ + "uniform", + "grain_equal", + "grain_family_equal", + "grain_family_equal_sqrt_count", + "nation_grain_family_equal", + "nation_grain_family_equal_sqrt_count" + ], + "title": "Target Weight Rule", + "type": "string" + }, + "population": { + "const": "held_out_local_targets", + "title": "Population", + "type": "string" + }, + "grains": { + "items": { + "enum": [ + "constituency", + "local_authority", + "la" + ], + "type": "string" + }, + "minItems": 1, + "title": "Grains", + "type": "array" + }, + "n_folds": { + "minimum": 2, + "title": "N Folds", + "type": "integer" + }, + "seed": { + "title": "Seed", + "type": "integer" + }, + "solve_seed": { + "title": "Solve Seed", + "type": "integer" + }, + "mean_holdout_loss": { + "minimum": 0, + "title": "Mean Holdout Loss", + "type": "number" + }, + "worst_holdout_loss": { + "minimum": 0, + "title": "Worst Holdout Loss", + "type": "number" + }, + "fold_losses": { + "items": { + "minimum": 0, + "type": "number" + }, + "title": "Fold Losses", + "type": "array" + }, + "folds": { + "items": { + "$ref": "#/$defs/UKRotatedHoldoutFold" + }, + "title": "Folds", + "type": "array" + } + }, + "required": [ + "report_only", + "method", + "target_loss_cap", + "loss_weight_scale", + "target_weight_rule", + "population", + "grains", + "n_folds", + "seed", + "solve_seed", + "mean_holdout_loss", + "worst_holdout_loss", + "fold_losses", + "folds" + ], + "title": "UKMeasuredRotatedHoldout", + "type": "object" + }, + "UKRotatedHoldoutFold": { + "additionalProperties": false, + "properties": { + "fold": { + "minimum": 0, + "title": "Fold", + "type": "integer" + }, + "n_train_targets": { + "minimum": 0, + "title": "N Train Targets", + "type": "integer" + }, + "n_holdout_targets": { + "exclusiveMinimum": 0, + "title": "N Holdout Targets", + "type": "integer" + }, + "holdout_target_indices": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "Holdout Target Indices", + "type": "array" + }, + "training_national_rows": { + "minimum": 0, + "title": "Training National Rows", + "type": "integer" + }, + "holdout_loss": { + "minimum": 0, + "title": "Holdout Loss", + "type": "number" + } + }, + "required": [ + "fold", + "n_train_targets", + "n_holdout_targets", + "holdout_target_indices", + "training_national_rows", + "holdout_loss" + ], + "title": "UKRotatedHoldoutFold", + "type": "object" + }, + "UKSkippedRotatedHoldout": { + "additionalProperties": false, + "properties": { + "skipped": { + "const": true, + "title": "Skipped", + "type": "boolean" + } + }, + "required": [ + "skipped" + ], + "title": "UKSkippedRotatedHoldout", + "type": "object" + }, + "UKWeakestAreasByFit": { + "additionalProperties": false, + "properties": { + "limit": { + "exclusiveMinimum": 0, + "title": "Limit", + "type": "integer" + }, + "n_areas_scored": { + "minimum": 0, + "title": "N Areas Scored", + "type": "integer" + }, + "bottom_by_fit": { + "items": { + "$ref": "#/$defs/UKAreaFit" + }, + "title": "Bottom By Fit", + "type": "array" + }, + "countries": { + "items": { + "$ref": "#/$defs/UKCountryFit" + }, + "title": "Countries", + "type": "array" + } + }, + "required": [ + "limit", + "n_areas_scored", + "bottom_by_fit", + "countries" + ], + "title": "UKWeakestAreasByFit", + "type": "object" + }, + "UKWeakestFamily": { + "additionalProperties": false, + "properties": { + "family": { + "title": "Family", + "type": "string" + }, + "n_targets": { + "exclusiveMinimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_within_10pct": { + "minimum": 0, + "title": "N Within 10Pct", + "type": "integer" + }, + "pass_rate": { + "maximum": 1, + "minimum": 0, + "title": "Pass Rate", + "type": "number" + }, + "worst_target": { + "title": "Worst Target", + "type": "string" + }, + "worst_abs_relative_error": { + "minimum": 0, + "title": "Worst Abs Relative Error", + "type": "number" + }, + "loss_contribution": { + "minimum": 0, + "title": "Loss Contribution", + "type": "number" + }, + "loss_share": { + "maximum": 1, + "minimum": 0, + "title": "Loss Share", + "type": "number" + } + }, + "required": [ + "family", + "n_targets", + "n_within_10pct", + "pass_rate", + "worst_target", + "worst_abs_relative_error", + "loss_contribution", + "loss_share" + ], + "title": "UKWeakestFamily", + "type": "object" + }, + "UKWeightSummary": { + "additionalProperties": false, + "properties": { + "n_records": { + "exclusiveMinimum": 0, + "title": "N Records", + "type": "integer" + }, + "positive_weight_records": { + "minimum": 0, + "title": "Positive Weight Records", + "type": "integer" + }, + "zero_weight_records": { + "minimum": 0, + "title": "Zero Weight Records", + "type": "integer" + }, + "total_weight": { + "minimum": 0, + "title": "Total Weight", + "type": "number" + }, + "effective_sample_size": { + "minimum": 0, + "title": "Effective Sample Size", + "type": "number" + }, + "ess_fraction": { + "maximum": 1, + "minimum": 0, + "title": "Ess Fraction", + "type": "number" + }, + "median_positive_weight": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Median Positive Weight" + }, + "max_weight": { + "minimum": 0, + "title": "Max Weight", + "type": "number" + }, + "max_to_median_positive_weight": { + "anyOf": [ + { + "minimum": 1, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Max To Median Positive Weight" + }, + "top_1pct_weight_share": { + "maximum": 1, + "minimum": 0, + "title": "Top 1Pct Weight Share", + "type": "number" + } + }, + "required": [ + "n_records", + "positive_weight_records", + "zero_weight_records", + "total_weight", + "effective_sample_size", + "ess_fraction", + "median_positive_weight", + "max_weight", + "max_to_median_positive_weight", + "top_1pct_weight_share" + ], + "title": "UKWeightSummary", + "type": "object" + }, + "UKWithheldModelFigures": { + "additionalProperties": false, + "description": "Target rows whose model figures stay in the build (microcosm#1262).\n\nA row resting on fewer than ``min_source_households`` distinct source\nhouseholds keeps its place without its model figures. While any does, the\npayload's totals are restated over the published rows (``totals_basis``)\nso that none can be differenced against them, and its loss trajectory is\nempty.", + "properties": { + "targets": { + "exclusiveMinimum": 0, + "title": "Targets", + "type": "integer" + }, + "min_source_households": { + "exclusiveMinimum": 0, + "title": "Min Source Households", + "type": "integer" + }, + "totals_basis": { + "const": "published_rows", + "title": "Totals Basis", + "type": "string" + }, + "note": { + "title": "Note", + "type": "string" + } + }, + "required": [ + "targets", + "min_source_households", + "totals_basis", + "note" + ], + "title": "UKWithheldModelFigures", + "type": "object" + }, + "UKZeroWeightStratum": { + "additionalProperties": false, + "properties": { + "stratum": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "minProperties": 1, + "title": "Stratum", + "type": "object" + }, + "rows": { + "minimum": 0, + "title": "Rows", + "type": "integer" + }, + "positive_weight_rows": { + "minimum": 0, + "title": "Positive Weight Rows", + "type": "integer" + }, + "zero_weight_rows": { + "minimum": 0, + "title": "Zero Weight Rows", + "type": "integer" + }, + "weight_sum": { + "minimum": 0, + "title": "Weight Sum", + "type": "number" + } + }, + "required": [ + "stratum", + "rows", + "positive_weight_rows", + "zero_weight_rows", + "weight_sum" + ], + "title": "UKZeroWeightStratum", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "schema_version": { + "const": 8, + "title": "Schema Version", + "type": "integer" + }, + "weight_entity": { + "title": "Weight Entity", + "type": "string" + }, + "options": { + "additionalProperties": { + "$ref": "#/$defs/JsonValue" + }, + "title": "Options", + "type": "object" + }, + "target_surface": { + "$ref": "#/$defs/TargetSurface" + }, + "target_registry": { + "$ref": "#/$defs/TargetRegistryRef" + }, + "l0_lambda": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "L0 Lambda" + }, + "n_nonzero": { + "minimum": 0, + "title": "N Nonzero", + "type": "integer" + }, + "n_records": { + "exclusiveMinimum": 0, + "title": "N Records", + "type": "integer" + }, + "initial_loss": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Initial Loss" + }, + "final_loss": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Final Loss" + }, + "fraction_within_10pct": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Fraction Within 10Pct" + }, + "effective_sample_size": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Effective Sample Size" + }, + "realized_max_weight_ratio": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Realized Max Weight Ratio" + }, + "top_1pct_weight_share": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Top 1Pct Weight Share" + }, + "loss_trajectory": { + "items": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "title": "Loss Trajectory", + "type": "array" + }, + "skipped": { + "items": { + "$ref": "#/$defs/SkippedTarget" + }, + "title": "Skipped", + "type": "array" + }, + "past_cap_census": { + "anyOf": [ + { + "$ref": "#/$defs/PastCapCensus" + }, + { + "type": "null" + } + ] + }, + "diagnostic_warnings": { + "items": { + "$ref": "#/$defs/DiagnosticsWarning" + }, + "title": "Diagnostic Warnings", + "type": "array" + }, + "targets": { + "items": { + "$ref": "#/$defs/TargetDiagnosticV8" + }, + "title": "Targets", + "type": "array" + }, + "target_loss_basis": { + "anyOf": [ + { + "$ref": "#/$defs/TargetLossBasis" + }, + { + "type": "null" + } + ], + "default": null + }, + "build": { + "anyOf": [ + { + "additionalProperties": { + "$ref": "#/$defs/JsonValue" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Build" + }, + "uk_diagnostics": { + "anyOf": [ + { + "$ref": "#/$defs/UKDiagnosticsV1" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "schema_version", + "weight_entity", + "options", + "target_surface", + "target_registry", + "l0_lambda", + "n_nonzero", + "n_records", + "initial_loss", + "final_loss", + "fraction_within_10pct", + "effective_sample_size", + "realized_max_weight_ratio", + "top_1pct_weight_share", + "loss_trajectory", + "skipped", + "past_cap_census", + "diagnostic_warnings", + "targets" + ], + "type": "object" +} diff --git a/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py b/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py new file mode 100644 index 000000000..2a437107d --- /dev/null +++ b/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py @@ -0,0 +1,745 @@ +"""The JSON Schema generated for calibration diagnostics schema 8. + +The pydantic models stay the authority. These tests pin what the generated +schema accepts and refuses, and keep its list of unexpressed model rules +complete: every validator in the schema-8 model tree must be listed. +""" + +from __future__ import annotations + +import importlib.util +import json +from collections.abc import Callable +from functools import cache +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator +from pydantic import ValidationError + +from microcosm import diagnostics +from test_support.microcosm_diagnostics.json_schema import ( + json_schema_errors, + load_strict_json, + published_json_schema_errors, +) +from test_support.microcosm_diagnostics.payloads import ( + schema_8_payload, + uk_schema_8_payload, +) +from test_support.paths import paths_for + +_TEST_PATHS = paths_for("microcosm-diagnostics") +DRAFT_2020_12 = "https://json-schema.org/draft/2020-12/schema" +#: The documented location consumers such as the collector read. +PUBLISHED_SCHEMA_PATH = ( + "packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/" + "calibration_diagnostics.v8.schema.json" +) + +DocumentBuilder = Callable[[], object] +DocumentMutation = Callable[[dict], None] + + +@cache +def _generated_schema_text() -> str: + return json.dumps(diagnostics.calibration_diagnostics_v8_json_schema()) + + +def _generated_schema() -> dict: + """A fresh copy per test, so no test can change another's schema.""" + + return json.loads(_generated_schema_text()) + + +def _set_value(path: tuple[object, ...], value: object) -> DocumentMutation: + def mutate(document: dict) -> None: + owner = document + for part in path[:-1]: + owner = owner[part] + owner[path[-1]] = value + + return mutate + + +def _delete_value(path: tuple[object, ...]) -> DocumentMutation: + def mutate(document: dict) -> None: + owner = document + for part in path[:-1]: + owner = owner[part] + del owner[path[-1]] + + return mutate + + +def _document(builder: DocumentBuilder, mutation: DocumentMutation | None) -> object: + document = builder() + if mutation is not None: + mutation(document) + return document + + +def _payload_with_every_optional_block() -> dict: + payload = schema_8_payload() + target = payload["targets"][0] + target["period"] = "2024" + target["filter"] = {"kind": "callable", "name": "is_adult"} + target["metadata"] = {"ledger": {"values": [1, 2.5, None, True, "text"]}} + target["hierarchy"]["dimensions"] = [ + {"id": "sex", "label": "Sex", "value_id": "female", "value_label": "Female"} + ] + target.update( + { + "registry": { + "family": "census_population", + "se": None, + "signed": False, + "notes": "", + }, + "target_loss_weight": 1.0, + "target_loss_weight_share": 1.0, + "target_loss_scale": 2.0, + "final_capped_scaled_error": 0.0, + "final_loss_contribution": 0.0, + } + ) + payload["target_registry"]["n_specs"] = 2 + payload["l0_lambda"] = None + payload["loss_trajectory"] = [0.1, None, 0.01] + payload["skipped"] = [{"name": "ghost@2024", "reason": "measure is missing"}] + payload["diagnostic_warnings"] = [ + { + "code": "target_loss_attribution_withheld", + "severity": "warning", + "message": "Attribution could not be validated.", + } + ] + payload["past_cap_census"] = { + "cap": 1.0, + "scale_basis": "target", + "n_targets": 1, + "initial_past_cap": 1, + "final_past_cap": 0, + "escaped": 1, + "frozen": 0, + "pushed_out": 0, + "pushed_out_rows": [ + {"name": "population@2024", "init_rel": 1.5, "final_rel": 0.0} + ], + } + payload["target_loss_basis"] = { + "formula": "sum(weight * min(scaled_error, cap) ** 2)", + "cap": 1.0, + "target_count": 1, + "total_target_weight": 1.0, + "weight_kind": "provided", + "scale_kind": "target", + "hash_algorithm": "sha256", + "sha256": "b" * 64, + } + payload["build"] = {"release_id": "fixture", "timing": {"seconds": 1.5}} + return payload + + +def _uk_payload_with_skipped_holdout() -> dict: + payload = uk_schema_8_payload() + payload["uk_diagnostics"]["rotated_holdout"] = {"skipped": True} + return payload + + +def _legacy_schema_6_payload() -> dict: + """A document the parser accepts through its schema-6 compatibility branch.""" + + payload = schema_8_payload() + payload["schema_version"] = 6 + del payload["targets"][0]["hierarchy"] + return payload + + +def _array_document() -> list: + return [] + + +def test_generated_schema_declares_draft_2020_12_and_passes_its_metaschema() -> None: + schema = _generated_schema() + + assert schema["$schema"] == DRAFT_2020_12 + assert schema["$id"] == "calibration_diagnostics.v8.schema.json" + assert schema["title"] == "Microcosm calibration diagnostics, schema 8" + assert diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID == schema["$id"] + Draft202012Validator.check_schema(schema) + + +def test_generated_schema_pins_version_8_and_forbids_undeclared_fields() -> None: + schema = _generated_schema() + object_definitions = { + name: definition + for name, definition in schema["$defs"].items() + if definition.get("type") == "object" + } + + assert schema["properties"]["schema_version"]["const"] == 8 + assert schema["additionalProperties"] is False + assert "UKDiagnosticsV1" in object_definitions + assert all( + definition["additionalProperties"] is False + for definition in object_definitions.values() + ) + + +@pytest.mark.parametrize( + "builder", + [ + pytest.param(schema_8_payload, id="minimal-us-registry"), + pytest.param(_payload_with_every_optional_block, id="every-optional-block"), + pytest.param(uk_schema_8_payload, id="complete-uk-extension"), + pytest.param(_uk_payload_with_skipped_holdout, id="skipped-uk-holdout"), + ], +) +def test_generated_schema_accepts_schema_8_documents( + builder: DocumentBuilder, +) -> None: + document = load_strict_json(json.dumps(builder(), allow_nan=False)) + diagnostics.CalibrationDiagnosticsV8.model_validate(document) + + assert json_schema_errors(document, _generated_schema()) == [] + + +@pytest.mark.parametrize( + ("builder", "mutation", "error_path"), + [ + pytest.param( + schema_8_payload, + _delete_value(("targets",)), + "$", + id="missing-required-field", + ), + pytest.param( + schema_8_payload, + _set_value(("unexpected",), 1), + "$", + id="undeclared-top-level-field", + ), + pytest.param( + uk_schema_8_payload, + _set_value(("uk_diagnostics", "weights", "unexpected"), 1), + "$.uk_diagnostics.weights", + id="undeclared-uk-field", + ), + pytest.param( + schema_8_payload, + _set_value(("n_records",), "2"), + "$.n_records", + id="string-for-integer", + ), + pytest.param( + schema_8_payload, + _set_value(("n_nonzero",), True), + "$.n_nonzero", + id="boolean-for-integer", + ), + pytest.param( + schema_8_payload, + _set_value(("final_loss",), "0.01"), + "$.final_loss", + id="string-for-number", + ), + pytest.param( + schema_8_payload, + _set_value(("n_nonzero",), -1), + "$.n_nonzero", + id="negative-count", + ), + pytest.param( + schema_8_payload, + _set_value(("target_registry", "n_specs"), 0), + "$.target_registry.n_specs", + id="zero-for-positive-count", + ), + pytest.param( + schema_8_payload, + _set_value(("schema_version",), 7), + "$.schema_version", + id="schema-version-7", + ), + pytest.param( + schema_8_payload, + _set_value(("target_surface", "sha256"), "A" * 64), + "$.target_surface.sha256", + id="uppercase-digest", + ), + pytest.param( + schema_8_payload, + _set_value(("targets", 0, "measure", "kind"), "expression"), + "$.targets[0].measure.kind", + id="unknown-selector-kind", + ), + pytest.param( + uk_schema_8_payload, + _set_value( + ( + "uk_diagnostics", + "target_pass_rates_by_geography_level", + 0, + "geography_level", + ), + "ward", + ), + "$.uk_diagnostics.target_pass_rates_by_geography_level[0].geography_level", + id="unknown-uk-geography-level", + ), + pytest.param( + uk_schema_8_payload, + _set_value(("uk_diagnostics", "weights", "ess_fraction"), 1.5), + "$.uk_diagnostics.weights.ess_fraction", + id="fraction-above-one", + ), + pytest.param( + schema_8_payload, + _set_value(("weight_entity",), None), + "$.weight_entity", + id="null-for-required-text", + ), + pytest.param( + uk_schema_8_payload, + _set_value(("uk_diagnostics", "zero_weight_rows_by_stratum"), []), + "$.uk_diagnostics.zero_weight_rows_by_stratum", + id="empty-required-list", + ), + pytest.param( + uk_schema_8_payload, + _set_value(("uk_diagnostics", "rotated_holdout", "n_folds"), 1), + "$.uk_diagnostics.rotated_holdout.n_folds", + id="too-few-holdout-folds", + ), + pytest.param( + schema_8_payload, + _set_value(("targets", 0, "period"), 2024.5), + "$.targets[0].period", + id="fractional-period", + ), + pytest.param( + schema_8_payload, + _set_value(("loss_trajectory",), ["0.1"]), + "$.loss_trajectory[0]", + id="string-in-loss-trajectory", + ), + pytest.param( + _legacy_schema_6_payload, + None, + "$.schema_version", + id="legacy-schema-6-document", + ), + pytest.param(_array_document, None, "$", id="array-document"), + ], +) +def test_generated_schema_and_models_reject_invalid_documents( + builder: DocumentBuilder, + mutation: DocumentMutation | None, + error_path: str, +) -> None: + document = _document(builder, mutation) + + errors = json_schema_errors(document, _generated_schema()) + + assert any(error.startswith(f"{error_path}: ") for error in errors), errors + with pytest.raises(ValidationError): + diagnostics.CalibrationDiagnosticsV8.model_validate(document) + + +_DUPLICATED_DIMENSION = { + "id": "sex", + "label": "Sex", + "value_id": "female", + "value_label": "Female", +} + +#: One or more documents per documented gap: the generated schema accepts +#: each, and the models refuse it with the given message. +GAP_CASES = [ + pytest.param( + "finite-numbers", + schema_8_payload, + _set_value(("final_loss",), float("nan")), + "must be finite or null", + id="finite-numbers-top-level-scalar", + ), + pytest.param( + "finite-numbers", + schema_8_payload, + _set_value(("options", "nested"), [{"value": float("inf")}]), + "must be finite or null", + id="finite-numbers-nested-solver-option", + ), + pytest.param( + "strict-integers", + schema_8_payload, + _set_value(("n_records",), 2.0), + "valid integer", + id="strict-integers", + ), + pytest.param( + "non-blank-text", + schema_8_payload, + _set_value(("targets", 0, "source"), " "), + "non-whitespace", + id="non-blank-text", + ), + pytest.param( + "digest-pattern-anchoring", + schema_8_payload, + _set_value(("target_surface", "sha256"), "a" * 64 + "\n"), + "should match pattern", + id="digest-pattern-anchoring", + ), + pytest.param( + "hierarchy-identity", + schema_8_payload, + _set_value( + ("targets", 0, "hierarchy", "category", "provider_id"), + "another-provider", + ), + "provider_id must match provider id", + id="hierarchy-category-provider", + ), + pytest.param( + "hierarchy-identity", + schema_8_payload, + _set_value( + ("targets", 0, "hierarchy", "dimensions"), + [_DUPLICATED_DIMENSION, _DUPLICATED_DIMENSION], + ), + "dimension ids must be unique", + id="hierarchy-dimension-ids", + ), + pytest.param( + "target-hierarchy", + schema_8_payload, + _set_value(("targets", 0, "hierarchy", "target", "id"), "wrong-target"), + "hierarchy target id must match target_name", + id="target-hierarchy", + ), + pytest.param( + "target-surface-shape", + schema_8_payload, + _set_value(("target_surface", "constraint_matrix", "rows"), 2), + "rows must equal n_targets", + id="target-surface-shape", + ), + pytest.param( + "diagnostics-counts", + schema_8_payload, + _set_value(("n_nonzero",), 3), + "n_nonzero cannot exceed n_records", + id="diagnostics-counts", + ), + pytest.param( + "uk-extension-presence", + uk_schema_8_payload, + _delete_value(("uk_diagnostics",)), + "require a uk_diagnostics block", + id="uk-registry-without-uk-extension", + ), + pytest.param( + "uk-extension-presence", + uk_schema_8_payload, + _set_value(("target_registry", "country"), "us"), + "only valid when target_registry.country is 'uk'", + id="uk-extension-on-another-registry", + ), + pytest.param( + "uk-weight-summary", + uk_schema_8_payload, + _set_value(("uk_diagnostics", "weights", "ess_fraction"), 0.25), + "ess_fraction must equal", + id="uk-weight-summary", + ), + pytest.param( + "uk-zero-weight-strata", + uk_schema_8_payload, + _set_value(("uk_diagnostics", "zero_weight_rows_by_stratum", 0, "rows"), 3), + "stratum row counts do not reconcile", + id="uk-zero-weight-strata", + ), + pytest.param( + "uk-geography-pass-rates", + uk_schema_8_payload, + _set_value( + ( + "uk_diagnostics", + "target_pass_rates_by_geography_level", + 4, + "geography_level", + ), + "national", + ), + "levels must be unique", + id="uk-geography-pass-rates", + ), + pytest.param( + "uk-local-fit", + uk_schema_8_payload, + _delete_value(("uk_diagnostics", "weakest_areas_by_fit")), + "require both family and area summaries", + id="uk-local-fit", + ), + pytest.param( + "uk-rotated-holdout", + uk_schema_8_payload, + _set_value(("uk_diagnostics", "rotated_holdout", "fold_losses"), [0.1, 0.3]), + "fold rows must match fold_losses", + id="uk-rotated-holdout", + ), + pytest.param( + "uk-consumption-drift", + uk_schema_8_payload, + _set_value( + ( + "uk_diagnostics", + "consumption_drift", + "totals", + "petrol_spending", + "change", + ), + 3.0, + ), + "final minus design", + id="uk-consumption-drift", + ), +] + + +@pytest.mark.parametrize( + ("gap_key", "builder", "mutation", "model_error"), + GAP_CASES, +) +def test_generated_schema_accepts_documents_only_the_models_reject( + gap_key: str, + builder: DocumentBuilder, + mutation: DocumentMutation, + model_error: str, +) -> None: + documented_keys = { + gap.key for gap in diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS + } + document = _document(builder, mutation) + + assert gap_key in documented_keys + assert json_schema_errors(document, _generated_schema()) == [] + with pytest.raises(ValidationError, match=model_error): + diagnostics.CalibrationDiagnosticsV8.model_validate(document) + + +def test_every_documented_gap_has_a_characterizing_document() -> None: + documented_keys = [ + gap.key for gap in diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS + ] + characterized_keys = {case.values[0] for case in GAP_CASES} + + assert len(documented_keys) == len(set(documented_keys)) + assert set(documented_keys) == characterized_keys + + +@pytest.mark.parametrize( + ("mutation", "error_path"), + [ + pytest.param( + _set_value(("uk_diagnostics", "rotated_holdout"), {"skipped": 1}), + "$.uk_diagnostics.rotated_holdout", + id="one-for-true", + ), + pytest.param( + _set_value(("uk_diagnostics", "schema_version"), True), + "$.uk_diagnostics.schema_version", + id="true-for-one", + ), + ], +) +def test_generated_schema_requires_the_literal_values_the_writer_emits( + mutation: DocumentMutation, + error_path: str, +) -> None: + document = _document(uk_schema_8_payload, mutation) + + errors = json_schema_errors(document, _generated_schema()) + model = diagnostics.CalibrationDiagnosticsV8.model_validate(document) + rewritten = json.loads(model.model_dump_json(exclude_defaults=True)) + + assert any(error.startswith(f"{error_path}: ") for error in errors), errors + assert json_schema_errors(rewritten, _generated_schema()) == [] + + +def _validator_qualified_names(core_schema: object) -> set[str]: + """Every validator function pydantic runs for a model, by qualified name.""" + + names: set[str] = set() + pending = [core_schema] + visited: set[int] = set() + while pending: + node = pending.pop() + if id(node) in visited: + continue + visited.add(id(node)) + if isinstance(node, dict): + if str(node.get("type", "")).startswith("function-"): + names.add(node["function"]["function"].__qualname__) + pending.extend(node.values()) + elif isinstance(node, (list, tuple)): + pending.extend(node) + return names + + +def test_every_model_validator_is_documented_as_a_schema_gap() -> None: + model_validators = _validator_qualified_names( + diagnostics.CalibrationDiagnosticsV8.__pydantic_core_schema__ + ) + documented_validators = { + validator + for gap in diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS + for validator in gap.validators + } + + assert "CalibrationDiagnosticsV8.reconcile_counts" in model_validators + assert documented_validators == model_validators + + +def test_generated_schema_description_lists_every_gap() -> None: + description = _generated_schema()["description"] + + assert "microcosm.diagnostics.parse_calibration_diagnostics" in description + assert diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE in ( + description + ) + for gap in diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS: + assert gap.rule in description, gap.key + + +def test_rendered_schema_is_deterministic_json_text() -> None: + rendered = diagnostics.render_calibration_diagnostics_v8_json_schema() + + assert rendered == diagnostics.render_calibration_diagnostics_v8_json_schema() + assert rendered.endswith("}\n") + assert json.loads(rendered) == _generated_schema() + assert list(json.loads(rendered))[:5] == [ + "$schema", + "$id", + "$comment", + "title", + "description", + ] + + +def _load_generator_tool(): + path = ( + _TEST_PATHS.repository + / "tools" + / "generate_calibration_diagnostics_json_schema.py" + ) + spec = importlib.util.spec_from_file_location( + "generate_calibration_diagnostics_json_schema", path + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_committed_schema_matches_the_generated_schema() -> None: + committed = diagnostics.committed_calibration_diagnostics_v8_json_schema_path() + + assert committed.read_bytes() == ( + diagnostics.render_calibration_diagnostics_v8_json_schema().encode("utf-8") + ) + assert diagnostics.load_calibration_diagnostics_v8_json_schema() == ( + diagnostics.calibration_diagnostics_v8_json_schema() + ) + diagnostics.assert_calibration_diagnostics_v8_json_schema_current() + + +def test_committed_schema_lives_at_its_documented_path() -> None: + committed = diagnostics.committed_calibration_diagnostics_v8_json_schema_path() + package_readme = (_TEST_PATHS.package / "README.md").read_text(encoding="utf-8") + + assert committed.resolve() == ( + (_TEST_PATHS.repository / PUBLISHED_SCHEMA_PATH).resolve() + ) + assert committed.name == diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID + assert f"`{PUBLISHED_SCHEMA_PATH}`" in package_readme + + +def test_generator_tool_writes_and_checks_the_committed_schema( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + tool = _load_generator_tool() + committed = diagnostics.committed_calibration_diagnostics_v8_json_schema_path() + written = tmp_path / committed.name + + assert tool.main(["--check"]) == 0 + assert tool.main(["--out", str(written)]) == 0 + assert written.read_bytes() == committed.read_bytes() + + written.write_text("{}\n", encoding="utf-8") + assert tool.main(["--check", "--out", str(written)]) == 1 + assert "regenerate it with" in capsys.readouterr().err + with pytest.raises(ValueError, match="differs from the schema the models generate"): + diagnostics.assert_calibration_diagnostics_v8_json_schema_current(written) + + +LEGACY_AREA_SUPPORT_FIGURES = { + "nonzero_households": 60, + "nonzero_source_households": 55, + "effective_sample_size": 52.5, +} + + +def _uk_payload_with_area_fit(area_fields: dict[str, object]) -> dict: + """The complete UK document, its one weakest area carrying ``area_fields``.""" + + payload = uk_schema_8_payload() + (area,) = payload["uk_diagnostics"]["weakest_areas_by_fit"]["bottom_by_fit"] + del area["support_below_floor"] + area.update(area_fields) + return payload + + +@pytest.mark.parametrize( + "area_fields", + [ + pytest.param({"support_below_floor": True}, id="floor-flag-only"), + pytest.param({}, id="neither-flag-nor-figures"), + pytest.param(LEGACY_AREA_SUPPORT_FIGURES, id="legacy-support-figures"), + pytest.param( + {"support_below_floor": None} + | {field: None for field in LEGACY_AREA_SUPPORT_FIGURES}, + id="explicit-nulls", + ), + ], +) +def test_published_schema_accepts_uk_area_fits_with_the_floor_flag_or_legacy_support_figures( + area_fields: dict[str, object], +) -> None: + # Released diagnostics withhold an area's support figures and carry the + # floor flag instead (microcosm#1301); earlier documents carry the figures. + document = load_strict_json( + json.dumps(_uk_payload_with_area_fit(area_fields), allow_nan=False) + ) + diagnostics.CalibrationDiagnosticsV8.model_validate(document) + + assert published_json_schema_errors(document) == [] + + +def test_published_schema_and_models_refuse_a_negative_legacy_support_figure() -> None: + document = _uk_payload_with_area_fit( + LEGACY_AREA_SUPPORT_FIGURES | {"effective_sample_size": -1.0} + ) + + assert any( + error.startswith( + "$.uk_diagnostics.weakest_areas_by_fit.bottom_by_fit[0]" + ".effective_sample_size: " + ) + for error in published_json_schema_errors(document) + ) + with pytest.raises(ValidationError): + diagnostics.CalibrationDiagnosticsV8.model_validate(document) diff --git a/packages/microcosm-diagnostics/tests/engine_free/shared/test_schema.py b/packages/microcosm-diagnostics/tests/engine_free/shared/test_schema.py index 399b6ab10..864afdf3f 100644 --- a/packages/microcosm-diagnostics/tests/engine_free/shared/test_schema.py +++ b/packages/microcosm-diagnostics/tests/engine_free/shared/test_schema.py @@ -12,218 +12,10 @@ parse_calibration_diagnostics, write_calibration_diagnostics, ) - - -def _payload() -> dict: - digest = "a" * 64 - return { - "schema_version": 8, - "weight_entity": "household", - "options": {"epochs": 1}, - "target_surface": { - "schema_version": 1, - "weight_entity": "household", - "n_targets": 1, - "n_records": 2, - "constraint_matrix": {"rows": 1, "columns": 2, "nnz": 2}, - "sha256": digest, - "names_sha256": digest, - "values_sha256": digest, - }, - "target_registry": {"country": "us", "version": digest, "n_specs": 1}, - "l0_lambda": 0.0, - "n_nonzero": 2, - "n_records": 2, - "initial_loss": 0.1, - "final_loss": 0.01, - "fraction_within_10pct": 1.0, - "effective_sample_size": 2.0, - "realized_max_weight_ratio": 1.0, - "top_1pct_weight_share": 0.5, - "loss_trajectory": [0.1], - "skipped": [], - "past_cap_census": None, - "diagnostic_warnings": [], - "targets": [ - { - "name": "population@2024", - "target_name": "population", - "period": 2024, - "entity": "household", - "measure": {"kind": "column", "name": "household_count"}, - "filter": None, - "source": "Census fixture", - "metadata": {}, - "target": 2.0, - "compiled_target": 2.0, - "initial_estimate": 1.8, - "final_estimate": 2.0, - "relative_error": 0.0, - "within_tolerance": None, - "hierarchy": { - "provider": {"id": "census", "label": "Census"}, - "category": { - "id": "census.population", - "label": "Population", - "provider_id": "census", - }, - "geography": { - "id": "0100000US", - "label": "United States", - "level": "country", - }, - "dimensions": [], - "target": {"id": "population", "label": "Population"}, - }, - } - ], - } - - -def _uk_payload() -> dict: - payload = _payload() - payload["target_registry"]["country"] = "uk" - payload["n_nonzero"] = 1 - payload["effective_sample_size"] = 1.0 - payload["top_1pct_weight_share"] = 1.0 - payload["uk_diagnostics"] = { - "schema_version": 1, - "weights": { - "n_records": 2, - "positive_weight_records": 1, - "zero_weight_records": 1, - "total_weight": 1.0, - "effective_sample_size": 1.0, - "ess_fraction": 0.5, - "median_positive_weight": 1.0, - "max_weight": 1.0, - "max_to_median_positive_weight": 1.0, - "top_1pct_weight_share": 1.0, - }, - "zero_weight_rows_by_stratum": [ - { - "stratum": {"source": "frs"}, - "rows": 2, - "positive_weight_rows": 1, - "zero_weight_rows": 1, - "weight_sum": 1.0, - } - ], - "target_pass_rates_by_geography_level": [ - { - "geography_level": level, - "n_targets": 1 if level == "national" else 0, - "n_scored": 1 if level == "national" else 0, - "n_skipped": 0, - "n_within_10pct": 1 if level == "national" else 0, - "pass_rate": 1.0 if level == "national" else None, - } - for level in ( - "national", - "region", - "country", - "local_authority", - "constituency", - ) - ], - "target_observation_basis": {"population@2024": "annual_stock"}, - "weakest_families": [ - { - "family": "population", - "n_targets": 1, - "n_within_10pct": 1, - "pass_rate": 1.0, - "worst_target": "population@2024", - "worst_abs_relative_error": 0.0, - "loss_contribution": 0.0, - "loss_share": 0.0, - } - ], - "weakest_areas_by_fit": { - "limit": 15, - "n_areas_scored": 1, - "bottom_by_fit": [ - { - "geography_level": "constituency", - "area_code": "E14000001", - "country": "England", - "n_targets": 1, - "n_within_10pct": 1, - "pass_rate": 1.0, - "worst_target": "population@2024", - "worst_abs_relative_error": 0.0, - "loss_contribution": 0.0, - "support_below_floor": True, - } - ], - "countries": [ - { - "country": "England", - "geography_level": "constituency", - "n_areas": 1, - "n_targets": 1, - "n_within_10pct": 1, - "pass_rate": 1.0, - "worst_target": "population@2024", - "worst_abs_relative_error": 0.0, - "loss_contribution": 0.0, - } - ], - }, - "rotated_holdout": { - "report_only": True, - "method": "rotated_folds", - "target_loss_cap": 10.0, - "loss_weight_scale": "held_local_grains_only", - "target_weight_rule": "grain_equal", - "population": "held_out_local_targets", - "grains": ["constituency", "local_authority"], - "n_folds": 2, - "seed": 20260529, - "solve_seed": 7, - "mean_holdout_loss": 0.15, - "worst_holdout_loss": 0.2, - "fold_losses": [0.1, 0.2], - "folds": [ - { - "fold": 0, - "n_train_targets": 1, - "n_holdout_targets": 1, - "holdout_target_indices": [0], - "training_national_rows": 1, - "holdout_loss": 0.1, - }, - { - "fold": 1, - "n_train_targets": 1, - "n_holdout_targets": 1, - "holdout_target_indices": [1], - "training_national_rows": 1, - "holdout_loss": 0.2, - }, - ], - }, - "consumption_drift": { - "report_only": True, - "totals": { - "petrol_spending": { - "design": 100.0, - "final": 102.0, - "change": 2.0, - "relative_change": 0.02, - } - }, - "shares": { - "fuel_car_households_without_fuel": { - "design": 0.0, - "final": 0.0, - "change": 0.0, - "relative_change": None, - } - }, - }, - } - return payload +from test_support.microcosm_diagnostics.payloads import ( + schema_8_payload, + uk_schema_8_payload, +) def _delete_path(payload: dict, path: tuple[object, ...]) -> None: @@ -234,14 +26,16 @@ def _delete_path(payload: dict, path: tuple[object, ...]) -> None: def test_schema_8_round_trips_as_a_typed_model() -> None: - model = parse_calibration_diagnostics(_payload()) + model = parse_calibration_diagnostics(schema_8_payload()) assert isinstance(model, CalibrationDiagnosticsV8) - assert json.loads(model.model_dump_json(exclude_defaults=True)) == _payload() + assert ( + json.loads(model.model_dump_json(exclude_defaults=True)) == schema_8_payload() + ) assert calibration_diagnostics_json_schema()["$defs"] def test_schema_8_round_trips_the_complete_uk_extension() -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() model = parse_calibration_diagnostics(payload) assert isinstance(model, CalibrationDiagnosticsV8) @@ -252,7 +46,7 @@ def test_uk_area_fit_still_parses_legacy_support_figures() -> None: # Released diagnostics carry the floor flag in place of an area's support # figures (microcosm#1301); payloads written before still parse, and a # row may carry neither. - payload = _uk_payload() + payload = uk_schema_8_payload() (area,) = payload["uk_diagnostics"]["weakest_areas_by_fit"]["bottom_by_fit"] del area["support_below_floor"] bare = parse_calibration_diagnostics(payload) @@ -336,7 +130,7 @@ def test_every_declared_uk_record_forbids_undeclared_fields() -> None: def test_schema_8_rejects_missing_fields_across_the_uk_extension( field_path: tuple[object, ...], ) -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() _delete_path(payload, field_path) with pytest.raises(ValidationError): @@ -368,7 +162,7 @@ def test_schema_8_rejects_missing_fields_across_the_uk_extension( def test_schema_8_rejects_undeclared_fields_across_the_uk_extension( field_path: tuple[object, ...], ) -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() owner = payload for part in field_path: owner = owner[part] @@ -379,7 +173,7 @@ def test_schema_8_rejects_undeclared_fields_across_the_uk_extension( def test_schema_8_requires_the_typed_uk_extension_for_uk_registries() -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() del payload["uk_diagnostics"] with pytest.raises(ValidationError, match="require a uk_diagnostics block"): @@ -387,7 +181,7 @@ def test_schema_8_requires_the_typed_uk_extension_for_uk_registries() -> None: def test_schema_8_schematizes_an_explicitly_skipped_uk_holdout() -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() payload["uk_diagnostics"]["rotated_holdout"] = {"skipped": True} model = parse_calibration_diagnostics(payload) @@ -399,7 +193,7 @@ def test_schema_8_schematizes_an_explicitly_skipped_uk_holdout() -> None: def test_schema_8_rejects_incomplete_hierarchy() -> None: - payload = _payload() + payload = schema_8_payload() del payload["targets"][0]["hierarchy"] with pytest.raises(ValidationError, match="hierarchy"): parse_calibration_diagnostics(payload) @@ -425,7 +219,7 @@ def test_schema_8_rejects_inconsistent_hierarchy_identity( value: str, message: str, ) -> None: - payload = _payload() + payload = schema_8_payload() owner = payload for part in field_path[:-1]: owner = owner[part] @@ -444,7 +238,7 @@ def test_schema_8_rejects_inconsistent_hierarchy_identity( ], ) def test_schema_8_rejects_whitespace_only_text(field_path: tuple[object, ...]) -> None: - payload = _payload() + payload = schema_8_payload() payload["targets"][0]["hierarchy"]["dimensions"] = [ {"id": "sex", "label": "Sex", "value_id": "female", "value_label": "Female"} ] @@ -465,7 +259,7 @@ def test_schema_8_requires_positive_declared_target_counts( section: str, field: str, ) -> None: - payload = _payload() + payload = schema_8_payload() payload[section][field] = 0 with pytest.raises(ValidationError, match="greater than 0"): @@ -473,7 +267,7 @@ def test_schema_8_requires_positive_declared_target_counts( def test_schema_8_reconciles_target_surface_matrix_shape() -> None: - payload = _payload() + payload = schema_8_payload() payload["target_surface"]["constraint_matrix"]["rows"] = 2 with pytest.raises(ValidationError, match="rows must equal n_targets"): @@ -501,7 +295,9 @@ def test_schema_8_rejects_non_finite_numbers_recursively( location: str, value: float, ) -> None: - payload = _uk_payload() if location == "UK extension" else _payload() + payload = ( + uk_schema_8_payload() if location == "UK extension" else schema_8_payload() + ) if location == "top-level scalar": payload["final_loss"] = value elif location == "target scalar": @@ -524,7 +320,7 @@ def test_schema_8_rejects_non_finite_numbers_recursively( def test_writer_strict_serialization_rejects_a_non_finite_mutated_mapping( tmp_path: Path, ) -> None: - payload = _payload() + payload = schema_8_payload() payload["build"] = {"nested": {"value": 1.0}} model = CalibrationDiagnosticsV8.model_validate(payload) assert model.build is not None @@ -546,7 +342,7 @@ def test_writer_is_atomic_and_removes_stale_output_on_failure( ) -> None: path = tmp_path / "calibration_diagnostics.json" path.write_text("stale") - model = CalibrationDiagnosticsV8.model_validate(_payload()) + model = CalibrationDiagnosticsV8.model_validate(schema_8_payload()) def fail_write(self: Path, data: bytes) -> int: raise OSError("synthetic write failure") @@ -565,7 +361,7 @@ def test_unexpected_writer_failure_does_not_escape( ) -> None: path = tmp_path / "calibration_diagnostics.json" path.write_text("stale") - model = CalibrationDiagnosticsV8.model_validate(_payload()) + model = CalibrationDiagnosticsV8.model_validate(schema_8_payload()) def fail_write(self: Path, data: bytes) -> int: raise RuntimeError("synthetic unexpected failure") @@ -579,7 +375,7 @@ def fail_write(self: Path, data: bytes) -> int: def test_writer_round_trips_valid_output(tmp_path: Path) -> None: - model = CalibrationDiagnosticsV8.model_validate(_payload()) + model = CalibrationDiagnosticsV8.model_validate(schema_8_payload()) path = tmp_path / "calibration_diagnostics.json" outcome = write_calibration_diagnostics(model, path) @@ -595,7 +391,7 @@ def test_writer_round_trips_valid_output(tmp_path: Path) -> None: ], ) def test_uk_consumption_drift_values_reconcile(change: dict, match: str) -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() payload["uk_diagnostics"]["consumption_drift"]["totals"]["petrol_spending"].update( change ) diff --git a/test_support/microcosm_diagnostics/__init__.py b/test_support/microcosm_diagnostics/__init__.py new file mode 100644 index 000000000..85e1acbbb --- /dev/null +++ b/test_support/microcosm_diagnostics/__init__.py @@ -0,0 +1 @@ +"""Shared support for microcosm-diagnostics tests.""" diff --git a/test_support/microcosm_diagnostics/json_schema.py b/test_support/microcosm_diagnostics/json_schema.py new file mode 100644 index 000000000..307dd193c --- /dev/null +++ b/test_support/microcosm_diagnostics/json_schema.py @@ -0,0 +1,48 @@ +"""Check calibration-diagnostics documents the way a JSON Schema intake would.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Iterator, Mapping + +from jsonschema import Draft202012Validator, ValidationError + +from microcosm import diagnostics + + +def _refuse_non_finite_literal(literal: str) -> float: + raise ValueError(f"strict JSON has no {literal} literal") + + +def load_strict_json(text: str | bytes) -> object: + """Decode RFC 8259 JSON, refusing the NaN and Infinity tokens Python accepts.""" + + return json.loads(text, parse_constant=_refuse_non_finite_literal) + + +def _leaf_errors(errors: Iterable[ValidationError]) -> Iterator[ValidationError]: + """Descend into ``anyOf`` branches so each error names its innermost path.""" + + for error in errors: + if error.context: + yield from _leaf_errors(error.context) + else: + yield error + + +def json_schema_errors(document: object, schema: Mapping[str, object]) -> list[str]: + """Return every ``: `` error, sorted for stable output.""" + + validator = Draft202012Validator(schema) + return sorted( + f"{error.json_path}: {error.message}" + for error in _leaf_errors(validator.iter_errors(document)) + ) + + +def published_json_schema_errors(document: object) -> list[str]: + """Return the errors against the committed schema-8 file consumers read.""" + + return json_schema_errors( + document, diagnostics.load_calibration_diagnostics_v8_json_schema() + ) diff --git a/test_support/microcosm_diagnostics/payloads.py b/test_support/microcosm_diagnostics/payloads.py new file mode 100644 index 000000000..30b85fbbe --- /dev/null +++ b/test_support/microcosm_diagnostics/payloads.py @@ -0,0 +1,223 @@ +"""Valid calibration-diagnostics schema-8 documents shared by tests. + +Each call returns a fresh, fully reconciled document, so a test can mutate one +field and know that any refusal comes from that change. +""" + +from __future__ import annotations + + +def schema_8_payload() -> dict: + """A minimal US-registry document with one target and no optional blocks.""" + + digest = "a" * 64 + return { + "schema_version": 8, + "weight_entity": "household", + "options": {"epochs": 1}, + "target_surface": { + "schema_version": 1, + "weight_entity": "household", + "n_targets": 1, + "n_records": 2, + "constraint_matrix": {"rows": 1, "columns": 2, "nnz": 2}, + "sha256": digest, + "names_sha256": digest, + "values_sha256": digest, + }, + "target_registry": {"country": "us", "version": digest, "n_specs": 1}, + "l0_lambda": 0.0, + "n_nonzero": 2, + "n_records": 2, + "initial_loss": 0.1, + "final_loss": 0.01, + "fraction_within_10pct": 1.0, + "effective_sample_size": 2.0, + "realized_max_weight_ratio": 1.0, + "top_1pct_weight_share": 0.5, + "loss_trajectory": [0.1], + "skipped": [], + "past_cap_census": None, + "diagnostic_warnings": [], + "targets": [ + { + "name": "population@2024", + "target_name": "population", + "period": 2024, + "entity": "household", + "measure": {"kind": "column", "name": "household_count"}, + "filter": None, + "source": "Census fixture", + "metadata": {}, + "target": 2.0, + "compiled_target": 2.0, + "initial_estimate": 1.8, + "final_estimate": 2.0, + "relative_error": 0.0, + "within_tolerance": None, + "hierarchy": { + "provider": {"id": "census", "label": "Census"}, + "category": { + "id": "census.population", + "label": "Population", + "provider_id": "census", + }, + "geography": { + "id": "0100000US", + "label": "United States", + "level": "country", + }, + "dimensions": [], + "target": {"id": "population", "label": "Population"}, + }, + } + ], + } + + +def uk_schema_8_payload() -> dict: + """The minimal document on a UK registry, carrying every UK extension block.""" + + payload = schema_8_payload() + payload["target_registry"]["country"] = "uk" + payload["n_nonzero"] = 1 + payload["effective_sample_size"] = 1.0 + payload["top_1pct_weight_share"] = 1.0 + payload["uk_diagnostics"] = { + "schema_version": 1, + "weights": { + "n_records": 2, + "positive_weight_records": 1, + "zero_weight_records": 1, + "total_weight": 1.0, + "effective_sample_size": 1.0, + "ess_fraction": 0.5, + "median_positive_weight": 1.0, + "max_weight": 1.0, + "max_to_median_positive_weight": 1.0, + "top_1pct_weight_share": 1.0, + }, + "zero_weight_rows_by_stratum": [ + { + "stratum": {"source": "frs"}, + "rows": 2, + "positive_weight_rows": 1, + "zero_weight_rows": 1, + "weight_sum": 1.0, + } + ], + "target_pass_rates_by_geography_level": [ + { + "geography_level": level, + "n_targets": 1 if level == "national" else 0, + "n_scored": 1 if level == "national" else 0, + "n_skipped": 0, + "n_within_10pct": 1 if level == "national" else 0, + "pass_rate": 1.0 if level == "national" else None, + } + for level in ( + "national", + "region", + "country", + "local_authority", + "constituency", + ) + ], + "target_observation_basis": {"population@2024": "annual_stock"}, + "weakest_families": [ + { + "family": "population", + "n_targets": 1, + "n_within_10pct": 1, + "pass_rate": 1.0, + "worst_target": "population@2024", + "worst_abs_relative_error": 0.0, + "loss_contribution": 0.0, + "loss_share": 0.0, + } + ], + "weakest_areas_by_fit": { + "limit": 15, + "n_areas_scored": 1, + "bottom_by_fit": [ + { + "geography_level": "constituency", + "area_code": "E14000001", + "country": "England", + "n_targets": 1, + "n_within_10pct": 1, + "pass_rate": 1.0, + "worst_target": "population@2024", + "worst_abs_relative_error": 0.0, + "loss_contribution": 0.0, + "support_below_floor": True, + } + ], + "countries": [ + { + "country": "England", + "geography_level": "constituency", + "n_areas": 1, + "n_targets": 1, + "n_within_10pct": 1, + "pass_rate": 1.0, + "worst_target": "population@2024", + "worst_abs_relative_error": 0.0, + "loss_contribution": 0.0, + } + ], + }, + "rotated_holdout": { + "report_only": True, + "method": "rotated_folds", + "target_loss_cap": 10.0, + "loss_weight_scale": "held_local_grains_only", + "target_weight_rule": "grain_equal", + "population": "held_out_local_targets", + "grains": ["constituency", "local_authority"], + "n_folds": 2, + "seed": 20260529, + "solve_seed": 7, + "mean_holdout_loss": 0.15, + "worst_holdout_loss": 0.2, + "fold_losses": [0.1, 0.2], + "folds": [ + { + "fold": 0, + "n_train_targets": 1, + "n_holdout_targets": 1, + "holdout_target_indices": [0], + "training_national_rows": 1, + "holdout_loss": 0.1, + }, + { + "fold": 1, + "n_train_targets": 1, + "n_holdout_targets": 1, + "holdout_target_indices": [1], + "training_national_rows": 1, + "holdout_loss": 0.2, + }, + ], + }, + "consumption_drift": { + "report_only": True, + "totals": { + "petrol_spending": { + "design": 100.0, + "final": 102.0, + "change": 2.0, + "relative_change": 0.02, + } + }, + "shares": { + "fuel_car_households_without_fuel": { + "design": 0.0, + "final": 0.0, + "change": 0.0, + "relative_change": None, + } + }, + }, + } + return payload diff --git a/tools/generate_calibration_diagnostics_json_schema.py b/tools/generate_calibration_diagnostics_json_schema.py new file mode 100644 index 000000000..cd44c9ef7 --- /dev/null +++ b/tools/generate_calibration_diagnostics_json_schema.py @@ -0,0 +1,43 @@ +"""Regenerate or check the committed calibration-diagnostics schema-8 JSON Schema.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from microcosm.diagnostics import ( + assert_calibration_diagnostics_v8_json_schema_current, + committed_calibration_diagnostics_v8_json_schema_path, + write_calibration_diagnostics_v8_json_schema, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--out", + type=Path, + help="write or check this file instead of the committed schema", + ) + parser.add_argument( + "--check", + action="store_true", + help="exit 1 unless the file equals freshly generated output", + ) + args = parser.parse_args(argv) + path = args.out or committed_calibration_diagnostics_v8_json_schema_path() + if args.check: + try: + assert_calibration_diagnostics_v8_json_schema_current(path) + except (OSError, ValueError) as error: + print(f"stale: {error}", file=sys.stderr) + return 1 + print(f"current: {path}") + return 0 + print(write_calibration_diagnostics_v8_json_schema(path)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/uv.lock b/uv.lock index dc7cbbc95..1a6ffe763 100644 --- a/uv.lock +++ b/uv.lock @@ -954,6 +954,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "jsonschema" }, { name = "pytest" }, ] @@ -961,7 +962,10 @@ dev = [ requires-dist = [{ name = "pydantic", specifier = ">=2.10,<3" }] [package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=8" }] +dev = [ + { name = "jsonschema", specifier = ">=4.23,<5" }, + { name = "pytest", specifier = ">=8" }, +] [[package]] name = "microcosm-emitter"