diff --git a/changelog.d/551.changed.md b/changelog.d/551.changed.md new file mode 100644 index 00000000..80b935fa --- /dev/null +++ b/changelog.d/551.changed.md @@ -0,0 +1 @@ +Use the ACS-local Populace dataset by default for US state and congressional-district simulations while retaining the sparse Populace dataset as the national default. diff --git a/docs/bundles.md b/docs/bundles.md index 4c5b642b..dc06c6ca 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -143,14 +143,20 @@ python scripts/bundle.py certify-data \ --country us \ --data-producer populace \ --manifest-uri hf://dataset/policyengine/populace-us@/releases//release_manifest.json \ + --regional-manifest-uri hf://dataset/policyengine/populace-us@/releases//release_manifest.json \ --model-version ``` -US state and congressional-district regions scope the certified national -Populace dataset with row filters. If a Populace release also publishes derived -`states/*.h5` or `districts/*.h5` area slices, the bundle certification omits -those slices from `data_releases.us.datasets`; they are not runtime dataset -dependencies. +US state and congressional-district regions scope one certified ACS-local +Populace dataset with row filters. Pass its non-default local-area release +manifest through `--regional-manifest-uri`; certification adds the pinned H5 +artifact to `data_releases.us.datasets` and writes both regional path templates +under `data_releases.us.region_datasets`. If a legacy release publishes derived +`states/*.h5` or `districts/*.h5` slices, certification still omits them. + +`policyengine bundle install` continues to materialize only each country's +certified national `default_dataset`. Regional runtimes materialize the +additional certified dataset when the selected region declares it. Use `python scripts/bundle.py generate` to regenerate derived bundle metadata, and `python scripts/bundle.py generate --include-tros` when TRACE TRO sidecars diff --git a/docs/engineering/runbooks/build-m-us-populace-certification.md b/docs/engineering/runbooks/build-m-us-populace-certification.md index dd0d628f..b63f919b 100644 --- a/docs/engineering/runbooks/build-m-us-populace-certification.md +++ b/docs/engineering/runbooks/build-m-us-populace-certification.md @@ -1,112 +1,108 @@ -# Build M US Populace certification runbook +# US Populace certification runbook -Fill-in-the-id runbook for certifying the next US Populace default -(`sparse-rmloss100` lineage: Build I → J → **M**) into the -`policyengine.py` bundle. It replays the exact steps that landed -[#470](https://github.com/PolicyEngine/policyengine.py/pull/470) (the Build J -certification), parameterized for the Build M release id. Read the -[data certification](../skills/data-certification.md) skill first for the -validation semantics; this file is the concrete checklist. +Use this runbook to certify a new national US Populace release together with +the shared ACS-local release used for state and congressional-district +simulations. Read the [data certification](../skills/data-certification.md) +skill first for validation semantics. + +This historical filename is retained so existing links remain valid; the +procedure itself is release-independent. ## When to use -A new US Populace `sparse-rmloss100` release has been published to -`policyengine/populace-us` and is ready to become the certified default. You -have the release id, the release manifest is reachable, and the model version -it was built with is known. +Both release manifests have been published to `policyengine/populace-us`, and +the model version used by the national release is known. The national release +will remain the default dataset; the local-area release will remain +non-default and be selected only for supported regional runs. ## Prerequisites -- A clean worktree branched from current `origin/main` - (`git fetch origin && git checkout -b certify-us-buildm origin/main`). +- A clean worktree branched from current `origin/main`. - Network access: certification fetches the release manifest and PyPI wheel metadata and runs reachability `HEAD` checks against Hugging Face. - `HUGGING_FACE_TOKEN` (or `HF_TOKEN`) exported — required to regenerate the UK TRO in the `--include-tros` step and to run the UK data-release fetch in the test suite. The US populace repo is public. -## Fill in these three values +## Step 1 — identify the releases + +Set values from the two published release manifests rather than copying an +older certification: ``` -BUILD_M_RELEASE_ID = populace-us-2024-buildm-sparse-rmloss100--Z -MODEL_VERSION = 1.764.6 # policyengine-us Build M was built with -CURRENT_RELEASE_ID = populace-us-2024-buildj-sparse-rmloss100-75d5add-20260710T094201Z +NATIONAL_RELEASE_ID = populace-us-2024---Z +LOCAL_AREA_RELEASE_ID = populace-us-2024---Z +MODEL_VERSION = ``` -`CURRENT_RELEASE_ID` is the outgoing default (Build J) — the string you are -replacing in the pinned test constants. `MODEL_VERSION` stays `1.764.6` unless -Build M was built against a newer `policyengine-us`; if it was, see step 4. +The local-area manifest must declare `dataset_role: non_default_local_area`, +`is_default: false`, no default datasets, and exactly one pinned H5 microdata +artifact. -## Step 1 — certify the release +## Step 2 — certify both releases ```bash -python scripts/certify_data_release.py \ +python scripts/bundle.py certify-data \ --country us \ --data-producer populace \ --model-version "$MODEL_VERSION" \ - --manifest-uri "hf://dataset/policyengine/populace-us@$BUILD_M_RELEASE_ID/releases/$BUILD_M_RELEASE_ID/release_manifest.json" + --manifest-uri "hf://dataset/policyengine/populace-us@$NATIONAL_RELEASE_ID/releases/$NATIONAL_RELEASE_ID/release_manifest.json" \ + --regional-manifest-uri "hf://dataset/policyengine/populace-us@$LOCAL_AREA_RELEASE_ID/releases/$LOCAL_AREA_RELEASE_ID/release_manifest.json" ``` This one command: - rewrites `data_releases.us` in `src/policyengine/data/bundle/manifest.json` - from the Build M release manifest (default dataset, per-artifact + from the national release manifest (default dataset, per-artifact repo/revision/sha256 pins, certified artifact, certification block); - runs `generate(check=False)`, which re-normalizes `manifest.json` and updates `pyproject.toml` **only if the model pins moved**; - writes the changelog fragment - `changelog.d/certify-us-$BUILD_M_RELEASE_ID.changed.md`. + `changelog.d/certify-us-$NATIONAL_RELEASE_ID.changed.md`. -## Step 2 — regenerate the US TRO sidecar +## Step 3 — regenerate TRACE sidecars -The certify step does not touch TRO sidecars. Rebind them so -`src/policyengine/data/bundle/us.trace.tro.jsonld` records the new -`manifest.json` sha256 (the `bundle_manifest` artifact hash is asserted by -`tests/test_certify_data_release.py::TestVendoredSidecarBinding`): +The certify step does not touch TRACE sidecars. Regenerate them so they record +the new bundle-manifest SHA-256: ```bash HUGGING_FACE_TOKEN="$HUGGING_FACE_TOKEN" python scripts/bundle.py generate --include-tros ``` -Only `us.trace.tro.jsonld` should change (UK regenerates identically because -only US was certified). If UK cannot be reached, the run writes a *limited* UK -TRO — do not commit a degraded `uk.trace.tro.jsonld`; `git checkout` it and -rerun with a valid token. +Both country sidecars contain the bundle-manifest hash, so both may change when +the bundle changes. If UK cannot be reached, the run writes a *limited* UK +sidecar; do not commit that degraded output. -## Step 3 — update the pinned test constants +## Step 4 — update pinned expectations -Three files hard-code the certified release id. Replace `CURRENT_RELEASE_ID` -with `BUILD_M_RELEASE_ID` in each (this is the whole of #470's test diff): +Update tests that intentionally pin the outgoing national release, local-area +release, model version, or artifact hashes. Derive every replacement from the +new manifests; do not copy identifiers from this runbook. -- `tests/test_release_manifests.py` — `US_DATA_RELEASE_ID`. -- `tests/test_models.py` — the `us_latest.default_dataset_uri` `@` - assertion. -- `tests/test_us_regions.py` — the national `dataset_path` `@` assertion. +If the model version changed, refresh the household snapshots with: -## Step 4 — only if the model version changed - -Build J → M on the same `policyengine-us` needs nothing here (this is the -common case; #470 left `pyproject.toml` and the snapshots untouched). If Build -M was built against a newer `policyengine-us`: +```bash +PE_UPDATE_SNAPSHOTS=1 pytest tests/test_household_calculator_snapshot.py +``` -- `pyproject.toml` model pins are already rewritten by step 1's `generate`; - commit them. -- Update `US_MODEL_VERSION` and `US_BUILT_WITH_MODEL_VERSION` in - `tests/test_release_manifests.py`. -- Refresh the household snapshots: - `PE_UPDATE_SNAPSHOTS=1 pytest tests/test_household_calculator_snapshot.py` - and commit `tests/fixtures/household_calculator_snapshots/`. +## Step 5 — certify and verify the local-area release -## Step 5 — verify the local-area overlay survived +The certification command passes the immutable ACS-local release manifest with +`--regional-manifest-uri`. Confirm all of the following: -Certification rewrites only `data_releases.us`, so the non-default -`dataset_overlays.us.populace_us_2024_acs_local` entry (see -[Non-default dataset overlays](../../release-bundles.md#non-default-dataset-overlays)) -must still be present and resolvable. No manual re-add is needed — confirm it: +- `default_dataset` remains the national Populace dataset; +- the local-area artifact appears in `data_releases.us.datasets` with its + repository type, immutable revision, and SHA-256; +- `region_datasets.national` selects the national artifact; +- `region_datasets.state` and `region_datasets.congressional_district` select + the shared local-area artifact; +- no derived `states/*.h5` or `districts/*.h5` artifacts are runtime inputs. ```bash -python -c "import json; b=json.load(open('src/policyengine/data/bundle/manifest.json')); assert 'populace_us_2024_acs_local' in b['dataset_overlays']['us'], 'overlay lost'; print('overlay preserved')" -pytest tests/test_release_manifests.py -k "local_area or DatasetOverlays" -q +pytest tests/test_certify_data_release.py \ + tests/test_release_manifests.py \ + tests/test_us_regions.py \ + -q ``` ## Step 6 — check, format, lint, test @@ -118,17 +114,12 @@ make lint make test # needs HUGGING_FACE_TOKEN for UK ``` -## Step 7 — commit exactly these files - -Matches #470's file set (add `pyproject.toml` and the snapshot dir only when -step 4 applied): +## Step 7 — inspect and commit the generated changes -- `src/policyengine/data/bundle/manifest.json` -- `src/policyengine/data/bundle/us.trace.tro.jsonld` -- `changelog.d/certify-us-$BUILD_M_RELEASE_ID.changed.md` -- `tests/test_release_manifests.py` -- `tests/test_models.py` -- `tests/test_us_regions.py` +The expected diff normally includes the bundle manifest, TRACE sidecars, one +Towncrier fragment, and pinned test expectations. Include package pins and +snapshots only when the certified model version changed. Investigate any other +generated difference before committing it. ## Step 8 — open the PR @@ -136,10 +127,3 @@ Follow [github-prs](../skills/github-prs.md): open/find the issue, put `Fixes #ISSUE` first, push to the canonical repo, and open a **draft** PR. The certify step already wrote the changelog fragment, so the changelog check passes. - -## Verified against #470 - -Every step above is checked against the merged Build J certification (#470), -whose diff was exactly: `manifest.json`, `us.trace.tro.jsonld`, the three test -constants, and the auto-written changelog fragment — with `pyproject.toml` and -the snapshots untouched because `policyengine-us` stayed `1.764.6`. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index f5319517..5f1d9abe 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -31,21 +31,30 @@ For US Populace certification, certify the Populace release manifest directly: ```bash python scripts/bundle.py certify-data --country us --data-producer populace \ --manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ + --regional-manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ --model-version "" ``` -US state and congressional-district regions are row filters over the certified -national Populace dataset. Certification writes: +US state and congressional-district regions are row filters over one shared, +certified ACS-local Populace dataset. Its release manifest must declare +`dataset_role: non_default_local_area`, `is_default: false`, no default +datasets, and exactly one pinned H5 microdata artifact. Certification writes: ```json "region_datasets": { - "national": {"path_template": "populace_us_2024.h5"} + "national": {"path_template": "populace_us_2024.h5"}, + "state": {"path_template": "populace_us_2024_acs_local.h5"}, + "congressional_district": {"path_template": "populace_us_2024_acs_local.h5"} } ``` +The ACS-local artifact is a second entry in `data_releases.us.datasets`; it is +not a dataset overlay and it never replaces the national `default_dataset`. + If the Populace release publishes derived `states/*.h5` or `districts/*.h5` files for compatibility checks, certification omits them from the runtime -bundle. The national H5 is the canonical `.py` dataset. +bundle. The country-wide default and ACS-local dataset are each single shared +files; region registries select one and then filter its rows. The script fetches and validates the manifest (every artifact must carry a revision pin; the certified dataset must be reachable), writes the canonical @@ -74,8 +83,9 @@ A certification PR should normally change only: Hard failures (certification refuses): missing national default dataset, default dataset absent from artifacts, any artifact without a revision pin, unreachable certified dataset, missing required supplemental release files -(for example Populace-US `us_source_coverage.json`), missing or malformed US -state overlay artifacts when `--regional-manifest-uri` is used, unknown country. +(for example Populace-US `us_source_coverage.json`), a malformed shared +local-area release or malformed legacy per-state artifacts when +`--regional-manifest-uri` is used, unknown country. Certification gate: the model version must either exactly match the build-time model (`compatibility_basis: built_with_model_package`) or be @@ -92,8 +102,9 @@ publisher-claim basis above. Concrete, fill-in-the-id runbooks that replay a specific certification live under `docs/engineering/runbooks/`. See -`runbooks/build-m-us-populace-certification.md` for the next US Populace -`sparse-rmloss100` default. +`runbooks/build-m-us-populace-certification.md` for the reusable US national +and local-area certification procedure. The historical filename is retained +so existing links remain valid. ## Legacy paths diff --git a/docs/microsim.md b/docs/microsim.md index 55820429..98cdc7bc 100644 --- a/docs/microsim.md +++ b/docs/microsim.md @@ -134,15 +134,16 @@ pe.us.load_datasets() # or pe.uk.load_datasets() ### US local-area dataset -Alongside the certified national default, the bundle registers a **non-default** -US dataset for finer geographic work: `populace_us_2024_acs_local`. It is a +Alongside the certified national default, the bundle registers a US dataset for +finer geographic work: `populace_us_2024_acs_local`. It is a Populace US 2024 build of roughly **1.6 million households** on an **ACS 2024 multispine**, with each household **PUMA-assigned** to a 119th-Congress congressional district, county, and state, and calibrated to **state administrative totals and state and congressional-district population**. Its -release gate summary records **four reviewed limitations**, so read that gate -summary before relying on it. It ships in its own immutable release and is never -selected implicitly — you load it by name. +release validation summary records **four reviewed limitations**, so read that +summary before relying on it. It ships in its own immutable release. State and +congressional-district region simulations select it through the bundle's +`region_datasets` metadata; direct microsimulations can still load it by name. Two-line load: diff --git a/docs/regions.md b/docs/regions.md index 3bf64d73..6d703c9f 100644 --- a/docs/regions.md +++ b/docs/regions.md @@ -23,9 +23,9 @@ ca_snap = Aggregate( ca_snap.run() ``` -Each state is a region in the US registry. State regions scope the certified -national Populace dataset by `state_fips`; they do not require separate state -H5 files: +Each state is a region in the US registry. State regions load the ACS-local +Populace dataset declared by `region.dataset_path`, then scope its rows by +`state_fips`. They do not require separate state H5 files: ```python states = pe.us.model.region_registry.get_by_type("state") @@ -33,8 +33,9 @@ for region in states: print(region.code, region.label, region.scoping_strategy) ``` -For state-specific simulations, pass `scoping_strategy=region.scoping_strategy` -with the certified national dataset. +For state-specific simulations, use both `region.dataset_path` and +`region.scoping_strategy`. The regional simulation API performs both steps from +the registry entry. ## US congressional districts @@ -49,7 +50,9 @@ for row in impacts.district_results: print(row["district_geoid"], row["avg_change"], row["winner_percentage"]) ``` -`district_geoid` is the SSDD integer (state FIPS × 100 + district number; at-large districts use `00`). Congressional district regions scope the certified national Populace dataset by `congressional_district_geoid`. +`district_geoid` is the SSDD integer (state FIPS × 100 + district number; +at-large districts use `00`). Congressional district regions load the same +ACS-local file and scope its rows by `congressional_district_geoid`. ## UK parliamentary constituencies diff --git a/docs/release-bundles.md b/docs/release-bundles.md index 8abd1c1b..80b6f850 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -201,31 +201,32 @@ US Populace certification uses the Populace release manifest directly: ```bash python scripts/bundle.py certify-data --country us --data-producer populace \ --manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ + --regional-manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ --model-version "" ``` That produces one US bundle manifest entry containing the Populace national -default dataset. State and congressional-district regions are runtime row -filters over that national dataset, so derived `states/*.h5` or -`districts/*.h5` files are not vendored into `data_releases.us.datasets`. +default dataset and the certified ACS-local regional dataset. State and +congressional-district simulations use the latter with runtime row filters, so +derived `states/*.h5` or `districts/*.h5` files are not vendored into +`data_releases.us.datasets`. ### Non-default dataset overlays `certify_data_release` rewrites `data_releases.{country}` wholesale from the -certified release manifest, so that block only ever holds datasets from the -certified release. A staged artifact published in its own release — one that is -deliberately not the certified default — is registered instead under the +certified release manifest or manifests. A staged artifact that is not part of +that certification and is deliberately not a default is registered under the sibling `dataset_overlays.{country}` map: ```json "dataset_overlays": { - "us": { - "populace_us_2024_acs_local": { + "uk": { + "populace_uk_2023": { "data_package_name": "populace-data", - "path": "populace_us_2024_acs_local.h5", - "repo_id": "policyengine/populace-us", + "path": "populace_uk_2023.h5", + "repo_id": "policyengine/populace-uk-private", "repo_type": "dataset", - "revision": "populace-us-2024-buildo-acs-local-...", + "revision": "populace-uk-2023-...", "sha256": "..." } } @@ -237,8 +238,15 @@ sibling `dataset_overlays.{country}` map: their own pinned, sha-verified revision. Overlays are strictly additive: an overlay may not shadow the certified default or any certified dataset, so default resolution is untouched. Because certification only rewrites -`data_releases`, overlays survive re-certification without any manual -re-add step. +`data_releases`, overlays survive re-certification without any manual re-add +step. + +Datasets used automatically for a region type are not overlays. They are +certified into `data_releases.{country}.datasets`, and +`data_releases.{country}.region_datasets` maps each region type to its artifact +path. The national `default_dataset` remains unchanged. Ordinary bundle +installation downloads only that national default; a regional runtime is +responsible for materializing the additional certified dataset. Cross-package overlays must declare `data_package_name` and `repo_type` explicitly. Ordinary artifacts inherit these values from the country release's diff --git a/scripts/bundle.py b/scripts/bundle.py index 4c535e21..121ab8e2 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -202,11 +202,11 @@ def _parser() -> argparse.ArgumentParser: ) certify.add_argument( "--regional-artifact-prefix", - help="Regional artifact prefix to import. Defaults to states/.", + help="Legacy per-state artifact prefix to import. Defaults to states/.", ) certify.add_argument( "--regional-path-template", - help="Region dataset path template to certify.", + help="Legacy per-state dataset path template to certify.", ) certify.add_argument( "--no-generate", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index c4c1ec54..8842adbb 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -60,12 +60,12 @@ def main(argv=None) -> int: parser.add_argument( "--regional-artifact-prefix", default="states/", - help="Regional artifact path prefix to import. Defaults to states/.", + help="Legacy per-state artifact path prefix to import. Defaults to states/.", ) parser.add_argument( "--regional-path-template", default="states/{state_code}.h5", - help="Region dataset path template to certify into the bundle manifest.", + help="Legacy per-state dataset path template to certify.", ) parser.add_argument( "--model-version", diff --git a/src/policyengine/core/region.py b/src/policyengine/core/region.py index d5d177a6..20be0f2e 100644 --- a/src/policyengine/core/region.py +++ b/src/policyengine/core/region.py @@ -1,10 +1,10 @@ """Region definitions for geographic simulations. This module provides the Region and RegionRegistry classes for defining -geographic regions that a tax-benefit model supports. Regions can have: -1. A dedicated dataset, usually for the national default. -2. A scoping strategy that derives the region from a parent dataset - (row filter or weight replacement). +geographic regions that a tax-benefit model supports. Regions can select an +input dataset, apply a scoping strategy to a selected dataset, or do both. For +example, US states and congressional districts select the shared ACS-local +dataset and then filter its rows. """ from typing import Literal, Optional, Union @@ -22,10 +22,8 @@ class Region(BaseModel): """Geographic region for tax-benefit simulations. - Regions can either have: - 1. A dedicated dataset (``dataset_path`` is set). - 2. A scoping strategy that derives the region from a parent dataset - (``scoping_strategy`` is set). + ``dataset_path`` selects the input dataset. ``scoping_strategy`` optionally + scopes that dataset to the requested geography. A region may declare both. The unique identifier is the code field, which uses a prefixed format: - National: "us", "uk" @@ -56,7 +54,7 @@ class Region(BaseModel): # Dataset configuration dataset_path: Optional[str] = Field( default=None, - description="URI to a dedicated dataset when the region has one.", + description="URI to the input dataset selected before regional scoping.", ) # Scoping strategy for regions that derive from a parent dataset @@ -173,11 +171,11 @@ def get_children(self, parent_code: str) -> list[Region]: return [r for r in self.regions if r.parent_code == parent_code] def get_dataset_regions(self) -> list[Region]: - """Get all regions that have a dedicated dataset on disk.""" + """Get all regions that select an input dataset.""" return [r for r in self.regions if r.dataset_path is not None] def get_filter_regions(self) -> list[Region]: - """Get all regions that derive from a parent dataset via a scoping strategy.""" + """Get all regions that apply a scoping strategy to their input dataset.""" return [r for r in self.regions if r.scoping_strategy is not None] def __len__(self) -> int: diff --git a/src/policyengine/countries/us/regions.py b/src/policyengine/countries/us/regions.py index 266fb5c4..7f6ceb87 100644 --- a/src/policyengine/countries/us/regions.py +++ b/src/policyengine/countries/us/regions.py @@ -43,7 +43,7 @@ def build_us_region_registry() -> RegionRegistry: ) ) - # 2. State regions (filtered from the certified national dataset) + # 2. State regions (filtered from the configured state dataset) for abbrev, name in US_STATES.items(): regions.append( Region( @@ -51,6 +51,11 @@ def build_us_region_registry() -> RegionRegistry: label=name, region_type="state", parent_code="us", + dataset_path=resolve_region_dataset_path( + "us", + "state", + state_code=abbrev, + ), scoping_strategy=RowFilterStrategy( variable_name="state_fips", variable_value=US_STATE_FIPS[abbrev], @@ -60,7 +65,7 @@ def build_us_region_registry() -> RegionRegistry: ) ) - # 3. Congressional district regions (filtered from the national dataset) + # 3. Congressional districts (filtered from the configured district dataset) for state_abbrev, count in DISTRICT_COUNTS.items(): state_name = US_STATES[state_abbrev] state_fips = US_STATE_FIPS[state_abbrev] @@ -81,6 +86,11 @@ def build_us_region_registry() -> RegionRegistry: label=label, region_type="congressional_district", parent_code=f"state/{state_abbrev.lower()}", + dataset_path=resolve_region_dataset_path( + "us", + "congressional_district", + district_code=district_code, + ), scoping_strategy=RowFilterStrategy( variable_name="congressional_district_geoid", variable_value=district_geoid, diff --git a/src/policyengine/data/bundle/manifest.json b/src/policyengine/data/bundle/manifest.json index 2c67fafe..723705b3 100644 --- a/src/policyengine/data/bundle/manifest.json +++ b/src/policyengine/data/bundle/manifest.json @@ -171,6 +171,13 @@ "revision": "populace-us-2024-spm-20260915", "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" }, + "populace_us_2024_acs_local": { + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "revision": "populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z", + "sha256": "769756c31f3ca646d12c272511744dec04c0e68870c6946dd945fbba65b6a7ec" + }, "source_enrichment": { "path": "releases/populace-us-2024-spm-20260915/source_enrichment.json", "repo_id": "policyengine/populace-us", @@ -212,10 +219,18 @@ }, "policyengine_version": "6.2.1", "region_datasets": { + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, "national": { "path_template": "populace_us_2024.h5" + }, + "state": { + "path_template": "populace_us_2024_acs_local.h5" } }, + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z/releases/populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z/releases/populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z/release_manifest.json", "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", "schema_version": 1, "source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", @@ -248,16 +263,6 @@ "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" } - }, - "us": { - "populace_us_2024_acs_local": { - "data_package_name": "populace-data", - "path": "populace_us_2024_acs_local.h5", - "repo_id": "policyengine/populace-us", - "repo_type": "dataset", - "revision": "populace-us-2024-buildo-acs-local-77e2061-20260724T110908Z", - "sha256": "71763290ded993789af0ad818fd833a6aabb9ca7ff7d14f78315340d8617c6f4" - } } }, "extras": { diff --git a/src/policyengine/data/bundle/uk.trace.tro.jsonld b/src/policyengine/data/bundle/uk.trace.tro.jsonld index 81179410..a6036246 100644 --- a/src/policyengine/data/bundle/uk.trace.tro.jsonld +++ b/src/policyengine/data/bundle/uk.trace.tro.jsonld @@ -75,7 +75,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for uk", "trov:mimeType": "application/json", - "trov:sha256": "0b80f402511f5cec0cfbe018b7f8b42e83b54a7bdfc5c9c210a2f77bb63bd21b" + "trov:sha256": "83f4217fb6c917bec0fb1a0e3f31581f68f730d7cb697065d2c49a39725f7606" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -102,7 +102,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "a318203a1ff8cf141dfe8b138902e445ae1ab7bcce6e3d714e87661a4bb6deaa" + "trov:sha256": "d07b591e5233a5b4c0d8029aac8562df60efcf04e4534d10e8fa6e49c6417f42" } }, "trov:hasPerformance": { diff --git a/src/policyengine/data/bundle/us.trace.tro.jsonld b/src/policyengine/data/bundle/us.trace.tro.jsonld index 213e1300..eefa6c02 100644 --- a/src/policyengine/data/bundle/us.trace.tro.jsonld +++ b/src/policyengine/data/bundle/us.trace.tro.jsonld @@ -74,7 +74,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "0b80f402511f5cec0cfbe018b7f8b42e83b54a7bdfc5c9c210a2f77bb63bd21b" + "trov:sha256": "83f4217fb6c917bec0fb1a0e3f31581f68f730d7cb697065d2c49a39725f7606" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -101,7 +101,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "7f7f19f41f2cfd707098768565562f6509b454b48c79bcd0558dab061b5b40a6" + "trov:sha256": "3534b9ac087d3a69265d9cce82b073df648fa6946cb99df427ec7bd72b045fd5" } }, "trov:hasPerformance": { diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 19d08ff0..3b4b27f1 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -42,7 +42,7 @@ import warnings from dataclasses import dataclass, field from pathlib import Path -from typing import Optional +from typing import Literal, Optional import requests @@ -472,6 +472,115 @@ def merge_us_state_release_manifest( return DataReleaseManifest.model_validate(primary_payload) +def merge_us_local_area_release_manifest( + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, + *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, +) -> DataReleaseManifest: + """Add one certified local-area dataset to a primary US release. + + The supplemental release remains non-default nationally. State and + congressional-district simulations load the shared artifact and apply the + existing row-filtering strategy at runtime. + """ + + if regional_manifest.dataset_role != "non_default_local_area": + raise CertificationError( + "US local-area release must declare dataset_role='non_default_local_area'." + ) + if regional_manifest.is_default is not False: + raise CertificationError("US local-area release must declare is_default=false.") + if regional_manifest.default_datasets: + raise CertificationError( + "US local-area release must not declare default datasets." + ) + + local_area_artifacts = [ + (name, artifact) + for name, artifact in regional_manifest.artifacts.items() + if artifact.kind == "microdata" + ] + if len(local_area_artifacts) != 1: + raise CertificationError( + "US local-area release must contain exactly one microdata artifact." + ) + artifact_name, artifact = local_area_artifacts[0] + if not artifact.path.endswith(".h5"): + raise CertificationError("US local-area artifact must be an H5 file.") + if not artifact.repo_id or not artifact.revision or not artifact.sha256: + raise CertificationError( + "US local-area artifact must declare repo_id, revision, and sha256." + ) + + primary_payload = primary_manifest.model_dump(mode="json", exclude_none=True) + merged_artifacts = primary_payload.setdefault("artifacts", {}) + if artifact_name in merged_artifacts: + raise CertificationError( + f"Regional artifact {artifact_name!r} conflicts with the primary manifest." + ) + if any(item.get("path") == artifact.path for item in merged_artifacts.values()): + raise CertificationError( + f"Regional artifact path {artifact.path!r} conflicts with the primary manifest." + ) + artifact_payload = artifact.model_dump( + mode="json", + exclude_none=True, + ) + if regional_repo_type is not None: + artifact_payload["repo_type"] = regional_repo_type + merged_artifacts[artifact_name] = artifact_payload + + metadata = primary_payload.setdefault("metadata", {}) + region_datasets = metadata.setdefault("region_datasets", {}) + default_dataset = primary_manifest.default_datasets["national"] + default_artifact = primary_manifest.artifacts[default_dataset] + region_datasets.setdefault( + "national", + {"path_template": default_artifact.path}, + ) + regional_template = {"path_template": artifact.path} + for region_type in ("state", "congressional_district"): + existing = region_datasets.get(region_type) + if existing is not None and existing != regional_template: + raise CertificationError( + f"US local-area release conflicts with the {region_type!r} " + "dataset template in the primary manifest." + ) + region_datasets[region_type] = regional_template + + return DataReleaseManifest.model_validate(primary_payload) + + +def merge_us_regional_release_manifest( + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, + *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, + artifact_prefix: str = "states/", + path_template: str = "states/{state_code}.h5", +) -> DataReleaseManifest: + """Merge a typed shared local-area release or a legacy per-state release.""" + + if regional_manifest.dataset_role == "non_default_local_area": + return merge_us_local_area_release_manifest( + primary_manifest, + regional_manifest, + regional_repo_type=regional_repo_type, + ) + if regional_manifest.dataset_role is not None: + raise CertificationError( + "Unsupported US regional release dataset_role " + f"{regional_manifest.dataset_role!r}." + ) + return merge_us_state_release_manifest( + primary_manifest, + regional_manifest, + artifact_prefix=artifact_prefix, + path_template=path_template, + ) + + def build_country_manifest_payload( *, country: str, @@ -512,20 +621,26 @@ def build_country_manifest_payload( payload["sha256"] = artifact.sha256 if artifact.repo_id: payload["repo_id"] = artifact.repo_id + if artifact.repo_type: + payload["repo_type"] = artifact.repo_type datasets[name] = payload region_datasets = {} raw_regions = manifest.metadata.get("region_datasets") if isinstance(raw_regions, dict): for region, template in sorted(raw_regions.items()): + if not isinstance(template, dict) or "path_template" not in template: + continue + path_template = template.get("path_template") if ( country == "us" and manifest.data_package.name in POPULACE_DATA_PACKAGES and region in {"state", "congressional_district"} + and isinstance(path_template, str) + and "{" in path_template ): continue - if isinstance(template, dict) and "path_template" in template: - region_datasets[region] = {"path_template": template["path_template"]} + region_datasets[region] = {"path_template": template["path_template"]} region_datasets.setdefault( "national", {"path_template": default_artifact.path}, @@ -681,15 +796,16 @@ def certify( if regional_manifest_uri is not None: if country != "us": raise CertificationError( - "Regional data release overlays are only supported for US " + "Regional data release manifests are only supported for US " "Populace certification." ) - state_manifest, _, regional_uri_parts = fetch_release_manifest( + regional_manifest, _, regional_uri_parts = fetch_release_manifest( regional_manifest_uri, token=token ) - manifest = merge_us_state_release_manifest( + manifest = merge_us_regional_release_manifest( manifest, - state_manifest, + regional_manifest, + regional_repo_type=regional_uri_parts["repo_type"], artifact_prefix=regional_artifact_prefix, path_template=regional_path_template, ) diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 99ca72be..0995c0d5 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -106,6 +106,7 @@ class DataReleaseArtifact(BaseModel): path: str repo_id: str revision: str + repo_type: Optional[Literal["model", "dataset"]] = None sha256: Optional[str] = None size_bytes: Optional[int] = None preservation_mirrors: list[PreservationMirror] = Field(default_factory=list) @@ -125,6 +126,10 @@ def uri(self) -> str: class DataReleaseManifest(BaseModel): schema_version: int data_package: PackageVersion + dataset_role: Optional[str] = None + """Producer-declared role for a release that supplements a default release.""" + is_default: Optional[bool] = None + """Whether the producer intends this release to provide default datasets.""" compatible_model_packages: list[CompatiblePackage] = Field(default_factory=list) compatible_core_packages: list[CompatiblePackage] = Field(default_factory=list) default_datasets: dict[str, str] = Field(default_factory=dict) @@ -324,7 +329,6 @@ def get_release_manifest(country_id: str) -> CountryReleaseManifest: release_payload = bundle["data_releases"][country_id] except KeyError as exc: raise ValueError(f"No bundled data release for country '{country_id}'") from exc - release_payload = _apply_dataset_overlays(country_id, release_payload, bundle) manifest = CountryReleaseManifest.model_validate(release_payload) manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest() diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 3d090b4e..b30db02e 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -8,6 +8,7 @@ from policyengine import bundle from policyengine.cli import main as cli_main from policyengine.provenance.dataset_materialization import MaterializedDataset +from policyengine.provenance.manifest import get_release_manifest def _sha256(payload: bytes) -> str: @@ -48,6 +49,20 @@ def test_bundle_manifest_exposes_data_releases(): ) +def test_bundle_and_country_manifest_share_certified_dataset_metadata(): + bundle_release = bundle.get_current_bundle()["data_releases"]["us"] + country_release = get_release_manifest("us") + + assert set(bundle_release["datasets"]) == set(country_release.datasets) + assert { + region_type: template["path_template"] + for region_type, template in bundle_release["region_datasets"].items() + } == { + region_type: template.path_template + for region_type, template in country_release.region_datasets.items() + } + + def test_bundle_install_requirements_are_country_scoped(): manifest = bundle.get_current_bundle() @@ -82,6 +97,18 @@ def test_selected_dataset_plan_uses_certified_release_metadata(tmp_path): assert plan.sha256 == release["datasets"][plan.dataset]["sha256"] +def test_selected_us_dataset_plan_installs_only_national_default(tmp_path): + manifest = bundle.get_current_bundle() + + entries = bundle._selected_dataset_plans(manifest, ["us"], data_dir=tmp_path) + + assert len(entries) == 1 + plan, release = entries[0] + assert plan.dataset == "populace_us_2024" + assert "populace_us_2024_acs_local" in release["datasets"] + assert plan.source_uri == release["default_dataset_uri"] + + def test_install_bundle_package_only_uses_explicit_python(monkeypatch, tmp_path): calls = [] diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 9b25a1db..b18a77f0 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -14,6 +14,8 @@ CertificationError, build_country_manifest_payload, certify_data_release, + merge_us_local_area_release_manifest, + merge_us_regional_release_manifest, merge_us_state_release_manifest, parse_manifest_uri, required_supplemental_release_files, @@ -36,6 +38,11 @@ "hf://model/policyengine/policyengine-us-data" f"@{US_DATA_VERSION}/releases/{US_DATA_VERSION}/release_manifest.json" ) +US_LOCAL_AREA_TAG = "populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z" +US_LOCAL_AREA_MANIFEST_URI = ( + "hf://dataset/policyengine/populace-us" + f"@{US_LOCAL_AREA_TAG}/releases/{US_LOCAL_AREA_TAG}/release_manifest.json" +) def _release_manifest_payload() -> dict: @@ -157,6 +164,26 @@ def _state_release_manifest_payload( } +def _local_area_release_manifest_payload() -> dict: + return { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "dataset_role": "non_default_local_area", + "is_default": False, + "default_datasets": {}, + "artifacts": { + "populace_us_2024_acs_local": { + "kind": "microdata", + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "revision": US_LOCAL_AREA_TAG, + "sha256": "7" * 64, + "size_bytes": 1, + } + }, + } + + def _manifest() -> DataReleaseManifest: return DataReleaseManifest.model_validate(_release_manifest_payload()) @@ -444,6 +471,109 @@ def test__given_duplicate_state_artifact__then_warns_and_ignores_duplicate(self) ) +class TestMergeUSLocalAreaReleaseManifest: + def test__given_typed_local_area_release__then_certifies_shared_dataset(self): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + regional = DataReleaseManifest.model_validate( + _local_area_release_manifest_payload() + ) + + merged = merge_us_local_area_release_manifest( + primary, + regional, + regional_repo_type="dataset", + ) + payload = build_country_manifest_payload( + country="us", + manifest=merged, + uri_parts=parse_manifest_uri(MANIFEST_URI), + policyengine_version="9.9.9", + model_package="policyengine-us", + model_version="1.723.0", + model_wheel={}, + ) + + local_dataset = payload["datasets"]["populace_us_2024_acs_local"] + assert local_dataset == { + "path": "populace_us_2024_acs_local.h5", + "revision": US_LOCAL_AREA_TAG, + "sha256": "7" * 64, + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + } + assert payload["region_datasets"] == { + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, + "national": {"path_template": "populace_us_2024.h5"}, + "state": {"path_template": "populace_us_2024_acs_local.h5"}, + } + + @pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("dataset_role", "other", "dataset_role='non_default_local_area'"), + ("is_default", True, "is_default=false"), + ("default_datasets", {"national": "local"}, "default datasets"), + ], + ) + def test__given_invalid_release_role__then_rejects_it( + self, + field, + value, + message, + ): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + regional_payload = _local_area_release_manifest_payload() + regional_payload[field] = value + regional = DataReleaseManifest.model_validate(regional_payload) + + with pytest.raises(CertificationError, match=message): + merge_us_local_area_release_manifest(primary, regional) + + def test__given_more_than_one_microdata_artifact__then_rejects_it(self): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + regional_payload = _local_area_release_manifest_payload() + regional_payload["artifacts"]["second"] = { + **regional_payload["artifacts"]["populace_us_2024_acs_local"], + "path": "second.h5", + } + regional = DataReleaseManifest.model_validate(regional_payload) + + with pytest.raises(CertificationError, match="exactly one microdata"): + merge_us_local_area_release_manifest(primary, regional) + + def test__given_legacy_regional_release__then_uses_state_merge(self): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + states = DataReleaseManifest.model_validate(_state_release_manifest_payload()) + + merged = merge_us_regional_release_manifest(primary, states) + + assert "states/CA" in merged.artifacts + assert merged.metadata["region_datasets"]["state"] == { + "path_template": "states/{state_code}.h5" + } + + def test__given_unknown_regional_role__then_rejects_it(self): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + regional_payload = _local_area_release_manifest_payload() + regional_payload["dataset_role"] = "unknown" + regional = DataReleaseManifest.model_validate(regional_payload) + + with pytest.raises(CertificationError, match="Unsupported.*dataset_role"): + merge_us_regional_release_manifest(primary, regional) + + class TestCertifyDataRelease: def test__given_fetched_populace_manifest__then_updates_bundle_manifest( self, tmp_path @@ -501,7 +631,7 @@ def test__given_fetched_populace_manifest__then_updates_bundle_manifest( assert result.build_id == UK_TAG assert result.bundle_path == bundle_path - def test__given_us_regional_manifest__then_validates_but_does_not_vendor_state_artifacts( + def test__given_us_local_area_manifest__then_certifies_shared_regional_dataset( self, tmp_path ): bundle_path = tmp_path / "manifest.json" @@ -514,7 +644,7 @@ def test__given_us_regional_manifest__then_validates_but_does_not_vendor_state_a regional_response = MagicMock() regional_response.status_code = 200 regional_response.content = json.dumps( - _state_release_manifest_payload() + _local_area_release_manifest_payload() ).encode() with ( @@ -547,7 +677,7 @@ def test__given_us_regional_manifest__then_validates_but_does_not_vendor_state_a country="us", data_producer="populace", manifest_uri=MANIFEST_URI, - regional_manifest_uri=US_DATA_MANIFEST_URI, + regional_manifest_uri=US_LOCAL_AREA_MANIFEST_URI, model_version="1.723.0", bundle_path=bundle_path, ) @@ -555,12 +685,19 @@ def test__given_us_regional_manifest__then_validates_but_does_not_vendor_state_a written = json.loads(bundle_path.read_text()) release = written["data_releases"]["us"] assert release["source_manifest_uri"] == MANIFEST_URI - assert release["regional_source_manifest_uri"] == US_DATA_MANIFEST_URI + assert release["regional_source_manifest_uri"] == US_LOCAL_AREA_MANIFEST_URI assert release["region_datasets"] == { - "national": {"path_template": "populace_us_2024.h5"} + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, + "national": {"path_template": "populace_us_2024.h5"}, + "state": {"path_template": "populace_us_2024_acs_local.h5"}, } - assert "states/CA" not in release["datasets"] - assert result.dataset_count == 4 + assert "populace_us_2024_acs_local" in release["datasets"] + assert ( + release["datasets"]["populace_us_2024_acs_local"]["repo_type"] == "dataset" + ) + assert result.dataset_count == 5 def test__given_us_without_data_producer__then_legacy_update_is_explicitly_unsupported( self, tmp_path diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 485cc1d0..e1c73d83 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -61,12 +61,15 @@ US_RELEASE_MANIFEST_DATASET_URI = ( f"hf://policyengine/populace-us/populace_us_2024.h5@{US_DATA_RELEASE_REVISION}" ) -# Non-default local-area overlay: a staged Populace US artifact published in its -# own immutable release (Build L), loadable by name but never the default. +# Certified local-area dataset: a Populace US artifact published in its own +# immutable release. It remains non-default nationally and is selected for +# states and congressional districts. US_LOCAL_AREA_DATASET = "populace_us_2024_acs_local" -US_LOCAL_AREA_RELEASE_ID = "populace-us-2024-buildo-acs-local-77e2061-20260724T110908Z" +US_LOCAL_AREA_RELEASE_ID = ( + "populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z" +) US_LOCAL_AREA_SHA256 = ( - "71763290ded993789af0ad818fd833a6aabb9ca7ff7d14f78315340d8617c6f4" + "769756c31f3ca646d12c272511744dec04c0e68870c6946dd945fbba65b6a7ec" ) US_LOCAL_AREA_DATASET_URI = ( "hf://policyengine/populace-us/populace_us_2024_acs_local.h5" @@ -316,13 +319,8 @@ def test__given_no_dataset__then_managed_resolution_uses_certified_default(self) == get_release_manifest("us").default_dataset_uri ) - def test__given_local_area_overlay__then_default_resolution_is_unchanged(self): - """Regression: the non-default local-area overlay never changes defaults. - - The overlay adds a loadable name; it must never become the certified - default, and default resolution must keep pointing at the certified - national Populace artifact. - """ + def test__given_local_area_dataset__then_national_default_is_unchanged(self): + """The certified regional dataset never changes the national default.""" manifest = get_release_manifest("us") assert manifest.default_dataset == "populace_us_2024" @@ -332,9 +330,16 @@ def test__given_local_area_overlay__then_default_resolution_is_unchanged(self): assert manifest.region_datasets["national"].path_template == ( "populace_us_2024.h5" ) + assert manifest.region_datasets["state"].path_template == ( + "populace_us_2024_acs_local.h5" + ) + assert ( + manifest.region_datasets["congressional_district"].path_template + == "populace_us_2024_acs_local.h5" + ) def test__given_local_area_name__then_resolves_to_its_immutable_tag(self): - """The local-area overlay resolves to its own Build L release tag.""" + """The local-area dataset resolves to its own immutable release tag.""" assert ( resolve_dataset_reference("us", US_LOCAL_AREA_DATASET) == US_LOCAL_AREA_DATASET_URI @@ -346,7 +351,7 @@ def test__given_local_area_name__then_resolves_to_its_immutable_tag(self): == US_LOCAL_AREA_DATASET_URI ) - def test__given_local_area_overlay__then_registered_with_sha_but_not_default(self): + def test__given_local_area_dataset__then_registered_with_sha_but_not_default(self): manifest = get_release_manifest("us") reference = manifest.datasets[US_LOCAL_AREA_DATASET] @@ -357,19 +362,18 @@ def test__given_local_area_overlay__then_registered_with_sha_but_not_default(sel assert US_LOCAL_AREA_DATASET in resolve_default_datasets("us") assert US_LOCAL_AREA_DATASET != manifest.default_dataset - def test__given_us_manifest__then_has_no_inherited_area_artifacts(self): - manifest = get_release_manifest("us") - - assert "state" not in manifest.region_datasets - assert "congressional_district" not in manifest.region_datasets - assert resolve_region_dataset_path("us", "state", state_code="CA") is None + def test__given_us_manifest__then_local_regions_resolve_to_acs_local(self): + assert ( + resolve_region_dataset_path("us", "state", state_code="CA") + == US_LOCAL_AREA_DATASET_URI + ) assert ( resolve_region_dataset_path( "us", "congressional_district", district_code="CA-01", ) - is None + == US_LOCAL_AREA_DATASET_URI ) assert not any( key.startswith(("states/", "districts/")) diff --git a/tests/test_us_regions.py b/tests/test_us_regions.py index 57414498..d1ccd09d 100644 --- a/tests/test_us_regions.py +++ b/tests/test_us_regions.py @@ -1,12 +1,21 @@ """Tests for US region definitions.""" -from policyengine.countries.us.data import DISTRICT_COUNTS, US_STATE_FIPS, US_STATES +from policyengine.countries.us.data import ( + DISTRICT_COUNTS, + US_STATE_FIPS, + US_STATES, +) from policyengine.countries.us.regions import ( build_us_region_registry, us_region_registry, ) from policyengine.provenance.manifest import CountryReleaseManifest +US_LOCAL_AREA_DATASET_URI = ( + "hf://policyengine/populace-us/populace_us_2024_acs_local.h5" + "@populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z" +) + class TestUSStates: """Tests for US state definitions.""" @@ -134,7 +143,7 @@ def test__given_california_region__then_has_correct_format(self): assert ca.label == "California" assert ca.region_type == "state" assert ca.parent_code == "us" - assert ca.dataset_path is None + assert ca.dataset_path == US_LOCAL_AREA_DATASET_URI assert ca.requires_filter assert ca.scoping_strategy is not None assert ca.scoping_strategy.variable_name == "state_fips" @@ -167,7 +176,7 @@ def test__given_ca_first_district__then_has_correct_format(self): assert "1st" in ca01.label.lower() or "1 " in ca01.label assert ca01.region_type == "congressional_district" assert ca01.parent_code == "state/ca" - assert ca01.dataset_path is None + assert ca01.dataset_path == US_LOCAL_AREA_DATASET_URI assert ca01.requires_filter assert ca01.scoping_strategy is not None assert ca01.scoping_strategy.variable_name == "congressional_district_geoid" @@ -250,24 +259,51 @@ def test__given_california__then_children_include_districts_and_places( assert len(district_children) == DISTRICT_COUNTS["CA"] assert len(place_children) >= 10 # CA has many large cities - def test__given_us_registry__then_dataset_regions_are_national_only(self): + def test__given_us_registry__then_all_filter_regions_have_datasets(self): """Given: US region registry When: Getting regions with datasets - Then: Only the national canonical Populace dataset is dedicated + Then: National, state, and district regions declare their input dataset """ # When dataset_regions = us_region_registry.get_dataset_regions() # Then - assert len(dataset_regions) == 1 - assert dataset_regions[0].region_type == "national" + assert len(dataset_regions) == 1 + 51 + 436 + assert {region.region_type for region in dataset_regions} == { + "national", + "state", + "congressional_district", + } + + def test__given_states_and_districts__then_all_use_local_area_dataset(self): + local_regions = [ + *us_region_registry.get_by_type("state"), + *us_region_registry.get_by_type("congressional_district"), + ] + + assert len(local_regions) == 51 + 436 + assert {region.dataset_path for region in local_regions} == { + US_LOCAL_AREA_DATASET_URI + } + assert all(region.requires_filter for region in local_regions) - def test__given_certified_state_template__then_state_filters_national_dataset( + def test__given_dc_state_and_district__then_both_use_local_area_dataset(self): + dc = us_region_registry.get("state/dc") + dc_al = us_region_registry.get("congressional_district/DC-01") + + assert dc is not None + assert dc_al is not None + assert dc.dataset_path == US_LOCAL_AREA_DATASET_URI + assert dc_al.dataset_path == US_LOCAL_AREA_DATASET_URI + assert dc.scoping_strategy is not None + assert dc_al.scoping_strategy is not None + + def test__given_certified_state_template__then_state_uses_it_and_filters_rows( self, monkeypatch ): """Given: US bundle manifest with a certified state template When: Building the region registry - Then: State regions still filter the national certified dataset + Then: State regions use the declared dataset and retain row filtering """ manifest = CountryReleaseManifest.model_validate( { @@ -312,7 +348,9 @@ def test__given_certified_state_template__then_state_filters_national_dataset( ca = registry.get("state/ca") assert ca is not None - assert ca.dataset_path is None + assert ca.dataset_path == ( + "hf://policyengine/policyengine-us-data/states/CA.h5@1.115.5" + ) assert ca.requires_filter assert ca.scoping_strategy is not None assert ca.scoping_strategy.variable_name == "state_fips"