Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,24 @@ adopt the facts-only surface explicitly.
`--year` is inert for `--suite uk` because the UK packages are year-pinned.
The US off-year bundle behavior is unchanged and out of scope here.

The New Zealand feed builds the same way from the curated NZ suite:

```bash
uv run chronicle build-bundle --suite nz --out /tmp/chronicle-nz --replace
uv run chronicle build-consumer-artifact --facts /tmp/chronicle-nz --out /tmp/chronicle-nz-artifact --replace
```

`NZ_BUNDLE_SOURCES` in `chronicle/bundle.py` lists every package under
`packages/ird`, `packages/mbie`, `packages/msd`, `packages/stats_nz` and
`packages/treasury`, including the national, regional-council and
area-outside-region (`rc_99`) Census 2023 place-summary packages; a test
fails if an alias under those directories is added or dropped without
updating it. `--year` is inert for
`--suite nz` too: every NZ package pins its `artifact_year` and uses
year-independent declarations, so one build at any `--year` emits its facts with
the publisher's own calendar, tax, fiscal-year or monthly period. `--suite`
cannot be combined with `--source`.

Builds without an Axiom CLI still pass when the source package is otherwise
valid, but `agent_acceptance.json` warns with
`concept_alignment_validation_skipped`. For strict agent review, require every
Expand Down
8 changes: 8 additions & 0 deletions changelog.d/nz-bundle-suite.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Add `chronicle build-bundle --suite nz`, a curated New Zealand bundle suite for
Microcosm's NZ feed. It builds all 48 packages under `packages/ird`,
`packages/mbie`, `packages/msd`, `packages/stats_nz` and `packages/treasury`,
including the 36 national, regional-council and area-outside-region Census
2023 place-summary packages. A drift guard fails when an alias under those
directories is added or dropped without updating the
suite. Like `--suite uk`, it refuses `--source`, and `--year` does not change
what it builds because every NZ package pins its artifact year.
4 changes: 4 additions & 0 deletions changelog.d/nz-treasury-r2-prefix.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Route `packages/treasury` and `db/data/treasury` to the `raw/nz` and
`derived/nz` R2 prefixes, so `verify-raw` and `publish-raw` compute the
`raw/nz/treasury/...` keys the two NZ Treasury manifests declare instead of
`raw/treasury/...`.
2 changes: 1 addition & 1 deletion chronicle/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
# and ``derived/{source_id}`` shapes. Publisher directories are the stable
# routing input because they are shared by packages/ and db/data/.
R2_COUNTRY_PUBLISHERS = {
"nz": frozenset({"ird", "mbie", "msd", "stats_nz"}),
"nz": frozenset({"ird", "mbie", "msd", "stats_nz", "treasury"}),
"uk": frozenset(
{
"desnz",
Expand Down
135 changes: 124 additions & 11 deletions chronicle/bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,72 @@
"welshgov-household-projections-2022-based",
"welshgov-transport-revenue-outturn-2024-25",
)
# Every package under these publisher directories is a New Zealand package:
# Inland Revenue, MBIE, MSD, Stats NZ and the New Zealand Treasury. "treasury"
# names no country, so a non-NZ treasury package must not be added under
# packages/treasury/; the drift guard would then pull it into the NZ suite.
NZ_BUNDLE_SOURCE_PREFIXES = (
"ird",
"mbie",
"msd",
"stats_nz",
"treasury",
)
# The curated NZ suite (`build-bundle --suite nz`). Each member pins one
# artifact_year and declares nothing by year, so it reads the same file and
# emits the same facts at any --year; the suite's periods (calendar, tax,
# fiscal and monthly) are the publishers' own, whatever --year is passed.
NZ_BUNDLE_SOURCES = (
"ird-taxable-income-distribution-2025",
"ird-wage-salary-distribution-2026",
"ird-working-for-families-statistics-sept-2025",
"mbie-tenancy-bond-rents-tla-2026",
"msd-annual-report-benefit-expenses-2025",
"msd-benefit-fact-sheets-national-june-2026",
"msd-benefit-fact-sheets-supplementary-june-2026",
"msd-nzs-vp-fact-sheet-june-2026",
"stats-nz-census-2023-family-type",
"stats-nz-census-2023-family-type-rc-01",
"stats-nz-census-2023-family-type-rc-02",
"stats-nz-census-2023-family-type-rc-03",
"stats-nz-census-2023-family-type-rc-04",
"stats-nz-census-2023-family-type-rc-05",
"stats-nz-census-2023-family-type-rc-06",
"stats-nz-census-2023-family-type-rc-07",
"stats-nz-census-2023-family-type-rc-08",
"stats-nz-census-2023-family-type-rc-09",
"stats-nz-census-2023-family-type-rc-12",
"stats-nz-census-2023-family-type-rc-13",
"stats-nz-census-2023-family-type-rc-14",
"stats-nz-census-2023-family-type-rc-15",
"stats-nz-census-2023-family-type-rc-16",
"stats-nz-census-2023-family-type-rc-17",
"stats-nz-census-2023-family-type-rc-18",
"stats-nz-census-2023-family-type-rc-99",
"stats-nz-census-2023-households-by-region",
"stats-nz-census-2023-households-by-region-rc-01",
"stats-nz-census-2023-households-by-region-rc-02",
"stats-nz-census-2023-households-by-region-rc-03",
"stats-nz-census-2023-households-by-region-rc-04",
"stats-nz-census-2023-households-by-region-rc-05",
"stats-nz-census-2023-households-by-region-rc-06",
"stats-nz-census-2023-households-by-region-rc-07",
"stats-nz-census-2023-households-by-region-rc-08",
"stats-nz-census-2023-households-by-region-rc-09",
"stats-nz-census-2023-households-by-region-rc-12",
"stats-nz-census-2023-households-by-region-rc-13",
"stats-nz-census-2023-households-by-region-rc-14",
"stats-nz-census-2023-households-by-region-rc-15",
"stats-nz-census-2023-households-by-region-rc-16",
"stats-nz-census-2023-households-by-region-rc-17",
"stats-nz-census-2023-households-by-region-rc-18",
"stats-nz-census-2023-households-by-region-rc-99",
"stats-nz-household-net-worth-2024",
"stats-nz-subnational-population-estimates-2025",
"treasury-an24-01-fiscal-totals",
"treasury-estimates-of-appropriations-2026-27",
)
BUNDLE_SUITES = ("nz", "uk")

_KEY_DOMAINS = {
"aggregate_fact_key": "aggregate_fact",
Expand All @@ -304,29 +370,76 @@
}


def uk_bundle_sources_from_aliases() -> tuple[str, ...]:
"""Return UK-package aliases implied by the source-package directory prefixes."""
def _prefixed_aliases(prefixes: Sequence[str]) -> tuple[str, ...]:
"""Return the sorted aliases whose package directory sits under ``prefixes``."""
return tuple(
sorted(
alias
for alias, path in SOURCE_PACKAGE_ALIASES.items()
if path.parts and path.parts[0] in UK_BUNDLE_SOURCE_PREFIXES
if path.parts and path.parts[0] in prefixes
)
)


def assert_uk_bundle_sources_match_aliases() -> None:
"""Fail loudly if the curated UK suite omits a UK-prefixed alias."""
expected = uk_bundle_sources_from_aliases()
if UK_BUNDLE_SOURCES != expected:
missing = sorted(set(expected) - set(UK_BUNDLE_SOURCES))
extra = sorted(set(UK_BUNDLE_SOURCES) - set(expected))
def _assert_suite_matches_aliases(
country: str,
sources: tuple[str, ...],
expected: tuple[str, ...],
) -> None:
if sources != expected:
missing = sorted(set(expected) - set(sources))
extra = sorted(set(sources) - set(expected))
raise ValueError(
"UK_BUNDLE_SOURCES drifted from UK-prefixed SOURCE_PACKAGE_ALIASES: "
f"missing={missing}, extra={extra}"
f"{country}_BUNDLE_SOURCES drifted from {country}-prefixed "
f"SOURCE_PACKAGE_ALIASES: missing={missing}, extra={extra}"
)


def uk_bundle_sources_from_aliases() -> tuple[str, ...]:
"""Return UK-package aliases implied by the source-package directory prefixes."""
return _prefixed_aliases(UK_BUNDLE_SOURCE_PREFIXES)


def assert_uk_bundle_sources_match_aliases() -> None:
"""Fail loudly if the curated UK suite omits a UK-prefixed alias."""
_assert_suite_matches_aliases(
"UK", UK_BUNDLE_SOURCES, uk_bundle_sources_from_aliases()
)


def nz_bundle_sources_from_aliases() -> tuple[str, ...]:
"""Return NZ-package aliases implied by the source-package directory prefixes.

Nested packages count: each Census 2023 place-summary family has a national
root, 16 regional-council ``rc_XX`` children and an area-outside-region
``rc_99`` child.

``--year`` does not select or relabel NZ facts: every member pins its
``artifact_year`` and uses year-independent declarations. A single build
at any year preserves all members' published calendar, tax, fiscal and
monthly reference periods.
"""
return _prefixed_aliases(NZ_BUNDLE_SOURCE_PREFIXES)


def assert_nz_bundle_sources_match_aliases() -> None:
"""Fail loudly if the curated NZ suite omits or adds an NZ-prefixed alias."""
_assert_suite_matches_aliases(
"NZ", NZ_BUNDLE_SOURCES, nz_bundle_sources_from_aliases()
)


def bundle_suite_sources(suite: str) -> tuple[str, ...]:
"""Return a curated suite's sources once they match its prefixed aliases."""
if suite == "uk":
assert_uk_bundle_sources_match_aliases()
return UK_BUNDLE_SOURCES
if suite == "nz":
assert_nz_bundle_sources_match_aliases()
return NZ_BUNDLE_SOURCES
raise ValueError(f"Unknown bundle suite: {suite!r}")


@dataclass(frozen=True)
class BuildBundleIssue:
"""One bundle-level build issue."""
Expand Down
15 changes: 8 additions & 7 deletions chronicle/harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@
verify_raw_artifacts,
)
from chronicle.bundle import (
UK_BUNDLE_SOURCES,
BUNDLE_SUITES,
BuildBundleReport,
assert_uk_bundle_sources_match_aliases,
build_bundle,
bundle_suite_sources,
)
from chronicle.concepts import ConceptAlignmentReport, validate_concept_alignments
from chronicle.consumer_contract import (
Expand Down Expand Up @@ -276,11 +276,12 @@ def build_bundle_dir(
replace: bool = False,
) -> BuildBundleReport:
"""Build a merged Chronicle consumer bundle from source-package suites."""
if suite == "uk":
if suite is not None:
if suite not in BUNDLE_SUITES:
raise ValueError(f"Unknown bundle suite: {suite!r}")
if sources:
raise ValueError("--suite uk cannot be combined with --source.")
assert_uk_bundle_sources_match_aliases()
sources = list(UK_BUNDLE_SOURCES)
raise ValueError(f"--suite {suite} cannot be combined with --source.")
sources = list(bundle_suite_sources(suite))
return build_bundle(
output_dir,
year=year,
Expand Down Expand Up @@ -744,7 +745,7 @@ def main(argv: list[str] | None = None) -> int:
)
bundle_parser.add_argument(
"--suite",
choices=["uk"],
choices=list(BUNDLE_SUITES),
help="Curated source-package suite to build.",
)
bundle_parser.add_argument(
Expand Down
6 changes: 6 additions & 0 deletions docs/agent-source-package-harness.md
Original file line number Diff line number Diff line change
Expand Up @@ -672,6 +672,12 @@ uv run chronicle build-consumer-artifact --facts /tmp/chronicle-uk --out /tmp/ch
`--year` is inert for `--suite uk` because the UK packages are year-pinned.
The US off-year bundle behavior is unchanged and out of scope here.

`--suite nz` builds the curated New Zealand suite (every package under
`packages/ird`, `packages/mbie`, `packages/msd`, `packages/stats_nz` and
`packages/treasury`) the same way, and
`--year` is inert for it as well: each NZ package pins its `artifact_year` and
uses year-independent declarations.

The first agent-facing gate is now
`/tmp/chronicle-suite/reports/agent_acceptance.json`; it summarizes whether raw
artifacts have R2 pointers, the full source document was parsed, facts have
Expand Down
46 changes: 46 additions & 0 deletions tests/test_chronicle_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

from chronicle.cli import main as cli_main
from chronicle.artifacts import (
R2_COUNTRY_PUBLISHERS,
build_artifact_key,
build_artifact_rows,
build_derived_r2_key,
Expand All @@ -24,8 +25,10 @@
publish_derived_artifacts,
publish_source_artifacts,
)
from chronicle.bundle import NZ_BUNDLE_SOURCE_PREFIXES, NZ_BUNDLE_SOURCES
from chronicle.epoch import Epoch
from chronicle.harness import main as harness_main
from chronicle.source_package import load_source_package


REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
Expand Down Expand Up @@ -97,6 +100,12 @@ def test_build_artifact_key_rejects_unknown_build_epoch():
("source_id", "package_path", "expected_country"),
[
("ird", "db/data/ird/wff", "nz"),
("treasury", "db/data/treasury/an24_01_fiscal_totals", "nz"),
(
"treasury",
"/repo/packages/treasury/estimates_of_appropriations_2026_27",
"nz",
),
("desnz", "db/data/desnz/qep", "uk"),
("ons", "db/data/ons/mye", "uk"),
("irs_soi", "db/data/irs_soi/table_1_1", None),
Expand Down Expand Up @@ -147,6 +156,43 @@ def test_country_aware_r2_keys_preserve_legacy_us_layout():
assert desnz_key == "raw/uk/desnz/desnz-qep/2026/def456/qep.xlsx"


def test_every_nz_suite_manifest_declares_its_default_canonical_raw_key():
"""Every NZ suite artifact sits under raw/nz, Treasury's included.

verify-raw and publish-raw route a package to raw/nz by its publisher
directory, so every NZ suite publisher must be an NZ R2 publisher.
"""
assert R2_COUNTRY_PUBLISHERS["nz"] == frozenset(NZ_BUNDLE_SOURCE_PREFIXES)
manifests = set()
for alias in NZ_BUNDLE_SOURCES:
artifact = load_source_package(alias).artifact
manifests.add(
REPOSITORY_ROOT
/ artifact.resource_package
/ artifact.resource_directory
/ artifact.manifest
)

checked = []
for manifest_path in sorted(manifests):
manifest = yaml.safe_load(manifest_path.read_text())
for year, artifact in manifest["files"].items():
expected_key = build_r2_key(
source_id=manifest["source_id"],
package_id=manifest["package_id"],
year=year,
sha256=artifact["sha256"],
filename=artifact["filename"],
package_path=manifest_path,
)
assert expected_key.startswith("raw/nz/")
assert artifact["storage"]["r2"]["key"] == expected_key, manifest_path
checked.append(manifest["source_id"])

assert set(checked) == set(NZ_BUNDLE_SOURCE_PREFIXES)
assert checked.count("treasury") == 4


def test_all_desnz_manifests_use_canonical_uk_r2_keys_and_pinned_bytes():
manifests = sorted(DESNZ_DATA_ROOT.glob("*/manifest.yaml"))

Expand Down
Loading
Loading