Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
b7bb1c1
Register the 86 proposed AB2026 keys and key their rows
claude Nov 5, 2026
3c146d2
Read the 26 AB2026 sign conventions signs.py declared unknown
claude Nov 5, 2026
640bdd3
Sort the variables in the AB2025 registry's name searches
claude Nov 5, 2026
2fefa6f
Ingest the AB2026 harvest with exact accounting
claude Nov 5, 2026
070113e
Revert the AB2025 name-search sort, which changes the public feed
claude Nov 5, 2026
b2b1995
Require AB2025's ingest to match main byte for byte
claude Nov 5, 2026
3812ce4
Merge the closed reader of value_raw into tranche 2
MaxGhenis Nov 5, 2026
dccb4f7
Drop nine red book repetitions of Table 6.1 cells; stable merge prove…
claude Nov 5, 2026
48c970d
Give packages with no expressible leg not_expressible; cut lineage fr…
claude Nov 5, 2026
b285db1
Require the AB2025 differential in CI; rewrite new drop reasons for s…
claude Nov 5, 2026
c58128e
Merge remote-tracking branch 'origin/uk/ab2026-harvest' into uk/ab202…
claude Nov 5, 2026
367e4f0
Tally IFS partial-year support, RF income paths, CenTax percentages a…
claude Nov 5, 2026
a4b3d4e
Test that each AB2026 claim preserves its staged row
claude Nov 5, 2026
83153e6
Tally the red book's two business gains; file IFS's maximum gain as n…
claude Nov 5, 2026
ea006ed
Test the dispositions, ingest's accounting check and verdict_rows' re…
claude Nov 5, 2026
327c211
Tighten two exchequer sign phrases; give AB2026 its own restatement r…
claude Nov 5, 2026
5ff3c4b
Test the leg lists against their scopes, producer scopes and the comp…
claude Nov 5, 2026
b227956
Test the stage's row-call check, recostings' verdicts and twins throu…
claude Nov 5, 2026
002d90c
Pin the sign phrases that match only dropped rows
claude Nov 5, 2026
fbea857
Merge remote-tracking branch 'origin/uk/ab2026-harvest' into uk/ab202…
claude Nov 5, 2026
5303262
Bring the AB2026 ingest's docstring up to date with the code
claude Nov 5, 2026
589f822
Give the ingest's and key decisions' text real verbs
MaxGhenis Nov 5, 2026
b8af215
Answer the reviews of 589f8229: IFS's one-off payment, guards and rea…
MaxGhenis Nov 5, 2026
0193b91
Merge uk/ab2026-harvest into uk/ab2026-tranche2
MaxGhenis Nov 5, 2026
8a3e498
Answer the reviews of 0193b91f: keep OBR row 2740 on borrowing, tight…
MaxGhenis Nov 5, 2026
412c2f4
Answer the review of 8a3e4982: close the preservation test's program …
MaxGhenis Nov 5, 2026
a28223f
Pin the alias path of the preservation helper
MaxGhenis Nov 5, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
# tests/test_uk_ab2026_ingest.py compares AB2025's ingest with main's
- run: git fetch --no-tags --depth=1 origin main:refs/remotes/origin/main
- uses: astral-sh/setup-uv@v5
- run: uvx ruff format --check be nz scorecard_db tests pipeline
# The database is a DERIVED artifact (scorecard_db/build_db.py):
Expand Down
1,688 changes: 1,681 additions & 7 deletions data/uk/ab2026_measures.json

Large diffs are not rendered by default.

1,208 changes: 1,208 additions & 0 deletions pipeline/ab2026_key_decisions.py

Large diffs are not rendered by default.

204 changes: 204 additions & 0 deletions pipeline/apply_ab2026_key_decisions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
"""Give each staged AB2026 row its registered key, and write KEY_DECISIONS.md.

Tranche 1 staged 973 rows with `measure_key: null` and a proposed key in
`conditions.proposed_measure_key` (harvest README rule 2). Tranche 2
registers each proposed key, or folds it into a registered key
(pipeline/ab2026_key_decisions.py). AB2025's rows name their registry key in
`measure_key`, and the contract checker rejects a registered key left in
`proposed_measure_key`, so this script moves each row's key into
`measure_key` and drops the condition. It changes nothing else in a row, and
it writes each claims file as the stagers do: sorted keys, UTF-8, gzip level 9
with no name or time in the header.

Every registered key is new to the registry and we fold none, so the rows a
decision covers are exactly the rows that now name its key. `render`
rebuilds KEY_DECISIONS.md from the rewritten claims, and the tests check it.

Usage:
python pipeline/apply_ab2026_key_decisions.py # rewrite and write
python pipeline/apply_ab2026_key_decisions.py --check # check only

After a rewrite, re-pin COUNTS.json with pipeline/harvest_counts.py and
regenerate PROPOSED_KEYS.md with pipeline/ab2026_proposed_keys.py.
"""

import gzip
import io
import json
import sys
from collections import defaultdict
from pathlib import Path

try:
from pipeline import ab2026_key_decisions as D
except ModuleNotFoundError: # direct script invocation without PYTHONPATH
import ab2026_key_decisions as D

REPO = Path(__file__).resolve().parent.parent
HARVEST = REPO / "sources/harvest-uk-ab2026-2026-10-29"
REGISTRY = REPO / "data/uk/ab2026_measures.json"
OUT = HARVEST / "KEY_DECISIONS.md"


def families(harvest=HARVEST):
return sorted(
p.name for p in harvest.iterdir() if (p / "claims_staged.jsonl.gz").exists()
)


def read(path):
with gzip.open(path, "rt", encoding="utf-8") as fh:
return [json.loads(line) for line in fh if line.strip()]


def write(path, rows):
buf = io.BytesIO()
with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0, filename="") as gz:
for r in rows:
gz.write(
(json.dumps(r, sort_keys=True, ensure_ascii=False) + "\n").encode()
)
path.write_bytes(buf.getvalue())


def rekey(row):
"""The row with its proposed key moved into measure_key (or unchanged)."""
pk = (row.get("conditions") or {}).get("proposed_measure_key")
if not pk:
return row
if pk not in D.DECISIONS:
raise ValueError(f"{pk} has no decision in pipeline/ab2026_key_decisions.py")
out = dict(row)
out["conditions"] = {
k: v for k, v in row["conditions"].items() if k != "proposed_measure_key"
}
out["measure_key"] = D.registered_key(pk)
return out


def apply(harvest=HARVEST):
changed = {}
for fam in families(harvest):
path = harvest / fam / "claims_staged.jsonl.gz"
rows = read(path)
new = [rekey(r) for r in rows]
n = sum(a is not b for a, b in zip(rows, new))
if n:
write(path, new)
changed[fam] = n
return changed


def _ranges(idx):
out, start, prev = [], None, None
for i in idx:
if start is None:
start = prev = i
elif i == prev + 1:
prev = i
else:
out.append((start, prev))
start = prev = i
if start is not None:
out.append((start, prev))
return ", ".join(str(a) if a == b else f"{a}–{b}" for a, b in out)


def _why(d):
v = d["verdict"]
kind = v[0]
if kind == "legs":
return "We combine the named registry measures."
if kind == "receipts":
return "We register it as the receipts of the named Table 6.1 measures."
if kind == "words":
return f"Its producer gives the scope in words: {v[1]}."
if kind in ("scope", "scope_why"):
return "We can't estimate this quantity within the household model."
if kind == "gap":
return "We need further policyengine-uk development to estimate this quantity."
if kind == "delta":
return "We represent this change with parameter deltas on current law."
if kind == "partial":
return "We can estimate part of this change."
raise ValueError(kind)


def render(harvest=HARVEST):
"""KEY_DECISIONS.md from the decisions and the rows that name each key."""
reg = {m["measure_key"]: m for m in json.loads(REGISTRY.read_text())["measures"]}
rows_of = defaultdict(lambda: defaultdict(list))
for fam in families(harvest):
for i, r in enumerate(read(harvest / fam / "claims_staged.jsonl.gz")):
rows_of[r["measure_key"]][fam].append(i)
kinds = defaultdict(int)
for d in D.DECISIONS.values():
kinds[d["kind"]] += 1
total = sum(d["rows"] for d in D.DECISIONS.values())
out = [
"# Key decisions for the AB2026 harvest (#230)",
"",
"Generated by `pipeline/apply_ab2026_key_decisions.py` from `pipeline/ab2026_key_decisions.py` and the staged families; don't edit by hand.",
"",
f"We register {len(D.DECISIONS) - len(D.FOLDED)} AB2026 keys ({kinds['measure']} measures, {kinds['package']} packages and {kinds['option']} options) for {total} staged figures. "
"We give two producers' packages one key only when they identify the same legs (rule 3). "
f"For {len(D.RENAMED)} key, we retain the proposed name in `proposed_as`. "
"We store registered keys in `measure_key` and copy each producer's scope from NOTES.md into `source_note`. "
"We number each family's rows from 0, in the order of its `claims_staged.jsonl.gz`, and register the keys in `data/uk/ab2026_measures.json`.",
"",
"| proposed key | decision | registered key | verdict | why | rows |",
"|---|---|---|---|---|---|",
]
for pk, d in sorted(D.DECISIONS.items()):
key = D.registered_key(pk)
if pk in D.FOLDED:
decision, why = "folded", D.FOLDED[pk][1]
elif pk in D.RENAMED:
decision, why = "registered, renamed", _why(d)
else:
decision, why = "registered", _why(d)
m = reg.get(key)
verdict = m["computability"] if m else "MISSING"
cover = "; ".join(
f"{fam} {_ranges(idx)}" for fam, idx in sorted(rows_of[key].items())
)
n = sum(len(v) for v in rows_of[key].values())
out.append(
f"| `{pk}` | {decision} | `{key}` | {verdict} | {why} Scope: {d['scope'].replace('|', '/')} | {n}: {cover} |"
)
return "\n".join(out) + "\n"


def problems(harvest=HARVEST):
reg = {m["measure_key"] for m in json.loads(REGISTRY.read_text())["measures"]}
out = []
for fam in families(harvest):
for i, r in enumerate(read(harvest / fam / "claims_staged.jsonl.gz")):
if (r.get("conditions") or {}).get("proposed_measure_key"):
out.append(f"{fam} row {i} still names a proposed key")
counted = defaultdict(int)
for fam in families(harvest):
for r in read(harvest / fam / "claims_staged.jsonl.gz"):
counted[r["measure_key"]] += 1
for pk, d in D.DECISIONS.items():
key = D.registered_key(pk)
if key not in reg:
out.append(f"{key} is not in the registry")
if pk not in D.FOLDED and counted[key] != d["rows"]:
out.append(f"{key}: {counted[key]} rows, the decision records {d['rows']}")
if OUT.read_text() != render(harvest):
out.append("KEY_DECISIONS.md is stale")
return out


def main(argv):
if "--check" not in argv:
print(json.dumps(apply(), indent=1))
OUT.write_text(render())
bad = problems()
print("\n".join(bad) if bad else "ok")
return 1 if bad else 0


if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
Loading
Loading