diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 9443fa258..6d9e666f8 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -72,6 +72,10 @@ jobs: fetch-depth: 0 # Fetch all history for all tags and branches - name: Install uv uses: astral-sh/setup-uv@v8.1.0 + # Restore uv's package cache but never save one from a job that + # holds the data: its location can be moved (UV_CACHE_DIR). + with: + save-cache: false - name: Set up Python uses: actions/setup-python@v6 with: @@ -86,21 +90,161 @@ jobs: run: uv run --frozen make data env: TESTING: "1" - - name: Save calibration log (constituencies) - uses: actions/upload-artifact@v7 - with: - name: constituency_calibration_log.csv - path: constituency_calibration_log.csv - - - name: Save calibration log (local authorities) - uses: actions/upload-artifact@v7 - with: - name: la_calibration_log.csv - path: la_calibration_log.csv - - name: Run tests run: uv run --frozen make test env: # Relaxes built-dataset outcome tolerances to match the reduced # fidelity of the TESTING build above (see test_cgt_band_donors). TESTING: "1" + # After the tests, whatever they did. The raw calibration logs stay on + # the runner: a target supported by a few households, divided by a + # published count, could give one household's value back. Only the + # summary the fit comparison needs is uploaded (utils/fit_report.py). + - name: Summarise the calibration fit + if: ${{ !cancelled() }} + run: uv run --frozen python -m policyengine_uk_data.utils.fit_report summarise constituency_calibration_log.csv --out fit_summary.json + - name: Save the fit summary + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: fit_summary.json + path: fit_summary.json + # The no-worsening check (W3). The base is built the same way as the head: + # TESTING, one clone and the head's lockfile, so both builds run one model + # and environment and the comparison measures the data change. As in the + # test job, only the fit summary leaves it. No dataset or calibration log + # is uploaded, and nothing is cached: setup-uv restores uv's package cache + # and saves none. + base-build: + name: Build the base for comparison + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + PYTHONUNBUFFERED: "1" + PE_UK_DATA_OA_CLONES: "1" + steps: + - name: Check out the head, for its lockfile + uses: actions/checkout@v6 + with: + path: head + - name: Check out the base + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + # Restore uv's package cache but never save one from a job that + # holds the data: its location can be moved (UV_CACHE_DIR). + with: + save-cache: false + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.13 + - name: Sync the head's locked environment for the base + run: | + cp head/uv.lock base/uv.lock + cd base && uv sync --frozen --all-extras + - name: Download data inputs + working-directory: base + run: uv run --frozen make download + - name: Build the base datasets + working-directory: base + run: uv run --frozen make data + env: + TESTING: "1" + - name: Summarise the base's calibration fit + run: | + cp head/policyengine_uk_data/utils/fit_report.py base/policyengine_uk_data/utils/fit_report.py + uv run --frozen --project base python -m policyengine_uk_data.utils.fit_report summarise base/constituency_calibration_log.csv --out base_fit_summary.json + - name: Save the base fit summary + uses: actions/upload-artifact@v7 + with: + name: base_fit_summary.json + path: base_fit_summary.json + fit-comparison: + name: Calibration fit against the base + needs: [test, base-build] + # Report a failed side too: an earlier passing comment must not stand. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + issues: write + pull-requests: write + steps: + - uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + # Restore uv's package cache but never save one from a job that + # holds the data: its location can be moved (UV_CACHE_DIR). + with: + save-cache: false + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.13 + - name: Sync locked environment + run: uv sync --frozen + - name: Download the fit summaries + uses: actions/download-artifact@v8 + continue-on-error: true + with: + pattern: "*fit_summary.json" + path: logs + merge-multiple: true + - name: Compare the final fits + id: compare + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + marker='' + if [ ! -f logs/fit_summary.json ] || [ ! -f logs/base_fit_summary.json ]; then + { + echo "$marker" + echo '## Calibration fit against the base build: FAIL' + echo + echo "One of the builds produced no fit summary, so the fit could not be compared. See the [workflow run]($RUN_URL)." + } > fit.md + echo "status=1" >> "$GITHUB_OUTPUT" + else + set +e + uv run --frozen python -m policyengine_uk_data.utils.fit_report compare \ + logs/base_fit_summary.json logs/fit_summary.json --markdown body.md + echo "status=$?" >> "$GITHUB_OUTPUT" + set -e + { echo "$marker"; cat body.md 2>/dev/null || echo '## Calibration fit against the base build: FAIL'; echo; echo "[Workflow run]($RUN_URL)"; } > fit.md + fi + cat fit.md >> "$GITHUB_STEP_SUMMARY" + - name: Post or update the pull request comment + if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }} + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + marker='' + if [ ! -s fit.md ]; then + # The job failed before the comparison could write a comment. + { + echo "$marker" + echo '## Calibration fit against the base build: FAIL' + echo + echo "The comparison did not run. See the [workflow run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)." + } > fit.md + fi + existing=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"$marker\")) | .id" \ + | tail -n 1) + if [ -n "$existing" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@fit.md > /dev/null + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@fit.md > /dev/null + fi + - name: Fail on a fit regression + if: steps.compare.outputs.status != '0' + run: | + echo "::error::The head build's calibration fit is worse than the base's; see the job summary." + exit 1 diff --git a/changelog.d/fit-comparison.added.md b/changelog.d/fit-comparison.added.md new file mode 100644 index 000000000..89d7d993e --- /dev/null +++ b/changelog.d/fit-comparison.added.md @@ -0,0 +1 @@ +Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points over the targets both builds have (`targets/fit_policy.yaml`). National targets in a worse error band on the head are flagged in a pull request comment. The raw calibration logs stay on the runner; only each target's error band (within 10%, 10-25%, 25-50%, 50-100%, over 100%) is uploaded, with no estimate or error (`python -m policyengine_uk_data.utils.fit_report summarise` and `compare`). diff --git a/policyengine_uk_data/targets/fit_policy.yaml b/policyengine_uk_data/targets/fit_policy.yaml new file mode 100644 index 000000000..5b71730bc --- /dev/null +++ b/policyengine_uk_data/targets/fit_policy.yaml @@ -0,0 +1,47 @@ +# When a pull request's build fails or flags against its base build's fit +# (utils/fit_report.py). Both builds run the same way in pull request CI: +# TESTING, 32 epochs, one output-area clone, the head's lockfile. +# +# The share allowance was adopted under cos d1344 (15 October 2026), from the final-epoch fits of +# five builds of the same data code: docs, changelog and CI pull requests +# while main's data code was unchanged from b34a4ea2 to 2b9c1c2c (runs +# 38006434333, 37959581668, 38009046475, 37960852139, 37864653693). +schema_version: 1 + +share_within_10pct: + # Fail when the share of national, or of local, targets within 10% of + # their values falls by more than this. Across the 20 ordered pairs of the + # five builds the national share moved by at most 0.93pp and the local by + # at most 0.69pp, so 2pp is about twice the same-code spread. + # Alternatives: 1pp leaves no margin over that spread while #598 (builds + # of the same code fall into two weight sets) is unexplained. 5pp would + # pass the 2.4pp local drop of #551's build against the build of the + # 1.58.1 release branch (runs 38661397028 and 38228500493). + allowed_drop: 0.02 + +national_target: + # Flag, for review, each national target whose error band is worse on the + # head than on the base. Bands of |estimate / target - 1|: within 10%, + # 10-25%, 25-50%, 50-100%, over 100% (fit_report.BANDS). Across the 20 + # ordered pairs of the five same-code builds above, 2 to 12 of the 644 + # national targets move to a worse band per pair (mean 7), so a list stays + # short. #551's build moves 31 and #579's 27 against the 1.58.1 release + # branch's build. + # + # This replaces flagging an error rise of more than 0.2 (adopted under + # d1344 on 15 October), which needed each target's error in the uploaded + # summary. Sol's re-review of #599 found no safe way to publish them: the + # dataset repeats households (output-area clones, SPI and CGT donors), so + # a count of rows can't show how many respondents support an estimate, and + # one household can dominate a target. A band is a bound, not a value. + # The old rule gave 0 to 6 flags per same-code pair. Bands still can't see + # a move inside one band (say 30% to 45%). The replacement is a W3 + # threshold call under d1344's delegation (thresholds on evidence, with an + # independent review agreeing), pending that review. Per-target moves + # don't fail until #598 is fixed. + # + # Run ids of the five builds and their saved log folders: 38006434333 + # (docs-ebdbbce5), 37959581668 (docs-2a071400), 38009046475 + # (changelog-6af935fb), 37960852139 (changelog-bb41c3ca), 37864653693 + # (ci-push-gate-03b3ecd9); the 1.58.1 release branch is 38228500493. + flag: worse_band diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py new file mode 100644 index 000000000..a116f7d56 --- /dev/null +++ b/policyengine_uk_data/tests/test_fit_report.py @@ -0,0 +1,274 @@ +"""The final calibration fit summary and the no-worsening comparison (utils/fit_report.py).""" + +import json + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.utils.fit_report import ( + BANDS, + FitPolicy, + band, + compare, + load_policy, + main, + summarise, +) + +POLICY = FitPolicy(share_drop_allowance=0.02) + + +def _log(national: dict, local=(), epochs=(0, 30)): + """A calibration log: national {metric: (estimate, target)}, local [(est, target)].""" + rows = [] + for epoch in epochs: + for metric, (estimate, target) in national.items(): + rows.append(("UK", 0, metric, estimate, target, epoch)) + for i, (estimate, target) in enumerate(local): + rows.append((f"area {i}", f"E{i}", f"local/{i}", estimate, target, epoch)) + return pd.DataFrame( + rows, columns=["name", "index", "metric", "estimate", "target", "epoch"] + ) + + +LOCAL = [(100, 100)] * 10 + + +def test_summarise_reads_the_last_epoch_and_drops_zero_targets(): + log = _log( + {"a": (105, 100), "b": (150, 100), "zero": (5, 0)}, local=[(9, 10), (20, 10)] + ) + log.loc[log.epoch == 0, "estimate"] = 1 # earlier epochs are ignored + s = summarise(log) + assert s["epoch"] == 30 + # Bands of |estimate / target - 1|: 0.05 is within 10%, 0.5 is 25-50%. + assert s["national"] == {"a": 0, "b": 2} + assert "zero" not in s["national"] + assert s["local"] == {"E0|local/0": 0, "E1|local/1": 3} + + +@settings(max_examples=100, deadline=None) +@given( + estimates=st.lists(st.floats(0.01, 1e12, allow_nan=False), min_size=1, max_size=20), + target=st.floats(1.0, 1e12), +) +def test_the_summary_publishes_no_estimate_or_error(estimates, target): + """Sol re-reviews of #599: rows don't count respondents and one household + can dominate a target, so nothing per target but within 10% is published.""" + log = _log( + {f"t{i}": (e, target) for i, e in enumerate(estimates)}, + local=[(e, target) for e in estimates], + ) + s = summarise(log) + for group in ("national", "local"): + assert all(type(v) is int and 0 <= v < len(BANDS) for v in s[group].values()) + text = json.dumps(s) + for e in estimates: + assert repr(e) not in text and repr(abs(e / target - 1)) not in text + + +def test_a_malformed_log_is_refused(): + with pytest.raises(ValueError, match="no rows"): + summarise(_log({}, epochs=())) + with pytest.raises(ValueError, match="non-finite"): + summarise(_log({"a": (np.inf, 100)})) + with pytest.raises(ValueError, match="repeats"): + summarise(pd.concat([_log({"a": (1, 1)}), _log({"a": (2, 1)})])) + + +def _summary(national_errors, local=LOCAL, epochs=(0, 30)): + return summarise( + _log( + {f"t{i}": (100 * (1 + e), 100) for i, e in enumerate(national_errors)}, + local, + epochs, + ) + ) + + +def test_a_drop_beyond_the_allowance_fails_and_within_it_passes(): + base = _summary([0.0] * 100) + head = _summary([0.5, 0.5, 0.5] + [0.0] * 97) + result = compare(base, head, POLICY) + assert result["status"] == "fail" + assert ( + "national targets within 10% falls from 100.0% to 97.0%" + in result["failures"][0] + ) + assert [f["target"] for f in result["flags"]] == ["t0", "t1", "t2"] + assert compare(base, _summary([0.5] + [0.0] * 99), POLICY)["status"] == "pass" + + +def test_a_target_in_a_worse_band_flags_and_a_better_one_is_a_note(): + base = _summary([0.0, 0.5] + [0.0] * 98) + head = _summary([0.5, 0.0] + [0.0] * 98) + result = compare(base, head, POLICY) + assert result["status"] == "pass" + assert result["flags"] == [{"target": "t0", "base": "within 10%", "head": "25-50%"}] + assert "1 national targets are in a better band on the head." in result["notes"] + # Sol re-review of #599: a target worsening outside 10% flags too. + assert compare(_summary([0.15]), _summary([3.0]), POLICY)["flags"] == [ + {"target": "t0", "base": "10-25%", "head": "over 100%"} + ] + # A move inside one band is not a flag. + assert compare(_summary([0.01]), _summary([0.09]), POLICY)["flags"] == [] + assert compare(_summary([0.3]), _summary([0.45]), POLICY)["flags"] == [] + + +@pytest.mark.parametrize( + "error, expected", + [(0.0, 0), (0.1, 0), (0.1000001, 1), (0.25, 1), (0.5, 2), (1.0, 3), (1.01, 4)], +) +def test_band_edges_are_inclusive(error, expected): + assert band(error) == expected + + +def test_an_estimate_exactly_on_a_band_edge_is_inside_it(): + """Delta review of #599: 110 / 100 - 1 is 0.10000000000000009.""" + s = summarise(_log({"edge": (110, 100), "quarter": (125, 100)}, local=LOCAL)) + assert s["national"] == {"edge": 0, "quarter": 1} + + +def test_a_repeated_local_target_is_refused(): + log = _log({"a": (1, 1)}, local=[(1, 1)]) + with pytest.raises(ValueError, match="repeats a local target"): + summarise(pd.concat([log, log[log.name != "UK"]])) + + +def test_added_targets_do_not_count_against_the_existing_ones(): + """Sol review of #599, P2: 100 unchanged targets plus 3 poor new ones pass.""" + base = _summary([0.0] * 100) + head = summarise( + _log( + { + **{f"t{i}": (100, 100) for i in range(100)}, + **{f"new{i}": (200, 100) for i in range(3)}, + }, + LOCAL, + ) + ) + result = compare(base, head, POLICY) + assert result["status"] == "pass" + assert result["summary"]["national"]["compared"] == 100 + assert "3 national targets are new: new0, new1, new2" in result["notes"] + + +@pytest.mark.parametrize( + "head_log", + [ + _log({"a": (1, 1)}, local=()), # only national rows + _log({}, local=LOCAL), # only local rows + _log({"a": (1, 0)}, local=[(1, 0)]), # only zero targets + ], +) +def test_an_empty_target_group_fails(head_log): + """Sol review of #599, P2: empty groups used to give NaN shares and pass.""" + result = compare(_summary([0.0] * 10), summarise(head_log), POLICY) + assert result["status"] == "fail" + assert "has no" in result["failures"][0] + + +def test_an_incomplete_last_epoch_fails(): + log = _log({f"t{i}": (100, 100) for i in range(10)}, LOCAL) + log = log[~((log.epoch == 30) & (log.metric == "t9"))] + result = compare(_summary([0.0] * 10), summarise(log), POLICY) + assert result["status"] == "fail" and "incomplete" in result["failures"][0] + + +def test_builds_of_different_epochs_cannot_be_compared(): + base = _summary([0.0], epochs=(0, 30)) + assert ( + compare(base, _summary([0.0], epochs=(0, 510, 511)), POLICY)["status"] == "fail" + ) + # 32 epochs log last at 30, or at 31 once the final epoch is saved (#594). + result = compare(base, _summary([0.0], epochs=(0, 30, 31)), POLICY) + assert result["status"] == "pass" + assert "saves the final epoch" in result["notes"][0] + + +errors = st.lists(st.floats(0, 3), min_size=1, max_size=40) + + +@settings(max_examples=200, deadline=None) +@given(national=errors, local=errors) +def test_a_build_compared_with_itself_passes_with_nothing_flagged(national, local): + s = _summary(national, local=[(100 * (1 + e), 100) for e in local]) + result = compare(s, s, POLICY) + assert result["status"] == "pass" and result["flags"] == [] + assert all(row["change"] == 0 for row in result["summary"].values()) + + +@settings(max_examples=200, deadline=None) +@given(base=errors, data=st.data()) +def test_improving_every_target_never_fails_and_changes_are_antisymmetric(base, data): + better = [data.draw(st.floats(0, e)) for e in base] + forward = compare(_summary(base), _summary(better), POLICY) + assert forward["status"] == "pass" and forward["flags"] == [] + backward = compare(_summary(better), _summary(base), POLICY) + for name in ("national", "local"): + assert forward["summary"][name]["change"] == pytest.approx( + -backward["summary"][name]["change"] + ) + + +def test_the_shipped_policy_loads(): + policy = load_policy() + assert policy.share_drop_allowance == 0.02 + + +def test_the_commands_summarise_and_exit_one_on_a_failure(tmp_path): + base_log, head_log = tmp_path / "base.csv", tmp_path / "head.csv" + targets = {f"t{i}": (100, 100) for i in range(50)} + _log(targets, LOCAL).to_csv(base_log, index=False) + _log(dict(targets, **{f"t{i}": (200, 100) for i in range(5)}), LOCAL).to_csv( + head_log, index=False + ) + for log in (base_log, head_log): + assert ( + main(["summarise", str(log), "--out", str(log.with_suffix(".json"))]) == 0 + ) + out = tmp_path / "fit.md" + code = main( + [ + "compare", + str(tmp_path / "base.json"), + str(tmp_path / "head.json"), + "--markdown", + str(out), + ] + ) + assert code == 1 + assert "## Calibration fit against the base build: fail" in out.read_text() + assert ( + main(["compare", str(tmp_path / "base.json"), str(tmp_path / "base.json")]) == 0 + ) + + +def test_the_pull_request_workflow_uploads_only_summaries(): + """The datasets are licensed microdata, and raw calibration logs stay on the runner.""" + from pathlib import Path + + import yaml + + workflow = Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml" + jobs = yaml.safe_load(workflow.read_text())["jobs"] + uploads = sorted( + step["with"]["path"] + for job in jobs.values() + for step in job.get("steps", []) + if step.get("uses", "").startswith("actions/upload-artifact") + ) + assert uploads == ["base_fit_summary.json", "fit_summary.json"] + for job in jobs.values(): + for step in job.get("steps", []): + assert not step.get("uses", "").startswith("actions/cache"), step + assert jobs["fit-comparison"]["needs"] == ["test", "base-build"] + post = next( + s + for s in jobs["fit-comparison"]["steps"] + if s.get("name", "").startswith("Post") + ) + assert "if [ ! -s fit.md ]" in post["run"] diff --git a/policyengine_uk_data/utils/fit_report.py b/policyengine_uk_data/utils/fit_report.py new file mode 100644 index 000000000..b4db12d7d --- /dev/null +++ b/policyengine_uk_data/utils/fit_report.py @@ -0,0 +1,293 @@ +"""Final calibration fit of a build, summarised safely, and the comparison of two. + +A build writes ``constituency_calibration_log.csv``: each target's estimate +and value every tenth epoch and at the last one, nationally (rows named +``UK``) and for each constituency. The log stays on the runner that built +it. ``summarise`` reduces its last epoch to what the comparison needs, and +only the summary is uploaded: for every national and local target, the band +its error |estimate / target - 1| falls in (``BANDS``: within 10%, 10-25%, +25-50%, 50-100%, over 100%), with zero targets left out. No estimate or +error leaves the runner. An +estimate built from a few households, or dominated by one, divided by a +published count, could give that household's value back, and the dataset +repeats households (output-area clones, SPI and CGT donors), so a count of +rows can't show how many respondents support a figure (Sol re-review of +#599). A band, at least 10% of a public target wide, is a bound, not a +value. + +``compare`` checks that a pull request does not worsen the fit of the build +it would replace, built the same way (same environment, epochs and clones), +using the policy in ``targets/fit_policy.yaml``: + +- **Fail** when the share of national, or of local, targets within 10% + falls by more than the allowance, over the targets both builds have. + Added and removed targets are reported, not counted. +- **Flag**, for review, each national target whose band is worse on the + head than on the base. Per-target moves do not fail while builds of the + same code move 2 to 12 national targets to a worse band (#598). + + python -m policyengine_uk_data.utils.fit_report summarise LOG.csv --out summary.json + python -m policyengine_uk_data.utils.fit_report compare BASE.json HEAD.json [--markdown out.md] +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import numpy as np +import pandas as pd +import yaml + +POLICY_PATH = Path(__file__).parents[1] / "targets" / "fit_policy.yaml" +SCHEMA = "policyengine-uk-data-fit-summary/3" +# Upper edges of the error bands; band 0 is within 10%. +BAND_EDGES = (0.10, 0.25, 0.50, 1.00) +BANDS = ("within 10%", "10-25%", "25-50%", "50-100%", "over 100%") +_COLUMNS = ["name", "index", "metric", "estimate", "target", "epoch"] + + +@dataclass(frozen=True) +class FitPolicy: + share_drop_allowance: float + + +def load_policy(path=None) -> FitPolicy: + with open(path or POLICY_PATH) as f: + raw = yaml.safe_load(f) + if raw.get("schema_version") != 1: + raise ValueError("fit policy: schema_version must be 1") + if raw.get("national_target", {}).get("flag") != "worse_band": + raise ValueError("fit policy: national_target.flag must be worse_band") + return FitPolicy( + share_drop_allowance=float(raw["share_within_10pct"]["allowed_drop"]), + ) + + +def band(error: float) -> int: + """The index in ``BANDS`` of an error: 0 when within 10% (inclusive). + + The error is rounded to 9 decimal places first, so an estimate exactly + 10% off (110 / 100 - 1 = 0.10000000000000009 in floating point) is + within 10%. + """ + return int(np.searchsorted(BAND_EDGES, round(float(error), 9), side="left")) + + +def summarise(log) -> dict: + """The last epoch of a calibration log, reduced to what may leave the runner.""" + if isinstance(log, pd.DataFrame): + df = log + else: + df = pd.read_csv(log, usecols=_COLUMNS) + if df.empty: + raise ValueError("the calibration log has no rows") + epochs = sorted(df.epoch.unique()) + last = int(epochs[-1]) + rows = df[df.epoch == last] + previous = int((df.epoch == epochs[-2]).sum()) if len(epochs) > 1 else None + national = rows[rows.name == "UK"] + if national.metric.duplicated().any(): + raise ValueError( + "the calibration log repeats a national target at its last epoch" + ) + national = national[national.target != 0].set_index("metric") + local = rows[(rows.name != "UK") & (rows.target != 0)] + if local.duplicated(["index", "metric"]).any(): + raise ValueError("the calibration log repeats a local target at its last epoch") + n_errors = (national.estimate / national.target - 1).abs() + l_errors = (local.estimate / local.target - 1).abs() + if not np.isfinite(n_errors).all() or not np.isfinite(l_errors).all(): + raise ValueError("the calibration log has non-finite estimates") + return { + "schema": SCHEMA, + "epoch": last, + "rows_last_epoch": int(len(rows)), + "rows_previous_epoch": previous, + "national": {metric: band(error) for metric, error in n_errors.items()}, + "local": { + f"{i}|{m}": band(e) + for i, m, e in zip(local["index"], local.metric, l_errors) + }, + } + + +def _share(flags: list) -> float: + return float(np.mean(flags)) if flags else float("nan") + + +def compare(base: dict, head: dict, policy: FitPolicy) -> dict: + """Failures, flags and the summary for a head build against its base.""" + failures, notes = [], [] + for side, s in (("base", base), ("head", head)): + if s.get("schema") != SCHEMA: + failures.append(f"The {side} fit summary is missing or malformed.") + continue + if not s["national"] or not s["local"]: + missing = "national" if not s["national"] else "local" + failures.append(f"The {side} build's last epoch has no {missing} targets.") + previous = s["rows_previous_epoch"] + if previous and s["rows_last_epoch"] < previous: + failures.append( + f"The {side} build's last logged epoch ({s['epoch']}) has fewer rows " + f"({s['rows_last_epoch']}) than the one before ({previous}): the log " + "is incomplete." + ) + if failures: + return { + "status": "fail", + "failures": failures, + "notes": [], + "flags": [], + "summary": {}, + "targets_compared": 0, + } + # A log's last epoch is a multiple of ten or, since #594, the final one; + # the same epoch count gives last epochs less than ten apart. + if abs(base["epoch"] - head["epoch"]) >= 10: + failures.append( + f"The builds' logs end at epochs {base['epoch']} and {head['epoch']}, so " + "they ran different epoch counts and their fits cannot be compared." + ) + elif base["epoch"] != head["epoch"]: + notes.append( + f"The base log ends at epoch {base['epoch']} and the head's at " + f"{head['epoch']}: one of them saves the final epoch (#594)." + ) + summary = {} + for name in ("national", "local"): + b, h = base[name], head[name] + common = sorted(set(b) & set(h)) + added, removed = sorted(set(h) - set(b)), sorted(set(b) - set(h)) + before = _share([b[k] == 0 for k in common]) + after = _share([h[k] == 0 for k in common]) + summary[name] = { + "base": before, + "head": after, + "change": after - before, + "compared": len(common), + } + if added: + notes.append( + f"{len(added)} {name} targets are new: {', '.join(added[:10])}" + ) + if removed: + notes.append( + f"{len(removed)} {name} targets are gone: {', '.join(removed[:10])}" + ) + if not common: + failures.append(f"The builds have no {name} targets in common.") + elif after < before - policy.share_drop_allowance: + failures.append( + f"The share of {name} targets within 10% falls from {before:.1%} to " + f"{after:.1%}, over the {len(common)} targets both builds have, by " + f"more than the {policy.share_drop_allowance:.1%} allowed." + ) + common = sorted(set(base["national"]) & set(head["national"])) + b_band, h_band = base["national"], head["national"] + flags = [ + {"target": t, "base": BANDS[b_band[t]], "head": BANDS[h_band[t]]} + for t in common + if h_band[t] > b_band[t] + ] + # Largest moves first. + flags.sort( + key=lambda f: (BANDS.index(f["base"]) - BANDS.index(f["head"]), f["target"]) + ) + better = sum(1 for t in common if h_band[t] < b_band[t]) + if better: + notes.append(f"{better} national targets are in a better band on the head.") + return { + "status": "fail" if failures else "pass", + "failures": failures, + "notes": notes, + "flags": flags, + "summary": summary, + "targets_compared": summary["national"]["compared"], + } + + +def render(result: dict, policy: FitPolicy) -> str: + lines = [f"## Calibration fit against the base build: {result['status']}", ""] + if result["summary"]: + lines += [ + "| Targets within 10% (those both builds have) | Base | Head | Change |", + "|---|---:|---:|---:|", + ] + for name, row in result["summary"].items(): + lines.append( + f"| {name.capitalize()} ({row['compared']:,}) | {row['base']:.1%} " + f"| {row['head']:.1%} | {100 * row['change']:+.1f}pp |" + ) + lines += ["", f"A drop of more than {policy.share_drop_allowance:.1%} fails."] + if result["failures"]: + lines += ["", "### Failures", ""] + [f"- {f}" for f in result["failures"]] + if result["notes"]: + lines += ["", "### Notes", ""] + [f"- {n}" for n in result["notes"]] + flags = result["flags"] + lines += [ + "", + f"### National targets in a worse error band on the head ({len(flags)})", + "", + ] + if flags: + lines += ["| Target | Base | Head |", "|---|---|---|"] + lines += [ + f"| `{f['target']}` | {f['base']} | {f['head']} |" for f in flags[:30] + ] + if len(flags) > 30: + lines.append(f"\n{len(flags) - 30} more in the job log.") + else: + lines.append("None.") + lines += [ + "", + "Flags are for review: builds of the same code move 2 to 12 national " + "targets to a worse band (#598). Bands are of |estimate / target - 1|; " + "no target's estimate or error is published.", + ] + return "\n".join(lines) + "\n" + + +def main(argv: Optional[list] = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m policyengine_uk_data.utils.fit_report" + ) + sub = parser.add_subparsers(dest="command", required=True) + s = sub.add_parser("summarise", help="reduce a calibration log to a safe summary") + s.add_argument("log") + s.add_argument("--out", required=True) + c = sub.add_parser( + "compare", help="compare a head build's fit summary with its base's" + ) + c.add_argument("base") + c.add_argument("head") + c.add_argument("--markdown", help="write the comparison here as markdown") + c.add_argument("--policy", help="fit policy YAML (default targets/fit_policy.yaml)") + args = parser.parse_args(argv) + if args.command == "summarise": + summary = summarise(args.log) + Path(args.out).write_text(json.dumps(summary)) + print( + f"epoch {summary['epoch']}: {len(summary['national'])} national, " + f"{len(summary['local'])} local targets" + ) + return 0 + policy = load_policy(args.policy) + base = json.loads(Path(args.base).read_text()) + head = json.loads(Path(args.head).read_text()) + result = compare(base, head, policy) + text = render(result, policy) + if args.markdown: + Path(args.markdown).write_text(text) + print(text) + for flag in result["flags"]: + print(f"flag: {flag['target']} {flag['base']} -> {flag['head']}") + return 1 if result["status"] == "fail" else 0 + + +if __name__ == "__main__": + sys.exit(main())