From d0390f5440753458cb81e97d40b24923307076cb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 18:34:36 -0400 Subject: [PATCH 1/7] Build council_tax before CTR and stop imputing reported CTR on SPI rows council_tax is policyengine-uk's gross liability, but the FRS CTANNUAL is "Annual CT amount after discounts/reduction" (DWP DV summary 2024-25), so every reported CTR recipient entered the model with its own reduction already taken off, and full-CTR recipients with a zero bill. derive_council_tax adds CTREBAMT back for CTREB = 1 households, imputes missing bills from non-recipients only, and in Scotland nets water and sewerage off with the 2024-25 gross charges (CWATAMT1, CSEWAMT1; CSEWAMT is blank that year), at 65% of gross for recipients, matching the Water Charges Reduction Scheme maximum. Earlier releases keep the previous Scottish netting. Reported CTR is a household amount held on the reference person, and policyengine-uk reads any positive value as a claim. The stage-2 QRF spread it over people on SPI-synthetic rows, rising with income, so it is now set to zero there instead of imputed. Fixes #496 Fixes #497 Co-Authored-By: Claude Opus 5.5 --- changelog.d/ctr-gross-council-tax.fixed.md | 2 + policyengine_uk_data/datasets/frs.py | 187 ++++++++----- .../datasets/imputations/frs_only.py | 28 +- .../tests/test_council_tax_gross_bill.py | 259 ++++++++++++++++++ .../tests/test_frs_only_imputation.py | 28 ++ .../tests/test_legacy_benefit_proxies.py | 11 +- pyproject.toml | 5 +- uv.lock | 73 ++++- 8 files changed, 524 insertions(+), 69 deletions(-) create mode 100644 changelog.d/ctr-gross-council-tax.fixed.md create mode 100644 policyengine_uk_data/tests/test_council_tax_gross_bill.py diff --git a/changelog.d/ctr-gross-council-tax.fixed.md b/changelog.d/ctr-gross-council-tax.fixed.md new file mode 100644 index 000000000..dc72e8a1c --- /dev/null +++ b/changelog.d/ctr-gross-council-tax.fixed.md @@ -0,0 +1,2 @@ +- Build `council_tax` as the bill after discounts and before council tax reduction, as policyengine-uk expects: add reported CTR (CTREBAMT) back to the FRS CTANNUAL, which is net of it, impute missing bills from non-recipients only, and in Scotland net off water and sewerage using the 2024-25 gross charges (uk-data#496). +- Stop imputing reported council tax reduction onto SPI-synthetic rows; set it to zero there (uk-data#497). diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index e440d4869..dac0644b7 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -85,6 +85,16 @@ FRS_APPROVED_TRAINING_CODES = tuple(range(1, 10)) UNKNOWN_QUALIFYING_EDUCATION_OR_TRAINING_ENTRY_AGE = 1000 +SCOTLAND_GVTREGNO = 12 +# From FRS 2024-25, CWATAMT1 and CSEWAMT1 are DWP derived variables, the +# "Weeklyised gross annual dom. water/sew. charge on bill" (DV summary +# 2024-25). Earlier releases carry interview answers under the same names, +# populated for only about half of Scottish households. +FIRST_SURVEY_YEAR_WITH_GROSS_SCOTTISH_WATER_CHARGES = 2024 +# Scottish Water Charges Reduction Scheme: at most a 35% reduction in water +# and sewerage charges for households receiving council tax reduction. +SCOTTISH_WATER_CHARGES_MAXIMUM_REDUCTION = 0.35 + @lru_cache(maxsize=None) def load_legacy_jobseeker_max_annual_hours(year: int) -> int: @@ -525,6 +535,118 @@ def split_reported_education_grants( FRS_RELEASE_FOLDER_PATTERN = re.compile(r"^frs_(\d{4})_(\d{2})$") +def derive_council_tax(household: pd.DataFrame, year: int) -> np.ndarray: + """Annual council tax bill after discounts and before council tax reduction. + + policyengine-uk's ``council_tax`` is the gross liability that its council + tax reduction (CTR) formulas reduce, so it must not already be net of CTR. + The FRS CTANNUAL is net of it: the 2024-25 DV summary labels it "Annual + CT amount after discounts/reduction", derived from inputs including CTREB + and CTREBAMT, and the interview question it replaced asked for the amount + payable "after deducting any discounts or reduction". In 2022-23 to + 2024-25, recipients' CTANNUAL plus CTREBAMT (weekly) x 365.25/7 matches + the mean bill of non-recipients in the same region, band and single-adult + cell. So: + + - A household reporting a reduction (CTREB = 1) gets CTANNUAL plus its + annualised CTREBAMT, including when CTANNUAL is 0 (a full reduction). + If CTREBAMT is missing or not positive, the reduction is unknown and + the bill is the larger of CTANNUAL and the imputed cell mean. + - A missing or negative CTANNUAL is imputed as the mean bill of + non-recipients with a positive bill in the same (region, band, + single-adult) cell, or 0 if the cell has none (Northern Ireland, which + has no council tax, always has none). + - Other households keep CTANNUAL. + + In Scotland CTANNUAL also includes water and sewerage charges, which are + not council tax and which CTR does not cover, so they are netted off + first. From 2024-25 the netting uses the gross charges CWATAMT1 and + CSEWAMT1 (CSEWAMT is blank that year). Measured on the 2024-25 release: + + - Non-recipients: netting the full gross charges leaves a status-discount + (25%) bill at 0.75 of the undiscounted bill in every band, so DWP's + derivation applies the status discount to council tax only. The + discount is therefore not applied to the charges here. + - Recipients: the recipient identity above holds (1.00 overall, 0.98-1.02 + by band) when 65% of the gross charges are netted, and falls to 0.83 + when all of them are. This matches the Water Charges Reduction Scheme + maximum of 35%, so recipients' charges are netted at 65% of gross. + + Earlier releases keep the previous netting of CSEWAMT plus CWATAMTD, the + discounted charges. Without CTREB or CTREBAMT (neither is used here if + absent), the result is the previous CTANNUAL-based bill. + + Args: + household: Raw FRS household table with lower-case column names, one + row per household. + year: FRS survey year (2024 for FRS 2024-25). + + Returns: + Annual council tax per household, in the order of ``household``. + """ + + def column(name: str) -> pd.Series: + if name in household.columns: + return pd.to_numeric(household[name], errors="coerce") + return pd.Series(np.nan, index=household.index) + + ctannual = column("ctannual") + region = column("gvtregno") + band = column("ctband") + single_adult = column("adulth") == 1 + reports_reduction = column("ctreb") == 1 + weekly_reduction = column("ctrebamt") + reduction_known = reports_reduction & (weekly_reduction > 0) + + in_scotland = region == SCOTLAND_GVTREGNO + if year >= FIRST_SURVEY_YEAR_WITH_GROSS_SCOTTISH_WATER_CHARGES and { + "cwatamt1", + "csewamt1", + }.issubset(household.columns): + gross_weekly_charges = column("cwatamt1").clip(lower=0).fillna(0) + column( + "csewamt1" + ).clip(lower=0).fillna(0) + share_in_bill = np.where( + reports_reduction, 1 - SCOTTISH_WATER_CHARGES_MAXIMUM_REDUCTION, 1 + ) + weekly_charges = gross_weekly_charges * share_in_bill + else: + weekly_charges = column("csewamt").clip(lower=0).fillna(0) + column( + "cwatamtd" + ).clip(lower=0).fillna(0) + scottish_charges = np.where(in_scotland, weekly_charges * WEEKS_IN_YEAR, 0) + council_tax_only = (ctannual - scottish_charges).clip(lower=0) + + # Cell means from non-recipients only: a recipient's CTANNUAL is net of + # its reduction and would pull the imputed gross bill down. + in_pool = (ctannual > 0) & ~reports_reduction + cell_keys = [region, band, single_adult] + cell_mean = ( + council_tax_only[in_pool] + .groupby([key[in_pool] for key in cell_keys], dropna=False) + .mean() + ) + imputed = ( + cell_mean.reindex(pd.MultiIndex.from_arrays(cell_keys)).fillna(0).to_numpy() + ) + + missing_bill = ctannual.isna() | (ctannual < 0) + council_tax = np.select( + [ + missing_bill, + reports_reduction & ~reduction_known, + reduction_known, + ], + [ + imputed, + np.maximum(council_tax_only, imputed), + council_tax_only + weekly_reduction * WEEKS_IN_YEAR, + ], + default=council_tax_only, + ) + return np.nan_to_num(council_tax, nan=0.0) + + def survey_year_from_frs_folder_name(raw_frs_folder) -> int | None: """Survey year encoded in an FRS release folder name (``frs_2024_25`` -> 2024). @@ -931,68 +1053,9 @@ def determine_education_level(fted_val, typeed2_val, age_val): household.typeacc, 1, range(1, 8), ACCOMMODATIONS ).values - # Impute Council Tax - - # In Scotland, council tax bills are collected together with Scottish - # Water and sewerage charges, and the FRS CTANNUAL variable includes - # them. Net them off (they are weekly variables; CTANNUAL is annual) so - # council_tax is tax only: the water charges are already captured - # separately in water_and_sewerage_charges, so leaving them in both - # double-counts them and overstates Scottish council tax by roughly - # £500 per household (~25% of the Scottish total). - SCOTLAND_GVTREGNO = 12 - scottish_water_annual = pd.Series( - np.where( - household.gvtregno == SCOTLAND_GVTREGNO, - ( - np.maximum(household.csewamt.fillna(0), 0) - + np.maximum(household.cwatamtd.fillna(0), 0) - ) - * (365.25 / 7), - 0, - ), - index=household.index, - ) - ctannual_tax_only = np.maximum(household.ctannual - scottish_water_annual, 0) - - # Only ~25% of household report Council Tax bills - use - # these to build a model to impute missing values - CT_valid = household.ctannual > 0 - - # Find the mean reported Council Tax bill for a given - # (region, CT band, is-single-person-household) triplet - region = household.gvtregno[CT_valid] - band = household.ctband[CT_valid] - single_person = (household.adulth == 1)[CT_valid] - ctannual = ctannual_tax_only[CT_valid] - - # Build the table - ct_mean = ctannual.groupby([region, band, single_person], dropna=False).mean() - ct_mean = ct_mean.replace(-1, ct_mean.mean()) - - # For every household consult the table to find the imputed - # Council Tax bill - pairs = household.set_index( - [household.gvtregno, household.ctband, (household.adulth == 1)] - ) - hh_CT_mean = pd.Series(index=pairs.index) - has_mean = pairs.index.isin(ct_mean.index) - hh_CT_mean[has_mean] = ct_mean[pairs.index[has_mean]].values - hh_CT_mean[~has_mean] = 0 - ct_imputed = hh_CT_mean - - # For households which originally reported Council Tax, - # use the reported value. Otherwise, use the imputed value - council_tax = pd.Series( - np.where( - # 2018 FRS uses blanks for missing values, 2019 FRS - # uses -1 for missing values - (household.ctannual < 0) | household.ctannual.isna(), - np.maximum(ct_imputed, 0).values, - ctannual_tax_only, - ) - ) - pe_household["council_tax"] = council_tax.fillna(0) + # Council tax: the bill after discounts and before council tax + # reduction, with Scottish water and sewerage charges netted off. + pe_household["council_tax"] = derive_council_tax(household, year) BANDS = ["A", "B", "C", "D", "E", "F", "G", "H", "I"] # Band 1 is the most common pe_household["council_tax_band"] = ( diff --git a/policyengine_uk_data/datasets/imputations/frs_only.py b/policyengine_uk_data/datasets/imputations/frs_only.py index 242fbf247..3669ddfef 100644 --- a/policyengine_uk_data/datasets/imputations/frs_only.py +++ b/policyengine_uk_data/datasets/imputations/frs_only.py @@ -95,13 +95,30 @@ "incapacity_benefit_reported", "maternity_allowance_reported", "winter_fuel_allowance_reported", - "council_tax_benefit_reported", "jsa_contrib_reported", "jsa_income_reported", "esa_contrib_reported", "esa_income_reported", ] +# FRS-only person variables set to zero on SPI-donor rows instead of being +# imputed. +# +# ``council_tax_benefit_reported`` is the household's reported council tax +# reduction (FRS CTREBAMT), which the FRS build puts on the household +# reference person only. A person-level QRF on personal incomes cannot +# reproduce that. It put amounts on two or three people in some households. +# On the 2024-25 build, imputed receipt also rose with household income on +# these rows, while reported receipt in the FRS falls steeply with it. +# policyengine-uk does not treat the amount as noise: any positive value +# makes the benefit unit claim CTR (``would_claim_council_tax_reduction``), +# and where it has no CTR scheme for the household, +# ``council_tax_benefit`` is the reported amount itself. At zero, CTR on +# these rows comes only from the model's own eligibility and take-up. +SPI_DONOR_ZEROED_PERSON_VARIABLES = [ + "council_tax_benefit_reported", +] + def _one_hot_encode(df: pd.DataFrame, columns: list[str]) -> pd.DataFrame: """Return ``df`` with object-typed ``columns`` one-hot encoded. @@ -177,12 +194,17 @@ def impute_frs_only_variables( to predict values for every row of ``target_dataset``; predictions replace the existing (donor-leaked) values in ``FRS_ONLY_PERSON_VARIABLES`` only. Variables absent from either - frame are skipped silently. + frame are skipped silently. ``SPI_DONOR_ZEROED_PERSON_VARIABLES`` + are set to zero on ``target_dataset`` rather than imputed. """ from policyengine_uk_data.utils.qrf import QRF target_dataset = target_dataset.copy() + for column in SPI_DONOR_ZEROED_PERSON_VARIABLES: + if column in target_dataset.person.columns: + target_dataset.person[column] = 0.0 + train_person = train_dataset.person target_person = target_dataset.person @@ -203,7 +225,7 @@ def impute_frs_only_variables( if not outputs: logger.warning( "Stage-2 FRS-only imputation: no output variables available; " - "returning target_dataset unchanged." + "returning target_dataset without imputed values." ) return target_dataset diff --git a/policyengine_uk_data/tests/test_council_tax_gross_bill.py b/policyengine_uk_data/tests/test_council_tax_gross_bill.py new file mode 100644 index 000000000..d8105c15c --- /dev/null +++ b/policyengine_uk_data/tests/test_council_tax_gross_bill.py @@ -0,0 +1,259 @@ +"""Council tax is the bill after discounts and before council tax reduction. + +The FRS CTANNUAL is net of the reported reduction (CTREB, CTREBAMT), and in +Scotland it also carries water and sewerage charges. ``derive_council_tax`` +adds the reported reduction back and nets the Scottish charges off. The +fixtures below are synthetic households, not survey records. +""" + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.datasets.frs import ( + SCOTLAND_GVTREGNO, + SCOTTISH_WATER_CHARGES_MAXIMUM_REDUCTION, + WEEKS_IN_YEAR, + derive_council_tax, +) + +LONDON = 7 +WALES = 11 +NORTHERN_IRELAND = 13 +BAND_C = 3 +RECIPIENT, NON_RECIPIENT = 1, 2 + + +def _households(rows: list[dict]) -> pd.DataFrame: + """Raw-FRS-shaped household table; unspecified fields are blank.""" + columns = [ + "gvtregno", + "ctband", + "adulth", + "ctannual", + "ctreb", + "ctrebamt", + "cwatamt1", + "csewamt1", + "cwatamtd", + "csewamt", + ] + table = pd.DataFrame(rows).reindex(columns=columns).astype(float) + table.index = pd.RangeIndex(100, 100 + len(table), name="household_id") + return table + + +def _cell(region, ctannual, ctreb=NON_RECIPIENT, ctrebamt=np.nan, adults=2, **kw): + return dict( + gvtregno=region, + ctband=BAND_C, + adulth=adults, + ctannual=ctannual, + ctreb=ctreb, + ctrebamt=ctrebamt, + **kw, + ) + + +def test_full_reduction_recipient_with_zero_bill_gets_gross_bill(): + households = _households( + [_cell(LONDON, 0.0, RECIPIENT, 30.0), _cell(LONDON, 1_600.0)] + ) + council_tax = derive_council_tax(households, 2024) + assert council_tax[0] == pytest.approx(30.0 * WEEKS_IN_YEAR) + + +def test_partial_reduction_recipient_gets_bill_plus_reduction(): + households = _households( + [_cell(WALES, 900.0, RECIPIENT, 12.0), _cell(WALES, 1_700.0)] + ) + council_tax = derive_council_tax(households, 2024) + assert council_tax[0] == pytest.approx(900.0 + 12.0 * WEEKS_IN_YEAR) + + +def test_non_recipients_keep_their_bill_including_zero(): + households = _households( + [ + _cell(LONDON, 1_500.0), + _cell(LONDON, 0.0), # e.g. an exempt dwelling + _cell(WALES, 1_200.0, adults=1), + ] + ) + council_tax = derive_council_tax(households, 2024) + np.testing.assert_allclose(council_tax, [1_500.0, 0.0, 1_200.0]) + + +def test_recipient_with_unknown_reduction_gets_non_recipient_cell_mean(): + households = _households( + [ + _cell(LONDON, 0.0, RECIPIENT, np.nan), + _cell(LONDON, 0.0, RECIPIENT, 0.0), + _cell(LONDON, 1_900.0, RECIPIENT, np.nan), + _cell(LONDON, 1_400.0), + _cell(LONDON, 1_600.0), + ] + ) + council_tax = derive_council_tax(households, 2024) + # The cell mean (1,500), or the reported bill if that is larger: the + # bill before the reduction cannot be below the bill after it. + np.testing.assert_allclose(council_tax[:3], [1_500.0, 1_500.0, 1_900.0]) + + +def test_missing_bills_are_imputed_from_non_recipients_only(): + households = _households( + [ + _cell(LONDON, np.nan), + _cell(LONDON, -1.0), + _cell(LONDON, 1_400.0), + _cell(LONDON, 1_600.0), + # A recipient's net bill must not pull the imputed bill down. + _cell(LONDON, 200.0, RECIPIENT, 25.0), + # A different cell (single adult) must not enter the mean. + _cell(LONDON, 1_125.0, adults=1), + ] + ) + council_tax = derive_council_tax(households, 2024) + np.testing.assert_allclose(council_tax[:2], [1_500.0, 1_500.0]) + + +def test_northern_ireland_has_no_council_tax(): + households = _households([_cell(NORTHERN_IRELAND, np.nan, RECIPIENT, 10.0)]) + assert derive_council_tax(households, 2024)[0] == 0.0 + + +def test_scottish_gross_water_and_sewerage_are_netted_from_2024(): + water, sewerage = 4.0, 5.0 + gross_charges = (water + sewerage) * WEEKS_IN_YEAR + households = _households( + [ + _cell(SCOTLAND_GVTREGNO, 2_000.0, cwatamt1=water, csewamt1=sewerage), + _cell( + SCOTLAND_GVTREGNO, + 900.0, + RECIPIENT, + 10.0, + cwatamt1=water, + csewamt1=sewerage, + ), + ] + ) + council_tax = derive_council_tax(households, 2024) + assert council_tax[0] == pytest.approx(2_000.0 - gross_charges) + # Recipients' CTANNUAL carries the charges after the Water Charges + # Reduction Scheme's maximum 35% reduction. + recipient_charges = gross_charges * (1 - SCOTTISH_WATER_CHARGES_MAXIMUM_REDUCTION) + assert council_tax[1] == pytest.approx( + 900.0 - recipient_charges + 10.0 * WEEKS_IN_YEAR + ) + + +def test_scotland_before_2024_keeps_discounted_charge_netting(): + households = _households( + [ + _cell( + SCOTLAND_GVTREGNO, + 2_000.0, + cwatamtd=3.0, + csewamt=4.0, + cwatamt1=9.0, + csewamt1=9.0, + ) + ] + ) + council_tax = derive_council_tax(households, 2023) + assert council_tax[0] == pytest.approx(2_000.0 - 7.0 * WEEKS_IN_YEAR) + + +def test_charges_are_not_netted_outside_scotland(): + households = _households([_cell(LONDON, 2_000.0, cwatamt1=9.0, csewamt1=9.0)]) + assert derive_council_tax(households, 2024)[0] == pytest.approx(2_000.0) + + +def test_without_reduction_columns_the_bill_is_ctannual(): + households = _households( + [_cell(LONDON, 700.0, RECIPIENT, 20.0), _cell(LONDON, np.nan)] + ).drop(columns=["ctreb", "ctrebamt"]) + council_tax = derive_council_tax(households, 2020) + np.testing.assert_allclose(council_tax, [700.0, 700.0]) + + +# Property-based checks over arbitrary small household tables. + +_amount = st.one_of( + st.none(), + st.just(-1.0), + st.just(0.0), + st.floats(min_value=0.0, max_value=5_000.0), +) +_weekly = st.one_of(st.none(), st.just(0.0), st.floats(min_value=0.0, max_value=100.0)) +_household = st.fixed_dictionaries( + dict( + gvtregno=st.sampled_from([1, LONDON, WALES, SCOTLAND_GVTREGNO, 13]), + ctband=st.one_of(st.none(), st.integers(1, 9)), + adulth=st.integers(1, 4), + ctannual=_amount, + ctreb=st.one_of(st.none(), st.sampled_from([RECIPIENT, NON_RECIPIENT])), + ctrebamt=_weekly, + cwatamt1=_weekly, + csewamt1=_weekly, + cwatamtd=_weekly, + csewamt=_weekly, + ) +) + + +@settings(max_examples=300, deadline=None) +@given( + rows=st.lists(_household, min_size=1, max_size=12), + year=st.sampled_from([2022, 2023, 2024, 2025]), +) +def test_council_tax_invariants(rows, year): + households = _households(rows) + council_tax = derive_council_tax(households, year) + ctannual = households.ctannual.to_numpy() + reduction = households.ctrebamt.to_numpy() + recipient = (households.ctreb == 1).to_numpy() + known_reduction = recipient & (households.ctrebamt > 0).to_numpy() + has_bill = ctannual >= 0 + scotland = (households.gvtregno == SCOTLAND_GVTREGNO).to_numpy() + + # One finite, non-negative bill per household. + assert council_tax.shape == (len(households),) + assert np.isfinite(council_tax).all() and (council_tax >= 0).all() + + # Non-recipients with a bill outside Scotland keep it exactly. + keep = ~recipient & has_bill & ~scotland + np.testing.assert_allclose(council_tax[keep], ctannual[keep]) + + # A known reduction is added back in full, so recipients outside + # Scotland pay CTANNUAL plus it. + add_back = known_reduction & has_bill & ~scotland + np.testing.assert_allclose( + council_tax[add_back], + ctannual[add_back] + reduction[add_back] * WEEKS_IN_YEAR, + ) + + # The bill before the reduction is never below the bill after it. + no_charges = has_bill & recipient & ~scotland + assert (council_tax[no_charges] >= ctannual[no_charges] - 1e-9).all() + + # Recomputing on a subset of rows cannot change the kept bills (the + # cell means only feed missing bills and unknown reductions). + council_tax_keep_only = derive_council_tax(households[keep], year) + np.testing.assert_allclose(council_tax_keep_only, council_tax[keep]) + + +@settings(max_examples=200, deadline=None) +@given(rows=st.lists(_household, min_size=1, max_size=12)) +def test_recipients_do_not_move_the_imputed_bill(rows): + """Changing recipients' bills never changes a non-recipient's result.""" + households = _households(rows) + recipient = (households.ctreb == 1).to_numpy() + changed = households.copy() + changed.loc[recipient, "ctannual"] = 4_321.0 + np.testing.assert_allclose( + derive_council_tax(households, 2024)[~recipient], + derive_council_tax(changed, 2024)[~recipient], + ) diff --git a/policyengine_uk_data/tests/test_frs_only_imputation.py b/policyengine_uk_data/tests/test_frs_only_imputation.py index 1961274e4..4ae6abd98 100644 --- a/policyengine_uk_data/tests/test_frs_only_imputation.py +++ b/policyengine_uk_data/tests/test_frs_only_imputation.py @@ -258,3 +258,31 @@ def predict(self, x): assert not result.person["is_severely_disabled_for_benefits"].any() assert (result.person["pip_dl_category"] == "NONE").all() assert (result.person["pip_m_category"] == "NONE").all() + + +def test_spi_donor_rows_get_zero_reported_council_tax_reduction(): + """Reported CTR is zeroed on SPI-donor rows, not imputed (uk-data#497). + + Even when every training person reports CTR, the target rows must end + with none: the stage-2 QRF would otherwise spread a household-level + amount over individual people. + """ + from policyengine_uk_data.datasets.imputations.frs_only import ( + FRS_ONLY_PERSON_VARIABLES, + SPI_DONOR_ZEROED_PERSON_VARIABLES, + impute_frs_only_variables, + ) + + assert "council_tax_benefit_reported" in SPI_DONOR_ZEROED_PERSON_VARIABLES + assert not set(SPI_DONOR_ZEROED_PERSON_VARIABLES) & set(FRS_ONLY_PERSON_VARIABLES) + + train = _fake_dataset(person_rows=400, seed=0) + train.person["council_tax_benefit_reported"] = 1_000.0 + target = _fake_dataset(person_rows=60, seed=1) + target.person["council_tax_benefit_reported"] = 750.0 + + result = impute_frs_only_variables(train_dataset=train, target_dataset=target) + + assert (result.person["council_tax_benefit_reported"] == 0).all() + # The training (FRS) rows keep their reported amounts. + assert (train.person["council_tax_benefit_reported"] == 1_000.0).all() diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py index 5f1acd85f..fc02379b6 100644 --- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py +++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py @@ -479,7 +479,8 @@ def fake_read_csv(path, *args, **kwargs): "csewamt": 0, "ctannual": 0, "ctband": 1, - "ctrebamt": 0, + "ctreb": 1, + "ctrebamt": 10, "cwatamtd": 0, "gross4": 0, "gvtregno": 1, @@ -561,3 +562,11 @@ def fake_read_csv(path, *args, **kwargs): ].iloc[0] assert dataset.person["education_grants"].iloc[0] == 100 assert dataset.person["disabled_students_allowance_eligible_expenses"].iloc[0] == 0 + # A full council tax reduction (CTANNUAL 0, CTREBAMT £10 a week) leaves + # the bill before the reduction in council_tax (uk-data#496). + weekly_reduction_annualised = 10 * frs_module.WEEKS_IN_YEAR + assert dataset.household["council_tax"].iloc[0] == weekly_reduction_annualised + assert ( + dataset.person["council_tax_benefit_reported"].iloc[0] + == weekly_reduction_annualised + ) diff --git a/pyproject.toml b/pyproject.toml index beff7f1a4..7973343cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,8 +45,9 @@ dev = [ "yaml-changelog>=0.1.7", "itables", "quantile-forest", - "build", "towncrier>=24.8.0", - + "build", + "towncrier>=24.8.0", + "hypothesis>=6.168.3", ] [tool.setuptools] diff --git a/uv.lock b/uv.lock index a3e9f44d9..672737ea1 100644 --- a/uv.lock +++ b/uv.lock @@ -577,6 +577,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/f4/124858007ddf3c61e9b144107304c9152fa80b5b6c168da07d86fe583cc1/huggingface_hub-1.1.5-py3-none-any.whl", hash = "sha256:e88ecc129011f37b868586bbcfae6c56868cae80cd56a79d61575426a3aa0d7d", size = 516000, upload-time = "2025-11-20T15:49:30.926Z" }, ] +[[package]] +name = "hypothesis" +version = "6.168.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/b7/13118bbc45d6d8b9d04e2de779e2a4ff23145ea39efa692b994fb874ca72/hypothesis-6.168.3.tar.gz", hash = "sha256:a43388f9067678fef6e13bdff325b6cfa6961a590498bb37f7ff31589c83bc75", size = 511022, upload-time = "2026-09-28T05:20:58.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/cd/746a2fc1e5e5ba34f07ea6ee1ad07525a3dde5ff4836db3a7980af49a891/hypothesis-6.168.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d20972ca134e652e9928ecd200966a8a2857adc2812c1d74b12a872e5cd503eb", size = 791536, upload-time = "2026-09-28T05:18:57.162Z" }, + { url = "https://files.pythonhosted.org/packages/c3/0f/7a158e377b69556c8e12c25c8fd811d0103e54c24a12dab1f3b9819f202e/hypothesis-6.168.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:eafbec09d3e87d13d8242411d1f5868b5e879f1bd6d95e233e9ff80c527bea1c", size = 787313, upload-time = "2026-09-28T05:20:18.122Z" }, + { url = "https://files.pythonhosted.org/packages/45/c6/7df9104ee359e8fcd77791781c47e70794964ca69673665e0de2a6590f4c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73c5627497968cc62d140e9fde1ea21e12cac7b64dc419fea8286cd34ff1ad4e", size = 1124036, upload-time = "2026-09-28T05:20:04.372Z" }, + { url = "https://files.pythonhosted.org/packages/92/0f/8a61715404a73b9a82cb1f976803428d603cdea976a9c30266c72ad6a7ce/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dc56e6dc6eeb0aeb08ac463f847279bde2336c4786faf7ee0af4b93cd031a7", size = 1147875, upload-time = "2026-09-28T05:19:15.495Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3f/2b16e95cc3b9a069afa23830a012aefe377ce4a457a227e5a5026eb827ff/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:753bb501f8560d2e321ed3b62596b4496c56f15668b0a0669231ccc3e6c4e80d", size = 1149489, upload-time = "2026-09-28T05:19:18.886Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e7/d7cf6dc068bb02732b2a6e38a35a0dcb7f2137608198fc79d740f69d197c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71ab606c472449cb872ef2a7acaec679ee4f651b7f0a477a04ebc85d8933ef8c", size = 1191992, upload-time = "2026-09-28T05:19:21.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/f1e5b25dec15998e8375d722825ec3dae1075cbe7cb1f74221b2312a2e33/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26928956c54748e4dfa587333741ab750246123b93484955646ff316cca27eef", size = 1169911, upload-time = "2026-09-28T05:19:27.996Z" }, + { url = "https://files.pythonhosted.org/packages/03/36/901234a49147e6bf8a5b9e54b775706a223c2b52befea1106ea2aaf15d48/hypothesis-6.168.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:0bdfc53c041b61c854fa3761735736b991bc2bddafee45a361cfe4d43027c1ef", size = 1129406, upload-time = "2026-09-28T05:18:42.632Z" }, + { url = "https://files.pythonhosted.org/packages/55/c1/5fc9ff91ec9fba6bb527833386c66812524b5cc75fe7d4a4f9122ee1c420/hypothesis-6.168.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:650528e2b1e2a45e95c2df624d4b9364b8ac5a027ba84e1eea2bc5398dd01dcd", size = 1160399, upload-time = "2026-09-28T05:20:23.911Z" }, + { url = "https://files.pythonhosted.org/packages/9b/d2/49ad1ef5c55547c8abfd0fc571b3493806dda530fd5cb3d8263dbad86a72/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:209dc54cdb1b4d6d7020ad8d09e44c49756361b7183adacea4d6f5705085b595", size = 1299906, upload-time = "2026-09-28T05:18:58.417Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5f/f98a26094c4f462c4215807bed0f9a6b5508b27a3c329bcba1bc8adc7ac7/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:af8ca98cd11dc7f9427bb90483abcd9688d4df2e8e63893a30a2423b027ebb11", size = 1425538, upload-time = "2026-09-28T05:20:49.074Z" }, + { url = "https://files.pythonhosted.org/packages/d0/b8/c5b4406e3a6591e51a42f85469351d1d824e0a54b84167aec9b2353759fe/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:f8122cfdd0bc0ba843063effa22ac310bdebdb3a1319bf1e63f14baa119c72ab", size = 1377084, upload-time = "2026-09-28T05:20:06.618Z" }, + { url = "https://files.pythonhosted.org/packages/e0/01/0d84a8ea469d024c15f602799c0b8410fbe03d99fdb0370449b33c3c916f/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4bedbb379eab34f792af7ee9a05aae04e9c08bbb52e0f90f5a2110d4fe4b2fbd", size = 1281162, upload-time = "2026-09-28T05:19:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/22/b5/ea6435038de1a795f005cb603531e8ccc053febc50180f3ab55583c9c60e/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:5a953115b9f5c95133ab2d04efffeec96e5658c3207923dca7285f7db3e6bbef", size = 1300433, upload-time = "2026-09-28T05:18:55.634Z" }, + { url = "https://files.pythonhosted.org/packages/6c/78/fa6c77d9f64b6d592e69e582d83cbce7d7e56897faa4762885a2123b0166/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6173558e676ad25ed1e20507fa4024a0d816dd90f715f77006e5a28f19109026", size = 1336273, upload-time = "2026-09-28T05:19:08.273Z" }, + { url = "https://files.pythonhosted.org/packages/f1/56/4acd0d778bab818c9ea336fcba768bf43bc70db7f84cc373579298df4641/hypothesis-6.168.3-cp310-abi3-win32.whl", hash = "sha256:dc66390fb12d80585aa9222bf538ce8b7aa22cf5d118250647355a1c9e8f62f4", size = 678211, upload-time = "2026-09-28T05:19:23.355Z" }, + { url = "https://files.pythonhosted.org/packages/b0/db/0a02b146ad1c30f9716362f2e1a379dfd6b0f7be03ac1e22c9da24c9e2a9/hypothesis-6.168.3-cp310-abi3-win_amd64.whl", hash = "sha256:92325b276360fe86c5bf71a568c0d53a6d140b0de36dcf029f9164a17803bb24", size = 684906, upload-time = "2026-09-28T05:18:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c0/958deaf726848f96f52250740bf39f13f476b068e22f73e2b358eaa07532/hypothesis-6.168.3-cp310-abi3-win_arm64.whl", hash = "sha256:3cf6f1eeaf41cd8d60cf1f88fde905ca1dd77c906929a507d6ac7f66f2ccba2a", size = 683292, upload-time = "2026-09-28T05:19:57.404Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a2/6787da846d929e52fc3344d803299c45782dbeac287528af08380e984bf8/hypothesis-6.168.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1b230a850de63334c16654a34a2d547e0179d36b9071d4439b3e7237f6d077e7", size = 793254, upload-time = "2026-09-28T05:19:36.294Z" }, + { url = "https://files.pythonhosted.org/packages/89/ba/2893f5ca42501f4d562cba3229c8994c3a3e5fac66ef60c1e0e0b5220a5d/hypothesis-6.168.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d63b0226cd3e0d8bdd97c3384b22a21934ed4d53246c1c93575dada616672499", size = 784699, upload-time = "2026-09-28T05:18:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/b7/aa/7d7349daf75b71f6f35876f8de115e974c5c04d600e86df1b2779b0883ab/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0369f5df055f96e117ab12e5f249668ff144731ab5280bc7a205fdf81f990b89", size = 1123016, upload-time = "2026-09-28T05:20:36.372Z" }, + { url = "https://files.pythonhosted.org/packages/00/f0/7774e1ea072708ea46cb25c4aeee5f9978b6c271764809069e8a6789858e/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f076bcd0f77fdcdb7797826c879099573d03a02e228ebe649ea917081b962ac0", size = 1168989, upload-time = "2026-09-28T05:20:08.44Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7d/113992abed9efbd7944e3a496a382da58152dce126fddf6419ff31a0a57d/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b823ba1fcec8da730f29316d010b06d3f7e0c3828dcf91020e24c55e7d24652a", size = 1298626, upload-time = "2026-09-28T05:19:40.986Z" }, + { url = "https://files.pythonhosted.org/packages/d5/65/3659fa5e733027e5b37a27486e57f4053535e40853d23501c422bb275043/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dd2849c269d674e4618590f3b48d443bd4c06b5aef3d8d869086c2e6d213d248", size = 1335184, upload-time = "2026-09-28T05:20:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/b4/6c/35aab2221b5ea65125340f236e1a765a9e9f3b28d89a389ea0033fed29f7/hypothesis-6.168.3-cp313-cp313-win_amd64.whl", hash = "sha256:3ef7d26f5789e691401d5f87eafed9bd2763f0dbe47af6d6b66012a509404766", size = 682173, upload-time = "2026-09-28T05:19:12.762Z" }, + { url = "https://files.pythonhosted.org/packages/6e/79/27b0cb56ff5d2bc92458bf6fcebdb1f0dc01f57f043a178e9f9d74498169/hypothesis-6.168.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2d4c68729a13df9af4998d2652cfb5d541c5609b88c306880a5dfb284938ec2", size = 793287, upload-time = "2026-09-28T05:20:38.522Z" }, + { url = "https://files.pythonhosted.org/packages/e8/bd/5342f95c3bc36586777ef8cb8eba87ac7acf0184210327fb0e6d8654e295/hypothesis-6.168.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b345f818083ec99966a43ca4f7b38feb62c6920ce28572bc2bde4948a8b7eaba", size = 784857, upload-time = "2026-09-28T05:20:42.609Z" }, + { url = "https://files.pythonhosted.org/packages/4a/5f/fc774241518a0588d362680a3084275058aab86d6bc02a2c61e1073142d0/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0608c610fc002978fc5de0f471770d8817e9455cb8649a47983c9f8bccfc1897", size = 1123330, upload-time = "2026-09-28T05:19:29.735Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e6/131f16775a3dca5f4fe27f0d6ad9a9851600098a54a872d163f6ebf0b664/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dcf6448b1ecc37f2b23f2d1b3ddfc9ff6b6910614c15a642dc82f419b96037b", size = 1169178, upload-time = "2026-09-28T05:19:53.436Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/c9f5bff8b73321c9fd12f2d69666c6e4861b97b60f5f24fdd6ac293007fd/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f9f094041dcce5119ebd7bab71062743ab02b6e654d056b370beda78c19e2", size = 1299127, upload-time = "2026-09-28T05:20:14.486Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/d4618f8ab12dd76c4d58ea052252759aa2bf0c4e71f4b502ba5d577e0b9f/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d479985fe73af97badfb72dc6d20c6a353e486a36f6d065f600026e0cf954a86", size = 1335390, upload-time = "2026-09-28T05:20:10.326Z" }, + { url = "https://files.pythonhosted.org/packages/95/b6/727161e17cc297df1aeea945de05c532033b66751625bec192bc5b5ff707/hypothesis-6.168.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:785e2c45f8c08e274b4bf1ccae97f1a4e09407e9a68e80790cfab1d17a4a45fa", size = 624291, upload-time = "2026-09-28T05:18:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/77/23/2f9b506392a0b8712440bcb781abc5713be9ffac1303b6d10f3669050882/hypothesis-6.168.3-cp314-cp314-win_amd64.whl", hash = "sha256:320920b1e3dae8611eee8a03d063cf2187446f2a17c38cfb8a7fc1466f71eee2", size = 682064, upload-time = "2026-09-28T05:20:16.336Z" }, + { url = "https://files.pythonhosted.org/packages/50/93/efabfd95eb2b69c0c9fa1e1c83c8290aa2715d46baeb5a7764faf09c133a/hypothesis-6.168.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:35380baa981108a7f60c4eab71e46acd8d8f58440520346a1a6aba06dca7e074", size = 791875, upload-time = "2026-09-28T05:19:37.789Z" }, + { url = "https://files.pythonhosted.org/packages/1b/fc/2a0ada1623a9a33048bbf9f00a7c92513398c6b8865cba5efb854262ce30/hypothesis-6.168.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f0aaaed00438fa6673d856aaee12a4a8afbde82a9f3c5ff487cacfb064239afe", size = 783412, upload-time = "2026-09-28T05:18:48.266Z" }, + { url = "https://files.pythonhosted.org/packages/41/31/73c615d37eeb12da0209555612c32ced628d860267c23cfd2691c6207aea/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27e6df1576bf838e7f484d4ae5cba92114497ca30afb917056bf1ea33e95675", size = 1121662, upload-time = "2026-09-28T05:19:26.225Z" }, + { url = "https://files.pythonhosted.org/packages/92/54/1893344a3b8bbf83fa7c9bb7e96f6836580213854f46671cb749f7b4706c/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bc85014577982ec6d2e266edc5cd6e7a7d3674c648791ca983c67a52f89a0c4", size = 1167761, upload-time = "2026-09-28T05:18:49.85Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/a61d82968febd25a0f08ffbbfabfaeead66f68fc5fa73f3c9ece1c36fc8f/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ccfc29505aa1cdcc254cb9cd701fd0811fef5480addd97d4127a00df023410f7", size = 1297309, upload-time = "2026-09-28T05:19:51.711Z" }, + { url = "https://files.pythonhosted.org/packages/bc/01/fe8e4cf6d39d02f4efadaa351427fa13722086cfc54a2da3579741c5177a/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32d0699566aaa93f9e97a44705d7164386f1e91de78d277f53bada35e78bcc96", size = 1334260, upload-time = "2026-09-28T05:19:00.121Z" }, + { url = "https://files.pythonhosted.org/packages/39/6a/09177ebc62f4778f94379ecade6a51299d3d8c4f4b75ba636bf3a0202364/hypothesis-6.168.3-cp314-cp314t-win_amd64.whl", hash = "sha256:28d88fa174ecbd4ecbd7bb290f06d0db084a3971c3f511ae2830b65b5e25500f", size = 681992, upload-time = "2026-09-28T05:19:48.043Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f6/890bf33d608cd63348d3146a3ca83362fbf09e589c5e5230a00c403070f4/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:61f5782d807b1e6aa5c9beef1054cb2037e7d1add48a64972cd6ee447778781f", size = 791231, upload-time = "2026-09-28T05:18:43.998Z" }, + { url = "https://files.pythonhosted.org/packages/03/dd/fc36b204f8aa7437c676576d9437f0422de46aa8a729e6cd5bbb0224e759/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:7aacf3cf40c7ce8f9e4924d5b57bc0b068beafdfed2347bcf160a28b4cfbba7b", size = 783171, upload-time = "2026-09-28T05:19:20.353Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d2/3cd087d577db67c404991d7723e64d7cd75570cdc82a0f7329aed6f2086c/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01768a03a30dc54df7fe457c0b34016c84d598fa00d5965ededab93ba4eb3408", size = 1121152, upload-time = "2026-09-28T05:18:39.095Z" }, + { url = "https://files.pythonhosted.org/packages/42/6f/a05a68cc66e7a22701c50bf94fb9ddeb36f4f5d0de3cc130066709eda621/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1a8a4ffc6c6e6e577f2bfbcfebf7cffbb310283ba2532c729570a0a752cebaaa", size = 1144069, upload-time = "2026-09-28T05:20:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f4/fdd7a093fb2fb3a0ce24256c92ad5bf67936f4669bd06d5ceae4298150ec/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b987d73eba95183a7e59cca6d1925c588aa4307922d9852cc2b4d282a2ae4128", size = 1146692, upload-time = "2026-09-28T05:20:25.83Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/6dbd4377e935505ae4fc8ee4c7b18c69994c4bbaca015447c470218bdbee/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4569c39bd97d9573e946429ed676f3b55a7c8ed80920addd67d384a47d59b38", size = 1189480, upload-time = "2026-09-28T05:20:46.734Z" }, + { url = "https://files.pythonhosted.org/packages/ba/bf/ff288b496b690000d2686dcfa7f67855e4c5a6dddb464ed8e4880be83bb2/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6042b8707a4b25bbbfe20b110258fb7549a68951e5525608a8ce06091039e9fc", size = 1167109, upload-time = "2026-09-28T05:19:24.758Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d1/3fee2bc29fc747fabbb27e174515592e390809ca465d3bbbabbcfa3235a8/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:f413629de94d38a7a2ad259697a6752526143cb49c2e2d7bdba19381c80693f6", size = 1126823, upload-time = "2026-09-28T05:18:52.636Z" }, + { url = "https://files.pythonhosted.org/packages/58/6a/585294392fa6a6d9719446a042a777ba98ecca265d9e48cbedc10c88df0f/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f071737e4e775bebba07e319eb645a880d1e0186b4d24bad23255e849d86d483", size = 1155773, upload-time = "2026-09-28T05:20:51.268Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4d/fa283ff79996debf1ff08593e01f8b773eb8211b19b3d6d5d491317a65bf/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:f59a3912858d0609c26054aa1c474847c797937f5e0e1c77f0d3f08032e50f09", size = 1296671, upload-time = "2026-09-28T05:18:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/42/05/98f9c2f628da5afabb6c3e8df5d2d299d2d50bf306b14a6e26d3954c4951/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f09c05a23a8025dd5cad07c2ed299a46778e72d49ff0dd5bf353bcc0f7dc1580", size = 1422001, upload-time = "2026-09-28T05:19:04.188Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c9/f2109ade29a7ec284d55bca328d784743e8b13e321576fd34ee7e65f99af/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:44ace770bda3a0301739fc5a413c764de790c739df1f1a7218dd049f8594d9f5", size = 1374182, upload-time = "2026-09-28T05:20:27.764Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6d/e05d5f72441564a3bebc71fa155deafd0fd3b015d6014ec8a00edf6e42bb/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:03b131043608f94a2578a2a896a1a72092acb7079815eef5b8513b08447e0e62", size = 1278402, upload-time = "2026-09-28T05:20:56.332Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/6988a7f1f69c5c530687dce03a32c157e2d878e3a2e32af9269ce016b78a/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:e9784aca26eddfe99b03a0292320db742cc8a74200ef864fdce949f526f973cd", size = 1297770, upload-time = "2026-09-28T05:19:11.387Z" }, + { url = "https://files.pythonhosted.org/packages/a2/28/5bb82b60b836bd2329e1fe01ad94efc2b14dfa806f8e50ed14b795d78770/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:2b52ac363096232bebc2add117e9178d91f4248f4cdb919fd1026b5f86a4bb16", size = 1333977, upload-time = "2026-09-28T05:20:02.483Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7d/d419841b8f65481ea1a50c4ba36f2670d48b8c7a51e4e8586089564cf7f6/hypothesis-6.168.3-cp315-abi3.abi3t-win32.whl", hash = "sha256:d28e3a6b511a74ce37df5274b51f36c2b274fea7365e00b16f0c81e22acd5957", size = 675394, upload-time = "2026-09-28T05:19:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d2/1de6a2ad100e44817f2e9a8e8ba3eeaf4769621f4f87f2d4966c4971fcc4/hypothesis-6.168.3-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:7b9638789548361a57d984f56619ac694a914c328181d911271618409261ff4a", size = 681699, upload-time = "2026-09-28T05:19:09.957Z" }, + { url = "https://files.pythonhosted.org/packages/c0/79/be3370fd02734d6b1d950183ae9224580d19fa8d356b95348d60f1e3eeb7/hypothesis-6.168.3-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:65d78e4357ec48ed2c67825f06740ee3599be4cfe770a6092bed07108d679ac5", size = 679849, upload-time = "2026-09-28T05:19:06.884Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1364,7 +1433,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.56.16" +version = "1.57.4" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1392,6 +1461,7 @@ dependencies = [ dev = [ { name = "build" }, { name = "furo" }, + { name = "hypothesis" }, { name = "itables" }, { name = "l0-python" }, { name = "pytest" }, @@ -1410,6 +1480,7 @@ requires-dist = [ { name = "google-auth" }, { name = "google-cloud-storage" }, { name = "huggingface-hub" }, + { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.168.3" }, { name = "itables", marker = "extra == 'dev'" }, { name = "l0-python", marker = "extra == 'dev'", specifier = ">=0.4.0" }, { name = "microcalibrate", specifier = ">=0.18.0" }, From 872fdd381ee7304c32b503269461029a6030523f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 18:36:44 -0400 Subject: [PATCH 2/7] Clarify derive_council_tax docstring: England and Wales identity, #467, no-CTREB fallback Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/frs.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index dac0644b7..2e3baa2ed 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -543,10 +543,10 @@ def derive_council_tax(household: pd.DataFrame, year: int) -> np.ndarray: The FRS CTANNUAL is net of it: the 2024-25 DV summary labels it "Annual CT amount after discounts/reduction", derived from inputs including CTREB and CTREBAMT, and the interview question it replaced asked for the amount - payable "after deducting any discounts or reduction". In 2022-23 to - 2024-25, recipients' CTANNUAL plus CTREBAMT (weekly) x 365.25/7 matches - the mean bill of non-recipients in the same region, band and single-adult - cell. So: + payable "after deducting any discounts or reduction". In England and + Wales in 2022-23 to 2024-25, recipients' CTANNUAL plus CTREBAMT (weekly) + x 365.25/7 matches the mean bill of non-recipients in the same region, + band and single-adult cell. So: - A household reporting a reduction (CTREB = 1) gets CTANNUAL plus its annualised CTREBAMT, including when CTANNUAL is 0 (a full reduction). @@ -560,8 +560,10 @@ def derive_council_tax(household: pd.DataFrame, year: int) -> np.ndarray: In Scotland CTANNUAL also includes water and sewerage charges, which are not council tax and which CTR does not cover, so they are netted off - first. From 2024-25 the netting uses the gross charges CWATAMT1 and - CSEWAMT1 (CSEWAMT is blank that year). Measured on the 2024-25 release: + first. (They belong in ``water_and_sewerage_charges``, which is still + zero for Scotland in 2024-25: uk-data#467.) From 2024-25 the netting uses + the gross charges CWATAMT1 and CSEWAMT1, as CSEWAMT is blank that year. + Measured on the 2024-25 release: - Non-recipients: netting the full gross charges leaves a status-discount (25%) bill at 0.75 of the undiscounted bill in every band, so DWP's @@ -569,12 +571,13 @@ def derive_council_tax(household: pd.DataFrame, year: int) -> np.ndarray: discount is therefore not applied to the charges here. - Recipients: the recipient identity above holds (1.00 overall, 0.98-1.02 by band) when 65% of the gross charges are netted, and falls to 0.83 - when all of them are. This matches the Water Charges Reduction Scheme - maximum of 35%, so recipients' charges are netted at 65% of gross. + when all of them are. This matches a flat reduction at the Water + Charges Reduction Scheme's 35% maximum, so recipients' charges are + netted at 65% of gross. Earlier releases keep the previous netting of CSEWAMT plus CWATAMTD, the - discounted charges. Without CTREB or CTREBAMT (neither is used here if - absent), the result is the previous CTANNUAL-based bill. + discounted charges. A table without a CTREB column has no recipients, so + it gets the previous CTANNUAL-based bill. Args: household: Raw FRS household table with lower-case column names, one From be96fa5632de11bb9867b56564ab4283b6acb7d6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 18:39:17 -0400 Subject: [PATCH 3/7] Give the Scottish recipient identity range across bands and discount groups Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/frs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 2e3baa2ed..5c55603e7 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -570,8 +570,8 @@ def derive_council_tax(household: pd.DataFrame, year: int) -> np.ndarray: derivation applies the status discount to council tax only. The discount is therefore not applied to the charges here. - Recipients: the recipient identity above holds (1.00 overall, 0.98-1.02 - by band) when 65% of the gross charges are netted, and falls to 0.83 - when all of them are. This matches a flat reduction at the Water + across bands and discount groups) when 65% of the gross charges are + netted, and falls to 0.83 when all of them are. This matches a flat reduction at the Water Charges Reduction Scheme's 35% maximum, so recipients' charges are netted at 65% of gross. From a66eae17b1987e918caec9592ed3b712ffa31cf8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 18:44:21 -0400 Subject: [PATCH 4/7] Ignore the Hypothesis example database Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9a741bc49..5c1ab2385 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ **/_build !policyengine_uk_data/storage/*.csv **/version.json + +# Hypothesis example database (property-based tests) +.hypothesis/ From cdab7e2229634de5b50aa3ce3c68db93eac27855 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 08:49:09 -0400 Subject: [PATCH 5/7] Impute household CTR claims while clearing SPI reports --- .gitignore | 3 + changelog.d/ctr-gross-council-tax.fixed.md | 2 +- policyengine_uk_data/datasets/frs.py | 13 +- .../imputations/council_tax_reduction.py | 236 ++++++++++++++++ .../datasets/imputations/frs_only.py | 26 +- .../tests/test_council_tax_gross_bill.py | 22 ++ .../tests/test_ctr_award_bound.py | 130 +++++++++ .../tests/test_frs_only_imputation.py | 49 +++- .../test_household_council_tax_reduction.py | 267 ++++++++++++++++++ .../tests/test_legacy_benefit_proxies.py | 28 ++ .../tests/test_spi_donor_benefit_rules.py | 20 +- pyproject.toml | 1 + uv.lock | 26 +- 13 files changed, 806 insertions(+), 17 deletions(-) create mode 100644 policyengine_uk_data/datasets/imputations/council_tax_reduction.py create mode 100644 policyengine_uk_data/tests/test_ctr_award_bound.py create mode 100644 policyengine_uk_data/tests/test_household_council_tax_reduction.py diff --git a/.gitignore b/.gitignore index 70d8396d3..4cc5a31ff 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz # Hypothesis example database (property-based tests) .hypothesis/ + +# Local adoption-job notes and tooling +.job/ diff --git a/changelog.d/ctr-gross-council-tax.fixed.md b/changelog.d/ctr-gross-council-tax.fixed.md index dc72e8a1c..bf88c50bd 100644 --- a/changelog.d/ctr-gross-council-tax.fixed.md +++ b/changelog.d/ctr-gross-council-tax.fixed.md @@ -1,2 +1,2 @@ - Build `council_tax` as the bill after discounts and before council tax reduction, as policyengine-uk expects: add reported CTR (CTREBAMT) back to the FRS CTANNUAL, which is net of it, impute missing bills from non-recipients only, and in Scotland net off water and sewerage using the 2024-25 gross charges (uk-data#496). -- Stop imputing reported council tax reduction onto SPI-synthetic rows; set it to zero there (uk-data#497). +- Stop imputing reported council tax reduction onto SPI-synthetic rows; set it to zero and impute household receipt from FRS joint incomes and composition, providing a reference-person benefit-unit claim route for modeled CTR (uk-data#497, #514). diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index b1ceac9fb..0c83f48f2 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -119,6 +119,7 @@ FRS_RENTPROF_LOSS = 2 SCOTLAND_GVTREGNO = 12 +NORTHERN_IRELAND_GVTREGNO = 13 # From FRS 2024-25, CWATAMT1 and CSEWAMT1 are DWP derived variables, the # "Weeklyised gross annual dom. water/sew. charge on bill" (DV summary # 2024-25). Earlier releases carry interview answers under the same names, @@ -775,7 +776,8 @@ def derive_council_tax(household: pd.DataFrame, year: int) -> np.ndarray: - A missing or negative CTANNUAL is imputed as the mean bill of non-recipients with a positive bill in the same (region, band, single-adult) cell, or 0 if the cell has none (Northern Ireland, which - has no council tax, always has none). + has no council tax, always has none). A known reported reduction is + a lower bound on a missing gross bill in Great Britain. - Other households keep CTANNUAL. In Scotland CTANNUAL also includes water and sewerage charges, which are @@ -867,7 +869,14 @@ def column(name: str) -> pd.Series: ], default=council_tax_only, ) - return np.nan_to_num(council_tax, nan=0.0) + # A missing bill must still cover a known reported award; otherwise + # unsupported schemes can pay more reported CTR than the gross bill. + reported_floor = np.where( + reduction_known & (region != NORTHERN_IRELAND_GVTREGNO), + weekly_reduction * WEEKS_IN_YEAR, + 0.0, + ) + return np.maximum(np.nan_to_num(council_tax, nan=0.0), reported_floor) def survey_year_from_frs_folder_name(raw_frs_folder) -> int | None: diff --git a/policyengine_uk_data/datasets/imputations/council_tax_reduction.py b/policyengine_uk_data/datasets/imputations/council_tax_reduction.py new file mode 100644 index 000000000..4605d912a --- /dev/null +++ b/policyengine_uk_data/datasets/imputations/council_tax_reduction.py @@ -0,0 +1,236 @@ +"""Learn household CTR receipt without manufacturing reported awards. + +SPI incomes replace personal income records, while CTR is a household award. +A separate household receipt model learns from the original FRS and supplies +an explicit claim flag only to the household reference person's benefit unit. +PolicyEngine then calculates eligibility and the award from current policy. + +An alternative is to impute the household award itself. That would require an +award override, or synthetic reported CTR, and would make the imputed amount +less responsive to reforms. Receipt probabilities retain the FRS income +gradient without choosing a uniform take-up rate or a majority-label cutoff. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +from policyengine_uk.data import UKSingleYearDataset + +HOUSEHOLD_CTR_INCOME_PREDICTORS = ( + "employment_income", + "self_employment_income", + "savings_interest_income", + "dividend_income", + "private_pension_income", + "property_income", +) +HOUSEHOLD_CTR_RECEIPT_TARGET = "household_receives_council_tax_reduction" +HOUSEHOLD_CTR_MODEL_SEED = 42 +HOUSEHOLD_CTR_DRAW_SEED = 102 + + +def _require_columns(frame: pd.DataFrame, name: str, columns) -> None: + missing = sorted(set(columns) - set(frame.columns)) + if missing: + raise KeyError(f"Household CTR imputation needs {name} columns: {missing}") + + +def _require_unique_ids(frame: pd.DataFrame, column: str) -> None: + if frame[column].isna().any() or frame[column].duplicated().any(): + raise ValueError(f"Household CTR imputation needs unique, non-null {column}") + + +def _household_predictors(dataset: UKSingleYearDataset) -> pd.DataFrame: + """Aggregate persons by household ID, independently of table row order. + + Adults are people aged 18 or older; children are younger than 18. For a + household with no adult, oldest adult age falls back to its oldest member. + """ + person, household = dataset.person, dataset.household + _require_columns( + person, + "person", + ("person_household_id", "age", *HOUSEHOLD_CTR_INCOME_PREDICTORS), + ) + _require_columns( + household, + "household", + ("household_id", "region", "tenure_type", "council_tax"), + ) + _require_unique_ids(household, "household_id") + household_ids = pd.Index(household.household_id, name="household_id").sort_values() + person_households = person.person_household_id + if not person_households.isin(household_ids).all(): + raise ValueError("Household CTR imputation found a person without a household") + if not household_ids.isin(person_households).all(): + raise ValueError("Household CTR imputation found a household without a person") + + incomes = person[list(HOUSEHOLD_CTR_INCOME_PREDICTORS)].astype(float).fillna(0) + if not np.isfinite(incomes.to_numpy()).all(): + raise ValueError("Household CTR imputation needs finite personal incomes") + frame = incomes.groupby(person_households).sum().reindex(household_ids) + + ages = person.age.astype(float) + if not np.isfinite(ages.to_numpy()).all(): + raise ValueError("Household CTR imputation needs finite ages") + adults = ages.ge(18) + frame["adult_count"] = adults.groupby(person_households).sum() + frame["child_count"] = (~adults).groupby(person_households).sum() + frame["oldest_adult_age"] = ( + ages.where(adults) + .groupby(person_households) + .max() + .fillna(ages.groupby(person_households).max()) + ) + + household_by_id = household.set_index("household_id") + gross_tax = household_by_id.council_tax.astype(float).fillna(0) + if not np.isfinite(gross_tax.to_numpy()).all(): + raise ValueError("Household CTR imputation needs finite gross council tax") + frame["council_tax"] = gross_tax.clip(lower=0) + for column in ("region", "tenure_type"): + frame[column] = household_by_id[column].fillna("UNKNOWN").astype(str) + return frame + + +def _reference_person_benunits(dataset: UKSingleYearDataset) -> pd.Series: + """Return one benefit-unit ID per household, using explicit entity keys.""" + person, benunit = dataset.person, dataset.benunit + _require_columns(person, "person", ("person_benunit_id", "is_household_head")) + _require_columns(benunit, "benunit", ("benunit_id",)) + _require_unique_ids(benunit, "benunit_id") + if not person.person_benunit_id.isin(benunit.benunit_id).all(): + raise ValueError( + "Household CTR imputation found a person without a benefit unit" + ) + if not benunit.benunit_id.isin(person.person_benunit_id).all(): + raise ValueError( + "Household CTR imputation found a benefit unit without a person" + ) + if person.groupby("person_benunit_id").person_household_id.nunique().gt(1).any(): + raise ValueError( + "Household CTR imputation found a benefit unit spanning households" + ) + if not person.is_household_head.isin((False, True)).all(): + raise ValueError("Household CTR imputation needs boolean is_household_head") + heads = person.loc[ + person.is_household_head.astype(bool), + ["person_household_id", "person_benunit_id"], + ] + head_counts = heads.groupby("person_household_id").size() + if ( + not head_counts.reindex(dataset.household.household_id, fill_value=0) + .eq(1) + .all() + ): + raise ValueError( + "Household CTR imputation needs exactly one household reference person " + "per household" + ) + return heads.set_index("person_household_id").person_benunit_id + + +def _receipt_probabilities(fitted_model, inputs: pd.DataFrame) -> pd.Series: + """Read the binary classifier probabilities exposed by MicroImpute QRF. + + The locked QRF implementation uses a random-forest classifier for binary + targets. Its default predictions are majority labels, so request the class + probabilities and draw receipt separately with our own local generator. + """ + prediction = fitted_model.predict(inputs, return_probs=True) + if isinstance(prediction, pd.DataFrame): + # MicroImpute returns a frame directly for constant training targets. + if not prediction.index.equals(inputs.index): + raise ValueError("Household CTR predictions lost their household IDs") + probabilities = prediction[HOUSEHOLD_CTR_RECEIPT_TARGET].astype(float) + else: + frames = [ + value for value in prediction.values() if isinstance(value, pd.DataFrame) + ] + if len(frames) != 1 or not frames[0].index.equals(inputs.index): + raise ValueError("Household CTR predictions lost their household IDs") + info = prediction["probabilities"][HOUSEHOLD_CTR_RECEIPT_TARGET] + classes = np.asarray(info["classes"]) + class_probabilities = np.asarray(info["probabilities"]) + if class_probabilities.shape != (len(inputs), len(classes)): + raise ValueError( + "Household CTR classifier returned invalid probability dimensions" + ) + probabilities = pd.Series( + class_probabilities[:, classes == True].sum(axis=1), + index=inputs.index, + ) + if ( + not np.isfinite(probabilities.to_numpy()).all() + or not probabilities.between(0, 1).all() + ): + raise ValueError( + "Household CTR classifier returned invalid receipt probabilities" + ) + return probabilities + + +def impute_household_council_tax_reduction( + train_dataset: UKSingleYearDataset, + target_dataset: UKSingleYearDataset, +) -> UKSingleYearDataset: + """Return SPI households with learned CTR claim flags, leaving reports alone. + + Training uses the original FRS, before income imputation. The target is + the SPI-donor subsample after income imputation and reported CTR zeroing. + Its predictors sum the six first-stage income components and include + household composition, region, tenure and the already-built gross bill. + Receipt is learned from a positive household sum of reported FRS CTR. + + Household IDs are sorted before fitting and drawing to keep the seeded + results stable when input frames are shuffled. Each household can claim + only through its reference person's benefit unit, and a zero gross bill + forces non-receipt. Explicit flags on every target benefit unit leave + eligibility and the award calculation to the current policy model. + """ + from microimpute.models import QRF + + train_inputs = _household_predictors(train_dataset) + target_inputs = _household_predictors(target_dataset) + reference_benunits = _reference_person_benunits(target_dataset) + _require_columns( + train_dataset.person, "training person", ("council_tax_benefit_reported",) + ) + reported = train_dataset.person.council_tax_benefit_reported.astype(float).fillna(0) + if not np.isfinite(reported.to_numpy()).all(): + raise ValueError("Household CTR imputation needs finite reported CTR") + train_outputs = ( + reported.groupby(train_dataset.person.person_household_id).sum().gt(0) + ) + train_outputs = train_outputs.reindex(train_inputs.index).rename( + HOUSEHOLD_CTR_RECEIPT_TARGET + ) + + categories = ["region", "tenure_type"] + train_encoded = pd.get_dummies(train_inputs, columns=categories, dtype=float) + target_encoded = pd.get_dummies(target_inputs, columns=categories, dtype=float) + columns = sorted(set(train_encoded.columns) | set(target_encoded.columns)) + train_encoded = train_encoded.reindex(columns=columns, fill_value=0) + target_encoded = target_encoded.reindex(columns=columns, fill_value=0) + + model = QRF() + model.seed = HOUSEHOLD_CTR_MODEL_SEED + fitted = model.fit( + pd.concat([train_encoded, train_outputs], axis=1), + predictors=list(train_encoded.columns), + imputed_variables=[HOUSEHOLD_CTR_RECEIPT_TARGET], + ) + probabilities = _receipt_probabilities(fitted, target_encoded) + generator = np.random.default_rng(HOUSEHOLD_CTR_DRAW_SEED) + receives = probabilities.gt(generator.random(len(probabilities))) + receives &= target_inputs.council_tax.gt(0) + benunit_receives = pd.Series( + receives.reindex(reference_benunits.index).to_numpy(), + index=reference_benunits.to_numpy(), + ) + result = target_dataset.copy() + result.benunit["would_claim_council_tax_reduction"] = ( + result.benunit.benunit_id.map(benunit_receives).fillna(False).astype(bool) + ) + return result diff --git a/policyengine_uk_data/datasets/imputations/frs_only.py b/policyengine_uk_data/datasets/imputations/frs_only.py index d173612b0..080a2244f 100644 --- a/policyengine_uk_data/datasets/imputations/frs_only.py +++ b/policyengine_uk_data/datasets/imputations/frs_only.py @@ -122,7 +122,7 @@ # policyengine-uk 2.93.0 the others (housing benefit, income support, tax # credits, income-related ESA and JSA) can only be claimed with a report, # so these rows no longer receive them. Sure Start Maternity Grant needs -# one of these awards. Council tax reduction is kept (below). +# one of these awards. CTR keeps a household-imputed claim route (below). # - Benefits paid only to people out of work or incapable of it: ESA and JSA # (contributory), incapacity benefit and severe disablement allowance. On # the 2024-25 build, 41% of SPI-row ESA (contributory) reporters by weight @@ -139,10 +139,9 @@ # Kept as drawn: state pension (paid as reported once over pension age), # winter fuel payment (not read by the model), the disability benefits and # carer's allowance, whose drawn rates sit below the FRS rates as the income -# gradient implies. Council tax reduction is also kept as drawn for now. In -# 2.93.0 it too can only be claimed with a report, and zeroing it cut 2025 -# CTR by 21% on the 2024-25 build. It will be zeroed together with a -# household-level CTR imputation (#499). +# gradient implies. CTR reports are zeroed: a separate household receipt +# model sets would_claim_council_tax_reduction for the reference person's +# benefit unit. Eligibility and awards remain policyengine-uk calculations. # # Every column stays in the QRF chain above, so the values kept do not # change. They were drawn alongside the values later zeroed or restored. @@ -161,6 +160,7 @@ "incapacity_benefit_reported", "sda_reported", "child_benefit_reported", + "council_tax_benefit_reported", ] SPI_DONOR_RESTORED_PERSON_VARIABLES = [ "iidb_reported", @@ -364,6 +364,22 @@ def impute_frs_only_variables( target_dataset, year ), ) + # Gross council tax was constructed in create_frs, before reports and + # before the SPI copy exists. Learn household receipt from the untouched + # FRS, with the SPI households' updated joint incomes as predictors. + if ( + "council_tax_benefit_reported" in train_dataset.person.columns + and "council_tax" in train_dataset.household.columns + and "council_tax" in target_dataset.household.columns + ): + from policyengine_uk_data.datasets.imputations.council_tax_reduction import ( + impute_household_council_tax_reduction, + ) + + target_dataset = impute_household_council_tax_reduction( + train_dataset=train_dataset, target_dataset=target_dataset + ) + target_dataset.person = add_disability_benefit_categories_from_reported_amounts( target_dataset.person, int(str(target_dataset.time_period)[:4]), diff --git a/policyengine_uk_data/tests/test_council_tax_gross_bill.py b/policyengine_uk_data/tests/test_council_tax_gross_bill.py index d8105c15c..1b46db702 100644 --- a/policyengine_uk_data/tests/test_council_tax_gross_bill.py +++ b/policyengine_uk_data/tests/test_council_tax_gross_bill.py @@ -257,3 +257,25 @@ def test_recipients_do_not_move_the_imputed_bill(rows): derive_council_tax(households, 2024)[~recipient], derive_council_tax(changed, 2024)[~recipient], ) + + +@pytest.mark.parametrize("bill", [np.nan, -1.0]) +def test_missing_gross_bill_covers_known_reported_reduction(bill): + households = _households([_cell(LONDON, bill, RECIPIENT, 30.0)]) + assert derive_council_tax(households, 2024)[0] == pytest.approx( + 30.0 * WEEKS_IN_YEAR + ) + + +@settings(max_examples=150, deadline=None) +@given(rows=st.lists(_household, min_size=1, max_size=12), year=st.integers(2022, 2025)) +def test_gross_bill_is_at_least_the_reported_ctr_received(rows, year): + households = _households(rows) + gross = derive_council_tax(households, year) + gb_recipient = (households.ctreb == RECIPIENT) & ( + households.gvtregno != NORTHERN_IRELAND + ) + received = np.where( + gb_recipient, households.ctrebamt.fillna(0).clip(lower=0) * WEEKS_IN_YEAR, 0 + ) + assert (gross >= received - 1e-9).all() diff --git a/policyengine_uk_data/tests/test_ctr_award_bound.py b/policyengine_uk_data/tests/test_ctr_award_bound.py new file mode 100644 index 000000000..4e9435792 --- /dev/null +++ b/policyengine_uk_data/tests/test_ctr_award_bound.py @@ -0,0 +1,130 @@ +"""Modelled CTR on synthetic households with no reported SPI benefits. + +The build stores the imputed claim decision on the household reference +person's benefit unit. These small situations use that same route without +creating a dataset or running a production microsimulation. +""" + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st +from policyengine_uk import Simulation + +YEAR = 2025 +SCHEMES = [ + ("LONDON", "ENGLAND", "MERTON"), + ("SCOTLAND", "SCOTLAND", "CITY_OF_EDINBURGH"), + ("WALES", "WALES", "CARDIFF"), +] + + +def _situation(households, would_claim=True): + people, benunits, homes = {}, {}, {} + for index, household in enumerate(households): + region, country, local_authority = household["scheme"] + head = f"head_{index}" + members = [head] + people[head] = { + "age": {YEAR: household["age"]}, + "employment_income": {YEAR: household["earnings"]}, + "is_household_head": {YEAR: True}, + "council_tax_benefit_reported": {YEAR: 0}, + } + head_members = [head] + if household.get("partner_age") is not None: + partner = f"partner_{index}" + people[partner] = { + "age": {YEAR: household["partner_age"]}, + "employment_income": {YEAR: household["partner_earnings"]}, + "is_household_head": {YEAR: False}, + "council_tax_benefit_reported": {YEAR: 0}, + } + members.append(partner) + head_members.append(partner) + benunits[f"head_family_{index}"] = { + "members": head_members, + "would_claim_council_tax_reduction": {YEAR: would_claim}, + "claims_all_entitled_benefits": {YEAR: False}, + "would_claim_uc": {YEAR: False}, + } + if household.get("non_dependant_age") is not None: + non_dependant = f"non_dependant_{index}" + people[non_dependant] = { + "age": {YEAR: household["non_dependant_age"]}, + "employment_income": {YEAR: household["non_dependant_earnings"]}, + "is_household_head": {YEAR: False}, + "council_tax_benefit_reported": {YEAR: 0}, + } + members.append(non_dependant) + benunits[f"non_dependant_family_{index}"] = { + "members": [non_dependant], + "would_claim_council_tax_reduction": {YEAR: False}, + "claims_all_entitled_benefits": {YEAR: False}, + "would_claim_uc": {YEAR: False}, + } + homes[f"home_{index}"] = { + "members": members, + "region": {YEAR: region}, + "country": {YEAR: country}, + "local_authority": {YEAR: local_authority}, + "council_tax": {YEAR: household["bill"]}, + "savings": {YEAR: household["savings"]}, + "tenure_type": {YEAR: "OWNED_OUTRIGHT"}, + } + return {"people": people, "benunits": benunits, "households": homes} + + +def _calculate(simulation, variable): + return np.asarray(simulation.calculate(variable, YEAR)) + + +def test_zero_reported_ctr_can_receive_imputed_household_claim(): + """An eligible English pensioner receives CTR through the explicit flag.""" + household = { + "scheme": ("SOUTH_EAST", "ENGLAND", "MAIDSTONE"), + "age": 70, + "earnings": 0, + "bill": 2_000, + "savings": 0, + } + claimant = Simulation(situation=_situation([household])) + non_claimant = Simulation(situation=_situation([household], would_claim=False)) + + assert (_calculate(claimant, "council_tax_benefit_reported") == 0).all() + assert _calculate(claimant, "would_claim_council_tax_reduction").tolist() == [True] + assert _calculate(claimant, "council_tax_reduction")[0] == pytest.approx( + household["bill"], abs=0.01 + ) + assert _calculate(non_claimant, "council_tax_reduction")[0] == 0 + + +_household = st.fixed_dictionaries( + { + "scheme": st.sampled_from(SCHEMES), + "age": st.integers(25, 90), + "earnings": st.integers(0, 50_000), + "bill": st.integers(1, 5_000), + "savings": st.integers(0, 20_000), + "partner_age": st.one_of(st.none(), st.integers(25, 90)), + "partner_earnings": st.integers(0, 50_000), + "non_dependant_age": st.one_of(st.none(), st.integers(18, 90)), + "non_dependant_earnings": st.integers(0, 50_000), + } +) + + +@settings(max_examples=15, deadline=None, derandomize=True) +@given(households=st.lists(_household, min_size=1, max_size=3)) +def test_gross_council_tax_covers_ctr_received_by_zero_report_spi_households( + households, +): + simulation = Simulation(situation=_situation(households)) + gross = _calculate(simulation, "council_tax") + received = _calculate(simulation, "council_tax_reduction") + + assert (_calculate(simulation, "council_tax_benefit_reported") == 0).all() + np.testing.assert_allclose(gross, [household["bill"] for household in households]) + assert np.isfinite(received).all() + assert (received >= 0).all() + assert (gross >= received - 0.01).all() diff --git a/policyengine_uk_data/tests/test_frs_only_imputation.py b/policyengine_uk_data/tests/test_frs_only_imputation.py index 86073a287..c7285b1c9 100644 --- a/policyengine_uk_data/tests/test_frs_only_imputation.py +++ b/policyengine_uk_data/tests/test_frs_only_imputation.py @@ -275,7 +275,8 @@ def test_spi_donor_rows_get_zero_reported_council_tax_reduction(): ) assert "council_tax_benefit_reported" in SPI_DONOR_ZEROED_PERSON_VARIABLES - assert not set(SPI_DONOR_ZEROED_PERSON_VARIABLES) & set(FRS_ONLY_PERSON_VARIABLES) + # Keep CTR in the QRF chain so other person draws do not move (#514). + assert "council_tax_benefit_reported" in FRS_ONLY_PERSON_VARIABLES train = _fake_dataset(person_rows=400, seed=0) train.person["council_tax_benefit_reported"] = 1_000.0 @@ -287,3 +288,49 @@ def test_spi_donor_rows_get_zero_reported_council_tax_reduction(): assert (result.person["council_tax_benefit_reported"] == 0).all() # The training (FRS) rows keep their reported amounts. assert (train.person["council_tax_benefit_reported"] == 1_000.0).all() + + +def test_household_ctr_imputation_uses_original_frs_after_spi_reports_are_zeroed( + monkeypatch, +): + from policyengine_uk_data.datasets.imputations import ( + council_tax_reduction, + frs_only, + ) + + train = _fake_dataset(person_rows=12, seed=0) + target = _fake_dataset(person_rows=6, seed=1) + train.person["council_tax_benefit_reported"] = 1_000.0 + target.person["council_tax_benefit_reported"] = 750.0 + train.household["council_tax"] = 1_600.0 + target.household["council_tax"] = 1_600.0 + calls = [] + + def fake_person_draw(train, target, outputs): + assert "council_tax_benefit_reported" in outputs + target.person["council_tax_benefit_reported"] = 777.0 + return target + + def household_draw(train_dataset, target_dataset): + assert train_dataset is train + assert (train_dataset.person.council_tax_benefit_reported == 1_000).all() + assert (target_dataset.person.council_tax_benefit_reported == 0).all() + assert (target_dataset.household.council_tax == 1_600).all() + calls.append("household_receipt") + target_dataset.benunit["would_claim_council_tax_reduction"] = True + return target_dataset + + monkeypatch.setattr(frs_only, "_impute_outputs", fake_person_draw) + monkeypatch.setattr( + frs_only, + "_all_claimants_over_state_pension_age", + lambda dataset, year: np.zeros(len(dataset.benunit), dtype=bool), + ) + monkeypatch.setattr( + council_tax_reduction, "impute_household_council_tax_reduction", household_draw + ) + result = frs_only.impute_frs_only_variables(train, target) + assert calls == ["household_receipt"] + assert result.benunit.would_claim_council_tax_reduction.all() + assert (result.person.council_tax_benefit_reported == 0).all() + assert (target.person.council_tax_benefit_reported == 750).all() diff --git a/policyengine_uk_data/tests/test_household_council_tax_reduction.py b/policyengine_uk_data/tests/test_household_council_tax_reduction.py new file mode 100644 index 000000000..20a823619 --- /dev/null +++ b/policyengine_uk_data/tests/test_household_council_tax_reduction.py @@ -0,0 +1,267 @@ +"""Household CTR receipt is learned and routed by entity IDs, without reports.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.imputations.council_tax_reduction import ( + HOUSEHOLD_CTR_INCOME_PREDICTORS, + HOUSEHOLD_CTR_MODEL_SEED, + HOUSEHOLD_CTR_RECEIPT_TARGET, + impute_household_council_tax_reduction, +) + + +def _dataset(): + from policyengine_uk.data import UKSingleYearDataset + + person = pd.DataFrame( + { + "person_id": [91, 92, 93, 94, 95, 96, 97, 98], + "person_household_id": [710, 710, 710, 710, 101, 101, 330, 330], + "person_benunit_id": [7100, 7100, 7109, 7100, 1017, 1017, 3300, 3308], + "is_household_head": [True, False, False, False, True, False, True, False], + "age": [45, 43, 22, 12, 70, 66, 40, 8], + "employment_income": [10_000, 5_000, 7_000, 0, 0, 0, 15_000, 0], + "council_tax_benefit_reported": [40, 60, 0, 0, 0, 0, 0, 15], + }, + index=[18, 4, 77, 29, 55, 88, 12, 99], + ) + for i, column in enumerate(HOUSEHOLD_CTR_INCOME_PREDICTORS[1:], start=1): + person[column] = np.arange(1, len(person) + 1) * i + benunit = pd.DataFrame( + { + "benunit_id": [3308, 7109, 1017, 7100, 3300], + "would_claim_council_tax_reduction": True, + }, + index=[44, 11, 76, 32, 92], + ) + household = pd.DataFrame( + { + "household_id": [710, 101, 330], + "region": ["LONDON", "SCOTLAND", "WALES"], + "tenure_type": ["RENT_FROM_COUNCIL", "OWNED_OUTRIGHT", "RENT_PRIVATELY"], + "council_tax": [1_500, 0, 1_700], + }, + index=[37, 11, 88], + ) + return UKSingleYearDataset( + person=person, + benunit=benunit, + household=household, + fiscal_year=2023, + ) + + +def _single_person_dataset(employment_incomes, reports=None): + from policyengine_uk.data import UKSingleYearDataset + + n = len(employment_incomes) + household_ids = np.arange(n) * 7 + 10 + benunit_ids = household_ids * 10 + 3 + person = pd.DataFrame( + { + "person_id": household_ids * 100 + 8, + "person_household_id": household_ids, + "person_benunit_id": benunit_ids, + "is_household_head": True, + "age": 40, + "employment_income": employment_incomes, + "council_tax_benefit_reported": np.zeros(n) if reports is None else reports, + } + ) + for column in HOUSEHOLD_CTR_INCOME_PREDICTORS[1:]: + person[column] = 0.0 + return UKSingleYearDataset( + person=person, + benunit=pd.DataFrame({"benunit_id": benunit_ids}), + household=pd.DataFrame( + { + "household_id": household_ids, + "region": "LONDON", + "tenure_type": "RENT_PRIVATELY", + "council_tax": 2_000.0, + } + ), + fiscal_year=2023, + ) + + +def _fake_qrf(monkeypatch, probability): + """Match MicroImpute's fit and classifier probability result contract.""" + import microimpute.models + + fitted_models = [] + + class FakeQRF: + def fit(self, data, predictors, imputed_variables): + assert imputed_variables == [HOUSEHOLD_CTR_RECEIPT_TARGET] + assert HOUSEHOLD_CTR_RECEIPT_TARGET not in predictors + self.data = data.copy(deep=True) + self.predictors = predictors + fitted_models.append(self) + return self + + def predict(self, inputs, return_probs=False): + assert return_probs + self.inputs = inputs.copy(deep=True) + probabilities = np.asarray(probability(inputs), dtype=float) + return { + 0.5: pd.DataFrame( + {HOUSEHOLD_CTR_RECEIPT_TARGET: probabilities > 0.5}, + index=inputs.index, + ), + "probabilities": { + HOUSEHOLD_CTR_RECEIPT_TARGET: { + "classes": np.array([False, True]), + "probabilities": np.column_stack( + [1 - probabilities, probabilities] + ), + } + }, + } + + monkeypatch.setattr(microimpute.models, "QRF", FakeQRF) + return fitted_models + + +def test_training_aggregates_income_reports_and_composition_by_household(monkeypatch): + models = _fake_qrf(monkeypatch, lambda x: np.zeros(len(x))) + train = _dataset() + target = _dataset() + target.person.council_tax_benefit_reported = 0.0 + + impute_household_council_tax_reduction(train, target) + + model = models[0] + assert model.seed == HOUSEHOLD_CTR_MODEL_SEED + assert model.data.index.tolist() == [101, 330, 710] + assert model.data[HOUSEHOLD_CTR_RECEIPT_TARGET].tolist() == [False, True, True] + assert model.data.loc[710, "employment_income"] == 22_000 + for i, column in enumerate(HOUSEHOLD_CTR_INCOME_PREDICTORS[1:], start=1): + assert model.data.loc[710, column] == 10 * i + assert model.data.loc[710, "adult_count"] == 3 + assert model.data.loc[710, "child_count"] == 1 + assert model.data.loc[710, "oldest_adult_age"] == 45 + assert model.data.loc[101, "oldest_adult_age"] == 70 + assert model.data.loc[710, "council_tax"] == 1_500 + assert model.data.loc[710, "region_LONDON"] == 1 + assert model.data.loc[330, "region_WALES"] == 1 + assert model.data.loc[710, "tenure_type_RENT_FROM_COUNCIL"] == 1 + assert list(model.inputs.columns) == model.predictors + + +def test_receipt_maps_only_to_reference_person_benunit_and_gross_bill(monkeypatch): + _fake_qrf(monkeypatch, lambda x: np.ones(len(x))) + train, target = _dataset(), _dataset() + target.person.council_tax_benefit_reported = 0.0 + original_train = train.copy() + original_target = target.copy() + + result = impute_household_council_tax_reduction(train, target) + + assert result.benunit.set_index( + "benunit_id" + ).would_claim_council_tax_reduction.to_dict() == { + 3308: False, + 7109: False, + 1017: False, # This household has no gross council tax bill. + 7100: True, + 3300: True, + } + for name in ("person", "benunit", "household"): + pd.testing.assert_frame_equal( + getattr(train, name), getattr(original_train, name) + ) + pd.testing.assert_frame_equal( + getattr(target, name), getattr(original_target, name) + ) + pd.testing.assert_frame_equal(result.person, original_target.person) + pd.testing.assert_frame_equal(result.household, original_target.household) + assert result.person.council_tax_benefit_reported.eq(0).all() + + +def test_seeded_receipt_is_stable_when_all_frames_are_shuffled(monkeypatch): + _fake_qrf(monkeypatch, lambda x: np.full(len(x), 0.5)) + train = _single_person_dataset(np.arange(100) * 100, np.arange(100) % 2) + target = _single_person_dataset(np.arange(100) * 100) + shuffled_train, shuffled_target = train.copy(), target.copy() + for name in ("person", "benunit", "household"): + for dataset in (shuffled_train, shuffled_target): + frame = getattr(dataset, name).sample(frac=1, random_state=3) + # The arbitrary frame index must not serve as an entity join key. + frame.index = np.arange(len(frame)) * 13 + 5 + setattr(dataset, name, frame) + + result = impute_household_council_tax_reduction(train, target) + shuffled_result = impute_household_council_tax_reduction( + shuffled_train, shuffled_target + ) + + flags = result.benunit.set_index( + "benunit_id" + ).would_claim_council_tax_reduction.sort_index() + shuffled_flags = shuffled_result.benunit.set_index( + "benunit_id" + ).would_claim_council_tax_reduction.sort_index() + pd.testing.assert_series_equal(flags, shuffled_flags) + # A learned 50% probability produces both claimants and non-claimants. + # Majority-label predictions from this fake model would produce none. + assert 25 < flags.sum() < 75 + + +@pytest.mark.parametrize( + ("frame", "column", "match"), + [ + ("household", "council_tax", "council_tax"), + ("person", "is_household_head", "is_household_head"), + ("person", "person_benunit_id", "person_benunit_id"), + ], +) +def test_missing_inputs_raise_clear_errors(frame, column, match): + train, target = _dataset(), _dataset() + setattr(target, frame, getattr(target, frame).drop(columns=column)) + with pytest.raises(KeyError, match=match): + impute_household_council_tax_reduction(train, target) + + +@pytest.mark.parametrize("reference_person_count", [0, 2]) +def test_ambiguous_reference_person_is_rejected(reference_person_count): + train, target = _dataset(), _dataset() + household_mask = target.person.person_household_id.eq(710) + target.person.loc[household_mask, "is_household_head"] = False + target.person.loc[ + target.person.index[household_mask][:reference_person_count], + "is_household_head", + ] = True + with pytest.raises(ValueError, match="exactly one household reference person"): + impute_household_council_tax_reduction(train, target) + + +def test_real_qrf_receipt_probabilities_follow_training_income_gradient(): + """Exercise the locked model's probability API without a microsimulation.""" + incomes = np.r_[np.full(16, 1_000.0), np.full(16, 200_000.0)] + reports = np.r_[np.full(16, 1_000.0), np.zeros(16)] + train = _single_person_dataset(incomes, reports) + target = _single_person_dataset(np.r_[np.full(8, 1_000.0), np.full(8, 200_000.0)]) + + result = impute_household_council_tax_reduction(train, target) + + flags = result.benunit.would_claim_council_tax_reduction.to_numpy() + assert flags[:8].all() + assert not flags[8:].any() + assert result.person.council_tax_benefit_reported.eq(0).all() + + +@pytest.mark.parametrize("reported_amount", [0.0, 100.0]) +def test_real_qrf_supports_constant_receipt_targets(reported_amount): + train = _single_person_dataset([1_000, 2_000], [reported_amount] * 2) + target = _single_person_dataset([1_000, 2_000]) + + result = impute_household_council_tax_reduction(train, target) + + assert result.benunit.would_claim_council_tax_reduction.eq( + reported_amount > 0 + ).all() diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py index 9d18302cf..29e2e2408 100644 --- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py +++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py @@ -610,3 +610,31 @@ def test_create_frs_child_rows_are_child(tmp_path, monkeypatch): def test_create_frs_rejects_unknown_adult_empstati(tmp_path, monkeypatch): with pytest.raises(ValueError, match="EMPSTATI"): create_single_adult_frs(tmp_path, monkeypatch, empstati=12) + + +def test_create_frs_builds_gross_council_tax_before_policy_calculations( + tmp_path, monkeypatch +): + events = [] + derive = frs_module.derive_council_tax + initialise_simulation = FakeMicrosimulation.__init__ + + def observe_gross_bill(household, year): + events.append("gross_bill") + return derive(household, year) + + def observe_simulation(self, dataset): + # Every policy calculation must see the gross bill and the separate + # reported reduction, including this full-reduction household. + assert events[0] == "gross_bill" + gross = dataset.household.council_tax.iloc[0] + report = dataset.person.council_tax_benefit_reported.sum() + assert gross == report == 10 * frs_module.WEEKS_IN_YEAR + events.append("policy_calculation") + initialise_simulation(self, dataset) + + monkeypatch.setattr(frs_module, "derive_council_tax", observe_gross_bill) + monkeypatch.setattr(FakeMicrosimulation, "__init__", observe_simulation) + create_single_adult_frs(tmp_path, monkeypatch) + assert events[0] == "gross_bill" + assert "policy_calculation" in events[1:] diff --git a/policyengine_uk_data/tests/test_spi_donor_benefit_rules.py b/policyengine_uk_data/tests/test_spi_donor_benefit_rules.py index e162400d7..9cf491eea 100644 --- a/policyengine_uk_data/tests/test_spi_donor_benefit_rules.py +++ b/policyengine_uk_data/tests/test_spi_donor_benefit_rules.py @@ -74,6 +74,7 @@ def test_rule_sets_are_pinned(): "incapacity_benefit_reported", "sda_reported", "child_benefit_reported", + "council_tax_benefit_reported", } assert set(SPI_DONOR_RESTORED_PERSON_VARIABLES) == { "iidb_reported", @@ -92,7 +93,6 @@ def test_rule_sets_are_pinned(): "pip_dl_reported", "carers_allowance_reported", "maternity_allowance_reported", - "council_tax_benefit_reported", } assert set(SPI_DONOR_REDRAWN_TAKEUP_FLAGS) == {"would_claim_uc", "would_claim_pc"} assert not set(SPI_DONOR_ZEROED_PERSON_VARIABLES) & set( @@ -433,9 +433,8 @@ def test_stage_two_applies_the_rules_and_keeps_drawn_values(monkeypatch): pd.testing.assert_frame_equal(ruled.person[flags], recomputed[flags]) -def test_council_tax_reduction_keeps_the_stage_two_draw(): - """SPI rows carry the QRF's CTR draw through unchanged, as before the - rules: not zeroed, not the donor's. Zeroing waits for #499.""" +def test_council_tax_reduction_draw_is_zeroed_after_stage_two(): + """Preserve the person QRF chain, then clear its synthetic CTR reports.""" from policyengine_uk_data.tests.test_frs_only_imputation import _fake_dataset train = _fake_dataset(person_rows=400, seed=0) @@ -451,8 +450,15 @@ def test_council_tax_reduction_keeps_the_stage_two_draw(): drawn = frs_only._impute_outputs(train, target.copy(), outputs).person ruled = frs_only.impute_frs_only_variables(train, target).person - np.testing.assert_array_equal( - ruled.council_tax_benefit_reported, drawn.council_tax_benefit_reported - ) + assert (ruled.council_tax_benefit_reported == 0).all() assert (drawn.council_tax_benefit_reported > 0).any() assert (drawn.council_tax_benefit_reported != 999.0).any() + + +@settings(max_examples=80, deadline=None) +@given(dataset=datasets()) +def test_no_spi_person_carries_reported_ctr(dataset): + # These are SPI-only target rows, with arbitrary positive donor reports. + result = apply_spi_donor_benefit_rules(dataset) + assert (result.person.council_tax_benefit_reported == 0).all() + pd.testing.assert_frame_equal(result.household, dataset.household) diff --git a/pyproject.toml b/pyproject.toml index fffc84a7d..c5b4d0340 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dev = [ "build", "towncrier>=24.8.0", "hypothesis>=6.168.3", + "pytest-xdist>=3.8.0", ] [tool.setuptools] diff --git a/uv.lock b/uv.lock index 5ae821352..3b7fe29dd 100644 --- a/uv.lock +++ b/uv.lock @@ -295,6 +295,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -1433,7 +1442,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.57.4" +version = "1.58.0" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1465,6 +1474,7 @@ dev = [ { name = "itables" }, { name = "l0-python" }, { name = "pytest" }, + { name = "pytest-xdist" }, { name = "quantile-forest" }, { name = "ruff" }, { name = "tables" }, @@ -1493,6 +1503,7 @@ requires-dist = [ { name = "policyengine-uk", specifier = ">=2.122.0" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pytest", marker = "extra == 'dev'" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.8.0" }, { name = "pyyaml" }, { name = "quantile-forest", marker = "extra == 'dev'" }, { name = "requests" }, @@ -1713,6 +1724,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From 126909bcddeafe792d461aca51ee1e39b3015074 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 11:13:18 -0400 Subject: [PATCH 6/7] Use a lint-compliant council tax property strategy --- .../tests/test_council_tax_gross_bill.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/policyengine_uk_data/tests/test_council_tax_gross_bill.py b/policyengine_uk_data/tests/test_council_tax_gross_bill.py index 1b46db702..a6067ba85 100644 --- a/policyengine_uk_data/tests/test_council_tax_gross_bill.py +++ b/policyengine_uk_data/tests/test_council_tax_gross_bill.py @@ -189,18 +189,18 @@ def test_without_reduction_columns_the_bill_is_ctannual(): ) _weekly = st.one_of(st.none(), st.just(0.0), st.floats(min_value=0.0, max_value=100.0)) _household = st.fixed_dictionaries( - dict( - gvtregno=st.sampled_from([1, LONDON, WALES, SCOTLAND_GVTREGNO, 13]), - ctband=st.one_of(st.none(), st.integers(1, 9)), - adulth=st.integers(1, 4), - ctannual=_amount, - ctreb=st.one_of(st.none(), st.sampled_from([RECIPIENT, NON_RECIPIENT])), - ctrebamt=_weekly, - cwatamt1=_weekly, - csewamt1=_weekly, - cwatamtd=_weekly, - csewamt=_weekly, - ) + { + "gvtregno": st.sampled_from([1, LONDON, WALES, SCOTLAND_GVTREGNO, 13]), + "ctband": st.one_of(st.none(), st.integers(1, 9)), + "adulth": st.integers(1, 4), + "ctannual": _amount, + "ctreb": st.one_of(st.none(), st.sampled_from([RECIPIENT, NON_RECIPIENT])), + "ctrebamt": _weekly, + "cwatamt1": _weekly, + "csewamt1": _weekly, + "cwatamtd": _weekly, + "csewamt": _weekly, + } ) From 50bf0c5e92d595526c563a476c73571c571eefdb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 04:11:43 -0400 Subject: [PATCH 7/7] Use explicit NumPy equality for CTR receipt class mask --- .../datasets/imputations/council_tax_reduction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/policyengine_uk_data/datasets/imputations/council_tax_reduction.py b/policyengine_uk_data/datasets/imputations/council_tax_reduction.py index 4605d912a..5f2b5a192 100644 --- a/policyengine_uk_data/datasets/imputations/council_tax_reduction.py +++ b/policyengine_uk_data/datasets/imputations/council_tax_reduction.py @@ -158,7 +158,7 @@ def _receipt_probabilities(fitted_model, inputs: pd.DataFrame) -> pd.Series: "Household CTR classifier returned invalid probability dimensions" ) probabilities = pd.Series( - class_probabilities[:, classes == True].sum(axis=1), + class_probabilities[:, np.equal(classes, True)].sum(axis=1), index=inputs.index, ) if (