diff --git a/.gitignore b/.gitignore index 70d8396d..4cc5a31f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz # Hypothesis example database (property-based tests) .hypothesis/ + +# Local adoption-job notes and tooling +.job/ diff --git a/changelog.d/ctr-gross-council-tax.fixed.md b/changelog.d/ctr-gross-council-tax.fixed.md new file mode 100644 index 00000000..bf88c50b --- /dev/null +++ b/changelog.d/ctr-gross-council-tax.fixed.md @@ -0,0 +1,2 @@ +- Build `council_tax` as the bill after discounts and before council tax reduction, as policyengine-uk expects: add reported CTR (CTREBAMT) back to the FRS CTANNUAL, which is net of it, impute missing bills from non-recipients only, and in Scotland net off water and sewerage using the 2024-25 gross charges (uk-data#496). +- Stop imputing reported council tax reduction onto SPI-synthetic rows; set it to zero and impute household receipt from FRS joint incomes and composition, providing a reference-person benefit-unit claim route for modeled CTR (uk-data#497, #514). diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 711127b5..0c83f48f 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -118,6 +118,17 @@ # FRS RENTPROF: whether ROYYR1 is a profit (1) or a loss (2). FRS_RENTPROF_LOSS = 2 +SCOTLAND_GVTREGNO = 12 +NORTHERN_IRELAND_GVTREGNO = 13 +# From FRS 2024-25, CWATAMT1 and CSEWAMT1 are DWP derived variables, the +# "Weeklyised gross annual dom. water/sew. charge on bill" (DV summary +# 2024-25). Earlier releases carry interview answers under the same names, +# populated for only about half of Scottish households. +FIRST_SURVEY_YEAR_WITH_GROSS_SCOTTISH_WATER_CHARGES = 2024 +# Scottish Water Charges Reduction Scheme: at most a 35% reduction in water +# and sewerage charges for households receiving council tax reduction. +SCOTTISH_WATER_CHARGES_MAXIMUM_REDUCTION = 0.35 + @lru_cache(maxsize=None) def load_legacy_jobseeker_max_annual_hours(year: int) -> int: @@ -745,6 +756,129 @@ def split_reported_education_grants( FRS_RELEASE_FOLDER_PATTERN = re.compile(r"^frs_(\d{4})_(\d{2})$") +def derive_council_tax(household: pd.DataFrame, year: int) -> np.ndarray: + """Annual council tax bill after discounts and before council tax reduction. + + policyengine-uk's ``council_tax`` is the gross liability that its council + tax reduction (CTR) formulas reduce, so it must not already be net of CTR. + The FRS CTANNUAL is net of it: the 2024-25 DV summary labels it "Annual + CT amount after discounts/reduction", derived from inputs including CTREB + and CTREBAMT, and the interview question it replaced asked for the amount + payable "after deducting any discounts or reduction". In England and + Wales in 2022-23 to 2024-25, recipients' CTANNUAL plus CTREBAMT (weekly) + x 365.25/7 matches the mean bill of non-recipients in the same region, + band and single-adult cell. So: + + - A household reporting a reduction (CTREB = 1) gets CTANNUAL plus its + annualised CTREBAMT, including when CTANNUAL is 0 (a full reduction). + If CTREBAMT is missing or not positive, the reduction is unknown and + the bill is the larger of CTANNUAL and the imputed cell mean. + - A missing or negative CTANNUAL is imputed as the mean bill of + non-recipients with a positive bill in the same (region, band, + single-adult) cell, or 0 if the cell has none (Northern Ireland, which + has no council tax, always has none). A known reported reduction is + a lower bound on a missing gross bill in Great Britain. + - Other households keep CTANNUAL. + + In Scotland CTANNUAL also includes water and sewerage charges, which are + not council tax and which CTR does not cover, so they are netted off + first. (They belong in ``water_and_sewerage_charges``, which is still + zero for Scotland in 2024-25: uk-data#467.) From 2024-25 the netting uses + the gross charges CWATAMT1 and CSEWAMT1, as CSEWAMT is blank that year. + Measured on the 2024-25 release: + + - Non-recipients: netting the full gross charges leaves a status-discount + (25%) bill at 0.75 of the undiscounted bill in every band, so DWP's + derivation applies the status discount to council tax only. The + discount is therefore not applied to the charges here. + - Recipients: the recipient identity above holds (1.00 overall, 0.98-1.02 + across bands and discount groups) when 65% of the gross charges are + netted, and falls to 0.83 when all of them are. This matches a flat reduction at the Water + Charges Reduction Scheme's 35% maximum, so recipients' charges are + netted at 65% of gross. + + Earlier releases keep the previous netting of CSEWAMT plus CWATAMTD, the + discounted charges. A table without a CTREB column has no recipients, so + it gets the previous CTANNUAL-based bill. + + Args: + household: Raw FRS household table with lower-case column names, one + row per household. + year: FRS survey year (2024 for FRS 2024-25). + + Returns: + Annual council tax per household, in the order of ``household``. + """ + + def column(name: str) -> pd.Series: + if name in household.columns: + return pd.to_numeric(household[name], errors="coerce") + return pd.Series(np.nan, index=household.index) + + ctannual = column("ctannual") + region = column("gvtregno") + band = column("ctband") + single_adult = column("adulth") == 1 + reports_reduction = column("ctreb") == 1 + weekly_reduction = column("ctrebamt") + reduction_known = reports_reduction & (weekly_reduction > 0) + + in_scotland = region == SCOTLAND_GVTREGNO + if year >= FIRST_SURVEY_YEAR_WITH_GROSS_SCOTTISH_WATER_CHARGES and { + "cwatamt1", + "csewamt1", + }.issubset(household.columns): + gross_weekly_charges = column("cwatamt1").clip(lower=0).fillna(0) + column( + "csewamt1" + ).clip(lower=0).fillna(0) + share_in_bill = np.where( + reports_reduction, 1 - SCOTTISH_WATER_CHARGES_MAXIMUM_REDUCTION, 1 + ) + weekly_charges = gross_weekly_charges * share_in_bill + else: + weekly_charges = column("csewamt").clip(lower=0).fillna(0) + column( + "cwatamtd" + ).clip(lower=0).fillna(0) + scottish_charges = np.where(in_scotland, weekly_charges * WEEKS_IN_YEAR, 0) + council_tax_only = (ctannual - scottish_charges).clip(lower=0) + + # Cell means from non-recipients only: a recipient's CTANNUAL is net of + # its reduction and would pull the imputed gross bill down. + in_pool = (ctannual > 0) & ~reports_reduction + cell_keys = [region, band, single_adult] + cell_mean = ( + council_tax_only[in_pool] + .groupby([key[in_pool] for key in cell_keys], dropna=False) + .mean() + ) + imputed = ( + cell_mean.reindex(pd.MultiIndex.from_arrays(cell_keys)).fillna(0).to_numpy() + ) + + missing_bill = ctannual.isna() | (ctannual < 0) + council_tax = np.select( + [ + missing_bill, + reports_reduction & ~reduction_known, + reduction_known, + ], + [ + imputed, + np.maximum(council_tax_only, imputed), + council_tax_only + weekly_reduction * WEEKS_IN_YEAR, + ], + default=council_tax_only, + ) + # A missing bill must still cover a known reported award; otherwise + # unsupported schemes can pay more reported CTR than the gross bill. + reported_floor = np.where( + reduction_known & (region != NORTHERN_IRELAND_GVTREGNO), + weekly_reduction * WEEKS_IN_YEAR, + 0.0, + ) + return np.maximum(np.nan_to_num(council_tax, nan=0.0), reported_floor) + + def survey_year_from_frs_folder_name(raw_frs_folder) -> int | None: """Survey year encoded in an FRS release folder name (``frs_2024_25`` -> 2024). @@ -1179,68 +1313,9 @@ def determine_education_level(fted_val, typeed2_val, age_val): household.typeacc, 1, range(1, 8), ACCOMMODATIONS ).values - # Impute Council Tax - - # In Scotland, council tax bills are collected together with Scottish - # Water and sewerage charges, and the FRS CTANNUAL variable includes - # them. Net them off (they are weekly variables; CTANNUAL is annual) so - # council_tax is tax only: the water charges are already captured - # separately in water_and_sewerage_charges, so leaving them in both - # double-counts them and overstates Scottish council tax by roughly - # £500 per household (~25% of the Scottish total). - SCOTLAND_GVTREGNO = 12 - scottish_water_annual = pd.Series( - np.where( - household.gvtregno == SCOTLAND_GVTREGNO, - ( - np.maximum(household.csewamt.fillna(0), 0) - + np.maximum(household.cwatamtd.fillna(0), 0) - ) - * (365.25 / 7), - 0, - ), - index=household.index, - ) - ctannual_tax_only = np.maximum(household.ctannual - scottish_water_annual, 0) - - # Only ~25% of household report Council Tax bills - use - # these to build a model to impute missing values - CT_valid = household.ctannual > 0 - - # Find the mean reported Council Tax bill for a given - # (region, CT band, is-single-person-household) triplet - region = household.gvtregno[CT_valid] - band = household.ctband[CT_valid] - single_person = (household.adulth == 1)[CT_valid] - ctannual = ctannual_tax_only[CT_valid] - - # Build the table - ct_mean = ctannual.groupby([region, band, single_person], dropna=False).mean() - ct_mean = ct_mean.replace(-1, ct_mean.mean()) - - # For every household consult the table to find the imputed - # Council Tax bill - pairs = household.set_index( - [household.gvtregno, household.ctband, (household.adulth == 1)] - ) - hh_CT_mean = pd.Series(index=pairs.index) - has_mean = pairs.index.isin(ct_mean.index) - hh_CT_mean[has_mean] = ct_mean[pairs.index[has_mean]].values - hh_CT_mean[~has_mean] = 0 - ct_imputed = hh_CT_mean - - # For households which originally reported Council Tax, - # use the reported value. Otherwise, use the imputed value - council_tax = pd.Series( - np.where( - # 2018 FRS uses blanks for missing values, 2019 FRS - # uses -1 for missing values - (household.ctannual < 0) | household.ctannual.isna(), - np.maximum(ct_imputed, 0).values, - ctannual_tax_only, - ) - ) - pe_household["council_tax"] = council_tax.fillna(0) + # Council tax: the bill after discounts and before council tax + # reduction, with Scottish water and sewerage charges netted off. + pe_household["council_tax"] = derive_council_tax(household, year) BANDS = ["A", "B", "C", "D", "E", "F", "G", "H", "I"] # Band 1 is the most common pe_household["council_tax_band"] = ( diff --git a/policyengine_uk_data/datasets/imputations/council_tax_reduction.py b/policyengine_uk_data/datasets/imputations/council_tax_reduction.py new file mode 100644 index 00000000..5f2b5a19 --- /dev/null +++ b/policyengine_uk_data/datasets/imputations/council_tax_reduction.py @@ -0,0 +1,236 @@ +"""Learn household CTR receipt without manufacturing reported awards. + +SPI incomes replace personal income records, while CTR is a household award. +A separate household receipt model learns from the original FRS and supplies +an explicit claim flag only to the household reference person's benefit unit. +PolicyEngine then calculates eligibility and the award from current policy. + +An alternative is to impute the household award itself. That would require an +award override, or synthetic reported CTR, and would make the imputed amount +less responsive to reforms. Receipt probabilities retain the FRS income +gradient without choosing a uniform take-up rate or a majority-label cutoff. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +from policyengine_uk.data import UKSingleYearDataset + +HOUSEHOLD_CTR_INCOME_PREDICTORS = ( + "employment_income", + "self_employment_income", + "savings_interest_income", + "dividend_income", + "private_pension_income", + "property_income", +) +HOUSEHOLD_CTR_RECEIPT_TARGET = "household_receives_council_tax_reduction" +HOUSEHOLD_CTR_MODEL_SEED = 42 +HOUSEHOLD_CTR_DRAW_SEED = 102 + + +def _require_columns(frame: pd.DataFrame, name: str, columns) -> None: + missing = sorted(set(columns) - set(frame.columns)) + if missing: + raise KeyError(f"Household CTR imputation needs {name} columns: {missing}") + + +def _require_unique_ids(frame: pd.DataFrame, column: str) -> None: + if frame[column].isna().any() or frame[column].duplicated().any(): + raise ValueError(f"Household CTR imputation needs unique, non-null {column}") + + +def _household_predictors(dataset: UKSingleYearDataset) -> pd.DataFrame: + """Aggregate persons by household ID, independently of table row order. + + Adults are people aged 18 or older; children are younger than 18. For a + household with no adult, oldest adult age falls back to its oldest member. + """ + person, household = dataset.person, dataset.household + _require_columns( + person, + "person", + ("person_household_id", "age", *HOUSEHOLD_CTR_INCOME_PREDICTORS), + ) + _require_columns( + household, + "household", + ("household_id", "region", "tenure_type", "council_tax"), + ) + _require_unique_ids(household, "household_id") + household_ids = pd.Index(household.household_id, name="household_id").sort_values() + person_households = person.person_household_id + if not person_households.isin(household_ids).all(): + raise ValueError("Household CTR imputation found a person without a household") + if not household_ids.isin(person_households).all(): + raise ValueError("Household CTR imputation found a household without a person") + + incomes = person[list(HOUSEHOLD_CTR_INCOME_PREDICTORS)].astype(float).fillna(0) + if not np.isfinite(incomes.to_numpy()).all(): + raise ValueError("Household CTR imputation needs finite personal incomes") + frame = incomes.groupby(person_households).sum().reindex(household_ids) + + ages = person.age.astype(float) + if not np.isfinite(ages.to_numpy()).all(): + raise ValueError("Household CTR imputation needs finite ages") + adults = ages.ge(18) + frame["adult_count"] = adults.groupby(person_households).sum() + frame["child_count"] = (~adults).groupby(person_households).sum() + frame["oldest_adult_age"] = ( + ages.where(adults) + .groupby(person_households) + .max() + .fillna(ages.groupby(person_households).max()) + ) + + household_by_id = household.set_index("household_id") + gross_tax = household_by_id.council_tax.astype(float).fillna(0) + if not np.isfinite(gross_tax.to_numpy()).all(): + raise ValueError("Household CTR imputation needs finite gross council tax") + frame["council_tax"] = gross_tax.clip(lower=0) + for column in ("region", "tenure_type"): + frame[column] = household_by_id[column].fillna("UNKNOWN").astype(str) + return frame + + +def _reference_person_benunits(dataset: UKSingleYearDataset) -> pd.Series: + """Return one benefit-unit ID per household, using explicit entity keys.""" + person, benunit = dataset.person, dataset.benunit + _require_columns(person, "person", ("person_benunit_id", "is_household_head")) + _require_columns(benunit, "benunit", ("benunit_id",)) + _require_unique_ids(benunit, "benunit_id") + if not person.person_benunit_id.isin(benunit.benunit_id).all(): + raise ValueError( + "Household CTR imputation found a person without a benefit unit" + ) + if not benunit.benunit_id.isin(person.person_benunit_id).all(): + raise ValueError( + "Household CTR imputation found a benefit unit without a person" + ) + if person.groupby("person_benunit_id").person_household_id.nunique().gt(1).any(): + raise ValueError( + "Household CTR imputation found a benefit unit spanning households" + ) + if not person.is_household_head.isin((False, True)).all(): + raise ValueError("Household CTR imputation needs boolean is_household_head") + heads = person.loc[ + person.is_household_head.astype(bool), + ["person_household_id", "person_benunit_id"], + ] + head_counts = heads.groupby("person_household_id").size() + if ( + not head_counts.reindex(dataset.household.household_id, fill_value=0) + .eq(1) + .all() + ): + raise ValueError( + "Household CTR imputation needs exactly one household reference person " + "per household" + ) + return heads.set_index("person_household_id").person_benunit_id + + +def _receipt_probabilities(fitted_model, inputs: pd.DataFrame) -> pd.Series: + """Read the binary classifier probabilities exposed by MicroImpute QRF. + + The locked QRF implementation uses a random-forest classifier for binary + targets. Its default predictions are majority labels, so request the class + probabilities and draw receipt separately with our own local generator. + """ + prediction = fitted_model.predict(inputs, return_probs=True) + if isinstance(prediction, pd.DataFrame): + # MicroImpute returns a frame directly for constant training targets. + if not prediction.index.equals(inputs.index): + raise ValueError("Household CTR predictions lost their household IDs") + probabilities = prediction[HOUSEHOLD_CTR_RECEIPT_TARGET].astype(float) + else: + frames = [ + value for value in prediction.values() if isinstance(value, pd.DataFrame) + ] + if len(frames) != 1 or not frames[0].index.equals(inputs.index): + raise ValueError("Household CTR predictions lost their household IDs") + info = prediction["probabilities"][HOUSEHOLD_CTR_RECEIPT_TARGET] + classes = np.asarray(info["classes"]) + class_probabilities = np.asarray(info["probabilities"]) + if class_probabilities.shape != (len(inputs), len(classes)): + raise ValueError( + "Household CTR classifier returned invalid probability dimensions" + ) + probabilities = pd.Series( + class_probabilities[:, np.equal(classes, True)].sum(axis=1), + index=inputs.index, + ) + if ( + not np.isfinite(probabilities.to_numpy()).all() + or not probabilities.between(0, 1).all() + ): + raise ValueError( + "Household CTR classifier returned invalid receipt probabilities" + ) + return probabilities + + +def impute_household_council_tax_reduction( + train_dataset: UKSingleYearDataset, + target_dataset: UKSingleYearDataset, +) -> UKSingleYearDataset: + """Return SPI households with learned CTR claim flags, leaving reports alone. + + Training uses the original FRS, before income imputation. The target is + the SPI-donor subsample after income imputation and reported CTR zeroing. + Its predictors sum the six first-stage income components and include + household composition, region, tenure and the already-built gross bill. + Receipt is learned from a positive household sum of reported FRS CTR. + + Household IDs are sorted before fitting and drawing to keep the seeded + results stable when input frames are shuffled. Each household can claim + only through its reference person's benefit unit, and a zero gross bill + forces non-receipt. Explicit flags on every target benefit unit leave + eligibility and the award calculation to the current policy model. + """ + from microimpute.models import QRF + + train_inputs = _household_predictors(train_dataset) + target_inputs = _household_predictors(target_dataset) + reference_benunits = _reference_person_benunits(target_dataset) + _require_columns( + train_dataset.person, "training person", ("council_tax_benefit_reported",) + ) + reported = train_dataset.person.council_tax_benefit_reported.astype(float).fillna(0) + if not np.isfinite(reported.to_numpy()).all(): + raise ValueError("Household CTR imputation needs finite reported CTR") + train_outputs = ( + reported.groupby(train_dataset.person.person_household_id).sum().gt(0) + ) + train_outputs = train_outputs.reindex(train_inputs.index).rename( + HOUSEHOLD_CTR_RECEIPT_TARGET + ) + + categories = ["region", "tenure_type"] + train_encoded = pd.get_dummies(train_inputs, columns=categories, dtype=float) + target_encoded = pd.get_dummies(target_inputs, columns=categories, dtype=float) + columns = sorted(set(train_encoded.columns) | set(target_encoded.columns)) + train_encoded = train_encoded.reindex(columns=columns, fill_value=0) + target_encoded = target_encoded.reindex(columns=columns, fill_value=0) + + model = QRF() + model.seed = HOUSEHOLD_CTR_MODEL_SEED + fitted = model.fit( + pd.concat([train_encoded, train_outputs], axis=1), + predictors=list(train_encoded.columns), + imputed_variables=[HOUSEHOLD_CTR_RECEIPT_TARGET], + ) + probabilities = _receipt_probabilities(fitted, target_encoded) + generator = np.random.default_rng(HOUSEHOLD_CTR_DRAW_SEED) + receives = probabilities.gt(generator.random(len(probabilities))) + receives &= target_inputs.council_tax.gt(0) + benunit_receives = pd.Series( + receives.reindex(reference_benunits.index).to_numpy(), + index=reference_benunits.to_numpy(), + ) + result = target_dataset.copy() + result.benunit["would_claim_council_tax_reduction"] = ( + result.benunit.benunit_id.map(benunit_receives).fillna(False).astype(bool) + ) + return result diff --git a/policyengine_uk_data/datasets/imputations/frs_only.py b/policyengine_uk_data/datasets/imputations/frs_only.py index d173612b..080a2244 100644 --- a/policyengine_uk_data/datasets/imputations/frs_only.py +++ b/policyengine_uk_data/datasets/imputations/frs_only.py @@ -122,7 +122,7 @@ # policyengine-uk 2.93.0 the others (housing benefit, income support, tax # credits, income-related ESA and JSA) can only be claimed with a report, # so these rows no longer receive them. Sure Start Maternity Grant needs -# one of these awards. Council tax reduction is kept (below). +# one of these awards. CTR keeps a household-imputed claim route (below). # - Benefits paid only to people out of work or incapable of it: ESA and JSA # (contributory), incapacity benefit and severe disablement allowance. On # the 2024-25 build, 41% of SPI-row ESA (contributory) reporters by weight @@ -139,10 +139,9 @@ # Kept as drawn: state pension (paid as reported once over pension age), # winter fuel payment (not read by the model), the disability benefits and # carer's allowance, whose drawn rates sit below the FRS rates as the income -# gradient implies. Council tax reduction is also kept as drawn for now. In -# 2.93.0 it too can only be claimed with a report, and zeroing it cut 2025 -# CTR by 21% on the 2024-25 build. It will be zeroed together with a -# household-level CTR imputation (#499). +# gradient implies. CTR reports are zeroed: a separate household receipt +# model sets would_claim_council_tax_reduction for the reference person's +# benefit unit. Eligibility and awards remain policyengine-uk calculations. # # Every column stays in the QRF chain above, so the values kept do not # change. They were drawn alongside the values later zeroed or restored. @@ -161,6 +160,7 @@ "incapacity_benefit_reported", "sda_reported", "child_benefit_reported", + "council_tax_benefit_reported", ] SPI_DONOR_RESTORED_PERSON_VARIABLES = [ "iidb_reported", @@ -364,6 +364,22 @@ def impute_frs_only_variables( target_dataset, year ), ) + # Gross council tax was constructed in create_frs, before reports and + # before the SPI copy exists. Learn household receipt from the untouched + # FRS, with the SPI households' updated joint incomes as predictors. + if ( + "council_tax_benefit_reported" in train_dataset.person.columns + and "council_tax" in train_dataset.household.columns + and "council_tax" in target_dataset.household.columns + ): + from policyengine_uk_data.datasets.imputations.council_tax_reduction import ( + impute_household_council_tax_reduction, + ) + + target_dataset = impute_household_council_tax_reduction( + train_dataset=train_dataset, target_dataset=target_dataset + ) + target_dataset.person = add_disability_benefit_categories_from_reported_amounts( target_dataset.person, int(str(target_dataset.time_period)[:4]), diff --git a/policyengine_uk_data/tests/test_council_tax_gross_bill.py b/policyengine_uk_data/tests/test_council_tax_gross_bill.py new file mode 100644 index 00000000..a6067ba8 --- /dev/null +++ b/policyengine_uk_data/tests/test_council_tax_gross_bill.py @@ -0,0 +1,281 @@ +"""Council tax is the bill after discounts and before council tax reduction. + +The FRS CTANNUAL is net of the reported reduction (CTREB, CTREBAMT), and in +Scotland it also carries water and sewerage charges. ``derive_council_tax`` +adds the reported reduction back and nets the Scottish charges off. The +fixtures below are synthetic households, not survey records. +""" + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.datasets.frs import ( + SCOTLAND_GVTREGNO, + SCOTTISH_WATER_CHARGES_MAXIMUM_REDUCTION, + WEEKS_IN_YEAR, + derive_council_tax, +) + +LONDON = 7 +WALES = 11 +NORTHERN_IRELAND = 13 +BAND_C = 3 +RECIPIENT, NON_RECIPIENT = 1, 2 + + +def _households(rows: list[dict]) -> pd.DataFrame: + """Raw-FRS-shaped household table; unspecified fields are blank.""" + columns = [ + "gvtregno", + "ctband", + "adulth", + "ctannual", + "ctreb", + "ctrebamt", + "cwatamt1", + "csewamt1", + "cwatamtd", + "csewamt", + ] + table = pd.DataFrame(rows).reindex(columns=columns).astype(float) + table.index = pd.RangeIndex(100, 100 + len(table), name="household_id") + return table + + +def _cell(region, ctannual, ctreb=NON_RECIPIENT, ctrebamt=np.nan, adults=2, **kw): + return dict( + gvtregno=region, + ctband=BAND_C, + adulth=adults, + ctannual=ctannual, + ctreb=ctreb, + ctrebamt=ctrebamt, + **kw, + ) + + +def test_full_reduction_recipient_with_zero_bill_gets_gross_bill(): + households = _households( + [_cell(LONDON, 0.0, RECIPIENT, 30.0), _cell(LONDON, 1_600.0)] + ) + council_tax = derive_council_tax(households, 2024) + assert council_tax[0] == pytest.approx(30.0 * WEEKS_IN_YEAR) + + +def test_partial_reduction_recipient_gets_bill_plus_reduction(): + households = _households( + [_cell(WALES, 900.0, RECIPIENT, 12.0), _cell(WALES, 1_700.0)] + ) + council_tax = derive_council_tax(households, 2024) + assert council_tax[0] == pytest.approx(900.0 + 12.0 * WEEKS_IN_YEAR) + + +def test_non_recipients_keep_their_bill_including_zero(): + households = _households( + [ + _cell(LONDON, 1_500.0), + _cell(LONDON, 0.0), # e.g. an exempt dwelling + _cell(WALES, 1_200.0, adults=1), + ] + ) + council_tax = derive_council_tax(households, 2024) + np.testing.assert_allclose(council_tax, [1_500.0, 0.0, 1_200.0]) + + +def test_recipient_with_unknown_reduction_gets_non_recipient_cell_mean(): + households = _households( + [ + _cell(LONDON, 0.0, RECIPIENT, np.nan), + _cell(LONDON, 0.0, RECIPIENT, 0.0), + _cell(LONDON, 1_900.0, RECIPIENT, np.nan), + _cell(LONDON, 1_400.0), + _cell(LONDON, 1_600.0), + ] + ) + council_tax = derive_council_tax(households, 2024) + # The cell mean (1,500), or the reported bill if that is larger: the + # bill before the reduction cannot be below the bill after it. + np.testing.assert_allclose(council_tax[:3], [1_500.0, 1_500.0, 1_900.0]) + + +def test_missing_bills_are_imputed_from_non_recipients_only(): + households = _households( + [ + _cell(LONDON, np.nan), + _cell(LONDON, -1.0), + _cell(LONDON, 1_400.0), + _cell(LONDON, 1_600.0), + # A recipient's net bill must not pull the imputed bill down. + _cell(LONDON, 200.0, RECIPIENT, 25.0), + # A different cell (single adult) must not enter the mean. + _cell(LONDON, 1_125.0, adults=1), + ] + ) + council_tax = derive_council_tax(households, 2024) + np.testing.assert_allclose(council_tax[:2], [1_500.0, 1_500.0]) + + +def test_northern_ireland_has_no_council_tax(): + households = _households([_cell(NORTHERN_IRELAND, np.nan, RECIPIENT, 10.0)]) + assert derive_council_tax(households, 2024)[0] == 0.0 + + +def test_scottish_gross_water_and_sewerage_are_netted_from_2024(): + water, sewerage = 4.0, 5.0 + gross_charges = (water + sewerage) * WEEKS_IN_YEAR + households = _households( + [ + _cell(SCOTLAND_GVTREGNO, 2_000.0, cwatamt1=water, csewamt1=sewerage), + _cell( + SCOTLAND_GVTREGNO, + 900.0, + RECIPIENT, + 10.0, + cwatamt1=water, + csewamt1=sewerage, + ), + ] + ) + council_tax = derive_council_tax(households, 2024) + assert council_tax[0] == pytest.approx(2_000.0 - gross_charges) + # Recipients' CTANNUAL carries the charges after the Water Charges + # Reduction Scheme's maximum 35% reduction. + recipient_charges = gross_charges * (1 - SCOTTISH_WATER_CHARGES_MAXIMUM_REDUCTION) + assert council_tax[1] == pytest.approx( + 900.0 - recipient_charges + 10.0 * WEEKS_IN_YEAR + ) + + +def test_scotland_before_2024_keeps_discounted_charge_netting(): + households = _households( + [ + _cell( + SCOTLAND_GVTREGNO, + 2_000.0, + cwatamtd=3.0, + csewamt=4.0, + cwatamt1=9.0, + csewamt1=9.0, + ) + ] + ) + council_tax = derive_council_tax(households, 2023) + assert council_tax[0] == pytest.approx(2_000.0 - 7.0 * WEEKS_IN_YEAR) + + +def test_charges_are_not_netted_outside_scotland(): + households = _households([_cell(LONDON, 2_000.0, cwatamt1=9.0, csewamt1=9.0)]) + assert derive_council_tax(households, 2024)[0] == pytest.approx(2_000.0) + + +def test_without_reduction_columns_the_bill_is_ctannual(): + households = _households( + [_cell(LONDON, 700.0, RECIPIENT, 20.0), _cell(LONDON, np.nan)] + ).drop(columns=["ctreb", "ctrebamt"]) + council_tax = derive_council_tax(households, 2020) + np.testing.assert_allclose(council_tax, [700.0, 700.0]) + + +# Property-based checks over arbitrary small household tables. + +_amount = st.one_of( + st.none(), + st.just(-1.0), + st.just(0.0), + st.floats(min_value=0.0, max_value=5_000.0), +) +_weekly = st.one_of(st.none(), st.just(0.0), st.floats(min_value=0.0, max_value=100.0)) +_household = st.fixed_dictionaries( + { + "gvtregno": st.sampled_from([1, LONDON, WALES, SCOTLAND_GVTREGNO, 13]), + "ctband": st.one_of(st.none(), st.integers(1, 9)), + "adulth": st.integers(1, 4), + "ctannual": _amount, + "ctreb": st.one_of(st.none(), st.sampled_from([RECIPIENT, NON_RECIPIENT])), + "ctrebamt": _weekly, + "cwatamt1": _weekly, + "csewamt1": _weekly, + "cwatamtd": _weekly, + "csewamt": _weekly, + } +) + + +@settings(max_examples=300, deadline=None) +@given( + rows=st.lists(_household, min_size=1, max_size=12), + year=st.sampled_from([2022, 2023, 2024, 2025]), +) +def test_council_tax_invariants(rows, year): + households = _households(rows) + council_tax = derive_council_tax(households, year) + ctannual = households.ctannual.to_numpy() + reduction = households.ctrebamt.to_numpy() + recipient = (households.ctreb == 1).to_numpy() + known_reduction = recipient & (households.ctrebamt > 0).to_numpy() + has_bill = ctannual >= 0 + scotland = (households.gvtregno == SCOTLAND_GVTREGNO).to_numpy() + + # One finite, non-negative bill per household. + assert council_tax.shape == (len(households),) + assert np.isfinite(council_tax).all() and (council_tax >= 0).all() + + # Non-recipients with a bill outside Scotland keep it exactly. + keep = ~recipient & has_bill & ~scotland + np.testing.assert_allclose(council_tax[keep], ctannual[keep]) + + # A known reduction is added back in full, so recipients outside + # Scotland pay CTANNUAL plus it. + add_back = known_reduction & has_bill & ~scotland + np.testing.assert_allclose( + council_tax[add_back], + ctannual[add_back] + reduction[add_back] * WEEKS_IN_YEAR, + ) + + # The bill before the reduction is never below the bill after it. + no_charges = has_bill & recipient & ~scotland + assert (council_tax[no_charges] >= ctannual[no_charges] - 1e-9).all() + + # Recomputing on a subset of rows cannot change the kept bills (the + # cell means only feed missing bills and unknown reductions). + council_tax_keep_only = derive_council_tax(households[keep], year) + np.testing.assert_allclose(council_tax_keep_only, council_tax[keep]) + + +@settings(max_examples=200, deadline=None) +@given(rows=st.lists(_household, min_size=1, max_size=12)) +def test_recipients_do_not_move_the_imputed_bill(rows): + """Changing recipients' bills never changes a non-recipient's result.""" + households = _households(rows) + recipient = (households.ctreb == 1).to_numpy() + changed = households.copy() + changed.loc[recipient, "ctannual"] = 4_321.0 + np.testing.assert_allclose( + derive_council_tax(households, 2024)[~recipient], + derive_council_tax(changed, 2024)[~recipient], + ) + + +@pytest.mark.parametrize("bill", [np.nan, -1.0]) +def test_missing_gross_bill_covers_known_reported_reduction(bill): + households = _households([_cell(LONDON, bill, RECIPIENT, 30.0)]) + assert derive_council_tax(households, 2024)[0] == pytest.approx( + 30.0 * WEEKS_IN_YEAR + ) + + +@settings(max_examples=150, deadline=None) +@given(rows=st.lists(_household, min_size=1, max_size=12), year=st.integers(2022, 2025)) +def test_gross_bill_is_at_least_the_reported_ctr_received(rows, year): + households = _households(rows) + gross = derive_council_tax(households, year) + gb_recipient = (households.ctreb == RECIPIENT) & ( + households.gvtregno != NORTHERN_IRELAND + ) + received = np.where( + gb_recipient, households.ctrebamt.fillna(0).clip(lower=0) * WEEKS_IN_YEAR, 0 + ) + assert (gross >= received - 1e-9).all() diff --git a/policyengine_uk_data/tests/test_ctr_award_bound.py b/policyengine_uk_data/tests/test_ctr_award_bound.py new file mode 100644 index 00000000..4e943579 --- /dev/null +++ b/policyengine_uk_data/tests/test_ctr_award_bound.py @@ -0,0 +1,130 @@ +"""Modelled CTR on synthetic households with no reported SPI benefits. + +The build stores the imputed claim decision on the household reference +person's benefit unit. These small situations use that same route without +creating a dataset or running a production microsimulation. +""" + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st +from policyengine_uk import Simulation + +YEAR = 2025 +SCHEMES = [ + ("LONDON", "ENGLAND", "MERTON"), + ("SCOTLAND", "SCOTLAND", "CITY_OF_EDINBURGH"), + ("WALES", "WALES", "CARDIFF"), +] + + +def _situation(households, would_claim=True): + people, benunits, homes = {}, {}, {} + for index, household in enumerate(households): + region, country, local_authority = household["scheme"] + head = f"head_{index}" + members = [head] + people[head] = { + "age": {YEAR: household["age"]}, + "employment_income": {YEAR: household["earnings"]}, + "is_household_head": {YEAR: True}, + "council_tax_benefit_reported": {YEAR: 0}, + } + head_members = [head] + if household.get("partner_age") is not None: + partner = f"partner_{index}" + people[partner] = { + "age": {YEAR: household["partner_age"]}, + "employment_income": {YEAR: household["partner_earnings"]}, + "is_household_head": {YEAR: False}, + "council_tax_benefit_reported": {YEAR: 0}, + } + members.append(partner) + head_members.append(partner) + benunits[f"head_family_{index}"] = { + "members": head_members, + "would_claim_council_tax_reduction": {YEAR: would_claim}, + "claims_all_entitled_benefits": {YEAR: False}, + "would_claim_uc": {YEAR: False}, + } + if household.get("non_dependant_age") is not None: + non_dependant = f"non_dependant_{index}" + people[non_dependant] = { + "age": {YEAR: household["non_dependant_age"]}, + "employment_income": {YEAR: household["non_dependant_earnings"]}, + "is_household_head": {YEAR: False}, + "council_tax_benefit_reported": {YEAR: 0}, + } + members.append(non_dependant) + benunits[f"non_dependant_family_{index}"] = { + "members": [non_dependant], + "would_claim_council_tax_reduction": {YEAR: False}, + "claims_all_entitled_benefits": {YEAR: False}, + "would_claim_uc": {YEAR: False}, + } + homes[f"home_{index}"] = { + "members": members, + "region": {YEAR: region}, + "country": {YEAR: country}, + "local_authority": {YEAR: local_authority}, + "council_tax": {YEAR: household["bill"]}, + "savings": {YEAR: household["savings"]}, + "tenure_type": {YEAR: "OWNED_OUTRIGHT"}, + } + return {"people": people, "benunits": benunits, "households": homes} + + +def _calculate(simulation, variable): + return np.asarray(simulation.calculate(variable, YEAR)) + + +def test_zero_reported_ctr_can_receive_imputed_household_claim(): + """An eligible English pensioner receives CTR through the explicit flag.""" + household = { + "scheme": ("SOUTH_EAST", "ENGLAND", "MAIDSTONE"), + "age": 70, + "earnings": 0, + "bill": 2_000, + "savings": 0, + } + claimant = Simulation(situation=_situation([household])) + non_claimant = Simulation(situation=_situation([household], would_claim=False)) + + assert (_calculate(claimant, "council_tax_benefit_reported") == 0).all() + assert _calculate(claimant, "would_claim_council_tax_reduction").tolist() == [True] + assert _calculate(claimant, "council_tax_reduction")[0] == pytest.approx( + household["bill"], abs=0.01 + ) + assert _calculate(non_claimant, "council_tax_reduction")[0] == 0 + + +_household = st.fixed_dictionaries( + { + "scheme": st.sampled_from(SCHEMES), + "age": st.integers(25, 90), + "earnings": st.integers(0, 50_000), + "bill": st.integers(1, 5_000), + "savings": st.integers(0, 20_000), + "partner_age": st.one_of(st.none(), st.integers(25, 90)), + "partner_earnings": st.integers(0, 50_000), + "non_dependant_age": st.one_of(st.none(), st.integers(18, 90)), + "non_dependant_earnings": st.integers(0, 50_000), + } +) + + +@settings(max_examples=15, deadline=None, derandomize=True) +@given(households=st.lists(_household, min_size=1, max_size=3)) +def test_gross_council_tax_covers_ctr_received_by_zero_report_spi_households( + households, +): + simulation = Simulation(situation=_situation(households)) + gross = _calculate(simulation, "council_tax") + received = _calculate(simulation, "council_tax_reduction") + + assert (_calculate(simulation, "council_tax_benefit_reported") == 0).all() + np.testing.assert_allclose(gross, [household["bill"] for household in households]) + assert np.isfinite(received).all() + assert (received >= 0).all() + assert (gross >= received - 0.01).all() diff --git a/policyengine_uk_data/tests/test_frs_only_imputation.py b/policyengine_uk_data/tests/test_frs_only_imputation.py index 17ba59fd..c7285b1c 100644 --- a/policyengine_uk_data/tests/test_frs_only_imputation.py +++ b/policyengine_uk_data/tests/test_frs_only_imputation.py @@ -259,3 +259,78 @@ def predict(self, x): assert not result.person["is_severely_disabled_for_benefits"].any() assert (result.person["pip_dl_category"] == "NONE").all() assert (result.person["pip_m_category"] == "NONE").all() + + +def test_spi_donor_rows_get_zero_reported_council_tax_reduction(): + """Reported CTR is zeroed on SPI-donor rows, not imputed (uk-data#497). + + Even when every training person reports CTR, the target rows must end + with none: the stage-2 QRF would otherwise spread a household-level + amount over individual people. + """ + from policyengine_uk_data.datasets.imputations.frs_only import ( + FRS_ONLY_PERSON_VARIABLES, + SPI_DONOR_ZEROED_PERSON_VARIABLES, + impute_frs_only_variables, + ) + + assert "council_tax_benefit_reported" in SPI_DONOR_ZEROED_PERSON_VARIABLES + # Keep CTR in the QRF chain so other person draws do not move (#514). + assert "council_tax_benefit_reported" in FRS_ONLY_PERSON_VARIABLES + + train = _fake_dataset(person_rows=400, seed=0) + train.person["council_tax_benefit_reported"] = 1_000.0 + target = _fake_dataset(person_rows=60, seed=1) + target.person["council_tax_benefit_reported"] = 750.0 + + result = impute_frs_only_variables(train_dataset=train, target_dataset=target) + + assert (result.person["council_tax_benefit_reported"] == 0).all() + # The training (FRS) rows keep their reported amounts. + assert (train.person["council_tax_benefit_reported"] == 1_000.0).all() + + +def test_household_ctr_imputation_uses_original_frs_after_spi_reports_are_zeroed( + monkeypatch, +): + from policyengine_uk_data.datasets.imputations import ( + council_tax_reduction, + frs_only, + ) + + train = _fake_dataset(person_rows=12, seed=0) + target = _fake_dataset(person_rows=6, seed=1) + train.person["council_tax_benefit_reported"] = 1_000.0 + target.person["council_tax_benefit_reported"] = 750.0 + train.household["council_tax"] = 1_600.0 + target.household["council_tax"] = 1_600.0 + calls = [] + + def fake_person_draw(train, target, outputs): + assert "council_tax_benefit_reported" in outputs + target.person["council_tax_benefit_reported"] = 777.0 + return target + + def household_draw(train_dataset, target_dataset): + assert train_dataset is train + assert (train_dataset.person.council_tax_benefit_reported == 1_000).all() + assert (target_dataset.person.council_tax_benefit_reported == 0).all() + assert (target_dataset.household.council_tax == 1_600).all() + calls.append("household_receipt") + target_dataset.benunit["would_claim_council_tax_reduction"] = True + return target_dataset + + monkeypatch.setattr(frs_only, "_impute_outputs", fake_person_draw) + monkeypatch.setattr( + frs_only, + "_all_claimants_over_state_pension_age", + lambda dataset, year: np.zeros(len(dataset.benunit), dtype=bool), + ) + monkeypatch.setattr( + council_tax_reduction, "impute_household_council_tax_reduction", household_draw + ) + result = frs_only.impute_frs_only_variables(train, target) + assert calls == ["household_receipt"] + assert result.benunit.would_claim_council_tax_reduction.all() + assert (result.person.council_tax_benefit_reported == 0).all() + assert (target.person.council_tax_benefit_reported == 750).all() diff --git a/policyengine_uk_data/tests/test_household_council_tax_reduction.py b/policyengine_uk_data/tests/test_household_council_tax_reduction.py new file mode 100644 index 00000000..20a82361 --- /dev/null +++ b/policyengine_uk_data/tests/test_household_council_tax_reduction.py @@ -0,0 +1,267 @@ +"""Household CTR receipt is learned and routed by entity IDs, without reports.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.imputations.council_tax_reduction import ( + HOUSEHOLD_CTR_INCOME_PREDICTORS, + HOUSEHOLD_CTR_MODEL_SEED, + HOUSEHOLD_CTR_RECEIPT_TARGET, + impute_household_council_tax_reduction, +) + + +def _dataset(): + from policyengine_uk.data import UKSingleYearDataset + + person = pd.DataFrame( + { + "person_id": [91, 92, 93, 94, 95, 96, 97, 98], + "person_household_id": [710, 710, 710, 710, 101, 101, 330, 330], + "person_benunit_id": [7100, 7100, 7109, 7100, 1017, 1017, 3300, 3308], + "is_household_head": [True, False, False, False, True, False, True, False], + "age": [45, 43, 22, 12, 70, 66, 40, 8], + "employment_income": [10_000, 5_000, 7_000, 0, 0, 0, 15_000, 0], + "council_tax_benefit_reported": [40, 60, 0, 0, 0, 0, 0, 15], + }, + index=[18, 4, 77, 29, 55, 88, 12, 99], + ) + for i, column in enumerate(HOUSEHOLD_CTR_INCOME_PREDICTORS[1:], start=1): + person[column] = np.arange(1, len(person) + 1) * i + benunit = pd.DataFrame( + { + "benunit_id": [3308, 7109, 1017, 7100, 3300], + "would_claim_council_tax_reduction": True, + }, + index=[44, 11, 76, 32, 92], + ) + household = pd.DataFrame( + { + "household_id": [710, 101, 330], + "region": ["LONDON", "SCOTLAND", "WALES"], + "tenure_type": ["RENT_FROM_COUNCIL", "OWNED_OUTRIGHT", "RENT_PRIVATELY"], + "council_tax": [1_500, 0, 1_700], + }, + index=[37, 11, 88], + ) + return UKSingleYearDataset( + person=person, + benunit=benunit, + household=household, + fiscal_year=2023, + ) + + +def _single_person_dataset(employment_incomes, reports=None): + from policyengine_uk.data import UKSingleYearDataset + + n = len(employment_incomes) + household_ids = np.arange(n) * 7 + 10 + benunit_ids = household_ids * 10 + 3 + person = pd.DataFrame( + { + "person_id": household_ids * 100 + 8, + "person_household_id": household_ids, + "person_benunit_id": benunit_ids, + "is_household_head": True, + "age": 40, + "employment_income": employment_incomes, + "council_tax_benefit_reported": np.zeros(n) if reports is None else reports, + } + ) + for column in HOUSEHOLD_CTR_INCOME_PREDICTORS[1:]: + person[column] = 0.0 + return UKSingleYearDataset( + person=person, + benunit=pd.DataFrame({"benunit_id": benunit_ids}), + household=pd.DataFrame( + { + "household_id": household_ids, + "region": "LONDON", + "tenure_type": "RENT_PRIVATELY", + "council_tax": 2_000.0, + } + ), + fiscal_year=2023, + ) + + +def _fake_qrf(monkeypatch, probability): + """Match MicroImpute's fit and classifier probability result contract.""" + import microimpute.models + + fitted_models = [] + + class FakeQRF: + def fit(self, data, predictors, imputed_variables): + assert imputed_variables == [HOUSEHOLD_CTR_RECEIPT_TARGET] + assert HOUSEHOLD_CTR_RECEIPT_TARGET not in predictors + self.data = data.copy(deep=True) + self.predictors = predictors + fitted_models.append(self) + return self + + def predict(self, inputs, return_probs=False): + assert return_probs + self.inputs = inputs.copy(deep=True) + probabilities = np.asarray(probability(inputs), dtype=float) + return { + 0.5: pd.DataFrame( + {HOUSEHOLD_CTR_RECEIPT_TARGET: probabilities > 0.5}, + index=inputs.index, + ), + "probabilities": { + HOUSEHOLD_CTR_RECEIPT_TARGET: { + "classes": np.array([False, True]), + "probabilities": np.column_stack( + [1 - probabilities, probabilities] + ), + } + }, + } + + monkeypatch.setattr(microimpute.models, "QRF", FakeQRF) + return fitted_models + + +def test_training_aggregates_income_reports_and_composition_by_household(monkeypatch): + models = _fake_qrf(monkeypatch, lambda x: np.zeros(len(x))) + train = _dataset() + target = _dataset() + target.person.council_tax_benefit_reported = 0.0 + + impute_household_council_tax_reduction(train, target) + + model = models[0] + assert model.seed == HOUSEHOLD_CTR_MODEL_SEED + assert model.data.index.tolist() == [101, 330, 710] + assert model.data[HOUSEHOLD_CTR_RECEIPT_TARGET].tolist() == [False, True, True] + assert model.data.loc[710, "employment_income"] == 22_000 + for i, column in enumerate(HOUSEHOLD_CTR_INCOME_PREDICTORS[1:], start=1): + assert model.data.loc[710, column] == 10 * i + assert model.data.loc[710, "adult_count"] == 3 + assert model.data.loc[710, "child_count"] == 1 + assert model.data.loc[710, "oldest_adult_age"] == 45 + assert model.data.loc[101, "oldest_adult_age"] == 70 + assert model.data.loc[710, "council_tax"] == 1_500 + assert model.data.loc[710, "region_LONDON"] == 1 + assert model.data.loc[330, "region_WALES"] == 1 + assert model.data.loc[710, "tenure_type_RENT_FROM_COUNCIL"] == 1 + assert list(model.inputs.columns) == model.predictors + + +def test_receipt_maps_only_to_reference_person_benunit_and_gross_bill(monkeypatch): + _fake_qrf(monkeypatch, lambda x: np.ones(len(x))) + train, target = _dataset(), _dataset() + target.person.council_tax_benefit_reported = 0.0 + original_train = train.copy() + original_target = target.copy() + + result = impute_household_council_tax_reduction(train, target) + + assert result.benunit.set_index( + "benunit_id" + ).would_claim_council_tax_reduction.to_dict() == { + 3308: False, + 7109: False, + 1017: False, # This household has no gross council tax bill. + 7100: True, + 3300: True, + } + for name in ("person", "benunit", "household"): + pd.testing.assert_frame_equal( + getattr(train, name), getattr(original_train, name) + ) + pd.testing.assert_frame_equal( + getattr(target, name), getattr(original_target, name) + ) + pd.testing.assert_frame_equal(result.person, original_target.person) + pd.testing.assert_frame_equal(result.household, original_target.household) + assert result.person.council_tax_benefit_reported.eq(0).all() + + +def test_seeded_receipt_is_stable_when_all_frames_are_shuffled(monkeypatch): + _fake_qrf(monkeypatch, lambda x: np.full(len(x), 0.5)) + train = _single_person_dataset(np.arange(100) * 100, np.arange(100) % 2) + target = _single_person_dataset(np.arange(100) * 100) + shuffled_train, shuffled_target = train.copy(), target.copy() + for name in ("person", "benunit", "household"): + for dataset in (shuffled_train, shuffled_target): + frame = getattr(dataset, name).sample(frac=1, random_state=3) + # The arbitrary frame index must not serve as an entity join key. + frame.index = np.arange(len(frame)) * 13 + 5 + setattr(dataset, name, frame) + + result = impute_household_council_tax_reduction(train, target) + shuffled_result = impute_household_council_tax_reduction( + shuffled_train, shuffled_target + ) + + flags = result.benunit.set_index( + "benunit_id" + ).would_claim_council_tax_reduction.sort_index() + shuffled_flags = shuffled_result.benunit.set_index( + "benunit_id" + ).would_claim_council_tax_reduction.sort_index() + pd.testing.assert_series_equal(flags, shuffled_flags) + # A learned 50% probability produces both claimants and non-claimants. + # Majority-label predictions from this fake model would produce none. + assert 25 < flags.sum() < 75 + + +@pytest.mark.parametrize( + ("frame", "column", "match"), + [ + ("household", "council_tax", "council_tax"), + ("person", "is_household_head", "is_household_head"), + ("person", "person_benunit_id", "person_benunit_id"), + ], +) +def test_missing_inputs_raise_clear_errors(frame, column, match): + train, target = _dataset(), _dataset() + setattr(target, frame, getattr(target, frame).drop(columns=column)) + with pytest.raises(KeyError, match=match): + impute_household_council_tax_reduction(train, target) + + +@pytest.mark.parametrize("reference_person_count", [0, 2]) +def test_ambiguous_reference_person_is_rejected(reference_person_count): + train, target = _dataset(), _dataset() + household_mask = target.person.person_household_id.eq(710) + target.person.loc[household_mask, "is_household_head"] = False + target.person.loc[ + target.person.index[household_mask][:reference_person_count], + "is_household_head", + ] = True + with pytest.raises(ValueError, match="exactly one household reference person"): + impute_household_council_tax_reduction(train, target) + + +def test_real_qrf_receipt_probabilities_follow_training_income_gradient(): + """Exercise the locked model's probability API without a microsimulation.""" + incomes = np.r_[np.full(16, 1_000.0), np.full(16, 200_000.0)] + reports = np.r_[np.full(16, 1_000.0), np.zeros(16)] + train = _single_person_dataset(incomes, reports) + target = _single_person_dataset(np.r_[np.full(8, 1_000.0), np.full(8, 200_000.0)]) + + result = impute_household_council_tax_reduction(train, target) + + flags = result.benunit.would_claim_council_tax_reduction.to_numpy() + assert flags[:8].all() + assert not flags[8:].any() + assert result.person.council_tax_benefit_reported.eq(0).all() + + +@pytest.mark.parametrize("reported_amount", [0.0, 100.0]) +def test_real_qrf_supports_constant_receipt_targets(reported_amount): + train = _single_person_dataset([1_000, 2_000], [reported_amount] * 2) + target = _single_person_dataset([1_000, 2_000]) + + result = impute_household_council_tax_reduction(train, target) + + assert result.benunit.would_claim_council_tax_reduction.eq( + reported_amount > 0 + ).all() diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py index 64bf5670..29e2e240 100644 --- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py +++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py @@ -485,7 +485,8 @@ def create_single_adult_frs(tmp_path, monkeypatch, empstati=8, with_child=False) "csewamt": 0, "ctannual": 0, "ctband": 1, - "ctrebamt": 0, + "ctreb": 1, + "ctrebamt": 10, "cwatamtd": 0, "gross4": 0, "gvtregno": 1, @@ -573,6 +574,14 @@ def test_create_frs_smoke_includes_legacy_proxy_columns(tmp_path, monkeypatch): ].iloc[0] assert dataset.person["education_grants"].iloc[0] == 100 assert dataset.person["disabled_students_allowance_eligible_expenses"].iloc[0] == 0 + # A full council tax reduction (CTANNUAL 0, CTREBAMT £10 a week) leaves + # the bill before the reduction in council_tax (uk-data#496). + weekly_reduction_annualised = 10 * frs_module.WEEKS_IN_YEAR + assert dataset.household["council_tax"].iloc[0] == weekly_reduction_annualised + assert ( + dataset.person["council_tax_benefit_reported"].iloc[0] + == weekly_reduction_annualised + ) @pytest.mark.parametrize("empstati", sorted(FRS_EMPSTATI_EMPLOYMENT_STATUS)) @@ -601,3 +610,31 @@ def test_create_frs_child_rows_are_child(tmp_path, monkeypatch): def test_create_frs_rejects_unknown_adult_empstati(tmp_path, monkeypatch): with pytest.raises(ValueError, match="EMPSTATI"): create_single_adult_frs(tmp_path, monkeypatch, empstati=12) + + +def test_create_frs_builds_gross_council_tax_before_policy_calculations( + tmp_path, monkeypatch +): + events = [] + derive = frs_module.derive_council_tax + initialise_simulation = FakeMicrosimulation.__init__ + + def observe_gross_bill(household, year): + events.append("gross_bill") + return derive(household, year) + + def observe_simulation(self, dataset): + # Every policy calculation must see the gross bill and the separate + # reported reduction, including this full-reduction household. + assert events[0] == "gross_bill" + gross = dataset.household.council_tax.iloc[0] + report = dataset.person.council_tax_benefit_reported.sum() + assert gross == report == 10 * frs_module.WEEKS_IN_YEAR + events.append("policy_calculation") + initialise_simulation(self, dataset) + + monkeypatch.setattr(frs_module, "derive_council_tax", observe_gross_bill) + monkeypatch.setattr(FakeMicrosimulation, "__init__", observe_simulation) + create_single_adult_frs(tmp_path, monkeypatch) + assert events[0] == "gross_bill" + assert "policy_calculation" in events[1:] diff --git a/policyengine_uk_data/tests/test_spi_donor_benefit_rules.py b/policyengine_uk_data/tests/test_spi_donor_benefit_rules.py index e162400d..9cf491ee 100644 --- a/policyengine_uk_data/tests/test_spi_donor_benefit_rules.py +++ b/policyengine_uk_data/tests/test_spi_donor_benefit_rules.py @@ -74,6 +74,7 @@ def test_rule_sets_are_pinned(): "incapacity_benefit_reported", "sda_reported", "child_benefit_reported", + "council_tax_benefit_reported", } assert set(SPI_DONOR_RESTORED_PERSON_VARIABLES) == { "iidb_reported", @@ -92,7 +93,6 @@ def test_rule_sets_are_pinned(): "pip_dl_reported", "carers_allowance_reported", "maternity_allowance_reported", - "council_tax_benefit_reported", } assert set(SPI_DONOR_REDRAWN_TAKEUP_FLAGS) == {"would_claim_uc", "would_claim_pc"} assert not set(SPI_DONOR_ZEROED_PERSON_VARIABLES) & set( @@ -433,9 +433,8 @@ def test_stage_two_applies_the_rules_and_keeps_drawn_values(monkeypatch): pd.testing.assert_frame_equal(ruled.person[flags], recomputed[flags]) -def test_council_tax_reduction_keeps_the_stage_two_draw(): - """SPI rows carry the QRF's CTR draw through unchanged, as before the - rules: not zeroed, not the donor's. Zeroing waits for #499.""" +def test_council_tax_reduction_draw_is_zeroed_after_stage_two(): + """Preserve the person QRF chain, then clear its synthetic CTR reports.""" from policyengine_uk_data.tests.test_frs_only_imputation import _fake_dataset train = _fake_dataset(person_rows=400, seed=0) @@ -451,8 +450,15 @@ def test_council_tax_reduction_keeps_the_stage_two_draw(): drawn = frs_only._impute_outputs(train, target.copy(), outputs).person ruled = frs_only.impute_frs_only_variables(train, target).person - np.testing.assert_array_equal( - ruled.council_tax_benefit_reported, drawn.council_tax_benefit_reported - ) + assert (ruled.council_tax_benefit_reported == 0).all() assert (drawn.council_tax_benefit_reported > 0).any() assert (drawn.council_tax_benefit_reported != 999.0).any() + + +@settings(max_examples=80, deadline=None) +@given(dataset=datasets()) +def test_no_spi_person_carries_reported_ctr(dataset): + # These are SPI-only target rows, with arbitrary positive donor reports. + result = apply_spi_donor_benefit_rules(dataset) + assert (result.person.council_tax_benefit_reported == 0).all() + pd.testing.assert_frame_equal(result.household, dataset.household) diff --git a/pyproject.toml b/pyproject.toml index fffc84a7..c5b4d034 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dev = [ "build", "towncrier>=24.8.0", "hypothesis>=6.168.3", + "pytest-xdist>=3.8.0", ] [tool.setuptools] diff --git a/uv.lock b/uv.lock index 5ae82135..3b7fe29d 100644 --- a/uv.lock +++ b/uv.lock @@ -295,6 +295,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -1433,7 +1442,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.57.4" +version = "1.58.0" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1465,6 +1474,7 @@ dev = [ { name = "itables" }, { name = "l0-python" }, { name = "pytest" }, + { name = "pytest-xdist" }, { name = "quantile-forest" }, { name = "ruff" }, { name = "tables" }, @@ -1493,6 +1503,7 @@ requires-dist = [ { name = "policyengine-uk", specifier = ">=2.122.0" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pytest", marker = "extra == 'dev'" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.8.0" }, { name = "pyyaml" }, { name = "quantile-forest", marker = "extra == 'dev'" }, { name = "requests" }, @@ -1713,6 +1724,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"