diff --git a/.gitignore b/.gitignore index 70d8396d3..6c6b64b39 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz # Hypothesis example database (property-based tests) .hypothesis/ + +# Local build-job evidence and Git metadata +.job/ diff --git a/changelog.d/frs-benunit-reported-capital.changed.md b/changelog.d/frs-benunit-reported-capital.changed.md new file mode 100644 index 000000000..db1f50e77 --- /dev/null +++ b/changelog.d/frs-benunit-reported-capital.changed.md @@ -0,0 +1 @@ +- Fill policyengine-uk's `benunit_reported_capital` with each benefit unit's FRS total capital (TOTCAPB4; TOTCAPB3 for earlier survey years) in place of `pension_credit_reported_capital`. policyengine-uk's Universal Credit and Pension Credit reported capital both default to it, so Universal Credit's capital test now uses the FRS's own benefit-unit capital instead of the household wealth imputation, as Pension Credit already did. SPI-synthetic copies still get -1 (household proxy). diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 711127b51..0c0826ac9 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -790,25 +790,31 @@ def validate_frs_survey_year(raw_frs_folder, year: int) -> None: ) -def derive_pension_credit_reported_capital(benunit: pd.DataFrame) -> np.ndarray: - """Each benefit unit's capital as the FRS records it, for Pension Credit. +def derive_benunit_reported_capital(benunit: pd.DataFrame) -> np.ndarray: + """Each benefit unit's capital as the FRS records it. Uses ``TOTCAPB4``, DWP's derived benefit-unit total of the adults' savings and investments, which its below-average-resources statistics use in place of ``TOTCAPB3`` since it became available in 2019/20; ``TOTCAPB3`` is the - fallback for earlier survey years. Pension Credit counts the claimant's - capital and, under the State Pension Credit Act 2002 s. 5, the partner's, - and this is a benefit-unit measure. It is an approximation of Pension - Credit capital, not the assessed figure: it covers financial assets only - (second homes and land, which Pension Credit also counts, are not in it), - and no Schedule V disregard or reg. 19 valuation is applied to it. The - household wealth imputation instead draws a household's wealth from Wealth - and Assets Survey households with similar income, composition, tenure and - region, with no information on means-tested receipt, and policyengine-uk - spreads it over the household's pension-age adults. - - A missing or negative value gives -1, so policyengine-uk falls back to the - household proxy. + fallback for earlier survey years. This fills policyengine-uk's + ``benunit_reported_capital``, an input that records the observed figure + only. Each means test derives its own capital from it: Universal Credit + (Welfare Reform Act 2012 s. 5) and Pension Credit (State Pension Credit + Act 2002 s. 5) both assess the claimant's and partner's capital together, + which this benefit-unit measure covers. As a means-test figure it is an + approximation, not the assessed amount: it covers financial assets only + (second homes and land, which both tests also count, are not in it), and + no disregard or valuation rule is applied to it. The household wealth + imputation instead draws a household's wealth from Wealth and Assets + Survey households with similar income, composition, tenure and region, + with no information on means-tested receipt, and policyengine-uk shares + it out between the household's benefit units. + + Both source fields are derived pound amounts: DWP has already estimated + banded responses within those totals. No new band midpoint is applied + here. A finite, non-negative TOTCAPB4 takes precedence, with a row-wise + TOTCAPB3 fallback. If neither is usable, -1 tells policyengine-uk to use + the household proxy; an observed zero is kept. """ capital = np.full(len(benunit), np.nan) for column in ("totcapb3", "totcapb4"): # later columns take precedence @@ -1843,12 +1849,11 @@ def determine_education_level(fted_val, typeed2_val, age_val): pe_benunit["is_married"] = benunit.famtypb2.isin([5, 7]) - # Pension Credit capital as the FRS records it for the benefit unit, in - # place of the household wealth proxy (policyengine-uk - # `pension_credit_reported_capital`). - pe_benunit["pension_credit_reported_capital"] = ( - derive_pension_credit_reported_capital(benunit) - ) + # The benefit unit's capital as the FRS records it. policyengine-uk's + # Universal Credit and Pension Credit capital (`uc_reported_capital`, + # `pension_credit_reported_capital`) default to it in place of the + # household wealth proxy. + pe_benunit["benunit_reported_capital"] = derive_benunit_reported_capital(benunit) # Assign property_purchased to a share of households matching the UK # housing transaction rate, so only genuine purchasers are charged SDLT. diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index 799b5f35e..76cfd2b88 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -461,16 +461,24 @@ def impute_over_incomes( return dataset +REPORTED_CAPITAL_COLUMNS = ( + "benunit_reported_capital", + "pension_credit_reported_capital", + "uc_reported_capital", +) + + def clear_frs_reported_capital(dataset: UKSingleYearDataset) -> UKSingleYearDataset: - """Set ``pension_credit_reported_capital`` to -1 (none recorded). + """Set the FRS-recorded capital columns to -1 (none recorded). Used on the SPI-synthetic copy. The FRS benefit-unit capital belongs to the FRS donor, whose incomes the SPI imputation replaces; keeping it would assess an SPI-income unit on the donor's capital. With -1, policyengine-uk - uses the household capital proxy for these rows. + uses the household capital proxy for these rows in every means test. """ - if "pension_credit_reported_capital" in dataset.benunit.columns: - dataset.benunit["pension_credit_reported_capital"] = -1.0 + for column in REPORTED_CAPITAL_COLUMNS: + if column in dataset.benunit.columns: + dataset.benunit[column] = -1.0 return dataset diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index d6c4229b7..6d97e9172 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -1,6 +1,7 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 afcs_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 alcohol_and_tobacco_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +benunit_reported_capital,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 bsp_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 bus_fare_spending,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 bus_fare_spending_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 @@ -65,7 +66,10 @@ petrol_spending,1.0,1.104066,1.63457,1.795584,1.530966,1.483159,1.452034,1.40618 private_pension_income,1.0,1.003,1.05315,1.105808,1.161098,1.202781,1.252095,1.29091,1.329638,1.366867,1.406507,1.438856,1.47195,1.506835,1.542698 private_pension_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 private_transfer_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +property_finance_costs,1.0,0.9774,1.120296,1.705538,2.17115,2.377844,2.567358,2.834363,2.969279,3.108241,3.280127,3.461518,3.65294,3.854948,4.068126 +property_finance_costs_brought_forward,1.0,0.9774,1.120296,1.705538,2.17115,2.377844,2.567358,2.834363,2.969279,3.108241,3.280127,3.461518,3.65294,3.854948,4.068126 property_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +property_rental_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 recreation_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 restaurants_and_hotels_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 savings,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 @@ -80,6 +84,7 @@ statutory_sick_pay,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.34604 student_loan_repayments,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 sublet_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 transport_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +uc_reported_capital,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 universal_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 winter_fuel_allowance_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 working_tax_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 6f947d478..ed07471aa 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -1,6 +1,7 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 afcs_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 alcohol_and_tobacco_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +benunit_reported_capital,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 bsp_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 bus_fare_spending,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 bus_fare_spending_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 @@ -65,7 +66,10 @@ petrol_spending,0,0.104066,0.4805,0.098505,-0.147372,-0.031227,-0.020986,-0.0315 private_pension_income,0,0.003,0.05,0.05,0.05,0.0359,0.041,0.031,0.030001,0.027999,0.029001,0.023,0.023,0.0237,0.0238 private_pension_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 private_transfer_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +property_finance_costs,0,-0.0226,0.1462,0.522399,0.273,0.0952,0.0797,0.104,0.0476,0.0468,0.0553,0.0553,0.0553,0.0553,0.0553 +property_finance_costs_brought_forward,0,-0.0226,0.1462,0.522399,0.273,0.0952,0.0797,0.104,0.0476,0.0468,0.0553,0.0553,0.0553,0.0553,0.0553 property_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +property_rental_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 recreation_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 restaurants_and_hotels_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 savings,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 @@ -80,6 +84,7 @@ statutory_sick_pay,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.0 student_loan_repayments,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 sublet_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 transport_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +uc_reported_capital,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 universal_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 winter_fuel_allowance_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 working_tax_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 diff --git a/policyengine_uk_data/tests/test_benunit_reported_capital.py b/policyengine_uk_data/tests/test_benunit_reported_capital.py new file mode 100644 index 000000000..eb5454b9e --- /dev/null +++ b/policyengine_uk_data/tests/test_benunit_reported_capital.py @@ -0,0 +1,140 @@ +"""`benunit_reported_capital` from the FRS benefit-unit capital measure +(TOTCAPB4, falling back to TOTCAPB3). policyengine-uk's Universal Credit and +Pension Credit reported capital default to it. + +Invariants: +1. A finite, non-negative TOTCAPB4 is carried over unchanged; where TOTCAPB4 + is absent or invalid, a valid TOTCAPB3 is used. +2. A missing, non-numeric or negative value in both gives -1 + (policyengine-uk's "none recorded" sentinel), so the household proxy + applies. +3. Without either column every benefit unit gets -1. +4. The output is always -1 or a non-negative number, one per benefit unit. + +Uprating is checked against the engine for every row, including this +column, in test_uprating_factors_table.py. +""" + +import numpy as np +import pandas as pd + +from policyengine_uk_data.datasets.frs import derive_benunit_reported_capital + + +def test_values_carry_over_and_invalid_values_fall_back(): + benunit = pd.DataFrame( + {"totcapb3": [0.0, 300.0, 2_900.0, 1_250_000.0, np.nan, -5.0, "x"]} + ) + result = derive_benunit_reported_capital(benunit) + assert result.tolist() == [0.0, 300.0, 2_900.0, 1_250_000.0, -1.0, -1.0, -1.0] + + +def test_totcapb4_takes_precedence_with_totcapb3_fallback(): + benunit = pd.DataFrame( + { + "totcapb3": [100.0, 200.0, 300.0, np.nan, -1.0], + "totcapb4": [150.0, np.nan, -5.0, 400.0, np.nan], + } + ) + result = derive_benunit_reported_capital(benunit) + assert result.tolist() == [150.0, 200.0, 300.0, 400.0, -1.0] + + +def test_nullable_inputs_keep_the_totcapb3_fallback(): + """Nullable (pd.NA) columns must not wipe a valid TOTCAPB3 where TOTCAPB4 + is missing.""" + benunit = pd.DataFrame( + { + "totcapb3": pd.array([0.0, 250.0, pd.NA, -3.0], dtype="Float64"), + "totcapb4": pd.array([pd.NA, pd.NA, 400.0, pd.NA], dtype="Float64"), + } + ) + result = derive_benunit_reported_capital(benunit) + assert result.tolist() == [0.0, 250.0, 400.0, -1.0] + + +def test_missing_column_gives_sentinel(): + benunit = pd.DataFrame({"benunit_id": [101, 102, 201]}) + assert derive_benunit_reported_capital(benunit).tolist() == [-1.0] * 3 + + +def test_output_is_sentinel_or_non_negative_for_random_inputs(): + rng = np.random.default_rng(1_792) + for _ in range(50): + n = int(rng.integers(1, 200)) + values = rng.normal(5_000, 20_000, n) + values[rng.random(n) < 0.1] = np.nan + result = derive_benunit_reported_capital(pd.DataFrame({"totcapb4": values})) + assert len(result) == n + assert np.all((result == -1) | (result >= 0)) + keep = np.isfinite(values) & (values >= 0) + np.testing.assert_array_equal(result[keep], values[keep]) + assert np.all(result[~keep] == -1) + + +def test_uprating_rows_match_the_model(): + """Capital follows the engine's load-time GDP growth at calibration too.""" + from policyengine_uk_data.storage import STORAGE_FOLDER + from policyengine_uk_data.utils.uprating import ( + END_YEAR, + START_YEAR, + UPRATING_TABLE_DECIMALS, + policyengine_uk_load_time_index, + ) + + years = range(START_YEAR, END_YEAR + 1) + expected = policyengine_uk_load_time_index().loc["benunit_reported_capital"] + expected = expected.round(UPRATING_TABLE_DECIMALS) + factors = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable") + growth = pd.read_csv(STORAGE_FOLDER / "uprating_growth_factors.csv").set_index( + "Variable" + ) + for y in years: + assert factors.loc["benunit_reported_capital", str(y)] == expected[y] + expected_growth = ( + 0 + if y == START_YEAR + else round(expected[y] / expected[y - 1] - 1, UPRATING_TABLE_DECIMALS) + ) + assert growth.loc["benunit_reported_capital", str(y)] == expected_growth + + +def test_programme_capital_defaults_to_the_recorded_input(): + """Programme formulas and legacy stored inputs use the same capital index.""" + from policyengine_uk.system import system + + from policyengine_uk_data.storage import STORAGE_FOLDER + + assert not system.variables["benunit_reported_capital"].formulas + factors = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable") + for name in ("pension_credit_reported_capital", "uc_reported_capital"): + assert system.variables[name].formulas + # The engine also projects older datasets that store programme + # inputs directly, so its load-time index keeps these rows. + np.testing.assert_array_equal( + factors.loc[name], factors.loc["benunit_reported_capital"] + ) + + +def test_spi_copy_records_no_capital(): + """SPI-synthetic copies carry SPI-imputed incomes, so the FRS donor's + capital is cleared to -1 (household proxy) on them.""" + from types import SimpleNamespace + + from policyengine_uk_data.datasets.imputations.income import ( + clear_frs_reported_capital, + ) + + copy = SimpleNamespace( + benunit=pd.DataFrame( + { + "benunit_reported_capital": [0.0, 300.0, -1.0], + "uc_reported_capital": [5.0, -1.0, 7.0], + } + ) + ) + cleared = clear_frs_reported_capital(copy).benunit + assert cleared["benunit_reported_capital"].tolist() == [-1.0, -1.0, -1.0] + assert cleared["uc_reported_capital"].tolist() == [-1.0, -1.0, -1.0] + without = SimpleNamespace(benunit=pd.DataFrame({"benunit_id": [1, 2]})) + assert list(clear_frs_reported_capital(without).benunit.columns) == ["benunit_id"] diff --git a/policyengine_uk_data/tests/test_frs_capital_uc.py b/policyengine_uk_data/tests/test_frs_capital_uc.py new file mode 100644 index 000000000..ec4a944b3 --- /dev/null +++ b/policyengine_uk_data/tests/test_frs_capital_uc.py @@ -0,0 +1,112 @@ +"""MOCK FRS capital reaches the locked UK model's Universal Credit rules.""" + +from pathlib import Path + +import pandas as pd +import pytest +from policyengine_uk import Simulation + +from policyengine_uk_data.tests.test_legacy_benefit_proxies import ( + create_single_adult_frs, +) + + +def _simulation_from_mock_frs(tmp_path, monkeypatch, capital_fields): + read_csv = pd.read_csv + + def read_mock_frs_table(path, *args, **kwargs): + table = read_csv(path, *args, **kwargs) + if Path(path).stem == "benunit": + table = table.assign(**capital_fields) + return table + + # The existing MOCK fixture stubs processing-time policy calculations. + # Restore those stubs before exercising the real UK model below. + with monkeypatch.context() as frs_patch: + frs_patch.setattr(pd, "read_csv", read_mock_frs_table) + dataset = create_single_adult_frs(tmp_path, frs_patch) + + # -1 is the UK model's fallback if create_frs omits the reported input. + # Keeping that fallback makes the regression on main a capital/UC + # assertion failure, rather than a missing-column or helper failure. + reported_capital = float( + dataset.benunit.get("benunit_reported_capital", pd.Series([-1.0])).iloc[0] + ) + return Simulation( + situation={ + "people": { + "claimant": { + "age": {2025: 30}, + "is_claimant_or_partner": {2025: True}, + } + }, + "benunits": { + "benunit": { + "members": ["claimant"], + "benunit_reported_capital": {2025: reported_capital}, + "would_claim_uc": {2025: True}, + } + }, + "households": { + "household": { + "members": ["claimant"], + # Reported benefit-unit capital must override the + # household wealth proxy, including an explicit zero. + "savings": {2025: 50_000}, + } + }, + } + ) + + +@pytest.mark.parametrize( + "capital_fields, capital, tariff_steps, eligible", + [ + ({"totcapb4": 6_000}, 6_000, 0, True), + ({"totcapb4": 6_001}, 6_001, 1, True), + ({"totcapb4": 6_250}, 6_250, 1, True), + ({"totcapb4": 6_251}, 6_251, 2, True), + ({"totcapb4": 16_000}, 16_000, 40, True), + ({"totcapb4": 16_001}, 16_001, 0, False), + ({"totcapb3": 6_251}, 6_251, 2, True), + ({"totcapb4": -9, "totcapb3": 6_001}, 6_001, 1, True), + ({"totcapb4": None, "totcapb3": 6_250}, 6_250, 1, True), + ({"totcapb4": 0, "totcapb3": 16_001}, 0, 0, True), + ], + ids=[ + "tariff-threshold", + "first-pound-over-threshold", + "first-tariff-step-end", + "second-tariff-step-start", + "capital-limit", + "over-capital-limit", + "older-frs-total", + "negative-new-total-falls-back", + "missing-new-total-falls-back", + "explicit-zero-overrides-old-total-and-wealth", + ], +) +def test_mock_frs_capital_reaches_universal_credit( + tmp_path, monkeypatch, capital_fields, capital, tariff_steps, eligible +): + sim = _simulation_from_mock_frs(tmp_path, monkeypatch, capital_fields) + + def value(variable): + return sim.calculate(variable, 2025)[0] + + for variable in ( + "benunit_reported_capital", + "uc_reported_capital", + "uc_assessable_capital", + ): + assert value(variable) == capital, variable + assert bool(value("is_uc_eligible")) is eligible + + # UC Regs 2013 regs 18 and 72: each started £250 over £6,000 + # contributes £4.35 a month, until capital exceeds £16,000. + annual_tariff = tariff_steps * 4.35 * 12 + assert value("uc_tariff_income") == pytest.approx(annual_tariff) + standard_allowance = value("uc_standard_allowance") + assert standard_allowance > 0 + expected_uc = standard_allowance - annual_tariff if eligible else 0 + assert value("universal_credit") == pytest.approx(expected_uc) diff --git a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py deleted file mode 100644 index dcdbfe166..000000000 --- a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py +++ /dev/null @@ -1,94 +0,0 @@ -"""`pension_credit_reported_capital` from the FRS benefit-unit capital -measure (TOTCAPB4, falling back to TOTCAPB3). - -Invariants: -1. A finite, non-negative TOTCAPB4 is carried over unchanged; where TOTCAPB4 - is absent or invalid, a valid TOTCAPB3 is used. -2. A missing, non-numeric or negative value in both gives -1 - (policyengine-uk's "none recorded" sentinel), so the household proxy - applies. -3. Without either column every benefit unit gets -1. -4. The output is always -1 or a non-negative number, one per benefit unit. - -Uprating is checked against the engine for every row, including this -column, in test_uprating_factors_table.py. -""" - -import numpy as np -import pandas as pd - -from policyengine_uk_data.datasets.frs import derive_pension_credit_reported_capital - - -def test_values_carry_over_and_invalid_values_fall_back(): - benunit = pd.DataFrame( - {"totcapb3": [0.0, 300.0, 2_900.0, 1_250_000.0, np.nan, -5.0, "x"]} - ) - result = derive_pension_credit_reported_capital(benunit) - assert result.tolist() == [0.0, 300.0, 2_900.0, 1_250_000.0, -1.0, -1.0, -1.0] - - -def test_totcapb4_takes_precedence_with_totcapb3_fallback(): - benunit = pd.DataFrame( - { - "totcapb3": [100.0, 200.0, 300.0, np.nan, -1.0], - "totcapb4": [150.0, np.nan, -5.0, 400.0, np.nan], - } - ) - result = derive_pension_credit_reported_capital(benunit) - assert result.tolist() == [150.0, 200.0, 300.0, 400.0, -1.0] - - -def test_nullable_inputs_keep_the_totcapb3_fallback(): - """Nullable (pd.NA) columns must not wipe a valid TOTCAPB3 where TOTCAPB4 - is missing.""" - benunit = pd.DataFrame( - { - "totcapb3": pd.array([0.0, 250.0, pd.NA, -3.0], dtype="Float64"), - "totcapb4": pd.array([pd.NA, pd.NA, 400.0, pd.NA], dtype="Float64"), - } - ) - result = derive_pension_credit_reported_capital(benunit) - assert result.tolist() == [0.0, 250.0, 400.0, -1.0] - - -def test_missing_column_gives_sentinel(): - benunit = pd.DataFrame({"benunit_id": [101, 102, 201]}) - assert derive_pension_credit_reported_capital(benunit).tolist() == [-1.0] * 3 - - -def test_output_is_sentinel_or_non_negative_for_random_inputs(): - rng = np.random.default_rng(1_792) - for _ in range(50): - n = int(rng.integers(1, 200)) - values = rng.normal(5_000, 20_000, n) - values[rng.random(n) < 0.1] = np.nan - result = derive_pension_credit_reported_capital( - pd.DataFrame({"totcapb4": values}) - ) - assert len(result) == n - assert np.all((result == -1) | (result >= 0)) - keep = np.isfinite(values) & (values >= 0) - np.testing.assert_array_equal(result[keep], values[keep]) - assert np.all(result[~keep] == -1) - - -def test_spi_copy_records_no_capital(): - """SPI-synthetic copies carry SPI-imputed incomes, so the FRS donor's - capital is cleared to -1 (household proxy) on them.""" - from types import SimpleNamespace - - from policyengine_uk_data.datasets.imputations.income import ( - clear_frs_reported_capital, - ) - - copy = SimpleNamespace( - benunit=pd.DataFrame({"pension_credit_reported_capital": [0.0, 300.0, -1.0]}) - ) - assert clear_frs_reported_capital(copy).benunit[ - "pension_credit_reported_capital" - ].tolist() == [-1.0, -1.0, -1.0] - without = SimpleNamespace(benunit=pd.DataFrame({"benunit_id": [1, 2]})) - assert "pension_credit_reported_capital" not in ( - clear_frs_reported_capital(without).benunit.columns - ) diff --git a/pyproject.toml b/pyproject.toml index e41f83dfc..19b691c78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "policyengine", "google-cloud-storage", "google-auth", - "policyengine-uk>=2.122.0", + "policyengine-uk>=2.124.0", "microcalibrate>=0.18.0", "microimpute>=1.0.1", "ruff>=0.9.0", diff --git a/uv.lock b/uv.lock index dce9c8285..2491d1896 100644 --- a/uv.lock +++ b/uv.lock @@ -1391,7 +1391,7 @@ wheels = [ [[package]] name = "policyengine-core" -version = "3.32.13" +version = "3.33.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -1411,14 +1411,14 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/0a/b6c27953e1083d3ab2c40330c49580130fff48a2fe9cbfc8d731e92882e6/policyengine_core-3.32.13.tar.gz", hash = "sha256:a870f7e212fdfa1b5fdad7df92aa2b325fd0280ede16a40aeeceea74d44fb864", size = 457899, upload-time = "2026-10-03T11:30:20.572Z" } +sdist = { url = "https://files.pythonhosted.org/packages/84/dc/b6e44d9cdb9afee354f89584eebe310d2e6480f852ca51d55570eee5a648/policyengine_core-3.33.2.tar.gz", hash = "sha256:4f0420021a5439b8d3b7c86333e291b003f4be689aebfff49cb10ad1b6952ccb", size = 693096, upload-time = "2026-10-10T19:09:47.182Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/01/00ebf4c13152347bc5396b40e80445fa92ea96712b9ec0bfdca7c5d11413/policyengine_core-3.32.13-py3-none-any.whl", hash = "sha256:262543c3ac696207f304f55664711b72e894fbad7b566407afa9d30b16086a41", size = 251251, upload-time = "2026-10-03T11:30:19.171Z" }, + { url = "https://files.pythonhosted.org/packages/43/1b/9950def57dc9ec82aa2647aed02237199e347b5b7ecb898d5dfaf70014bc/policyengine_core-3.33.2-py3-none-any.whl", hash = "sha256:631e20811b74f6a7df906891efa5c78fbfd461ecefd65902b6e1456b1a0e4098", size = 301904, upload-time = "2026-10-10T19:09:45.432Z" }, ] [[package]] name = "policyengine-uk" -version = "2.122.2" +version = "2.126.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1426,9 +1426,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/62/4378fd8e493df42fef30c58924dbb0f7ca8111c927302f6e60be862d67d0/policyengine_uk-2.122.2.tar.gz", hash = "sha256:0083793f1d398feef28a64db5508d5a64bcd7834feeade6cde1055d419d505ed", size = 1976779, upload-time = "2026-10-06T02:10:56.786Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/db/ddd5cb838cbac73e85fe21df09a99fc391a508d86bc1c158420990752dc8/policyengine_uk-2.126.8.tar.gz", hash = "sha256:5063920bacd5d74a32444336de638586ba1399209083d7976f813264e6a8904c", size = 2208548, upload-time = "2026-10-11T13:32:58.405Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/63/11/09948c900ca6d58c1050c89b9c93d8f5884244aa6ac95661ef1a2ee9eb9a/policyengine_uk-2.122.2-py3-none-any.whl", hash = "sha256:14cefcedce43ce4a37d659a0ff17ad05695de0dc8e0c5f6e2259a69944cb5558", size = 3118697, upload-time = "2026-10-06T02:10:54.896Z" }, + { url = "https://files.pythonhosted.org/packages/75/da/6416e8f3969155c5ef0623e6e6b23d32af328382248951384d995f35cb09/policyengine_uk-2.126.8-py3-none-any.whl", hash = "sha256:8ac0b48746b87bc32eea4ab06dde3646ddd3f184e735be41c9652bfe85115bce", size = 3432792, upload-time = "2026-10-11T13:32:56.413Z" }, ] [[package]] @@ -1490,7 +1490,7 @@ requires-dist = [ { name = "pandas" }, { name = "policyengine" }, { name = "policyengine-core", specifier = ">=3.32.13" }, - { name = "policyengine-uk", specifier = ">=2.122.0" }, + { name = "policyengine-uk", specifier = ">=2.124.0" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pyyaml" },