Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz

# Hypothesis example database (property-based tests)
.hypothesis/

# Local build-job evidence and Git metadata
.job/
1 change: 1 addition & 0 deletions changelog.d/frs-benunit-reported-capital.changed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Fill policyengine-uk's `benunit_reported_capital` with each benefit unit's FRS total capital (TOTCAPB4; TOTCAPB3 for earlier survey years) in place of `pension_credit_reported_capital`. policyengine-uk's Universal Credit and Pension Credit reported capital both default to it, so Universal Credit's capital test now uses the FRS's own benefit-unit capital instead of the household wealth imputation, as Pension Credit already did. SPI-synthetic copies still get -1 (household proxy).
47 changes: 26 additions & 21 deletions policyengine_uk_data/datasets/frs.py
Original file line number Diff line number Diff line change
Expand Up @@ -790,25 +790,31 @@ def validate_frs_survey_year(raw_frs_folder, year: int) -> None:
)


def derive_pension_credit_reported_capital(benunit: pd.DataFrame) -> np.ndarray:
"""Each benefit unit's capital as the FRS records it, for Pension Credit.
def derive_benunit_reported_capital(benunit: pd.DataFrame) -> np.ndarray:
"""Each benefit unit's capital as the FRS records it.

Uses ``TOTCAPB4``, DWP's derived benefit-unit total of the adults' savings
and investments, which its below-average-resources statistics use in place
of ``TOTCAPB3`` since it became available in 2019/20; ``TOTCAPB3`` is the
fallback for earlier survey years. Pension Credit counts the claimant's
capital and, under the State Pension Credit Act 2002 s. 5, the partner's,
and this is a benefit-unit measure. It is an approximation of Pension
Credit capital, not the assessed figure: it covers financial assets only
(second homes and land, which Pension Credit also counts, are not in it),
and no Schedule V disregard or reg. 19 valuation is applied to it. The
household wealth imputation instead draws a household's wealth from Wealth
and Assets Survey households with similar income, composition, tenure and
region, with no information on means-tested receipt, and policyengine-uk
spreads it over the household's pension-age adults.

A missing or negative value gives -1, so policyengine-uk falls back to the
household proxy.
fallback for earlier survey years. This fills policyengine-uk's
``benunit_reported_capital``, an input that records the observed figure
only. Each means test derives its own capital from it: Universal Credit
(Welfare Reform Act 2012 s. 5) and Pension Credit (State Pension Credit
Act 2002 s. 5) both assess the claimant's and partner's capital together,
which this benefit-unit measure covers. As a means-test figure it is an
approximation, not the assessed amount: it covers financial assets only
(second homes and land, which both tests also count, are not in it), and
no disregard or valuation rule is applied to it. The household wealth
imputation instead draws a household's wealth from Wealth and Assets
Survey households with similar income, composition, tenure and region,
with no information on means-tested receipt, and policyengine-uk shares
it out between the household's benefit units.

Both source fields are derived pound amounts: DWP has already estimated
banded responses within those totals. No new band midpoint is applied
here. A finite, non-negative TOTCAPB4 takes precedence, with a row-wise
TOTCAPB3 fallback. If neither is usable, -1 tells policyengine-uk to use
the household proxy; an observed zero is kept.
"""
capital = np.full(len(benunit), np.nan)
for column in ("totcapb3", "totcapb4"): # later columns take precedence
Expand Down Expand Up @@ -1843,12 +1849,11 @@ def determine_education_level(fted_val, typeed2_val, age_val):

pe_benunit["is_married"] = benunit.famtypb2.isin([5, 7])

# Pension Credit capital as the FRS records it for the benefit unit, in
# place of the household wealth proxy (policyengine-uk
# `pension_credit_reported_capital`).
pe_benunit["pension_credit_reported_capital"] = (
derive_pension_credit_reported_capital(benunit)
)
# The benefit unit's capital as the FRS records it. policyengine-uk's
# Universal Credit and Pension Credit capital (`uc_reported_capital`,
# `pension_credit_reported_capital`) default to it in place of the
# household wealth proxy.
pe_benunit["benunit_reported_capital"] = derive_benunit_reported_capital(benunit)

# Assign property_purchased to a share of households matching the UK
# housing transaction rate, so only genuine purchasers are charged SDLT.
Expand Down
16 changes: 12 additions & 4 deletions policyengine_uk_data/datasets/imputations/income.py
Original file line number Diff line number Diff line change
Expand Up @@ -461,16 +461,24 @@ def impute_over_incomes(
return dataset


REPORTED_CAPITAL_COLUMNS = (
"benunit_reported_capital",
"pension_credit_reported_capital",
"uc_reported_capital",
)


def clear_frs_reported_capital(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
"""Set ``pension_credit_reported_capital`` to -1 (none recorded).
"""Set the FRS-recorded capital columns to -1 (none recorded).

Used on the SPI-synthetic copy. The FRS benefit-unit capital belongs to the
FRS donor, whose incomes the SPI imputation replaces; keeping it would
assess an SPI-income unit on the donor's capital. With -1, policyengine-uk
uses the household capital proxy for these rows.
uses the household capital proxy for these rows in every means test.
"""
if "pension_credit_reported_capital" in dataset.benunit.columns:
dataset.benunit["pension_credit_reported_capital"] = -1.0
for column in REPORTED_CAPITAL_COLUMNS:
if column in dataset.benunit.columns:
dataset.benunit[column] = -1.0
return dataset


Expand Down
5 changes: 5 additions & 0 deletions policyengine_uk_data/storage/uprating_factors.csv
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034
afcs_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
alcohol_and_tobacco_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
benunit_reported_capital,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269
bsp_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
bus_fare_spending,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
bus_fare_spending_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
Expand Down Expand Up @@ -65,7 +66,10 @@ petrol_spending,1.0,1.104066,1.63457,1.795584,1.530966,1.483159,1.452034,1.40618
private_pension_income,1.0,1.003,1.05315,1.105808,1.161098,1.202781,1.252095,1.29091,1.329638,1.366867,1.406507,1.438856,1.47195,1.506835,1.542698
private_pension_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269
private_transfer_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269
property_finance_costs,1.0,0.9774,1.120296,1.705538,2.17115,2.377844,2.567358,2.834363,2.969279,3.108241,3.280127,3.461518,3.65294,3.854948,4.068126
property_finance_costs_brought_forward,1.0,0.9774,1.120296,1.705538,2.17115,2.377844,2.567358,2.834363,2.969279,3.108241,3.280127,3.461518,3.65294,3.854948,4.068126
property_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269
property_rental_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269
recreation_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
restaurants_and_hotels_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
savings,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269
Expand All @@ -80,6 +84,7 @@ statutory_sick_pay,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.34604
student_loan_repayments,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379
sublet_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269
transport_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
uc_reported_capital,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269
universal_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
winter_fuel_allowance_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
working_tax_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179
5 changes: 5 additions & 0 deletions policyengine_uk_data/storage/uprating_growth_factors.csv
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034
afcs_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
alcohol_and_tobacco_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
benunit_reported_capital,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315
bsp_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
bus_fare_spending,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
bus_fare_spending_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
Expand Down Expand Up @@ -65,7 +66,10 @@ petrol_spending,0,0.104066,0.4805,0.098505,-0.147372,-0.031227,-0.020986,-0.0315
private_pension_income,0,0.003,0.05,0.05,0.05,0.0359,0.041,0.031,0.030001,0.027999,0.029001,0.023,0.023,0.0237,0.0238
private_pension_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315
private_transfer_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315
property_finance_costs,0,-0.0226,0.1462,0.522399,0.273,0.0952,0.0797,0.104,0.0476,0.0468,0.0553,0.0553,0.0553,0.0553,0.0553
property_finance_costs_brought_forward,0,-0.0226,0.1462,0.522399,0.273,0.0952,0.0797,0.104,0.0476,0.0468,0.0553,0.0553,0.0553,0.0553,0.0553
property_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315
property_rental_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315
recreation_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
restaurants_and_hotels_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
savings,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315
Expand All @@ -80,6 +84,7 @@ statutory_sick_pay,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.0
student_loan_repayments,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377
sublet_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315
transport_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
uc_reported_capital,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315
universal_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
winter_fuel_allowance_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
working_tax_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02
140 changes: 140 additions & 0 deletions policyengine_uk_data/tests/test_benunit_reported_capital.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
"""`benunit_reported_capital` from the FRS benefit-unit capital measure
(TOTCAPB4, falling back to TOTCAPB3). policyengine-uk's Universal Credit and
Pension Credit reported capital default to it.

Invariants:
1. A finite, non-negative TOTCAPB4 is carried over unchanged; where TOTCAPB4
is absent or invalid, a valid TOTCAPB3 is used.
2. A missing, non-numeric or negative value in both gives -1
(policyengine-uk's "none recorded" sentinel), so the household proxy
applies.
3. Without either column every benefit unit gets -1.
4. The output is always -1 or a non-negative number, one per benefit unit.

Uprating is checked against the engine for every row, including this
column, in test_uprating_factors_table.py.
"""

import numpy as np
import pandas as pd

from policyengine_uk_data.datasets.frs import derive_benunit_reported_capital


def test_values_carry_over_and_invalid_values_fall_back():
benunit = pd.DataFrame(
{"totcapb3": [0.0, 300.0, 2_900.0, 1_250_000.0, np.nan, -5.0, "x"]}
)
result = derive_benunit_reported_capital(benunit)
assert result.tolist() == [0.0, 300.0, 2_900.0, 1_250_000.0, -1.0, -1.0, -1.0]


def test_totcapb4_takes_precedence_with_totcapb3_fallback():
benunit = pd.DataFrame(
{
"totcapb3": [100.0, 200.0, 300.0, np.nan, -1.0],
"totcapb4": [150.0, np.nan, -5.0, 400.0, np.nan],
}
)
result = derive_benunit_reported_capital(benunit)
assert result.tolist() == [150.0, 200.0, 300.0, 400.0, -1.0]


def test_nullable_inputs_keep_the_totcapb3_fallback():
"""Nullable (pd.NA) columns must not wipe a valid TOTCAPB3 where TOTCAPB4
is missing."""
benunit = pd.DataFrame(
{
"totcapb3": pd.array([0.0, 250.0, pd.NA, -3.0], dtype="Float64"),
"totcapb4": pd.array([pd.NA, pd.NA, 400.0, pd.NA], dtype="Float64"),
}
)
result = derive_benunit_reported_capital(benunit)
assert result.tolist() == [0.0, 250.0, 400.0, -1.0]


def test_missing_column_gives_sentinel():
benunit = pd.DataFrame({"benunit_id": [101, 102, 201]})
assert derive_benunit_reported_capital(benunit).tolist() == [-1.0] * 3


def test_output_is_sentinel_or_non_negative_for_random_inputs():
rng = np.random.default_rng(1_792)
for _ in range(50):
n = int(rng.integers(1, 200))
values = rng.normal(5_000, 20_000, n)
values[rng.random(n) < 0.1] = np.nan
result = derive_benunit_reported_capital(pd.DataFrame({"totcapb4": values}))
assert len(result) == n
assert np.all((result == -1) | (result >= 0))
keep = np.isfinite(values) & (values >= 0)
np.testing.assert_array_equal(result[keep], values[keep])
assert np.all(result[~keep] == -1)


def test_uprating_rows_match_the_model():
"""Capital follows the engine's load-time GDP growth at calibration too."""
from policyengine_uk_data.storage import STORAGE_FOLDER
from policyengine_uk_data.utils.uprating import (
END_YEAR,
START_YEAR,
UPRATING_TABLE_DECIMALS,
policyengine_uk_load_time_index,
)

years = range(START_YEAR, END_YEAR + 1)
expected = policyengine_uk_load_time_index().loc["benunit_reported_capital"]
expected = expected.round(UPRATING_TABLE_DECIMALS)
factors = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable")
growth = pd.read_csv(STORAGE_FOLDER / "uprating_growth_factors.csv").set_index(
"Variable"
)
for y in years:
assert factors.loc["benunit_reported_capital", str(y)] == expected[y]
expected_growth = (
0
if y == START_YEAR
else round(expected[y] / expected[y - 1] - 1, UPRATING_TABLE_DECIMALS)
)
assert growth.loc["benunit_reported_capital", str(y)] == expected_growth


def test_programme_capital_defaults_to_the_recorded_input():
"""Programme formulas and legacy stored inputs use the same capital index."""
from policyengine_uk.system import system

from policyengine_uk_data.storage import STORAGE_FOLDER

assert not system.variables["benunit_reported_capital"].formulas
factors = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable")
for name in ("pension_credit_reported_capital", "uc_reported_capital"):
assert system.variables[name].formulas
# The engine also projects older datasets that store programme
# inputs directly, so its load-time index keeps these rows.
np.testing.assert_array_equal(
factors.loc[name], factors.loc["benunit_reported_capital"]
)


def test_spi_copy_records_no_capital():
"""SPI-synthetic copies carry SPI-imputed incomes, so the FRS donor's
capital is cleared to -1 (household proxy) on them."""
from types import SimpleNamespace

from policyengine_uk_data.datasets.imputations.income import (
clear_frs_reported_capital,
)

copy = SimpleNamespace(
benunit=pd.DataFrame(
{
"benunit_reported_capital": [0.0, 300.0, -1.0],
"uc_reported_capital": [5.0, -1.0, 7.0],
}
)
)
cleared = clear_frs_reported_capital(copy).benunit
assert cleared["benunit_reported_capital"].tolist() == [-1.0, -1.0, -1.0]
assert cleared["uc_reported_capital"].tolist() == [-1.0, -1.0, -1.0]
without = SimpleNamespace(benunit=pd.DataFrame({"benunit_id": [1, 2]}))
assert list(clear_frs_reported_capital(without).benunit.columns) == ["benunit_id"]
Loading
Loading