From 2d8f56556602d3d7ba5861f8ead3dbfeed40bd9d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 17:12:04 -0400 Subject: [PATCH 1/4] Keep FRS-reported dividends instead of overwriting them with SPI draws The SPI income model predicts from age, gender and region only; applying it to the FRS half gave Universal Credit claimants six-figure dividends unrelated to anything they reported (policyengine-uk#1948). Only the SPI-donor copy is now imputed, as in the microcosm UK build. Co-Authored-By: Claude Opus 5.5 --- changelog.d/frs-reported-dividends.fixed.md | 1 + .../datasets/imputations/income.py | 15 +++-- .../tests/test_imputation_source_flags.py | 56 +++++++++++++++++++ 3 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 changelog.d/frs-reported-dividends.fixed.md diff --git a/changelog.d/frs-reported-dividends.fixed.md b/changelog.d/frs-reported-dividends.fixed.md new file mode 100644 index 00000000..389b32f3 --- /dev/null +++ b/changelog.d/frs-reported-dividends.fixed.md @@ -0,0 +1 @@ +- The enhanced FRS keeps FRS respondents' reported dividends instead of replacing them with draws from the SPI income model, which predicts from age, gender and region alone and gave Universal Credit claimants six-figure dividends (policyengine-uk#1948). The SPI-donor half still carries the SPI income distribution. diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index feafcb27..9af0d7c4 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -266,7 +266,7 @@ def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset: model = create_income_model() - # Impute just dividends on the original, full variable set on the copy + # Impute the full income set on the SPI-donor copy only. zero_weight_copy = impute_over_incomes( zero_weight_copy, @@ -292,11 +292,14 @@ def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset: target_dataset=zero_weight_copy, ) - dataset = impute_over_incomes( - dataset, - model, - ["dividend_income"], - ) + # The FRS half keeps its reported dividends. Replacing them with a draw + # from the SPI model, whose only predictors are age, gender and region, + # gave dividends to FRS respondents without regard to their investments, + # earnings or benefits: in 2026 the FRS half carried £47bn of dividends + # against £17bn reported, and Universal Credit claimants received as + # many as anyone else (policyengine-uk#1948). The SPI-donor half carries + # the SPI dividend distribution, and calibration to the HMRC dividend + # targets reweights between the two, as in the microcosm UK build. zero_weight_copy.validate() dataset.validate() diff --git a/policyengine_uk_data/tests/test_imputation_source_flags.py b/policyengine_uk_data/tests/test_imputation_source_flags.py index 2df1f9d8..91f96db9 100644 --- a/policyengine_uk_data/tests/test_imputation_source_flags.py +++ b/policyengine_uk_data/tests/test_imputation_source_flags.py @@ -131,3 +131,59 @@ def test_impute_capital_gains_marks_capital_gains_clone_households(monkeypatch): True, ] assert result.household.loc[2:, "household_weight"].eq(1).all() + + +def test_impute_income_keeps_reported_dividends_on_the_frs_half(monkeypatch): + """SPI draws replace incomes only on the SPI-donor copy (policyengine-uk#1948). + + The SPI income model predicts from age, gender and region alone, so using + it to overwrite the FRS half's dividends gave Universal Credit claimants + dividends unrelated to anything they reported. + """ + from policyengine_uk_data.datasets.imputations import income as income_module + from policyengine_uk_data.datasets import disability_benefits + from policyengine_uk_data.datasets.imputations import frs_only + + imputed_halves = [] + + def fake_impute_over_incomes(dataset, _model, output_variables): + imputed_halves.append( + ( + bool(dataset.household["household_is_spi_synthetic"].all()), + tuple(output_variables), + ) + ) + dataset = dataset.copy() + for column in output_variables: + dataset.person[column] = 123_456.0 + return dataset + + monkeypatch.setattr(income_module, "create_income_model", lambda: object()) + monkeypatch.setattr( + income_module, + "subsample_dataset", + lambda dataset, _sample_size: dataset.copy(), + ) + monkeypatch.setattr(income_module, "impute_over_incomes", fake_impute_over_incomes) + monkeypatch.setattr( + frs_only, + "impute_frs_only_variables", + lambda train_dataset, target_dataset: target_dataset, + ) + monkeypatch.setattr( + disability_benefits, + "strip_internal_disability_reported_amounts", + lambda dataset: dataset, + ) + monkeypatch.setattr(income_module, "stack_datasets", _stack_without_remapping) + + dataset = _fake_dataset() + dataset.person["dividend_income"] = [150.0, 0.0] + result = income_module.impute_income(dataset) + + # Only the SPI-donor copy is imputed, with the full income set. + assert imputed_halves == [(True, tuple(income_module.IMPUTATIONS))] + frs_half = result.person.iloc[:2] + assert frs_half["dividend_income"].tolist() == [150.0, 0.0] + assert frs_half["employment_income"].tolist() == [20_000.0, 80_000.0] + assert result.person.iloc[2:]["dividend_income"].eq(123_456.0).all() From c9b203e34a79bfcee8908caf82100a52a70a1140 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 17:24:42 -0400 Subject: [PATCH 2/4] Key FRS dividends on person_id, not row position MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit frs.py summed investment-account dividends to person.index while account.person_id holds household_id * 1e3 + person, so only 8 people got any (about £40m against £8bn). Extract the calculation into a tested helper and annualise with WEEKS_IN_YEAR like the savings lines beside it. Correct the income.py comments that relied on the near-zero FRS dividends. Co-Authored-By: Claude Opus 5.5 --- changelog.d/frs-reported-dividends.fixed.md | 3 +- policyengine_uk_data/datasets/frs.py | 37 +++++++++++------ .../datasets/imputations/income.py | 12 +++--- .../tests/test_frs_dividend_income.py | 40 +++++++++++++++++++ 4 files changed, 72 insertions(+), 20 deletions(-) create mode 100644 policyengine_uk_data/tests/test_frs_dividend_income.py diff --git a/changelog.d/frs-reported-dividends.fixed.md b/changelog.d/frs-reported-dividends.fixed.md index 389b32f3..cf31851c 100644 --- a/changelog.d/frs-reported-dividends.fixed.md +++ b/changelog.d/frs-reported-dividends.fixed.md @@ -1 +1,2 @@ -- The enhanced FRS keeps FRS respondents' reported dividends instead of replacing them with draws from the SPI income model, which predicts from age, gender and region alone and gave Universal Credit claimants six-figure dividends (policyengine-uk#1948). The SPI-donor half still carries the SPI income distribution. +- FRS investment-account dividends are now summed to the person who holds the account. They were keyed on each person's row position, so almost none matched and the base FRS carried about £40m of dividends instead of about £8bn in 2024-25. +- The enhanced FRS keeps those reported dividends instead of replacing them with draws from the SPI income model, which predicts from age, gender and region alone and gave Universal Credit claimants six-figure dividends (policyengine-uk#1948). The SPI-donor half still carries the SPI income distribution. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index e440d486..3625f0f1 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -570,6 +570,29 @@ def validate_frs_survey_year(raw_frs_folder, year: int) -> None: ) +def frs_dividend_income(account: pd.DataFrame, person_ids) -> np.ndarray: + """Annual dividends each person reports on FRS investment accounts. + + Gilt-edged stock taxed at source (account type 6), unit and investment + trusts (7) and stocks and shares (8). Amounts taxed at source are grossed + up at the basic rate. Summed to ``person_ids``, which must be the same + ``household_id * 1e3 + person`` keys as ``account.person_id``: keying on + the positional index instead matched almost no accounts, leaving the FRS + with about £40m of dividends rather than about £8bn in 2024-25. + """ + INVERTED_BASIC_RATE = 1.25 + dividends = ( + account.accint * np.where(account.invtax == 1, INVERTED_BASIC_RATE, 1) + ) * ( + ((account.account == 6) & (account.invtax == 1)) # GGES + | account.account.isin((7, 8)) # Stocks/shares/UITs + ) + return np.maximum( + 0, + sum_to_entity(dividends, account.person_id, person_ids) * WEEKS_IN_YEAR, + ) + + def create_frs( raw_frs_folder: str, year: int, @@ -1070,19 +1093,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): 0, taxable_savings_interest + pe_person["tax_free_savings_income"].values, ) - pe_person["dividend_income"] = np.maximum( - 0, - sum_to_entity( - (account.accint * np.where(account.invtax == 1, INVERTED_BASIC_RATE, 1)) - * ( - ((account.account == 6) & (account.invtax == 1)) # GGES - | account.account.isin((7, 8)) # Stocks/shares/UITs - ), - account.person_id, - person.index, - ) - * 52, - ) + pe_person["dividend_income"] = frs_dividend_income(account, person.person_id) is_head = person.hrpid == 1 household_property_income = ( household.tentyp2.isin((5, 6)) * household.subrent diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index 9af0d7c4..aef5e55d 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -223,8 +223,9 @@ def impute_over_incomes( # Housing costs (rent, mortgage interest, mortgage capital) used to be # rescaled here by new_income_total / original_income_total across - # INCOME_COMPONENTS. Because FRS dividend_income is near-zero and the - # SPI-trained QRF predicts materially larger dividends, the ratio + # INCOME_COMPONENTS. Because FRS dividend_income was then near-zero (a + # keying error in frs.py, since fixed) and the SPI-trained QRF predicts + # materially larger dividends, the ratio # inflated rent/mortgage by ~2.5× uniformly in the built enhanced FRS # — pushing AHC poverty rates 10–18 pp above HBAI for non-pensioners # (see issue #367). Housing costs now pass through unchanged; their @@ -295,11 +296,10 @@ def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset: # The FRS half keeps its reported dividends. Replacing them with a draw # from the SPI model, whose only predictors are age, gender and region, # gave dividends to FRS respondents without regard to their investments, - # earnings or benefits: in 2026 the FRS half carried £47bn of dividends - # against £17bn reported, and Universal Credit claimants received as - # many as anyone else (policyengine-uk#1948). The SPI-donor half carries + # earnings or benefits, so Universal Credit claimants received them as + # often as anyone else (policyengine-uk#1948). The SPI-donor half carries # the SPI dividend distribution, and calibration to the HMRC dividend - # targets reweights between the two, as in the microcosm UK build. + # targets reweights between the two. zero_weight_copy.validate() dataset.validate() diff --git a/policyengine_uk_data/tests/test_frs_dividend_income.py b/policyengine_uk_data/tests/test_frs_dividend_income.py new file mode 100644 index 00000000..77f37f6c --- /dev/null +++ b/policyengine_uk_data/tests/test_frs_dividend_income.py @@ -0,0 +1,40 @@ +"""FRS dividends land on the person who holds the account (policyengine-uk#1948).""" + +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.frs import WEEKS_IN_YEAR, frs_dividend_income + + +def test_dividends_are_keyed_on_person_id_not_row_position(): + # Person ids are household_id * 1000 + person number, so they never + # coincide with the positional index the rows happen to sit at. + person_ids = pd.Series([1_001, 1_002, 2_001, 2_002]) + account = pd.DataFrame( + { + "person_id": [1_002, 2_001, 2_001, 2_002, 1_001, 2_002], + # 8: stocks and shares; 7: unit/investment trusts; 6: gilts; 1: bank account + "account": [8, 7, 6, 6, 1, 8], + "accint": [10.0, 4.0, 2.0, 3.0, 50.0, 0.0], + "invtax": [2, 1, 1, 2, 2, 2], + } + ) + dividends = frs_dividend_income(account, person_ids) + weekly = np.array( + [ + 0.0, # 1,001 has only a bank account + 10.0, # 1,002: shares, not taxed at source + 4.0 * 1.25 + 2.0 * 1.25, # 2,001: trusts and gilts taxed at source + 0.0, # 2,002: gilts not taxed at source are excluded; shares pay 0 + ] + ) + assert dividends == pytest.approx(weekly * WEEKS_IN_YEAR) + + +def test_dividends_are_never_negative(): + person_ids = pd.Series([1_001]) + account = pd.DataFrame( + {"person_id": [1_001], "account": [8], "accint": [-5.0], "invtax": [2]} + ) + assert frs_dividend_income(account, person_ids).tolist() == [0.0] From a0d0e697b20a1845a1cf0be663942e2a61c11152 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 09:45:52 -0400 Subject: [PATCH 3/4] Test FRS dividend identity and conservation through imputation Add a tiny account-to-output regression using real income-draw application and ID-remapping stack output. Property tests conserve eligible annual gross dividends and FRS-half reported amounts. Ignore workspace-local hub job artifacts. Dividend regressions and repository formatting pass; merge-neighbour checks continue in the locked environment. --- .gitignore | 3 + .../tests/test_frs_dividend_income.py | 42 +++++ .../tests/test_imputation_source_flags.py | 175 ++++++++++++++++++ 3 files changed, 220 insertions(+) diff --git a/.gitignore b/.gitignore index 70d8396d..bded48fc 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz # Hypothesis example database (property-based tests) .hypothesis/ + +# Workspace-local hub job notes and tooling +.job/ diff --git a/policyengine_uk_data/tests/test_frs_dividend_income.py b/policyengine_uk_data/tests/test_frs_dividend_income.py index 77f37f6c..7c3809d4 100644 --- a/policyengine_uk_data/tests/test_frs_dividend_income.py +++ b/policyengine_uk_data/tests/test_frs_dividend_income.py @@ -3,6 +3,7 @@ import numpy as np import pandas as pd import pytest +from hypothesis import given, settings, strategies as st from policyengine_uk_data.datasets.frs import WEEKS_IN_YEAR, frs_dividend_income @@ -38,3 +39,44 @@ def test_dividends_are_never_negative(): {"person_id": [1_001], "account": [8], "accint": [-5.0], "invtax": [2]} ) assert frs_dividend_income(account, person_ids).tolist() == [0.0] + + +@st.composite +def dividend_account_cases(draw): + household_ids = draw( + st.lists(st.integers(1, 50), min_size=1, max_size=8, unique=True) + ) + person_ids = [household_id * 1_000 + 1 for household_id in household_ids] + rows = draw( + st.lists( + st.tuples( + st.sampled_from(person_ids), + st.sampled_from([1, 6, 7, 8, 9]), + st.integers(0, 1_000_000), + st.sampled_from([1, 2]), + ), + max_size=40, + ) + ) + account = pd.DataFrame(rows, columns=["person_id", "account", "pence", "invtax"]) + account["accint"] = account.pop("pence") / 100 + # Neither frame's index can be mistaken for an FRS person identifier. + account.index = np.arange(len(account)) * 7 + 100 + person_ids = pd.Series(person_ids[::-1], index=np.arange(len(person_ids)) * 3 + 10) + return account, person_ids + + +@given(dividend_account_cases()) +@settings(max_examples=100, deadline=None) +def test_eligible_gross_dividends_are_conserved_when_joining_to_people(case): + account, person_ids = case + expected_by_person = {person_id: 0.0 for person_id in person_ids} + for row in account.itertuples(): + if row.account in (7, 8) or (row.account == 6 and row.invtax == 1): + gross_weekly_amount = row.accint / (0.8 if row.invtax == 1 else 1) + expected_by_person[row.person_id] += gross_weekly_amount * WEEKS_IN_YEAR + + dividends = frs_dividend_income(account, person_ids) + + assert dividends.sum() == pytest.approx(sum(expected_by_person.values())) + assert dict(zip(person_ids, dividends)) == pytest.approx(expected_by_person) diff --git a/policyengine_uk_data/tests/test_imputation_source_flags.py b/policyengine_uk_data/tests/test_imputation_source_flags.py index 91f96db9..cd75f940 100644 --- a/policyengine_uk_data/tests/test_imputation_source_flags.py +++ b/policyengine_uk_data/tests/test_imputation_source_flags.py @@ -4,6 +4,9 @@ import numpy as np import pandas as pd +import pytest +from hypothesis import given, settings, strategies as st +from policyengine_uk.data import UKSingleYearDataset class _FakeDataset: @@ -187,3 +190,175 @@ def fake_impute_over_incomes(dataset, _model, output_variables): assert frs_half["dividend_income"].tolist() == [150.0, 0.0] assert frs_half["employment_income"].tolist() == [20_000.0, 80_000.0] assert result.person.iloc[2:]["dividend_income"].eq(123_456.0).all() + + +def _reported_dividend_dataset(dividends_by_person) -> UKSingleYearDataset: + person_ids = list(dividends_by_person) + household_ids = [person_id // 1_000 for person_id in person_ids] + person = pd.DataFrame( + { + "person_id": person_ids, + "person_household_id": household_ids, + "person_benunit_id": [ + household_id * 100 + 1 for household_id in household_ids + ], + "age": 40, + "gender": "FEMALE", + "employment_status": "FT_EMPLOYED", + "employment_income": 20_000.0, + "self_employment_income": 0.0, + "savings_interest_income": 0.0, + "dividend_income": list(dividends_by_person.values()), + "private_pension_income": 0.0, + "property_income": 0.0, + }, + index=np.arange(len(person_ids)) * 7 + 10, + ) + household_ids = list(dict.fromkeys(household_ids))[::-1] + return UKSingleYearDataset( + person=person, + benunit=pd.DataFrame( + {"benunit_id": [household_id * 100 + 1 for household_id in household_ids]}, + index=np.arange(len(household_ids)) * 3 + 50, + ), + household=pd.DataFrame( + { + "household_id": household_ids, + "household_weight": 2.0, + "region": "LONDON", + }, + index=np.arange(len(household_ids)) * 5 + 20, + ), + fiscal_year=2025, + ) + + +def _stub_income_training(monkeypatch): + """Avoid model training and simulation; retain income draws and real stacking.""" + from policyengine_uk_data.datasets.imputations import income as income_module + from policyengine_uk_data.datasets.imputations import frs_only + + class FakeSimulation: + def __init__(self, dataset): + self.dataset = dataset + + def calculate_dataframe(self, variables): + person = self.dataset.person + regions = self.dataset.household.set_index("household_id").region + return pd.DataFrame( + { + "age": person.age.to_numpy(), + "gender": person.gender.to_numpy(), + "region": person.person_household_id.map(regions).to_numpy(), + } + )[variables] + + class FakeIncomeModel: + def __init__(self): + self.predictions = [] + + def predict(self, inputs): + self.predictions.append(inputs.copy()) + return pd.DataFrame( + 123_456.0, index=inputs.index, columns=income_module.IMPUTATIONS + ) + + def reordered_donor_copy(dataset, _sample_size): + dataset = dataset.copy() + dataset.person = dataset.person.iloc[::-1].copy() + dataset.benunit = dataset.benunit.iloc[::-1].copy() + dataset.household = dataset.household.iloc[::-1].copy() + return dataset + + model = FakeIncomeModel() + monkeypatch.setattr(income_module, "Microsimulation", FakeSimulation) + monkeypatch.setattr(income_module, "create_income_model", lambda: model) + monkeypatch.setattr(income_module, "subsample_dataset", reordered_donor_copy) + monkeypatch.setattr( + frs_only, + "impute_frs_only_variables", + lambda train_dataset, target_dataset: target_dataset, + ) + return income_module, model + + +def _income_halves(dataset): + source = dataset.household.set_index("household_id").household_is_spi_synthetic + is_spi = dataset.person.person_household_id.map(source) + assert is_spi.notna().all() + return ( + dataset.person.loc[~is_spi].set_index("person_id"), + dataset.person.loc[is_spi].set_index("person_id"), + ) + + +def test_account_dividends_survive_imputation_and_real_stacking(monkeypatch): + from policyengine_uk_data.datasets.frs import WEEKS_IN_YEAR, frs_dividend_income + + income_module, model = _stub_income_training(monkeypatch) + dataset = _reported_dividend_dataset( + {2_001: 0.0, 1_002: 0.0, 1_001: 0.0, 2_002: 0.0} + ) + account = pd.DataFrame( + { + "person_id": [1_002, 2_001, 2_001, 1_001, 2_002], + "account": [8, 7, 6, 1, 6], + "accint": [10.0, 4.0, 2.0, 50.0, 3.0], + "invtax": [2, 1, 1, 2, 2], + }, + index=[43, 19, 71, 7, 28], + ) + dataset.person["dividend_income"] = frs_dividend_income( + account, dataset.person.person_id + ) + original_person = dataset.person.copy() + + result = income_module.impute_income(dataset) + frs_half, spi_half = _income_halves(result) + + assert frs_half.dividend_income.to_dict() == pytest.approx( + {2_001: 7.5 * WEEKS_IN_YEAR, 1_002: 10 * WEEKS_IN_YEAR, 1_001: 0, 2_002: 0} + ) + assert frs_half.employment_income.eq(20_000.0).all() + person_offset = original_person.person_id.max() + 1 + assert ( + spi_half.index.tolist() + == (original_person.person_id.iloc[::-1] + person_offset).tolist() + ) + assert spi_half.dividend_income.eq(123_456.0).all() + assert len(model.predictions) == 1 + assert len(model.predictions[0]) == len(original_person) + assert result.person.person_id.is_unique + assert result.benunit.benunit_id.is_unique + assert result.household.household_id.is_unique + assert result.person.person_benunit_id.isin(result.benunit.benunit_id).all() + pd.testing.assert_frame_equal(dataset.person, original_person) + result.validate() + + +@given( + st.dictionaries( + st.integers(1, 50).map(lambda household_id: household_id * 1_000 + 1), + st.integers(0, 1_000_000).map(lambda pence: pence / 100), + min_size=1, + max_size=8, + ) +) +@settings(max_examples=50, deadline=None) +def test_frs_dividend_totals_are_conserved_through_income_imputation( + reported_dividends, +): + # A fresh patch context per example avoids state leaking between property cases. + with pytest.MonkeyPatch.context() as monkeypatch: + income_module, _model = _stub_income_training(monkeypatch) + dataset = _reported_dividend_dataset(reported_dividends) + result = income_module.impute_income(dataset) + + frs_half, spi_half = _income_halves(result) + assert frs_half.dividend_income.sum() == pytest.approx( + sum(reported_dividends.values()) + ) + assert frs_half.dividend_income.to_dict() == pytest.approx(reported_dividends) + assert spi_half.dividend_income.sum() == pytest.approx( + len(reported_dividends) * 123_456 + ) From 7a9174a56e43b94101a72f6062d49d08319d8c8c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 23:34:30 -0400 Subject: [PATCH 4/4] Pin FRS dividend call site and stage-2 training inputs Fold in the investigation patch on top of Round 1 without rewriting history. Keep the account-to-output and conservation regressions, add create_frs holder-keying coverage and assert stage 2 learns from FRS-reported dividends. Correct earnings-group and join documentation, hoist the gross-up constant, and remove the redundant annualisation definition. Conflict resolutions retain the existing gift-aid explanation and all Round 1 tests. Keep precise historical aggregates per d932/notice 9. Co-Authored-By: Claude Opus 5.5 --- changelog.d/frs-reported-dividends.fixed.md | 2 +- policyengine_uk_data/datasets/frs.py | 7 +-- .../datasets/imputations/income.py | 14 +++--- .../tests/test_imputation_source_flags.py | 19 +++++--- .../tests/test_legacy_benefit_proxies.py | 48 +++++++++++++++++-- policyengine_uk_data/utils/datasets.py | 6 ++- 6 files changed, 71 insertions(+), 25 deletions(-) diff --git a/changelog.d/frs-reported-dividends.fixed.md b/changelog.d/frs-reported-dividends.fixed.md index cf31851c..0b49741f 100644 --- a/changelog.d/frs-reported-dividends.fixed.md +++ b/changelog.d/frs-reported-dividends.fixed.md @@ -1,2 +1,2 @@ - FRS investment-account dividends are now summed to the person who holds the account. They were keyed on each person's row position, so almost none matched and the base FRS carried about £40m of dividends instead of about £8bn in 2024-25. -- The enhanced FRS keeps those reported dividends instead of replacing them with draws from the SPI income model, which predicts from age, gender and region alone and gave Universal Credit claimants six-figure dividends (policyengine-uk#1948). The SPI-donor half still carries the SPI income distribution. +- The enhanced FRS keeps those reported dividends instead of replacing them with draws from the SPI income model, which predicts from age, gender and region within earnings groups and gave Universal Credit claimants six-figure dividends (policyengine-uk#1948). The SPI-donor half still carries the SPI income distribution. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 2a959139..fae6595b 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -48,6 +48,8 @@ # loaders (e.g. LCFS/ETB in `datasets/imputations/consumption.py`) can import # the same value rather than re-defining `* 52` locally and drifting. WEEKS_IN_YEAR = 365.25 / 7 +# Grosses up an amount taxed at source at the basic rate (1 / 0.8). +INVERTED_BASIC_RATE = 1.25 LEGACY_JOBSEEKER_MIN_AGE = 18 HOURS_WORKED_WEEKS_PER_YEAR = 52 @@ -800,7 +802,6 @@ def frs_dividend_income(account: pd.DataFrame, person_ids) -> np.ndarray: the positional index instead matched almost no accounts, leaving the FRS with about £40m of dividends rather than about £8bn in 2024-25. """ - INVERTED_BASIC_RATE = 1.25 dividends = ( account.accint * np.where(account.invtax == 1, INVERTED_BASIC_RATE, 1) ) * ( @@ -1290,8 +1291,6 @@ def determine_education_level(fted_val, typeed2_val, age_val): * 52 ).astype(float) - WEEKS_IN_YEAR = 365.25 / 7 - pe_person["employment_income"] = np.maximum(0, person.inearns) * WEEKS_IN_YEAR pension_payment = sum_to_entity( @@ -1326,8 +1325,6 @@ def determine_education_level(fted_val, typeed2_val, age_val): ) ) - INVERTED_BASIC_RATE = 1.25 - pe_person["tax_free_savings_income"] = np.maximum( 0, sum_to_entity( diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index 0bd516c4..1e63dd5b 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -489,7 +489,6 @@ def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset: Returns: Combined dataset with original data plus synthetic high-income individuals. """ - # Impute wealth, assuming same time period as trained data dataset = dataset.copy() # gift_aid and charitable_investment_gifts are in IMPUTATIONS but are not # columns on the raw FRS build, so initialise them to zero everywhere @@ -551,12 +550,13 @@ def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset: ) # The FRS half keeps its reported dividends. Replacing them with a draw - # from the SPI model, whose only predictors are age, gender and region, - # gave dividends to FRS respondents without regard to their investments, - # earnings or benefits, so Universal Credit claimants received them as - # often as anyone else (policyengine-uk#1948). The SPI-donor half carries - # the SPI dividend distribution, and calibration to the HMRC dividend - # targets reweights between the two. + # from the SPI model, which predicts from age, gender and region within + # earnings groups (whether the person has pay or a trade), gave + # dividends to FRS respondents without regard to their investments or + # benefits, so Universal Credit claimants received them + # (policyengine-uk#1948). The SPI-donor half carries the SPI dividend + # distribution, and calibration to the HMRC dividend targets reweights + # between the two. zero_weight_copy.validate() dataset.validate() diff --git a/policyengine_uk_data/tests/test_imputation_source_flags.py b/policyengine_uk_data/tests/test_imputation_source_flags.py index cd75f940..1a324d5e 100644 --- a/policyengine_uk_data/tests/test_imputation_source_flags.py +++ b/policyengine_uk_data/tests/test_imputation_source_flags.py @@ -139,15 +139,17 @@ def test_impute_capital_gains_marks_capital_gains_clone_households(monkeypatch): def test_impute_income_keeps_reported_dividends_on_the_frs_half(monkeypatch): """SPI draws replace incomes only on the SPI-donor copy (policyengine-uk#1948). - The SPI income model predicts from age, gender and region alone, so using - it to overwrite the FRS half's dividends gave Universal Credit claimants - dividends unrelated to anything they reported. + The SPI income model predicts from age, gender and region within earnings + groups, so using it to overwrite the FRS half's dividends gave Universal + Credit claimants dividends unrelated to any investments they reported. + The stage-2 model then trains on the FRS half's reported dividends. """ from policyengine_uk_data.datasets.imputations import income as income_module from policyengine_uk_data.datasets import disability_benefits from policyengine_uk_data.datasets.imputations import frs_only imputed_halves = [] + stage_2_training = [] def fake_impute_over_incomes(dataset, _model, output_variables): imputed_halves.append( @@ -168,10 +170,13 @@ def fake_impute_over_incomes(dataset, _model, output_variables): lambda dataset, _sample_size: dataset.copy(), ) monkeypatch.setattr(income_module, "impute_over_incomes", fake_impute_over_incomes) + + def fake_impute_frs_only_variables(train_dataset, target_dataset): + stage_2_training.append(train_dataset.person["dividend_income"].tolist()) + return target_dataset + monkeypatch.setattr( - frs_only, - "impute_frs_only_variables", - lambda train_dataset, target_dataset: target_dataset, + frs_only, "impute_frs_only_variables", fake_impute_frs_only_variables ) monkeypatch.setattr( disability_benefits, @@ -190,6 +195,8 @@ def fake_impute_over_incomes(dataset, _model, output_variables): assert frs_half["dividend_income"].tolist() == [150.0, 0.0] assert frs_half["employment_income"].tolist() == [20_000.0, 80_000.0] assert result.person.iloc[2:]["dividend_income"].eq(123_456.0).all() + # Stage 2 learns from the FRS half's reported dividends, not SPI draws. + assert stage_2_training == [[150.0, 0.0]] def _reported_dividend_dataset(dividends_by_person) -> UKSingleYearDataset: diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py index 64bf5670..54f23f4f 100644 --- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py +++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py @@ -3,6 +3,7 @@ import pytest import policyengine_uk import policyengine_uk_data.datasets.frs as frs_module +from policyengine_uk_data.utils.datasets import sum_to_entity as real_sum_to_entity from policyengine_uk_data.datasets.frs import ( FRS_EMPSTATI_EMPLOYMENT_STATUS, @@ -382,12 +383,21 @@ def calculate(self, variable, year=None): raise KeyError(variable) -def create_single_adult_frs(tmp_path, monkeypatch, empstati=8, with_child=False): +def create_single_adult_frs( + tmp_path, monkeypatch, empstati=8, with_child=False, accounts=None +): monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) monkeypatch.setattr(frs_module, "load_take_up_rate", lambda *args, **kwargs: 0.0) monkeypatch.setattr(frs_module, "load_parameter", lambda *args, **kwargs: 0.0) + # The real join for any table with rows; empty tables give zeros. monkeypatch.setattr( - frs_module, "sum_to_entity", lambda values, ids, index: np.zeros(len(index)) + frs_module, + "sum_to_entity", + lambda values, ids, index: ( + real_sum_to_entity(values, ids, index) + if len(values) + else np.zeros(len(index)) + ), ) monkeypatch.setattr( frs_module, @@ -521,8 +531,12 @@ def create_single_adult_frs(tmp_path, monkeypatch, empstati=8, with_child=False) ] ), "oddjob": pd.DataFrame(columns=["person", "sernum", "ojamt", "ojnow"]), - "accounts": pd.DataFrame( - columns=["person", "sernum", "accint", "acctax", "invtax", "account"] + "accounts": ( + accounts + if accounts is not None + else pd.DataFrame( + columns=["person", "sernum", "accint", "acctax", "invtax", "account"] + ) ), "job": pd.DataFrame(columns=["person", "sernum", "deduc1", "spnamt", "salsac"]), "benefits": pd.DataFrame( @@ -598,6 +612,32 @@ def test_create_frs_child_rows_are_child(tmp_path, monkeypatch): } +def test_create_frs_keys_account_dividends_on_person_id(tmp_path, monkeypatch): + # One stocks-and-shares account (type 8) held by the adult. create_frs + # must join it on person_id (household_id * 1000 + person), not on the + # person table's row position, which matches no account id. + accounts = pd.DataFrame( + [ + { + "person": 1, + "sernum": 100, + "accint": 10.0, + "acctax": 0, + "invtax": 0, + "account": 8, + } + ] + ) + person = create_single_adult_frs( + tmp_path, monkeypatch, with_child=True, accounts=accounts + ).person.set_index("person_id") + + assert person.loc[100_001, "dividend_income"] == pytest.approx( + 10.0 * frs_module.WEEKS_IN_YEAR + ) + assert person.loc[100_002, "dividend_income"] == 0 + + def test_create_frs_rejects_unknown_adult_empstati(tmp_path, monkeypatch): with pytest.raises(ValueError, match="EMPSTATI"): create_single_adult_frs(tmp_path, monkeypatch, empstati=12) diff --git a/policyengine_uk_data/utils/datasets.py b/policyengine_uk_data/utils/datasets.py index 2f4cd1b4..60d887d5 100644 --- a/policyengine_uk_data/utils/datasets.py +++ b/policyengine_uk_data/utils/datasets.py @@ -16,10 +16,12 @@ def sum_to_entity(values: pd.Series, foreign_key: pd.Series, primary_key) -> np. Args: values (pd.Series): The values in the non-entity table. foreign_key (pd.Series): E.g. pension.person_id. - primary_key ([type]): E.g. person.index. + primary_key ([type]): The entity's own ids in the same key space as + ``foreign_key``, e.g. person.person_id. A positional index such as + person.index matches the wrong rows or none. Returns: - pd.Series: A value for each person. + np.ndarray: A value for each entry of ``primary_key``. """ return values.groupby(foreign_key).sum().reindex(primary_key).fillna(0).values