Solve Pension Credit take-up over entitled benefit units

.gitignore

@@ -19,3 +19,5 @@
 **/_build
 !policyengine_uk_data/storage/*.csv
 **/version.json
+# Build output: household weights by local area, derived from FRS microdata
+policyengine_uk_data/storage/local_geography_weights.csv.gz

changelog.d/hb-dwp-age-targets.changed.md

@@ -0,0 +1 @@
+Calibrate Housing Benefit to DWP's Great Britain spending and claims by age group, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Pension-age Housing Benefit is calibrated to DWP's figures over Pension Credit qualifying age. Working-age Housing Benefit is calibrated to DWP's figures under that age less all supported and temporary accommodation Housing Benefit, for which policyengine-uk has no rules (£585m and 107k claims in 2025-26, against £5.78bn and 460k in total); treating the model's working-age Housing Benefit as general needs is an approximation. Modelled GB Housing Benefit for 2025-26 falls from about £11.7bn to £8.5bn (DWP: £12.9bn), because pension-age Housing Benefit was about 1.6 times DWP's figure while about £5.2bn of supported and temporary accommodation Housing Benefit remains unmodelled as such. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims.

changelog.d/pc-takeup-over-eligible.fixed.md

@@ -0,0 +1 @@
+Solve Pension Credit take-up over entitled benefit units in Great Britain, at DWP's FYE 2024 caseload take-up of 62%, after the imputations so entitlement reflects imputed capital. Entitled non-reporters, including in Northern Ireland, are drawn at the solved probability. Units with no entitlement in the calibration year are drawn at DWP's Savings Credit-only caseload take-up (37% in FYE 2024), the rate at which units a reform or a later year newly entitles claim. Reports on SPI-synthetic households, which are imputed from SPI incomes rather than observed, no longer anchor take-up. Calibrate Pension Credit to DWP's Great Britain spending and caseload, replacing the OBR target that the model compared with UK-wide Pension Credit.

policyengine_uk_data/datasets/create_datasets.py

@@ -100,6 +100,7 @@ def main():
             "Impute capital gains",
             "Impute salary sacrifice",
             "Impute student loan plan",
+            "Assign Pension Credit take-up",
             "Clone and assign OA geography",
             "Calibrate constituency weights",
             "Calibrate local authority weights",
@@ -202,6 +203,18 @@ def main():
             )
             update_dataset("Impute student loan plan", "completed")
 
+            # Pension Credit entitlement needs the imputed capital, so its
+            # take-up is solved here rather than in the FRS build.
+            update_dataset("Assign Pension Credit take-up", "processing")
+            from policyengine_uk_data.datasets.pension_credit_takeup import (
+                assign_pension_credit_takeup,
+            )
+
+            frs, pension_credit_takeup = assign_pension_credit_takeup(
+                frs, year=frs_release.calibration_year
+            )
+            update_dataset("Assign Pension Credit take-up", "completed")
+
             # Clone households and assign OA geography
             update_dataset("Clone and assign OA geography", "processing")
             from policyengine_uk_data.calibration.clone_and_assign import (
@@ -397,6 +410,7 @@ def main():
                 "long_geography_weights": "local_geography_weights.csv.gz",
                 "imputations_applied": "consumption, wealth, VAT, services, income, capital_gains, cgt_band_donors, salary_sacrifice, student_loan_plan",
                 "calibration": "national, LA and  constituency targets",
+                "pension_credit_take_up": pension_credit_takeup,
             },
         )
 

policyengine_uk_data/datasets/frs.py

@@ -24,6 +24,7 @@
     add_disability_benefit_flags_from_reported_amounts,
     drop_internal_disability_reported_amounts,
 )
+from policyengine_uk_data.utils.benefit_units import claimant_or_partner_variable
 from policyengine_uk_data.utils.datasets import (
     sum_to_entity,
     categorical,
@@ -392,6 +393,38 @@ def derive_is_parent_from_frs_microdata(
     return is_adult_record & has_dependent_children
 
 
+def derive_all_claimants_over_state_pension_age(
+    person_benunit_ids,
+    is_claimant_or_partner,
+    is_over_state_pension_age,
+    benunit_ids,
+) -> np.ndarray:
+    """Identify benefit units whose claimant and any partner have all reached
+    State Pension age.
+
+    Such a unit cannot claim Universal Credit (Welfare Reform Act 2012
+    s.4(1)(b)). ``is_claimant_or_partner`` should be the variable that
+    ``claimant_or_partner_variable`` names, so the rule matches the
+    pension-age route of policyengine-uk's ``housing_benefit_eligible``.
+    """
+
+    claimant = np.asarray(is_claimant_or_partner, dtype=bool)
+    over = claimant & np.asarray(is_over_state_pension_age, dtype=bool)
+    counts = (
+        pd.DataFrame(
+            {
+                "benunit": np.asarray(person_benunit_ids),
+                "claimants": claimant.astype(int),
+                "over": over.astype(int),
+            }
+        )
+        .groupby("benunit")[["claimants", "over"]]
+        .sum()
+        .reindex(np.asarray(benunit_ids), fill_value=0)
+    )
+    return ((counts.claimants > 0) & (counts.over == counts.claimants)).to_numpy()
+
+
 def _as_non_negative_array(values) -> np.ndarray:
     values = np.asarray(values, dtype=float)
     return np.maximum(np.nan_to_num(values, nan=0.0), 0.0)
@@ -1532,15 +1565,30 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray:
     pe_benunit["child_benefit_opts_out"] = (
         generator.random(len(pe_benunit)) < child_benefit_opts_out_rate
     )
+    # The enhanced dataset redraws this once entitlement can be computed
+    # (datasets/pension_credit_takeup.py).
     pe_benunit["would_claim_pc"] = assign_takeup_with_reported_anchors(
         generator.random(len(pe_benunit)),
         pension_credit_rate,
         reported_mask=_reported_benunit_mask("pension_credit_reported"),
     )
+    # A benefit unit whose claimant and any partner have all reached State
+    # Pension age cannot claim Universal Credit, so it never gets
+    # would_claim_uc, even if it reports UC. The draw still covers every unit,
+    # so the random stream and every other unit's value are unchanged. Ages
+    # are not rolled forward, so if State Pension age rises above a claimant's
+    # survey age in a later year, that unit stays without would_claim_uc there.
     pe_benunit["would_claim_uc"] = assign_takeup_with_reported_anchors(
         generator.random(len(pe_benunit)),
         universal_credit_rate,
         reported_mask=_reported_benunit_mask("universal_credit_reported"),
+    ) & ~derive_all_claimants_over_state_pension_age(
+        person_benunit_ids=sim.calculate("person_benunit_id", year).values,
+        is_claimant_or_partner=sim.calculate(
+            claimant_or_partner_variable(sim.tax_benefit_system.variables), year
+        ).values,
+        is_over_state_pension_age=sim.calculate("is_SP_age", year).values,
+        benunit_ids=pe_benunit.benunit_id,
     )
     pe_benunit["would_claim_tfc"] = generator.random(len(pe_benunit)) < tfc_rate
 

policyengine_uk_data/datasets/pension_credit_takeup.py

@@ -0,0 +1,148 @@
+"""Pension Credit take-up solved over entitled benefit units.
+
+The FRS build first draws ``would_claim_pc`` against an unweighted count of
+every benefit unit, mostly units with no entitlement. Entitled non-reporters
+then claim at about the take-up rate on top of every reporter, so take-up
+among the entitled ends up well above it. This step redraws the flag once
+the imputations are in; entitlement depends on imputed capital, so it has
+to wait for them.
+
+For the calibration year it:
+- finds the benefit units with positive Pension Credit entitlement;
+- keeps reporters of Pension Credit as claimants, except on SPI-synthetic
+  households, whose reports the second-stage imputation (frs_only.py)
+  predicts from their SPI incomes rather than observes;
+- solves the probability that makes weighted take-up among entitled units
+  in Great Britain equal DWP's caseload take-up rate, which covers Great
+  Britain only;
+- applies that probability to every entitled non-reporter, in Northern
+  Ireland too;
+- draws every unit with no entitlement in the calibration year at DWP's
+  take-up for Savings Credit-only awards (37% in FYE 2024), the rate at
+  which a unit claims once a reform or a later year entitles it. This step
+  finds no entitlement for them in the calibration year, so their rate only
+  matters under a reform, in a later year, or where the saved dataset's
+  calibration-year entitlement differs from this step's (#479).
+
+Weights are the survey grossing weights, before calibration.
+"""
+
+import numpy as np
+
+from policyengine_uk_data.parameters import load_take_up_rate
+from policyengine_uk_data.utils.takeup import solve_fill_probability
+
+# Separate from the FRS build's take-up generator, so redrawing here leaves
+# every other stochastic input unchanged.
+PENSION_CREDIT_TAKEUP_SEED = 1_792
+
+
+def spi_synthetic_benunits(dataset) -> np.ndarray:
+    """Benefit units in households flagged ``household_is_spi_synthetic``."""
+    household = dataset.household
+    if "household_is_spi_synthetic" not in household.columns:
+        return np.zeros(len(dataset.benunit), dtype=bool)
+    synthetic = household.loc[
+        household["household_is_spi_synthetic"].astype(bool), "household_id"
+    ]
+    person = dataset.person
+    benunits = person.loc[
+        person["person_household_id"].isin(synthetic), "person_benunit_id"
+    ]
+    return dataset.benunit["benunit_id"].isin(benunits).to_numpy()
+
+
+def pension_credit_takeup_flags(
+    draws, rate, weights, entitled, reported, great_britain, newly_entitled_rate
+):
+    """Claim flags and the fill probability solved over GB entitled units.
+
+    Only Great Britain enters the solve, because DWP's take-up rate covers
+    Great Britain. Reporters claim. Every other entitled unit, wherever it
+    lives, claims when its draw is below the probability; every unit that
+    isn't entitled claims when its draw is below ``newly_entitled_rate``.
+    """
+    entitled = np.asarray(entitled, dtype=bool)
+    reported = np.asarray(reported, dtype=bool)
+    eligible = entitled & np.asarray(great_britain, dtype=bool)
+    probability = solve_fill_probability(rate, weights, eligible, reported)
+    claim_probability = np.where(entitled, probability, newly_entitled_rate)
+    claims = reported | (np.asarray(draws, dtype=np.float64) < claim_probability)
+    return claims, probability
+
+
+def assign_pension_credit_takeup(dataset, year: int):
+    """Return a copy of ``dataset`` with ``would_claim_pc`` solved for
+    ``year``, and a summary of weighted aggregates.
+
+    The probability is zero when entitled GB reporters already make up the
+    take-up rate of entitled GB units. The summary's GB reporter baseline
+    (reporters only, before any fill) is for comparison with DWP's absolute
+    caseload and spending, which calibration targets later and separately.
+    """
+    from policyengine_uk import Microsimulation
+
+    sim = Microsimulation(dataset=dataset)
+    entitlement = sim.calculate("pension_credit_entitlement", year).values
+    entitled = entitlement > 0
+    reported_amount = sim.calculate(
+        "pension_credit_reported", year, map_to="benunit"
+    ).values
+    spi_synthetic = spi_synthetic_benunits(dataset)
+    reported = (reported_amount > 0) & ~spi_synthetic
+    weights = sim.calculate("benunit_weight", year).values
+    country = sim.calculate("country", year).values
+    in_ni = (
+        sim.map_result(
+            sim.map_result(
+                (country == "NORTHERN_IRELAND").astype(float), "household", "person"
+            ),
+            "person",
+            "benunit",
+        )
+        > 0
+    )
+    gb = ~in_ni
+    rate = load_take_up_rate("pension_credit", year)
+    newly_entitled_rate = load_take_up_rate("pension_credit_newly_entitled", year)
+    draws = np.random.default_rng(PENSION_CREDIT_TAKEUP_SEED).random(len(entitled))
+    claims, probability = pension_credit_takeup_flags(
+        draws, rate, weights, entitled, reported, gb, newly_entitled_rate
+    )
+
+    dataset = dataset.copy()
+    dataset.benunit["would_claim_pc"] = claims
+
+    def total(values, mask, scale):
+        return round(float((weights * values)[mask].sum()) / scale, 3)
+
+    summary = {
+        "year": year,
+        "rate": rate,
+        "fill_probability": round(probability, 4),
+        "newly_entitled_rate": newly_entitled_rate,
+        "entitled_k": total(1, entitled, 1e3),
+        "gb_entitled_k": total(1, gb & entitled, 1e3),
+        "reporters_k": total(1, reported, 1e3),
+        "spi_synthetic_units_not_anchored": int(
+            (spi_synthetic & (reported_amount > 0)).sum()
+        ),
+        "entitled_reporters_k": total(1, entitled & reported, 1e3),
+        "gb_reporters_k": total(1, gb & reported, 1e3),
+        "gb_entitled_reporters_k": total(1, gb & entitled & reported, 1e3),
+        "gb_reported_pension_credit_bn": total(reported_amount, gb, 1e9),
+        "gb_reporters_modelled_pension_credit_bn": total(
+            entitlement, gb & reported, 1e9
+        ),
+        "gb_claims_after_fill_k": total(1, gb & entitled & claims, 1e3),
+        "gb_pension_credit_after_fill_bn": total(entitlement, gb & claims, 1e9),
+        "gb_not_entitled_k": total(1, gb & ~entitled, 1e3),
+        "gb_not_entitled_flagged_share": round(
+            float(
+                weights[gb & ~entitled & claims].sum()
+                / max(weights[gb & ~entitled].sum(), 1)
+            ),
+            4,
+        ),
+    }
+    return dataset, summary

policyengine_uk_data/parameters/take_up/pension_credit.yaml

@@ -1,10 +1,16 @@
-description: Share of eligible Pension Credit recipients that participate.
+description: Share of benefit units entitled to Pension Credit that receive it (caseload take-up, Great Britain).
 metadata:
   label: Pension Credit take-up rate
   name: PC_takeup
   unit: /1
   reference:
     - title: "Income-related benefits: estimates of take-up: financial year 2019 to 2020"
       href: https://www.gov.uk/government/statistics/income-related-benefits-estimates-of-take-up-financial-year-2019-to-2020/income-related-benefits-estimates-of-take-up-financial-year-2019-to-2020#pension-credit-2
+    - title: "Income-related benefits: estimates of take-up: financial year ending 2024"
+      href: https://www.gov.uk/government/statistics/income-related-benefits-estimates-of-take-up-financial-year-ending-2024/income-related-benefits-estimates-of-take-up-financial-year-ending-2024
 values:
   2015-01-01: 0.7
+  # FYE 2023 and FYE 2024 caseload take-up (65% and 62%) from the FYE 2024
+  # edition; expenditure take-up was 71% in FYE 2024.
+  2022-04-01: 0.65
+  2023-04-01: 0.62

policyengine_uk_data/parameters/take_up/pension_credit_newly_entitled.yaml

@@ -0,0 +1,14 @@
+description: Probability that a benefit unit with no Pension Credit entitlement in the calibration year claims it once a reform or a later year entitles it. Set to DWP's caseload take-up of Savings Credit-only awards (Great Britain), the group with the smallest awards, which is closest to what a reform gives the units it newly entitles.
+metadata:
+  label: Pension Credit take-up for newly entitled units
+  name: PC_takeup_newly_entitled
+  unit: /1
+  reference:
+    - title: "Income-related benefits: estimates of take-up: financial year ending 2024"
+      href: https://www.gov.uk/government/statistics/income-related-benefits-estimates-of-take-up-financial-year-ending-2024/income-related-benefits-estimates-of-take-up-financial-year-ending-2024
+values:
+  # Savings Credit-only caseload take-up, FYE 2023 and FYE 2024 (42% and
+  # 37%, table PC1). Mean award claimed in FYE 2024: £9 a week, against £87
+  # for Guarantee Credit, whose take-up was 69%.
+  2022-04-01: 0.42
+  2023-04-01: 0.37

policyengine_uk_data/targets/build_loss_matrix.py

@@ -113,7 +113,7 @@ def create_target_matrix(
             col = _compute_column(target, ctx, year)
             if col is None:
                 continue
-            df[target.name] = col
+            df[target.name] = restrict_to_countries(col, ctx.country, target.countries)
             target_names.append(target.name)
             target_values.append(val)
         except Exception as e:
@@ -122,14 +122,30 @@ def create_target_matrix(
     return df, pd.Series(target_values, index=target_names)
 
 
+def restrict_to_countries(column, household_country, countries):
+    """Zero a household column outside the countries a target covers.
+
+    ``countries`` of None means the target covers the whole UK, and the
+    column is returned unchanged.
+    """
+    if countries is None:
+        return column
+    in_scope = np.isin(np.asarray(household_country), countries)
+    return np.asarray(column, dtype=float) * in_scope
+
+
 def _resolve_value(target: Target, year: int) -> float | None:
     """Get the target value for a year, falling back to nearest year.
 
-    VOA council tax targets are population-uprated when extrapolating
-    from their base year (2024).
+    A missing year takes the value of the nearest listed year (the earlier
+    on a tie) if that year is earlier and at most three years away, unless
+    the target sets ``carry_forward=False``; otherwise there is no value. VOA council tax targets are
+    population-uprated when extrapolating from their base year (2024).
     """
     if year in target.values:
         return target.values[year]
+    if not target.carry_forward:
+        return None
     available = sorted(target.values.keys())
     if not available:
         return None

policyengine_uk_data/targets/schema.py

@@ -19,6 +19,11 @@ class Unit(str, Enum):
     RATE = "rate"
 
 
+# DWP statistics cover Great Britain: benefits for Northern Ireland residents
+# are the Northern Ireland Executive's responsibility.
+GREAT_BRITAIN = ("ENGLAND", "SCOTLAND", "WALES")
+
+
 class Target(BaseModel):
     """A single calibration target from an official statistical source.
 
@@ -41,6 +46,15 @@ class Target(BaseModel):
     is_count: bool = False
     reference_url: str | None = None
     forecast_vintage: str | None = None
+    # Countries a national source covers when that is less than the UK, as
+    # values of the model's `country` variable. The loss matrix column only
+    # counts households in these countries. None means the whole UK.
+    countries: tuple[str, ...] | None = None
+    # Whether the loss matrix may fill a year the target does not list with
+    # the nearest listed year's value (the earlier on a tie), used only when
+    # that year is earlier and at most three years away (see _resolve_value).
+    # False for a figure that exists only in the years it lists.
+    carry_forward: bool = True
 
     # For targets needing custom simulation logic (UC splits,
     # counterfactuals). Excluded from serialisation.