Uprate Scottish households' water charges by Scottish Water's path

changelog.d/scottish-water-uprating.fixed.md

@@ -0,0 +1 @@
+Uprate Scottish households' water and sewerage charges by Scottish Water's published charge rises, and hold Northern Ireland's flat, instead of applying the Ofwat England and Wales series to every household.

docs/book/assumptions/water-bills.md

@@ -77,3 +77,13 @@ The model uses these resulting year-on-year growth rates:
 | 2027 | 5.1% |
 | 2028 | 3.8% |
 | 2029 | 4.3% |
+
+### Scotland and Northern Ireland
+
+The Ofwat series covers England and Wales only. The dataset projection uprates `water_and_sewerage_charges` by country (`uprate_water_charges` in `policyengine_uk/data/economic_assumptions.py`):
+
+- England and Wales take `ofwat.water_bills`.
+- Scotland takes `scottish_water.water_bills`, the published rise in Scottish Water's household charges: 1.8% (2021-22), 4.2%, 5%, 8.8%, 9.9% and 8.67% (2026-27). From 2027-28 it assumes 4% a year: CPI (the OBR's 2.0%) plus 2%, the cap in the Water Industry Commission for Scotland's draft determination for 2027-33. The final determination is due on 29 October 2026.
+- Northern Ireland has no domestic water charges, so any charge a dataset gives its households is held flat.
+
+The variable has no `uprating` attribute, so like council tax it is held at its last projected value after the projection horizon.

docs/book/validation/hbai.md

@@ -108,6 +108,6 @@ Each of the variables above is either derived from other variables, or is a dire
 | `maintenance_expenses` | gov.obr.consumer_price_index | Uprated based on CPI inflation |
 | `student_loan_repayments` | gov.economic_assumptions.indices.obr.average_earnings | Uprated based on OBR average earnings projections |
 | `rent` | gov.obr.rent | Social rents uprated by CPI+1%, private rents uprated with outturn data by region to 2025, then with backed-out private rent level growth from OBR aggregate rent forecasts (assuming social rent grows at CPI+1%) |
-| `water_and_sewerage_charges` | gov.obr.consumer_price_index | Uprated based on CPI as proxy |
+| `water_and_sewerage_charges` | gov.economic_assumptions.yoy_growth.ofwat.water_bills (England and Wales), gov.economic_assumptions.yoy_growth.scottish_water.water_bills (Scotland) | Uprated by each country's published bill increases, then projections; Northern Ireland has no domestic water charges and is held flat |
 | `mortgage_interest_repayment` | gov.obr.mortgage_interest | Uprated based on outturn Ofwat data to 2025, then Ofwat projections onwards |
 | `housing_service_charges` | gov.economic_assumptions.indices.obr.lagged_average_earnings | Uprated based on lagged average earnings as proxy |

policyengine_uk/data/economic_assumptions.py

@@ -56,6 +56,28 @@
     "UNKNOWN": "england",
 }
 
+# The water and sewerage charges growth series for each Region label. Ofwat
+# regulates the water companies of England and Wales, and its average bill
+# series covers both. Scottish Water sets Scotland's charges under limits from
+# the Water Industry Commission for Scotland, so Scotland has its own series.
+# Northern Ireland has no domestic water charges (NI Water is funded by the
+# Executive), and the FRS gives its households none, so it has no series: any
+# charge a dataset does give them is held flat rather than taking the England
+# and Wales rate. An unknown region takes the Ofwat series, which covers most
+# households.
+WATER_CHARGES = "water_and_sewerage_charges"
+_OFWAT_WATER_GROWTH = "gov.economic_assumptions.yoy_growth.ofwat.water_bills"
+_SCOTTISH_WATER_GROWTH = (
+    "gov.economic_assumptions.yoy_growth.scottish_water.water_bills"
+)
+_WATER_GROWTH_SERIES = {
+    **dict.fromkeys(_ENGLAND_REGIONS, _OFWAT_WATER_GROWTH),
+    "WALES": _OFWAT_WATER_GROWTH,
+    "SCOTLAND": _SCOTTISH_WATER_GROWTH,
+    "NORTHERN_IRELAND": None,
+    "UNKNOWN": _OFWAT_WATER_GROWTH,
+}
+
 
 # The last year a single-year dataset is projected to: 2032, fiscal year
 # 2032-33. An OBR forecast runs five years past the fiscal year it is made in,
@@ -126,7 +148,7 @@ def uprating_input_paths() -> list:
     ``apply_single_year_uprating`` and the functions it calls read these, and
     no other parameters: the growth rates in ``uprating_indices.yaml``, the
     council tax and rent series, the cumulative residential property index,
-    the indices of the input variables that
+    the water charge series by country, the indices of the input variables that
     file leaves out (after ``LEGACY_PROJECTION_END_YEAR``) and the capital
     gains projection's metadata (``CGT_INDEX_PATH``). A path may name a node
     (the regional private rent series), which covers its parameters.
@@ -136,6 +158,7 @@ def uprating_input_paths() -> list:
     return sorted(
         set(uprating)
         | {f"{COUNCIL_TAX_GROWTH}.{name}" for name in series}
+        | {value for value in _WATER_GROWTH_SERIES.values() if value}
         | {
             SOCIAL_RENT_GROWTH,
             PRIVATE_RENT_GROWTH,
@@ -290,6 +313,10 @@ def apply_single_year_uprating(
 
     current_year = uprate_council_tax(current_year, previous_year, parameters)
 
+    # Water and sewerage charges are uprated by each country's regulated path.
+
+    current_year = uprate_water_charges(current_year, previous_year, parameters)
+
     # Rent is uprated by OBR forecasts/outturns by region.
 
     current_year = uprate_rent(current_year, previous_year, parameters)
@@ -393,6 +420,36 @@ def uprate_council_tax(
     return current_year
 
 
+def uprate_water_charges(
+    current_year: UKSingleYearDataset,
+    previous_year: UKSingleYearDataset,
+    parameters: ParameterNode,
+):
+    # Uprate water and sewerage charges for a single year dataset.
+
+    if WATER_CHARGES not in current_year.household.columns:
+        return current_year
+    region = pd.Series(np.asarray(current_year.household["region"]).astype(str))
+    unrecognised = set(region) - set(_WATER_GROWTH_SERIES)
+    if unrecognised:
+        raise ValueError(
+            "Cannot uprate water and sewerage charges for unrecognised regions: "
+            f"{sorted(unrecognised)}"
+        )
+    growth_by_region = {
+        name: 0.0
+        if series is None
+        else parameters.get_child(series)(current_year.time_period)
+        for name, series in _WATER_GROWTH_SERIES.items()
+    }
+    growth_rates = region.map(growth_by_region).to_numpy()
+
+    current_year.household[WATER_CHARGES] = previous_year.household[WATER_CHARGES] * (
+        1 + growth_rates
+    )
+    return current_year
+
+
 def uprate_rent(
     current_year: UKSingleYearDataset,
     previous_year: UKSingleYearDataset,
@@ -586,7 +643,7 @@ def reset_growthfactor_uprating(
     growthfactor_uprated_variables = {
         variable for variables in uprating.values() for variable in variables
     }
-    growthfactor_uprated_variables.update({"council_tax", "rent"})
+    growthfactor_uprated_variables.update({"council_tax", "rent", WATER_CHARGES})
     growthfactor_uprated_variables.update(_unmapped_uprating_indices(uprating))
 
     first_year = min(dataset.datasets.keys())

policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml

@@ -1214,6 +1214,44 @@ ofwat:
           href: https://www.ofwat.gov.uk/price-review/
         - title: Ofwat 2025-26 media statement
           href: https://www.ofwat.gov.uk/average-bills-2025-26-press-statement/
+scottish_water:
+  water_bills:
+    description: Growth in Scottish Water's household water and waste water charges, used to project the water_and_sewerage_charges of Scottish households.
+    # Scottish Water sets its own charges within limits from the Water
+    # Industry Commission for Scotland (WICS), so Scottish households do not
+    # follow the Ofwat path for England and Wales. Each value is a charging
+    # year from 1 April, stored at 1 January (2025 holds 2025-26), as in
+    # ofwat.water_bills.
+    values:
+      # Outturn: the published rise in household charges.
+      2021-01-01: 0.018
+      2022-01-01: 0.042
+      2023-01-01: 0.05
+      2024-01-01: 0.088
+      2025-01-01: 0.099
+      2026-01-01: 0.0867
+      # Projection (an assumption, not a final determination): CPI plus 2%,
+      # the cap in the WICS draft determination for 2027-33 (30 June 2026).
+      # The final determination is due on 29 October 2026. CPI is the OBR
+      # March 2026 forecast in consumer_price_index, 2.0% in each of 2027 to
+      # 2030, so 0.020 + 0.02 = 0.04. Later years hold this rate.
+      2027-01-01: 0.04
+    metadata:
+      unit: /1
+      label: Scottish Water household charges growth
+      reference:
+        - title: WICS, letter on Scottish Water's Charges Scheme 2022-23 (2021-22 1.8%, 2022-23 4.2%)
+          href: https://wics.scot/publications/price-setting/strategic-review-charges-2021-27/scottish-waters-charges-scheme/charges-scheme-2022-23
+        - title: Scottish Water, Water and Waste Water Charges for 2023/24 Announced (2 February 2023, 5%)
+          href: https://www.scottishwater.co.uk/About-Us/News-and-Views/2023/01/010223-Charges
+        - title: Scottish Water, Water and Waste Water Charges 2024 to 2025 (1 February 2024, 8.8%)
+          href: https://www.scottishwater.co.uk/About-Us/News-and-Views/2024/02/Water-and-Waste-Water-Charges-2024-and-2025
+        - title: Scottish Water, Charges 2025 to 2026 (28 January 2025, 9.9%)
+          href: https://www.scottishwater.co.uk/About-Us/News-and-Views/2025/01/Charges-2025-to-2026-Announcement
+        - title: Scottish Water, Average Water Bills to Rise £42 in 2026 (22 January 2026, 8.67%)
+          href: https://www.scottishwater.co.uk/about-us/news-and-views/2026/01/230126-charges-2026
+        - title: WICS publishes 2027-33 Draft Determination (cap of CPI plus 2%)
+          href: https://wics.scot/latest/wics-publishes-2027-33-draft-determination
 finance_ni:
   domestic_rates:
     description: Growth in the average Northern Ireland domestic rates bill (regional rate plus district rate), used to project the domestic_rates dataset input.

policyengine_uk/tests/test_projection_horizon.py

@@ -205,7 +205,6 @@ def test_unmapped_inputs_preserve_legacy_years_and_grow_after_2030(projected):
         "rail_usage",
         "stocks_and_shares_isa",
         "cash_isa",
-        "water_and_sewerage_charges",
         "employee_pension_contributions_reported",
         "pension_contributions_via_salary_sacrifice",
         "housing_service_charges",

policyengine_uk/tests/test_water_charges_uprating.py

@@ -0,0 +1,156 @@
+"""Water and sewerage charges are projected by each country's own path.
+
+Ofwat's average bill series covers England and Wales. Scottish Water sets
+Scotland's charges under limits from the Water Industry Commission for
+Scotland, and Northern Ireland has no domestic water charges
+(PolicyEngine/policyengine-uk#2501).
+"""
+
+from types import SimpleNamespace
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from policyengine_uk import Microsimulation, parameters
+from policyengine_uk.data import UKSingleYearDataset
+from policyengine_uk.data.economic_assumptions import (
+    extend_single_year_dataset,
+    reset_growthfactor_uprating,
+    uprate_water_charges,
+    uprating_input_paths,
+)
+from policyengine_uk.variables.household.demographic.geography import Region
+
+OFWAT = parameters.gov.economic_assumptions.yoy_growth.ofwat.water_bills
+SCOTTISH_WATER_PATH = "gov.economic_assumptions.yoy_growth.scottish_water.water_bills"
+# One household per known region. Region.UNKNOWN is left out of the
+# dataset-level tests because rent uprating cannot project it yet (#1985).
+KNOWN_REGIONS = [region.name for region in Region if region.name != "UNKNOWN"]
+BASE_YEAR = 2024
+
+
+def _dataset(water=400.0):
+    ids = np.arange(1, len(KNOWN_REGIONS) + 1)
+    return UKSingleYearDataset(
+        person=pd.DataFrame(
+            {
+                "person_id": ids,
+                "person_benunit_id": ids,
+                "person_household_id": ids,
+                "age": 40,
+            }
+        ),
+        benunit=pd.DataFrame({"benunit_id": ids}),
+        household=pd.DataFrame(
+            {
+                "household_id": ids,
+                "household_weight": 1.0,
+                "region": KNOWN_REGIONS,
+                "tenure_type": "OWNED_OUTRIGHT",
+                "rent": 0.0,
+                "council_tax": 0.0,
+                "water_and_sewerage_charges": water,
+            }
+        ),
+        fiscal_year=BASE_YEAR,
+    )
+
+
+def _water(dataset, region):
+    household = dataset.household
+    return household["water_and_sewerage_charges"][household["region"] == region].iloc[
+        0
+    ]
+
+
+@pytest.fixture(scope="module")
+def projected():
+    return extend_single_year_dataset(_dataset(), parameters)
+
+
+def test_scottish_charges_follow_scottish_water_published_rises(projected):
+    # Scottish Water: +9.9% from 1 April 2025, +8.67% from 1 April 2026.
+    # 400 x 1.099 = 439.60; 439.60 x 1.0867 = 477.71332.
+    assert _water(projected[2025], "SCOTLAND") == pytest.approx(439.60)
+    assert _water(projected[2026], "SCOTLAND") == pytest.approx(477.71332)
+
+
+def test_scottish_charges_assume_cpi_plus_two_per_cent_from_2027(projected):
+    # WICS draft determination cap CPI + 2%, with OBR CPI at 2.0%: 4% a year.
+    # 477.71332 x 1.04 = 496.8218528; x 1.04 again = 516.69472691.
+    assert _water(projected[2027], "SCOTLAND") == pytest.approx(496.8218528)
+    assert _water(projected[2028], "SCOTLAND") == pytest.approx(516.69472691)
+    for year in (2031, 2032):
+        assert _water(projected[year], "SCOTLAND") == pytest.approx(
+            _water(projected[year - 1], "SCOTLAND") * 1.04
+        )
+
+
+def test_england_and_wales_charges_follow_ofwat(projected):
+    for region in ("LONDON", "NORTH_EAST", "WALES"):
+        expected = 400.0
+        for year in range(BASE_YEAR + 1, 2033):
+            expected *= 1 + OFWAT(year)
+            assert _water(projected[year], region) == pytest.approx(expected), (
+                region,
+                year,
+            )
+
+
+def test_northern_ireland_charges_are_held_flat(projected):
+    # Northern Ireland has no domestic water charges, so a charge a dataset
+    # gives a household there is not grown by the England and Wales rate.
+    for year in range(BASE_YEAR + 1, 2033):
+        assert _water(projected[year], "NORTHERN_IRELAND") == 400.0
+
+
+def test_microsimulation_uses_the_scottish_path():
+    sim = Microsimulation(dataset=_dataset())
+    water = dict(
+        zip(KNOWN_REGIONS, sim.calculate("water_and_sewerage_charges", 2026).values)
+    )
+    assert water["SCOTLAND"] == pytest.approx(477.71332)
+    assert water["NORTHERN_IRELAND"] == 400.0
+    assert water["LONDON"] == pytest.approx(
+        400.0 * (1 + OFWAT(2025)) * (1 + OFWAT(2026))
+    )
+
+
+def _uprated(regions, water, year):
+    previous_year = SimpleNamespace(
+        household=pd.DataFrame({"water_and_sewerage_charges": water}, dtype=float)
+    )
+    current_year = SimpleNamespace(
+        time_period=str(year),
+        household=pd.DataFrame(
+            {"region": regions, "water_and_sewerage_charges": [0.0] * len(water)}
+        ),
+    )
+    uprate_water_charges(current_year, previous_year, parameters)
+    return current_year.household["water_and_sewerage_charges"].to_numpy()
+
+
+def test_unknown_region_takes_ofwat_growth():
+    assert _uprated(["UNKNOWN"], [400.0], 2025) == pytest.approx(
+        [400.0 * (1 + OFWAT(2025))]
+    )
+
+
+@pytest.mark.parametrize("region", ["NORTHERN IRELAND", "ATLANTIS", ""])
+def test_unrecognised_regions_are_rejected(region):
+    with pytest.raises(ValueError, match="unrecognised regions"):
+        _uprated(["LONDON", region], [400.0, 400.0], 2025)
+
+
+def test_scottish_series_is_part_of_the_uprating_fingerprint():
+    assert SCOTTISH_WATER_PATH in uprating_input_paths()
+
+
+def test_reset_growthfactor_uprating_restores_water_charges(projected):
+    reset = reset_growthfactor_uprating(projected.copy())
+    for year in range(BASE_YEAR + 1, 2033):
+        np.testing.assert_array_equal(
+            reset[year].household["water_and_sewerage_charges"].to_numpy(),
+            np.full(len(KNOWN_REGIONS), 400.0),
+        )

policyengine_uk/variables/input/consumption/property/water_and_sewerage_charges.py

@@ -7,4 +7,7 @@ class water_and_sewerage_charges(Variable):
     label = "water and sewerage charges"
     documentation = "Total amount spent on water and sewerage charges"
     definition_period = YEAR
-    uprating = "gov.economic_assumptions.indices.ofwat.water_bills"
+    # No `uprating` attribute: the dataset projection uprates this by country
+    # (uprate_water_charges in policyengine_uk/data/economic_assumptions.py),
+    # as it does council tax, because Scotland and Northern Ireland do not
+    # follow the Ofwat path for England and Wales.