diff --git a/changelog.d/spi-tax-year-start.fixed.md b/changelog.d/spi-tax-year-start.fixed.md new file mode 100644 index 000000000..a2013007d --- /dev/null +++ b/changelog.d/spi-tax-year-start.fixed.md @@ -0,0 +1 @@ +Map the HMRC Survey of Personal Incomes 2023-24 outturn to PolicyEngine year 2023, the tax year it describes, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors. Defer the CGT year mapping and aggregate/band feasibility to a follow-up on top of #571. diff --git a/policyengine_uk_data/storage/incomes_projection.csv b/policyengine_uk_data/storage/incomes_projection.csv index aa9bda2d5..ad0e914ca 100644 --- a/policyengine_uk_data/storage/incomes_projection.csv +++ b/policyengine_uk_data/storage/incomes_projection.csv @@ -1,85 +1,99 @@ total_income_lower_bound,total_income_upper_bound,employment_income_count,employment_income_amount,self_employment_income_count,self_employment_income_amount,state_pension_count,state_pension_amount,private_pension_income_count,private_pension_income_amount,property_income_count,property_income_amount,savings_interest_income_count,savings_interest_income_amount,dividend_income_count,dividend_income_amount,year -12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2024 -15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2024 -20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2024 -30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2024 -40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2024 -50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2024 -70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2024 -100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2024 -150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2024 -200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2024 -300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2024 -500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2024 -1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2024 -12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2024 -12570,15000.0,1375891,17795688094,449188,4843188790,1153320,12201198978,1082502,4506163433,102180,699345737,1264606,286001092,116344,182664931,2025 -15000,20000.0,3136222,51175765762,658607,8085406650,2073953,22954798078,2165005,14088235102,234711,1764021335,2873184,1430005461,376347,892448664,2025 -20000,30000.0,7557283,181115878830,972229,15808329316,2347108,25746597844,2691081,31284169123,436036,3987314498,5311344,3267092769,852850,3475852690,2025 -30000,40000.0,5432746,179009880239,601952,12686939451,1173554,12718198935,1466943,25793901032,368253,3966438506,3733116,2640813005,674794,4905858152,2025 -40000,50000.0,3540896,146366902077,363195,8790236620,574637,6224679479,788102,17506703913,328797,3945562513,2397692,1951905265,634326,8767916697,2025 -50000,70000.0,3470078,183221877421,314634,8276717642,446153,4818439597,650513,18024653733,401639,5636517876,2256056,2442491080,650513,13047495084,2025 -70000,100000.0,1659163,124253916871,144671,5396983767,178056,1954259836,262026,9809969589,228641,3935124517,1082502,1690955363,339926,9853468288,2025 -100000,150000.0,800242,86661842021,85993,5014361783,74865,875797927,110274,5624935044,138601,2661688997,494714,1315187504,224594,7170903298,2025 -150000,200000.0,267085,41277572384,40467,3675184841,21245,274009977,29339,1874978348,54631,1200369548,147706,678469744,93075,3601108643,2025 -200000,300000.0,182103,38539774216,34397,4420290809,14164,185085985,20234,1626362435,41479,1095989587,112297,765105112,75876,3862058545,2025 -300000,500000.0,94087,30536979570,21245,4873395789,8093,99263992,10117,1160207597,23269,798506699,68795,702477135,49573,3580232651,2025 -500000,1000000.0,44514,24324283727,15175,6716022709,3035,44461996,4047,662975769,13152,494761014,38444,681601143,32374,3987314498,2025 -1000000,inf,21245,37381474991,11129,19634549151,2023,20679998,2023,383282867,7082,458228027,21245,1294311512,19222,11586175635,2025 -12570,inf,27581555,1141661836204,3712882,108221607319,8070207,88117472623,9282205,132346537984,2378470,30643868854,19801701,19146416186,4139813,74913497775,2025 -12570,15000.0,1385161,18400750621,452214,4991391904,1161091,12481831123,1089796,4690916055,102869,719766739,1273126,294352368,117128,187998775,2026 -15000,20000.0,3157352,52915768057,663044,8332822659,2087926,23482767029,2179591,14665852495,236292,1815531028,2892541,1471761839,378882,918508301,2026 -20000,30000.0,7608199,187273911644,978779,16292069209,2362921,26338779235,2709211,32566819510,438974,4103744689,5347128,3362492376,858596,3577348119,2026 -30000,40000.0,5469348,185096308020,606008,13075163824,1181461,13010722273,1476826,26851450524,370734,4082259115,3758267,2717925148,679340,5049109958,2026 -40000,50000.0,3564752,151343451852,365642,9059220649,578508,6367849438,793412,18224478468,331013,4060773541,2413846,2008901196,638600,9023941202,2026 -50000,70000.0,3493457,189451515267,316754,8529987828,449159,4929265511,654896,18763664221,404345,5801105058,2271256,2513812192,654896,13428483931,2026 -70000,100000.0,1670341,128478613802,145646,5562133182,179256,1999208544,263792,10212178171,230181,4050030754,1089796,1740331518,342216,10141191065,2026 -100000,150000.0,805634,89608389118,86573,5167802845,75369,895941607,111017,5855557283,139535,2739410722,498047,1353591180,226107,7380294769,2026 -150000,200000.0,268884,42681031026,40740,3787646663,21389,280312309,29537,1951852428,54999,1235420522,148701,698281164,93702,3706261565,2026 -200000,300000.0,183330,39850146315,34629,4555553110,14259,189343031,20370,1693043266,41759,1127992650,113054,787446298,76388,3974831244,2026 -300000,500000.0,94721,31575252545,21389,5022523247,8148,101547101,10185,1207776088,23426,821823217,69258,722989575,49907,3684775991,2026 -500000,1000000.0,44814,25151321854,15278,6921535135,3056,45484639,4074,690157764,13241,509208111,38703,701504001,32592,4103744689,2026 -1000000,inf,21389,38652464322,11204,20235372584,2037,21155646,2037,398997458,7130,471608356,21389,1332105606,19352,11924493731,2026 -12570,inf,27767379,1180478924442,3737897,111533222839,8124578,90144207486,9344742,137772743731,2394495,31538674501,19935110,19705494460,4167704,77100983340,2026 -12570,15000.0,1395755,18842365727,455673,5176573740,1169971,12731467935,1098130,4836334659,103655,743015304,1282863,303859990,118023,194071161,2027 -15000,20000.0,3181500,54185738125,668115,8641972377,2103895,23952422725,2196261,15120494566,238099,1874172930,2914664,1519299949,381780,948176246,2027 -20000,30000.0,7666388,191768455916,986265,16896508881,2380993,26865555219,2729932,33576392346,442331,4236296209,5388023,3471101344,865163,3692896957,2027 -30000,40000.0,5511178,189538590149,610643,13560255535,1190497,13270936915,1488121,27683846669,373570,4214116647,3787011,2805714505,684536,5212196906,2027 -40000,50000.0,3592016,154975670769,368438,9395319907,582933,6495206523,799480,18789438101,333544,4191937086,2432308,2073788982,643484,9315415747,2027 -50000,70000.0,3520175,193998321682,319176,8846452421,452594,5027850896,659905,19345338636,407437,5988481552,2288627,2595008672,659905,13862225814,2027 -70000,100000.0,1683116,131562080221,146759,5768489656,180627,2039192746,265809,10528756143,231942,4180847305,1098130,1796544465,344833,10468752935,2027 -100000,150000.0,811796,91758976290,87235,5359529569,75945,913860453,111866,6037079816,140602,2827894066,501856,1397312362,227836,7618679307,2027 -150000,200000.0,270941,43705369023,41052,3928169262,21552,285918560,29762,2012359939,55420,1275324775,149838,720835742,94419,3825974325,2027 -200000,300000.0,184732,40806543526,34894,4724565222,14368,193129895,20526,1745527682,42078,1164426968,113918,812880922,76972,4103218841,2027 -300000,500000.0,95445,32333053614,21552,5208860063,8210,103578044,10263,1245217200,23605,848368220,69788,746342238,50288,3803794763,2027 -500000,1000000.0,45157,25754949603,15394,7178325748,3079,46394332,4105,711552685,13342,525655603,38999,724162677,32841,4236296209,2027 -1000000,inf,21552,39580117355,11289,20986109757,2053,21578759,2053,411366396,7184,486841371,21552,1375132801,19500,12309656523,2027 -12570,inf,27979749,1208810232000,3766485,115671132138,8186717,91947093002,9416212,142043704838,2412808,32557378036,20087577,20341984648,4199579,79591355733,2027 -12570,15000.0,1401052,19238055016,457402,5364999173,1174411,12986098807,1102298,4981427321,104049,766048722,1287731,313279627,118471,200087353,2028 -15000,20000.0,3193574,55323637503,670650,8956537080,2111879,24431474027,2204596,15574117602,239003,1932272150,2925725,1566398134,383229,977569638,2028 -20000,30000.0,7695482,195795589515,990008,17511535761,2390029,27402869517,2740292,34583702323,444010,4367621074,5408471,3578705225,868446,3807376486,2028 -30000,40000.0,5532093,193518896614,612960,14053843987,1195015,13536357231,1493768,28514377081,374987,4344753948,3801383,2892691444,687133,5373774619,2028 -40000,50000.0,3605648,158230156643,369836,9737306191,585145,6625111425,802514,19353131432,334810,4321886822,2441538,2138076285,645926,9604192937,2028 -50000,70000.0,3533535,198072282416,320388,9168460125,454312,5128408512,662409,19925709286,408983,6174124031,2297313,2675453747,662409,14291953775,2028 -70000,100000.0,1689503,134324881179,147316,5978460617,181313,2079976843,266818,10844624536,232822,4310453259,1102298,1852237209,346142,10793283491,2028 -100000,150000.0,814876,93685912890,87566,5554614528,76234,932137770,112290,6218195484,141135,2915558570,503760,1440628941,228701,7854857795,2028 -150000,200000.0,271969,44623180866,41207,4071153218,21634,291636965,29875,2072731828,55630,1314859747,150407,743181596,94777,3944579242,2028 -200000,300000.0,185433,41663480094,35026,4896537706,14423,196992516,20604,1797894459,42238,1200524117,114351,838080169,77264,4230418317,2028 -300000,500000.0,95807,33012047069,21634,5398460706,8241,105649617,10302,1282574391,23694,874667571,70053,769478791,50479,3921712116,2028 -500000,1000000.0,45328,26295803010,15453,7439614237,3091,47322224,4121,732899652,13392,541950887,39147,746611665,32966,4367621074,2028 -1000000,inf,21634,40411298999,11332,21749996646,2060,22010337,2060,423707611,7211,501933417,21634,1417761814,19574,12691254952,2028 -12570,inf,28085934,1234195221815,3780779,119881519976,8217786,93786045792,9451947,146305093007,2421965,33566654314,20163811,20972584647,4215517,82058681794,2028 -12570,15000.0,1408997,19661286328,459996,5554383918,1181071,13245818324,1108549,5120904049,104639,788723905,1295034,322552761,119143,206009975,2029 -15000,20000.0,3211685,56540740564,674454,9272703298,2123856,24920098881,2217098,16010182775,240358,1989467760,2942317,1612763805,385402,1006505878,2029 -20000,30000.0,7739124,200103032447,995622,18129693871,2403583,27950921718,2755833,35552023516,446528,4496903457,5439143,3684635555,873371,3920075527,2029 -30000,40000.0,5563466,197776253000,616436,14549945400,1201792,13807081812,1502240,29312761111,377114,4473359460,3822941,2978315640,691030,5532839332,2029 -40000,50000.0,3626095,161711171571,371934,10081033598,588463,6757612399,807065,19895006537,336709,4449815463,2455385,2201363734,649589,9888478806,2029 -50000,70000.0,3553574,202429811895,322204,9492107237,456888,5230975711,666166,20483616198,411303,6356879232,2310341,2754647667,666166,14714998223,2029 -70000,100000.0,1699085,137279987377,148152,6189500583,182341,2121575986,268331,11148266977,234142,4438043464,1108549,1907063770,348105,11112766658,2029 -100000,150000.0,819498,95746974247,88062,5750692706,76666,950780349,112927,6392300917,141936,3001859637,506617,1483271821,229998,8087363023,2029 -150000,200000.0,273511,45604877162,41441,4214865136,21757,297469649,30045,2130766972,55945,1353779836,151260,765179908,95315,4061339509,2029 -200000,300000.0,186485,42580063881,35225,5069385738,14504,200932329,20721,1848234335,42477,1236059851,114999,862887496,77702,4355639474,2029 -300000,500000.0,96351,33738301982,21757,5589026646,8288,107762590,10360,1318485640,23829,900557891,70450,792255504,50765,4037795512,2029 -500000,1000000.0,45585,26874302614,15540,7702233001,3108,48268660,4144,753420366,13468,557992733,39369,768711507,33153,4496903457,2029 -1000000,inf,21757,41300335185,11396,22517772642,2072,22450540,2072,435571149,7252,516790738,21757,1459727824,19685,13066918422,2029 -12570,inf,28245211,1261347138253,3802220,124113343773,8264389,95661748947,9505550,150401540542,2435700,34560233426,20278162,21593376992,4239424,84487633796,2029 +12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2023 +15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2023 +20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2023 +30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2023 +40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2023 +50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2023 +70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2023 +100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2023 +150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2023 +200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2023 +300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2023 +500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2023 +1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2023 +12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2023 +12570,15000.0,1373373,17761903015,448366,4941310379,1151209,12094996607,1080521,4567498426,101993,694923847,1262291,284192738,116131,181509960,2024 +15000,20000.0,3130482,51078608670,657401,8249214625,2070157,22754993616,2161042,14279995081,234281,1752867615,2867925,1420963688,375658,886805805,2024 +20000,30000.0,7543451,180772030685,970449,16128601446,2342812,25522492840,2686155,31709989076,435238,3962103130,5301622,3246435287,851289,3453875241,2024 +30000,40000.0,5422802,178670030328,600851,12943973135,1171406,12607496463,1464258,26144990993,367579,3941359134,3726283,2624115424,673559,4874838929,2024 +40000,50000.0,3534415,146089024798,362530,8968324244,573585,6170498269,786660,17744993887,328196,3920615139,2393304,1939563574,633165,8712478086,2024 +50000,70000.0,3463727,182874031042,314058,8444401521,445336,4776498660,649323,18269993706,400904,5600878770,2251927,2427047467,649323,12964997152,2024 +70000,100000.0,1656126,124018021051,144406,5506325080,177731,1937249457,261547,9943496574,228222,3910243141,1080521,1680263631,339304,9791165849,2024 +100000,150000.0,798778,86497314682,85836,5115951287,74728,868174756,110072,5701498036,138347,2644859419,493808,1306871713,224183,7125562435,2024 +150000,200000.0,266596,41199206993,40393,3749643011,21206,271624924,29285,1900499345,54531,1192779738,147436,674179852,92905,3578339214,2024 +200000,300000.0,181770,38466606529,34334,4509844608,14138,183474949,20197,1648499432,41403,1089059761,112091,760267433,75737,3837639157,2024 +300000,500000.0,93914,30479005174,21206,4972129363,8079,98399972,10098,1175999595,23226,793457826,68669,698035447,49482,3557595219,2024 +500000,1000000.0,44433,24278104121,15147,6852087366,3029,44074988,4039,671999768,13128,491632692,38374,677291451,32315,3962103130,2024 +1000000,inf,21206,37310506333,11108,20032339376,2020,20499994,2020,388499866,7069,455330700,21206,1286127717,19187,11512917471,2024 +12570,inf,27531073,1139494393423,3706087,110414145442,8055437,87350475494,9265216,134147953786,2374117,30450110911,19765458,19025355421,4132236,74439827648,2024 +12570,15000.0,1389420,18703271362,453605,4975405206,1164661,12506225444,1093147,4731469975,103185,725361239,1277040,296640268,117488,189460025,2025 +15000,20000.0,3167060,53785738946,665083,8306133847,2094346,23528661429,2186293,14792641761,237019,1829642527,2901436,1483201339,380047,925647551,2025 +20000,30000.0,7631593,190352820961,981789,16239888094,2370187,26390255387,2717542,32848366262,440324,4135641689,5363569,3388627876,861236,3605153619,2025 +30000,40000.0,5486165,188139416066,607871,13033285986,1185093,13036150251,1481367,27083586753,371874,4113989114,3769823,2739050647,681429,5088354957,2025 +40000,50000.0,3575713,153831640195,366766,9030205290,580287,6380294676,795852,18382032776,332030,4092336540,2421268,2024515696,640563,9094081200,2025 +50000,70000.0,3504199,192566225856,317728,8502667524,450540,4938899201,656910,18925879900,405588,5846195057,2278240,2533351191,656910,13532858929,2025 +70000,100000.0,1675477,130590888799,146093,5544318483,179807,2003115770,264603,10300464520,230889,4081510253,1093147,1753858517,343268,10220015063,2025 +100000,150000.0,808111,91081611425,86839,5151251128,75601,897692623,111358,5906179762,139964,2760703221,499578,1364112180,226802,7437659267,2025 +150000,200000.0,269711,43382735940,40865,3775515385,21454,280860148,29627,1968726587,55168,1245023021,149158,703708664,93990,3735069064,2025 +200000,300000.0,183894,40505309577,34735,4540962340,14303,189713081,20433,1707679968,41887,1136760150,113401,793566848,76622,4005726243,2025 +300000,500000.0,95012,32094370976,21454,5006436839,8173,101745563,10216,1218217557,23498,828210966,69471,728609125,50060,3713416490,2025 +500000,1000000.0,44952,25564826536,15324,6899366470,3065,45573533,4087,696124318,13281,513166011,38822,706956550,32692,4135641689,2025 +1000000,inf,21454,39287936884,11238,20170561645,2043,21196992,2043,402446871,7151,475274006,21454,1342459606,19411,12017178729,2025 +12570,inf,27852760,1199886793524,3749390,111175998236,8149560,90320384099,9373476,138963817010,2401857,31783813794,19996408,19858658506,4180519,77700262824,2025 +12570,15000.0,1398781,19339192185,456661,5127654184,1172507,12793873312,1100511,4925460161,103880,746541897,1285644,305302209,118279,194992287,2026 +15000,20000.0,3188397,55614481669,669563,8560304178,2108456,24069829452,2201023,15399139815,238616,1883068368,2920983,1526511044,382608,952676600,2026 +20000,30000.0,7683009,196824914547,988403,16736833822,2386155,26997241142,2735851,34195148706,443290,4256403057,5399705,3487576327,867038,3710424654,2026 +30000,40000.0,5523127,194536252750,611967,13432108672,1193078,13335986588,1491347,28194013337,374380,4234118224,3795221,2819031344,686020,5236935698,2026 +40000,50000.0,3599803,159061994896,369237,9306532437,584197,6527043842,801213,19135695799,334267,4211833391,2437581,2083631863,644879,9359629759,2026 +50000,70000.0,3527807,199113576344,319868,8762851848,453575,5052495732,661335,19701840645,408321,6016904845,2293589,2607325433,661335,13928020474,2026 +70000,100000.0,1686765,135031046027,147078,5713976388,181019,2049188183,266385,10722783386,232444,4200690975,1100511,1805071453,345581,10518441062,2026 +100000,150000.0,813556,94178432949,87424,5308881047,76110,918339889,112108,6148333029,140907,2841316177,502944,1403944464,228330,7654840053,2026 +150000,200000.0,271528,44857771222,41141,3891047353,21599,287320036,29827,2049444343,55540,1281377884,150163,724257065,94623,3844133651,2026 +200000,300000.0,185133,41882510886,34970,4679917228,14399,194076553,20570,1777694817,42169,1169953720,114165,816739121,77139,4122694060,2026 +300000,500000.0,95652,33185596057,21599,5159635395,8228,104085749,10285,1268164455,23656,852394853,69939,749884622,50397,3821848818,2026 +500000,1000000.0,45255,26434043756,15428,7110489273,3086,46621742,4114,724665403,13371,528150536,39084,727599790,32912,4256403057,2026 +1000000,inf,21599,40623746898,11314,20787787231,2057,21684531,2057,418947186,7200,489152079,21599,1381659631,19542,12368082181,2026 +12570,inf,28040411,1240683560188,3774651,114578019055,8204466,92397786751,9436627,144661331081,2418039,32711906006,20131129,20438534364,4208684,79969122354,2026 +12570,15000.0,1409479,19803329740,460153,5317891383,1181475,13049750972,1108928,5078149643,104675,770655304,1295477,315163512,119184,201290564,2027 +15000,20000.0,3212783,56949220436,674684,8877893514,2124582,24551226406,2217856,15876513825,240441,1943891736,2943324,1575817561,385534,983448186,2027 +20000,30000.0,7741770,201548681379,995963,17357774368,2404405,27537186374,2756775,35255199818,446680,4393885462,5441003,3600225523,873670,3830271882,2027 +30000,40000.0,5565369,199205092061,616647,13930443123,1202203,13602706522,1502753,29068028989,377243,4370880827,3824248,2910086445,691266,5406089443,2027 +40000,50000.0,3627335,162879457626,372061,9651807021,588665,6657584818,807341,19728903209,336824,4347876191,2456224,2150933459,649811,9661947090,2027 +50000,70000.0,3554789,203892270698,322315,9087955752,457044,5153545723,666393,20312598570,411443,6211251701,2311131,2691542404,666393,14377897456,2027 +70000,100000.0,1699666,138271769783,148203,5925966281,182403,2090171978,268423,11055190142,234222,4336373873,1108928,1863375510,348224,10858188159,2027 +100000,150000.0,819778,96438700451,88092,5505841806,76692,936706701,112966,6338931623,141984,2933091081,506791,1449292064,230077,7902092442,2027 +150000,200000.0,273605,45934350640,41455,4035406143,21764,293066441,30055,2112977208,55965,1322766566,151312,747650668,95347,3968299698,2027 +200000,300000.0,186549,42887684526,35237,4853543279,14509,197958087,20728,1832803434,42492,1207743386,115038,843119907,77729,4255857647,2027 +300000,500000.0,96383,33982045116,21764,5351059105,8291,106167466,10364,1307477609,23837,879927324,70474,774105999,50783,3945295062,2027 +500000,1000000.0,45601,27068456627,15546,7374290130,3109,47554177,4146,747130062,13473,545209872,39382,751101363,33164,4393885462,2027 +1000000,inf,21764,41598710401,11400,21559019120,2073,22118222,2073,431934567,7255,504951759,21764,1426287428,19691,12767572941,2027 +12570,inf,28254869,1270459769485,3803520,118828891025,8267215,94245743887,9508800,149145838699,2436533,33768505081,20285095,21098701843,4240873,82552136032,2027 +12570,15000.0,1414828,20219199254,461900,5511460727,1185959,13310747543,1113137,5230496886,105072,794545560,1300393,324933557,119636,207530557,2028 +15000,20000.0,3224975,58145152885,677245,9201045663,2132645,25042253852,2226273,16352817849,241353,2004152234,2954494,1624667787,386997,1013935006,2028 +20000,30000.0,7771150,205781199511,999742,17989591147,2413530,28087933375,2767237,36312874929,448376,4530095582,5461652,3711832244,876985,3949010023,2028 +30000,40000.0,5586490,203388394865,618987,14437506271,1206765,13874762269,1508456,29940085621,378675,4506377804,3838761,3000298907,693890,5573677811,2028 +40000,50000.0,3641101,166299922860,373473,10003129345,590899,6790737306,810405,20320781004,338102,4482660027,2465546,2217612235,652277,9961466726,2028 +50000,70000.0,3568279,208174004156,323538,9418754091,458779,5256617250,668922,20921987542,413005,6403800038,2319902,2774980016,668922,14823611199,2028 +70000,100000.0,1706116,141175474083,148765,6141669334,183095,2131975666,269441,11386851840,235111,4470801138,1113137,1921140011,349546,11194791177,2028 +100000,150000.0,822889,98463911161,88427,5706252478,76983,955440947,113394,6529103009,142523,3024016685,508714,1494220009,230950,8147056715,2028 +150000,200000.0,274643,46898971051,41613,4182293483,21847,298927805,30169,2176367670,56177,1363772230,151886,770827782,95709,4091316691,2028 +200000,300000.0,187257,43788325012,35371,5030210518,14564,201917272,20806,1887788531,42653,1245183341,115475,869256561,78024,4387788915,2028 +300000,500000.0,96749,34695667359,21847,5545835742,8323,108290827,10403,1346702646,23927,907205005,70741,798103227,50975,4067598913,2028 +500000,1000000.0,45774,27636893655,15605,7642711653,3121,48505266,4161,769544369,13524,562111337,39532,774385449,33290,4530095582,2028 +1000000,inf,21847,42472282457,11443,22343759705,2081,22560589,2081,444892839,7282,520605225,21847,1470502231,19766,13163366745,2028 +12570,inf,28362098,1297139398310,3817955,123154220156,8298590,96130669967,9544887,153620294735,2445780,34815326206,20362079,21752760018,4256968,85111246060,2028 +12570,15000.0,1422852,20664015438,464519,5706015573,1192684,13576959973,1119449,5376947400,105668,818064254,1307768,334551650,120315,213673499,2029 +15000,20000.0,3243265,59424328419,681086,9525843046,2144739,25543094187,2238899,16810686123,242722,2063475507,2971249,1672758251,389192,1043947668,2029 +20000,30000.0,7815221,210308322801,1005412,18624624636,2427217,28649686723,2782930,37329611831,450918,4664187241,5492625,3821703158,881959,4065901443,2029 +30000,40000.0,5618171,207862877187,622498,14947150982,1213609,14152254887,1517011,30778388563,380822,4639767412,3860531,3089108304,697825,5738659694,2029 +40000,50000.0,3661750,169958470171,375591,10356240323,594250,6926550766,815001,20889749667,340020,4615347584,2479528,2283253963,655976,10256327964,2029 +50000,70000.0,3588515,212753768415,325373,9751236593,461381,5361748600,672716,21507789598,415347,6593353691,2333058,2857119933,672716,15262392804,2029 +70000,100000.0,1715792,144281291224,149609,6358470576,184134,2174614775,270970,11705676293,236444,4603137670,1119449,1978006107,351528,11526159046,2029 +100000,150000.0,827556,100630087015,88928,5907683483,77420,974549585,114037,6711913650,143331,3113528132,511599,1538449195,232260,8388211085,2029 +150000,200000.0,276201,47930734034,41849,4329928657,21971,304906304,30340,2237304550,56496,1404140138,152747,793644426,96252,4212420414,2029 +200000,300000.0,188319,44751654736,35571,5207777207,14647,205955579,20924,1940645383,42895,1282040996,116130,894986714,78466,4517668270,2029 +300000,500000.0,97298,35458961402,21971,5741604028,8370,110456624,10462,1384409445,24063,934058440,71143,821727229,51265,4188000585,2029 +500000,1000000.0,46033,28244896841,15693,7912499766,3139,49475363,4185,791091112,13601,578749935,39756,797307400,33479,4664187241,2029 +1000000,inf,21971,43406659648,11508,23132495567,2092,23011797,2092,457349549,7324,536015235,21971,1514029366,19878,13553004810,2029 +12570,inf,28522942,1325676067330,3839607,127501570438,8345652,98053265162,9599017,157921563165,2459650,35845866235,20477554,22396645696,4281109,87630554524,2029 diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py index 17b375b8e..c76c60bc1 100644 --- a/policyengine_uk_data/targets/sources/hmrc_spi.py +++ b/policyengine_uk_data/targets/sources/hmrc_spi.py @@ -53,8 +53,11 @@ ] _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")] -# SPI year: the ODS is for tax year 2023-24, mapped to calendar 2024 -_SPI_YEAR = 2024 +# SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is +# tax year N to N+1 (policyengine-uk's convert_to_fiscal_year_parameters +# reads each parameter at 30 April N), so the outturn belongs to 2023. +# Later years come from incomes_projection.csv. +_SPI_YEAR = 2023 # HMRC Property Rental Income Statistics show ~1.9x more property income # than the SPI (£46.68bn vs £24.5bn for 2020-21), because SPI only covers @@ -229,8 +232,13 @@ def get_income_band_table(include_aggregate: bool = True) -> pd.DataFrame: def get_targets() -> list[Target]: """Build income-band targets from the live HMRC SPI ODS. - Also reads incomes_projection.csv if available, which contains - projected future year data generated by the microsimulation. + Also reads incomes_projection.csv if available, which projects the + same bands to later years with PolicyEngine's uprating indices. Each + band is returned as one target holding the base year and every + projected year: the registry keeps only the first target of a name, + and calibration reads targets without a year filter, so a band split + across two targets would lose its projected years and be calibrated + to the base-year outturn carried forward. """ config = load_config() ref = config["hmrc"]["spi_collated"] @@ -291,7 +299,18 @@ def get_targets() -> list[Target]: if proj_path.exists(): targets.extend(_read_projection_csv(proj_path, ref)) - return targets + return _merge_by_name(targets) + + +def _merge_by_name(targets: list[Target]) -> list[Target]: + """Combine targets that share a name into one, merging their years.""" + merged: dict[str, Target] = {} + for t in targets: + if t.name in merged: + merged[t.name].values.update(t.values) + else: + merged[t.name] = t + return list(merged.values()) def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: @@ -355,12 +374,4 @@ def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: ) ) - # Merge targets with the same name across years - merged: dict[str, Target] = {} - for t in targets: - if t.name in merged: - merged[t.name].values.update(t.values) - else: - merged[t.name] = t - - return list(merged.values()) + return _merge_by_name(targets) diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index 6370578cc..5ca8756c7 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -5,9 +5,12 @@ to have been generated and will be skipped otherwise. """ +import re + import pytest import pandas as pd from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.sources._common import load_config from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR PROJECTION_PATH = STORAGE_FOLDER / "incomes_projection.csv" @@ -152,3 +155,74 @@ def test_projection_is_the_spi_table_uprated_with_the_committed_table( from policyengine_uk_data.utils.incomes_projection import project_income_table pd.testing.assert_frame_equal(project_income_table(base_targets), projections) + + +def test_spi_year_is_the_start_of_the_configured_tax_year(): + """The SPI ODS name ends with its tax year (..._2324.ods is 2023-24). + + PolicyEngine UK's year N is tax year N to N+1 (each parameter takes its + value at 30 April N), so a 2023-24 outturn is the 2023 value. Mapping it + to the end year would treat it as 2024-25 and understate every later + year by a year's growth. A refreshed URL without a matching _SPI_YEAR + fails here. + """ + url = load_config()["hmrc"]["spi_collated"] + match = re.search(r"_(\d{2})(\d{2})\.ods$", url) + assert match, url + start, end = int(match[1]), int(match[2]) + assert end == (start + 1) % 100, url + assert _SPI_YEAR == 2000 + start + + +def test_calibration_reads_the_projected_spi_targets( + projections, base_targets, monkeypatch +): + """Calibration (build_loss_matrix) fetches targets without a year filter, + and the registry keeps only the first target of each name. Every SPI band + must therefore be one target carrying its outturn and every projected + year; if the projected years sit on a second target of the same name they + are dropped, and the outturn is carried forward unuprated instead. + + The official table is served from the committed incomes.csv, so the + outturn targets are always present and the test needs no download. + """ + from policyengine_uk_data.targets import get_all_targets + from policyengine_uk_data.targets import registry + from policyengine_uk_data.targets.build_loss_matrix import _resolve_value + from policyengine_uk_data.targets.sources import hmrc_spi + + def committed_table(include_aggregate: bool = True) -> pd.DataFrame: + if include_aggregate: + return base_targets + return _without_aggregate(base_targets).reset_index(drop=True) + + monkeypatch.setattr(hmrc_spi, "get_income_band_table", committed_table) + monkeypatch.setattr(registry, "discover_source_modules", lambda: [hmrc_spi]) + + names = [t.name for t in hmrc_spi.get_targets()] + assert len(names) == len(set(names)) + + spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"} + expected = _without_aggregate(projections[projections["year"] >= _SPI_YEAR]) + checked = 0 + for row in expected.itertuples(): + label = hmrc_spi._format_band_label( + row.total_income_lower_bound, row.total_income_upper_bound + ) + for variable in hmrc_spi.INCOME_VARIABLES: + amount = getattr(row, f"{variable}_amount") + if variable == "property_income": + amount *= hmrc_spi._PROPERTY_INCOME_SCALE + amount_target = spi[f"hmrc/{variable}_income_band_{label}"] + count_target = spi[f"hmrc/{variable}_count_income_band_{label}"] + assert _resolve_value(amount_target, row.year) == pytest.approx(amount) + assert _resolve_value(count_target, row.year) == pytest.approx( + getattr(row, f"{variable}_count") + ) + checked += 2 + assert checked == 2 * len(hmrc_spi.INCOME_VARIABLES) * len(expected) + + years = set(projections["year"]) + assert _SPI_YEAR in years + for target in spi.values(): + assert set(target.values) == years, target.name diff --git a/policyengine_uk_data/tests/test_property_income_targets.py b/policyengine_uk_data/tests/test_property_income_targets.py index 5a2f6208c..b5a1e2eec 100644 --- a/policyengine_uk_data/tests/test_property_income_targets.py +++ b/policyengine_uk_data/tests/test_property_income_targets.py @@ -5,6 +5,7 @@ """ from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR def test_property_income_targets_scaled(): @@ -13,7 +14,7 @@ def test_property_income_targets_scaled(): Raw SPI 2023-24 total is scaled up to better match HMRC rental income statistics, which cover more landlords than SPI. """ - base_year = 2024 + base_year = _SPI_YEAR targets = get_all_targets(year=base_year) total = sum( t.values[base_year]