From ca87af441bcf5d6830d2255ac84f9e230a422197 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 17:29:48 -0400 Subject: [PATCH 1/3] Map the 2023-24 SPI and CGT outturns to PolicyEngine year 2023 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PolicyEngine UK's year N is tax year N to N+1: convert_to_fiscal_year_parameters gives every parameter, the uprating indices included, its value at 30 April N. hmrc_spi mapped the 2023-24 SPI to 2024 and hmrc_cgt followed, so both treated 2023-24 outturns as 2024-25 values and every projected year carried one year too little uprating. The FRS release (2024-25 -> 2024), OBR targets, the SPI microdata (2022-23 -> 2022) and #533's salary sacrifice tables already use the start year; the end-year mapping arrived without stated reasoning in the February 2026 registry refactor (de061fc). - _SPI_YEAR and _CGT_BASE_YEAR move to 2023; incomes_projection.csv is regenerated from the same ODS (its 2023 rows equal main's 2024 rows). - The CGT base year is now gated at the £6,000 AEA the outturn was realised under. - Tests tie _SPI_YEAR to the tax year in the configured ODS name, check the CGT base-year AEA against policyengine-uk, and (as in #541) check that the committed projection is the SPI table uprated with the committed factors. Co-Authored-By: Claude Opus 5.5 --- changelog.d/spi-cgt-tax-year-start.fixed.md | 1 + .../storage/incomes_projection.csv | 182 ++++++++++-------- .../targets/sources/hmrc_cgt.py | 30 +-- .../targets/sources/hmrc_spi.py | 6 +- .../tests/test_hmrc_cgt_targets.py | 16 +- .../tests/test_income_projection.py | 29 +++ .../tests/test_property_income_targets.py | 3 +- 7 files changed, 165 insertions(+), 102 deletions(-) create mode 100644 changelog.d/spi-cgt-tax-year-start.fixed.md diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md new file mode 100644 index 00000000..b49b12b5 --- /dev/null +++ b/changelog.d/spi-cgt-tax-year-start.fixed.md @@ -0,0 +1 @@ +Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Income-band, local income and capital gains targets for 2024 onwards now carry one more year of uprating. diff --git a/policyengine_uk_data/storage/incomes_projection.csv b/policyengine_uk_data/storage/incomes_projection.csv index b8db818e..961476a1 100644 --- a/policyengine_uk_data/storage/incomes_projection.csv +++ b/policyengine_uk_data/storage/incomes_projection.csv @@ -1,85 +1,99 @@ total_income_lower_bound,total_income_upper_bound,employment_income_count,employment_income_amount,self_employment_income_count,self_employment_income_amount,state_pension_count,state_pension_amount,private_pension_income_count,private_pension_income_amount,property_income_count,property_income_amount,savings_interest_income_count,savings_interest_income_amount,dividend_income_count,dividend_income_amount,year -12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2024 -15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2024 -20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2024 -30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2024 -40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2024 -50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2024 -70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2024 -100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2024 -150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2024 -200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2024 -300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2024 -500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2024 -1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2024 -12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2024 -12570,15000.0,1375891,17529896907,449188,5033426819,1153320,12181568310,1082502,4556072351,102180,688579832,1264606,281598319,116344,179852941,2025 -15000,20000.0,3136222,50411419508,658607,8402997371,2073953,22917865804,2165005,14244272179,234711,1736865546,2873184,1407991597,376347,878710084,2025 -20000,30000.0,7557283,178410785091,972229,16429272568,2347108,25705173808,2691081,31630663221,436036,3925932773,5311344,3216798319,852850,3422344538,2025 -30000,40000.0,5432746,176336241079,601952,13185276074,1173554,12697736459,1466943,26079586563,368253,3905378151,3733116,2600159664,674794,4830336134,2025 -40000,50000.0,3540896,144180808882,363195,9135512708,574637,6214664511,788102,17700602929,328797,3884823529,2397692,1921857143,634326,8632941176,2025 -50000,70000.0,3470078,180485329104,314634,8601822962,446153,4810687146,650513,18224289406,401639,5549747899,2256056,2404890756,650513,12846638655,2025 -70000,100000.0,1659163,122398096749,144671,5608974584,178056,1951115602,262026,9918621878,228641,3874546218,1082502,1664924370,339926,9701781513,2025 -100000,150000.0,800242,85367486122,85993,5211323401,74865,874388844,110274,5687235142,138601,2620714286,494714,1294941176,224594,7060512605,2025 -150000,200000.0,267085,40661062649,40467,3819544259,21245,273569119,29339,1895745047,54631,1181890756,147706,668025210,93075,3545672269,2025 -200000,300000.0,182103,37964155432,34397,4593917616,14164,184788197,20234,1644375538,41479,1079117647,112297,753326891,75876,3802605042,2025 -300000,500000.0,94087,30080888184,21245,5064820333,8093,99104285,10117,1173057709,23269,786214286,68795,691663025,49573,3525117647,2025 -500000,1000000.0,44514,23960983347,15175,6979824715,3035,44390461,4047,670318691,13152,487144538,38444,671108403,32374,3925932773,2025 -1000000,inf,21245,36823156225,11129,20405784400,2023,20646726,2023,387527993,7082,451173950,21245,1274386555,19222,11407815126,2025 -12570,inf,27581555,1124610309278,3712882,112472497809,8070207,87975699272,9282205,133812368648,2378470,30172129412,19801701,18851671429,4139813,73760260504,2025 -12570,15000.0,1385161,17918556701,452214,5189404032,1161091,12410509297,1089796,4724677003,102869,708285714,1273126,289657143,117128,185000000,2026 -15000,20000.0,3157352,51529103886,663044,8663391762,2087926,23348585287,2179591,14771403962,236292,1786571429,2892541,1448285714,378882,903857143,2026 -20000,30000.0,7608199,182366375892,978779,16938387379,2362921,26188278092,2709211,32801205857,438974,4038285714,5347128,3308857143,858596,3520285714,2026 -30000,40000.0,5469348,180245836638,606008,13593865031,1181461,12936378335,1476826,27044702842,370734,4017142857,3758267,2674571429,679340,4968571429,2026 -40000,50000.0,3564752,147377478192,365642,9418606486,578508,6331463217,793412,18355641688,331013,3996000000,2413846,1976857143,638600,8880000000,2026 -50000,70000.0,3493457,184486915147,316754,8868378615,449159,4901099434,654896,18898708010,404345,5708571429,2271256,2473714286,654896,13214285714,2026 -70000,100000.0,1670341,125111816019,145646,5782787029,179256,1987784964,263792,10285676141,230181,3985428571,1089796,1712571429,342216,9979428571,2026 -100000,150000.0,805634,87260190325,86573,5372813322,75369,890822150,111017,5897700258,139535,2695714286,498047,1332000000,226107,7262571429,2026 -150000,200000.0,268884,41562569389,40740,3937905346,21389,278710590,29537,1965900086,54999,1215714286,148701,687142857,93702,3647142857,2026 -200000,300000.0,183330,38805868358,34629,4736275197,14259,188261116,20370,1705228252,41759,1110000000,113054,774885714,76388,3911428571,2026 -300000,500000.0,94721,30747819191,21389,5221770377,8148,100966855,10185,1216468562,23426,808714286,69258,711457143,49907,3626000000,2026 -500000,1000000.0,44814,24492228390,15278,7196117441,3056,45224737,4074,695124892,13241,501085714,38703,690314286,32592,4038285714,2026 -1000000,inf,21389,37639571768,11204,21038124452,2037,21034762,2037,401869078,7130,464085714,21389,1310857143,19352,11734285714,2026 -12570,inf,27767379,1149544329897,3737897,115957826468,8124578,89629118836,9344742,138764306632,2394495,31035600000,19935110,19391171429,4167704,75871142857,2026 -12570,15000.0,1395755,18293814433,455673,5353812445,1169971,12658528698,1098130,4825839793,103655,730243697,1282863,298636975,118023,190735294,2027 -15000,20000.0,3181500,52608247423,668115,8937861525,2103895,23815198060,2196261,15087683032,238099,1841957983,2914664,1493184874,381780,931878151,2027 -20000,30000.0,7666388,186185567010,986265,17475021911,2380993,26711641067,2729932,33503531438,442331,4163478992,5388023,3411436975,865163,3629420168,2027 -30000,40000.0,5511178,184020618557,610643,14024539877,1190497,13194907033,1488121,27623772610,373570,4141680672,3787011,2757487395,684536,5122605042,2027 -40000,50000.0,3592016,150463917526,368438,9717002629,582933,6457995150,799480,18748664944,333544,4119882353,2432308,2038142857,643484,9155294118,2027 -50000,70000.0,3520175,188350515464,319176,9149342682,452594,4999046079,659905,19303359173,407437,5885546218,2288627,2550403361,659905,13623949580,2027 -70000,100000.0,1683116,127731958763,146759,5965994741,180627,2027510105,265809,10505908699,231942,4108983193,1098130,1765663866,344833,10288806723,2027 -100000,150000.0,811796,89087628866,87235,5543032428,75945,908624899,111866,6023979328,140602,2779285714,501856,1373294118,227836,7487722689,2027 -150000,200000.0,270941,42432989691,41052,4062664330,21552,284280517,29762,2007993109,55420,1253403361,149838,708445378,94419,3760210084,2027 -200000,300000.0,184732,39618556701,34894,4886327783,14368,192023444,20526,1741739879,42078,1144411765,113918,798908403,76972,4032689076,2027 -300000,500000.0,95445,31391752577,21552,5387204207,8210,102984640,10263,1242515073,23605,833785714,69788,733513445,50288,3738411765,2027 -500000,1000000.0,45157,25005154639,15394,7424101665,3079,46128537,4105,710008613,13342,516620168,38999,711715126,32841,4163478992,2027 -1000000,inf,21552,38427835052,11289,21704645048,2053,21455133,2053,410473730,7184,478473109,21552,1351495798,19500,12098067227,2027 -12570,inf,27979749,1173618556701,3766485,119631551271,8186717,91420323363,9416212,141735469423,2412808,31997752941,20087577,19992328571,4199579,78223268908,2027 -12570,15000.0,1401052,18709278351,457402,5543514461,1174411,12906548100,1102298,4927002584,104049,754453782,1287731,308537815,118471,197058824,2028 -15000,20000.0,3193574,53803013481,670650,9254557406,2111879,24281810833,2204596,15403962102,239003,1903025210,2925725,1542689076,383229,962773109,2028 -20000,30000.0,7695482,190413957177,990008,18094215600,2390029,27235004042,2740292,34205857020,444010,4301512605,5408471,3524537815,868446,3749747899,2028 -30000,40000.0,5532093,188199841396,612960,14521472393,1195015,13453435732,1493768,28202842377,374987,4278991597,3801383,2848907563,687133,5292436975,2028 -40000,50000.0,3605648,153881046788,369836,10061305872,585145,6584527082,802514,19141688200,334810,4256470588,2441538,2105714286,645926,9458823529,2028 -50000,70000.0,3533535,192628072958,320388,9473531989,454312,5096992724,662409,19708010336,408983,6080672269,2297313,2634957983,662409,14075630252,2028 -70000,100000.0,1689503,130632831086,147316,6177388256,181313,2067235247,266818,10726141258,232822,4245210084,1102298,1824201681,346142,10629915966,2028 -100000,150000.0,814876,91110864393,87566,5739439089,76234,926427648,112290,6150258398,141135,2871428571,503760,1418823529,228701,7735966387,2028 -150000,200000.0,271969,43396669310,41207,4206617003,21634,289850445,29875,2050086133,55630,1294957983,150407,731932773,94777,3884873950,2028 -200000,300000.0,185433,40518318795,35026,5059465381,14423,195785772,20604,1778251507,42238,1182352941,114351,825394958,77264,4166386555,2028 -300000,500000.0,95807,32104678826,21634,5578089395,8241,105002425,10302,1268561585,23694,861428571,70053,757831933,50479,3862352941,2028 -500000,1000000.0,45328,25573037272,15453,7687160386,3091,47032336,4121,724892334,13392,533747899,39147,735310924,32966,4301512605,2028 -1000000,inf,21634,39300555115,11332,22473707274,2060,21875505,2060,419078381,7211,494336134,21634,1396302521,19574,12499159664,2028 -12570,inf,28085934,1200272164948,3780779,123870464505,8217786,93211527890,9451947,144706632214,2421965,33058588235,20163811,20655142857,4215517,80816638655,2028 -12570,15000.0,1408997,19178350515,459996,5754294479,1181071,13164106710,1108549,5043152455,104639,779226891,1295034,318668908,119143,203529412,2029 -15000,20000.0,3211685,55151942902,674454,9606441718,2123856,24766370251,2217098,15767097330,240358,1965512605,2942317,1593344538,385402,994386555,2029 -20000,30000.0,7739124,195187946075,995622,18782208589,2403583,27778496362,2755833,35012230835,446528,4442756303,5439143,3640268908,873371,3872873950,2029 -30000,40000.0,5563466,192918318795,616436,15073619632,1201792,13721907842,1502240,28867700258,377114,4419495798,3822941,2942453782,691030,5466218487,2029 -40000,50000.0,3626095,157739095956,371934,10443865031,588463,6715925627,807065,19592937123,336709,4396235294,2455385,2174857143,649589,9769411765,2029 -50000,70000.0,3553574,197457573354,322204,9833742331,456888,5198706548,666166,20172609819,411303,6280336134,2310341,2721478992,666166,14537815126,2029 -70000,100000.0,1699085,133908009516,148152,6412269939,182341,2108488278,268331,10979000861,234142,4384605042,1108549,1884100840,348105,10978957983,2029 -100000,150000.0,819498,93395162569,88062,5957668712,76666,944915117,112927,6295245478,141936,2965714286,506617,1465411765,229998,7989983193,2029 -150000,200000.0,273511,44484694687,41441,4366564417,21757,295634600,30045,2098415159,55945,1337478992,151260,755966387,95315,4012436975,2029 -200000,300000.0,186485,41534179223,35225,5251840491,14504,199692805,20721,1820172265,42477,1221176471,114999,852497479,77702,4303193277,2029 -300000,500000.0,96351,32909595559,21757,5790184049,8288,107097817,10360,1298466839,23829,889714286,70450,782715966,50765,3989176471,2029 -500000,1000000.0,45585,26214195083,15540,7979447853,3108,47970897,4144,741981051,13468,551273950,39369,759455462,33153,4442756303,2029 -1000000,inf,21757,40285884219,11396,23328220859,2072,22312045,2072,428957795,7252,510568067,21757,1442151261,19685,12909579832,2029 -12570,inf,28245211,1230364948454,3802220,128580368098,8264389,95071624899,9505550,148117967270,2435700,34144094118,20278162,21333371429,4239424,83470319328,2029 +12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2023 +15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2023 +20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2023 +30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2023 +40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2023 +50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2023 +70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2023 +100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2023 +150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2023 +200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2023 +300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2023 +500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2023 +1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2023 +12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2023 +12570,15000.0,1373373,17685394191,448366,5039678604,1151209,12073283706,1080521,4566320072,101993,695117698,1262291,284272014,116131,181560593,2024 +15000,20000.0,3130482,50858589212,657401,8413434343,2070157,22714143921,2161042,14276311031,234281,1753356582,2867925,1421360070,375658,887053182,2024 +20000,30000.0,7543451,179993360996,970449,16449678604,2342812,25476674938,2686155,31701808318,435238,3963208370,5301622,3247340889,851289,3454838710,2024 +30000,40000.0,5422802,177900414938,600851,13201652893,1171406,12584863524,1464258,26138245931,367579,3942458588,3726283,2624847428,673559,4876198779,2024 +40000,50000.0,3534415,145459751037,362530,9146859504,573585,6159421009,786660,17740415913,328196,3921708806,2393304,1940104621,633165,8714908457,2024 +50000,70000.0,3463727,182086307054,314058,8612506887,445336,4767923904,649323,18265280289,400904,5602441151,2251927,2427724499,649323,12968613775,2024 +70000,100000.0,1656126,123483817427,144406,5615941230,177731,1933771712,261547,9940931284,228222,3911333915,1080521,1680732345,339304,9793897123,2024 +100000,150000.0,798778,86124730290,85836,5217796143,74728,866616212,110072,5700027125,138347,2645597210,493808,1307236269,224183,7127550131,2024 +150000,200000.0,266596,41021742739,40393,3824288338,21206,271137304,29285,1900009042,54531,1193112467,147436,674367916,92905,3579337402,2024 +200000,300000.0,181770,38300912863,34334,4599623508,14138,183145575,20197,1648074141,41403,1089363557,112091,760479512,75737,3838709677,2024 +300000,500000.0,93914,30347717842,21206,5071111111,8079,98223325,10098,1175696203,23226,793679163,68669,698230166,49482,3558587620,2024 +500000,1000000.0,44433,24173526971,15147,6988494031,3029,43995864,4039,671826401,13128,491769834,38374,677480384,32315,3963208370,2024 +1000000,inf,21206,37149792531,11108,20431129477,2020,20463193,2020,388399638,7069,455457716,21206,1286486486,19187,11516129032,2024 +12570,inf,27531073,1134586058091,3706087,112612194674,8055437,87193664185,9265216,134113345389,2374117,30458605057,19765458,19030662598,4132236,74460592851,2024 +12570,15000.0,1389420,18344564315,453605,5273774105,1164661,12463688999,1093147,4782640145,103185,714394071,1277040,292155187,117488,186595466,2025 +15000,20000.0,3167060,52754190871,665083,8804242424,2094346,23448635236,2186293,14952622061,237019,1801979076,2901436,1460775937,380047,911652136,2025 +20000,30000.0,7631593,186702074689,981789,17213774105,2370187,26300496278,2717542,33203616637,440324,4073112467,5363569,3337393200,861236,3550645161,2025 +30000,40000.0,5486165,184531120332,607871,13814876033,1185093,12991811414,1481367,27376491863,371874,4051787271,3769823,2697637315,681429,5011421099,2025 +40000,50000.0,3575713,150881327801,366766,9571735537,580287,6358593879,795852,18580831826,332030,4030462075,2421268,1993905841,640563,8956582389,2025 +50000,70000.0,3504199,188873029046,317728,9012561983,450540,4922100910,656910,19130560579,405588,5757802964,2278240,2495047951,656910,13328247602,2025 +70000,100000.0,1675477,128086307054,146093,5876804408,179807,1996302730,264603,10411862568,230889,4019799477,1093147,1727340889,343268,10065492589,2025 +100000,150000.0,808111,89334771784,86839,5460165289,75601,894639371,111358,5970054250,139964,2718962511,499578,1343487358,226802,7325204882,2025 +150000,200000.0,269711,42550705394,40865,4001928375,21454,279904880,29627,1990018083,55168,1226198779,149158,693068875,93990,3678596338,2025 +200000,300000.0,183894,39728464730,34735,4813278237,14303,189067825,20433,1726148282,41887,1119572799,113401,781568439,76622,3945161290,2025 +300000,500000.0,95012,31478838174,21454,5306666667,8173,101399504,10216,1231392405,23498,815688753,69471,717592851,50060,3657271142,2025 +500000,1000000.0,44952,25074522822,15324,7313112948,3065,45418528,4087,703652803,13281,505407149,38822,696267655,32692,4073112467,2025 +1000000,inf,21454,38534439834,11238,21380165289,2043,21124897,2043,406799277,7151,468088056,21454,1322162162,19411,11835483871,2025 +12570,inf,27852760,1176874356846,3749390,117843085399,8149560,90013184450,9373476,140466690778,2401857,31303255449,19996408,19558403662,4180519,76525466434,2025 +12570,15000.0,1398781,18751286307,456661,5437199265,1172507,12697932175,1100511,4959629295,103880,734838710,1285644,300516129,118279,191935484,2026 +15000,20000.0,3188397,53923817427,669563,9077070707,2108456,23889330025,2201023,15505967450,238616,1853548387,2920983,1502580645,382608,937741935,2026 +20000,30000.0,7683009,190841493776,988403,17747199265,2386155,26794789082,2735851,34432368897,443290,4189677419,5399705,3432903226,867038,3652258065,2026 +30000,40000.0,5523127,188622406639,611967,14242975207,1193078,13235980149,1491347,28389602170,374380,4167741935,3795221,2774838710,686020,5154838710,2026 +40000,50000.0,3599803,154226556017,369237,9868347107,584197,6478097601,801213,19268444846,334267,4145806452,2437581,2050967742,644879,9212903226,2026 +50000,70000.0,3527807,193060580913,319868,9291845730,453575,5014607113,661335,19838517179,408321,5922580645,2293589,2566451613,661335,13709677419,2026 +70000,100000.0,1686765,130926141079,147078,6058916437,181019,2033821340,266385,10797169982,232444,4134838710,1100511,1776774194,345581,10353548387,2026 +100000,150000.0,813556,91315435685,87424,5629366391,76110,911453267,112108,6190985533,140907,2796774194,502944,1381935484,228330,7534838710,2026 +150000,200000.0,271528,43494107884,41141,4125941230,21599,285165426,29827,2063661844,55540,1261290323,150163,712903226,94623,3783870968,2026 +200000,300000.0,185133,40609294606,34970,4962433425,14399,192621175,20570,1790027125,42169,1151612903,114165,803935484,77139,4058064516,2026 +300000,500000.0,95652,32176763485,21599,5471111111,8228,103305211,10285,1276962025,23656,839032258,69939,738129032,50397,3761935484,2026 +500000,1000000.0,45255,25630456432,15428,7539733701,3086,46272126,4114,729692586,13371,519870968,39084,716193548,32912,4189677419,2026 +1000000,inf,21599,39388796680,11314,22042699725,2057,21521919,2057,421853526,7200,481483871,21599,1360000000,19542,12174193548,2026 +12570,inf,28040411,1202967136929,3774651,121494839302,8204466,91704896609,9436627,145664882459,2418039,32199096774,20131129,20118129032,4208684,78715483871,2026 +12570,15000.0,1409479,19143983402,460153,5609458219,1181475,12951695616,1108928,5065822785,104675,757619878,1295477,309832607,119184,197885789,2027 +15000,20000.0,3212783,55053112033,674684,9364646465,2124582,24366749380,2217856,15837974684,240441,1911011334,2943324,1549163034,385534,966813426,2027 +20000,30000.0,7741770,194838174274,995963,18309458219,2404405,27330272953,2756775,35169620253,446680,4319564080,5441003,3539328684,873670,3765483871,2027 +30000,40000.0,5565369,192572614108,616647,14694214876,1202203,13500496278,1502753,28997468354,377243,4296948561,3824248,2860863121,691266,5314646905,2027 +40000,50000.0,3627335,157456431535,372061,10180991736,588665,6607559967,807341,19681012658,336824,4274333043,2456224,2114551003,649811,9498517873,2027 +50000,70000.0,3554789,197103734440,322315,9586225895,457044,5114822167,666393,20263291139,411443,6106190061,2311131,2646015693,666393,14134699215,2027 +70000,100000.0,1699666,133668049793,148203,6250872360,182403,2074466501,268423,11028354430,234222,4263025283,1108928,1831857018,348224,10674524847,2027 +100000,150000.0,819778,93227800830,88092,5807713499,76692,929668321,112966,6323544304,141984,2883478640,506791,1424777681,230077,7768430689,2027 +150000,200000.0,273605,44404979253,41455,4256657484,21764,290864351,30055,2107848101,55965,1300392328,151312,735004359,95347,3901176983,2027 +200000,300000.0,186549,41459751037,35237,5119651056,14509,196470637,20728,1828354430,42492,1187314734,115038,828858762,77729,4183870968,2027 +300000,500000.0,96383,32850622407,21764,5644444444,8291,105369727,10364,1304303797,23837,865043592,70474,761012206,50783,3878561465,2027 +500000,1000000.0,45601,26167219917,15546,7778604224,3109,47196857,4146,745316456,13473,535987794,39382,738396687,33164,4319564080,2027 +1000000,inf,21764,40213692946,11400,22741046832,2073,21952026,2073,430886076,7255,496410636,21764,1402162162,19691,12551612903,2027 +12570,inf,28254869,1228160165975,3803520,125343985308,8267215,93537584781,9508800,148783797468,2436533,33197319965,20285095,20741823017,4240873,81155789015,2027 +12570,15000.0,1414828,19578755187,461900,5808218549,1185959,13205459057,1113137,5172016275,105072,782737576,1300393,320104621,119636,204446382,2028 +15000,20000.0,3224975,56303402490,677245,9696464646,2132645,24844168734,2226273,16169981917,241353,1974367916,2954494,1600523104,386997,998866609,2028 +20000,30000.0,7771150,199263070539,999742,18958218549,2413530,27865756824,2767237,35906871609,448376,4462772450,5461652,3656669573,876985,3890322581,2028 +30000,40000.0,5586490,196946058091,618987,15214876033,1206765,13765012407,1508456,29605334539,378675,4439407149,3838761,2955710549,693890,5490845684,2028 +40000,50000.0,3641101,161032365145,373473,10541735537,590899,6737022333,810405,20093580470,338102,4416041848,2465546,2184655623,652277,9813426330,2028 +50000,70000.0,3568279,201580082988,323538,9925895317,458779,5215037221,668922,20688065099,413005,6308631212,2319902,2733740192,668922,14603312990,2028 +70000,100000.0,1706116,136703734440,148765,6472359963,183095,2115111663,269441,11259538879,235111,4404359198,1113137,1892589364,349546,11028421970,2028 +100000,150000.0,822889,95345062241,88427,6013498623,76983,947883375,113394,6456103074,142523,2979075850,508714,1472013949,230950,8025980820,2028 +150000,200000.0,274643,45413443983,41613,4407483930,21847,296563275,30169,2152034358,56177,1343504795,151886,759372276,95709,4030514385,2028 +200000,300000.0,187257,42401327801,35371,5301056015,14564,200320099,20806,1866681736,42653,1226678291,115475,856338274,78024,4322580645,2028 +300000,500000.0,96749,33596680498,21847,5844444444,8323,107434243,10403,1331645570,23927,893722755,70741,786242371,50975,4007149085,2028 +500000,1000000.0,45774,26761493776,15605,8054224059,3121,48121588,4161,760940325,13524,553757629,39532,762877071,33290,4462772450,2028 +1000000,inf,21847,41126970954,11443,23546831956,2081,22382134,2081,439918626,7282,512868352,21847,1448648649,19766,12967741935,2028 +12570,inf,28362098,1256052448133,3817955,129785307622,8298590,95370272953,9544887,151902712477,2445780,34297925022,20362079,21429485615,4256968,83846381866,2028 +12570,15000.0,1422852,20069626556,464519,6029063361,1192684,13468982630,1119449,5293942134,105668,808439407,1307768,330615519,120315,211159547,2029 +15000,20000.0,3243265,57715020747,681086,10065151515,2144739,25339950372,2238899,16551175407,242722,2039197908,2971249,1653077594,389192,1031665214,2029 +20000,30000.0,7815221,204258921162,1005412,19679063361,2427217,28421836228,2782930,36753345389,450918,4609311247,5492625,3776739320,881959,4018064516,2029 +30000,40000.0,5618171,201883817427,622498,15793388430,1213609,14039702233,1517011,30303254973,380822,4585178727,3860531,3052763731,697825,5671142110,2029 +40000,50000.0,3661750,165069709544,375591,10942561983,594250,6871464020,815001,20567269439,340020,4561046207,2479528,2256390584,655976,10135658239,2029 +50000,70000.0,3588515,206634024896,325373,10303305785,461381,5319106700,672716,21175768535,415347,6515780296,2333058,2823504795,672716,15082824760,2029 +70000,100000.0,1715792,140131120332,149609,6718457300,184134,2157320099,270970,11524972875,236444,4548979948,1119449,1954734089,351528,11390549259,2029 +100000,150000.0,827556,97735518672,88928,6242148760,77420,966799007,114037,6608300181,143331,3076896251,511599,1520348736,232260,8289520488,2029 +150000,200000.0,276201,46552033195,41849,4575068871,21971,302481390,30340,2202766727,56496,1387619878,152747,784306888,96252,4162859634,2029 +200000,300000.0,188319,43464398340,35571,5502617080,14647,204317618,20924,1910687161,42895,1266957280,116130,884456844,78466,4464516129,2029 +300000,500000.0,97298,34439004149,21971,6066666667,8370,109578164,10462,1363037975,24063,923068875,71143,812059285,51265,4138727114,2029 +500000,1000000.0,46033,27432448133,15693,8360468320,3139,49081886,4185,778878843,13601,571940715,39756,787926765,33479,4609311247,2029 +1000000,inf,21971,42158091286,11508,24442148760,2092,22828784,2092,450289331,7324,529708806,21971,1496216216,19878,13393548387,2029 +12570,inf,28522942,1287543734440,3839607,134720110193,8345652,97273449132,9599017,155483688969,2459650,35424125545,20477554,22133140366,4281109,86599546643,2029 diff --git a/policyengine_uk_data/targets/sources/hmrc_cgt.py b/policyengine_uk_data/targets/sources/hmrc_cgt.py index f2936068..8b39a8e3 100644 --- a/policyengine_uk_data/targets/sources/hmrc_cgt.py +++ b/policyengine_uk_data/targets/sources/hmrc_cgt.py @@ -29,17 +29,19 @@ for the simulation year, so the target's meaning tracks the policy in force. ``_AEA_FALLBACK`` is used only if the parameter lookup fails. -Year handling: HMRC's figures are the 2023-24 outturn, mapped to -calendar 2024 following ``hmrc_spi._SPI_YEAR``. The amount is projected -to 2029 with PolicyEngine's ``capital_gains`` uprating factors so the -target constrains the projection years the calibration actually runs -for; the taxpayer count is held flat, as the repo has no administrative -basis for forecasting CGT taxpayer numbers and count uprating factors -would be a nominal-income index applied to a headcount. - -Caveat: the 2023-24 outturn was realised under a £6,000 AEA, while the -gate applied in later years uses the £3,000 AEA then in force. The -targets are therefore approximate for the projection years; the +Year handling: HMRC's figures are the 2023-24 outturn. PolicyEngine +UK's year N is tax year N to N+1, so the outturn is the 2023 value, the +year whose £6,000 AEA it was realised under (``hmrc_spi._SPI_YEAR`` +follows the same convention). The amount is projected to 2029 with +PolicyEngine's ``capital_gains`` uprating factors so the target +constrains the projection years the calibration actually runs for; the +taxpayer count is held flat, as the repo has no administrative basis for +forecasting CGT taxpayer numbers and count uprating factors would be a +nominal-income index applied to a headcount. + +Caveat: from 2024 the gate uses the £3,000 AEA then in force, while the +projected values carry the 2023-24 outturn realised under a £6,000 AEA. +The targets are therefore approximate for the projection years; the alternative (a fixed £6,000 gate) would be wrong in a different and less transparent way, because it would not describe any year's actual policy. @@ -55,9 +57,9 @@ logger = logging.getLogger(__name__) -# HMRC CGT statistics, tax year 2023-24 outturn, mapped to calendar 2024 -# following the hmrc_spi._SPI_YEAR convention. -_CGT_BASE_YEAR = 2024 +# HMRC CGT statistics, tax year 2023-24 outturn. PolicyEngine UK's year N +# is tax year N to N+1, so the outturn is the 2023 value. +_CGT_BASE_YEAR = 2023 _MAX_YEAR = 2029 # Total chargeable gains of CGT taxpayers, 2023-24 (£65.9bn). diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py index 17b375b8..2e1d1207 100644 --- a/policyengine_uk_data/targets/sources/hmrc_spi.py +++ b/policyengine_uk_data/targets/sources/hmrc_spi.py @@ -53,8 +53,10 @@ ] _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")] -# SPI year: the ODS is for tax year 2023-24, mapped to calendar 2024 -_SPI_YEAR = 2024 +# SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is +# tax year N to N+1 (parameters take their 6 April N value), so the +# outturn belongs to 2023. Later years come from incomes_projection.csv. +_SPI_YEAR = 2023 # HMRC Property Rental Income Statistics show ~1.9x more property income # than the SPI (£46.68bn vs £24.5bn for 2020-21), because SPI only covers diff --git a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py index 66a15b08..70bb85db 100644 --- a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py +++ b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py @@ -89,6 +89,20 @@ def test_aea_tracks_the_year(): assert compute_cgt_taxpayers(_dummy_ctx(**args), target, 2023)[0] == 0.0 +def test_base_year_is_the_tax_year_the_outturn_was_realised_in(): + """PolicyEngine UK's year N is tax year N to N+1, so the 2023-24 outturn + is the 2023 value. The base-year gate must then be 2023-24's £6,000 AEA, + read from the real parameter tree; mapping the outturn to 2024 would gate + it at 2024-25's £3,000.""" + from policyengine_uk.system import system + + from policyengine_uk_data.targets.sources.hmrc_cgt import _annual_exempt_amount + + ctx = SimpleNamespace(sim=SimpleNamespace(tax_benefit_system=system)) + assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR) == 6_000 + assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR + 1) == 3_000 + + def test_targets_load_with_expected_base_year_values(): targets = {t.name: t for t in get_targets()} # Aggregate targets plus the per-size-band targets (tested in @@ -296,7 +310,7 @@ def test_registry_keeps_custom_compute_for_the_calibration_years(): def test_registry_year_filter_excludes_years_before_the_outturn(): - """The targets describe a 2023-24 outturn mapped to calendar 2024, + """The targets describe the 2023-24 outturn, PolicyEngine year 2023, so they must not appear for earlier years.""" from policyengine_uk_data.targets import get_all_targets diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index 686604e0..faa6159a 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -5,9 +5,12 @@ to have been generated and will be skipped otherwise. """ +import re + import pytest import pandas as pd from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.sources._common import load_config from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR PROJECTION_PATH = STORAGE_FOLDER / "incomes_projection.csv" @@ -143,3 +146,29 @@ def test_projection_keeps_top_open_ended_band(projections): ] assert len(top_band) == 1 assert top_band.iloc[0]["dividend_income_amount"] > 0 + + +def test_projection_is_the_spi_table_uprated_with_the_committed_table( + projections, base_targets +): + """Regenerate incomes_projection.csv whenever uprating_factors.csv changes.""" + from policyengine_uk_data.utils.incomes_projection import project_income_table + + pd.testing.assert_frame_equal(project_income_table(base_targets), projections) + + +def test_spi_year_is_the_start_of_the_configured_tax_year(): + """The SPI ODS name ends with its tax year (..._2324.ods is 2023-24). + + PolicyEngine UK's year N is tax year N to N+1 (each parameter takes its + value at 30 April N), so a 2023-24 outturn is the 2023 value. Mapping it + to the end year would treat it as 2024-25 and understate every later + year by a year's growth. A refreshed URL without a matching _SPI_YEAR + fails here. + """ + url = load_config()["hmrc"]["spi_collated"] + match = re.search(r"_(\d{2})(\d{2})\.ods$", url) + assert match, url + start, end = int(match[1]), int(match[2]) + assert end == (start + 1) % 100, url + assert _SPI_YEAR == 2000 + start diff --git a/policyengine_uk_data/tests/test_property_income_targets.py b/policyengine_uk_data/tests/test_property_income_targets.py index 5a2f6208..b5a1e2ee 100644 --- a/policyengine_uk_data/tests/test_property_income_targets.py +++ b/policyengine_uk_data/tests/test_property_income_targets.py @@ -5,6 +5,7 @@ """ from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR def test_property_income_targets_scaled(): @@ -13,7 +14,7 @@ def test_property_income_targets_scaled(): Raw SPI 2023-24 total is scaled up to better match HMRC rental income statistics, which cover more landlords than SPI. """ - base_year = 2024 + base_year = _SPI_YEAR targets = get_all_targets(year=base_year) total = sum( t.values[base_year] From a608ddbab0c85f56d6779c2bdfd1f91bae20b3e7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 17:41:11 -0400 Subject: [PATCH 2/3] Calibrate SPI income bands to their projected values build_loss_matrix fetches targets without a year filter, and the registry keeps only the first target of each name. hmrc_spi emitted each band twice: the ODS outturn (base year only), then the projection (later years). The projected target was dropped, so _resolve_value carried the base-year outturn forward unuprated: the 2025 calibration has targeted raw 2023-24 SPI values since the February 2026 registry refactor. get_targets now merges the two into one target per band, and a test checks every projected band, variable and year through the calibration's own resolution path. Co-Authored-By: Claude Opus 5.5 --- changelog.d/spi-cgt-tax-year-start.fixed.md | 2 +- .../targets/sources/hmrc_spi.py | 32 ++++++++------ .../tests/test_income_projection.py | 42 +++++++++++++++++++ 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md index b49b12b5..514abf0d 100644 --- a/changelog.d/spi-cgt-tax-year-start.fixed.md +++ b/changelog.d/spi-cgt-tax-year-start.fixed.md @@ -1 +1 @@ -Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Income-band, local income and capital gains targets for 2024 onwards now carry one more year of uprating. +Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Together these raise the 2025 SPI income targets by two years of uprating (employment income +8.6%) and the capital gains targets by one (+3.8%). diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py index 2e1d1207..080320cd 100644 --- a/policyengine_uk_data/targets/sources/hmrc_spi.py +++ b/policyengine_uk_data/targets/sources/hmrc_spi.py @@ -231,8 +231,13 @@ def get_income_band_table(include_aggregate: bool = True) -> pd.DataFrame: def get_targets() -> list[Target]: """Build income-band targets from the live HMRC SPI ODS. - Also reads incomes_projection.csv if available, which contains - projected future year data generated by the microsimulation. + Also reads incomes_projection.csv if available, which projects the + same bands to later years with PolicyEngine's uprating indices. Each + band is returned as one target holding the base year and every + projected year: the registry keeps only the first target of a name, + and calibration reads targets without a year filter, so a band split + across two targets would lose its projected years and be calibrated + to the base-year outturn carried forward. """ config = load_config() ref = config["hmrc"]["spi_collated"] @@ -293,7 +298,18 @@ def get_targets() -> list[Target]: if proj_path.exists(): targets.extend(_read_projection_csv(proj_path, ref)) - return targets + return _merge_by_name(targets) + + +def _merge_by_name(targets: list[Target]) -> list[Target]: + """Combine targets that share a name into one, merging their years.""" + merged: dict[str, Target] = {} + for t in targets: + if t.name in merged: + merged[t.name].values.update(t.values) + else: + merged[t.name] = t + return list(merged.values()) def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: @@ -357,12 +373,4 @@ def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: ) ) - # Merge targets with the same name across years - merged: dict[str, Target] = {} - for t in targets: - if t.name in merged: - merged[t.name].values.update(t.values) - else: - merged[t.name] = t - - return list(merged.values()) + return _merge_by_name(targets) diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index faa6159a..ad7c71ef 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -172,3 +172,45 @@ def test_spi_year_is_the_start_of_the_configured_tax_year(): start, end = int(match[1]), int(match[2]) assert end == (start + 1) % 100, url assert _SPI_YEAR == 2000 + start + + +def test_calibration_reads_the_projected_spi_targets(projections): + """Calibration (build_loss_matrix) fetches targets without a year filter, + and the registry keeps only the first target of each name. Every SPI band + must therefore be one target carrying all its years; if the projected + years sit on a second target of the same name they are dropped, and the + base-year outturn is carried forward unuprated instead.""" + from policyengine_uk_data.targets import get_all_targets + from policyengine_uk_data.targets.build_loss_matrix import _resolve_value + from policyengine_uk_data.targets.sources.hmrc_spi import ( + _PROPERTY_INCOME_SCALE, + INCOME_VARIABLES, + _format_band_label, + get_targets, + ) + + names = [t.name for t in get_targets()] + assert len(names) == len(set(names)) + + spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"} + checked = 0 + for row in _without_aggregate(projections).itertuples(): + if row.year <= _SPI_YEAR: + continue + label = _format_band_label( + row.total_income_lower_bound, row.total_income_upper_bound + ) + for variable in INCOME_VARIABLES: + amount = getattr(row, f"{variable}_amount") + if variable == "property_income": + amount *= _PROPERTY_INCOME_SCALE + amount_target = spi[f"hmrc/{variable}_income_band_{label}"] + count_target = spi[f"hmrc/{variable}_count_income_band_{label}"] + assert _resolve_value(amount_target, row.year) == pytest.approx(amount) + assert _resolve_value(count_target, row.year) == pytest.approx( + getattr(row, f"{variable}_count") + ) + checked += 2 + assert checked == 2 * len(INCOME_VARIABLES) * len( + _without_aggregate(projections[projections["year"] > _SPI_YEAR]) + ) From 47fa84eeac7060a863c1ebd3a8a46f7b49ce5b30 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:32:02 -0400 Subject: [PATCH 3/3] Serve the SPI merge test the committed table, and fix the sampling date Review r2 (APPROVE_WITH_NITS): with the live ODS download failing, the merge test passed even without the merge, because only the projection targets were emitted. The test now serves get_income_band_table from incomes.csv, checks the outturn year as well as the projected years through the calibration resolver, and requires every SPI target to carry all of them. The comment now names 30 April N, the date convert_to_fiscal_year_parameters samples. Co-Authored-By: Claude Opus 5.5 --- .../targets/sources/hmrc_spi.py | 5 +- .../tests/test_income_projection.py | 50 +++++++++++-------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py index 080320cd..c76c60bc 100644 --- a/policyengine_uk_data/targets/sources/hmrc_spi.py +++ b/policyengine_uk_data/targets/sources/hmrc_spi.py @@ -54,8 +54,9 @@ _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")] # SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is -# tax year N to N+1 (parameters take their 6 April N value), so the -# outturn belongs to 2023. Later years come from incomes_projection.csv. +# tax year N to N+1 (policyengine-uk's convert_to_fiscal_year_parameters +# reads each parameter at 30 April N), so the outturn belongs to 2023. +# Later years come from incomes_projection.csv. _SPI_YEAR = 2023 # HMRC Property Rental Income Statistics show ~1.9x more property income diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index ad7c71ef..944bab16 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -174,36 +174,43 @@ def test_spi_year_is_the_start_of_the_configured_tax_year(): assert _SPI_YEAR == 2000 + start -def test_calibration_reads_the_projected_spi_targets(projections): +def test_calibration_reads_the_projected_spi_targets( + projections, base_targets, monkeypatch +): """Calibration (build_loss_matrix) fetches targets without a year filter, and the registry keeps only the first target of each name. Every SPI band - must therefore be one target carrying all its years; if the projected - years sit on a second target of the same name they are dropped, and the - base-year outturn is carried forward unuprated instead.""" + must therefore be one target carrying its outturn and every projected + year; if the projected years sit on a second target of the same name they + are dropped, and the outturn is carried forward unuprated instead. + + The official table is served from the committed incomes.csv, so the + outturn targets are always present and the test needs no download. + """ from policyengine_uk_data.targets import get_all_targets from policyengine_uk_data.targets.build_loss_matrix import _resolve_value - from policyengine_uk_data.targets.sources.hmrc_spi import ( - _PROPERTY_INCOME_SCALE, - INCOME_VARIABLES, - _format_band_label, - get_targets, - ) + from policyengine_uk_data.targets.sources import hmrc_spi - names = [t.name for t in get_targets()] + def committed_table(include_aggregate: bool = True) -> pd.DataFrame: + if include_aggregate: + return base_targets + return _without_aggregate(base_targets).reset_index(drop=True) + + monkeypatch.setattr(hmrc_spi, "get_income_band_table", committed_table) + + names = [t.name for t in hmrc_spi.get_targets()] assert len(names) == len(set(names)) spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"} + expected = _without_aggregate(projections[projections["year"] >= _SPI_YEAR]) checked = 0 - for row in _without_aggregate(projections).itertuples(): - if row.year <= _SPI_YEAR: - continue - label = _format_band_label( + for row in expected.itertuples(): + label = hmrc_spi._format_band_label( row.total_income_lower_bound, row.total_income_upper_bound ) - for variable in INCOME_VARIABLES: + for variable in hmrc_spi.INCOME_VARIABLES: amount = getattr(row, f"{variable}_amount") if variable == "property_income": - amount *= _PROPERTY_INCOME_SCALE + amount *= hmrc_spi._PROPERTY_INCOME_SCALE amount_target = spi[f"hmrc/{variable}_income_band_{label}"] count_target = spi[f"hmrc/{variable}_count_income_band_{label}"] assert _resolve_value(amount_target, row.year) == pytest.approx(amount) @@ -211,6 +218,9 @@ def test_calibration_reads_the_projected_spi_targets(projections): getattr(row, f"{variable}_count") ) checked += 2 - assert checked == 2 * len(INCOME_VARIABLES) * len( - _without_aggregate(projections[projections["year"] > _SPI_YEAR]) - ) + assert checked == 2 * len(hmrc_spi.INCOME_VARIABLES) * len(expected) + + years = set(projections["year"]) + assert _SPI_YEAR in years + for target in spi.values(): + assert set(target.values) == years, target.name