Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
6f97e2c
Unpivot every measure cube of a Stat-Xplore response, and read its ag…
juaristi22 Sep 29, 2026
4778785
Add DWP State Pension and Pension Credit caseloads, February 2023 to …
juaristi22 Sep 29, 2026
2263a14
Port every relief row of HMRC Tables 6.1 and 6.2 for 2023-24 and 2024-25
juaristi22 Sep 29, 2026
f64719a
Register the chronicle#302 packages in the alias map and UK bundle
juaristi22 Sep 29, 2026
f9deb33
Add Northern Ireland State Pension and Pension Credit, February 2023 …
juaristi22 Sep 29, 2026
bb2146e
Add the OBR salary-sacrifice costing, HMRC's forgone-NICs series and …
juaristi22 Sep 29, 2026
a88ab08
Add Winter Fuel Payment recipients and beneficiaries, winters 2023-24…
juaristi22 Sep 29, 2026
3fe30e2
Give repeated rows one label so the bundle's label checks pass
juaristi22 Sep 29, 2026
4b0ba05
Add DWP workplace pension participation and amounts saved, 2023 to 2025
juaristi22 Sep 29, 2026
374ea0c
Add ONS workplace pension membership and contribution rate bands, 202…
juaristi22 Sep 29, 2026
1d99c3f
Extend the DWP forecast tables to 2030-31 and add caseloads by benefi…
juaristi22 Sep 29, 2026
bd842ce
Add Attendance Allowance cases with entitlement, February 2023 to Mar…
juaristi22 Sep 29, 2026
ec9a0a5
Add ONS ASHE pension tables P2-P4, P6-P8 and P10-P12, 2024 provisional
juaristi22 Sep 29, 2026
8723c91
Add HMRC's count of employers operating salary sacrifice for pensions
juaristi22 Sep 29, 2026
3cd8b6a
Cross Pension Credit by region, type, partner and age band
juaristi22 Sep 29, 2026
91db5f1
Record the chronicle#302 pension packages in the source checklist
juaristi22 Sep 29, 2026
c28c11a
Test the chronicle#302 pension packages
juaristi22 Sep 29, 2026
f9aa24b
Keep the #302 text artifacts out of whitespace checks
juaristi22 Sep 29, 2026
d0a9902
Stamp the devolved benefits' forecast lines England and Wales
juaristi22 Sep 29, 2026
59d0aaf
Correct chronicle#302 package headers after review
juaristi22 Sep 29, 2026
c412539
Re-pin the default bundle for chronicle#302
juaristi22 Sep 29, 2026
3d93e72
Note the England and Wales forecast lines in the source checklist
juaristi22 Sep 29, 2026
c6855da
Address review round one on chronicle#302
juaristi22 Sep 30, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@ db/data/ons/oa21_parncp_lad_rgn_ctry_dec2024_lookup/ew_oa21_parncp_lad24_rgn24_d
db/data/ons/oa21_pcon_jul2024_lookup/ew_oa21_pcon24_lookup.csv -diff
db/data/ons/oa21_wd24_lad24_lookup/ew_oa21_ward24_lad24_bestfit.csv -diff
db/data/welshgov/council_tax_dwellings_2023_24_to_2026_27/statswales_council_tax_dwellings_by_authority_and_band.csv -diff
db/data/hmrc/pension_contribution_relief_2023_24/Tables_6_1_and_6_2.csv -diff
db/data/hmrc/winter_fuel_payment_charge_2025/winter-fuel-payments-charge.html -diff
1 change: 1 addition & 0 deletions changelog.d/302.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add UK pension facts for chronicle#302: State Pension and Pension Credit caseloads and amounts from 2023, Northern Ireland, workplace pension participation and contribution bands, salary sacrifice, Winter Fuel Payment and Attendance Allowance, and the unported pension rows of the DWP forecast tables and HMRC Tables 6.1/6.2.
41 changes: 41 additions & 0 deletions chronicle/bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@
"desnz-subnational-electricity-consumption-2024",
"desnz-subnational-gas-consumption-2024",
"desnz-weekly-road-fuel-prices-september-2026",
"dfc-ni-pension-credit-statistics-may-2026",
"dfc-ni-state-pension-statistics-may-2026",
"dfc-ni-uc-statistics-may-2026",
"dfe-funded-early-education-childcare-2026",
"dfi-ni-bus-concessionary-journeys-2024-25",
Expand All @@ -77,13 +79,22 @@
"dft-nts0621-local-bus-use-frequency-2025",
"dft-nts0705-local-bus-trips-2024",
"dft-veh1103-cars-fuel-type-2025",
"dwp-attendance-allowance-entitled-country-award-age-gender-february-2023-march-2026",
"dwp-benefit-cap-november-2025",
"dwp-benefit-expenditure-caseload-spring-2026",
"dwp-benefit-statistics-february-2026",
"dwp-esa-caseload-payment-type-phase-may-2018-march-2026",
"dwp-hb-claimants-client-type-tenure-accommodation-type-september-2025-february-2026",
"dwp-hb-claimants-client-type-tenure-january-2023-february-2026",
"dwp-pension-credit-amount-band-type-february-2023-march-2026",
"dwp-pension-credit-gender-type-february-2023-march-2026",
"dwp-pension-credit-region-type-partner-age-february-2023-march-2026",
"dwp-pension-credit-type-partner-age-february-2023-march-2026",
"dwp-pip-daily-living-foi-2025",
"dwp-state-pension-age-gender-type-february-2023-march-2026",
"dwp-state-pension-amount-band-type-february-2023-march-2026",
"dwp-state-pension-category-protected-payment-february-2023-march-2026",
"dwp-state-pension-region-type-gender-february-2023-march-2026",
"dwp-uc-childcare-element-march-2021-may-2026",
"dwp-uc-deductions-march-2025-february-2026",
"dwp-uc-households-by-constituency-children-may-2025",
Expand All @@ -105,6 +116,12 @@
"dwp-uc-people-employment-indicator-january-2023-may-2026",
"dwp-uc-scotland-youngest-child-april-december-2025",
"dwp-uc-two-child-limit-2025",
"dwp-winter-fuel-payment-recipients-winter-2023-24",
"dwp-winter-fuel-payment-recipients-winters-2024-25-2025-26",
"dwp-winter-fuel-payment-statistics-winter-2023-24",
"dwp-winter-fuel-payment-statistics-winter-2024-25",
"dwp-winter-fuel-payment-statistics-winter-2025-26",
"dwp-workplace-pension-savings-trends-2009-to-2025",
"hmrc-cgt-age-2026",
"hmrc-cgt-asset-type-2026",
"hmrc-cgt-badr-ir-2026",
Expand All @@ -117,6 +134,7 @@
"hmrc-child-benefit-august-2025",
"hmrc-hydrocarbon-oils-quantities-june-2026",
"hmrc-income-tax-liabilities-july-2026",
"hmrc-pension-contribution-relief-2023-24",
"hmrc-property-rental-income-2026",
"hmrc-salary-sacrifice-reform-2029-headcounts",
"hmrc-salary-sacrifice-relief-2024-25",
Expand All @@ -126,6 +144,7 @@
"hmrc-tax-free-childcare-march-2026",
"hmrc-vat-firm-sector-targets-2024-25",
"hmrc-vat-firm-targets-2024-25",
"hmrc-winter-fuel-payment-charge-2025",
"isc-annual-census-2023",
"isc-annual-census-2024",
"mhclg-council-tax-collection-england-2025-26",
Expand All @@ -151,14 +170,36 @@
"obr-efo-expenditure-march-2026",
"obr-efo-receipts-march-2026",
"obr-fuel-duty-receipts-by-vehicle-april-2024",
"obr-salary-sacrifice-costing-february-2026",
"ofgem-energy-price-cap-levels-2024-2026",
"ofgem-energy-price-cap-q1-2024",
"ons-ashe-employee-contribution-bands-by-age-2024-full-pay",
"ons-ashe-employee-contribution-bands-by-age-2024-qualifying-earnings",
"ons-ashe-employee-contribution-bands-by-business-size-2024-full-pay",
"ons-ashe-employee-contribution-bands-by-business-size-2024-qualifying-earnings",
"ons-ashe-employee-contribution-bands-by-industry-2024-full-pay",
"ons-ashe-employee-contribution-bands-by-industry-2024-qualifying-earnings",
"ons-ashe-employee-contribution-bands-by-occupation-2024-full-pay",
"ons-ashe-employee-contribution-bands-by-occupation-2024-qualifying-earnings",
"ons-ashe-employer-contribution-bands-by-age-2024-full-pay",
"ons-ashe-employer-contribution-bands-by-age-2024-qualifying-earnings",
"ons-ashe-employer-contribution-bands-by-business-size-2024-full-pay",
"ons-ashe-employer-contribution-bands-by-business-size-2024-qualifying-earnings",
"ons-ashe-employer-contribution-bands-by-industry-2024-full-pay",
"ons-ashe-employer-contribution-bands-by-industry-2024-qualifying-earnings",
"ons-ashe-employer-contribution-bands-by-occupation-2024-full-pay",
"ons-ashe-employer-contribution-bands-by-occupation-2024-qualifying-earnings",
"ons-ashe-pension-membership-by-age-earnings-2024",
"ons-ashe-pension-membership-by-business-size-earnings-2024",
"ons-ashe-pension-membership-by-industry-earnings-2024",
"ons-ashe-pension-membership-by-occupation-earnings-2024",
"ons-census2021-ts003-household-composition-country",
"ons-census2021-ts041-households-lad",
"ons-census2021-ts041-households-pcon24",
"ons-census2021-ts046-central-heating-ltla",
"ons-census2021-ts054-tenure-lad",
"ons-consumer-trends-current-price-2026",
"ons-employee-workplace-pensions-summary-2024",
"ons-families-households-2025",
"ons-households-by-type-country-2025",
"ons-lad-population-by-age-2024",
Expand Down
126 changes: 126 additions & 0 deletions chronicle/source_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,132 @@
"dwp-esa-caseload-payment-type-phase-may-2018-march-2026": Path(
"dwp/esa_caseload_payment_type_phase_may_2018_march_2026"
),
# chronicle#302: UK pension facts (State Pension, Pension Credit, NI, workplace
# pensions, salary sacrifice, Winter Fuel Payment).
"dfc-ni-pension-credit-statistics-may-2026": Path(
"dfc_ni/pension_credit_statistics_may_2026"
),
"dfc-ni-state-pension-statistics-may-2026": Path(
"dfc_ni/state_pension_statistics_may_2026"
),
"dwp-attendance-allowance-entitled-country-award-age-gender-february-2023-march-2026": Path(
"dwp/attendance_allowance_entitled_country_award_age_gender_february_2023_march_2026"
),
"dwp-pension-credit-amount-band-type-february-2023-march-2026": Path(
"dwp/pension_credit_amount_band_type_february_2023_march_2026"
),
"dwp-pension-credit-gender-type-february-2023-march-2026": Path(
"dwp/pension_credit_gender_type_february_2023_march_2026"
),
"dwp-pension-credit-region-type-partner-age-february-2023-march-2026": Path(
"dwp/pension_credit_region_type_partner_age_february_2023_march_2026"
),
"dwp-pension-credit-type-partner-age-february-2023-march-2026": Path(
"dwp/pension_credit_type_partner_age_february_2023_march_2026"
),
"dwp-state-pension-age-gender-type-february-2023-march-2026": Path(
"dwp/state_pension_age_gender_type_february_2023_march_2026"
),
"dwp-state-pension-amount-band-type-february-2023-march-2026": Path(
"dwp/state_pension_amount_band_type_february_2023_march_2026"
),
"dwp-state-pension-category-protected-payment-february-2023-march-2026": Path(
"dwp/state_pension_category_protected_payment_february_2023_march_2026"
),
"dwp-state-pension-region-type-gender-february-2023-march-2026": Path(
"dwp/state_pension_region_type_gender_february_2023_march_2026"
),
"dwp-winter-fuel-payment-recipients-winter-2023-24": Path(
"dwp/winter_fuel_payment_recipients_winter_2023_24"
),
"dwp-winter-fuel-payment-recipients-winters-2024-25-2025-26": Path(
"dwp/winter_fuel_payment_recipients_winters_2024_25_2025_26"
),
"dwp-winter-fuel-payment-statistics-winter-2023-24": Path(
"dwp/winter_fuel_payment_statistics_winter_2023_24"
),
"dwp-winter-fuel-payment-statistics-winter-2024-25": Path(
"dwp/winter_fuel_payment_statistics_winter_2024_25"
),
"dwp-winter-fuel-payment-statistics-winter-2025-26": Path(
"dwp/winter_fuel_payment_statistics_winter_2025_26"
),
"dwp-workplace-pension-savings-trends-2009-to-2025": Path(
"dwp/workplace_pension_savings_trends_2009_to_2025"
),
"hmrc-pension-contribution-relief-2023-24": Path(
"hmrc/pension_contribution_relief_2023_24"
),
"hmrc-winter-fuel-payment-charge-2025": Path(
"hmrc/winter_fuel_payment_charge_2025"
),
"obr-salary-sacrifice-costing-february-2026": Path(
"obr/salary_sacrifice_costing_february_2026"
),
"ons-ashe-employee-contribution-bands-by-age-2024-full-pay": Path(
"ons/ashe_employee_contribution_bands_by_age_2024_full_pay"
),
"ons-ashe-employee-contribution-bands-by-age-2024-qualifying-earnings": Path(
"ons/ashe_employee_contribution_bands_by_age_2024_qualifying_earnings"
),
"ons-ashe-employee-contribution-bands-by-business-size-2024-full-pay": Path(
"ons/ashe_employee_contribution_bands_by_business_size_2024_full_pay"
),
"ons-ashe-employee-contribution-bands-by-business-size-2024-qualifying-earnings": Path(
"ons/ashe_employee_contribution_bands_by_business_size_2024_qualifying_earnings"
),
"ons-ashe-employee-contribution-bands-by-industry-2024-full-pay": Path(
"ons/ashe_employee_contribution_bands_by_industry_2024_full_pay"
),
"ons-ashe-employee-contribution-bands-by-industry-2024-qualifying-earnings": Path(
"ons/ashe_employee_contribution_bands_by_industry_2024_qualifying_earnings"
),
"ons-ashe-employee-contribution-bands-by-occupation-2024-full-pay": Path(
"ons/ashe_employee_contribution_bands_by_occupation_2024_full_pay"
),
"ons-ashe-employee-contribution-bands-by-occupation-2024-qualifying-earnings": Path(
"ons/ashe_employee_contribution_bands_by_occupation_2024_qualifying_earnings"
),
"ons-ashe-employer-contribution-bands-by-age-2024-full-pay": Path(
"ons/ashe_employer_contribution_bands_by_age_2024_full_pay"
),
"ons-ashe-employer-contribution-bands-by-age-2024-qualifying-earnings": Path(
"ons/ashe_employer_contribution_bands_by_age_2024_qualifying_earnings"
),
"ons-ashe-employer-contribution-bands-by-business-size-2024-full-pay": Path(
"ons/ashe_employer_contribution_bands_by_business_size_2024_full_pay"
),
"ons-ashe-employer-contribution-bands-by-business-size-2024-qualifying-earnings": Path(
"ons/ashe_employer_contribution_bands_by_business_size_2024_qualifying_earnings"
),
"ons-ashe-employer-contribution-bands-by-industry-2024-full-pay": Path(
"ons/ashe_employer_contribution_bands_by_industry_2024_full_pay"
),
"ons-ashe-employer-contribution-bands-by-industry-2024-qualifying-earnings": Path(
"ons/ashe_employer_contribution_bands_by_industry_2024_qualifying_earnings"
),
"ons-ashe-employer-contribution-bands-by-occupation-2024-full-pay": Path(
"ons/ashe_employer_contribution_bands_by_occupation_2024_full_pay"
),
"ons-ashe-employer-contribution-bands-by-occupation-2024-qualifying-earnings": Path(
"ons/ashe_employer_contribution_bands_by_occupation_2024_qualifying_earnings"
),
"ons-ashe-pension-membership-by-age-earnings-2024": Path(
"ons/ashe_pension_membership_by_age_earnings_2024"
),
"ons-ashe-pension-membership-by-business-size-earnings-2024": Path(
"ons/ashe_pension_membership_by_business_size_earnings_2024"
),
"ons-ashe-pension-membership-by-industry-earnings-2024": Path(
"ons/ashe_pension_membership_by_industry_earnings_2024"
),
"ons-ashe-pension-membership-by-occupation-earnings-2024": Path(
"ons/ashe_pension_membership_by_occupation_earnings_2024"
),
"ons-employee-workplace-pensions-summary-2024": Path(
"ons/employee_workplace_pensions_summary_2024"
),
# end chronicle#302
"hmrc-cgt-age-2026": Path("hmrc/cgt_age_2026"),
"hmrc-cgt-asset-type-2026": Path("hmrc/cgt_asset_type_2026"),
"hmrc-cgt-badr-ir-2026": Path("hmrc/cgt_badr_ir_2026"),
Expand Down
76 changes: 45 additions & 31 deletions chronicle/sources/rows.py
Original file line number Diff line number Diff line change
Expand Up @@ -1393,50 +1393,64 @@ def source_rows_from_statxplore_table(
) -> list[SourceRow]:
"""Unpivot a Stat-Xplore Open Data API /table response into source rows.

The response is self-describing: it echoes the submitted query, names the
The response is self-describing: it echoes the submitted query, names each
measure, and lists each dimension field with its items (labels plus value
URIs, whose trailing segments carry publisher codes such as GSS geography
identifiers). One source row is emitted per cube cell, columns are the
measure label, the cell value, and a label/URI pair per dimension field
in response order.
in response order. A request for several measures (a caseload count and
the mean of an amount, say) returns one cube per measure; their rows follow
one another in response order, so a single-measure response unpivots
exactly as before. A cube without a values array, or one whose values do
not match the fields, is a malformed response and raises rather than
returning another cube's rows as if complete.
"""
data = json.loads(content.decode("utf-8"))
fields = data.get("fields") or []
cubes = data.get("cubes") or {}
if not fields or not cubes:
return []
measure_uri, cube = next(iter(cubes.items()))
measure_label = measure_uri
for measure in data.get("measures") or []:
if measure.get("uri") == measure_uri:
measure_label = str(measure.get("label") or measure_uri)
break
measure_labels = {
str(measure.get("uri")): str(measure.get("label") or measure.get("uri"))
for measure in data.get("measures") or []
if measure.get("uri")
}
field_keys = _statxplore_field_keys(fields)
item_axes = [field.get("items") or [] for field in fields]
values = cube.get("values") if isinstance(cube, dict) else None
if values is None or any(not axis for axis in item_axes):
if any(not axis for axis in item_axes):
return []
rows: list[SourceRow] = []
for indices in itertools.product(*(range(len(axis)) for axis in item_axes)):
node: Any = values
for index in indices:
node = node[index]
row_values: dict[str, Scalar] = {
"measure": measure_label,
"value": _json_scalar(node),
}
for key, axis, index in zip(field_keys, item_axes, indices):
item = axis[index]
labels = item.get("labels") or []
uris = item.get("uris") or []
row_values[key] = str(labels[0]) if labels else None
row_values[f"{key}_uri"] = str(uris[0]) if uris else None
rows.append(
SourceRow(
artifact=artifact,
sheet_name=sheet_name,
row_number=len(rows) + 1,
values=row_values,
for measure_uri, cube in cubes.items():
values = cube.get("values") if isinstance(cube, dict) else None
if not isinstance(values, list):
raise ValueError(f"Stat-Xplore cube {measure_uri!r} has no values array.")
measure_label = measure_labels.get(measure_uri, measure_uri)
for indices in itertools.product(*(range(len(axis)) for axis in item_axes)):
node: Any = values
try:
for index in indices:
node = node[index]
except (IndexError, KeyError, TypeError) as error:
raise ValueError(
f"Stat-Xplore cube {measure_uri!r} does not match the response's "
"fields."
) from error
row_values: dict[str, Scalar] = {
"measure": measure_label,
"value": _json_scalar(node),
}
for key, axis, index in zip(field_keys, item_axes, indices):
item = axis[index]
labels = item.get("labels") or []
uris = item.get("uris") or []
row_values[key] = str(labels[0]) if labels else None
row_values[f"{key}_uri"] = str(uris[0]) if uris else None
rows.append(
SourceRow(
artifact=artifact,
sheet_name=sheet_name,
row_number=len(rows) + 1,
values=row_values,
)
)
)
return rows
32 changes: 32 additions & 0 deletions chronicle/suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -1519,7 +1519,39 @@ def _source_cell_age_range(cell: SourceCell) -> tuple[int, int | None] | None:
return None


def _statxplore_age_range(label: str) -> tuple[int, int | None] | None:
"""Read a DWP Stat-Xplore 'Age (bands and single year)' item label.

The field's value sets label single years as bare digits ('66'), bands as
'65-69', the open top band as '90 and over' and the open bottom band as
'Under 65'. 'Unknown' and a recode's 'Total' carry no age.
"""
label = label.strip()
if re.fullmatch(r"\d+", label):
lower = int(label)
return lower, lower + 1
match = re.fullmatch(r"(\d+)\s*[-–]\s*(\d+)", label)
if match:
return int(match.group(1)), int(match.group(2)) + 1
match = re.fullmatch(r"(\d+)\s+and\s+over", label, re.I)
if match:
return int(match.group(1)), None
match = re.fullmatch(r"under\s+(\d+)", label, re.I)
if match:
return 0, int(match.group(1))
return None


def _source_row_age_range(row: SourceRow) -> tuple[int, int | None] | None:
# Stat-Xplore names its age column after the field label, and its labels
# carry no "Age" prefix, so they are read only from that column.
matched, value = _source_row_value_without_interpretation(
row, "age_bands_and_single_year"
)
if matched and value is not None:
age_range = _statxplore_age_range(str(value))
if age_range is not None:
return age_range
for variable in (
"C_AGE_NAME",
"AGE_NAME",
Expand Down
Binary file not shown.
18 changes: 18 additions & 0 deletions db/data/dfc_ni/pension_credit_statistics_may_2026/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: dfc_ni
package_id: dfc-ni-pension-credit-statistics-may-2026
dataset: dfc_ni_dfc-ni-pension-credit-statistics-may-2026
source_page: https://www.communities-ni.gov.uk/publications/benefits-statistics-summary-publication-accredited-official-statistics-may-2026
table: 'Benefits Statistics Summary Publication May 2026: Pension Credit workbook'
files:
2026:
filename: bss-pension-credit-may-2026.xlsx
source_url: https://www.communities-ni.gov.uk/system/files/2026-08/bss-pension-credit-may-2026.xlsx
sha256: 7179424803d543fd30ffaff75044f77fa92db8a30300b980c33c4624b28d350e
size_bytes: 112885
fetched_at: '2026-09-29T11:56:58+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/dfc_ni/dfc-ni-pension-credit-statistics-may-2026/2026/7179424803d543fd30ffaff75044f77fa92db8a30300b980c33c4624b28d350e/bss-pension-credit-may-2026.xlsx
uri: r2://ledger-raw/raw/dfc_ni/dfc-ni-pension-credit-statistics-may-2026/2026/7179424803d543fd30ffaff75044f77fa92db8a30300b980c33c4624b28d350e/bss-pension-credit-may-2026.xlsx
Binary file not shown.
Loading
Loading