diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 3b78e7e91..03101d3b5 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -41,6 +41,13 @@ The enhanced FRS dataset is licensed under strict UK Data Service terms. Violati If your change is a non-bugfix update to a cloud-hosted dataset (FRS, enhanced FRS), bump both the filename and URL in the class definition and in `storage/upload_completed_datasets.py`. That lets us store historical dataset versions separately and reproducibly. +## Changelog fragments + +- A pull request adds its own fragment, `changelog.d/..md`, unless it changes only the exempt paths below. `` uses ASCII letters, digits, `.`, `_` and `-`. The type sets the version bump: `breaking` is major, `added` and `removed` are minor, and `changed` and `fixed` are patch. +- Towncrier ignores a fragment without a type, so the `Changelog entry` check rejects any other name. It also rejects two fragments that towncrier would file as the same entry, such as `1.fixed.md` and `01.fixed.md`. +- Exempt paths: `docs/`, `.github/`, `.claude/`, `tools/`, `changelog.d/`, root Markdown files and `.gitignore`. This is a policy choice, so that docs and CI changes can merge without a data release; a change under `.github/` can still change how the next release is built. +- Merging a fragment into `main` releases data: the versioning workflow bumps the version, then the Push workflow builds, tests, uploads and tags it. + ## Repo-specific anti-patterns - **Don't** hardcode dataset years in variable transforms; use `dataset.time_period` and the uprating pipeline. diff --git a/.github/check_changelog_fragment.py b/.github/check_changelog_fragment.py new file mode 100644 index 000000000..37d536cc3 --- /dev/null +++ b/.github/check_changelog_fragment.py @@ -0,0 +1,163 @@ +"""Check the towncrier changelog fragments a pull request would merge. + +Towncrier compiles only fragments whose names carry a type from +``[tool.towncrier]`` in pyproject.toml, and leaves any other file in +changelog.d, so that entry never reaches CHANGELOG.md. This check accepts one +name form, ``changelog.d/..md`` with an ASCII ````, which +towncrier and .github/bump_version.py both read as ````. It also rejects +two fragments that towncrier would file as the same entry, because towncrier +then stops versioning with an error. + +A pull request that changes a file outside the exempt paths must add a typed +fragment of its own. The exempt paths are docs, CI and release tooling +(.github/), agent settings, offline tools, changelog fragments, root Markdown +files and .gitignore. That is a release-policy choice, not a guarantee: a +change under .github/ can still change how the next release is built. +Exempting these paths lets docs and CI changes merge without a data release. + +Usage: python .github/check_changelog_fragment.py +""" + +import os +import re +import subprocess +import sys +import tomllib +from collections.abc import Callable, Iterable +from pathlib import Path + +FRAGMENT_DIR = "changelog.d/" +GITKEEP = "changelog.d/.gitkeep" +EXEMPT_DIRS = (".claude/", ".github/", "changelog.d/", "docs/", "tools/") +EXEMPT_ROOT_FILES = frozenset({".gitignore"}) + + +def fragment_types(pyproject: Path) -> list[str]: + config = tomllib.loads(pyproject.read_text())["tool"]["towncrier"] + return [fragment_type["directory"] for fragment_type in config["type"]] + + +def fragment_pattern(types: Iterable[str]) -> re.Pattern: + alternatives = "|".join(re.escape(fragment_type) for fragment_type in types) + return re.compile( + rf"changelog\.d/(?P[A-Za-z0-9][A-Za-z0-9._-]*)\.(?P{alternatives})\.md" + ) + + +def towncrier_entry(match: re.Match) -> tuple[str, str]: + """The entry towncrier files a fragment under: it drops a number's leading zeros.""" + name = match["name"] + return (str(int(name)) if name.isdigit() else name, match["type"]) + + +def needs_fragment(path: str) -> bool: + if path.startswith(EXEMPT_DIRS) or path in EXEMPT_ROOT_FILES: + return False + return "/" in path or not path.endswith(".md") + + +def check( + changes: Iterable[tuple[str, str]], + types: list[str], + read_bytes: Callable[[str], bytes], + fragments_at_head: Iterable[str], +) -> list[str]: + """Return one message per problem. + + ``changes`` holds ``(status, path)`` pairs as printed by + ``git diff --name-status --no-renames``, so a rename is a deletion plus an + addition. ``read_bytes`` returns a path's content at the pull request's + head, and ``fragments_at_head`` lists every file in changelog.d there. + """ + changes = list(changes) + pattern = fragment_pattern(types) + problems = [] + added_fragments = [] + for status, path in changes: + if not path.startswith(FRAGMENT_DIR) or path == GITKEEP or status == "D": + continue + if not pattern.fullmatch(path): + problems.append( + f"{path}: name it changelog.d/..md, where uses " + "only ASCII letters, digits, '.', '_' and '-', and is one " + f"of: {', '.join(types)}. Towncrier ignores a fragment without a " + "type, so its entry would never reach CHANGELOG.md, and this is " + "the form that towncrier and .github/bump_version.py both read " + "the same way." + ) + continue + try: + text = read_bytes(path).decode("utf-8") + except UnicodeDecodeError: + problems.append(f"{path} is not UTF-8 text.") + continue + if not text.strip(): + problems.append(f"{path} is empty. Describe the change in it.") + elif status == "A": + added_fragments.append(path) + entries = {} + for path in sorted(fragments_at_head): + if match := pattern.fullmatch(path): + entries.setdefault(towncrier_entry(match), []).append(path) + for (name, fragment_type), paths in sorted(entries.items()): + if len(paths) > 1: + problems.append( + f"{' and '.join(paths)}: towncrier files each as " + f"{name}.{fragment_type} and stops versioning with an error. " + "Rename all but one." + ) + needing = sorted({path for _, path in changes if needs_fragment(path)}) + if needing and not added_fragments: + listed = ", ".join(needing[:5]) + (", ..." if len(needing) > 5 else "") + problems.append( + f"This pull request changes files outside the exempt paths ({listed}) " + "but adds no changelog fragment. Add one with: echo 'Description.' > " + "changelog.d/..md, where is one of: " + f"{', '.join(types)}." + ) + return problems + + +def changed_paths(base: str, head: str) -> list[tuple[str, str]]: + output = subprocess.run( + ["git", "diff", "--name-status", "--no-renames", "-z", base, head], + check=True, + capture_output=True, + ).stdout + fields = output.split(b"\0")[:-1] + return [ + (status.decode(), os.fsdecode(path)) + for status, path in zip(fields[::2], fields[1::2]) + ] + + +def main(argv: list[str]) -> int: + base, head = argv + changes = changed_paths(base, head) + types = fragment_types(Path("pyproject.toml")) + # Git keeps no empty directories, so changelog.d is absent once a pull + # request deletes every file in it. + fragment_dir = Path(FRAGMENT_DIR) + fragments_at_head = [ + f"{FRAGMENT_DIR}{path.name}" + for path in (fragment_dir.iterdir() if fragment_dir.is_dir() else []) + if path.is_file() + ] + problems = check( + changes, types, lambda path: Path(path).read_bytes(), fragments_at_head + ) + for problem in problems: + print(f"::error::{problem}") + if problems: + return 1 + if any(needs_fragment(path) for _, path in changes): + print("Changelog fragment check passed.") + else: + print( + "::notice::No fragment needed: this pull request changes only exempt paths." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/workflows/changelog_entry.yaml b/.github/workflows/changelog_entry.yaml index 8c88b1bf8..3ed5b2281 100644 --- a/.github/workflows/changelog_entry.yaml +++ b/.github/workflows/changelog_entry.yaml @@ -15,12 +15,13 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v6 - - name: Check for changelog fragment - run: | - FRAGMENTS=$(find changelog.d -type f ! -name '.gitkeep' | wc -l) - if [ "$FRAGMENTS" -eq 0 ]; then - echo "::error::No changelog fragment found in changelog.d/" - echo "Add one with: echo 'Description.' > changelog.d/\$(git branch --show-current)..md" - echo "Types: added, changed, fixed, removed, breaking" - exit 1 - fi + with: + # The pull request's merge commit and its first parent, the tip of + # main, so the check sees exactly what merging would change. + fetch-depth: 2 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.13 + - name: Check for a typed changelog fragment + run: python .github/check_changelog_fragment.py HEAD^1 HEAD diff --git a/changelog.d/218.md b/changelog.d/218.md deleted file mode 100644 index c5e09f604..000000000 --- a/changelog.d/218.md +++ /dev/null @@ -1 +0,0 @@ -Restrict income projection reweighting to employment and self-employment targets only, fixing ~2.5x inflation of dividend, property, and pension income (#218). diff --git a/changelog.d/281.md b/changelog.d/281.md deleted file mode 100644 index 4dce3b60a..000000000 --- a/changelog.d/281.md +++ /dev/null @@ -1 +0,0 @@ -Impute below-threshold England student loan holders into the FRS base dataset and add SLC liable-to-repay calibration targets for Plans 2 and 5. diff --git a/changelog.d/316.md b/changelog.d/316.md deleted file mode 100644 index 3addacfb4..000000000 --- a/changelog.d/316.md +++ /dev/null @@ -1 +0,0 @@ -Add Scotland and Wales private-rent country targets so constituency calibration no longer relies only on UK-wide rent anchors for those countries. diff --git a/changelog.d/317.md b/changelog.d/317.md deleted file mode 100644 index 738028e7c..000000000 --- a/changelog.d/317.md +++ /dev/null @@ -1 +0,0 @@ -Fix private rent target to use UK-wide figures and add social rent calibration target (#317). diff --git a/changelog.d/341.md b/changelog.d/341.md deleted file mode 100644 index dad119f0e..000000000 --- a/changelog.d/341.md +++ /dev/null @@ -1 +0,0 @@ -Split specific student-finance grant capacity out of the generic FRS education-grants residual and seed Disabled Students' Allowance expenses where reported grants plausibly identify DSA (#341). diff --git a/changelog.d/368.md b/changelog.d/368.md deleted file mode 100644 index 3d6318809..000000000 --- a/changelog.d/368.md +++ /dev/null @@ -1 +0,0 @@ -- Set Marriage Allowance take-up rate to 0.5 (HMRC outturn ~2.1m claimants of ~4.2m eligible couples) instead of the placeholder 1.0, so microsimulation no longer overstates Marriage Allowance cost by ~£500m/year. diff --git a/changelog.d/378.md b/changelog.d/378.md deleted file mode 100644 index 9f235903b..000000000 --- a/changelog.d/378.md +++ /dev/null @@ -1 +0,0 @@ -- Calibrate against self-employed NICs (combined OBR Class 2 + Class 4 line) and skip the inert Class 3 target so calibration diagnostics are no longer misleading (#378, partial close of #88). diff --git a/changelog.d/402.md b/changelog.d/402.md deleted file mode 100644 index 0672cba4a..000000000 --- a/changelog.d/402.md +++ /dev/null @@ -1 +0,0 @@ -- Override CPI and fixed fuel uplifts for `petrol_spending` / `diesel_spending` with a UK road-fuel volume index from HMRC clearances and OBR-implied forecast volumes, so household-level petrol and diesel litres no longer grow with consumer-price inflation. Resolves #402. diff --git a/changelog.d/409.md b/changelog.d/409.md deleted file mode 100644 index 42036bf54..000000000 --- a/changelog.d/409.md +++ /dev/null @@ -1 +0,0 @@ -- Update the UK data build pipeline to target FRS 2024-25 and flatten the UK Data Service TAB zip layout during prerequisite extraction. diff --git a/changelog.d/414.md b/changelog.d/414.md deleted file mode 100644 index 23f183731..000000000 --- a/changelog.d/414.md +++ /dev/null @@ -1 +0,0 @@ -- Set SPI trading and property allowance overrides on the concrete deduction variables, matching the PE-UK removal of the pass-through allowance variables. diff --git a/changelog.d/428.md b/changelog.d/428.md deleted file mode 100644 index ba3375978..000000000 --- a/changelog.d/428.md +++ /dev/null @@ -1,2 +0,0 @@ -- Impute household `bus_fare_spending` (COICOP 7.3.2 bus & coach fares) from the detailed LCFS codes as a new output of the consumption QRF, providing the passenger fare households pay (distinct from the ETB-based `bus_subsidy_spending`) as a building block for modelling bus fare reforms. Re-enables the `bus_subsidy_spending` smoke-test target (GBP 2.5bn) and records the `bus_fare_spending` target (GBP 3.4bn, DfT Annual Bus Statistics year ending March 2025) to enable once a dataset built with the imputation is published. Refs #427. -- Retry OBR detailed-forecast-table downloads with exponential backoff on transient HTTP errors (429/5xx) and connection failures, so an occasional OBR rate-limit no longer drops the OBR target set and reds the build. diff --git a/changelog.d/429.md b/changelog.d/429.md deleted file mode 100644 index 4ddf4c038..000000000 --- a/changelog.d/429.md +++ /dev/null @@ -1 +0,0 @@ -- Bump the locked `policyengine-uk` to `>=2.89.1` so the dataset build runs against a version that defines the `bus_fare_spending` input variable; without it the imputed bus fare column was dropped when materialising the dataset through the model and never reached the published dataset. diff --git a/changelog.d/431.md b/changelog.d/431.md deleted file mode 100644 index a8180a96f..000000000 --- a/changelog.d/431.md +++ /dev/null @@ -1 +0,0 @@ -- Calibrate `bus_fare_spending` and `bus_subsidy_spending` to DfT Annual Bus Statistics (year ending March 2025, England) totals — passenger fare receipts £3.4bn (table BUS05aii) and net government support £3.0bn (table BUS05bii), uplifted England→UK by population — via post-calibration scaling steps mirroring the rail subsidy calibration. Without anchoring, imputed bus fare inherited the broader transport-consumption over-estimate (~£10bn, ~3× too high) and bus subsidy drifted low (~£1.5bn). Adds tests asserting both bus totals match the DfT targets within 20% in the built dataset (skipped in PR CI where no dataset is built, active on the post-merge build, like test_energy_calibration). diff --git a/changelog.d/433.md b/changelog.d/433.md deleted file mode 100644 index feab253de..000000000 --- a/changelog.d/433.md +++ /dev/null @@ -1,2 +0,0 @@ -- Populate `employment_sector` (public/private, from FRS `mjobsect`) and `sic_industry_division` (SIC 2007, from FRS `sic`) Person-level variables in the FRS dataset. -- Add a national calibration target for public-sector employment (`employment_sector == PUBLIC`) against the ONS Public Sector Employment headcount. diff --git a/changelog.d/436.md b/changelog.d/436.md deleted file mode 100644 index 94147feb1..000000000 --- a/changelog.d/436.md +++ /dev/null @@ -1 +0,0 @@ -- Fix household weights (and all household-level variables) landing on the wrong households when the raw FRS household table is not ordered by `sernum` (the case from FRS 2024-25 onward), by sorting the household frame by `household_id`. Previously this scrambled the grossing weights, collapsing the modelled UK population (e.g. 18-24 from ~5.0m to ~3.0m) and skewing it old. diff --git a/changelog.d/462.md b/changelog.d/462.md deleted file mode 100644 index f72d41c5c..000000000 --- a/changelog.d/462.md +++ /dev/null @@ -1 +0,0 @@ -Sort the benunit table by benunit_id in create_frs so benunit-level variables align with the model's sorted entity order (raw 2024-25 FRS tables are no longer ordered by sernum). diff --git a/changelog.d/64.md b/changelog.d/64.md deleted file mode 100644 index 7dc7ae6a4..000000000 --- a/changelog.d/64.md +++ /dev/null @@ -1 +0,0 @@ -Use country-specific age profiles for Northern Ireland instead of UK-wide mean in local area demographic targets. diff --git a/changelog.d/68.md b/changelog.d/68.md deleted file mode 100644 index 85c2bf2fa..000000000 --- a/changelog.d/68.md +++ /dev/null @@ -1 +0,0 @@ -Average imputed earnings percentiles with national reference values to improve stability for local areas. diff --git a/changelog.d/73.md b/changelog.d/73.md deleted file mode 100644 index 80370f5d2..000000000 --- a/changelog.d/73.md +++ /dev/null @@ -1 +0,0 @@ -Assign `is_parent` from FRS adult-table membership and benefit-unit dependent-child counts (#73). diff --git a/changelog.d/bus-fare-distribution-anchoring.md b/changelog.d/bus-fare-distribution-anchoring.md deleted file mode 100644 index 4df2f3ad0..000000000 --- a/changelog.d/bus-fare-distribution-anchoring.md +++ /dev/null @@ -1 +0,0 @@ -- Anchor the distribution of bus_fare_spending to DfT BUS05ai (London/outside-London receipts split) and NTS0705a (local-bus trip rates by income quintile), preserving the national BUS05aii total, with post-build validation tests. diff --git a/changelog.d/cgt-band-donors.md b/changelog.d/cgt-band-donors.md deleted file mode 100644 index 04fafeca6..000000000 --- a/changelog.d/cgt-band-donors.md +++ /dev/null @@ -1 +0,0 @@ -- Stack donor households carrying HMRC Table 2.1a size-of-gain band mean gains, and calibrate to per-band taxpayer-count and gains targets. The spline-based capital gains imputation cannot produce gains above ~£2m (only ~1.6% of built gains came from £1m+ gains against HMRC's ~61%), and the aggregate #440 targets gave calibration no records to upweight and no constraint on the distribution's shape. Following the SPI synthetic-household precedent, 300 donors per HMRC band, entering at band-exact initial weights (published band taxpayer count / donors per band), (from £12,300 to £5m+) are stacked after the capital gains imputation, sampled with probability proportional to Advani & Summers percent_with_gains at the household's first adult's income; per-band count and amount targets from the committed HMRC size-of-gain table let calibration decide each band's weight. diff --git a/changelog.d/cgt-donor-sparsity.md b/changelog.d/cgt-donor-sparsity.md deleted file mode 100644 index d8c1e6983..000000000 --- a/changelog.d/cgt-donor-sparsity.md +++ /dev/null @@ -1 +0,0 @@ -- Reduce CGT band donors from 300 to 30 per HMRC size-of-gain band. The L0-regularised calibration pays a gate penalty per active record, so the first full build with 300 light donors per band (weight ~7 each in the top band) shrank total donor weight ~8x and left the £2m+ bands 94-97% under target, failing the built-dataset outcome tests before upload. Thirty donors per band carry the same band-exact initial totals with a tenth of the gate cost, making the per-band HMRC targets cheap for the optimiser to satisfy. diff --git a/changelog.d/frs-year-semantics.md b/changelog.d/frs-year-semantics.md deleted file mode 100644 index 1872aa188..000000000 --- a/changelog.d/frs-year-semantics.md +++ /dev/null @@ -1,3 +0,0 @@ -- Threshold FRS disability categories against the survey fiscal year's DWP rates, the same fiscal-converted parameter tree the disability flags read, and convert reported amounts back to weekly with the survey's 365.25/7 factor in both paths (uk-data#475, uk-data#476). -- Validate `create_frs`'s `year` against the FRS release folder being read, and add a `policy_year` argument (the release's calibration year in the build) so Disabled Students' Allowance expenses are seeded for the FRS 2024-25 dataset (uk-data#477, uk-data#478). -- Document that the saved base-year dataset is not uprate-invertible on the post-calibration rail, bus and road-fuel scalings (uk-data#479). diff --git a/changelog.d/fuel-litre-calibration.md b/changelog.d/fuel-litre-calibration.md deleted file mode 100644 index 09541c2fd..000000000 --- a/changelog.d/fuel-litre-calibration.md +++ /dev/null @@ -1 +0,0 @@ -- Calibrate final petrol and diesel litre proxies to HMRC/OBR road-fuel clearances, and uprate fuel-spending proxies by road-fuel litres and pump prices. diff --git a/changelog.d/fuel-uprating-household-weight.md b/changelog.d/fuel-uprating-household-weight.md deleted file mode 100644 index 03623d7f0..000000000 --- a/changelog.d/fuel-uprating-household-weight.md +++ /dev/null @@ -1 +0,0 @@ -Restore the pre-fuel-change household-weight uprating row when calibrating fuel litre proxies. diff --git a/changelog.d/hmrc-cgt-targets.md b/changelog.d/hmrc-cgt-targets.md deleted file mode 100644 index 5f1fd525f..000000000 --- a/changelog.d/hmrc-cgt-targets.md +++ /dev/null @@ -1 +0,0 @@ -- Add HMRC capital gains calibration targets (`hmrc/capital_gains_total`, `hmrc/cgt_taxpayers`) so the calibration constrains the distribution of gains, not just CGT revenue. Both apply an annual-exempt-amount gate that tracks the policy year, matching HMRC's CGT-taxpayer concept. diff --git a/changelog.d/longwise-local-geography.md b/changelog.d/longwise-local-geography.md deleted file mode 100644 index 2a986da6d..000000000 --- a/changelog.d/longwise-local-geography.md +++ /dev/null @@ -1 +0,0 @@ -- Add an OA-first long-format local geography weights artifact (`local_geography_weights.csv.gz`) so UK constituency and local-authority consumers can migrate away from dense area-by-household weight matrices. diff --git a/changelog.d/national-age-bands.md b/changelog.d/national-age-bands.md deleted file mode 100644 index 64e5998ba..000000000 --- a/changelog.d/national-age-bands.md +++ /dev/null @@ -1 +0,0 @@ -- Narrow the national gender x age calibration targets from 15-year to five-year bands. The wide bands pinned only their own totals, leaving age composition inside them unconstrained nationally, which collapsed the modelled 18-24 population to ~3.4M against ONS's ~5.4M. Also skip the superseded 15-year rows in `demographics.csv` so the two sets cannot double-count. diff --git a/changelog.d/obr-efo-fallback.md b/changelog.d/obr-efo-fallback.md deleted file mode 100644 index 200a3aa5e..000000000 --- a/changelog.d/obr-efo-fallback.md +++ /dev/null @@ -1 +0,0 @@ -- Commit the OBR EFO receipts and expenditure workbooks and fall back to them when the download fails. obr.uk serves 403 Forbidden to GitHub Actions runner IPs, and the swallowed error silently dropped 28 OBR targets, calibrating a degraded dataset that failed the reform-impact and target-registry tests (blocking the data upload). Permanent HTTP errors no longer burn the retry budget, and the committed workbooks — following the committed-not-fetched precedent of the HMRC size-of-gain table and ONS demographics — guarantee the full OBR target set builds offline. Update them together with the sources.yaml URLs on each EFO vintage change. diff --git a/changelog.d/salary-sacrifice-needs-pay.fixed.md b/changelog.d/salary-sacrifice-needs-pay.fixed.md new file mode 100644 index 000000000..7f9682293 --- /dev/null +++ b/changelog.d/salary-sacrifice-needs-pay.fixed.md @@ -0,0 +1,2 @@ +- Limit every salary sacrifice amount, reported or imputed, to the person's pay, and train the salary sacrifice QRF only on SALSAC respondents with pay. Stage 1 had drawn amounts for people without pay from the few SALSAC respondents who report a sacrifice but have no recorded pay, so about a million people with no pay carried salary sacrifice, and from 2029 policyengine-uk returned the excess over the £2,000 cap to their employment income. +- Stage 2 of the salary sacrifice imputation now takes the employee pension contribution it converts to salary sacrifice out of the donor's pay (at most half the pay). It had left pay unchanged, so converted donors lost the income tax relief on that contribution. diff --git a/changelog.d/scottish-ct-water-charges.md b/changelog.d/scottish-ct-water-charges.md deleted file mode 100644 index 1223fb625..000000000 --- a/changelog.d/scottish-ct-water-charges.md +++ /dev/null @@ -1 +0,0 @@ -Net Scottish Water and sewerage charges out of FRS-reported council tax (CTANNUAL includes them in Scotland), fixing a ~£500-per-household double count that overstated Scottish council tax by roughly 25% relative to Scottish Government receipts; the charges remain captured in water_and_sewerage_charges. diff --git a/changelog.d/spi-2022-23.md b/changelog.d/spi-2022-23.md deleted file mode 100644 index d61f1e622..000000000 --- a/changelog.d/spi-2022-23.md +++ /dev/null @@ -1 +0,0 @@ -Update SPI private prerequisites and income imputation to the 2022-23 Public Use Tape. diff --git a/changelog.d/spi-prior-diagnostics.md b/changelog.d/spi-prior-diagnostics.md deleted file mode 100644 index 51c1bd578..000000000 --- a/changelog.d/spi-prior-diagnostics.md +++ /dev/null @@ -1 +0,0 @@ -- Give zero-weight SPI synthetic households meaningful calibration prior mass, tag SPI and capital-gains synthetic rows in the enhanced FRS, and add source-weight/loss diagnostics to calibration target logs. diff --git a/changelog.d/spi-unknown-region.fixed.md b/changelog.d/spi-unknown-region.fixed.md new file mode 100644 index 000000000..f5111f98e --- /dev/null +++ b/changelog.d/spi-unknown-region.fixed.md @@ -0,0 +1 @@ +Take Scottish taxpayer status in the SPI dataset from HMRC's `SCOT_TXP` flag rather than the region code. `GORCODE` 13, 14 and -1 (address abroad, address unknown, composite records) stay `UNKNOWN`, which policyengine-uk 2.104.5 and later can simulate; `load_spi_dataset` relabels them `SOUTH_EAST` only when the imported model cannot simulate them, and rebuilds a cached SPI H5 that predates the flag. diff --git a/changelog.d/typed-changelog-fragments.fixed.md b/changelog.d/typed-changelog-fragments.fixed.md new file mode 100644 index 000000000..39ba2afb3 --- /dev/null +++ b/changelog.d/typed-changelog-fragments.fixed.md @@ -0,0 +1 @@ +Remove 32 changelog fragments that had no type, so towncrier never compiled them. Their changes shipped in 1.57.3 or earlier, but their entries never reached this changelog; #555 lists them. The changelog check now rejects a fragment without a type, and requires a typed fragment from each pull request unless it changes only docs, CI and the other exempt paths in CONTRIBUTING.md. diff --git a/changelog.d/uprating-factors-load-time.fixed.md b/changelog.d/uprating-factors-load-time.fixed.md new file mode 100644 index 000000000..83d7d1872 --- /dev/null +++ b/changelog.d/uprating-factors-load-time.fixed.md @@ -0,0 +1 @@ +Regenerate `storage/uprating_factors.csv` (last built 2026-05-20) from the uprating policyengine-uk applies to a dataset at load (the year-on-year growth in its `uprating_indices.yaml`) rather than from each variable's `uprating` attribute, keeping the household-weight and road-fuel overrides and storing six decimal places. The stale table moved the FRS to the 2025 calibration year with 2024-25 growth of 3.7% for earnings (policyengine-uk 2.93.0: 5.3%), 4.7% for self-employment income (0.7%) and 2.8% for GDP-indexed incomes and wealth (4.4%), and uprated salary-sacrificed pension contributions, water charges and service charges, which policyengine-uk carries unchanged, so calibration now fits the values the model runs on. The SPI income projections are regenerated with the new table, and the salary sacrifice headcount targets compare contributions with the £2,000 cap as simulated instead of deflating them through a table row that no longer exists. diff --git a/policyengine_uk_data/datasets/create_datasets.py b/policyengine_uk_data/datasets/create_datasets.py index 310081839..e005aa6d7 100644 --- a/policyengine_uk_data/datasets/create_datasets.py +++ b/policyengine_uk_data/datasets/create_datasets.py @@ -337,13 +337,15 @@ def main(): # materialisation on purpose. Rail and bus (subsidy and fares) are # fitted by simulating the saved base-year file at the calibration # year, the configuration consumers actually run: policyengine-uk - # re-uprates rail_usage (gov.dft.rail.ridership_index) and - # bus_fare_spending (CPI) at load, so a factor fitted on a - # calibration-year file and then down-rated would be uprated a - # second time. rail_usage, rail_subsidy_spending, - # bus_subsidy_spending and bus_fare_spending are not in + # re-uprates bus_fare_spending (CPI) at load, so a factor fitted + # on a calibration-year file and then down-rated would be uprated + # a second time. (rail_usage declares gov.dft.rail.ridership_index, + # but policyengine-uk carries it forward unchanged at load because + # uprating_indices.yaml does not list it.) rail_usage, + # rail_subsidy_spending and bus_subsidy_spending are not in # uprating_factors.csv, so `uprate_dataset` leaves them untouched - # in either direction: uprating the saved file back to the + # in either direction, and bus_fare_spending moves by the same CPI + # factor both ways: uprating the saved file back to the # calibration year reproduces the calibrated weights and monetary # levels but keeps these post-calibration scalings, which the # weight solve never saw. The saved file is therefore not diff --git a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py index 35800b339..83a57b925 100644 --- a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py +++ b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py @@ -8,9 +8,18 @@ not asked. 2. Headcount-targeted imputation: converts a fraction of pension - contributors without SS into below-cap (≤£2,000) SS users, moving - employee pension contributions to salary sacrifice. Targets the - OBR/ASHE estimate of ~4.3mn below-cap SS users. + contributors without SS into SS users, moving employee pension + contributions to salary sacrifice and taking the amount moved out of + their pay, so the calibrator has enough records to reach the OBR + headcounts (7.7mn users: 4.3mn at or below £2,000, 3.3mn above). + +A salary sacrifice is pay given up for an employer pension contribution, +so every amount, reported or imputed, is limited to the person's pay: +nobody without pay sacrifices anything. Without that limit, stage 1 drew +amounts for people with no pay from the few SALSAC respondents who have a +job but no recorded pay, and from 2029 policyengine-uk returns the excess +over the £2,000 cap to employment income, giving those people pay they do +not have. Exact monetary totals (~£24bn SS contributions) and final headcount calibration happen via weight optimisation in a subsequent step. @@ -23,6 +32,10 @@ from policyengine_uk.data import UKSingleYearDataset from policyengine_uk import Microsimulation +# Trained only on SALSAC respondents with pay; the name changed with that +# training set so a forest cached by an older build is not reused. +MODEL_FILE = "salary_sacrifice_paid_respondents.pkl" + PREDICTORS = [ "age", "employment_income", @@ -33,14 +46,37 @@ ] +def limit_salary_sacrifice_to_pay( + salary_sacrifice: np.ndarray, employment_income: np.ndarray +) -> np.ndarray: + """Limit salary sacrifice amounts to the pay they are sacrificed from. + + A person without pay sacrifices nothing. The upper bound is an + imputation bound, not a legal one (an employee may sacrifice, for + example, a whole bonus, as long as cash pay stays above the National + Minimum Wage): no FRS 2024-25 respondent who reports a sacrifice and + has pay reports more than the pay they still receive, so imputed + amounts are held to the same bound. + + Args: + salary_sacrifice: Annual salary sacrifice pension contributions. + employment_income: Annual employment income after the sacrifice. + + Returns: + Amounts between zero and the person's pay (zero without pay). + """ + return np.clip(salary_sacrifice, 0, np.maximum(employment_income, 0)) + + def save_salary_sacrifice_model(): """ Train and save salary sacrifice imputation model using FRS data. - Uses FRS respondents who were asked about salary sacrifice (SALSAC field) - as training data. The model learns to predict the SS pension amount - directly - non-participants have 0, participants have their reported - SPNAMT value. + Uses FRS respondents with pay who were asked about salary sacrifice + (SALSAC field) as training data. The model learns to predict the SS + pension amount directly - non-participants have 0, participants have + their reported SPNAMT value. Respondents without pay are left out: only + predictions for people with pay are kept. Returns: Trained QRF model for salary sacrifice imputation. @@ -71,12 +107,15 @@ def save_salary_sacrifice_model(): ) ss_asked = dataset.person.salary_sacrifice_asked.values - # Build training DataFrame with only those who were asked + # Build training DataFrame with only those who were asked and have pay # This includes both participants (with amounts) and non-participants (0) - training_mask = ss_asked == 1 + training_mask = (ss_asked == 1) & (employment_income > 0) if training_mask.sum() == 0: - raise ValueError("No training data found - no respondents were asked SALSAC.") + raise ValueError( + "No training data found - no SALSAC respondents with positive " + "employment income." + ) train_df = pd.DataFrame( { @@ -89,7 +128,7 @@ def save_salary_sacrifice_model(): # Train QRF model model = QRF() model.fit(train_df[PREDICTORS], train_df[IMPUTATIONS]) - model.save(STORAGE_FOLDER / "salary_sacrifice.pkl") + model.save(STORAGE_FOLDER / MODEL_FILE) return model @@ -106,7 +145,7 @@ def create_salary_sacrifice_model(overwrite_existing: bool = False): """ from policyengine_uk_data.utils.qrf import QRF - model_path = STORAGE_FOLDER / "salary_sacrifice.pkl" + model_path = STORAGE_FOLDER / MODEL_FILE if model_path.exists() and not overwrite_existing: return QRF(file_path=model_path) return save_salary_sacrifice_model() @@ -119,9 +158,12 @@ def impute_salary_sacrifice( Impute salary sacrifice pension amounts for FRS non-respondents. Stage 1: QRF predicts SS amounts for respondents not asked SALSAC. - Stage 2: Converts a fraction of pension contributors to below-cap - SS users, targeting ~4.3mn (OBR/ASHE). Moves employee pension - contributions to salary sacrifice to keep total pension consistent. + Every amount, reported or predicted, is then limited to the person's + pay (see ``limit_salary_sacrifice_to_pay``). + Stage 2: Converts a fraction of pension contributors to SS users, + for the OBR headcount targets. Moves employee pension + contributions to salary sacrifice to keep total pension consistent, + and takes the amount moved out of pay (at most half the pay). Args: dataset: PolicyEngine UK dataset with salary_sacrifice_asked @@ -133,9 +175,13 @@ def impute_salary_sacrifice( dataset = dataset.copy() sim = Microsimulation(dataset=dataset) - # Get variables needed for imputation + # Get variables needed for imputation. The predictors come from the + # simulation, as in training; the limits use the stored pay, which is in + # the same year as the stored salary sacrifice (the simulation may + # calculate a later year and uprate pay but not salary sacrifice). age = sim.calculate("age").values employment_income = sim.calculate("employment_income").values + pay = dataset.person["employment_income"].values.copy() current_ss = dataset.person.pension_contributions_via_salary_sacrifice.values # Get indicator for who was asked @@ -166,23 +212,28 @@ def impute_salary_sacrifice( # For those not asked, use the imputed values final_ss = np.where( ss_asked == 1, - current_ss, # Keep reported values exactly + current_ss, # Keep reported values imputed_ss, # Use imputed for non-respondents ) + # A sacrifice comes out of pay. This also covers SPI-synthetic rows + # copied from a SALSAC respondent, whose kept amounts come from the + # FRS-only imputation rather than the survey. + final_ss = limit_salary_sacrifice_to_pay(final_ss, pay) # Stage 2: Headcount-targeted imputation for SS users. # ASHE data shows many more SS users than the FRS captures due to # self-reporting bias in auto-enrolment. Impute additional SS users # from pension contributors to create enough records for calibration # to hit OBR headcount targets (7.7mn total, 3.3mn above 2k, - # 4.3mn below 2k). Donors keep their full employee pension amount - # so those above 2k become above-cap records and the rest below-cap. + # 4.3mn below 2k). Donors move their employee pension amount (at + # most half their pay), so those above 2k become above-cap records + # and the rest below-cap. person_weight = sim.calculate("person_weight").values employee_pension = dataset.person["employee_pension_contributions"].values.copy() has_ss = final_ss > 0 # Donor pool: employed pension contributors not already SS users - is_donor = (employee_pension > 0) & ~has_ss & (employment_income > 0) + is_donor = (employee_pension > 0) & ~has_ss & (pay > 0) # Create enough SS records for the calibrator to work with. # Target ~70% of the 7.7mn total so the calibrator can gently @@ -198,17 +249,25 @@ def impute_salary_sacrifice( rng = np.random.default_rng(seed=2024) newly_imputed = is_donor & (rng.random(len(final_ss)) < imputation_rate) - # Move full employee pension to SS so the above/below - # 2k split reflects the natural pension distribution - ss_new = employee_pension.copy() + # Move full employee pension to SS so the above/below 2k + # split reflects the natural pension distribution. The FRS + # pay these contributions were deducted from is before the + # sacrifice, so the sacrifice comes out of it: pay falls by + # the amount moved, which keeps pay plus sacrifice unchanged. At most + # half the pay moves, so the sacrifice never exceeds the pay + # that remains. + ss_new = limit_salary_sacrifice_to_pay(employee_pension, pay / 2) final_ss = np.where(newly_imputed, ss_new, final_ss) - # Reduce employee pension correspondingly + # Reduce employee pension and pay correspondingly dataset.person["employee_pension_contributions"] = np.where( newly_imputed, - 0.0, + employee_pension - ss_new, employee_pension, ) + dataset.person["employment_income"] = np.where( + newly_imputed, pay - ss_new, pay + ) dataset.person["pension_contributions_via_salary_sacrifice"] = final_ss diff --git a/policyengine_uk_data/datasets/spi.py b/policyengine_uk_data/datasets/spi.py index 41de96410..0f7b01a22 100644 --- a/policyengine_uk_data/datasets/spi.py +++ b/policyengine_uk_data/datasets/spi.py @@ -1,3 +1,5 @@ +from functools import cache + from policyengine_uk_data.storage import STORAGE_FOLDER import pandas as pd import numpy as np @@ -23,10 +25,11 @@ 7: (74, 90), } -# SPI GORCODE → policyengine-uk region enum. -# NB the SPI codebook does not include a "region unknown" code; we surface -# unknown codes explicitly rather than silently mapping them to SOUTH_EAST -# (which the previous implementation did, distorting regional income totals). +# SPI GORCODE → policyengine-uk region enum. GORCODE also takes 13 ("Address +# abroad"), 14 ("Address unknown or not available") and -1 (composite +# records). None of those is a UK region, so they become "UNKNOWN" rather +# than SOUTH_EAST (which the previous implementation used, distorting +# regional income totals). REGION_MAP = { 1: "NORTH_EAST", 2: "NORTH_WEST", @@ -43,6 +46,57 @@ } +def _spi_shaped_household(region: str) -> UKSingleYearDataset: + # The fixture policyengine-uk's own test_rent_uprating.py simulates with + # Region.UNKNOWN, so the model keeps supporting this shape. + ids = [1] + return UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": [40], + "employment_income": [60_000.0], + } + ), + benunit=pd.DataFrame({"benunit_id": ids}), + household=pd.DataFrame( + { + "household_id": ids, + "household_weight": [1.0], + "region": [region], + "rent": [0.0], + "tenure_type": ["OWNED_OUTRIGHT"], + "council_tax": [0.0], + } + ), + fiscal_year=SPI_FISCAL_YEAR, + ) + + +@cache +def model_simulates_unknown_region() -> bool: + """Whether the imported policyengine-uk can build a simulation of an + SPI-shaped household in Region.UNKNOWN. + + Releases before 2.104.5 have no rent index for it + (PolicyEngine/policyengine-uk#1985). This builds one household rather than + reading the installed version, which need not be the imported code + (``make data-local`` puts a checkout on PYTHONPATH). A failure counts as + "no" only if the same household labelled SOUTH_EAST builds; otherwise + that error is raised, since relabelling would not help. + """ + from policyengine_uk import Microsimulation + + try: + Microsimulation(dataset=_spi_shaped_household("UNKNOWN")) + except Exception: + Microsimulation(dataset=_spi_shaped_household("SOUTH_EAST")) + return False + return True + + def _get_marriage_allowance(fiscal_year: int) -> float: """Return the maximum Marriage Allowance transfer for the given UK fiscal year in £. This equals ``max`` × ``personal_allowance`` at the start of @@ -98,10 +152,10 @@ def create_spi( existing call sites don't break. seed: Seed for the random age imputation. Fixed by default so builds are deterministic. - unknown_region: Fallback region label for SPI GORCODE values outside - the documented 1-12 range. Defaults to ``"UNKNOWN"`` so regional - totals are not silently distorted; pass ``"SOUTH_EAST"`` to - reproduce legacy behaviour if needed. + unknown_region: Region label for SPI GORCODE values outside 1-12 + (address abroad, address unknown, composite records). Defaults to + ``"UNKNOWN"`` so regional totals are not silently distorted; pass + ``"SOUTH_EAST"`` to reproduce legacy behaviour if needed. """ df = pd.read_csv(spi_data_file_path, delimiter="\t") rng = np.random.default_rng(seed) @@ -121,6 +175,16 @@ def create_spi( person["dividend_income"] = df.DIVIDENDS person["gift_aid"] = df.GIFTAID household["region"] = df.GORCODE.map(REGION_MAP).fillna(unknown_region) + # GORCODE is the address at the end of the tax year; SCOT_TXP marks + # records HMRC taxed under the Scottish system for the year. The two + # disagree for some records and SCOT_TXP is also set on records with no + # UK region, so SCOT_TXP decides the rates. HMRC documents it as "." or 1; + # the 2022-23 tape holds 0 or 1. WELSH_TXP is not used: policyengine-uk + # has no separate Welsh rates. policyengine-uk carries dataset inputs to + # 2030; from 2031 it derives the status from the region again. + person["pays_scottish_income_tax"] = ( + pd.to_numeric(df.SCOT_TXP, errors="coerce") == 1 + ) household["rent"] = 0 household["tenure_type"] = "OWNED_OUTRIGHT" household["council_tax"] = 0 diff --git a/policyengine_uk_data/storage/incomes_projection.csv b/policyengine_uk_data/storage/incomes_projection.csv index b8db818ea..aa9bda2d5 100644 --- a/policyengine_uk_data/storage/incomes_projection.csv +++ b/policyengine_uk_data/storage/incomes_projection.csv @@ -13,73 +13,73 @@ total_income_lower_bound,total_income_upper_bound,employment_income_count,employ 500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2024 1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2024 12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2024 -12570,15000.0,1375891,17529896907,449188,5033426819,1153320,12181568310,1082502,4556072351,102180,688579832,1264606,281598319,116344,179852941,2025 -15000,20000.0,3136222,50411419508,658607,8402997371,2073953,22917865804,2165005,14244272179,234711,1736865546,2873184,1407991597,376347,878710084,2025 -20000,30000.0,7557283,178410785091,972229,16429272568,2347108,25705173808,2691081,31630663221,436036,3925932773,5311344,3216798319,852850,3422344538,2025 -30000,40000.0,5432746,176336241079,601952,13185276074,1173554,12697736459,1466943,26079586563,368253,3905378151,3733116,2600159664,674794,4830336134,2025 -40000,50000.0,3540896,144180808882,363195,9135512708,574637,6214664511,788102,17700602929,328797,3884823529,2397692,1921857143,634326,8632941176,2025 -50000,70000.0,3470078,180485329104,314634,8601822962,446153,4810687146,650513,18224289406,401639,5549747899,2256056,2404890756,650513,12846638655,2025 -70000,100000.0,1659163,122398096749,144671,5608974584,178056,1951115602,262026,9918621878,228641,3874546218,1082502,1664924370,339926,9701781513,2025 -100000,150000.0,800242,85367486122,85993,5211323401,74865,874388844,110274,5687235142,138601,2620714286,494714,1294941176,224594,7060512605,2025 -150000,200000.0,267085,40661062649,40467,3819544259,21245,273569119,29339,1895745047,54631,1181890756,147706,668025210,93075,3545672269,2025 -200000,300000.0,182103,37964155432,34397,4593917616,14164,184788197,20234,1644375538,41479,1079117647,112297,753326891,75876,3802605042,2025 -300000,500000.0,94087,30080888184,21245,5064820333,8093,99104285,10117,1173057709,23269,786214286,68795,691663025,49573,3525117647,2025 -500000,1000000.0,44514,23960983347,15175,6979824715,3035,44390461,4047,670318691,13152,487144538,38444,671108403,32374,3925932773,2025 -1000000,inf,21245,36823156225,11129,20405784400,2023,20646726,2023,387527993,7082,451173950,21245,1274386555,19222,11407815126,2025 -12570,inf,27581555,1124610309278,3712882,112472497809,8070207,87975699272,9282205,133812368648,2378470,30172129412,19801701,18851671429,4139813,73760260504,2025 -12570,15000.0,1385161,17918556701,452214,5189404032,1161091,12410509297,1089796,4724677003,102869,708285714,1273126,289657143,117128,185000000,2026 -15000,20000.0,3157352,51529103886,663044,8663391762,2087926,23348585287,2179591,14771403962,236292,1786571429,2892541,1448285714,378882,903857143,2026 -20000,30000.0,7608199,182366375892,978779,16938387379,2362921,26188278092,2709211,32801205857,438974,4038285714,5347128,3308857143,858596,3520285714,2026 -30000,40000.0,5469348,180245836638,606008,13593865031,1181461,12936378335,1476826,27044702842,370734,4017142857,3758267,2674571429,679340,4968571429,2026 -40000,50000.0,3564752,147377478192,365642,9418606486,578508,6331463217,793412,18355641688,331013,3996000000,2413846,1976857143,638600,8880000000,2026 -50000,70000.0,3493457,184486915147,316754,8868378615,449159,4901099434,654896,18898708010,404345,5708571429,2271256,2473714286,654896,13214285714,2026 -70000,100000.0,1670341,125111816019,145646,5782787029,179256,1987784964,263792,10285676141,230181,3985428571,1089796,1712571429,342216,9979428571,2026 -100000,150000.0,805634,87260190325,86573,5372813322,75369,890822150,111017,5897700258,139535,2695714286,498047,1332000000,226107,7262571429,2026 -150000,200000.0,268884,41562569389,40740,3937905346,21389,278710590,29537,1965900086,54999,1215714286,148701,687142857,93702,3647142857,2026 -200000,300000.0,183330,38805868358,34629,4736275197,14259,188261116,20370,1705228252,41759,1110000000,113054,774885714,76388,3911428571,2026 -300000,500000.0,94721,30747819191,21389,5221770377,8148,100966855,10185,1216468562,23426,808714286,69258,711457143,49907,3626000000,2026 -500000,1000000.0,44814,24492228390,15278,7196117441,3056,45224737,4074,695124892,13241,501085714,38703,690314286,32592,4038285714,2026 -1000000,inf,21389,37639571768,11204,21038124452,2037,21034762,2037,401869078,7130,464085714,21389,1310857143,19352,11734285714,2026 -12570,inf,27767379,1149544329897,3737897,115957826468,8124578,89629118836,9344742,138764306632,2394495,31035600000,19935110,19391171429,4167704,75871142857,2026 -12570,15000.0,1395755,18293814433,455673,5353812445,1169971,12658528698,1098130,4825839793,103655,730243697,1282863,298636975,118023,190735294,2027 -15000,20000.0,3181500,52608247423,668115,8937861525,2103895,23815198060,2196261,15087683032,238099,1841957983,2914664,1493184874,381780,931878151,2027 -20000,30000.0,7666388,186185567010,986265,17475021911,2380993,26711641067,2729932,33503531438,442331,4163478992,5388023,3411436975,865163,3629420168,2027 -30000,40000.0,5511178,184020618557,610643,14024539877,1190497,13194907033,1488121,27623772610,373570,4141680672,3787011,2757487395,684536,5122605042,2027 -40000,50000.0,3592016,150463917526,368438,9717002629,582933,6457995150,799480,18748664944,333544,4119882353,2432308,2038142857,643484,9155294118,2027 -50000,70000.0,3520175,188350515464,319176,9149342682,452594,4999046079,659905,19303359173,407437,5885546218,2288627,2550403361,659905,13623949580,2027 -70000,100000.0,1683116,127731958763,146759,5965994741,180627,2027510105,265809,10505908699,231942,4108983193,1098130,1765663866,344833,10288806723,2027 -100000,150000.0,811796,89087628866,87235,5543032428,75945,908624899,111866,6023979328,140602,2779285714,501856,1373294118,227836,7487722689,2027 -150000,200000.0,270941,42432989691,41052,4062664330,21552,284280517,29762,2007993109,55420,1253403361,149838,708445378,94419,3760210084,2027 -200000,300000.0,184732,39618556701,34894,4886327783,14368,192023444,20526,1741739879,42078,1144411765,113918,798908403,76972,4032689076,2027 -300000,500000.0,95445,31391752577,21552,5387204207,8210,102984640,10263,1242515073,23605,833785714,69788,733513445,50288,3738411765,2027 -500000,1000000.0,45157,25005154639,15394,7424101665,3079,46128537,4105,710008613,13342,516620168,38999,711715126,32841,4163478992,2027 -1000000,inf,21552,38427835052,11289,21704645048,2053,21455133,2053,410473730,7184,478473109,21552,1351495798,19500,12098067227,2027 -12570,inf,27979749,1173618556701,3766485,119631551271,8186717,91420323363,9416212,141735469423,2412808,31997752941,20087577,19992328571,4199579,78223268908,2027 -12570,15000.0,1401052,18709278351,457402,5543514461,1174411,12906548100,1102298,4927002584,104049,754453782,1287731,308537815,118471,197058824,2028 -15000,20000.0,3193574,53803013481,670650,9254557406,2111879,24281810833,2204596,15403962102,239003,1903025210,2925725,1542689076,383229,962773109,2028 -20000,30000.0,7695482,190413957177,990008,18094215600,2390029,27235004042,2740292,34205857020,444010,4301512605,5408471,3524537815,868446,3749747899,2028 -30000,40000.0,5532093,188199841396,612960,14521472393,1195015,13453435732,1493768,28202842377,374987,4278991597,3801383,2848907563,687133,5292436975,2028 -40000,50000.0,3605648,153881046788,369836,10061305872,585145,6584527082,802514,19141688200,334810,4256470588,2441538,2105714286,645926,9458823529,2028 -50000,70000.0,3533535,192628072958,320388,9473531989,454312,5096992724,662409,19708010336,408983,6080672269,2297313,2634957983,662409,14075630252,2028 -70000,100000.0,1689503,130632831086,147316,6177388256,181313,2067235247,266818,10726141258,232822,4245210084,1102298,1824201681,346142,10629915966,2028 -100000,150000.0,814876,91110864393,87566,5739439089,76234,926427648,112290,6150258398,141135,2871428571,503760,1418823529,228701,7735966387,2028 -150000,200000.0,271969,43396669310,41207,4206617003,21634,289850445,29875,2050086133,55630,1294957983,150407,731932773,94777,3884873950,2028 -200000,300000.0,185433,40518318795,35026,5059465381,14423,195785772,20604,1778251507,42238,1182352941,114351,825394958,77264,4166386555,2028 -300000,500000.0,95807,32104678826,21634,5578089395,8241,105002425,10302,1268561585,23694,861428571,70053,757831933,50479,3862352941,2028 -500000,1000000.0,45328,25573037272,15453,7687160386,3091,47032336,4121,724892334,13392,533747899,39147,735310924,32966,4301512605,2028 -1000000,inf,21634,39300555115,11332,22473707274,2060,21875505,2060,419078381,7211,494336134,21634,1396302521,19574,12499159664,2028 -12570,inf,28085934,1200272164948,3780779,123870464505,8217786,93211527890,9451947,144706632214,2421965,33058588235,20163811,20655142857,4215517,80816638655,2028 -12570,15000.0,1408997,19178350515,459996,5754294479,1181071,13164106710,1108549,5043152455,104639,779226891,1295034,318668908,119143,203529412,2029 -15000,20000.0,3211685,55151942902,674454,9606441718,2123856,24766370251,2217098,15767097330,240358,1965512605,2942317,1593344538,385402,994386555,2029 -20000,30000.0,7739124,195187946075,995622,18782208589,2403583,27778496362,2755833,35012230835,446528,4442756303,5439143,3640268908,873371,3872873950,2029 -30000,40000.0,5563466,192918318795,616436,15073619632,1201792,13721907842,1502240,28867700258,377114,4419495798,3822941,2942453782,691030,5466218487,2029 -40000,50000.0,3626095,157739095956,371934,10443865031,588463,6715925627,807065,19592937123,336709,4396235294,2455385,2174857143,649589,9769411765,2029 -50000,70000.0,3553574,197457573354,322204,9833742331,456888,5198706548,666166,20172609819,411303,6280336134,2310341,2721478992,666166,14537815126,2029 -70000,100000.0,1699085,133908009516,148152,6412269939,182341,2108488278,268331,10979000861,234142,4384605042,1108549,1884100840,348105,10978957983,2029 -100000,150000.0,819498,93395162569,88062,5957668712,76666,944915117,112927,6295245478,141936,2965714286,506617,1465411765,229998,7989983193,2029 -150000,200000.0,273511,44484694687,41441,4366564417,21757,295634600,30045,2098415159,55945,1337478992,151260,755966387,95315,4012436975,2029 -200000,300000.0,186485,41534179223,35225,5251840491,14504,199692805,20721,1820172265,42477,1221176471,114999,852497479,77702,4303193277,2029 -300000,500000.0,96351,32909595559,21757,5790184049,8288,107097817,10360,1298466839,23829,889714286,70450,782715966,50765,3989176471,2029 -500000,1000000.0,45585,26214195083,15540,7979447853,3108,47970897,4144,741981051,13468,551273950,39369,759455462,33153,4442756303,2029 -1000000,inf,21757,40285884219,11396,23328220859,2072,22312045,2072,428957795,7252,510568067,21757,1442151261,19685,12909579832,2029 -12570,inf,28245211,1230364948454,3802220,128580368098,8264389,95071624899,9505550,148117967270,2435700,34144094118,20278162,21333371429,4239424,83470319328,2029 +12570,15000.0,1375891,17795688094,449188,4843188790,1153320,12201198978,1082502,4506163433,102180,699345737,1264606,286001092,116344,182664931,2025 +15000,20000.0,3136222,51175765762,658607,8085406650,2073953,22954798078,2165005,14088235102,234711,1764021335,2873184,1430005461,376347,892448664,2025 +20000,30000.0,7557283,181115878830,972229,15808329316,2347108,25746597844,2691081,31284169123,436036,3987314498,5311344,3267092769,852850,3475852690,2025 +30000,40000.0,5432746,179009880239,601952,12686939451,1173554,12718198935,1466943,25793901032,368253,3966438506,3733116,2640813005,674794,4905858152,2025 +40000,50000.0,3540896,146366902077,363195,8790236620,574637,6224679479,788102,17506703913,328797,3945562513,2397692,1951905265,634326,8767916697,2025 +50000,70000.0,3470078,183221877421,314634,8276717642,446153,4818439597,650513,18024653733,401639,5636517876,2256056,2442491080,650513,13047495084,2025 +70000,100000.0,1659163,124253916871,144671,5396983767,178056,1954259836,262026,9809969589,228641,3935124517,1082502,1690955363,339926,9853468288,2025 +100000,150000.0,800242,86661842021,85993,5014361783,74865,875797927,110274,5624935044,138601,2661688997,494714,1315187504,224594,7170903298,2025 +150000,200000.0,267085,41277572384,40467,3675184841,21245,274009977,29339,1874978348,54631,1200369548,147706,678469744,93075,3601108643,2025 +200000,300000.0,182103,38539774216,34397,4420290809,14164,185085985,20234,1626362435,41479,1095989587,112297,765105112,75876,3862058545,2025 +300000,500000.0,94087,30536979570,21245,4873395789,8093,99263992,10117,1160207597,23269,798506699,68795,702477135,49573,3580232651,2025 +500000,1000000.0,44514,24324283727,15175,6716022709,3035,44461996,4047,662975769,13152,494761014,38444,681601143,32374,3987314498,2025 +1000000,inf,21245,37381474991,11129,19634549151,2023,20679998,2023,383282867,7082,458228027,21245,1294311512,19222,11586175635,2025 +12570,inf,27581555,1141661836204,3712882,108221607319,8070207,88117472623,9282205,132346537984,2378470,30643868854,19801701,19146416186,4139813,74913497775,2025 +12570,15000.0,1385161,18400750621,452214,4991391904,1161091,12481831123,1089796,4690916055,102869,719766739,1273126,294352368,117128,187998775,2026 +15000,20000.0,3157352,52915768057,663044,8332822659,2087926,23482767029,2179591,14665852495,236292,1815531028,2892541,1471761839,378882,918508301,2026 +20000,30000.0,7608199,187273911644,978779,16292069209,2362921,26338779235,2709211,32566819510,438974,4103744689,5347128,3362492376,858596,3577348119,2026 +30000,40000.0,5469348,185096308020,606008,13075163824,1181461,13010722273,1476826,26851450524,370734,4082259115,3758267,2717925148,679340,5049109958,2026 +40000,50000.0,3564752,151343451852,365642,9059220649,578508,6367849438,793412,18224478468,331013,4060773541,2413846,2008901196,638600,9023941202,2026 +50000,70000.0,3493457,189451515267,316754,8529987828,449159,4929265511,654896,18763664221,404345,5801105058,2271256,2513812192,654896,13428483931,2026 +70000,100000.0,1670341,128478613802,145646,5562133182,179256,1999208544,263792,10212178171,230181,4050030754,1089796,1740331518,342216,10141191065,2026 +100000,150000.0,805634,89608389118,86573,5167802845,75369,895941607,111017,5855557283,139535,2739410722,498047,1353591180,226107,7380294769,2026 +150000,200000.0,268884,42681031026,40740,3787646663,21389,280312309,29537,1951852428,54999,1235420522,148701,698281164,93702,3706261565,2026 +200000,300000.0,183330,39850146315,34629,4555553110,14259,189343031,20370,1693043266,41759,1127992650,113054,787446298,76388,3974831244,2026 +300000,500000.0,94721,31575252545,21389,5022523247,8148,101547101,10185,1207776088,23426,821823217,69258,722989575,49907,3684775991,2026 +500000,1000000.0,44814,25151321854,15278,6921535135,3056,45484639,4074,690157764,13241,509208111,38703,701504001,32592,4103744689,2026 +1000000,inf,21389,38652464322,11204,20235372584,2037,21155646,2037,398997458,7130,471608356,21389,1332105606,19352,11924493731,2026 +12570,inf,27767379,1180478924442,3737897,111533222839,8124578,90144207486,9344742,137772743731,2394495,31538674501,19935110,19705494460,4167704,77100983340,2026 +12570,15000.0,1395755,18842365727,455673,5176573740,1169971,12731467935,1098130,4836334659,103655,743015304,1282863,303859990,118023,194071161,2027 +15000,20000.0,3181500,54185738125,668115,8641972377,2103895,23952422725,2196261,15120494566,238099,1874172930,2914664,1519299949,381780,948176246,2027 +20000,30000.0,7666388,191768455916,986265,16896508881,2380993,26865555219,2729932,33576392346,442331,4236296209,5388023,3471101344,865163,3692896957,2027 +30000,40000.0,5511178,189538590149,610643,13560255535,1190497,13270936915,1488121,27683846669,373570,4214116647,3787011,2805714505,684536,5212196906,2027 +40000,50000.0,3592016,154975670769,368438,9395319907,582933,6495206523,799480,18789438101,333544,4191937086,2432308,2073788982,643484,9315415747,2027 +50000,70000.0,3520175,193998321682,319176,8846452421,452594,5027850896,659905,19345338636,407437,5988481552,2288627,2595008672,659905,13862225814,2027 +70000,100000.0,1683116,131562080221,146759,5768489656,180627,2039192746,265809,10528756143,231942,4180847305,1098130,1796544465,344833,10468752935,2027 +100000,150000.0,811796,91758976290,87235,5359529569,75945,913860453,111866,6037079816,140602,2827894066,501856,1397312362,227836,7618679307,2027 +150000,200000.0,270941,43705369023,41052,3928169262,21552,285918560,29762,2012359939,55420,1275324775,149838,720835742,94419,3825974325,2027 +200000,300000.0,184732,40806543526,34894,4724565222,14368,193129895,20526,1745527682,42078,1164426968,113918,812880922,76972,4103218841,2027 +300000,500000.0,95445,32333053614,21552,5208860063,8210,103578044,10263,1245217200,23605,848368220,69788,746342238,50288,3803794763,2027 +500000,1000000.0,45157,25754949603,15394,7178325748,3079,46394332,4105,711552685,13342,525655603,38999,724162677,32841,4236296209,2027 +1000000,inf,21552,39580117355,11289,20986109757,2053,21578759,2053,411366396,7184,486841371,21552,1375132801,19500,12309656523,2027 +12570,inf,27979749,1208810232000,3766485,115671132138,8186717,91947093002,9416212,142043704838,2412808,32557378036,20087577,20341984648,4199579,79591355733,2027 +12570,15000.0,1401052,19238055016,457402,5364999173,1174411,12986098807,1102298,4981427321,104049,766048722,1287731,313279627,118471,200087353,2028 +15000,20000.0,3193574,55323637503,670650,8956537080,2111879,24431474027,2204596,15574117602,239003,1932272150,2925725,1566398134,383229,977569638,2028 +20000,30000.0,7695482,195795589515,990008,17511535761,2390029,27402869517,2740292,34583702323,444010,4367621074,5408471,3578705225,868446,3807376486,2028 +30000,40000.0,5532093,193518896614,612960,14053843987,1195015,13536357231,1493768,28514377081,374987,4344753948,3801383,2892691444,687133,5373774619,2028 +40000,50000.0,3605648,158230156643,369836,9737306191,585145,6625111425,802514,19353131432,334810,4321886822,2441538,2138076285,645926,9604192937,2028 +50000,70000.0,3533535,198072282416,320388,9168460125,454312,5128408512,662409,19925709286,408983,6174124031,2297313,2675453747,662409,14291953775,2028 +70000,100000.0,1689503,134324881179,147316,5978460617,181313,2079976843,266818,10844624536,232822,4310453259,1102298,1852237209,346142,10793283491,2028 +100000,150000.0,814876,93685912890,87566,5554614528,76234,932137770,112290,6218195484,141135,2915558570,503760,1440628941,228701,7854857795,2028 +150000,200000.0,271969,44623180866,41207,4071153218,21634,291636965,29875,2072731828,55630,1314859747,150407,743181596,94777,3944579242,2028 +200000,300000.0,185433,41663480094,35026,4896537706,14423,196992516,20604,1797894459,42238,1200524117,114351,838080169,77264,4230418317,2028 +300000,500000.0,95807,33012047069,21634,5398460706,8241,105649617,10302,1282574391,23694,874667571,70053,769478791,50479,3921712116,2028 +500000,1000000.0,45328,26295803010,15453,7439614237,3091,47322224,4121,732899652,13392,541950887,39147,746611665,32966,4367621074,2028 +1000000,inf,21634,40411298999,11332,21749996646,2060,22010337,2060,423707611,7211,501933417,21634,1417761814,19574,12691254952,2028 +12570,inf,28085934,1234195221815,3780779,119881519976,8217786,93786045792,9451947,146305093007,2421965,33566654314,20163811,20972584647,4215517,82058681794,2028 +12570,15000.0,1408997,19661286328,459996,5554383918,1181071,13245818324,1108549,5120904049,104639,788723905,1295034,322552761,119143,206009975,2029 +15000,20000.0,3211685,56540740564,674454,9272703298,2123856,24920098881,2217098,16010182775,240358,1989467760,2942317,1612763805,385402,1006505878,2029 +20000,30000.0,7739124,200103032447,995622,18129693871,2403583,27950921718,2755833,35552023516,446528,4496903457,5439143,3684635555,873371,3920075527,2029 +30000,40000.0,5563466,197776253000,616436,14549945400,1201792,13807081812,1502240,29312761111,377114,4473359460,3822941,2978315640,691030,5532839332,2029 +40000,50000.0,3626095,161711171571,371934,10081033598,588463,6757612399,807065,19895006537,336709,4449815463,2455385,2201363734,649589,9888478806,2029 +50000,70000.0,3553574,202429811895,322204,9492107237,456888,5230975711,666166,20483616198,411303,6356879232,2310341,2754647667,666166,14714998223,2029 +70000,100000.0,1699085,137279987377,148152,6189500583,182341,2121575986,268331,11148266977,234142,4438043464,1108549,1907063770,348105,11112766658,2029 +100000,150000.0,819498,95746974247,88062,5750692706,76666,950780349,112927,6392300917,141936,3001859637,506617,1483271821,229998,8087363023,2029 +150000,200000.0,273511,45604877162,41441,4214865136,21757,297469649,30045,2130766972,55945,1353779836,151260,765179908,95315,4061339509,2029 +200000,300000.0,186485,42580063881,35225,5069385738,14504,200932329,20721,1848234335,42477,1236059851,114999,862887496,77702,4355639474,2029 +300000,500000.0,96351,33738301982,21757,5589026646,8288,107762590,10360,1318485640,23829,900557891,70450,792255504,50765,4037795512,2029 +500000,1000000.0,45585,26874302614,15540,7702233001,3108,48268660,4144,753420366,13468,557992733,39369,768711507,33153,4496903457,2029 +1000000,inf,21757,41300335185,11396,22517772642,2072,22450540,2072,435571149,7252,516790738,21757,1459727824,19685,13066918422,2029 +12570,inf,28245211,1261347138253,3802220,124113343773,8264389,95661748947,9505550,150401540542,2435700,34560233426,20278162,21593376992,4239424,84487633796,2029 diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index 149ef19ff..d6c4229b7 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -1,81 +1,85 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 -afcs_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -alcohol_and_tobacco_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -benunit_rent,1.0,1.0,1.0,1.11,1.184,1.223,1.275,1.312,1.351,1.392,1.392,1.392,1.392,1.392,1.392 -bsp_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -capital_gains,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -capital_gains_before_response,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -carers_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -child_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -child_tax_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -childcare_expenses,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -clothing_and_footwear_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -communication_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -corporate_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -diesel_spending,1.0,1.138,1.57,1.539,1.589,1.54,1.507,1.46,1.415,1.355,1.284,1.284,1.284,1.284,1.284 -dividend_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -domestic_energy_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -education_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -employee_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -employer_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -pension_contributions_via_salary_sacrifice,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -employment_income,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -employment_income_before_lsr,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -esa_contrib_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -esa_income_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -food_and_non_alcoholic_beverages_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -free_school_fruit_veg,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -free_school_meals,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -free_school_milk,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -gross_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -health_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -household_furnishings_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +afcs_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +alcohol_and_tobacco_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +bsp_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +bus_fare_spending,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +bus_fare_spending_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +capital_gains,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +capital_gains_badr,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +capital_gains_before_response,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +capital_gains_carried_interest,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +capital_gains_residential_property,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +carers_allowance_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +child_benefit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +child_tax_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +childcare_expenses,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +clothing_and_footwear_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +communication_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +corporate_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +diesel_spending,1.0,1.13844,1.569717,1.539242,1.589317,1.539688,1.507377,1.459781,1.414994,1.35456,1.283802,1.283802,1.283802,1.283802,1.283802 +dividend_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +domestic_rates,1.0,1.0066,1.019887,1.08414,1.137371,1.191283,1.242865,1.296681,1.352828,1.411405,1.472519,1.536279,1.6028,1.672201,1.744607 +education_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +employee_pension_contributions,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +employer_pension_contributions,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +employment_income,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +employment_income_before_lsr,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +esa_contrib_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +esa_income_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +food_and_non_alcoholic_beverages_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +free_school_fruit_veg,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +free_school_meals,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +free_school_milk,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +gross_financial_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +health_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +household_furnishings_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +household_lifetime_isa_balance,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 household_weight,1.0,1.0,1.003,1.017,1.027,1.039,1.046,1.054,1.058,1.064,1.064,1.064,1.064,1.064,1.064 -housing_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -housing_service_charges,1.0,1.0,1.0,1.064,1.137,1.191,1.235,1.262,1.289,1.318,1.318,1.318,1.318,1.318,1.318 -housing_water_and_electricity_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -iidb_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -incapacity_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -income_support_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -jsa_contrib_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -jsa_income_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -lump_sum_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -main_residence_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -maintenance_expenses,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -maintenance_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -maternity_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -miscellaneous_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -miscellaneous_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -mortgage_capital_repayment,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -mortgage_interest_repayment,1.0,1.0,1.262,1.874,2.288,2.599,2.927,3.167,3.3,3.455,3.455,3.455,3.455,3.455,3.455 -net_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -non_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -other_investment_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -other_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -owned_land,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -pension_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -pension_credit_reported_capital,1.0,1.0,1.102,1.161,1.204,1.256,1.293,1.335,1.376,1.417,1.462,1.508,1.555,1.604,1.655 -pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -petrol_spending,1.0,1.104,1.635,1.796,1.531,1.483,1.452,1.406,1.363,1.305,1.237,1.237,1.237,1.237,1.237 -private_pension_income,1.0,1.003,1.053,1.106,1.161,1.216,1.261,1.288,1.315,1.346,1.346,1.346,1.346,1.346,1.346 -private_transfer_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -property_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -recreation_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -restaurants_and_hotels_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -savings,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -savings_interest_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -sda_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -self_employment_income,1.0,1.0,1.063,1.089,1.141,1.194,1.231,1.27,1.315,1.365,1.365,1.365,1.365,1.365,1.365 -state_pension,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -state_pension_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -statutory_maternity_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -statutory_paternity_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -statutory_sick_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -student_loan_repayments,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -sublet_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -transport_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -universal_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -water_and_sewerage_charges,1.0,1.0,1.0,1.092,1.14,1.21,1.283,1.349,1.4,1.46,1.46,1.46,1.46,1.46,1.46 -winter_fuel_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -working_tax_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +housing_benefit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +housing_water_and_electricity_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +iidb_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +incapacity_benefit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +income_support_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +jsa_contrib_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +jsa_income_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +lifetime_isa_balance,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +lump_sum_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +main_residence_value,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +maintenance_expenses,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +maintenance_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +maternity_allowance_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +miscellaneous_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +miscellaneous_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +mortgage_capital_repayment,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +mortgage_interest_repayment,1.0,0.9774,1.120296,1.705538,2.17115,2.377844,2.567358,2.834363,2.969279,3.108241,3.280127,3.461518,3.65294,3.854948,4.068126 +net_financial_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +non_residential_property_value,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +other_investment_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +other_residential_property_value,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +owned_land,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +pension_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +pension_credit_reported_capital,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +pension_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +personal_pension_contributions,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +petrol_spending,1.0,1.104066,1.63457,1.795584,1.530966,1.483159,1.452034,1.406186,1.363043,1.304828,1.236668,1.236668,1.236668,1.236668,1.236668 +private_pension_income,1.0,1.003,1.05315,1.105808,1.161098,1.202781,1.252095,1.29091,1.329638,1.366867,1.406507,1.438856,1.47195,1.506835,1.542698 +private_pension_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +private_transfer_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +property_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +recreation_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +restaurants_and_hotels_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +savings,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +savings_interest_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +sda_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +self_employment_income,1.0,1.0635,1.09498,1.08841,1.118123,1.125838,1.160289,1.203336,1.247137,1.291161,1.337772,1.386066,1.436103,1.487946,1.541661 +state_pension,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +state_pension_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +statutory_maternity_pay,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +statutory_paternity_pay,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +statutory_sick_pay,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +student_loan_repayments,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +sublet_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +transport_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +universal_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +winter_fuel_allowance_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +working_tax_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 956f740d7..6f947d478 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -1,81 +1,85 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 -afcs_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -alcohol_and_tobacco_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -benunit_rent,0,0.0,0.0,0.11,0.067,0.033,0.043,0.029,0.03,0.03,0.0,0.0,0.0,0.0,0.0 -bsp_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -capital_gains,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -capital_gains_before_response,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -carers_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -child_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -child_tax_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -childcare_expenses,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -clothing_and_footwear_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -communication_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -corporate_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -diesel_spending,0,0.138,0.379,-0.019,0.033,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 -dividend_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -domestic_energy_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -education_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -employee_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -employer_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -pension_contributions_via_salary_sacrifice,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -employment_income,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -employment_income_before_lsr,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -esa_contrib_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -esa_income_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -food_and_non_alcoholic_beverages_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -free_school_fruit_veg,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -free_school_meals,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -free_school_milk,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -gross_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -health_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -household_furnishings_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -household_weight,0,0.0,0.003,0.014,0.01,0.012,0.007,0.008,0.004,0.006,0.0,0.0,0.0,0.0,0.0 -housing_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -housing_service_charges,0,0.0,0.0,0.064,0.069,0.047,0.037,0.022,0.021,0.022,0.0,0.0,0.0,0.0,0.0 -housing_water_and_electricity_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -iidb_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -incapacity_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -income_support_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -jsa_contrib_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -jsa_income_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -lump_sum_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -main_residence_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -maintenance_expenses,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -maintenance_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -maternity_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -miscellaneous_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -miscellaneous_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -mortgage_capital_repayment,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -mortgage_interest_repayment,0,0.0,0.262,0.485,0.221,0.136,0.126,0.082,0.042,0.047,0.0,0.0,0.0,0.0,0.0 -net_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -non_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -other_investment_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -other_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -owned_land,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -pension_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -pension_credit_reported_capital,0,0.0,0.102,0.054,0.037,0.043,0.029,0.032,0.031,0.03,0.032,0.031,0.031,0.032,0.032 -pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -petrol_spending,0,0.104,0.481,0.099,-0.147,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 -private_pension_income,0,0.003,0.05,0.05,0.05,0.047,0.037,0.021,0.021,0.024,0.0,0.0,0.0,0.0,0.0 -private_transfer_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -property_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -recreation_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -restaurants_and_hotels_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -savings,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -savings_interest_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -sda_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -self_employment_income,0,0.0,0.063,0.024,0.048,0.046,0.031,0.032,0.035,0.038,0.0,0.0,0.0,0.0,0.0 -state_pension,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -state_pension_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -statutory_maternity_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -statutory_paternity_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -statutory_sick_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -student_loan_repayments,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -sublet_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -transport_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -universal_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -water_and_sewerage_charges,0,0.0,0.0,0.092,0.044,0.061,0.06,0.051,0.038,0.043,0.0,0.0,0.0,0.0,0.0 -winter_fuel_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -working_tax_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +afcs_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +alcohol_and_tobacco_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +bsp_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +bus_fare_spending,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +bus_fare_spending_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +capital_gains,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +capital_gains_badr,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +capital_gains_before_response,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +capital_gains_carried_interest,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +capital_gains_residential_property,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +carers_allowance_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +child_benefit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +child_tax_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +childcare_expenses,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +clothing_and_footwear_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +communication_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +corporate_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +diesel_spending,0,0.13844,0.378832,-0.019414,0.032532,-0.031227,-0.020985,-0.031575,-0.030681,-0.04271,-0.052237,0.0,0.0,0.0,0.0 +dividend_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +domestic_rates,0,0.0066,0.0132,0.063,0.0491,0.047401,0.0433,0.0433,0.043301,0.0433,0.0433,0.0433,0.0433,0.0433,0.0433 +education_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +employee_pension_contributions,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +employer_pension_contributions,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +employment_income,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +employment_income_before_lsr,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +esa_contrib_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +esa_income_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +food_and_non_alcoholic_beverages_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +free_school_fruit_veg,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +free_school_meals,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +free_school_milk,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +gross_financial_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +health_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +household_furnishings_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +household_lifetime_isa_balance,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +household_weight,0,0.0,0.003,0.013958,0.009833,0.011685,0.006737,0.007648,0.003795,0.005671,0.0,0.0,0.0,0.0,0.0 +housing_benefit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +housing_water_and_electricity_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +iidb_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +incapacity_benefit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +income_support_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +jsa_contrib_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +jsa_income_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +lifetime_isa_balance,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +lump_sum_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +main_residence_value,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +maintenance_expenses,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +maintenance_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +maternity_allowance_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +miscellaneous_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +miscellaneous_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +mortgage_capital_repayment,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +mortgage_interest_repayment,0,-0.0226,0.1462,0.522399,0.273,0.0952,0.0797,0.104,0.0476,0.0468,0.0553,0.0553,0.0553,0.0553,0.0553 +net_financial_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +non_residential_property_value,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +other_investment_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +other_residential_property_value,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +owned_land,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +pension_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +pension_credit_reported_capital,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +pension_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +personal_pension_contributions,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +petrol_spending,0,0.104066,0.4805,0.098505,-0.147372,-0.031227,-0.020986,-0.031575,-0.030681,-0.04271,-0.052237,0.0,0.0,0.0,0.0 +private_pension_income,0,0.003,0.05,0.05,0.05,0.0359,0.041,0.031,0.030001,0.027999,0.029001,0.023,0.023,0.0237,0.0238 +private_pension_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +private_transfer_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +property_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +recreation_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +restaurants_and_hotels_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +savings,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +savings_interest_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +sda_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +self_employment_income,0,0.0635,0.0296,-0.006,0.027299,0.0069,0.0306,0.0371,0.0364,0.0353,0.0361,0.0361,0.0361,0.0361,0.0361 +state_pension,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +state_pension_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +statutory_maternity_pay,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +statutory_paternity_pay,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +statutory_sick_pay,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +student_loan_repayments,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +sublet_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +transport_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +universal_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +winter_fuel_allowance_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +working_tax_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 diff --git a/policyengine_uk_data/targets/compute/income.py b/policyengine_uk_data/targets/compute/income.py index 19c49ac71..b522504ef 100644 --- a/policyengine_uk_data/targets/compute/income.py +++ b/policyengine_uk_data/targets/compute/income.py @@ -1,8 +1,6 @@ """Income and salary sacrifice compute functions.""" import numpy as np -import pandas as pd -from policyengine_uk_data.storage import STORAGE_FOLDER def compute_income_band(target, ctx) -> np.ndarray: @@ -96,27 +94,21 @@ def compute_ss_ni_relief(target, ctx) -> np.ndarray: def compute_ss_headcount(target, ctx) -> np.ndarray: """Compute salary sacrifice user headcounts. - The 2k cap is defined at 2023-24 FRS base-year prices. The dataset - is uprated to 2025 for calibration then downrated to 2023 for - saving, but PE does not uprate SS when loading. To keep the - above/below classification consistent, deflate SS to base-year - prices before applying the threshold. + The 2k cap is applied to the contributions the calibration-year + simulation holds. uprating_factors.csv follows policyengine-uk's + load-time uprating, so these are the amounts the model runs on in that + year (the survey amounts, as policyengine-uk does not uprate this + variable at load), the values test_salary_sacrifice_headcount checks. """ ss = ctx.sim.calculate("pension_contributions_via_salary_sacrifice") - uprating = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index( - "Variable" - ) - row = "pension_contributions_via_salary_sacrifice" - price_adj = uprating.loc[row, "2023"] / uprating.loc[row, str(ctx.time_period)] - ss_base = ss * price_adj name = target.name if "below_cap" in name: - mask = (ss_base > 0) & (ss_base <= 2000) + mask = (ss > 0) & (ss <= 2000) elif "above_cap" in name: - mask = ss_base > 2000 + mask = ss > 2000 else: - mask = ss_base > 0 + mask = ss > 0 return ctx.household_from_person(mask) diff --git a/policyengine_uk_data/tests/test_changelog_fragment_check.py b/policyengine_uk_data/tests/test_changelog_fragment_check.py new file mode 100644 index 000000000..82ac858a3 --- /dev/null +++ b/policyengine_uk_data/tests/test_changelog_fragment_check.py @@ -0,0 +1,334 @@ +"""Tests for .github/check_changelog_fragment.py, the PR changelog check.""" + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st + +REPO = Path(__file__).resolve().parents[2] + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, REPO / ".github" / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +checker = _load("check_changelog_fragment") +bump_version = _load("bump_version") +TYPES = checker.fragment_types(REPO / "pyproject.toml") +PATTERN = checker.fragment_pattern(TYPES) +BUMP = {"breaking": "major", "added": "minor", "removed": "minor"} +relaxed = settings(deadline=None, suppress_health_check=[HealthCheck.too_slow]) + +# The untyped fragments that sat on main until this check landed. Towncrier +# never compiled them, so several releases say "No significant changes." +HISTORICAL_UNTYPED = """218 281 316 317 341 368 378 402 409 414 428 429 431 433 436 +462 64 68 73 bus-fare-distribution-anchoring cgt-band-donors cgt-donor-sparsity +frs-year-semantics fuel-litre-calibration fuel-uprating-household-weight +hmrc-cgt-targets longwise-local-geography national-age-bands obr-efo-fallback +scottish-ct-water-charges spi-2022-23 spi-prior-diagnostics""".split() + +valid_names = st.from_regex(r"[A-Za-z0-9][A-Za-z0-9._-]{0,10}", fullmatch=True) +any_names = st.text( + st.characters(exclude_categories=("Cs",), exclude_characters="/\0"), + min_size=1, + max_size=12, +) +types = st.sampled_from(TYPES) +stems = st.text(st.sampled_from("abcxyz_-019"), min_size=1, max_size=8) +needing_paths = st.one_of( + st.sampled_from(["pyproject.toml", "uv.lock", "Makefile", ".gitattributes"]), + stems.map(lambda s: f"policyengine_uk_data/{s}.py"), + stems.map(lambda s: f"policyengine_uk_data/storage/{s}.md"), +) +exempt_paths = st.one_of( + st.sampled_from(["README.md", "CLAUDE.md", ".gitignore", ".claude/settings.json"]), + stems.map(lambda s: f"docs/{s}.md"), + stems.map(lambda s: f".github/workflows/{s}.yaml"), + stems.map(lambda s: f"tools/{s}/build.py"), + any_names.map(lambda n: ("D", f"changelog.d/{n}")), + st.sampled_from(["A", "M", "D"]).map(lambda s: (s, "changelog.d/.gitkeep")), +) + + +def _changes(paths): + return [p if isinstance(p, tuple) else ("M", p) for p in paths] + + +def _check(changes, contents=None, at_head=None): + contents = contents or {} + if at_head is None: + at_head = [p for s, p in changes if s != "D" and p.startswith("changelog.d/")] + return checker.check( + changes, TYPES, lambda path: contents.get(path, b"Entry."), at_head + ) + + +def _towncrier_parse(): + builder = pytest.importorskip("towncrier._builder") + if not hasattr(builder, "parse_newfragment_basename"): + pytest.skip("towncrier no longer exposes parse_newfragment_basename") + return builder.parse_newfragment_basename + + +def test_types_come_from_towncrier_config(): + assert TYPES == ["breaking", "added", "changed", "fixed", "removed"] + + +@pytest.mark.parametrize("stem", HISTORICAL_UNTYPED) +def test_historical_untyped_fragments_are_rejected_like_towncrier_ignores_them(stem): + path = f"changelog.d/{stem}.md" + problems = _check([("A", path), ("A", "changelog.d/x.fixed.md")]) + assert len(problems) == 1 and problems[0].startswith(f"{path}: name it") + assert _towncrier_parse()(f"{stem}.md", TYPES) == (None, None, None) + + +@pytest.mark.parametrize( + "name", + ["x.added.1.md", "x.added.md.bak", "x.added.mdx", "x.Added.md", "x.added"] + + ["sub/x.added.md", "x.feature.md", "added.md", "².added.md", "x .added.md"] + + [".x.added.md", "-x.added.md", "x\r.added.md", "é.added.md"], +) +def test_near_miss_names_are_rejected(name): + path = f"changelog.d/{name}" + problems = _check([("A", path), ("A", "changelog.d/y.fixed.md")]) + assert len(problems) == 1 and problems[0].startswith(f"{path}: name it") + + +@pytest.mark.parametrize("name", ["x.added.1.md", "x.added.md.bak"]) +def test_rejected_suffixes_would_split_towncrier_from_bump_version(tmp_path, name): + # Towncrier files these under Added, but bump_version.py sees no type and + # bumps the patch number instead of the minor one. + (tmp_path / name).write_text("Entry.") + assert _towncrier_parse()(name, TYPES)[1] == "added" + assert bump_version.infer_bump(tmp_path) == "patch" + + +@pytest.mark.parametrize( + "path, needed", + [ + ("policyengine_uk_data/datasets/frs.py", True), + ("policyengine_uk_data/tests/test_x.py", True), + ("policyengine_uk_data/storage/BRMA_DATA_SOURCES.md", True), + ("pyproject.toml", True), + ("uv.lock", True), + ("Makefile", True), + (".gitattributes", True), + ("README.md", False), + ("CHANGELOG.md", False), + ("docs/index.md", False), + (".github/workflows/push.yaml", False), + (".github/bump_version.py", False), + (".claude/settings.json", False), + ("tools/brma_households/build.py", False), + ("changelog.d/x.fixed.md", False), + (".gitignore", False), + ], +) +def test_which_paths_need_a_fragment(path, needed): + assert checker.needs_fragment(path) is needed + + +@relaxed +@given(st.lists(needing_paths, min_size=1), st.lists(exempt_paths), valid_names, types) +def test_a_typed_fragment_satisfies_any_change(needing, exempt, name, t): + fragment = ("A", f"changelog.d/{name}.{t}.md") + assert _check(_changes(needing + exempt) + [fragment]) == [] + + +@relaxed +@given(st.lists(needing_paths, min_size=1), st.lists(exempt_paths), valid_names, types) +def test_a_change_needing_a_fragment_fails_without_an_added_one(needing, exempt, n, t): + # Editing a fragment already on main does not count as adding one. + edited = ("M", f"changelog.d/{n}.{t}.md") + problems = _check(_changes(needing + exempt) + [edited]) + assert len(problems) == 1 and "adds no changelog fragment" in problems[0] + + +@relaxed +@given(st.lists(exempt_paths)) +def test_changes_only_to_exempt_paths_need_no_fragment(exempt): + assert _check(_changes(exempt)) == [] + + +@relaxed +@given(st.lists(st.one_of(needing_paths, exempt_paths)), any_names) +@example(others=[], name=".gitkeep") +def test_a_badly_named_fragment_is_always_rejected(others, name): + path = f"changelog.d/{name}" + if PATTERN.fullmatch(path) or path == checker.GITKEEP: + return + problems = _check(_changes(others) + [("A", path)]) + assert any(p.startswith(f"{path}: name it") for p in problems) + + +@relaxed +@given(st.lists(needing_paths, min_size=1), valid_names, types) +def test_an_empty_fragment_is_rejected_and_does_not_count(needing, name, t): + path = f"changelog.d/{name}.{t}.md" + problems = _check(_changes(needing) + [("A", path)], {path: b" \n"}) + assert problems[0] == f"{path} is empty. Describe the change in it." + assert "adds no changelog fragment" in problems[1] + + +def test_a_fragment_that_is_not_utf8_is_rejected(): + path = "changelog.d/x.fixed.md" + assert _check([("A", path)], {path: b"\xff"}) == [f"{path} is not UTF-8 text."] + + +@relaxed +@given(st.lists(st.one_of(needing_paths, exempt_paths)), valid_names, types, st.data()) +def test_problems_do_not_depend_on_change_order(paths, name, t, data): + changes = _changes(paths) + [("A", f"changelog.d/{name}.{t}.md")] + shuffled = data.draw(st.permutations(changes)) + assert sorted(_check(changes)) == sorted(_check(shuffled)) + + +@relaxed +@given(st.one_of(valid_names, any_names), types) +def test_towncrier_reads_every_accepted_name_as_the_same_entry(name, t): + basename = f"{name}.{t}.md" + if match := PATTERN.fullmatch(f"changelog.d/{basename}"): + issue, category, counter = _towncrier_parse()(basename, TYPES) + assert (issue, category, counter) == (*checker.towncrier_entry(match), 0) + + +@relaxed +@given(valid_names, valid_names, types) +def test_a_collision_is_reported_exactly_when_towncrier_would_collide(a, b, t): + paths = [f"changelog.d/{a}.{t}.md", f"changelog.d/{b}.{t}.md"] + if paths[0] == paths[1]: + return + parse = _towncrier_parse() + collide = parse(Path(paths[0]).name, TYPES) == parse(Path(paths[1]).name, TYPES) + problems = _check([("A", path) for path in paths]) + assert bool(problems) is collide + assert all("stops versioning" in p for p in problems) + + +def test_a_collision_with_a_fragment_already_on_main_is_reported(): + problems = _check( + [("A", "changelog.d/01.fixed.md")], + at_head=["changelog.d/1.fixed.md", "changelog.d/01.fixed.md"], + ) + assert problems == [ + "changelog.d/01.fixed.md and changelog.d/1.fixed.md: towncrier files " + "each as 1.fixed and stops versioning with an error. Rename all but one." + ] + + +@pytest.mark.parametrize( + "names, collide", + [ + (["1.fixed.md", "01.fixed.md"], True), + (["0.added.md", "000.added.md"], True), + (["1.fixed.md", "1.0.fixed.md"], False), + (["foo.fixed.md", "foo.added.md"], False), + ], +) +def test_towncrier_build_fails_exactly_when_the_check_reports_a_collision( + tmp_path, names, collide +): + pytest.importorskip("towncrier") + (tmp_path / "pyproject.toml").write_text((REPO / "pyproject.toml").read_text()) + (tmp_path / "changelog.d").mkdir() + for name in names: + (tmp_path / "changelog.d" / name).write_text(f"Entry {name}.") + build = subprocess.run( + [sys.executable, "-m", "towncrier", "build", "--draft"] + + ["--name", "x", "--version", "0.0.0", "--dir", str(tmp_path)], + capture_output=True, + text=True, + ) + assert (build.returncode != 0) is collide + assert bool(_check([("A", f"changelog.d/{name}") for name in names])) is collide + + +def test_changed_paths_keeps_raw_path_bytes(monkeypatch): + output = b"M\0docs/a\rb.md\0A\0changelog.d/\xff.fixed.md\0D\0changelog.d/x.md\0" + monkeypatch.setattr( + checker.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 0, stdout=output), + ) + assert checker.changed_paths("a", "b") == [ + ("M", "docs/a\rb.md"), + ("A", os.fsdecode(b"changelog.d/\xff.fixed.md")), + ("D", "changelog.d/x.md"), + ] + + +def _git(repo, *args): + identity = ["-c", "user.name=t", "-c", "user.email=t@t"] + return subprocess.run( + ["git", *identity, "-c", "commit.gpgsign=false", *args], + cwd=repo, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +@pytest.mark.parametrize( + "edits, expected", + [ + ({"policyengine_uk_data/x.py": "y = 2", "changelog.d/b.fixed.md": "B."}, 0), + ({"policyengine_uk_data/x.py": "y = 2"}, 1), + ({"docs/a b.md": "Docs.", "docs/c\rd.md": "CR.", ".github/w.yaml": "x"}, 0), + ({"policyengine_uk_data/x.py": "y = 2", "changelog.d/b.md": "B."}, 1), + ({"changelog.d/1.fixed.md": "A.", "changelog.d/01.fixed.md": "B."}, 1), + # Deleting every file removes changelog.d itself, as in a real checkout. + ({"changelog.d/old.md": None, "changelog.d/.gitkeep": None}, 0), + # Same content, so git would report a rename unless told not to. + ( + { + "changelog.d/old.md": None, + "changelog.d/old.fixed.md": "An untyped fragment already on main.", + "policyengine_uk_data/x.py": "y = 2", + }, + 0, + ), + ], +) +def test_main_reads_the_pull_requests_own_diff(tmp_path, monkeypatch, edits, expected): + files = { + "pyproject.toml": (REPO / "pyproject.toml").read_text(), + "changelog.d/.gitkeep": "", + "changelog.d/old.md": "An untyped fragment already on main.", + "policyengine_uk_data/x.py": "y = 1", + } + for path, text in files.items(): + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text(text) + _git(tmp_path, "init", "-q") + _git(tmp_path, "add", ".") + _git(tmp_path, "commit", "-qm", "base") + base = _git(tmp_path, "rev-parse", "HEAD") + for path, text in edits.items(): + target = tmp_path / path + if text is None: + target.unlink() + if not any(target.parent.iterdir()): + target.parent.rmdir() + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + _git(tmp_path, "add", "-A") + _git(tmp_path, "commit", "-qm", "change") + monkeypatch.chdir(tmp_path) + assert checker.main([base, "HEAD"]) == expected + + +@settings(relaxed, max_examples=50) +@given(valid_names, types) +def test_accepted_names_give_bump_version_the_same_type(tmp_path_factory, name, t): + directory = tmp_path_factory.mktemp("changelog.d") + (directory / f"{name}.{t}.md").write_text("Entry.") + assert bump_version.infer_bump(directory) == BUMP.get(t, "patch") diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index 686604e06..6370578cc 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -143,3 +143,12 @@ def test_projection_keeps_top_open_ended_band(projections): ] assert len(top_band) == 1 assert top_band.iloc[0]["dividend_income_amount"] > 0 + + +def test_projection_is_the_spi_table_uprated_with_the_committed_table( + projections, base_targets +): + """Regenerate incomes_projection.csv whenever uprating_factors.csv changes.""" + from policyengine_uk_data.utils.incomes_projection import project_income_table + + pd.testing.assert_frame_equal(project_income_table(base_targets), projections) diff --git a/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py b/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py index 9ed48abfe..41d5c784a 100644 --- a/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py +++ b/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py @@ -1,3 +1,7 @@ +import numpy as np +import pytest + + def test_pension_contributions_via_salary_sacrifice(baseline): """Test that pension_contributions_via_salary_sacrifice loads and has reasonable values.""" values = baseline.calculate( @@ -20,3 +24,50 @@ def test_pension_contributions_via_salary_sacrifice(baseline): assert total < total_employment, ( f"Salary sacrifice contributions ({total / 1e9:.1f}B) cannot exceed total employment income ({total_employment / 1e9:.1f}B)" ) + + +def test_salary_sacrifice_needs_pay(baseline): + """A salary sacrifice comes out of pay: none without pay, none above it.""" + pay = baseline.calculate("employment_income_before_lsr", period=2025).values + ss = baseline.calculate( + "pension_contributions_via_salary_sacrifice", period=2025 + ).values + weight = baseline.calculate("person_weight", period=2025).values + + without_pay = (ss > 0) & (pay <= 0) + assert not without_pay.any(), ( + f"{without_pay.sum()} people ({weight[without_pay].sum() / 1e3:.0f}k " + "weighted) have salary sacrifice but no pay" + ) + # Both columns uprate by the same index, so only rounding can separate them. + above_pay = ss > pay * (1 + 1e-9) + 1e-6 + assert not above_pay.any(), ( + f"{above_pay.sum()} people sacrifice more than their pay" + ) + + +def test_salary_sacrifice_cap_gives_nobody_pay_they_do_not_have(baseline): + """From 2029 the excess over the cap returns to pay, so it needs pay.""" + pay = baseline.calculate("employment_income_before_lsr", period=2029).values + employment_income = baseline.calculate("employment_income", period=2029).values + phantom = (pay <= 0) & (employment_income > 0) + assert not phantom.any(), ( + f"{phantom.sum()} people without pay have employment income in 2029" + ) + + +def test_model_pay_limit_never_binds_on_the_dataset(baseline): + """policyengine-uk limits salary sacrifice to pay as the imputation does.""" + limited = "pension_contributions_via_salary_sacrifice_from_pay" + if limited not in baseline.tax_benefit_system.variables: + pytest.skip(f"policyengine-uk has no {limited} variable") + for period in (2025, 2029): + ss = baseline.calculate( + "pension_contributions_via_salary_sacrifice", period=period + ).values + from_pay = baseline.calculate(limited, period=period).values + differing = ~np.isclose(from_pay, ss, rtol=1e-6, atol=1e-2) + assert not differing.any(), ( + f"policyengine-uk limits the salary sacrifice of {differing.sum()} " + f"people in {period}" + ) diff --git a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py index d19185811..dcdbfe166 100644 --- a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py +++ b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py @@ -9,6 +9,9 @@ applies. 3. Without either column every benefit unit gets -1. 4. The output is always -1 or a non-negative number, one per benefit unit. + +Uprating is checked against the engine for every row, including this +column, in test_uprating_factors_table.py. """ import numpy as np @@ -70,34 +73,6 @@ def test_output_is_sentinel_or_non_negative_for_random_inputs(): assert np.all(result[~keep] == -1) -def test_uprating_rows_match_the_model(): - """The build uprates the column with these rows (calibration materialises - the calibration year from them), and policyengine-uk projects the saved - dataset with the variable's own uprating index at runtime. The rows must - equal what ``create_policyengine_uprating_factors_table`` derives from the - locked policyengine-uk, or a unit near the 10,000 pound deemed-income - disregard can be assessed differently in calibration and at runtime.""" - from policyengine_uk.system import system - - from policyengine_uk_data.storage import STORAGE_FOLDER - from policyengine_uk_data.utils.uprating import END_YEAR, START_YEAR - - variable = system.variables["pension_credit_reported_capital"] - index = system.parameters.get_child(variable.uprating) - years = range(START_YEAR, END_YEAR + 1) - expected = {y: round(index(y) / index(START_YEAR), 3) for y in years} - factors = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable") - growth = pd.read_csv(STORAGE_FOLDER / "uprating_growth_factors.csv").set_index( - "Variable" - ) - for y in years: - assert factors.loc["pension_credit_reported_capital", str(y)] == expected[y] - expected_growth = ( - 0 if y == START_YEAR else round(expected[y] / expected[y - 1] - 1, 3) - ) - assert growth.loc["pension_credit_reported_capital", str(y)] == expected_growth - - def test_spi_copy_records_no_capital(): """SPI-synthetic copies carry SPI-imputed incomes, so the FRS donor's capital is cleared to -1 (household proxy) on them.""" diff --git a/policyengine_uk_data/tests/test_road_fuel_volume_uprating.py b/policyengine_uk_data/tests/test_road_fuel_volume_uprating.py index e44a1ade9..77cf0f224 100644 --- a/policyengine_uk_data/tests/test_road_fuel_volume_uprating.py +++ b/policyengine_uk_data/tests/test_road_fuel_volume_uprating.py @@ -32,6 +32,7 @@ END_YEAR, HOUSEHOLD_WEIGHT_UPRATING_INDEX, START_YEAR, + UPRATING_TABLE_DECIMALS, VOLUME_OVERRIDDEN_VARIABLES, _apply_household_weight_uprating_override, _apply_road_fuel_litre_proxy_override, @@ -109,7 +110,9 @@ def test__given_uprating_table__then_only_fuel_rows_are_overridden(): household_weight_index=df.loc["household_weight"], ) for year in range(START_YEAR, END_YEAR + 1): - assert out.loc[variable, year] == round(expected[year], 3) + assert out.loc[variable, year] == round( + expected[year], UPRATING_TABLE_DECIMALS + ) def test__given_generated_uprating_table__then_household_weight_row_is_restored(): @@ -144,7 +147,9 @@ def test__given_storage_csv__then_fuel_rows_reflect_litre_proxy_index(): ) assert variable in df.index for year in range(START_YEAR, END_YEAR + 1): - assert df.loc[variable, str(year)] == round(expected[year], 3) + assert df.loc[variable, str(year)] == round( + expected[year], UPRATING_TABLE_DECIMALS + ) def test__given_storage_csv__then_household_weight_row_is_unchanged(): diff --git a/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py b/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py index 17a18a42d..4f45fa600 100644 --- a/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py +++ b/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py @@ -6,8 +6,14 @@ """ import os +from types import SimpleNamespace + +import numpy as np +import pandas as pd from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.compute.income import compute_ss_headcount REDUCED_BUILD = os.environ.get("TESTING") == "1" # The 32-epoch reduced build stops well short of the OBR salary-sacrifice @@ -90,3 +96,53 @@ def test_salary_sacrifice_above_cap_users(baseline): f"Expected ~{TARGET / 1e6:.1f}mn above-cap SS users, " f"got {total_above_cap / 1e6:.1f}mn ({total_above_cap / TARGET * 100:.0f}% of target)" ) + + +def _headcount_masks(contributions): + """Run compute_ss_headcount for each target on one person per household.""" + contributions = np.asarray(contributions, dtype=float) + ctx = SimpleNamespace( + sim=SimpleNamespace(calculate=lambda variable: contributions), + household_from_person=lambda values: np.asarray(values), + time_period=PERIOD, + ) + return { + kind: compute_ss_headcount( + SimpleNamespace(name=f"obr/salary_sacrifice_users_{kind}"), ctx + ) + for kind in ("total", "below_cap", "above_cap") + } + + +def test_headcount_targets_split_users_at_the_cap_on_simulated_amounts(): + """Calibration classifies the amounts the calibration-year simulation holds. + + The same amounts the tests above classify, with no adjustment through the + uprating table (which has no salary sacrifice row, as policyengine-uk does + not uprate it at load). Below and above the cap partition the users. + """ + edges = [0.0, 0.01, 1_999.99, 2_000.0, 2_000.01, 1e6] + rng = np.random.default_rng(0) + for contributions in (edges, rng.uniform(0, 5_000, 1_000)): + masks = _headcount_masks(contributions) + contributions = np.asarray(contributions) + np.testing.assert_array_equal( + masks["below_cap"], (contributions > 0) & (contributions <= 2_000) + ) + np.testing.assert_array_equal(masks["above_cap"], contributions > 2_000) + np.testing.assert_array_equal( + masks["below_cap"].astype(int) + masks["above_cap"], masks["total"] + ) + + +def test_calibration_year_contributions_are_the_survey_amounts(): + """The cap split classifies survey-year contributions only while + policyengine-uk carries them unchanged at load, so the table has no row. + + If policyengine-uk starts uprating them (policyengine-uk#1863), the + regenerated table gains a row and the split moves to nominal + calibration-year amounts. Decide then which of the two the headcount + targets should classify (policyengine-uk-data#541). + """ + table = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv", index_col="Variable") + assert "pension_contributions_via_salary_sacrifice" not in table.index diff --git a/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py new file mode 100644 index 000000000..574b1fb8e --- /dev/null +++ b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py @@ -0,0 +1,248 @@ +"""Salary sacrifice imputation keeps every sacrifice within the person's pay. + +Invariants of ``limit_salary_sacrifice_to_pay``, for every input: + +- the result is between zero and the person's pay, so zero without pay; +- an amount already within those bounds is unchanged; +- applying it twice changes nothing; +- it is non-decreasing in the sacrifice and in the pay. + +``impute_salary_sacrifice`` applies it to every record, reported or +imputed. Stage 2 moves employee pension contributions to salary sacrifice +and takes the amount moved out of pay, so for every donor the pension total +and pay plus salary sacrifice are both unchanged, and the sacrifice never +exceeds the pay that remains. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.datasets.imputations import ( + salary_sacrifice as salary_sacrifice_module, +) +from policyengine_uk_data.datasets.imputations.salary_sacrifice import ( + limit_salary_sacrifice_to_pay, +) + +amounts = st.floats(min_value=-1e5, max_value=1e7, allow_nan=False) +pairs = st.lists(st.tuples(amounts, amounts), min_size=1, max_size=40) + + +def _split(rows): + ss, pay = np.array(rows, dtype=float).T + return ss, pay + + +@given(pairs) +def test_result_is_between_zero_and_pay(rows): + ss, pay = _split(rows) + limited = limit_salary_sacrifice_to_pay(ss, pay) + assert np.all(limited >= 0) + assert np.all(limited <= np.maximum(pay, 0)) + assert np.all(limited[pay <= 0] == 0) + + +@given(pairs) +def test_amounts_within_pay_are_unchanged(rows): + ss, pay = _split(rows) + within = (ss >= 0) & (ss <= pay) + limited = limit_salary_sacrifice_to_pay(ss, pay) + assert np.array_equal(limited[within], ss[within]) + + +@given(pairs) +def test_limit_is_idempotent(rows): + ss, pay = _split(rows) + once = limit_salary_sacrifice_to_pay(ss, pay) + assert np.array_equal(limit_salary_sacrifice_to_pay(once, pay), once) + + +@given(pairs, st.floats(min_value=0, max_value=1e6), st.floats(0, 1e6)) +def test_limit_is_monotone_in_sacrifice_and_pay(rows, more_ss, more_pay): + ss, pay = _split(rows) + limited = limit_salary_sacrifice_to_pay(ss, pay) + assert np.all(limit_salary_sacrifice_to_pay(ss + more_ss, pay) >= limited) + assert np.all(limit_salary_sacrifice_to_pay(ss, pay + more_pay) >= limited) + + +# The build imputes on the survey-year dataset (2024) while the simulation +# calculates a later default year, uprating pay but not salary sacrifice, so +# the wiring is checked in both years. +YEARS = pytest.mark.parametrize("year", [2024, 2025]) + + +class _StubModel: + """Stands in for the trained QRF: predicts a fixed amount per row.""" + + def __init__(self, predictions): + self.predictions = np.asarray(predictions, dtype=float) + + def predict(self, X): + assert len(X) == len(self.predictions) + return pd.DataFrame( + {"pension_contributions_via_salary_sacrifice": self.predictions} + ) + + +def _dataset(pay, asked, reported, employee_pension, year): + from policyengine_uk.data import UKSingleYearDataset + + n = len(pay) + person = pd.DataFrame( + { + "person_id": np.arange(n), + "person_benunit_id": np.arange(n), + "person_household_id": np.arange(n), + "age": 40, + "employment_income": np.asarray(pay, dtype=float), + "salary_sacrifice_asked": np.asarray(asked, dtype=int), + "pension_contributions_via_salary_sacrifice": np.asarray( + reported, dtype=float + ), + "employee_pension_contributions": np.asarray(employee_pension, dtype=float), + } + ) + benunit = pd.DataFrame({"benunit_id": np.arange(n)}) + household = pd.DataFrame( + { + "household_id": np.arange(n), + "household_weight": 1.0, + "region": "LONDON", + "tenure_type": "OWNED_OUTRIGHT", + "council_tax": 0.0, + "rent": 0.0, + } + ) + return UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=year + ) + + +def _impute(monkeypatch, predictions, year=2024, **columns): + monkeypatch.setattr( + salary_sacrifice_module, + "create_salary_sacrifice_model", + lambda: _StubModel(predictions), + ) + dataset = _dataset(year=year, **columns) + before = dataset.person.copy() + after = salary_sacrifice_module.impute_salary_sacrifice(dataset).person + return before, after + + +def _assert_invariants(before, after): + pay_before = before.employment_income.values + pay = after.employment_income.values + ss = after.pension_contributions_via_salary_sacrifice.values + assert np.all(ss >= 0) + assert np.all(ss <= np.maximum(pay, 0)) + assert np.all(ss[pay <= 0] == 0) + # Stage 2 moves employee pension to salary sacrifice out of pay: whoever + # lost employee pension gained exactly that much salary sacrifice and + # lost exactly that much pay. Nobody else's pay changes. + pension_before = before.employee_pension_contributions.values + pension_after = after.employee_pension_contributions.values + moved = pension_after != pension_before + assert np.all(pension_after <= pension_before) + assert np.allclose(pension_after[moved] + ss[moved], pension_before[moved]) + assert np.allclose(pay[moved] + ss[moved], pay_before[moved]) + assert np.array_equal(pay[~moved], pay_before[~moved]) + + +@YEARS +def test_nobody_without_pay_keeps_a_sacrifice(monkeypatch, year): + pay = [0, 0, -500, 30_000, 30_000, 8_000, 0] + before, after = _impute( + monkeypatch, + year=year, + # The model predicts a large sacrifice for everyone. + predictions=[25_000] * 7, + pay=pay, + asked=[0, 1, 0, 0, 1, 0, 1], + reported=[0, 6_000, 0, 0, 1_200, 0, 0], + employee_pension=[0] * 7, + ) + ss = after.pension_contributions_via_salary_sacrifice.values + # Unpaid: imputed (0, 2) and reported (1) amounts are all dropped. + assert ss[[0, 1, 2, 6]].tolist() == [0, 0, 0, 0] + # Paid and not asked: the prediction, limited to pay. + assert ss[3] == 25_000 + assert ss[5] == 8_000 + # Paid and asked: the reported amount is kept. + assert ss[4] == 1_200 + _assert_invariants(before, after) + + +@YEARS +def test_stage_two_takes_the_sacrifice_out_of_pay(monkeypatch, year): + n = 40 + before, after = _impute( + monkeypatch, + year=year, + predictions=[0] * n, + pay=[30_000] * n, + asked=[0] * n, + reported=[0] * n, + employee_pension=[1_500] * n, + ) + ss = after.pension_contributions_via_salary_sacrifice.values + moved = ss > 0 + assert moved.any() and not moved.all() + assert np.all(ss[moved] == 1_500) + assert np.all(after.employee_pension_contributions.values[moved] == 0) + assert np.all(after.employment_income.values[moved] == 28_500) + assert np.all(after.employment_income.values[~moved] == 30_000) + _assert_invariants(before, after) + + +@YEARS +def test_stage_two_moves_at_most_half_the_pay(monkeypatch, year): + n = 40 + # Every donor's employee pension exceeds half their pay, so each one + # moved sacrifices half and keeps the rest as employee pension. + before, after = _impute( + monkeypatch, + year=year, + predictions=[0] * n, + pay=[1_000] * n, + asked=[0] * n, + reported=[0] * n, + employee_pension=[1_500] * n, + ) + ss = after.pension_contributions_via_salary_sacrifice.values + moved = ss > 0 + assert moved.any() + assert np.all(ss[moved] == 500) + assert np.all(after.employee_pension_contributions.values[moved] == 1_000) + assert np.all(after.employment_income.values[moved] == 500) + _assert_invariants(before, after) + + +row = st.tuples( + st.sampled_from([-1_000.0, 0.0, 500.0, 15_000.0, 60_000.0]), # pay + st.integers(0, 1), # asked SALSAC + st.sampled_from([0.0, 800.0, 6_000.0, 70_000.0]), # reported amount + st.sampled_from([0.0, 300.0, 3_000.0, 90_000.0]), # employee pension + st.sampled_from([-100.0, 0.0, 1_500.0, 27_000.0]), # model prediction +) + + +@settings(max_examples=10, deadline=None) +@given(st.lists(row, min_size=1, max_size=12)) +def test_imputation_keeps_every_sacrifice_within_pay(rows): + pay, asked, reported, employee_pension, predictions = map(list, zip(*rows)) + with pytest.MonkeyPatch.context() as monkeypatch: + before, after = _impute( + monkeypatch, + predictions=predictions, + pay=pay, + asked=asked, + reported=reported, + employee_pension=employee_pension, + ) + _assert_invariants(before, after) diff --git a/policyengine_uk_data/tests/test_spi_allowance_deductions.py b/policyengine_uk_data/tests/test_spi_allowance_deductions.py index f635bb1f6..f40892905 100644 --- a/policyengine_uk_data/tests/test_spi_allowance_deductions.py +++ b/policyengine_uk_data/tests/test_spi_allowance_deductions.py @@ -10,6 +10,7 @@ def test_spi_overrides_allowance_deductions_not_policy_parameters(tmp_path): "DIVIDENDS": 0, "GIFTAID": 0, "GORCODE": 7, + "SCOT_TXP": 0, "INCBBS": 0, "INCPROP": 1_000, "PAY": 0, diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index eff98ef7a..298f9585b 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -30,6 +30,10 @@ allow_module_level=True, ) +from policyengine_core.errors import ParameterNotFoundError + +from policyengine_uk_data.datasets.spi import model_simulates_unknown_region + SPI_COLUMNS = [ "SEX", @@ -38,6 +42,7 @@ "DIVIDENDS", "GIFTAID", "GORCODE", + "SCOT_TXP", "INCBBS", "INCPROP", "PAY", @@ -357,7 +362,17 @@ def save(self, path): assert dataset_path.read_text() == "rebuilt h5" -def test_income_projection_loads_local_h5_dataset(monkeypatch): +@pytest.mark.parametrize( + "model_simulates, expected", + [ + (False, ["SOUTH_EAST", "LONDON", "SOUTH_EAST"]), + (True, ["UNKNOWN", "LONDON", "SOUTH_EAST"]), + ], +) +def test_income_projection_loads_local_h5_dataset( + monkeypatch, model_simulates, expected +): + """UNKNOWN becomes SOUTH_EAST only for a model that cannot simulate it.""" from policyengine_uk_data.utils import incomes_projection calls = {} @@ -375,16 +390,55 @@ def __init__(self, path): lambda: "/tmp/spi_2022_23.h5", ) monkeypatch.setattr(incomes_projection, "UKSingleYearDataset", FakeDataset) + monkeypatch.setattr( + incomes_projection, + "model_simulates_unknown_region", + lambda: model_simulates, + ) dataset = incomes_projection.load_spi_dataset() assert isinstance(dataset, FakeDataset) assert calls == {"path": "/tmp/spi_2022_23.h5"} - assert dataset.household["region"].tolist() == [ - "SOUTH_EAST", - "LONDON", - "SOUTH_EAST", - ] + assert dataset.household["region"].tolist() == expected + + +def test_income_projection_rebuilds_spi_dataset_without_scottish_flag( + tmp_path, monkeypatch +): + """A cached H5 from before create_spi read SCOT_TXP is rebuilt; a current + one is reused.""" + from policyengine_uk_data.datasets.spi import create_spi + from policyengine_uk_data.utils import incomes_projection + + tab_dir = tmp_path / "spi_2022_23" + tab_dir.mkdir() + tab = tab_dir / "put2223uk.tab" + _write_fake_spi(tab, gor_values=(11, 13), maind_values=(0, 0)) + dataset_path = tmp_path / "spi_2022_23.h5" + stale = create_spi(tab, 2022) + stale.person = stale.person.drop(columns="pays_scottish_income_tax") + stale.save(dataset_path) + + builds = [] + + def counting_create_spi(path, fiscal_year): + builds.append(fiscal_year) + return create_spi(path, fiscal_year) + + monkeypatch.setattr(incomes_projection, "STORAGE_FOLDER", tmp_path) + monkeypatch.setattr(incomes_projection, "SPI_RELEASE_NAME", "spi_2022_23") + monkeypatch.setattr(incomes_projection, "SPI_TAB_FILENAME", "put2223uk.tab") + monkeypatch.setattr(incomes_projection, "SPI_H5_FILENAME", "spi_2022_23.h5") + monkeypatch.setattr(incomes_projection, "SPI_FISCAL_YEAR", 2022) + monkeypatch.setattr(incomes_projection, "create_spi", counting_create_spi) + + assert not incomes_projection._has_scottish_taxpayer_flag(dataset_path) + assert incomes_projection.ensure_spi_dataset() == str(dataset_path) + assert builds == [2022] + assert incomes_projection._has_scottish_taxpayer_flag(dataset_path) + assert incomes_projection.ensure_spi_dataset() == str(dataset_path) + assert builds == [2022] def _write_income_model_cache(path, income_module, metadata): @@ -458,6 +512,263 @@ def test_income_model_cache_accepts_current_spi_release(tmp_path, monkeypatch): assert income_module.create_income_model().metadata == current_metadata +def _set_spi_columns(path, **columns): + df = pd.read_csv(path, sep="\t") + for col, values in columns.items(): + df[col] = list(values) + df.to_csv(path, sep="\t", index=False) + + +# HMRC's GORCODE codes 1-12 (SN 9422, Annex A) as policyengine-uk regions, +# written out here rather than read from REGION_MAP so a wrong mapping fails. +HMRC_REGIONS = { + 1: "NORTH_EAST", + 2: "NORTH_WEST", + 3: "YORKSHIRE", # Yorkshire and the Humber + 4: "EAST_MIDLANDS", + 5: "WEST_MIDLANDS", + 6: "EAST_OF_ENGLAND", + 7: "LONDON", + 8: "SOUTH_EAST", + 9: "SOUTH_WEST", + 10: "WALES", + 11: "SCOTLAND", + 12: "NORTHERN_IRELAND", +} + + +@pytest.mark.parametrize("not_scottish", [0, ".", ""]) +def test_create_spi_region_and_scottish_taxpayer_invariants(tmp_path, not_scottish): + """For every GORCODE (documented 1-14, composite -1, undocumented 99) and + SCOT_TXP value, the region follows GORCODE alone and is always a Region + member, and Scottish taxpayer status follows SCOT_TXP alone. HMRC + documents "not a Scottish taxpayer" as "."; the 2022-23 tape writes 0. + """ + from itertools import product + + from policyengine_uk.variables.household.demographic.geography import Region + + from policyengine_uk_data.datasets.spi import create_spi + + cases = list(product([-1, *range(1, 15), 99], (not_scottish, 1))) + gor = [g for g, _ in cases] + scot = [s for _, s in cases] + tab = tmp_path / "spi.tab" + _write_fake_spi(tab, gor_values=gor, maind_values=[0] * len(cases)) + _set_spi_columns(tab, SCOT_TXP=scot) + + ds = create_spi(tab, 2022) + + regions = ds.household["region"].tolist() + assert regions == [HMRC_REGIONS.get(g, "UNKNOWN") for g in gor] + assert set(regions) <= {region.name for region in Region} + assert ds.person["pays_scottish_income_tax"].tolist() == [s == 1 for s in scot] + # Address abroad, address unknown and composite records stay UNKNOWN even + # when they are Scottish taxpayers. + assert {r for r, g in zip(regions, gor) if g in (-1, 13, 14)} == {"UNKNOWN"} + + +def test_create_spi_scottish_taxpayer_status_survives_h5_round_trip(tmp_path): + from policyengine_uk.data import UKSingleYearDataset + + from policyengine_uk_data.datasets.spi import create_spi + + tab = tmp_path / "spi.tab" + _write_fake_spi(tab, gor_values=(13, 11, 7), maind_values=(0, 0, 0)) + _set_spi_columns(tab, SCOT_TXP=(1, 0, 1)) + ds = create_spi(tab, 2022) + ds.save(tmp_path / "spi.h5") + + loaded = UKSingleYearDataset(str(tmp_path / "spi.h5")) + + assert loaded.person["pays_scottish_income_tax"].tolist() == [True, False, True] + assert loaded.household["region"].tolist() == ["UNKNOWN", "SCOTLAND", "LONDON"] + + +def _missing(name): + return ParameterNotFoundError(name, "2023-01-01", "rent") + + +UNKNOWN_RENT_INDEX = _missing( + "gov.economic_assumptions.yoy_growth.ons.private_rental_prices.UNKNOWN" +) + + +@pytest.mark.parametrize( + "errors, expected", + [ + ({}, True), + ({"UNKNOWN": UNKNOWN_RENT_INDEX}, False), + # Any failure that relabelling to SOUTH_EAST cures. + ({"UNKNOWN": KeyError("UNKNOWN")}, False), + ], +) +def test_unknown_region_probe_reads_the_simulation(monkeypatch, errors, expected): + import policyengine_uk + + regions = [] + + def simulate(dataset): + (region,) = dataset.household["region"] + regions.append(region) + if region in errors: + raise errors[region] + + monkeypatch.setattr(policyengine_uk, "Microsimulation", simulate) + + assert model_simulates_unknown_region.__wrapped__() is expected + assert regions == (["UNKNOWN"] if expected else ["UNKNOWN", "SOUTH_EAST"]) + + +@pytest.mark.parametrize( + "error", + [ + _missing("gov.hmrc.unrelated.UNKNOWN"), + _missing("gov.hmrc.income_tax.rates.uk"), + KeyError("employment_income"), + ], +) +def test_unknown_region_probe_raises_failures_relabelling_would_not_cure( + monkeypatch, error +): + """When the SOUTH_EAST household fails too, its error is raised, not read + as a missing UNKNOWN index.""" + import policyengine_uk + + original = RuntimeError("the UNKNOWN household failed") + + def simulate(dataset): + (region,) = dataset.household["region"] + raise original if region == "UNKNOWN" else error + + monkeypatch.setattr(policyengine_uk, "Microsimulation", simulate) + + with pytest.raises(type(error)) as raised: + model_simulates_unknown_region.__wrapped__() + assert raised.value is error + assert raised.value.__context__ is original + + +def test_unknown_region_probe_agrees_with_policyengine_uk_release(): + """2.104.5 is the first policyengine-uk release with + PolicyEngine/policyengine-uk#1985. For a final release not installed from + a direct URL, whose imported economic_assumptions.py is the installed file + and matches its RECORD, the probe agrees with the release number.""" + import base64 + import hashlib + from importlib.metadata import PackageNotFoundError, distribution + from pathlib import Path + + from packaging.version import Version + from policyengine_uk.data import economic_assumptions + + try: + release = distribution("policyengine-uk") + except PackageNotFoundError: + pytest.skip("policyengine-uk is not installed as a distribution") + version = Version(release.version) + if version.is_prerelease or version.is_devrelease or version.local: + pytest.skip(f"policyengine-uk {version} is not a final release") + if release.read_text("direct_url.json") is not None: + pytest.skip("policyengine-uk was installed from a direct URL") + # #1985 changed this module, so it must be the installed file, unmodified. + path = "policyengine_uk/data/economic_assumptions.py" + installed = Path(release.locate_file(path)) + if not installed.exists() or not installed.samefile(economic_assumptions.__file__): + pytest.skip("the imported policyengine-uk is not the installed release") + record = {file.as_posix(): file.hash for file in release.files or []}.get(path) + if record is None: + pytest.skip("policyengine-uk's RECORD does not list the module") + digest = hashlib.new(record.mode, installed.read_bytes()).digest() + if base64.urlsafe_b64encode(digest).rstrip(b"=").decode() != record.value: + pytest.skip("the installed economic_assumptions.py has been modified") + + assert model_simulates_unknown_region() == (version >= Version("2.104.5")) + + +# GORCODE, SCOT_TXP: abroad, unknown, composite, London, Scotland, abroad and +# Scottish, Scotland but not Scottish. +SIMULATED_RECORDS = ((13, 0), (14, 0), (-1, 0), (7, 0), (11, 1), (13, 1), (11, 0)) +# The data year, an uprated year and 2030, the last year policyengine-uk +# carries dataset inputs to. +YEARS = [2022, 2026, 2030] + + +def _spi_income_tax(tmp_path, region_rule=False, **kwargs): + """Income tax on SIMULATED_RECORDS, each paid £60,000, in YEARS. With + region_rule, the Scottish taxpayer flag is dropped, so policyengine-uk + derives it from the region as it did before create_spi read SCOT_TXP.""" + from policyengine_uk import Microsimulation + + from policyengine_uk_data.datasets.spi import create_spi + + tab = tmp_path / "spi.tab" + gor, scot = zip(*SIMULATED_RECORDS) + _write_fake_spi(tab, gor_values=gor, maind_values=[0] * len(gor)) + _set_spi_columns( + tab, SCOT_TXP=scot, PAY=[60_000] * len(gor), AGERANGE=[3] * len(gor) + ) + dataset = create_spi(tab, 2022, **kwargs) + if region_rule: + dataset.person = dataset.person.drop(columns="pays_scottish_income_tax") + sim = Microsimulation(dataset=dataset) + return {year: sim.calculate("income_tax", year).values for year in YEARS} + + +def test_spi_income_tax_follows_scottish_taxpayer_flag(tmp_path): + """Equal pay, so income tax depends only on SCOT_TXP, in the data year + and after uprating. Uses the legacy SOUTH_EAST label so it runs on any + policyengine-uk release.""" + for year, tax in _spi_income_tax(tmp_path, unknown_region="SOUTH_EAST").items(): + abroad, unknown, composite, london, scotland, abroad_scot, scotland_ruk = tax + + assert london > 0, year + assert abroad == unknown == composite == scotland_ruk == london, year + assert abroad_scot == scotland != london, year + + +def test_spi_income_tax_changes_only_where_scottish_flag_and_region_disagree( + tmp_path, +): + """Records whose SCOT_TXP agrees with their region get exactly the income + tax they got when the region decided Scottish status. The two records + where they disagree change.""" + label = "UNKNOWN" if model_simulates_unknown_region() else "SOUTH_EAST" + flag = _spi_income_tax(tmp_path, unknown_region=label) + region = _spi_income_tax(tmp_path, region_rule=True, unknown_region=label) + agrees = np.array([(g == 11) == (s == 1) for g, s in SIMULATED_RECORDS]) + + for year in YEARS: + assert (flag[year][agrees] == region[year][agrees]).all(), year + assert (flag[year][~agrees] != region[year][~agrees]).all(), year + + +def test_create_spi_output_with_unknown_region_can_be_simulated(tmp_path): + """SPI records with an address abroad (13), an unknown address (14) or a + composite record (-1) keep region UNKNOWN and still run through + policyengine-uk, giving record for record the income tax of the legacy + SOUTH_EAST relabelling: the region label does not move income tax. + Models without PolicyEngine/policyengine-uk#1985 must fail on the missing + rent index, and on nothing else. + """ + if not model_simulates_unknown_region(): + with pytest.raises( + ParameterNotFoundError, match=r"private_rental_prices\.UNKNOWN'" + ): + _spi_income_tax(tmp_path) + pytest.xfail( + "policyengine-uk before 2.104.5 (PolicyEngine/policyengine-uk#1985) " + "has no rent index for Region.UNKNOWN" + ) + + tax = _spi_income_tax(tmp_path) + legacy = _spi_income_tax(tmp_path, unknown_region="SOUTH_EAST") + + for year in YEARS: + assert (tax[year] > 0).all(), year + assert (tax[year] == legacy[year]).all(), year + + def test_create_spi_marks_every_taxpayer_as_their_benefit_units_claimant(tmp_path): from policyengine_uk_data.datasets.spi import create_spi diff --git a/policyengine_uk_data/tests/test_uprating_factors_table.py b/policyengine_uk_data/tests/test_uprating_factors_table.py new file mode 100644 index 000000000..c10d44cce --- /dev/null +++ b/policyengine_uk_data/tests/test_uprating_factors_table.py @@ -0,0 +1,276 @@ +"""uprating_factors.csv follows policyengine-uk's load-time uprating. + +``uprate_dataset`` moves the build between the FRS base year and the +calibration year with this table, and policyengine-uk moves the saved file +to later years with ``extend_single_year_dataset``. Calibrated values are the +values the model runs on only where the two agree. + +Invariants, checked for every row and every year (or pair of years): +- the committed table is what the generator builds from the locked + policyengine-uk, and its rows are exactly the variables policyengine-uk + uprates at load; +- outside ``OVERRIDDEN_VARIABLES`` each row grows each year by one plus the + growth parameter policyengine-uk applies to it, and agrees with + policyengine-uk's own ``extend_single_year_dataset`` (differential), as + does ``uprate_dataset`` from the FRS base year; +- the fuel rows divide by the household-weight override, so fuel spending + times household weight stays on policyengine-uk's path; +- every row is a level index (1 in ``START_YEAR``, finite, above 0.5), so + ``uprate_dataset`` there and back between any two years is the identity; +- every variable a target projection uprates has a row. + +The engine's other load-time changes are outside the table: council tax by +country, rent by region and tenure, and student loan plan reassignment, which +zeroes the repayments of loans it writes off. + +test_income_projection checks that incomes_projection.csv is the committed +SPI table projected with this table. +""" + +import numpy as np +import pandas as pd +import pytest +from policyengine_uk.data import UKSingleYearDataset + +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.utils import uprating +from policyengine_uk_data.utils.uprating import ( + END_YEAR, + OVERRIDDEN_VARIABLES, + START_YEAR, + UPRATING_TABLE_DECIMALS, + VOLUME_OVERRIDDEN_VARIABLES, + build_uprating_factors_table, + policyengine_uk_load_time_index, + policyengine_uk_uprating_indices, + uprate_dataset, +) + +TABLE = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv", index_col="Variable") +TABLE.columns = TABLE.columns.astype(int) +YEARS = list(range(START_YEAR, END_YEAR + 1)) +ENGINE_ROWS = TABLE.index.difference(OVERRIDDEN_VARIABLES) +# Each stored level is within half a unit in the last stored place of the +# unrounded level, and every level exceeds 0.5, so a ratio of two stored +# levels is within 2 units in that place, relative, of the unrounded ratio. +ROUNDING = 2 * 10**-UPRATING_TABLE_DECIMALS +# The engine's own uprating moves council tax by country and rent by region +# and tenure. Neither is a single index, so neither is in the table and +# `uprate_dataset` leaves both unchanged. +NOT_SINGLE_INDICES = ("council_tax", "rent") +# Dataset columns that declare an `uprating` attribute policyengine-uk does not +# apply at load (policyengine-uk#1862): both paths carry them unchanged. +CARRIED_UNCHANGED = ( + "housing_service_charges", + "pension_contributions_via_salary_sacrifice", + "rail_usage", + "water_and_sewerage_charges", +) + + +def _unit_dataset(variables, year: int) -> UKSingleYearDataset: + """One person, benefit unit and household with every variable at 1.""" + from policyengine_uk.system import system + + tables = { + "person": { + "person_id": [1], + "person_benunit_id": [1], + "person_household_id": [1], + }, + "benunit": {"benunit_id": [1]}, + "household": { + "household_id": [1], + "region": ["LONDON"], + "tenure_type": ["RENT_PRIVATELY"], + **{variable: [1.0] for variable in NOT_SINGLE_INDICES}, + }, + } + for variable in variables: + tables[system.variables[variable].entity.key][variable] = [1.0] + return UKSingleYearDataset( + **{name: pd.DataFrame(columns) for name, columns in tables.items()}, + fiscal_year=year, + ) + + +def _value(dataset: UKSingleYearDataset, variable: str) -> float: + (column,) = [table[variable] for table in dataset.tables if variable in table] + return float(column.iloc[0]) + + +def _project_with_policyengine_uk(dataset, end_year: int) -> dict: + from policyengine_uk.data.economic_assumptions import extend_single_year_dataset + from policyengine_uk.system import system + + return extend_single_year_dataset( + dataset, system.parameters, end_year=end_year + ).datasets + + +def test_committed_table_is_what_the_generator_builds(): + """Fails when policyengine-uk's growth parameters or indices change. + + Regenerate with ``python -m policyengine_uk_data.utils.uprating``, then + ``python -m policyengine_uk_data.utils.incomes_projection``. + """ + pd.testing.assert_frame_equal( + TABLE, build_uprating_factors_table(), check_names=False, rtol=0, atol=1e-12 + ) + + +def test_rows_are_the_variables_policyengine_uk_uprates_at_load(): + engine = policyengine_uk_load_time_index() + assert set(TABLE.index) == set(engine.index) + assert set(OVERRIDDEN_VARIABLES) <= set(TABLE.index) + + +def test_each_row_grows_by_its_policyengine_uk_growth_parameter(): + from policyengine_uk.system import system + + for index_name, variables in policyengine_uk_uprating_indices().items(): + growth = system.parameters.get_child(index_name) + for variable in set(variables).difference(OVERRIDDEN_VARIABLES): + for year in YEARS[1:]: + factor = TABLE.loc[variable, year] / TABLE.loc[variable, year - 1] + assert factor == pytest.approx(1 + growth(str(year)), rel=ROUNDING), ( + variable, + year, + ) + + +def test_table_agrees_with_policyengine_uks_own_load_time_uprating(): + """Differential: policyengine-uk projects a dataset of ones from START_YEAR.""" + projected = _project_with_policyengine_uk( + _unit_dataset(TABLE.index, START_YEAR), END_YEAR + ) + for year in YEARS: + engine = projected[year] + for variable in ENGINE_ROWS: + assert TABLE.loc[variable, year] == pytest.approx( + _value(engine, variable), rel=ROUNDING + ), (variable, year) + engine_weight = _value(engine, "household_weight") + for variable in VOLUME_OVERRIDDEN_VARIABLES: + weighted = TABLE.loc[variable, year] * TABLE.loc["household_weight", year] + assert weighted == pytest.approx( + _value(engine, variable) * engine_weight, rel=ROUNDING + ), (variable, year) + + +@pytest.mark.parametrize("year", range(CURRENT_FRS_RELEASE.base_year, 2031)) +def test_uprate_dataset_from_the_base_year_matches_policyengine_uk(year): + """What calibration sees in ``year`` is what policyengine-uk runs on. + + policyengine-uk projects a dataset to 2030 at load, so later years follow + each variable's own ``uprating`` attribute and are not compared. + """ + base_year = CURRENT_FRS_RELEASE.base_year + variables = list(TABLE.index) + list(CARRIED_UNCHANGED) + dataset = _unit_dataset(variables, base_year) + engine = _project_with_policyengine_uk(dataset, year)[year] + calibration = uprate_dataset(dataset, year) + for variable in ENGINE_ROWS.union(CARRIED_UNCHANGED): + assert _value(calibration, variable) == pytest.approx( + _value(engine, variable), rel=ROUNDING + ), variable + for variable in CARRIED_UNCHANGED: + assert _value(calibration, variable) == 1.0, variable + for variable in VOLUME_OVERRIDDEN_VARIABLES: + assert _value(calibration, variable) * _value( + calibration, "household_weight" + ) == pytest.approx( + _value(engine, variable) * _value(engine, "household_weight"), + rel=ROUNDING, + ), variable + + +def test_student_loan_plan_reassignment_is_outside_the_table(): + """The table grows repayments by average earnings, as the engine does for a + loan it keeps. It does not zero a loan the engine writes off at load.""" + base_year = CURRENT_FRS_RELEASE.base_year + year = base_year + 1 + dataset = UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [1, 1], + # Ages are carried unchanged, so in `year` these borrowers + # started university in 2012 (Plan 2) and 1982 (written off). + "age": [year - 2012 + 18.0, year - 1982 + 18.0], + "student_loan_plan": ["PLAN_2", "PLAN_1"], + "highest_education": ["TERTIARY", "TERTIARY"], + "student_loan_repayments": [1.0, 1.0], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + { + "household_id": [1], + "region": ["LONDON"], + "tenure_type": ["RENT_PRIVATELY"], + **{variable: [1.0] for variable in NOT_SINGLE_INDICES}, + } + ), + fiscal_year=base_year, + ) + engine = _project_with_policyengine_uk(dataset, year)[year].person + calibration = uprate_dataset(dataset, year).person + growth = ( + TABLE.loc["student_loan_repayments", year] + / TABLE.loc["student_loan_repayments", base_year] + ) + assert list(engine.student_loan_plan) == ["PLAN_2", "NONE"] + assert engine.student_loan_repayments.tolist() == [ + pytest.approx(growth, rel=ROUNDING), + 0, + ] + assert calibration.student_loan_repayments.tolist() == [ + pytest.approx(growth, rel=1e-15), + pytest.approx(growth, rel=1e-15), + ] + + +def test_every_row_is_a_level_index(): + levels = TABLE.to_numpy() + assert (TABLE[START_YEAR] == 1).all() + assert np.isfinite(levels).all() + assert (levels > 0.5).all() + + +@pytest.mark.parametrize("start", YEARS) +def test_uprate_dataset_there_and_back_is_the_identity(start): + dataset = _unit_dataset(TABLE.index, start) + for end in YEARS: + there = uprate_dataset(dataset, end) + back = uprate_dataset(there, start) + for variable in TABLE.index: + assert _value(there, variable) == pytest.approx( + TABLE.loc[variable, end] / TABLE.loc[variable, start], rel=1e-15 + ) + assert _value(back, variable) == pytest.approx(1.0, rel=1e-12) + + +def test_every_variable_a_target_projection_uprates_has_a_row(): + """A missing row loses targets: the CGT projection stops at the first year + it cannot uprate, behind a log warning, and the SPI income projection + cannot be regenerated.""" + from policyengine_uk_data.utils.incomes_projection import ALL_INCOME_VARIABLES + + needed = set(ALL_INCOME_VARIABLES) | {"household_weight", "capital_gains"} + assert needed <= set(TABLE.index), needed - set(TABLE.index) + + +def test_a_variable_listed_under_two_indices_raises(monkeypatch): + monkeypatch.setattr( + uprating, + "policyengine_uk_uprating_indices", + lambda: { + "gov.economic_assumptions.yoy_growth.obr.consumer_price_index": ["x"], + "gov.economic_assumptions.yoy_growth.obr.average_earnings": ["x"], + }, + ) + with pytest.raises(ValueError, match="listed under two indices"): + policyengine_uk_load_time_index() diff --git a/policyengine_uk_data/utils/incomes_projection.py b/policyengine_uk_data/utils/incomes_projection.py index 144e3f833..7e0fe5e8b 100644 --- a/policyengine_uk_data/utils/incomes_projection.py +++ b/policyengine_uk_data/utils/incomes_projection.py @@ -7,6 +7,7 @@ SPI_RELEASE_NAME, SPI_TAB_FILENAME, create_spi, + model_simulates_unknown_region, ) from policyengine_uk_data.storage import STORAGE_FOLDER from policyengine_uk_data.targets.sources.hmrc_spi import ( @@ -31,6 +32,12 @@ def _read_spi_dataset_year(dataset_path) -> int: return int(store["time_period"].iloc[0]) +def _has_scottish_taxpayer_flag(dataset_path) -> bool: + # SPI H5s built before create_spi read SCOT_TXP lack this column. + with pd.HDFStore(dataset_path, mode="r") as store: + return "pays_scottish_income_tax" in store.select("person", stop=0) + + def ensure_spi_dataset() -> str: """Create the SPI H5 projection input from the current TAB release if needed. @@ -42,6 +49,7 @@ def ensure_spi_dataset() -> str: if ( dataset_path.exists() and _read_spi_dataset_year(dataset_path) == SPI_FISCAL_YEAR + and _has_scottish_taxpayer_flag(dataset_path) ): return str(dataset_path) @@ -64,9 +72,13 @@ def ensure_spi_dataset() -> str: def load_spi_dataset() -> UKSingleYearDataset: dataset = UKSingleYearDataset(ensure_spi_dataset()) - dataset.household["region"] = dataset.household["region"].replace( - {"UNKNOWN": "SOUTH_EAST"} - ) + # Older policyengine-uk releases cannot simulate an unknown region. The + # stand-in label does not move income tax, which follows the Scottish + # taxpayer flag. + if not model_simulates_unknown_region(): + dataset.household["region"] = dataset.household["region"].replace( + {"UNKNOWN": "SOUTH_EAST"} + ) return dataset diff --git a/policyengine_uk_data/utils/uprating.py b/policyengine_uk_data/utils/uprating.py index ff3939fb1..45826756e 100644 --- a/policyengine_uk_data/utils/uprating.py +++ b/policyengine_uk_data/utils/uprating.py @@ -1,9 +1,15 @@ +from pathlib import Path + from policyengine_uk_data.storage import STORAGE_FOLDER import pandas as pd +import yaml from policyengine_uk.data import UKSingleYearDataset START_YEAR = 2020 END_YEAR = 2034 +# Decimal places stored in uprating_factors.csv. At 3 places the rounding +# alone moved the factor between two years by up to 0.08%. +UPRATING_TABLE_DECIMALS = 6 # These variables are named as spending, but PolicyEngine UK derives fuel # litres through ``litres = spending / price`` and uprates household weights @@ -31,6 +37,10 @@ 2033: 1.064, 2034: 1.064, } +# Rows of the table that deliberately differ from policyengine-uk's own +# index. The fuel rows divide by the household-weight row, so together they +# keep weighted fuel spending on the engine's path. +OVERRIDDEN_VARIABLES = ("household_weight",) + VOLUME_OVERRIDDEN_VARIABLES class UpratingYearOutOfRangeError(ValueError): @@ -55,32 +65,69 @@ def _check_year_in_range(year: int, *, kind: str) -> None: ) -def create_policyengine_uprating_factors_table(): - from policyengine_uk.system import system +def policyengine_uk_uprating_indices() -> dict[str, list[str]]: + """The growth index policyengine-uk projects each dataset variable by. - df = pd.DataFrame() + Read from the ``uprating_indices.yaml`` that + ``policyengine_uk.data.economic_assumptions`` loads: a mapping from a + year-on-year growth parameter to the variables it uprates. + """ + from policyengine_uk.data import economic_assumptions + + path = Path(economic_assumptions.__file__).with_name("uprating_indices.yaml") + with open(path) as f: + return yaml.safe_load(f) + + +def policyengine_uk_load_time_index( + start_year: int = START_YEAR, end_year: int = END_YEAR +) -> pd.DataFrame: + """Level index (``start_year`` = 1) that policyengine-uk applies to data. + + policyengine-uk projects a single-year dataset to later years with + ``economic_assumptions.extend_single_year_dataset`` (to 2030 by default): + each year, every variable listed in ``uprating_indices.yaml`` is + multiplied by one plus the year-on-year growth parameter it is listed + under. Variables the file does not list are carried forward unchanged, + and a variable's own ``uprating`` attribute only applies to periods with + no stored value (for a dataset column, after the last year the dataset is + projected to), so neither appears here; past that year this index keeps + compounding the same growth parameters. Council tax and rent, which the + engine uprates by country and by region and tenure, are not single + indices and are not covered, nor is the engine's yearly reassignment of + student loan plans, which zeroes the repayments of loans it writes off. + """ + from policyengine_uk.system import system - variable_names = [] - years = [] - index_values = [] + rows = {} + for index_name, variables in policyengine_uk_uprating_indices().items(): + growth = system.parameters.get_child(index_name) + level = [1.0] + for year in range(start_year + 1, end_year + 1): + level.append(level[-1] * (1 + growth(str(year)))) + for variable in variables: + if variable in rows: + raise ValueError( + f"{variable} is listed under two indices in " + "policyengine-uk's uprating_indices.yaml." + ) + rows[variable] = level + df = pd.DataFrame.from_dict( + rows, orient="index", columns=range(start_year, end_year + 1) + ) + df.index.name = "Variable" + return df.sort_index() - for variable in system.variables.values(): - if variable.uprating is not None: - parameter = system.parameters.get_child(variable.uprating) - start_value = parameter(START_YEAR) - for year in range(START_YEAR, END_YEAR + 1): - variable_names.append(variable.name) - years.append(year) - growth = parameter(year) / start_value - index_values.append(round(growth, 3)) - df["Variable"] = variable_names - df["Year"] = years - df["Value"] = index_values +def build_uprating_factors_table() -> pd.DataFrame: + """The uprating factor table: policyengine-uk's load-time index, overridden. - # Convert to there is a column for each year - df = df.pivot(index="Variable", columns="Year", values="Value") - df = df.sort_values("Variable") + ``uprate_dataset`` moves the build between the FRS base year and the + calibration year with this table, and policyengine-uk moves the saved file + with its own load-time index, so the two must agree for calibrated values + to be the values the model runs on. Only ``OVERRIDDEN_VARIABLES`` differ. + """ + df = policyengine_uk_load_time_index().round(UPRATING_TABLE_DECIMALS) # Keep the population calibration row stable. Current PolicyEngine UK # population indices would inflate final calibrated population above the @@ -90,14 +137,20 @@ def create_policyengine_uprating_factors_table(): # Ensure petrol/diesel use sourced road-fuel clearances and model pump # prices. This keeps litres aligned after PolicyEngine divides by price. df = _apply_road_fuel_litre_proxy_override(df) + return df + +def create_policyengine_uprating_factors_table(): + df = build_uprating_factors_table() df.to_csv(STORAGE_FOLDER / "uprating_factors.csv") # Create a table with growth factors by year df_growth = df.copy() for year in range(END_YEAR, START_YEAR, -1): - df_growth[year] = round(df_growth[year] / df_growth[year - 1] - 1, 3) + df_growth[year] = round( + df_growth[year] / df_growth[year - 1] - 1, UPRATING_TABLE_DECIMALS + ) df_growth[START_YEAR] = 0 df_growth.to_csv(STORAGE_FOLDER / "uprating_growth_factors.csv") @@ -224,7 +277,7 @@ def _apply_road_fuel_litre_proxy_override(df: pd.DataFrame) -> pd.DataFrame: + ", ".join(str(year) for year in missing_years) ) for year in range(START_YEAR, END_YEAR + 1): - df.loc[variable, year] = round(index[year], 3) + df.loc[variable, year] = round(index[year], UPRATING_TABLE_DECIMALS) return df diff --git a/uv.lock b/uv.lock index 5ae821352..dce9c8285 100644 --- a/uv.lock +++ b/uv.lock @@ -1433,7 +1433,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.57.4" +version = "1.58.0" source = { editable = "." } dependencies = [ { name = "google-auth" },