diff --git a/.github/check_test_shards.py b/.github/check_test_shards.py new file mode 100644 index 0000000000..69f40fa83f --- /dev/null +++ b/.github/check_test_shards.py @@ -0,0 +1,195 @@ +"""Check that the shards of the Test job ran every test exactly once. + +CI splits ``pytest policyengine_uk/tests`` over several runners with +``--shard K/N`` (see policyengine_uk/tests/conftest.py). This script compares +what the shards ran with what an unsharded ``pytest --collect-only -q`` lists: + + python .github/check_test_shards.py \ + --collected collected.txt --junit junit-1.xml junit-2.xml junit-3.xml + +It fails if a collected test is missing from every shard, ran in more than one, +ran without being collected, or failed. So splitting the suite cannot drop a +test without the Test job going red. + +Standard library only: the job that runs it does not install the package. +""" + +import argparse +import re +import sys +import xml.etree.ElementTree as ElementTree +from collections import Counter +from pathlib import Path + +_COLLECTED_SUMMARY = re.compile(r"^(\d+) tests? collected\b(.*)$") +_NOTHING_COLLECTED = re.compile(r"^no tests collected\b") + + +class ShardCheckError(Exception): + """An input that cannot be checked, as opposed to a check that failed.""" + + +def junit_key(nodeid: str) -> tuple[str, str]: + """The ``(classname, name)`` pytest gives ``nodeid`` in JUnit XML. + + Follows _pytest.junitxml.mangle_test_address; a test compares the two. + pytest also rewrites characters that XML does not allow, such as control + characters. Its node ids do not contain them, because it escapes them in + parameter ids. If one ever did, the names would differ and the check + would fail, not pass. + """ + path, bracket, params = nodeid.partition("[") + names = path.split("::") + names[0] = re.sub(r"\.py$", "", names[0].replace("/", ".")) + names[-1] += bracket + params + return ".".join(names[:-1]), names[-1] + + +def read_collected(text: str) -> list[str]: + """Node ids from the output of ``pytest --collect-only -q``. + + The listing is one node id per line, a blank line, then any warnings and + ``N tests collected``. The ids found must number N, so a listing this + cannot parse is an error, never a shorter list. + """ + ids = [] + total = None + listing_ended = False + for line in text.splitlines(): + summary = _COLLECTED_SUMMARY.match(line) + if summary: + if "error" in summary.group(2): + raise ShardCheckError(f"collection reported errors: {line!r}") + total = int(summary.group(1)) + elif _NOTHING_COLLECTED.match(line): + total = 0 + elif not line.strip(): + listing_ended = listing_ended or bool(ids) + elif "::" in line and not listing_ended: + ids.append(line) + if total is None: + raise ShardCheckError("no 'N tests collected' line in the collection listing") + if len(ids) != total: + raise ShardCheckError( + f"the collection listing says {total} tests but lists {len(ids)}" + ) + if len(set(ids)) != len(ids): + raise ShardCheckError("the collection listing repeats a node id") + return ids + + +def read_junit(path: Path) -> list[tuple[tuple[str, str], str]]: + """``((classname, name), outcome)`` for each test case in a JUnit file. + + The outcome is ``failed`` for a failure or an error, else ``skipped`` for a + skip or an expected failure, else ``passed``. + """ + cases = [] + for case in ElementTree.parse(path).getroot().iter("testcase"): + children = {child.tag for child in case} + if children & {"failure", "error"}: + outcome = "failed" + elif "skipped" in children: + outcome = "skipped" + else: + outcome = "passed" + cases.append(((case.get("classname", ""), case.get("name", "")), outcome)) + return cases + + +def check(collected: list[str], shards: list[list[tuple[tuple[str, str], str]]]): + """Compare the collected node ids with what each shard ran. + + Returns ``(problems, summary)``: ``problems`` is empty when every collected + test ran in exactly one shard and none failed. + """ + expected = {} + for nodeid in collected: + key = junit_key(nodeid) + if key in expected: + raise ShardCheckError( + f"{nodeid!r} and {expected[key]!r} have the same JUnit name" + ) + expected[key] = nodeid + + runs = Counter() + outcomes = Counter() + failed = [] + for number, cases in enumerate(shards, start=1): + for key, outcome in cases: + runs[key] += 1 + outcomes[outcome] += 1 + if outcome == "failed": + failed.append((number, key)) + + def name(key): + return expected.get(key) or f"{key[0]}::{key[1]}" + + problems = [] + missing = sorted(expected[key] for key in expected if key not in runs) + if missing: + problems.append((f"{len(missing)} collected tests ran in no shard", missing)) + repeated = sorted(name(key) for key, count in runs.items() if count > 1) + if repeated: + problems.append((f"{len(repeated)} tests ran more than once", repeated)) + extra = sorted(name(key) for key in runs if key not in expected) + if extra: + problems.append((f"{len(extra)} tests ran but were not collected", extra)) + if failed: + problems.append( + ( + f"{len(failed)} tests failed", + [f"shard {number}: {name(key)}" for number, key in failed], + ) + ) + + sizes = "/".join(str(len(cases)) for cases in shards) + summary = ( + f"{len(collected)} tests collected; {len(shards)} shards ran " + f"{sizes} = {sum(len(cases) for cases in shards)} " + f"({outcomes['passed']} passed, {outcomes['skipped']} skipped or " + f"expected failures, {outcomes['failed']} failed)" + ) + return problems, summary + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--collected", + required=True, + type=Path, + help="output of an unsharded `pytest --collect-only -q`", + ) + parser.add_argument( + "--junit", + required=True, + nargs="+", + type=Path, + help="the JUnit XML file of each shard", + ) + args = parser.parse_args(argv) + + try: + collected = read_collected(args.collected.read_text(encoding="utf-8")) + shards = [read_junit(path) for path in args.junit] + problems, summary = check(collected, shards) + except (ShardCheckError, OSError, ElementTree.ParseError) as error: + print(f"Cannot check the shards: {error}", file=sys.stderr) + return 2 + + print(summary) + for title, names in problems: + print(f"\n{title}:") + for line in names[:50]: + print(f" {line}") + if len(names) > 50: + print(f" ... and {len(names) - 50} more") + if problems: + return 1 + print("Every collected test ran in exactly one shard.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/code_changes.yaml b/.github/workflows/code_changes.yaml index 5ba890a038..6d30dcf3b9 100644 --- a/.github/workflows/code_changes.yaml +++ b/.github/workflows/code_changes.yaml @@ -31,19 +31,20 @@ jobs: uses: astral-sh/setup-uv@v8.1.0 - name: Check formatting run: uvx ruff format --check . - Test: + # The tests run as four jobs: the YAML policy tests, and the Python tests + # split over three runners. As one job they took 59 minutes on 10 October + # 2026 (run 38075384588: 2.6 minutes of YAML tests, then 55.8 minutes of + # pytest on four workers), and runs were cancelled at a 60-minute limit + # (issues #2099 and #2257). + Test-YAML: + name: Test (YAML) # Linux arm64 keeps the CPU architecture of the macOS runners this job # used to run on. GitHub-hosted macOS runners are scarce, so PR runs # queued for hours; the parameter tree and every test result are # identical on both. runs-on: ubuntu-24.04-arm - # The whole job took about 18 minutes when this limit was set; by - # October 2026 it takes about an hour (57 to 60 minutes on 9 October, - # and every run on 10 October was cancelled at a 60-minute limit, one - # at 99% with every test passing; issue #2257). 120 minutes lets a - # slow run finish and still stops a hung one long before the 6-hour - # GitHub limit. The suite's runtime needs its own attention. - timeout-minutes: 120 + # About 4 minutes on 10 October (run 38085920460). + timeout-minutes: 30 permissions: contents: "read" id-token: "write" @@ -61,7 +62,70 @@ jobs: with: workload_identity_provider: "projects/322898545428/locations/global/workloadIdentityPools/policyengine-research-id-pool/providers/prod-github-provider" service_account: "policyengine-research@policyengine-research.iam.gserviceaccount.com" - + + - name: Install package + run: uv pip install -e .[dev] --system + - name: Install policyengine + run: uv pip install policyengine --system + - name: Install JB + run: uv pip install "jupyter-book>=2.0.0a0" --system + - name: UV sync + run: uv sync --extra dev + - name: Run YAML tests + run: make test-yaml + - name: List the Python tests + # Every Python test, collected without --shard. The Test job + # checks that the shards below ran each of these exactly once. + run: | + # Only export token if non-empty (defensive against missing secrets) + if [ -n "$HF_TOKEN_VALUE" ]; then + export HUGGING_FACE_TOKEN="$HF_TOKEN_VALUE" + fi + pytest policyengine_uk/tests --collect-only -q -p no:cacheprovider > collected.txt + tail -n 1 collected.txt + env: + HF_TOKEN_VALUE: ${{ secrets.HUGGING_FACE_TOKEN }} + - name: Upload the list of Python tests + uses: actions/upload-artifact@v7 + with: + name: test-collected + path: collected.txt + if-no-files-found: error + overwrite: true + retention-days: 7 + Test-Python: + # Shard K of N runs the tests whose node id hashes to K (--shard in + # policyengine_uk/tests/conftest.py), so each test is in exactly one + # shard. N is the number of jobs this matrix makes: to add a shard, add + # a number to the list. + name: Test (Python ${{ matrix.shard }}/${{ strategy.job-total }}) + runs-on: ubuntu-24.04-arm + # The shards took 15, 20 and 15 minutes on 10 October (run + # 38085920460). 60 minutes leaves room for a slow runner and still + # stops a hung shard. + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + permissions: + contents: "read" + id-token: "write" + steps: + - name: Checkout repo + uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + - uses: "google-github-actions/auth@v3" + with: + workload_identity_provider: "projects/322898545428/locations/global/workloadIdentityPools/policyengine-research-id-pool/providers/prod-github-provider" + service_account: "policyengine-research@policyengine-research.iam.gserviceaccount.com" + - name: Install package run: uv pip install -e .[dev] --system - name: Install policyengine @@ -78,12 +142,54 @@ jobs: fi # Four pytest-xdist workers, one per core; together they peak at # about 9 GB of the runner's 16 GB. - make test PYTEST_ARGS="-n 4 --dist worksteal" + make test-python PYTEST_ARGS="-n 4 --dist worksteal --shard ${{ matrix.shard }}/${{ strategy.job-total }} --junitxml=junit-${{ matrix.shard }}.xml" env: HF_TOKEN_VALUE: ${{ secrets.HUGGING_FACE_TOKEN }} + # Measure coverage with sys.monitoring instead of the C trace + # function: the same lines are reported, at less cost per line. + COVERAGE_CORE: sysmon + - name: Upload the test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: test-junit-${{ matrix.shard }} + path: junit-${{ matrix.shard }}.xml + if-no-files-found: error + overwrite: true + retention-days: 7 - name: Upload coverage to Codecov uses: codecov/codecov-action@v6 with: files: ./coverage.xml fail_ci_if_error: false verbose: true + Test: + # One result for the whole suite. It fails if any test job failed, or + # if the shards did not run every collected Python test exactly once. + needs: [Test-YAML, Test-Python] + if: ${{ !cancelled() }} + # The shards' runners, not ubuntu-latest: on 10 October this 5-second + # job waited 32 minutes for an ubuntu-latest runner after the shards + # had finished (run 38085920460). + runs-on: ubuntu-24.04-arm + timeout-minutes: 10 + steps: + - name: Check that every test job passed + run: | + echo "YAML: ${{ needs.Test-YAML.result }}; Python: ${{ needs.Test-Python.result }}" + test "${{ needs.Test-YAML.result }}" = success + test "${{ needs.Test-Python.result }}" = success + - name: Checkout repo + if: ${{ !cancelled() }} + uses: actions/checkout@v6 + - name: Download the test lists and results + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + pattern: test-* + merge-multiple: true + - name: Check that every Python test ran in exactly one shard + # Runs even when a test job failed, so its report names the + # failed tests. + if: ${{ !cancelled() }} + run: python3 .github/check_test_shards.py --collected collected.txt --junit junit-*.xml diff --git a/.github/workflows/pr_code_changes.yaml b/.github/workflows/pr_code_changes.yaml index 055741c182..2c8f588ba6 100644 --- a/.github/workflows/pr_code_changes.yaml +++ b/.github/workflows/pr_code_changes.yaml @@ -51,19 +51,83 @@ jobs: uses: astral-sh/setup-uv@v8.1.0 - name: Check formatting run: uvx ruff format --check . - Test: + # The tests run as four jobs: the YAML policy tests, and the Python tests + # split over three runners. As one job they took 59 minutes on 10 October + # 2026 (run 38075384588: 2.6 minutes of YAML tests, then 55.8 minutes of + # pytest on four workers), and runs were cancelled at a 60-minute limit + # (issues #2099 and #2257). + Test-YAML: + name: Test (YAML) # Linux arm64 keeps the CPU architecture of the macOS runners this job # used to run on. GitHub-hosted macOS runners are scarce, so PR runs # queued for hours; the parameter tree and every test result are # identical on both. runs-on: ubuntu-24.04-arm - # The whole job took about 18 minutes when this limit was set; by - # October 2026 it takes about an hour (57 to 60 minutes on 9 October, - # and every run on 10 October was cancelled at a 60-minute limit, one - # at 99% with every test passing; issue #2257). 120 minutes lets a - # slow run finish and still stops a hung one long before the 6-hour - # GitHub limit. The suite's runtime needs its own attention. - timeout-minutes: 120 + # About 4 minutes on 10 October (run 38085920460). + timeout-minutes: 30 + permissions: + contents: "read" + id-token: "write" + steps: + - name: Checkout repo + uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + - uses: "google-github-actions/auth@v3" + with: + workload_identity_provider: "projects/322898545428/locations/global/workloadIdentityPools/policyengine-research-id-pool/providers/prod-github-provider" + service_account: "policyengine-research@policyengine-research.iam.gserviceaccount.com" + + - name: Install package + run: uv pip install -e .[dev] --system + - name: Install policyengine + run: uv pip install policyengine --system + - name: UV sync + run: uv sync --extra dev + - name: Run YAML tests + run: make test-yaml + - name: List the Python tests + # Every Python test, collected without --shard. The Test job + # checks that the shards below ran each of these exactly once. + run: | + # Only export token if non-empty (avoids 'Bearer ' error for Dependabot PRs) + if [ -n "$HF_TOKEN_VALUE" ]; then + export HUGGING_FACE_TOKEN="$HF_TOKEN_VALUE" + fi + pytest policyengine_uk/tests --collect-only -q -p no:cacheprovider > collected.txt + tail -n 1 collected.txt + env: + HF_TOKEN_VALUE: ${{ secrets.HUGGING_FACE_TOKEN }} + - name: Upload the list of Python tests + uses: actions/upload-artifact@v7 + with: + name: test-collected + path: collected.txt + if-no-files-found: error + overwrite: true + retention-days: 7 + - name: Test documentation builds + run: make documentation + Test-Python: + # Shard K of N runs the tests whose node id hashes to K (--shard in + # policyengine_uk/tests/conftest.py), so each test is in exactly one + # shard. N is the number of jobs this matrix makes: to add a shard, add + # a number to the list. + name: Test (Python ${{ matrix.shard }}/${{ strategy.job-total }}) + runs-on: ubuntu-24.04-arm + # The shards took 15, 20 and 15 minutes on 10 October (run + # 38085920460). 60 minutes leaves room for a slow runner and still + # stops a hung shard. + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] permissions: contents: "read" id-token: "write" @@ -96,14 +160,54 @@ jobs: fi # Four pytest-xdist workers, one per core; together they peak at # about 9 GB of the runner's 16 GB. - make test PYTEST_ARGS="-n 4 --dist worksteal" + make test-python PYTEST_ARGS="-n 4 --dist worksteal --shard ${{ matrix.shard }}/${{ strategy.job-total }} --junitxml=junit-${{ matrix.shard }}.xml" env: HF_TOKEN_VALUE: ${{ secrets.HUGGING_FACE_TOKEN }} + # Measure coverage with sys.monitoring instead of the C trace + # function: the same lines are reported, at less cost per line. + COVERAGE_CORE: sysmon + - name: Upload the test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: test-junit-${{ matrix.shard }} + path: junit-${{ matrix.shard }}.xml + if-no-files-found: error + overwrite: true + retention-days: 7 - name: Upload coverage to Codecov uses: codecov/codecov-action@v6 with: files: ./coverage.xml fail_ci_if_error: false verbose: true - - name: Test documentation builds - run: make documentation + Test: + # One result for the whole suite. It fails if any test job failed, or + # if the shards did not run every collected Python test exactly once. + needs: [Test-YAML, Test-Python] + if: ${{ !cancelled() }} + # The shards' runners, not ubuntu-latest: on 10 October this 5-second + # job waited 32 minutes for an ubuntu-latest runner after the shards + # had finished (run 38085920460). + runs-on: ubuntu-24.04-arm + timeout-minutes: 10 + steps: + - name: Check that every test job passed + run: | + echo "YAML: ${{ needs.Test-YAML.result }}; Python: ${{ needs.Test-Python.result }}" + test "${{ needs.Test-YAML.result }}" = success + test "${{ needs.Test-Python.result }}" = success + - name: Checkout repo + if: ${{ !cancelled() }} + uses: actions/checkout@v6 + - name: Download the test lists and results + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + pattern: test-* + merge-multiple: true + - name: Check that every Python test ran in exactly one shard + # Runs even when a test job failed, so its report names the + # failed tests. + if: ${{ !cancelled() }} + run: python3 .github/check_test_shards.py --collected collected.txt --junit junit-*.xml diff --git a/Makefile b/Makefile index 5723221068..528f7b9be3 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,10 @@ test: test-yaml test-python test-yaml: policyengine-core test policyengine_uk/tests/policy -c policyengine_uk -# CI passes PYTEST_ARGS="-n 4 --dist worksteal" (pytest-xdist) to spread the -# tests over the runner's four cores. Without it this runs serially, as before. +# CI passes PYTEST_ARGS to spread the tests over a runner's four cores +# ("-n 4 --dist worksteal", pytest-xdist) and over several runners +# ("--shard K/N", policyengine_uk/tests/conftest.py). Without it this runs +# every test serially. test-python: pytest policyengine_uk/tests/ --cov=policyengine_uk --cov-report=xml --maxfail=0 -v $(PYTEST_ARGS) diff --git a/changelog.d/2169.fixed.md b/changelog.d/2169.fixed.md new file mode 100644 index 0000000000..b8ff4b838d --- /dev/null +++ b/changelog.d/2169.fixed.md @@ -0,0 +1 @@ +Put the petrol and diesel pump prices that convert spending into litres on the DESNZ annual ULSP and ULSD series (the basis Microcosm UK prices fuel spending on), replacing the January 2024 RAC prices, and re-derived the petrol and diesel spending litre proxies so weighted litres still follow the HMRC/OBR road-fuel volume path (part A of #2169). diff --git a/changelog.d/2309.fixed.md b/changelog.d/2309.fixed.md new file mode 100644 index 0000000000..db731a062c --- /dev/null +++ b/changelog.d/2309.fixed.md @@ -0,0 +1 @@ +- Stop paying Carer's Allowance to a carer whose net earnings exceed the earnings limit (£204 a week from April 2026). diff --git a/changelog.d/2321.fixed.md b/changelog.d/2321.fixed.md new file mode 100644 index 0000000000..4c44b28086 --- /dev/null +++ b/changelog.d/2321.fixed.md @@ -0,0 +1 @@ +- Round the High Income Child Benefit Charge as ITEPA 2003 s. 681C(3) requires: the percentage down to a whole number, and the Child Benefit and the charge down to whole pounds. diff --git a/changelog.d/2326.fixed.md b/changelog.d/2326.fixed.md new file mode 100644 index 0000000000..d486a2a5a0 --- /dev/null +++ b/changelog.d/2326.fixed.md @@ -0,0 +1 @@ +- Stop relieving a miscellaneous loss against other income in Income Tax, adjusted net income, total income and Universal Credit earnings: ITA 2007 s. 152(4) allows it only against miscellaneous income. diff --git a/changelog.d/2347.fixed.md b/changelog.d/2347.fixed.md new file mode 100644 index 0000000000..06c17f5efc --- /dev/null +++ b/changelog.d/2347.fixed.md @@ -0,0 +1 @@ +Set the Personal Savings Allowance on the rest-of-UK bands for Scottish taxpayers, as ITA 2007 s. 12B(8) requires; give the higher amount at exactly the basic rate limit and the lower amount at exactly the higher rate limit (ITA 2007 s. 10); and use the right bands for periods 2016 and 2017. diff --git a/changelog.d/2351.fixed.md b/changelog.d/2351.fixed.md new file mode 100644 index 0000000000..429087f2a7 --- /dev/null +++ b/changelog.d/2351.fixed.md @@ -0,0 +1 @@ +Band savings and dividend income above property income, which is non-savings income (ITA 2007 ss. 16, 16A), including in the income the annual allowance charge's marginal rate is read from. diff --git a/changelog.d/ci-test-runtime-2099.fixed.md b/changelog.d/ci-test-runtime-2099.fixed.md new file mode 100644 index 0000000000..413d63714e --- /dev/null +++ b/changelog.d/ci-test-runtime-2099.fixed.md @@ -0,0 +1 @@ +- Split the CI `Test` job into a YAML job and three parallel shards of the Python tests, with a check that every collected test ran in exactly one shard, and measured coverage with `sys.monitoring` (#2099). diff --git a/changelog.d/dataset-fiscal-year-default.fixed.md b/changelog.d/dataset-fiscal-year-default.fixed.md new file mode 100644 index 0000000000..9f1275ad38 --- /dev/null +++ b/changelog.d/dataset-fiscal-year-default.fixed.md @@ -0,0 +1 @@ +Warned when `UKSingleYearDataset` is given a `fiscal_year` alongside a file path, where it cannot take effect, and when it is built from DataFrames without one. Removed the process-wide `warnings.filterwarnings("ignore")` that importing the package installed, which hid these and every other warning from callers. diff --git a/changelog.d/docs-toc-patterns.changed.md b/changelog.d/docs-toc-patterns.changed.md new file mode 100644 index 0000000000..9b7931a53e --- /dev/null +++ b/changelog.d/docs-toc-patterns.changed.md @@ -0,0 +1 @@ +Documentation pages in docs/book/assumptions, validation, policy and programs/gov/ are now picked up by folder, so a new page no longer needs a myst.yml entry. This also publishes eight existing pages that were missing from the contents. diff --git a/changelog.d/remove-stale-xfail-1968.changed.md b/changelog.d/remove-stale-xfail-1968.changed.md new file mode 100644 index 0000000000..8c89b101f7 --- /dev/null +++ b/changelog.d/remove-stale-xfail-1968.changed.md @@ -0,0 +1 @@ +- Removed the stale non-strict expected failure for the 2023-24 Class 4 lower profits limit (#1968, fixed by #2058), so the alignment test now asserts that year. diff --git a/docs/book/assumptions/growthfactors.md b/docs/book/assumptions/growthfactors.md index 1a1d52e1ee..5871432b7b 100644 --- a/docs/book/assumptions/growthfactors.md +++ b/docs/book/assumptions/growthfactors.md @@ -32,6 +32,8 @@ We use HMRC road-fuel clearances and OBR-implied forecast volumes for petrol and We apply this to: `diesel_spending`, `petrol_spending` +The pump prices that turn spending into litres (`household.consumption.fuel.prices.petrol` and `.diesel`) are DESNZ calendar-year averages (Quarterly Energy Prices tables 4.1.1 and 4.1.2), the same basis Microcosm UK uses to price household fuel spending. The spending growth rates (`petrol_spending_litre_proxy`, `diesel_spending_litre_proxy`) are derived from those prices, so that weighted litres follow the volume path above: (1 + growth) = (1 + volume growth) × pricet / pricet−1 ÷ (1 + population growth). Whenever the prices change, re-derive the growth rates; a test checks that they match. + ## Average earnings ```{note} diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 66cb54d3c4..6894b55b4a 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -58,3 +58,24 @@ make test-microsimulation ``` When a command is not run, record why. + +## How CI Runs The Tests + +The test jobs in `.github/workflows/pr_code_changes.yaml` and +`code_changes.yaml` run the YAML policy tests in one job and split the Python +tests over several runners. Each of those runs one shard: + +```bash +make test-python PYTEST_ARGS="-n 4 --dist worksteal --shard K/N" +``` + +`--shard K/N` is defined in `policyengine_uk/tests/conftest.py`. It keeps the +tests whose node id hashes to shard K of N, so every test is in exactly one +shard, including a test added later. The final `Test` job runs +`.github/check_test_shards.py`, which fails unless every collected test ran in +exactly one shard. + +- To rerun what one CI shard ran, pass the same `--shard K/N` locally. +- To make CI faster, add a shard to the `shard` matrix in both workflows. +- Do not split tests by file or directory in a workflow: a test outside the + listed paths would silently not run. diff --git a/myst.yml b/myst.yml index 43cc808e87..b5b3053685 100644 --- a/myst.yml +++ b/myst.yml @@ -13,23 +13,30 @@ project: - file: docs/book/index.md - title: Usage children: - - file: docs/book/usage/getting-started - - file: docs/book/usage/simulations - - file: docs/book/usage/scenarios - - file: docs/book/usage/dynamics + - file: docs/book/usage/getting-started.md + - file: docs/book/usage/simulations.md + - file: docs/book/usage/scenarios.md + - file: docs/book/usage/dynamics.md + # Pages listed by `file:` keep their place at the top of a section. Any other + # Markdown page in a section's folder is added after them by `pattern:`, so a + # new page needs no change here. Notebooks are not pattern-matched. - title: Assumptions children: - - file: docs/book/assumptions/growthfactors - - file: docs/book/assumptions/water-bills + - file: docs/book/assumptions/growthfactors.md + - file: docs/book/assumptions/water-bills.md + - pattern: docs/book/assumptions/*.md - title: Validation children: - - file: docs/book/validation/hbai + - file: docs/book/validation/hbai.md + - pattern: docs/book/validation/*.md - title: Policy children: - file: docs/engineering/excise-duties.md - file: docs/engineering/bus-fares.md - - file: docs/book/policy/model-baseline - - file: docs/book/policy/uc-rebalancing + - pattern: docs/book/policy/*.md + - title: Programmes + children: + - pattern: docs/book/programs/gov/*/*.md site: options: logo: docs/book/logo.png diff --git a/policyengine_uk/data/dataset_schema.py b/policyengine_uk/data/dataset_schema.py index 6f294022b6..37f0956250 100644 --- a/policyengine_uk/data/dataset_schema.py +++ b/policyengine_uk/data/dataset_schema.py @@ -1,3 +1,4 @@ +import warnings import pandas as pd import numpy as np from typing import TYPE_CHECKING @@ -40,16 +41,28 @@ def validate_file_path(file_path: str, raise_exception: bool = True): return True + # Used when a dataset is built from DataFrames without a fiscal year. + # Deprecated: pass fiscal_year explicitly, as this will become required. + DEFAULT_FISCAL_YEAR = 2025 + def __init__( self, file_path: str = None, person: pd.DataFrame = None, benunit: pd.DataFrame = None, household: pd.DataFrame = None, - fiscal_year: int = 2025, + fiscal_year: int = None, ): file_path = str(file_path) if file_path else None if file_path is not None: + if fiscal_year is not None: + warnings.warn( + "fiscal_year is ignored when loading from a file: the " + "time period is read from the file itself. Drop the " + "argument, or use uprate_dataset to change the period.", + UserWarning, + stacklevel=2, + ) self.validate_file_path(file_path) # Read-only: the file may be a cached download the cache manager # has made unwritable, and the default append mode would refuse it. @@ -63,6 +76,17 @@ def __init__( raise ValueError( "Must provide either a file path or all three DataFrames (person, benunit, household)." ) + if fiscal_year is None: + warnings.warn( + "fiscal_year was not given, so this dataset is labelled " + f"{self.DEFAULT_FISCAL_YEAR}. The label is the year the " + "financial year starts, so FRS 2024/25 data is 2024. " + "Pass fiscal_year explicitly; the default will be " + "removed.", + FutureWarning, + stacklevel=2, + ) + fiscal_year = self.DEFAULT_FISCAL_YEAR self._person = person self._benunit = benunit self._household = household diff --git a/policyengine_uk/parameters/gov/dwp/JSA/eligibility/min_age.yaml b/policyengine_uk/parameters/gov/dwp/JSA/eligibility/min_age.yaml index 7729fa752f..c387d1daa0 100644 --- a/policyengine_uk/parameters/gov/dwp/JSA/eligibility/min_age.yaml +++ b/policyengine_uk/parameters/gov/dwp/JSA/eligibility/min_age.yaml @@ -9,7 +9,7 @@ metadata: href: https://www.legislation.gov.uk/uksi/1996/207/regulation/57 - title: The Jobseeker's Allowance Regulations 1996, Schedule 1 href: https://www.legislation.gov.uk/uksi/1996/207/schedule/1 -notes: > +documentation: > This is the standard age threshold used for legacy JSA modelling and proxying. Some 16 and 17 year olds can qualify under special cases in regulation 57. values: diff --git a/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml b/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml new file mode 100644 index 0000000000..22c25451e7 --- /dev/null +++ b/policyengine_uk/parameters/gov/dwp/carers_allowance/earnings_limit.yaml @@ -0,0 +1,62 @@ +description: Weekly earnings above which a person is treated as gainfully employed and so is not entitled to Carer's Allowance. The test uses net earnings in the preceding week. +metadata: + economy: false + period: week + unit: currency-GBP + label: Carer's Allowance earnings limit + reference: + - title: The Social Security (Invalid Care Allowance) Regulations 1976 reg. 8(1) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8 + - title: Social Security Contributions and Benefits Act 1992 s. 70(1)(b) + href: https://www.legislation.gov.uk/ukpga/1992/4/section/70 +values: + 2015-04-06: + value: 110 + reference: + - title: The Social Security (Invalid Care Allowance) (Amendment) Regulations 2015 (SI 2015/162) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2015-04-06 + 2017-04-10: + value: 116 + reference: + - title: The Social Security (Invalid Care Allowance) (Amendment) Regulations 2017 (SI 2017/386) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2017-04-10 + 2018-04-09: + value: 120 + reference: + - title: The Social Security (Invalid Care Allowance) (Amendment) Regulations 2018 (SI 2018/280) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2018-04-09 + 2019-04-08: + value: 123 + reference: + - title: The Social Security Benefits Up-rating Regulations 2019 (SI 2019/552) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2019-04-08 + 2020-04-06: + value: 128 + reference: + - title: The Social Security Benefits Up-rating Regulations 2020 (SI 2020/266) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2020-04-06 + 2022-04-11: + value: 132 + reference: + - title: The Social Security Benefits Up-rating Regulations 2022 (SI 2022/342) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2022-04-11 + 2023-04-10: + value: 139 + reference: + - title: The Social Security Benefits Up-rating Regulations 2023 (SI 2023/340) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2023-04-10 + 2024-04-08: + value: 151 + reference: + - title: The Social Security Benefits Up-Rating Regulations 2024 (SI 2024/386) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2024-04-08 + 2025-04-07: + value: 196 + reference: + - title: The Social Security Benefits Up-rating Regulations 2025 (SI 2025/352) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8/2025-04-07 + 2026-04-06: + value: 204 + reference: + - title: The Social Security Benefits Up-rating Regulations 2026 (SI 2026/218) + href: https://www.legislation.gov.uk/uksi/1976/409/regulation/8 diff --git a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml index 01fa695891..e21a645890 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml +++ b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml @@ -345,14 +345,14 @@ obr: href: https://obr.uk/docs/dlm_uploads/Fuel-duty-supplementary-release_receipts-by-vehicle-type.pdf petrol_spending_litre_proxy: - description: Unleaded-petrol spending proxy growth that preserves weighted road-fuel litres after dividing by model pump prices. + description: Unleaded-petrol spending proxy growth that preserves weighted road-fuel litres after dividing by model pump prices. Derived as (1 + road_fuel_volume) x (price_t / price_t-1) / (1 + population) - 1, using household.consumption.fuel.prices; re-derive whenever those prices change. values: - 2021-01-01: 0.09966723889828133 - 2022-01-01: 0.47125942029399615 - 2023-01-01: 0.09943596417596279 - 2024-01-01: -0.14810324567590538 - 2025-01-01: -0.0269132439679719 - 2026-01-01: -0.018120813771079436 + 2021-01-01: 0.4276130160591334 + 2022-01-01: 0.32114330849740713 + 2023-01-01: -0.11978022401038202 + 2024-01-01: -0.05378435951558702 + 2025-01-01: -0.07097011631498962 + 2026-01-01: 0.09631434800374827 2027-01-01: -0.027765858532455168 2028-01-01: -0.030878786058031737 2029-01-01: -0.04149818363243518 @@ -408,18 +408,18 @@ obr: href: https://www.gov.uk/government/statistics/hydrocarbon-oils-bulletin - title: OBR EFO March 2026 (fuel-duty receipts forecast) href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ - - title: RAC latest average petrol price - href: https://www.rac.co.uk/drive/advice/fuel-watch/ + - title: DESNZ annual average ULSP pump price (household.consumption.fuel.prices.petrol) + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics diesel_spending_litre_proxy: - description: Diesel spending proxy growth that preserves weighted road-fuel litres after dividing by model pump prices. + description: Diesel spending proxy growth that preserves weighted road-fuel litres after dividing by model pump prices. Derived as (1 + road_fuel_volume) x (price_t / price_t-1) / (1 + population) - 1, using household.consumption.fuel.prices; re-derive whenever those prices change. values: - 2021-01-01: 0.13390420538515913 - 2022-01-01: 0.3702255435452164 - 2023-01-01: -0.01858407316867594 - 2024-01-01: 0.03164670292538285 - 2025-01-01: -0.0269132439679719 - 2026-01-01: -0.018120813771079436 + 2021-01-01: 0.4035933750476013 + 2022-01-01: 0.38608334996374616 + 2023-01-01: -0.12613670608493466 + 2024-01-01: -0.07344928074891444 + 2025-01-01: -0.06483399761456943 + 2026-01-01: 0.1748436772712496 2027-01-01: -0.027765858532455168 2028-01-01: -0.030878786058031737 2029-01-01: -0.04149818363243518 @@ -475,8 +475,8 @@ obr: href: https://www.gov.uk/government/statistics/hydrocarbon-oils-bulletin - title: OBR EFO March 2026 (fuel-duty receipts forecast) href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ - - title: RAC latest average diesel price - href: https://www.rac.co.uk/drive/advice/fuel-watch/ + - title: DESNZ annual average ULSD pump price (household.consumption.fuel.prices.diesel) + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics consumer_price_index_ahc: description: Consumer price index year-on-year growth, modified to remove housing costs. diff --git a/policyengine_uk/parameters/gov/hmrc/child_benefit/README.md b/policyengine_uk/parameters/gov/hmrc/child_benefit/README.md index a5190ffabe..3e5e568dd5 100644 --- a/policyengine_uk/parameters/gov/hmrc/child_benefit/README.md +++ b/policyengine_uk/parameters/gov/hmrc/child_benefit/README.md @@ -29,11 +29,15 @@ data in a release; updating model code alone does not repair existing data. `gov.hmrc.child_benefit.opt_out_charge_share` is a behavioural assumption, not a statutory eligibility rule. It defaults to one: a flagged family continues -to opt out only while the charge would recover the full benefit. The charge -share is calculated independently of payment and clipped to [0, 1]. A reform -that reduces the share below the threshold, removes the charge or neutralises -`CB_HITC` restores payment. A zero charge never sustains an opt-out, even if -the assumed threshold is zero. Genuine nonclaimants remain outside payment. +to opt out only while the continuous charge share is one, that is while the +higher adjusted net income in the family is at or above the end of the +phase-out band. The share is the unrounded fraction of the band crossed, calculated independently of +payment and clipped to [0, 1]. A reform that reduces the share below the +threshold, removes the charge or neutralises `CB_HITC` restores payment. A zero +continuous charge share never sustains an opt-out, even if the assumed +threshold is zero. The statutory charge (`CB_HITC`) is different: ITEPA 2003 +s. 681C(3) rounds its percentage and amount down, so it can be nil while the +share is small but positive. Genuine nonclaimants remain outside payment. Microcosm selects fully charged families first and can draw from partly charged families if needed to meet its target. The model's default threshold is an diff --git a/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/index.yaml b/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/index.yaml index 0d122a1e7c..50bf8f1a10 100644 --- a/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/index.yaml +++ b/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/index.yaml @@ -1,5 +1,6 @@ description: Student loan interest rates. -label: Interest rates +metadata: + label: Interest rates documentation: | Interest rates applied to student loan balances vary by plan type. Plan 2 has income-contingent rates that vary between RPI and RPI+3%. diff --git a/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/plan_2/index.yaml b/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/plan_2/index.yaml index 0ff4c1ab69..aae6ba7a54 100644 --- a/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/plan_2/index.yaml +++ b/policyengine_uk/parameters/gov/hmrc/student_loans/interest_rates/plan_2/index.yaml @@ -1,5 +1,6 @@ description: Plan 2 student loan interest rates. -label: Plan 2 interest rates +metadata: + label: Plan 2 interest rates documentation: | Plan 2 student loan interest rates are income-contingent after graduation. diff --git a/policyengine_uk/parameters/household/consumption/fuel/prices/diesel.yaml b/policyengine_uk/parameters/household/consumption/fuel/prices/diesel.yaml index 992f1f1e40..16141abd1f 100644 --- a/policyengine_uk/parameters/household/consumption/fuel/prices/diesel.yaml +++ b/policyengine_uk/parameters/household/consumption/fuel/prices/diesel.yaml @@ -1,17 +1,35 @@ -description: Average price of diesel per litre. +description: Average pump price of diesel (ULSD) per litre, including fuel duty and VAT. Calendar-year average of DESNZ monthly typical retail prices; the same basis Microcosm UK uses to price household diesel spending. values: - 2012-04-01: 1.4052 - 2014-09-15: 1.3339 - 2017-05-01: 1.2225 - 2020-05-14: 1.1186 - 2022-01-01: 1.4559 - 2024-01-01: 1.52 + 2010-01-01: 1.19258627 + 2011-01-01: 1.38716127 + 2012-01-01: 1.41828260 + 2013-01-01: 1.40405189 + 2014-01-01: 1.33457999 + 2015-01-01: 1.14898456 + 2016-01-01: 1.10128630 + 2017-01-01: 1.20149237 + 2018-01-01: 1.29982166 + 2019-01-01: 1.31475463 + 2020-01-01: 1.19135223 + 2021-01-01: 1.34937007 + 2022-01-01: 1.77658132 + 2023-01-01: 1.58188711 + 2024-01-01: 1.48328885 + 2025-01-01: 1.42548575 + # Provisional: mean of the January to September 2026 monthly prices + # (DESNZ table 4.1.1, published 29 September 2026). Replace with the + # calendar-year average from table 4.1.2 when DESNZ publishes it, and + # re-derive the diesel_spending_litre_proxy growth series. + 2026-01-01: 1.70563033 metadata: unit: currency-GBP label: Price of diesel per litre economy: false + # Held flat in cash terms after the last DESNZ year. Litres and fuel duty + # do not depend on this projection: the spending litre proxy is derived + # from these prices, so later price changes cancel out of spending / price. reference: - - title: RAC - href: https://www.rac.co.uk/drive/advice/fuel-watch/ - - title: RAC Foundation historical averages - href: https://www.racfoundation.org/data/uk-pump-prices-over-time + - title: DESNZ, Quarterly Energy Prices table 4.1.2, annual typical retail prices of petroleum products (ULSD) + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics + - title: DESNZ, Quarterly Energy Prices table 4.1.1, monthly typical retail prices (ULSD), January to September 2026 + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics diff --git a/policyengine_uk/parameters/household/consumption/fuel/prices/petrol.yaml b/policyengine_uk/parameters/household/consumption/fuel/prices/petrol.yaml index 6ff14ca293..f701f127f8 100644 --- a/policyengine_uk/parameters/household/consumption/fuel/prices/petrol.yaml +++ b/policyengine_uk/parameters/household/consumption/fuel/prices/petrol.yaml @@ -1,18 +1,35 @@ -description: Average price of unleaded petrol per litre, including fuel duty. +description: Average pump price of unleaded petrol (ULSP) per litre, including fuel duty and VAT. Calendar-year average of DESNZ monthly typical retail prices; the same basis Microcosm UK uses to price household petrol spending. values: - 2012-04-01: 1.3492 - 2014-09-15: 1.2918 - 2017-05-01: 1.2023 - 2020-05-14: 1.0669 - 2022-01-01: 1.491 - 2022-03-22: 1.6703 - 2024-01-01: 1.44 + 2010-01-01: 1.16902571 + 2011-01-01: 1.33268790 + 2012-01-01: 1.35390547 + 2013-01-01: 1.34145278 + 2014-01-01: 1.27495853 + 2015-01-01: 1.11130760 + 2016-01-01: 1.08845640 + 2017-01-01: 1.17588883 + 2018-01-01: 1.25195971 + 2019-01-01: 1.24877998 + 2020-01-01: 1.13947293 + 2021-01-01: 1.31269579 + 2022-01-01: 1.64732274 + 2023-01-01: 1.47746334 + 2024-01-01: 1.41477665 + 2025-01-01: 1.35072210 + # Provisional: mean of the January to September 2026 monthly prices + # (DESNZ table 4.1.1, published 29 September 2026). Replace with the + # calendar-year average from table 4.1.2 when DESNZ publishes it, and + # re-derive the petrol_spending_litre_proxy growth series. + 2026-01-01: 1.50814483 metadata: unit: currency-GBP label: Price of unleaded petrol per litre economy: false + # Held flat in cash terms after the last DESNZ year. Litres and fuel duty + # do not depend on this projection: the spending litre proxy is derived + # from these prices, so later price changes cancel out of spending / price. reference: - - title: RAC latest average - href: https://www.rac.co.uk/drive/advice/fuel-watch/ - - title: RAC Foundation historical averages - href: https://www.racfoundation.org/data/uk-pump-prices-over-time + - title: DESNZ, Quarterly Energy Prices table 4.1.2, annual typical retail prices of petroleum products (ULSP) + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics + - title: DESNZ, Quarterly Energy Prices table 4.1.1, monthly typical retail prices (ULSP), January to September 2026 + href: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics diff --git a/policyengine_uk/reforms/conservatives/household_based_hitc.py b/policyengine_uk/reforms/conservatives/household_based_hitc.py index 861c04d3d0..05295e581d 100644 --- a/policyengine_uk/reforms/conservatives/household_based_hitc.py +++ b/policyengine_uk/reforms/conservatives/household_based_hitc.py @@ -1,4 +1,8 @@ from policyengine_uk.model_api import * +from policyengine_uk.utils.child_benefit import ( + child_benefit_charge_percentage, + round_down_to_pounds, +) class CB_HITC(Variable): @@ -15,10 +19,12 @@ def formula(person, period, parameters): hitc = parameters(period).gov.hmrc.income_tax.charges.CB_HITC personal_income = person("adjusted_net_income", period) income = person.benunit.sum(personal_income) - percentage = max_(income - hitc.phase_out_start, 0) / ( - hitc.phase_out_end - hitc.phase_out_start + # The same s. 681C(3) rounding as the baseline charge, on the + # household's combined income. + percentage = child_benefit_charge_percentage( + income, hitc.phase_out_start, hitc.phase_out_end ) - return min_(percentage, 1) * CB_received + return round_down_to_pounds(percentage * round_down_to_pounds(CB_received)) class make_cb_hitc_household_based(Variable): diff --git a/policyengine_uk/tests/code_health/test_ci_test_shards.py b/policyengine_uk/tests/code_health/test_ci_test_shards.py new file mode 100644 index 0000000000..17e3d71586 --- /dev/null +++ b/policyengine_uk/tests/code_health/test_ci_test_shards.py @@ -0,0 +1,332 @@ +"""The Test job's shards run every test exactly once. + +CI runs ``pytest policyengine_uk/tests`` as several jobs, each with +``--shard K/N`` (conftest.py), and then runs ``.github/check_test_shards.py`` +over their JUnit files and an unsharded ``pytest --collect-only -q`` listing. +These tests hold the two halves to the invariants the split relies on: + +1. ``shard_of(nodeid, N)`` is between 1 and N and depends only on its + arguments, so it is the same in every process and on every runner. +2. For any set of node ids and any N, the N shards are disjoint and their + union is the whole set. +3. ``parse_shard`` accepts exactly ``K/N`` with 1 <= K <= N. +4. ``junit_key`` gives a node id the name pytest gives it in JUnit XML. +5. The check passes when every collected test ran in exactly one shard and + none failed, and reports a test that is missing, repeated, not collected + or failed. +6. With real pytest runs, the collection listing and the shards' JUnit files + pass the check, with and without pytest-xdist, and fail it if a shard's + file is left out. +""" + +import argparse +import importlib.util +import os +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ElementTree +from pathlib import Path + +import pytest +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st + +TESTS_DIR = Path(__file__).resolve().parents[1] +REPO_ROOT = TESTS_DIR.parents[1] +CHECKER = REPO_ROOT / ".github" / "check_test_shards.py" + +if not CHECKER.exists(): + pytest.skip( + "the .github folder is not part of an installed package", + allow_module_level=True, + ) + +_spec = importlib.util.spec_from_file_location("check_test_shards", CHECKER) +checker = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(checker) + +PROPERTY_SETTINGS = settings( + max_examples=200, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.function_scoped_fixture], +) + + +@pytest.fixture +def conftest(request): + path = TESTS_DIR / "conftest.py" + for plugin in request.config.pluginmanager.get_plugins(): + if getattr(plugin, "__file__", None) and Path(plugin.__file__) == path: + return plugin + raise AssertionError(f"{path} is not loaded as a plugin") + + +# Node ids as pytest writes them: a path, optional classes, a test name and +# optional parameters. pytest escapes non-ASCII and control characters in +# parameter ids, so the parameters here are any printable ASCII text, +# including "[", "]" and "::". +_identifier = st.from_regex(r"[A-Za-z_][A-Za-z0-9_]{0,12}", fullmatch=True) +_path = st.lists(_identifier, min_size=1, max_size=4).map( + lambda parts: "/".join(parts) + ".py" +) +_printable = st.characters(min_codepoint=0x20, max_codepoint=0x7E) +_params = st.one_of( + st.just(""), + st.text(alphabet=_printable, max_size=20).map(lambda text: f"[{text}]"), +) +nodeids = st.builds( + lambda path, classes, name, params: "::".join([path, *classes, name + params]), + _path, + st.lists(_identifier, max_size=2), + _identifier, + _params, +) +# Node ids that could come from one collection: distinct, and with distinct +# JUnit names, which pytest's own naming guarantees for real suites. +collections = st.lists(nodeids, max_size=30, unique_by=checker.junit_key) + + +def junit_cases(ids, outcome="passed"): + return [(checker.junit_key(nodeid), outcome) for nodeid in ids] + + +@PROPERTY_SETTINGS +@given(nodeid=nodeids, shards=st.integers(1, 64)) +@example(nodeid="policyengine_uk/tests/test_x.py::test_y", shards=3) +def test_shard_is_in_range_and_repeatable(conftest, nodeid, shards): + shard = conftest.shard_of(nodeid, shards) + assert 1 <= shard <= shards + assert shard == conftest.shard_of(nodeid, shards) + + +def test_shard_does_not_depend_on_the_process(conftest): + # Pinned values: every runner and pytest-xdist worker must agree, which + # Python's own hash, salted per process, would not give. + assert [ + conftest.shard_of("policyengine_uk/tests/test_x.py::test_y", shards) + for shards in (1, 2, 3, 4, 7) + ] == [1, 2, 1, 4, 7] + assert conftest.shard_of("policyengine_uk/tests/test_x.py::test_y[2026-a]", 3) == 2 + + +@PROPERTY_SETTINGS +@given(ids=st.lists(nodeids, max_size=40, unique=True), shards=st.integers(1, 8)) +def test_shards_partition_any_collection(conftest, ids, shards): + members = { + shard: [nodeid for nodeid in ids if conftest.shard_of(nodeid, shards) == shard] + for shard in range(1, shards + 1) + } + assert sorted(sum(members.values(), [])) == sorted(ids) + assert sum(len(group) for group in members.values()) == len(ids) + + +@PROPERTY_SETTINGS +@given(shards=st.integers(1, 200), data=st.data()) +def test_parse_shard_accepts_k_of_n(conftest, shards, data): + shard = data.draw(st.integers(1, shards)) + assert conftest.parse_shard(f"{shard}/{shards}") == (shard, shards) + + +@pytest.mark.parametrize( + "value", + [ + "0/3", + "4/3", + "1/0", + "0/0", + "3", + "1/3/5", + "a/b", + "1-3", + "", + " 1/3", + "1/3 ", + "-1/3", + ], +) +def test_parse_shard_rejects_anything_else(conftest, value): + with pytest.raises(argparse.ArgumentTypeError): + conftest.parse_shard(value) + + +@PROPERTY_SETTINGS +@given(nodeid=nodeids) +@example(nodeid="policyengine_uk/tests/test_x.py::TestA::test_y[a/b.yaml-1]") +@example(nodeid="policyengine_uk/tests/test_x.py::test_y[a::b[c]-d e]") +def test_junit_key_matches_pytest(nodeid): + # Differential test against pytest's own naming, written out to XML and + # read back as the check reads it. + junitxml = pytest.importorskip("_pytest.junitxml") + names = junitxml.mangle_test_address(nodeid) + element = ElementTree.Element( + "testcase", + classname=".".join(names[:-1]), + name=junitxml.bin_xml_escape(names[-1]), + ) + read_back = ElementTree.fromstring(ElementTree.tostring(element)) + assert checker.junit_key(nodeid) == ( + read_back.get("classname"), + read_back.get("name"), + ) + + +@PROPERTY_SETTINGS +@given(ids=collections, shards=st.integers(1, 5), data=st.data()) +def test_check_passes_exactly_when_every_test_ran_once(conftest, ids, shards, data): + ran = [ + junit_cases(i for i in ids if conftest.shard_of(i, shards) == shard) + for shard in range(1, shards + 1) + ] + problems, summary = checker.check(ids, ran) + assert problems == [] + assert summary.startswith(f"{len(ids)} tests collected") + + if not ids: + return + victim = data.draw(st.sampled_from(ids)) + key = checker.junit_key(victim) + home = conftest.shard_of(victim, shards) - 1 + + missing = [list(cases) for cases in ran] + missing[home].remove((key, "passed")) + problems, _ = checker.check(ids, missing) + assert [names for title, names in problems if "ran in no shard" in title] == [ + [victim] + ] + + repeated = [list(cases) for cases in ran] + repeated[data.draw(st.integers(0, shards - 1))].append((key, "passed")) + problems, _ = checker.check(ids, repeated) + assert [names for title, names in problems if "more than once" in title] == [ + [victim] + ] + + failed = [list(cases) for cases in ran] + failed[home][failed[home].index((key, "passed"))] = (key, "failed") + problems, _ = checker.check(ids, failed) + assert [names for title, names in problems if "failed" in title] == [ + [f"shard {home + 1}: {victim}"] + ] + + uncollected = [nodeid for nodeid in ids if nodeid != victim] + problems, _ = checker.check(uncollected, ran) + assert [names for title, names in problems if "not collected" in title] == [ + [f"{key[0]}::{key[1]}"] + ] + + +def test_read_collected_requires_the_listed_count(): + listing = "a/test_x.py::test_one\na/test_x.py::test_two[p]\n\n2 tests collected in 0.01s\n" + assert checker.read_collected(listing) == [ + "a/test_x.py::test_one", + "a/test_x.py::test_two[p]", + ] + assert checker.read_collected("\nno tests collected in 0.01s\n") == [] + # Warnings follow the listing and may mention a node id. + with_warnings = listing.replace( + "\n\n2 tests", + "\n\n=== warnings summary ===\na/test_x.py::test_one\n x\n2 tests", + ) + assert checker.read_collected(with_warnings) == checker.read_collected(listing) + for bad in ( + "a/test_x.py::test_one\n", + "a/test_x.py::test_one\n\n2 tests collected in 0.01s\n", + "a/test_x.py::test_one\na/test_x.py::test_one\n\n2 tests collected in 0.01s\n", + "a/test_x.py::test_one\n\n1 test collected, 1 error in 0.01s\n", + ): + with pytest.raises(checker.ShardCheckError): + checker.read_collected(bad) + + +SAMPLE_SUITE = """ +import pytest + + +@pytest.mark.parametrize("value", range(40)) +def test_number(value): + assert value >= 0 + + +@pytest.mark.parametrize( + "name", ["parameters/gov/a.yaml", "b c", "d[e]", "f::g", "h-1", "caf\\u00e9", "tab\\there"] +) +def test_name(name): + assert name + + +class TestGroup: + @pytest.mark.parametrize("value", ["x", "y"]) + def test_member(self, value): + assert value + + @pytest.mark.skip(reason="skipped tests still count as run") + def test_skipped(self): + raise AssertionError + + @pytest.mark.xfail(strict=True) + def test_expected_failure(self): + raise AssertionError +""" + + +def _pytest(cwd, *args): + # A clean run of pytest: no options, plugins or coverage settings carried + # over from the session that runs this test. + env = { + name: value + for name, value in os.environ.items() + if not name.startswith(("PYTEST_", "COV_CORE_", "COVERAGE_")) + } + return subprocess.run( + [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", *args], + cwd=cwd, + env=env, + capture_output=True, + text=True, + timeout=600, + ) + + +@pytest.mark.parametrize("xdist", [(), ("-n", "2", "--dist", "worksteal")]) +def test_real_pytest_shards_pass_the_check(tmp_path, xdist): + suite = tmp_path / "suite" + suite.mkdir() + # The suite's real conftest.py, so this runs the --shard CI uses. + shutil.copy(TESTS_DIR / "conftest.py", suite / "conftest.py") + (suite / "test_sample.py").write_text(SAMPLE_SUITE, encoding="utf-8") + + listing = _pytest(tmp_path, "suite", "--collect-only", "-q") + assert listing.returncode == 0, listing.stdout + listing.stderr + collected = tmp_path / "collected.txt" + collected.write_text(listing.stdout, encoding="utf-8") + assert len(checker.read_collected(listing.stdout)) == 51 + + shards = 3 + junit = [] + for shard in range(1, shards + 1): + junit.append(tmp_path / f"junit-{shard}.xml") + run = _pytest( + tmp_path, + "suite", + "--shard", + f"{shard}/{shards}", + f"--junitxml={junit[-1]}", + *xdist, + ) + assert run.returncode == 0, run.stdout + run.stderr + assert f"shard: {shard} of {shards}" in run.stdout + + sizes = [len(checker.read_junit(path)) for path in junit] + assert sum(sizes) == 51 and all(sizes) + + def check(*paths): + return checker.main( + ["--collected", str(collected), "--junit", *map(str, paths)] + ) + + assert check(*junit) == 0 + assert check(*junit[:-1]) == 1 + assert check(*junit, junit[0]) == 1 + assert check(tmp_path / "absent.xml") == 2 diff --git a/policyengine_uk/tests/conftest.py b/policyengine_uk/tests/conftest.py index d611a0c1c5..ca783dd9ab 100644 --- a/policyengine_uk/tests/conftest.py +++ b/policyengine_uk/tests/conftest.py @@ -1,4 +1,6 @@ +import argparse import gc +import hashlib import os import re import sys @@ -29,7 +31,62 @@ def cloned_uk_tax_benefit_system(): yield +# CI splits the suite over several runners with ``--shard K/N``. The shard of a +# test is a function of its node id alone, so the shards partition whatever is +# collected: every test runs in exactly one of them, including a test added +# later. The Test job checks that on each run (.github/check_test_shards.py). +def shard_of(nodeid: str, shards: int) -> int: + """The shard, from 1 to ``shards``, that runs the test ``nodeid``. + + Uses SHA-256 rather than ``hash``, which Python salts per process: every + runner and every pytest-xdist worker must give the same answer. + """ + digest = hashlib.sha256(nodeid.encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big") % shards + 1 + + +def parse_shard(value: str) -> tuple[int, int]: + """Parse ``K/N`` into ``(K, N)``, requiring 1 <= K <= N.""" + match = re.fullmatch(r"([0-9]+)/([0-9]+)", value) + if match: + shard, shards = int(match.group(1)), int(match.group(2)) + if 1 <= shard <= shards: + return shard, shards + raise argparse.ArgumentTypeError( + f"expected K/N with 1 <= K <= N, such as 1/3; got {value!r}" + ) + + +def pytest_addoption(parser): + parser.addoption( + "--shard", + type=parse_shard, + default=None, + metavar="K/N", + help="Run only shard K of N. Every test is in exactly one shard.", + ) + + +def pytest_report_header(config): + shard = config.getoption("shard", default=None) + if shard is not None: + return f"shard: {shard[0]} of {shard[1]}, chosen by node id" + + def pytest_collection_modifyitems(config, items): + shard = config.getoption("shard", default=None) + if shard is not None: + this_shard, shards = shard + selected, deselected = [], [] + for item in items: + if shard_of(item.nodeid, shards) == this_shard: + selected.append(item) + else: + deselected.append(item) + items[:] = selected + if deselected: + config.hook.pytest_deselected(items=deselected) + has_default_dataset = bool(os.environ.get("POLICYENGINE_UK_DEFAULT_DATASET")) if has_default_dataset: return diff --git a/policyengine_uk/tests/policy/baseline/consumption/road_fuel_litres.yaml b/policyengine_uk/tests/policy/baseline/consumption/road_fuel_litres.yaml new file mode 100644 index 0000000000..5ef8ff3170 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/consumption/road_fuel_litres.yaml @@ -0,0 +1,46 @@ +# Litres divide spending by the DESNZ calendar-year average pump price +# (QEP table 4.1.2): 141.477665p ULSP and 148.328885p diesel in 2024, +# 135.072210p and 142.548575p in 2025. Microcosm UK prices household +# petrol and diesel spending on the same basis (PolicyEngine/policyengine-uk#2169). +- name: 2024 petrol spending at the DESNZ ULSP price buys 1,000 litres + period: 2024 + absolute_error_margin: 0.01 + input: + people: + adult: + age: 40 + benunits: + benunit: + members: [adult] + households: + household: + members: [adult] + petrol_spending: 1_414.77665 + diesel_spending: 1_483.28885 + output: + petrol_price: 1.41477665 + diesel_price: 1.48328885 + petrol_litres: 1_000 + diesel_litres: 1_000 + +- name: 2025 litres use the 2025 DESNZ prices, not the January 2024 RAC prices + period: 2025 + absolute_error_margin: 0.01 + input: + people: + adult: + age: 40 + benunits: + benunit: + members: [adult] + households: + household: + members: [adult] + petrol_spending: 1_350.7221 + diesel_spending: 1_425.48575 + output: + petrol_price: 1.3507221 + diesel_price: 1.42548575 + # Under the old £1.44 / £1.52 divisors these would be 938.0 and 937.8. + petrol_litres: 1_000 + diesel_litres: 1_000 diff --git a/policyengine_uk/tests/policy/baseline/finance/benefit/family/universal_credit/income/uc_earnings_deductions.yaml b/policyengine_uk/tests/policy/baseline/finance/benefit/family/universal_credit/income/uc_earnings_deductions.yaml index d5e2aef012..99132fcbd3 100644 --- a/policyengine_uk/tests/policy/baseline/finance/benefit/family/universal_credit/income/uc_earnings_deductions.yaml +++ b/policyengine_uk/tests/policy/baseline/finance/benefit/family/universal_credit/income/uc_earnings_deductions.yaml @@ -272,10 +272,10 @@ household: members: [parent, child] output: - # Child Benefit 27.05 x 52 = 1,406.60; the charge takes - # (70,000 - 60,000) / 200 = 50% of it, 703.30. - CB_HITC: [703.30, 0] - income_tax: [16_135.30, 0] + # Child Benefit 27.05 x 52 = 1,406.60, rounded down to £1,406; the charge + # takes (70,000 - 60,000) / 200 = 50% of it, £703 (s. 681C(3)). + CB_HITC: [703, 0] + income_tax: [16_135, 0] # 37,700 x 20% + (57,430 - 37,700) x 40%. uc_income_tax_on_earnings: [15_432, 0] diff --git a/policyengine_uk/tests/policy/baseline/gov/dfe/extended_childcare_entitlement/extended_childcare_entitlement_work_condition.yaml b/policyengine_uk/tests/policy/baseline/gov/dfe/extended_childcare_entitlement/extended_childcare_entitlement_work_condition.yaml index ef56a091ad..39860f1648 100644 --- a/policyengine_uk/tests/policy/baseline/gov/dfe/extended_childcare_entitlement/extended_childcare_entitlement_work_condition.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/dfe/extended_childcare_entitlement/extended_childcare_entitlement_work_condition.yaml @@ -506,14 +506,18 @@ # overlapped to nil by maternity allowance: entitled but not paid, so still # excluded. - name: 2022, working partner entitled to Carer's Allowance overlapped to nil - the partner on contributory ESA does not meet the condition + # Carer's Allowance entitlement needs net earnings within the earnings + # limit (£132 a week in 2022-23): pay of £8,400 less half of £3,100 of + # pension contributions is £6,850, or £131.73 a week. period: 2022 input: people: parent1: age: 40 is_parent: true - employment_income: 20_000 - adjusted_net_income: 20_000 + employment_income: 8_400 + employee_pension_contributions: 3_100 + adjusted_net_income: 5_300 care_hours: 35 maternity_allowance_reported: 5_000 minimum_wage: 10 diff --git a/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml new file mode 100644 index 0000000000..f7ad5724ac --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/gov/dwp/carers_allowance_earnings_limit.yaml @@ -0,0 +1,257 @@ +# Carer's Allowance earnings limit. SSCBA 1992 s. 70(1)(b) bars anyone +# gainfully employed; reg. 8(1) of the Social Security (Invalid Care +# Allowance) Regulations 1976 treats a person as gainfully employed when net +# earnings exceed £204 a week from 6 April 2026 (SI 2026/218). Net earnings +# are gross pay less income tax, primary Class 1 contributions and half of +# pension contributions (Computation of Earnings Regulations 1996 reg. +# 10(4)), and trading profit less a notional basic-rate tax and main-rate +# Class 4 (regs. 13(4) and 14). A fraction of a penny is disregarded (reg. 5). +# 2026-27: personal allowance and NI primary threshold £12,570, employee +# Class 1 main rate 8%, Class 4 lower profits limit £12,570 at 6%, basic rate +# 20%. Carer's Allowance £86.45 a week. + +- name: Case 1, pay of £8,000 (£153.85 a week, no tax or NI) is under the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 8_000 + care_hours: 40 + country: ENGLAND + output: + carers_allowance_earnings: 8_000 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 2, pay of £15,000 is £275.38 a week net, over the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 15_000 + care_hours: 40 + country: ENGLAND + output: + # Tax 20% x 2,430 = 486; NI 8% x 2,430 = 194.40. + carers_allowance_earnings: 15_000 - 486 - 194.40 + is_gainfully_employed_for_carers_allowance: true + carers_allowance_pre_overlap: 0 + carers_allowance: 0 + +- name: Case 3, earnings of exactly £204 a week have not exceeded the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 204 * 52 + care_hours: 40 + country: ENGLAND + output: + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 4, one pound a year more is over the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 204 * 52 + 1 + care_hours: 40 + country: ENGLAND + output: + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 5, half of pension contributions comes off earnings. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 11_000 + employee_pension_contributions: 1_000 + care_hours: 40 + country: ENGLAND + output: + # 11,000 - 500 = 10,500, or £201.92 a week; without the deduction it + # would be £211.54 and over the limit. + carers_allowance_earnings: 10_500 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 6, trading profit of £14,000 is £262.08 a week net, over the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: 14_000 + care_hours: 40 + country: ENGLAND + output: + # Tax 20% x 1,430 = 286; Class 4 6% x 1,430 = 85.80. + carers_allowance_earnings: 14_000 - 286 - 85.80 + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 7, a trading loss counts as nil earnings. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: -5_000 + care_hours: 40 + country: ENGLAND + output: + carers_allowance_earnings: 0 + carers_allowance: 86.45 * 52 + +- name: Case 8, a reported recipient over the limit is not entitled either, but stays a carer for Universal Credit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 15_000 + carers_allowance_reported: 4_000 + country: ENGLAND + output: + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + # UC regulation 30 does not apply the Carer's Allowance earnings limit. + is_carer_for_benefits: true + +- name: Case 9, savings and pension income are not earnings. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + savings_interest_income: 20_000 + private_pension_income: 20_000 + care_hours: 40 + country: ENGLAND + output: + carers_allowance_earnings: 0 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 10, the 2025-26 limit was £196. + period: 2025 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 200 * 52 + care_hours: 40 + country: ENGLAND + output: + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 11, trading profit takes notional basic-rate tax and main-rate Class 4 only. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: 60_000 + care_hours: 40 + country: ENGLAND + output: + # Reg. 14: 20% x (60,000 - 12,570) = 9,486, not the progressive tax; + # 6% x (50,270 - 12,570) = 2,262, with no additional-rate Class 4. + carers_allowance_earnings: 60_000 - 9_486 - 2_262 + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 12, half of personal pension contributions brings trading earnings under the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: 15_000 + capital_allowances: 5_000 + personal_pension_contributions: 7_700 + care_hours: 40 + country: ENGLAND + output: + # 15,000 - 20% x 2,430 (486) - 6% x 2,430 (145.80) - 7,700 / 2 (3,850) + # = 10,518.20, or £202.27 a week. Capital allowances are not deducted + # (reg. 13(6)(a)), though they cut the actual Class 4 to nil. + carers_allowance_earnings: 10_518.20 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 13, a fraction of a penny over £204 is disregarded. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 10_608.26 + care_hours: 40 + country: ENGLAND + output: + # 10,608.26 / 52 = £204.005, which counts as £204.00 (reg. 5). + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 86.45 * 52 + +- name: Case 14, a whole penny over £204 is over the limit. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 10_608.52 + care_hours: 40 + country: ENGLAND + output: + # 10,608.52 / 52 = £204.01. + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 15, pay and trading profit each take the full personal allowance. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 10_000 + self_employment_income: 5_000 + care_hours: 40 + country: ENGLAND + output: + # No tax or Class 1 on £10,000 of pay; no notional tax or Class 4 on + # £5,000 of profit under the full £12,570 (reg. 14(1)). £15,000 is + # £288.46 a week. + carers_allowance_earnings: 15_000 + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 16, the pay's use of the allowance does not shrink the profit's. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 6_000 + self_employment_income: 8_000 + personal_pension_contributions: 6_600 + care_hours: 40 + country: ENGLAND + output: + # 6,000 + 8,000 - 6,600 / 2 = 10,700, or £205.76 a week: over the limit. + # Sharing one allowance would deduct 20% x (8,000 - 6,570) = 286 and + # give £200.27, under it. + carers_allowance_earnings: 10_700 + is_gainfully_employed_for_carers_allowance: true + carers_allowance: 0 + +- name: Case 17, before April 2024 trading earnings are also net of Class 2. + period: 2018 + absolute_error_margin: 0.01 + input: + age: 45 + self_employment_income: 7_000 + personal_pension_contributions: 1_400 + care_hours: 40 + country: ENGLAND + output: + # 2018-19: no tax or Class 4 under the £11,850 allowance and lower + # profits limit; Class 2 at £2.95 for 52 weeks (reg. 14(2)(a)) = 153.40; + # half the pension 700. 7,000 - 153.40 - 700 = 6,146.60, £118.20 a week, + # under the £120 limit. Without Class 2 it would be £121.15, over it. + carers_allowance_earnings: 6_146.60 + is_gainfully_employed_for_carers_allowance: false + carers_allowance: 64.60 * 52 diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml index 248ef3aa10..45d122fe22 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/child_benefit/opt_out.yaml @@ -87,7 +87,7 @@ absolute_error_margin: 0.01 input: people: - person1: {age: 40, adjusted_net_income: 100_000} + person1: {age: 40, adjusted_net_income: 200_000} person2: {age: 8} person3: {age: 5} benunits: @@ -98,8 +98,9 @@ output: child_benefit_entitlement: 2_337.40 child_benefit: 2_337.40 - # (100,000 - 60,000) / (10,000,000 - 60,000) * 2,337.40. - CB_HITC: [9.406036, 0, 0] + # (200,000 - 60,000) / (10,000,000 - 60,000) = 1.41%, rounded down to 1% + # (ITEPA 2003 s. 681C(3)); 1% of £2,337 is £23.37, rounded down to £23. + CB_HITC: [23, 0, 0] - name: Case 9, the configurable opt-out threshold includes its exact boundary. period: 2026 @@ -111,9 +112,10 @@ gov.hmrc.child_benefit.opt_out_charge_share: 0.5 output: child_benefit: [1_000, 0, 0] - CB_HITC: [499.95, 0, 0] + # 49.995% is rounded down to 49% (s. 681C(3)): £490. + CB_HITC: [490, 0, 0] -- name: Case 10, a zero behavioural threshold still requires a positive charge. +- name: Case 10, a zero behavioural threshold still requires a positive continuous charge share. period: 2026 input: adjusted_net_income: [59_999, 60_000, 60_001] diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml index 6f2f1f0f16..e8e0e20c0e 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances.yaml @@ -94,21 +94,27 @@ period: 2020 absolute_error_margin: 0 input: - tax_band: "BASIC" + # 2020 parameters: taxable income £27,500, under the £37,500 basic rate limit. + adjusted_net_income: 40_000 + allowances: 12_500 output: savings_allowance: 1000 - name: Savings allowance at higher rate period: 2020 absolute_error_margin: 0 input: - tax_band: "HIGHER" + # Taxable income £47,500, between £37,500 and £150,000. + adjusted_net_income: 60_000 + allowances: 12_500 output: savings_allowance: 500 - name: No savings allowance at additional rate period: 2020 absolute_error_margin: 0 input: - tax_band: "ADDITIONAL" + # Taxable income £200,000, above the £150,000 higher rate limit. + adjusted_net_income: 200_000 + allowances: 0 output: savings_allowance: 0 - name: Allowances by default diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml new file mode 100644 index 0000000000..3ef8fabad3 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/allowances/savings_allowance.yaml @@ -0,0 +1,200 @@ +# Personal Savings Allowance, ITA 2007 s. 12B. A Scottish taxpayer's higher- +# and additional-rate income is what would be charged at those rates if they +# were not a Scottish taxpayer (s. 12B(8)(a)(iv) and (b)(iv)), so the +# allowance follows the rest-of-UK bands: in 2026-27, £37,700 and £125,140 of +# taxable income, with a £12,570 personal allowance tapered above £100,000. +# Savings income is taxed at the UK savings rates. + +- name: Case 1, a Scottish advanced-rate taxpayer below £125,140 gets £500. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 80_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: SCOTLAND + output: + # Taxable income 81,000 - 12,570 = 68,430: Scottish advanced rate, but + # UK higher rate. (1,000 - 500) x 40% = 200. + pays_scottish_income_tax: true + tax_band: ADDITIONAL + savings_allowance: 500 + savings_income_tax: 200 + +- name: Case 2, a Scottish higher-rate taxpayer below £37,700 gets £1,000. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 45_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: SCOTLAND + output: + # Taxable income 46,000 - 12,570 = 33,430: Scottish higher rate, but UK + # basic rate, so the whole £1,000 of interest is covered. + tax_band: HIGHER + savings_allowance: 1_000 + savings_income_tax: 0 + +- name: Case 3, an English taxpayer with the same income is unchanged. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 80_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + tax_band: HIGHER + savings_allowance: 500 + savings_income_tax: 200 + +- name: Case 4, a Scottish taxpayer above £125,140 gets nothing. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 130_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: SCOTLAND + output: + # The personal allowance is tapered to nil, so taxable income is + # 131,000, above £125,140. + savings_allowance: 0 + +- name: Case 5, 2016-17 bands for a higher-rate taxpayer. + period: 2016 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 60_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Taxable income above the £32,000 basic rate limit but below £150,000. + savings_allowance: 500 + +- name: Case 6, 2016-17 bands for an additional-rate taxpayer. + period: 2016 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 200_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + savings_allowance: 0 + +- name: Case 7, the 2016-17 rules also apply to period 2017. + period: 2017 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 60_000 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + savings_allowance: 500 + +- name: Case 8, taxable income exactly at the basic rate limit has no higher-rate income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 49_270 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + # 50,270 - 12,570 = 37,700: nothing above the limit (ITA 2007 s. 10(2)-(3)). + savings_allowance: 1_000 + savings_income_tax: 0 + +- name: Case 9, a pound over the basic rate limit is higher-rate income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 49_271 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + savings_allowance: 500 + +- name: Case 10, taxable income exactly at the higher rate limit has no additional-rate income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 124_140 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Adjusted net income 125,140 tapers the personal allowance to nil, so + # taxable income is 125,140. + savings_allowance: 500 + +- name: Case 11, a pound over the higher rate limit is additional-rate income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 40 + employment_income: 124_141 + savings_interest_income: 1_000 + households: + household: + members: [person] + country: ENGLAND + output: + savings_allowance: 0 diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml new file mode 100644 index 0000000000..08d5fcf444 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.yaml @@ -0,0 +1,63 @@ +# A miscellaneous loss is relieved only against miscellaneous income +# (ITA 2007 s. 152(4)). 2026-27: personal allowance £12,570, basic rate 20% +# to £50,270, higher rate 40%. + +- name: Case 1, miscellaneous income is taxed with other income. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 30_000 + miscellaneous_income: 5_000 + output: + taxable_miscellaneous_income: 5_000 + # (35,000 - 12,570) x 20%. + income_tax: 4_486 + +- name: Case 2, a miscellaneous loss does not reduce tax on pay. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 50_000 + miscellaneous_income: -10_000 + output: + taxable_miscellaneous_income: 0 + # (50,000 - 12,570) x 20%, as with no loss. + income_tax: 7_486 + +- name: Case 3, nor adjusted net income. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 45 + employment_income: 110_000 + miscellaneous_income: -20_000 + output: + taxable_miscellaneous_income: 0 + adjusted_net_income: 110_000 + # Personal allowance 12,570 - (110,000 - 100,000) / 2 = 7,570; taxable + # 102,430: 37,700 x 20% + 64,730 x 40% = 33,432. + income_tax: 33_432 + +- name: Case 4, total income ignores a miscellaneous loss. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 70 + private_pension_income: 40_000 + miscellaneous_income: -10_000 + output: + total_income: 40_000 + +- name: Case 5, a miscellaneous loss does not reduce Universal Credit earnings. + period: 2026 + absolute_error_margin: 0.01 + input: + age: 30 + employment_income: 30_000 + miscellaneous_income: -10_000 + output: + # 30,000 less income tax (30,000 - 12,570) x 20% = 3,486 and primary + # Class 1 (30,000 - 12,570) x 8% = 1,394.40. + uc_individual_earned_income_before_mif: 30_000 - 3_486 - 1_394.40 diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml index c20990a727..ecfd90acb5 100644 --- a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/charges/child_benefit_hitc.yaml @@ -47,6 +47,150 @@ members: [person1, person2, person3, person4] output: child_benefit: 2_337.40 - CB_HITC: [2_337.40, 0, 0, 0] + # s. 681C(3): the Child Benefit total is rounded down to £2,337. + CB_HITC: [2_337, 0, 0, 0] income_tax_pre_charges: [35_742, 4_486, 0, 0] - income_tax: [38_079.40, 4_486, 0, 0] + income_tax: [38_079, 4_486, 0, 0] + +# ITEPA 2003 s. 681C(3): the Child Benefit total and the charge are rounded +# down to whole pounds, and the appropriate percentage ((ANI - £60,000) / £200 +# from 2024-25) to a whole number. +- name: Case 6, the percentage, the Child Benefit and the charge are rounded down. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 71_336 + child: + age: 5 + benunit: + members: [parent, child] + output: + # (71,336 - 60,000) / 200 = 56.68, so 56%; £1,406.60 becomes £1,406; + # 56% of £1,406 is £787.36, so £787. + child_benefit: 1_406.60 + CB_HITC: [787, 0] + +- name: Case 7, a whole-number percentage is not rounded below itself. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 70_000 + child: + age: 5 + benunit: + members: [parent, child] + output: + # (70,000 - 60,000) / 200 = 50%; 50% of £1,406 is £703. + CB_HITC: [703, 0] + +- name: Case 8, just under the next £200 step keeps the lower percentage. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 70_199 + child: + age: 5 + benunit: + members: [parent, child] + output: + # 50.995% rounds down to 50%: £703. + CB_HITC: [703, 0] + +- name: Case 9, at or above £80,000 the whole Child Benefit, in whole pounds. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 90_000 + child1: + age: 5 + child2: + age: 3 + benunit: + members: [parent, child1, child2] + output: + child_benefit: 2_337.40 + CB_HITC: [2_337, 0, 0] + +- name: Case 10, the 2023-24 rules, 1% per £100 above £50,000. + period: 2023 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 55_550 + child: + age: 5 + benunit: + members: [parent, child] + child_benefit: 1_000 + output: + # (55,550 - 50,000) / 100 = 55.5%, so 55%; 55% of £1,000 is £550. + CB_HITC: [550, 0] + +- name: Case 11, the Child Benefit total is rounded down before the percentage applies. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 79_800 + child: + age: 5 + benunit: + members: [parent, child] + output: + # (79,800 - 60,000) / 200 = 99%; 99% of £1,406 is £1,391.94, so £1,391. + # Rounding only the charge (99% of £1,406.60 = £1,392.53) would give £1,392. + CB_HITC: [1_391, 0] + +- name: Case 12, an income on an exact £200 step gets that many points. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 60_200 + child: + age: 5 + benunit: + members: [parent, child] + output: + # 1% of £1,406 is £14.06, so £14. + CB_HITC: [14, 0] + +- name: Case 13, four children's £4,199 is not rounded down to £4,198. + period: 2026 + absolute_error_margin: 0 + input: + people: + parent: + age: 40 + employment_income: 90_000 + child1: + age: 2 + child2: + age: 4 + child3: + age: 6 + child4: + age: 8 + benunit: + members: [parent, child1, child2, child3, child4] + output: + # 52 x (27.05 + 3 x 17.90) = 4,199. + CB_HITC: [4_199, 0, 0, 0, 0] diff --git a/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml new file mode 100644 index 0000000000..7312c6d5bd --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/gov/hmrc/income_tax/property_income_band_base.yaml @@ -0,0 +1,141 @@ +# Savings and dividend income are the highest part of total income (ITA 2007 +# s. 16), so property income, like pay, fills the bands below them. 2026-27, +# single adult: personal allowance £12,570, basic rate limit £37,700, savings +# allowance £500 for a higher-rate taxpayer, dividend allowance £500 at +# 10.75% / 35.75%. + +- name: Case 1, dividends sit above property income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + property_income: 40_000 + dividend_income: 20_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Non-savings taxable income 40,000 - 12,570 = 27,430. The £500 dividend + # allowance and 9,770 fill the basic-rate band (9,770 x 10.75% = + # 1,050.275); 9,730 is at 35.75% (3,478.475). Total 4,528.75, as for pay. + non_savings_taxable_income: 27_430 + dividend_income_tax: 4_528.75 + income_tax: 5_486 + 4_528.75 + +- name: Case 2, savings sit above property income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + property_income: 40_000 + savings_interest_income: 20_000 + households: + household: + members: [person] + country: ENGLAND + output: + # £500 savings allowance, then 9,770 at 20% (1,954) to the basic rate + # limit and 9,730 at 40% (3,892). + savings_income_tax: 5_846 + +- name: Case 3, pay and property income together form the band base. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + employment_income: 10_000 + property_income: 30_000 + dividend_income: 20_000 + households: + household: + members: [person] + country: ENGLAND + output: + # The allowance covers the pay first: 0 earned and 27,430 property + # income taxable, so dividends sit on 27,430 as in Case 1. + taxable_property_income_after_allowances: 27_430 + non_savings_taxable_income: 27_430 + dividend_income_tax: 4_528.75 + +- name: Case 4, a Scottish taxpayer's dividends use UK bands above the property income. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + property_income: 40_000 + dividend_income: 20_000 + households: + household: + members: [person] + country: SCOTLAND + output: + dividend_income_tax: 4_528.75 + +- name: Case 5, from 2027-28 property income has its own rates but still sits below dividends. + period: 2027 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + property_income: 40_000 + dividend_income: 20_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Property income at the 22% property basic rate: 27,430 x 22% = 6,034.60. + property_income_tax: 6_034.60 + dividend_income_tax: 4_528.75 + +- name: Case 6, the annual allowance charge reads the rate on property income too. + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + employment_income: 10_000 + property_income: 60_000 + employee_pension_contributions: 70_000 + households: + household: + members: [person] + country: ENGLAND + output: + # Relief is limited to the £10,000 of pay, so taxable income is + # 70,000 - 12,570 - 10,000 = 47,430, in the higher-rate band; the + # £10,000 over the £60,000 annual allowance is charged at 40%. + taxed_income: 47_430 + personal_pension_contributions_tax: 4_000 + +- name: Case 7, Marriage Allowance still comes off the band base (pinned; #1947). + period: 2026 + absolute_error_margin: 0.01 + input: + people: + person: + age: 45 + employment_income: 20_000 + property_income: 10_000 + marriage_allowance: 1_260 + households: + household: + members: [person] + country: ENGLAND + output: + # earned_taxable_income subtracts Marriage Allowance, which the law gives + # as a tax reduction (ITA 2007 s. 55B), so the base is 30,000 - 12,570 - + # 1,260 = 16,170 rather than 17,430. Pinned here until #1947 is fixed. + non_savings_taxable_income: 16_170 + non_savings_income_after_allowances: 17_430 diff --git a/policyengine_uk/tests/policy/integration/income.yaml b/policyengine_uk/tests/policy/integration/income.yaml index dc8b7ea721..798953e25f 100644 --- a/policyengine_uk/tests/policy/integration/income.yaml +++ b/policyengine_uk/tests/policy/integration/income.yaml @@ -100,8 +100,13 @@ members: [person1, person2] output: property_income_tax: [17_792, 6_776.20] - savings_income_tax: [0, 0] - income_tax: [17_792, 6_776.20] + # Savings sit above the property income (ITA 2007 s. 16). Person 1: + # 75,900 - 12,570 = 63,330 of non-savings taxable income, so a higher- + # rate taxpayer with a £500 savings allowance and no starting rate: + # (2,900 - 500) x 40% = 960. Person 2: 46,451 - 12,570 = 33,881, a + # basic-rate taxpayer with £1,000: (1,525 - 1,000) x 20% = 105. + savings_income_tax: [960, 105] + income_tax: [18_752, 6_881.20] - name: Household net income can be negative period: 2020 diff --git a/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py new file mode 100644 index 0000000000..497d9ad7f9 --- /dev/null +++ b/policyengine_uk/tests/test_carers_allowance_earnings_limit_properties.py @@ -0,0 +1,170 @@ +"""Carer's Allowance earnings limit. + +SSCBA 1992 s. 70(1)(b) bars a person who is gainfully employed, and reg. 8(1) +of the Social Security (Invalid Care Allowance) Regulations 1976 treats a +person as gainfully employed when their earnings exceed the limit (£204 a +week from 6 April 2026). Earnings are net of income tax, primary Class 1 (or, +for trading profit, Class 4) contributions and half of pension contributions +(Computation of Earnings Regulations 1996 regs. 10(4), 13 and 14; reg. 14(2)(a), +which deducted Class 2, was omitted from 6 April 2024). + +Invariants, for a carer in England in 2026 caring 35 hours or more, with pay, +trading profit, employee pension contributions and unearned income drawn +freely: + +1. Bounds: 0 <= carers_allowance_earnings <= pay + max(0, profit). +2. Differential: the test is passed exactly when + carers_allowance_earnings / 52 <= 204, and the allowance is the full rate + then and nil otherwise. +3. Monotonic: raising pay never lowers net earnings, so a person over the + limit stays over it. +4. Unearned income is not earnings: savings interest and pension income + leave carers_allowance_earnings unchanged. +""" + +import numpy as np +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation +from policyengine_uk.utils.class_2 import class_2_contribution_weeks + +LIMIT = 204 +RATE = 86.45 +WEEKS = 52 + + +def _person(pay, profit, pension, savings=0.0, private_pension=0.0): + situation = { + "people": { + "carer": { + "age": {"2026": 45}, + "employment_income": {"2026": pay}, + "self_employment_income": {"2026": profit}, + "employee_pension_contributions": {"2026": pension}, + "savings_interest_income": {"2026": savings}, + "private_pension_income": {"2026": private_pension}, + "care_hours": {"2026": 40}, + } + }, + "benunits": {"benunit": {"members": ["carer"]}}, + "households": { + "household": {"members": ["carer"], "country": {"2026": "ENGLAND"}} + }, + } + sim = Simulation(situation=situation) + return { + name: float(np.asarray(sim.calculate(name, 2026))[0]) + for name in ( + "carers_allowance_earnings", + "is_gainfully_employed_for_carers_allowance", + "carers_allowance", + ) + } + + +money = st.floats(min_value=0, max_value=60_000, allow_nan=False) +profit_money = st.floats(min_value=-20_000, max_value=60_000, allow_nan=False) + + +@settings( + max_examples=12, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given(pay=money, profit=profit_money, pension_share=st.floats(0, 0.3)) +def test_bounds_and_differential(pay, profit, pension_share): + pension = pay * pension_share + result = _person(pay, profit, pension) + earnings = result["carers_allowance_earnings"] + assert -0.01 <= earnings <= pay + max(0, profit) + 0.01 + # Regulation 5: a fraction of a penny is disregarded. + over = np.floor(earnings / WEEKS * 100 + 1e-6) / 100 > LIMIT + assert bool(result["is_gainfully_employed_for_carers_allowance"]) == over + expected = 0 if over else RATE * WEEKS + assert abs(result["carers_allowance"] - expected) < 0.01 + + +@settings( + max_examples=8, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given(pay=money, rise=st.floats(min_value=1, max_value=20_000)) +def test_net_earnings_never_fall_as_pay_rises(pay, rise): + lower = _person(pay, 0, 0) + higher = _person(pay + rise, 0, 0) + assert ( + higher["carers_allowance_earnings"] >= lower["carers_allowance_earnings"] - 0.01 + ) + if lower["is_gainfully_employed_for_carers_allowance"]: + assert higher["is_gainfully_employed_for_carers_allowance"] + + +@settings( + max_examples=6, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given( + pay=st.floats(min_value=0, max_value=15_000), + savings=money, + private_pension=money, +) +def test_unearned_income_is_not_earnings(pay, savings, private_pension): + alone = _person(pay, 0, 0) + with_unearned = _person(pay, 0, 0, savings=savings, private_pension=private_pension) + assert ( + abs( + alone["carers_allowance_earnings"] + - with_unearned["carers_allowance_earnings"] + ) + < 0.01 + ) + + +@pytest.mark.parametrize("year", [2016, 2017, 2018, 2026]) +@pytest.mark.parametrize("country", ["ENGLAND", "SCOTLAND"]) +def test_trading_profit_takes_the_basic_rate_in_every_year(year, country): + """Reg. 14(1): the basic rate, or for a Scottish taxpayer the Scottish + basic rate, which were both 20% in each of these years. Before 2018-19 the + Scottish scale has no starter rate, so reading its second bracket by + position at an instant would pick the higher rate. Before 2024-25 reg. + 14(2)(a) also deducted Class 2: £30,000 is above the small profits + threshold in each year, so a full year of the flat rate.""" + profit = 30_000 + simulation = Simulation( + situation={ + "people": { + "carer": { + "age": {year: 45}, + "self_employment_income": {year: profit}, + "care_hours": {year: 40}, + } + }, + "benunits": {"family": {"members": ["carer"]}}, + "households": {"home": {"members": ["carer"], "country": {year: country}}}, + } + ) + hmrc = simulation.tax_benefit_system.parameters(year).gov.hmrc + allowance = hmrc.income_tax.allowances.personal_allowance.amount + class_4 = hmrc.national_insurance.class_4 + class_2 = hmrc.national_insurance.class_2 + class_2_weeks = class_2_contribution_weeks(year) + expected = ( + profit + - 0.20 * (profit - allowance) + - class_2.flat_rate * class_2_weeks + - class_4.rates.main + * ( + min(profit, class_4.thresholds.upper_profits_limit) + - class_4.thresholds.lower_profits_limit + ) + ) + assert bool(simulation.calculate("pays_scottish_income_tax", year)[0]) == ( + country == "SCOTLAND" + ) + assert simulation.calculate("carers_allowance_earnings", year)[0] == pytest.approx( + expected, abs=0.01 + ) diff --git a/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py b/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py new file mode 100644 index 0000000000..7e448efab2 --- /dev/null +++ b/policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py @@ -0,0 +1,182 @@ +"""High Income Child Benefit Charge rounding (ITEPA 2003 s. 681C(3)). + +The charge is the appropriate percentage of the Child Benefit concerned: +(ANI - L) / X percent, capped at 100% (s. 681C(2); L = £60,000 and X = £200 +from 2024-25). Subsection (3) rounds the Child Benefit total and the charge +down to whole pounds, and the percentage down to a whole number. + +Invariants, for adjusted net income in whole pounds and Child Benefit totals in whole +pence (the amounts the model produces), including their float32 storage: + +1. The charge is a whole number of pounds. +2. Bounds: 0 <= charge <= Child Benefit rounded down to pounds. +3. Differential: the charge equals floor(points / 100 x floor(CB)), where + points = min(100, max(0, floor((ANI - L) / X))), computed here from the + statute in integer arithmetic. +4. Monotonic: with the Child Benefit fixed, a higher income never lowers the + charge. +5. Precision: every exact step, held as float32 as in a microsimulation, + gives exactly that many points, under the statutory band and under wider + tapers a reform can set; a penny below a step gives one point fewer. +6. The model's charge from a full simulation matches the statute, with one to + four children. +""" + +import numpy as np +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation +from policyengine_uk.utils.scenario import Scenario +from policyengine_uk.utils.child_benefit import ( + child_benefit_charge_percentage, + round_down_to_pounds, +) + +L = 60_000 +X = 200 +END = L + 100 * X + + +def statutory_charge(ani_pounds: int, child_benefit_pence: int) -> int: + """s. 681C in integer arithmetic: whole pounds, whole percentage points.""" + if ani_pounds <= L: + return 0 + points = min(100, (ani_pounds - L) // X) + pounds = child_benefit_pence // 100 + return (points * pounds) // 100 + + +def model_charge(ani: float, child_benefit: float) -> float: + percentage = child_benefit_charge_percentage(np.array([ani]), L, END) + return float( + round_down_to_pounds(percentage * round_down_to_pounds(child_benefit))[0] + ) + + +incomes = st.integers(min_value=0, max_value=200_000) +benefit_pence = st.integers(min_value=0, max_value=1_000_000) + + +@settings(max_examples=500, deadline=None) +@given(ani=incomes, cb_pence=benefit_pence) +def test_charge_matches_the_statute(ani, cb_pence): + child_benefit = cb_pence / 100 + charge = model_charge(ani, child_benefit) + assert charge == int(charge) + assert 0 <= charge <= cb_pence // 100 + assert charge == statutory_charge(ani, cb_pence) + + +@settings(max_examples=300, deadline=None) +@given( + ani=incomes, rise=st.integers(min_value=0, max_value=50_000), cb_pence=benefit_pence +) +def test_a_higher_income_never_lowers_the_charge(ani, rise, cb_pence): + child_benefit = cb_pence / 100 + assert model_charge(ani + rise, child_benefit) >= model_charge(ani, child_benefit) + + +@settings(max_examples=8, deadline=None) +@given(pay=st.integers(min_value=55_000, max_value=90_000), children=st.integers(1, 4)) +def test_simulated_charge_matches_the_statute(pay, children): + people = {"parent": {"age": {"2026": 40}, "employment_income": {"2026": pay}}} + for i in range(children): + people[f"child{i}"] = {"age": {"2026": 3 + i}} + members = list(people) + sim = Simulation( + situation={ + "people": people, + "benunits": {"b": {"members": members}}, + "households": {"h": {"members": members, "country": {"2026": "ENGLAND"}}}, + } + ) + ani = float(sim.calculate("adjusted_net_income", 2026)[0]) + child_benefit = float(sim.calculate("child_benefit", 2026)[0]) + charge = float(sim.calculate("CB_HITC", 2026)[0]) + assert charge == statutory_charge(int(ani), round(child_benefit * 100)) + + +def test_every_exact_step_gives_exactly_its_points_in_float32(): + for points in range(0, 101): + income = np.array([L + X * points], dtype=np.float32) + assert child_benefit_charge_percentage(income, L, END)[0] == points / 100 + + +def test_a_float32_annual_amount_is_taken_to_the_penny_first(): + # Four children's £4,199 accumulated by month and stored as float32. + assert round_down_to_pounds(np.float32(4198.99951171875)) == 4199 + assert round_down_to_pounds(np.float32(1406.5999755859375)) == 1406 + + +# Structural reforms are chosen when the simulation is built, so the switch has +# to be set before data load. +HOUSEHOLD_BASED = Scenario( + applied_before_data_load=True, + parameter_changes={"gov.contrib.conservatives.cb_hitc_household": {"2025": True}}, +) + + +def _charge(pay: list[int], scenario=None) -> float: + people = { + f"parent{i}": {"age": {2026: 40}, "employment_income": {2026: p}} + for i, p in enumerate(pay) + } + people["child"] = {"age": {2026: 5}} + situation = { + "people": people, + "benunits": {"family": {"members": list(people)}}, + "households": {"home": {"members": list(people), "country": {2026: "ENGLAND"}}}, + } + return float( + Simulation(situation=situation, scenario=scenario) + .calculate("CB_HITC", 2026) + .sum() + ) + + +def test_the_household_based_reform_rounds_as_the_baseline_does(): + """A single parent's household income is their own, so the reform charges + what the baseline does: 99% of £1,406 at £79,800 is £1,391, not £1,392.53.""" + assert _charge([79_800]) == 1_391 + assert _charge([79_800], HOUSEHOLD_BASED) == 1_391 + + +def test_the_household_based_reform_charges_on_combined_income(): + """A couple on £50,000 and £30,000 pay nothing in the baseline; on £80,000 + combined the reform takes the whole Child Benefit, in whole pounds.""" + assert _charge([50_000, 30_000]) == 0 + assert _charge([50_000, 30_000], HOUSEHOLD_BASED) == 1_406 + + +WIDE_TAPER_ENDS = [END, 2_000_000, 10_000_000] + + +def test_exact_steps_and_the_income_just_below_under_wide_tapers(): + """A reform can widen the band (tests use £2,000,000 and £10,000,000). + Each step is width / 100. The largest float32 income below a step stays + one point lower: no slack rounds it up. (Above about £524,288, float32 + cannot hold a penny below a step, so the nearest float32 below it is + used.)""" + for end in WIDE_TAPER_ENDS: + step = (end - L) / 100 + for points in [1, 2, 37, 99, 100]: + on_step = np.float32(L + step * points) + below = np.nextafter(on_step, np.float32(0)) + assert ( + child_benefit_charge_percentage(np.array([on_step]), L, end)[0] + == points / 100 + ) + assert ( + child_benefit_charge_percentage(np.array([below]), L, end)[0] + == (points - 1) / 100 + ) + + +def test_a_penny_below_the_first_wide_taper_step_is_not_charged(): + # £79,399.99 is £79,399.9921875 in float32: 0 points under a £60,000 to + # £2,000,000 band, so no charge on one child's £1,406. + income = np.array([79_399.99], dtype=np.float32) + percentage = child_benefit_charge_percentage(income, L, 2_000_000) + assert percentage[0] == 0 + assert round_down_to_pounds(percentage * round_down_to_pounds(1_406.60))[0] == 0 diff --git a/policyengine_uk/tests/test_child_benefit_opt_out.py b/policyengine_uk/tests/test_child_benefit_opt_out.py index 012ef7ceca..54e235fea5 100644 --- a/policyengine_uk/tests/test_child_benefit_opt_out.py +++ b/policyengine_uk/tests/test_child_benefit_opt_out.py @@ -76,7 +76,9 @@ def test_cliff_charge_has_no_division_warnings(income, opts_out, taper_end): charge = simulation.calculate("CB_HITC", 2026).sum() entitlement = simulation.calculate("child_benefit_entitlement", 2026)[0] assert payment == (0 if opts_out and income > 60_000 else entitlement) - assert charge == (payment if income > 60_000 else 0) + # A cliff charges the whole Child Benefit, rounded down to whole pounds + # (ITEPA 2003 s. 681C(3)(a)). + assert charge == (np.floor(payment) if income > 60_000 else 0) @pytest.mark.usefixtures("cloned_uk_tax_benefit_system") @@ -106,9 +108,13 @@ def test_cliff_charge_has_no_division_warnings(income, opts_out, taper_end): 0, ), ( - {"gov.hmrc.income_tax.charges.CB_HITC.phase_out_end": {"2026": 10_000_000}}, + # A softer taper: (100,000 - 60,000) / (2,000,000 - 60,000) = 2.06%, + # rounded down to 2% (ITEPA 2003 s. 681C(3)(b)); 2% of £2,337 is + # £46.74, rounded down to £46. Payment resumes because the share is + # below the opt-out threshold. + {"gov.hmrc.income_tax.charges.CB_HITC.phase_out_end": {"2026": 2_000_000}}, 2_337.40, - 2_337.40 * 40_000 / 9_940_000, + 46, ), ], ) diff --git a/policyengine_uk/tests/test_council_tax_reduction_carer_income_properties.py b/policyengine_uk/tests/test_council_tax_reduction_carer_income_properties.py index 8795cdd8c6..e317bf68e5 100644 --- a/policyengine_uk/tests/test_council_tax_reduction_carer_income_properties.py +++ b/policyengine_uk/tests/test_council_tax_reduction_carer_income_properties.py @@ -192,6 +192,12 @@ def situation(families): "private_pension_income": {YEAR: float(pension) * (j == 0)}, } if j == fam["carer"]: + # Keep the carer's pay within the Carer's Allowance + # earnings limit (£204 a week in 2026-27), so the + # carer stays entitled; £10,000 is £192 a week. + person["employment_income"] = { + YEAR: min(a.get("employment_income", 0.0), 10_000.0) + } if fam["by_hours"]: person["care_hours"] = {YEAR: 35} else: diff --git a/policyengine_uk/tests/test_dataset_fiscal_year.py b/policyengine_uk/tests/test_dataset_fiscal_year.py new file mode 100644 index 0000000000..abdbafa419 --- /dev/null +++ b/policyengine_uk/tests/test_dataset_fiscal_year.py @@ -0,0 +1,112 @@ +"""`fiscal_year` labelling on UKSingleYearDataset. + +The year label is the year the financial year starts, so FRS 2024/25 data +is labelled 2024. A dataset loaded from a file takes its period from the +file; one built from DataFrames takes it from the argument. +""" + +import os +import subprocess +import sys +import textwrap +import warnings + +import pandas as pd +import pytest + +from policyengine_uk.data.dataset_schema import UKSingleYearDataset + + +@pytest.fixture +def frames(): + person = pd.DataFrame( + {"person_id": [1], "person_benunit_id": [1], "person_household_id": [1]} + ) + benunit = pd.DataFrame({"benunit_id": [1]}) + household = pd.DataFrame({"household_id": [1]}) + return person, benunit, household + + +def test_fiscal_year_sets_the_time_period(frames): + person, benunit, household = frames + dataset = UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=2024 + ) + assert dataset.time_period == "2024" + + +def test_omitting_fiscal_year_warns(frames): + person, benunit, household = frames + with pytest.warns(FutureWarning, match="fiscal_year was not given"): + dataset = UKSingleYearDataset( + person=person, benunit=benunit, household=household + ) + assert dataset.time_period == str(UKSingleYearDataset.DEFAULT_FISCAL_YEAR) + + +def test_fiscal_year_alongside_a_file_path_warns(tmp_path, frames): + person, benunit, household = frames + file_path = tmp_path / "dataset.h5" + UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=2024 + ).save(file_path) + + with pytest.warns(UserWarning, match="ignored when loading from a file"): + dataset = UKSingleYearDataset(file_path=file_path, fiscal_year=2030) + + # The file's own period wins, so the ignored argument cannot mislabel it. + assert dataset.time_period == "2024" + + +def test_loading_without_fiscal_year_does_not_warn(tmp_path, frames): + person, benunit, household = frames + file_path = tmp_path / "dataset.h5" + UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=2024 + ).save(file_path) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert UKSingleYearDataset(file_path=file_path).time_period == "2024" + + +# pytest.warns overrides the active filters, so the tests above would pass +# even if importing the package silenced warnings process-wide. Run a fresh +# interpreter under Python's default filters, as an ordinary caller would. +FRESH_PROCESS_SCRIPT = textwrap.dedent( + """ + import sys + import tempfile + from pathlib import Path + + import pandas as pd + + import policyengine_uk # noqa: F401 (import side effects are under test) + from policyengine_uk.data.dataset_schema import UKSingleYearDataset + + frames = dict( + person=pd.DataFrame( + {"person_id": [1], "person_benunit_id": [1], "person_household_id": [1]} + ), + benunit=pd.DataFrame({"benunit_id": [1]}), + household=pd.DataFrame({"household_id": [1]}), + ) + UKSingleYearDataset(**frames) + with tempfile.TemporaryDirectory() as directory: + file_path = Path(directory) / "dataset.h5" + UKSingleYearDataset(**frames, fiscal_year=2024).save(file_path) + UKSingleYearDataset(file_path=file_path, fiscal_year=2030) + """ +) + + +def test_warnings_reach_a_fresh_process_under_default_filters(): + result = subprocess.run( + [sys.executable, "-c", FRESH_PROCESS_SCRIPT], + capture_output=True, + text=True, + env={k: v for k, v in os.environ.items() if k != "PYTHONWARNINGS"}, + ) + assert result.returncode == 0, result.stderr + assert "FutureWarning: fiscal_year was not given" in result.stderr + assert "UserWarning: fiscal_year is ignored when loading" in result.stderr diff --git a/policyengine_uk/tests/test_miscellaneous_loss_properties.py b/policyengine_uk/tests/test_miscellaneous_loss_properties.py new file mode 100644 index 0000000000..e3f08a5734 --- /dev/null +++ b/policyengine_uk/tests/test_miscellaneous_loss_properties.py @@ -0,0 +1,72 @@ +"""A miscellaneous loss is relieved only against miscellaneous income +(ITA 2007 s. 152(4)). + +Invariants: + +1. For any pay, savings, dividends and pension income, a miscellaneous loss + leaves Income Tax, adjusted net income and total income exactly as they are + with no miscellaneous income. +2. For any pay, positive miscellaneous income is taxable in full. +""" + +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation + + +def _calc(misc, pay, savings, dividends, pension): + situation = { + "people": { + "p": { + "age": {"2026": 50}, + "employment_income": {"2026": pay}, + "savings_interest_income": {"2026": savings}, + "dividend_income": {"2026": dividends}, + "private_pension_income": {"2026": pension}, + "miscellaneous_income": {"2026": misc}, + } + }, + "benunits": {"b": {"members": ["p"]}}, + "households": {"h": {"members": ["p"], "country": {"2026": "ENGLAND"}}}, + } + sim = Simulation(situation=situation) + return { + name: float(sim.calculate(name, 2026)[0]) + for name in ( + "income_tax", + "adjusted_net_income", + "total_income", + "taxable_miscellaneous_income", + ) + } + + +amounts = st.floats(min_value=0, max_value=150_000, allow_nan=False) + + +@settings(max_examples=25, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@example(loss=20_000, pay=110_000, savings=0, dividends=0, pension=0) +@given( + loss=st.floats(min_value=1, max_value=500_000, allow_nan=False), + pay=amounts, + savings=st.floats(min_value=0, max_value=20_000), + dividends=st.floats(min_value=0, max_value=20_000), + pension=st.floats(min_value=0, max_value=40_000), +) +def test_a_miscellaneous_loss_changes_nothing_else( + loss, pay, savings, dividends, pension +): + with_loss = _calc(-loss, pay, savings, dividends, pension) + without = _calc(0, pay, savings, dividends, pension) + assert with_loss["taxable_miscellaneous_income"] == 0 + assert abs(with_loss["income_tax"] - without["income_tax"]) < 0.01 + assert abs(with_loss["adjusted_net_income"] - without["adjusted_net_income"]) < 0.01 + assert abs(with_loss["total_income"] - without["total_income"]) < 0.01 + + +@settings(max_examples=6, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given(income=st.floats(min_value=0, max_value=100_000, allow_nan=False), pay=amounts) +def test_positive_miscellaneous_income_is_taxable_in_full(income, pay): + result = _calc(income, pay, 0, 0, 0) + assert abs(result["taxable_miscellaneous_income"] - income) < 0.01 diff --git a/policyengine_uk/tests/test_ni_threshold_freeze.py b/policyengine_uk/tests/test_ni_threshold_freeze.py index 0403c92137..43cceb3712 100644 --- a/policyengine_uk/tests/test_ni_threshold_freeze.py +++ b/policyengine_uk/tests/test_ni_threshold_freeze.py @@ -26,7 +26,7 @@ 4. Alignment, in test_threshold_indexation.py: the UEL and UPL with the higher rate threshold in every year 2021-2040, and the PT and LPL with the personal allowance from 2023. The 2021-22 and 2022-23 lower thresholds - are intended exceptions, and the 2023-24 LPL is #1968. + are intended exceptions. 5. No cash cuts: no threshold falls from one year to the next, 2026-2040. 6. Differential: for any earnings or profits >= 0 in 2026-2030, primary and additional Class 1, secondary Class 1 and Class 4 equal an exact-rational diff --git a/policyengine_uk/tests/test_property_income_band_base_properties.py b/policyengine_uk/tests/test_property_income_band_base_properties.py new file mode 100644 index 0000000000..7d5730f867 --- /dev/null +++ b/policyengine_uk/tests/test_property_income_band_base_properties.py @@ -0,0 +1,127 @@ +"""Savings and dividends sit above all non-savings income (ITA 2007 s. 16). + +Property income is non-savings income, so it fills the bands before savings +and dividends, as pay does. + +Invariants, for single adults in 2026 with pay, property income, savings +interest and dividends drawn freely: + +1. Differential: replacing property income with pay equal to its taxable + amount leaves savings and dividend tax unchanged, in England and in + Scotland. (The taxable amount, because the £1,000 property allowance can + relieve small property income in full.) +2. Accounting: with no Marriage Allowance, non_savings_taxable_income equals + adjusted net income less savings, dividends and allowances, floored at nil + (earned_taxable_income also subtracts Marriage Allowance; #1947). +3. Monotonic: more property income never lowers savings or dividend tax. + (Main passes this too; the differential and the higher-rate check below + are what catch the bug.) +""" + +import numpy as np +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation + +YEAR = 2026 +PEOPLE = 10 +amount = st.integers(min_value=0, max_value=200_000) +rows_strategy = st.lists( + st.tuples(amount, amount, st.integers(0, 60_000), st.integers(0, 60_000)), + min_size=PEOPLE, + max_size=PEOPLE, +) +VARIABLES = [ + "taxable_property_income", + "savings_income_tax", + "dividend_income_tax", + "non_savings_taxable_income", + "adjusted_net_income", + "taxable_savings_interest_income", + "taxable_dividend_income", + "allowances", +] + + +def _calculate(rows, country): + """rows: (pay, property, savings, dividends), one single adult each.""" + people, benunits, households = {}, {}, {} + for i, (pay, prop, savings, dividends) in enumerate(rows): + name = f"p{i}" + people[name] = { + "age": {YEAR: 45}, + "employment_income": {YEAR: pay}, + "property_income": {YEAR: prop}, + "savings_interest_income": {YEAR: savings}, + "dividend_income": {YEAR: dividends}, + } + benunits[f"b{i}"] = {"members": [name]} + households[f"h{i}"] = {"members": [name], "country": {YEAR: country}} + simulation = Simulation( + situation={"people": people, "benunits": benunits, "households": households} + ) + return {v: np.asarray(simulation.calculate(v, YEAR)) for v in VARIABLES} + + +SETTINGS = settings( + max_examples=8, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) + + +@SETTINGS +@given(rows=rows_strategy, country=st.sampled_from(["ENGLAND", "SCOTLAND"])) +def test_moving_income_between_pay_and_property_leaves_savings_and_dividend_tax( + rows, country +): + as_property = _calculate(rows, country) + taxable_property = as_property["taxable_property_income"] + as_pay = _calculate( + [ + (pay + float(taxable), 0, s, d) + for (pay, _, s, d), taxable in zip(rows, taxable_property) + ], + country, + ) + for name in [ + "savings_income_tax", + "dividend_income_tax", + "non_savings_taxable_income", + ]: + np.testing.assert_allclose(as_property[name], as_pay[name], atol=0.01) + + +@SETTINGS +@given(rows=rows_strategy) +def test_non_savings_taxable_income_is_the_rest_after_allowances(rows): + v = _calculate(rows, "ENGLAND") + expected = np.maximum( + 0, + v["adjusted_net_income"] + - v["taxable_savings_interest_income"] + - v["taxable_dividend_income"] + - v["allowances"], + ) + np.testing.assert_allclose(v["non_savings_taxable_income"], expected, atol=0.01) + + +@SETTINGS +@given(rows=rows_strategy, rise=st.integers(min_value=1, max_value=100_000)) +def test_more_property_income_never_lowers_savings_or_dividend_tax(rows, rise): + before = _calculate(rows, "ENGLAND") + after = _calculate( + [(pay, prop + rise, s, d) for pay, prop, s, d in rows], "ENGLAND" + ) + for name in ["savings_income_tax", "dividend_income_tax"]: + assert np.all(after[name] >= before[name] - 0.01) + + +def test_savings_above_property_income_past_the_basic_rate_limit_pay_the_higher_rate(): + """£40,000 of property income leaves £10,270 of the basic-rate band, so + some of £20,000 of savings must pay 40%: savings tax exceeds 20% of + everything above the £500 savings allowance (£3,900). Main banded the + savings as if the property income were not there and charged £2,900.""" + v = _calculate([(0, 40_000, 20_000, 0)] + [(0, 0, 0, 0)] * (PEOPLE - 1), "ENGLAND") + assert v["savings_income_tax"][0] > (20_000 - 500) * 0.20 diff --git a/policyengine_uk/tests/test_road_fuel_prices.py b/policyengine_uk/tests/test_road_fuel_prices.py new file mode 100644 index 0000000000..ab0dd05262 --- /dev/null +++ b/policyengine_uk/tests/test_road_fuel_prices.py @@ -0,0 +1,58 @@ +from pathlib import Path + +import pytest +import yaml + +import policyengine_uk +from policyengine_uk.system import system + +PRICES = system.parameters.household.consumption.fuel.prices +YOY = system.parameters.gov.economic_assumptions.yoy_growth + + +@pytest.mark.parametrize( + "year, petrol_pence, diesel_pence", + [ + # DESNZ Quarterly Energy Prices table 4.1.2, annual typical retail + # prices (ULSP, ULSD), the series Microcosm UK uses to price spending. + (2023, 147.746334, 158.188711), + (2024, 141.477665, 148.328885), + (2025, 135.072210, 142.548575), + ], +) +def test_pump_prices_are_desnz_annual_averages(year, petrol_pence, diesel_pence): + assert PRICES.petrol(year) * 100 == pytest.approx(petrol_pence, abs=1e-5) + assert PRICES.diesel(year) * 100 == pytest.approx(diesel_pence, abs=1e-5) + + +@pytest.mark.parametrize("fuel", ["petrol", "diesel"]) +def test_spending_litre_proxy_is_derived_from_the_pump_prices(fuel): + """The proxy must be re-derived whenever the pump prices change. + + Spending grows by the proxy, household weights by population, and + litres are spending divided by the price, so weighted litres follow + the HMRC/OBR road-fuel volume path only if + (1 + proxy) = (1 + volume) x price_t / price_t-1 / (1 + population). + """ + proxy = getattr(YOY.obr, f"{fuel}_spending_litre_proxy") + price = getattr(PRICES, fuel) + volume = YOY.obr.road_fuel_volume + population = YOY.ons.population + # The years the YAML itself declares; the loaded parameter also carries + # values the system adds before the first declared date. + source = Path(policyengine_uk.__file__).parent / ( + "parameters/gov/economic_assumptions/yoy_growth.yaml" + ) + declared = yaml.safe_load(source.read_text())["obr"][ + f"{fuel}_spending_litre_proxy" + ]["values"] + years = sorted(int(str(date)[:4]) for date in declared) + assert years[0] == 2021 + for year in years: + expected = (1 + volume(year)) * price(year) / price(year - 1) / ( + 1 + population(year) + ) - 1 + assert proxy(year) == pytest.approx(expected, abs=1e-12), ( + f"{fuel}_spending_litre_proxy for {year} no longer matches the " + f"pump prices; re-derive it from household.consumption.fuel.prices." + ) diff --git a/policyengine_uk/tests/test_savings_allowance_properties.py b/policyengine_uk/tests/test_savings_allowance_properties.py new file mode 100644 index 0000000000..1ed0d174f6 --- /dev/null +++ b/policyengine_uk/tests/test_savings_allowance_properties.py @@ -0,0 +1,94 @@ +"""Personal Savings Allowance, ITA 2007 s. 12B. + +A Scottish taxpayer's higher- and additional-rate income is what would be +charged at those rates if they were not a Scottish taxpayer (s. 12B(8)(a)(iv) +and (b)(iv)), so the allowance follows the rest-of-UK bands for everyone. + +Invariants, for single adults in 2026 with pay, savings interest and +dividends drawn freely: + +1. Country: the allowance is the same for a Scottish and an English taxpayer + with the same income. +2. Differential: it equals the statute computed independently here: £1,000 + when taxable income is at or under the £37,700 basic rate limit, £500 at + or under the £125,140 higher rate limit, and nil above (ITA 2007 s. 10(2)- + (3)), with taxable income as total income less the personal allowance, + tapered by £1 for every £2 of income over £100,000. +3. Monotonic: a higher pay never raises the allowance. +""" + +import numpy as np +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st + +from policyengine_uk import Simulation + +YEAR = 2026 +PEOPLE = 12 +money = st.integers(min_value=0, max_value=300_000) +small = st.integers(min_value=0, max_value=20_000) +incomes = st.lists(st.tuples(money, small, small), min_size=PEOPLE, max_size=PEOPLE) + + +def _allowances(rows, country): + """One single-adult household per row, all in one simulation.""" + people, benunits, households = {}, {}, {} + for i, (pay, savings, dividends) in enumerate(rows): + name = f"p{i}" + people[name] = { + "age": {YEAR: 40}, + "employment_income": {YEAR: pay}, + "savings_interest_income": {YEAR: savings}, + "dividend_income": {YEAR: dividends}, + } + benunits[f"b{i}"] = {"members": [name]} + households[f"h{i}"] = {"members": [name], "country": {YEAR: country}} + simulation = Simulation( + situation={"people": people, "benunits": benunits, "households": households} + ) + return np.asarray(simulation.calculate("savings_allowance", YEAR)) + + +BOUNDARY_ROWS = [ + (49_270, 1_000, 0), # taxable 37,700: at the basic rate limit + (49_271, 1_000, 0), # taxable 37,701 + (124_140, 1_000, 0), # taxable 125,140 (allowance tapered to nil) + (124_141, 1_000, 0), # taxable 125,141 +] + [(0, 0, 0)] * (PEOPLE - 4) + + +def _statute(pay, savings, dividends): + income = pay + savings + dividends + allowance = max(0, 12_570 - max(0, income - 100_000) / 2) + taxable = income - allowance + if taxable > 125_140: + return 0 + if taxable > 37_700: + return 500 + return 1_000 + + +@settings( + max_examples=10, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given(rows=incomes) +@example(rows=BOUNDARY_ROWS) +def test_scottish_and_english_taxpayers_get_the_same_allowance(rows): + scottish = _allowances(rows, "SCOTLAND") + english = _allowances(rows, "ENGLAND") + assert np.array_equal(scottish, english) + assert list(scottish) == [_statute(*row) for row in rows] + + +@settings( + max_examples=10, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given(rows=incomes, rise=st.integers(min_value=1, max_value=100_000)) +def test_a_higher_pay_never_raises_the_allowance(rows, rise): + before = _allowances(rows, "SCOTLAND") + after = _allowances([(pay + rise, s, d) for pay, s, d in rows], "SCOTLAND") + assert np.all(after <= before) diff --git a/policyengine_uk/tests/test_threshold_indexation.py b/policyengine_uk/tests/test_threshold_indexation.py index 01d5396e02..23211c9960 100644 --- a/policyengine_uk/tests/test_threshold_indexation.py +++ b/policyengine_uk/tests/test_threshold_indexation.py @@ -35,8 +35,7 @@ £0.26 of the UPL. From 2023: LPL = PA and 52 x PT is within £0.26 of the LPL. Intended exceptions, as the law stood: in 2021-22 the PT and LPL were £9,568 a year, below the PA; in 2022-23 the PT rose to the PA only - from 6 July 2022 and the LPL was £11,908 for the whole year. The model's - 2023-24 LPL (£11,908, not £12,570) is #1968. + from 6 July 2022 and the LPL was £11,908 for the whole year. 6. Secondary threshold: from 2031, ST(Y) = ST(Y-1) x (1 + g_Y). 7. For any freeze end and September CPI path (function level): values up to each parameter's last stated year are unchanged; after it, each NICs @@ -322,19 +321,9 @@ def test_upper_thresholds_align_with_the_higher_rate_threshold(system, year): ) -@pytest.mark.parametrize( - "year", - [ - pytest.param( - 2023, - marks=pytest.mark.xfail( - reason="#1968: the model's 2023-24 LPL is £11,908, not £12,570", - strict=False, - ), - ), - *range(2024, 2041), - ], -) +# 2023-24 is the first aligned year: NICA 2022 s2 set the LPL to £12,570 +# from 6 April 2023 (#1968, fixed by #2058). +@pytest.mark.parametrize("year", range(2023, 2041)) def test_lower_thresholds_align_with_the_personal_allowance(system, year): values = thresholds(system, year) assert values["lower_profits_limit"] == pytest.approx( diff --git a/policyengine_uk/utils/child_benefit.py b/policyengine_uk/utils/child_benefit.py index e8d1092c38..f6c94270be 100644 --- a/policyengine_uk/utils/child_benefit.py +++ b/policyengine_uk/utils/child_benefit.py @@ -14,3 +14,38 @@ def child_benefit_charge_share( where=width > 0, ) return np.clip(share, 0, 1) + + +def child_benefit_charge_percentage( + income: np.ndarray, phase_out_start: float, phase_out_end: float +) -> np.ndarray: + """The appropriate percentage of ITEPA 2003 s. 681C(2), as a fraction. + + One percentage point per hundredth of the phase-out band crossed (per £200 + from 2024-25), capped at 100 and rounded down to a whole number + (s. 681C(3)(b)). Worked in float64, multiplying before dividing. With + whole-pound band edges, an income on an exact step (exact in float32 too, + below £16.7m) gives an integer-valued quotient, which IEEE division returns + exactly, so no slack is added: slack would round a penny income just + below a step up to it. A nonpositive band width is a cliff (100 points + above the start). + """ + income = np.asarray(income, dtype=np.float64) + width = float(phase_out_end) - float(phase_out_start) + excess = np.maximum(income - float(phase_out_start), 0) + if width > 0: + points = np.minimum(excess * 100 / width, 100) + else: + points = np.where(income > float(phase_out_start), 100.0, 0.0) + return np.floor(points) / 100 + + +def round_down_to_pounds(amount: np.ndarray) -> np.ndarray: + """ITEPA 2003 s. 681C(3)(a): amounts not a whole number of pounds are + rounded down. The amount is first taken to the nearest penny in float64, + so that £4,199 stored as 4,198.9995 is not rounded down to £4,198, then + to whole pounds by integer division of the pence. Child Benefit totals and + charges are whole pence (weekly rates in pence, and whole percentage + points of whole pounds), so the penny step loses nothing.""" + pence = np.round(np.asarray(amount, dtype=np.float64) * 100) + return np.floor_divide(pence, 100) diff --git a/policyengine_uk/variables/contrib/ubi_center/basic_income/bi_household_phaseout.py b/policyengine_uk/variables/contrib/ubi_center/basic_income/bi_household_phaseout.py index 1edff68552..798d2d59fe 100644 --- a/policyengine_uk/variables/contrib/ubi_center/basic_income/bi_household_phaseout.py +++ b/policyengine_uk/variables/contrib/ubi_center/basic_income/bi_household_phaseout.py @@ -1,5 +1,4 @@ from policyengine_uk.model_api import * -import warnings class bi_household_phaseout(Variable): @@ -27,7 +26,10 @@ def formula(person, period, parameters): uncapped_deduction = bi.phase_out.household.rate * income_over_threshold capped_deduction = min_(household_bi, uncapped_deduction) - warnings.filterwarnings("ignore") - - percent_reduction = where(household_bi > 0, capped_deduction / household_bi, 0) + percent_reduction = np.divide( + capped_deduction, + household_bi, + out=np.zeros_like(household_bi, dtype=float), + where=household_bi > 0, + ) return percent_reduction * remaining_bi diff --git a/policyengine_uk/variables/gov/dwp/LHA_allowed_bedrooms.py b/policyengine_uk/variables/gov/dwp/LHA_allowed_bedrooms.py index ce8e5543af..4c87a0df20 100644 --- a/policyengine_uk/variables/gov/dwp/LHA_allowed_bedrooms.py +++ b/policyengine_uk/variables/gov/dwp/LHA_allowed_bedrooms.py @@ -1,9 +1,4 @@ from policyengine_uk.model_api import * -import pandas as pd -import warnings -from policyengine_core.model_api import * - -warnings.filterwarnings("ignore") def child_bedrooms(boys_under_10, boys_10_to_15, girls_under_10, girls_10_to_15): diff --git a/policyengine_uk/variables/gov/dwp/LHA_cap.py b/policyengine_uk/variables/gov/dwp/LHA_cap.py index af611c409d..3b26f2ba9e 100644 --- a/policyengine_uk/variables/gov/dwp/LHA_cap.py +++ b/policyengine_uk/variables/gov/dwp/LHA_cap.py @@ -1,9 +1,4 @@ from policyengine_uk.model_api import * -import pandas as pd -import warnings -from policyengine_core.model_api import * - -warnings.filterwarnings("ignore") class LHA_cap(Variable): diff --git a/policyengine_uk/variables/gov/dwp/LHA_eligible.py b/policyengine_uk/variables/gov/dwp/LHA_eligible.py index 008e3f822b..662b36e484 100644 --- a/policyengine_uk/variables/gov/dwp/LHA_eligible.py +++ b/policyengine_uk/variables/gov/dwp/LHA_eligible.py @@ -1,9 +1,4 @@ from policyengine_uk.model_api import * -import pandas as pd -import warnings -from policyengine_core.model_api import * - -warnings.filterwarnings("ignore") class LHA_eligible(Variable): diff --git a/policyengine_uk/variables/gov/dwp/additional_state_pension.py b/policyengine_uk/variables/gov/dwp/additional_state_pension.py index 932e7ff563..a476661b78 100644 --- a/policyengine_uk/variables/gov/dwp/additional_state_pension.py +++ b/policyengine_uk/variables/gov/dwp/additional_state_pension.py @@ -69,10 +69,11 @@ def formula(person, period, parameters): max_(reported - max_for_type_data, 0), 0, ) - uprating = where( - max_for_type_data > 0, - max_for_type_period / max_for_type_data, - 1, + uprating = np.divide( + max_for_type_period, + max_for_type_data, + out=np.ones_like(max_for_type_data, dtype=float), + where=max_for_type_data > 0, ) # No State Pension is paid before State Pension age. A computed type # is already NONE there; this also holds if the type is an input. diff --git a/policyengine_uk/variables/gov/dwp/basic_state_pension.py b/policyengine_uk/variables/gov/dwp/basic_state_pension.py index 23d4a7a883..48ac56fb13 100644 --- a/policyengine_uk/variables/gov/dwp/basic_state_pension.py +++ b/policyengine_uk/variables/gov/dwp/basic_state_pension.py @@ -43,10 +43,11 @@ def formula(person, period, parameters): # Compute the person's share of the data-year maximum so reforms can # scale the current-period amount while preserving the original cap. - share = where( - max_sp_data_year > 0, - min_(reported, max_sp_data_year) / max_sp_data_year, - 0, + share = np.divide( + min_(reported, max_sp_data_year), + max_sp_data_year, + out=np.zeros_like(reported, dtype=float), + where=max_sp_data_year > 0, ) return ( where( diff --git a/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py new file mode 100644 index 0000000000..d9d7e6f167 --- /dev/null +++ b/policyengine_uk/variables/gov/dwp/carers_allowance_earnings.py @@ -0,0 +1,117 @@ +from policyengine_uk.model_api import * +from policyengine_uk.utils.class_2 import class_2_contribution_weeks, class_2_liable + + +class carers_allowance_earnings(Variable): + value_type = float + entity = Person + label = "earnings for the Carer's Allowance earnings limit" + documentation = ( + "Net earnings from employment and self-employment, as the Carer's " + "Allowance gainful-employment test counts them (Computation of " + "Earnings Regulations 1996), less half of the person's pension " + "contributions. A trading loss counts as nil.\n\n" + "Employment (reg. 10(4)): gross pay less the income tax and primary " + "Class 1 contributions deducted from it. The law deducts the tax " + "actually withheld; the model approximates it as the tax PAYE would " + "take on a standard code if the pay were the person's only income " + "(pay less net-pay pension contributions and the personal allowance, " + "at rest-of-UK or Scottish rates). That also avoids a circularity, " + "since Carer's Allowance is itself taxable, and makes unearned income " + "irrelevant here by assumption.\n\n" + "Self-employment (regs. 13(4) and 14): trading profit less a notional " + "income tax at the basic rate (the Scottish basic rate for a Scottish " + "taxpayer) on the profit above the full personal allowance, and Class 4 " + "contributions at the main rate on the profit between the lower and " + "upper profits limits. Regulation 14(1) deducts the personal reliefs " + "the claimant is entitled to, with no reduction for any allowance the " + "pay uses, and capital expenditure is not deducted (reg. 13(6)(a)). " + "Until 6 April 2024 reg. 14(2)(a) also deducted the Class 2 " + "contributions payable on the chargeable profit; that paragraph was " + "omitted with compulsory Class 2, whose flat rate is nil from " + "2024-25.\n\n" + "Not modelled: the deduction for care charges (Schedule 3, up to half " + "of net earnings) and the Schedule 1 disregards, so this can overstate " + "earnings for a carer who pays for care while at work." + ) + definition_period = YEAR + unit = GBP + reference = [ + dict( + title="Social Security Benefit (Computation of Earnings) Regulations 1996 reg. 10(4)", + href="https://www.legislation.gov.uk/uksi/1996/2745/regulation/10", + ), + dict( + title="Social Security Benefit (Computation of Earnings) Regulations 1996 reg. 13", + href="https://www.legislation.gov.uk/uksi/1996/2745/regulation/13", + ), + dict( + title="Social Security Benefit (Computation of Earnings) Regulations 1996 reg. 14", + href="https://www.legislation.gov.uk/uksi/1996/2745/regulation/14", + ), + ] + + def formula(person, period, parameters): + p = parameters(period).gov.hmrc + income_tax = p.income_tax + class_4 = p.national_insurance.class_4 + scottish = person("pays_scottish_income_tax", period) + personal_allowance = income_tax.allowances.personal_allowance.amount + + # Employment: PAYE on the pay alone (reg. 10(4)(a)). + pay = person("employment_income", period) + taxable_pay = max_(0, pay - person("employee_pension_contributions", period)) + pay_tax = where( + scottish, + income_tax.rates.scotland.rates.calc( + max_(0, taxable_pay - personal_allowance) + ), + income_tax.rates.uk.calc(max_(0, taxable_pay - personal_allowance)), + ) + primary_class_1 = person("ni_class_1_employee", period) + + # Self-employment: notional basic-rate tax and main-rate Class 4 + # (reg. 14(1)-(2)). The tax is on the profit as if it were the only + # income, less the full personal allowance. + profit = max_(0, person("self_employment_income", period)) + # The basic rate is the first band of the rest-of-UK scale (ITA 2007 + # s. 6(2)). The Scottish basic rate is the second bracket of the + # Scottish scale, after the starter rate; read it from the parameter + # tree, which keeps the null starter bracket of years before 2018-19 in + # place. The scale at an instant drops null brackets, so its index 1 + # was the higher rate in 2016-17 and 2017-18. + scottish_scale = person.simulation.tax_benefit_system.parameters.gov.hmrc.income_tax.rates.scotland.rates + basic_rate = where( + scottish, + scottish_scale.brackets[1].rate(period.start), + income_tax.rates.uk.rates[0], + ) + trading_tax = basic_rate * max_(0, profit - personal_allowance) + trading_class_4 = class_4.rates.main * max_( + 0, + min_(profit, class_4.thresholds.upper_profits_limit) + - class_4.thresholds.lower_profits_limit, + ) + # Reg. 14(2)(a), omitted from 6 April 2024: the Class 2 contributions + # payable on the chargeable profit (SSCBA 1992 s. 11), on the + # profit itself rather than ni_class_2's base after capital + # allowances. The flat rate is nil from 2024-25. + class_2 = p.national_insurance.class_2 + trading_class_2 = ( + class_2_liable(profit, class_2) + * class_2.flat_rate + * class_2_contribution_weeks(period.start.year) + ) + + half_pension_contributions = 0.5 * person("pension_contributions", period) + return max_( + 0, + pay + - pay_tax + - primary_class_1 + + profit + - trading_tax + - trading_class_4 + - trading_class_2 + - half_pension_contributions, + ) diff --git a/policyengine_uk/variables/gov/dwp/carers_allowance_pre_overlap.py b/policyengine_uk/variables/gov/dwp/carers_allowance_pre_overlap.py index ca06fafbb2..6bca6f6b93 100644 --- a/policyengine_uk/variables/gov/dwp/carers_allowance_pre_overlap.py +++ b/policyengine_uk/variables/gov/dwp/carers_allowance_pre_overlap.py @@ -25,9 +25,16 @@ def formula(person, period, parameters): weekly_care_hours = person("care_hours", period) meets_work_condition = weekly_care_hours >= ca.min_hours would_claim = person("would_claim_carers_allowance", period) + # SSCBA 1992 s. 70(1)(b): no allowance for anyone gainfully employed, + # which reg. 8 of the 1976 Regulations defines by an earnings limit. + # It applies to reported recipients too. + gainfully_employed = person( + "is_gainfully_employed_for_carers_allowance", period + ) eligible = ( ~(in_scotland & csp_replaces_ca) & (meets_work_condition | receives_ca) + & ~gainfully_employed & would_claim ) return eligible * ca.rate * WEEKS_IN_YEAR diff --git a/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py b/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py new file mode 100644 index 0000000000..b6e6b4b981 --- /dev/null +++ b/policyengine_uk/variables/gov/dwp/is_gainfully_employed_for_carers_allowance.py @@ -0,0 +1,39 @@ +from policyengine_uk.model_api import * + + +class is_gainfully_employed_for_carers_allowance(Variable): + value_type = bool + entity = Person + label = "gainfully employed for Carer's Allowance" + documentation = ( + "Whether the person's weekly net earnings exceed the Carer's Allowance " + "earnings limit, so that they are treated as gainfully employed and " + "are not entitled to the allowance. The regulation tests the earnings " + "of the preceding week; annual earnings are spread evenly over 52 " + "weeks. A fraction of a penny is disregarded, as regulation 5 of the " + "Computation of Earnings Regulations requires when that favours the " + "claimant." + ) + definition_period = YEAR + reference = [ + dict( + title="Social Security Contributions and Benefits Act 1992 s. 70(1)(b)", + href="https://www.legislation.gov.uk/ukpga/1992/4/section/70", + ), + dict( + title="The Social Security (Invalid Care Allowance) Regulations 1976 reg. 8(1)", + href="https://www.legislation.gov.uk/uksi/1976/409/regulation/8", + ), + dict( + title="Social Security Benefit (Computation of Earnings) Regulations 1996 reg. 5", + href="https://www.legislation.gov.uk/uksi/1996/2745/regulation/5", + ), + ] + + def formula(person, period, parameters): + limit = parameters(period).gov.dwp.carers_allowance.earnings_limit + weekly = person("carers_allowance_earnings", period) / WEEKS_IN_YEAR + # Disregard a fraction of a penny; the tolerance keeps an exact + # number of pence from being knocked down by floating-point error. + weekly_in_pence = np.floor(weekly * 100 + 1e-6) / 100 + return weekly_in_pence > limit diff --git a/policyengine_uk/variables/gov/dwp/new_state_pension.py b/policyengine_uk/variables/gov/dwp/new_state_pension.py index f26e4a9cfd..69c9a3bfce 100644 --- a/policyengine_uk/variables/gov/dwp/new_state_pension.py +++ b/policyengine_uk/variables/gov/dwp/new_state_pension.py @@ -48,9 +48,10 @@ def formula(person, period, parameters): # flows through ``additional_state_pension`` — mirrors the # BASIC / ASP split so ``new_state_pension`` is always the headline # flat-rate component. - share = where( - max_new_data_year > 0, - min_(reported_weekly, max_new_data_year) / max_new_data_year, - 0, + share = np.divide( + min_(reported_weekly, max_new_data_year), + max_new_data_year, + out=np.zeros_like(reported_weekly, dtype=float), + where=max_new_data_year > 0, ) return eligible * share * max_new_period * WEEKS_IN_YEAR diff --git a/policyengine_uk/variables/gov/dwp/universal_credit/income/uc_individual_earned_income_before_mif.py b/policyengine_uk/variables/gov/dwp/universal_credit/income/uc_individual_earned_income_before_mif.py index 73a93702a1..9bf59a96a5 100644 --- a/policyengine_uk/variables/gov/dwp/universal_credit/income/uc_individual_earned_income_before_mif.py +++ b/policyengine_uk/variables/gov/dwp/universal_credit/income/uc_individual_earned_income_before_mif.py @@ -35,7 +35,10 @@ def formula(person, period, parameters): # The deductions are the person's own and come off only their own # earnings, so one partner's tax, NI or pension contributions never # reduce the other partner's earned income. - earnings_components = ["employment_income", "miscellaneous_income"] + # Miscellaneous income as income tax counts it, so a loss does not + # reduce pay (and matches uc_income_tax_on_earnings). Whether it is + # earnings at all is issue #1961. + earnings_components = ["employment_income", "taxable_miscellaneous_income"] bi = parameters(period).gov.contrib.ubi_center.basic_income if bi.interactions.include_in_means_tests: earnings_components.append("basic_income") diff --git a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py index 758358a59c..03fb18e275 100644 --- a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py +++ b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py @@ -7,10 +7,12 @@ class child_benefit_opts_out(Variable): "Whether a family would opt out of Child Benefit payments because of the " "High Income Child Benefit Charge. This is independent of " "would_claim_child_benefit and never identifies a claim by itself. " - "For claimants, payments resume when the simulated charge share falls below " - "the assumed " - "opt_out_charge_share threshold (one by default), or the charge is zero " - "or neutralised. Entitlement and nonclaimant status are unchanged." + "For claimants, payments resume when the continuous charge share (the " + "unrounded share of the phase-out band crossed) falls below the assumed " + "opt_out_charge_share threshold (one by default), or is zero, or the " + "charge is neutralised. The statutory charge itself is rounded " + "(ITEPA 2003 s. 681C(3)) and can be nil where the share is small but " + "positive. Entitlement and nonclaimant status are unchanged." ) entity = BenUnit definition_period = YEAR diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py index 6c046ff0c0..56f2599a13 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/savings_allowance.py @@ -5,26 +5,36 @@ class savings_allowance(Variable): value_type = float entity = Person label = "Savings Allowance for the year" + documentation = ( + "The Personal Savings Allowance (ITA 2007 s. 12B): the basic amount " + "with no higher-rate income, the higher amount with higher-rate but " + "no additional-rate income, and the additional amount (nil) with any " + "additional-rate income. For a Scottish or Welsh taxpayer, s. " + "12B(8)(a)(iv) and (b)(iv) count as higher- or additional-rate income " + "what would be charged at those rates if they were not a Scottish or " + "Welsh taxpayer, so the bands are the rest-of-UK ones for everyone. " + "tax_band follows the Scottish bands for a Scottish taxpayer, as the " + "Marriage Allowance conditions need, so it is not read here. Taxable " + "income is taken, as in tax_band, as adjusted net income less " + "allowances." + ) definition_period = YEAR - reference = "Income Tax Act 2007 s. 12B" + reference = "https://www.legislation.gov.uk/ukpga/2007/3/section/12B" unit = GBP def formula(person, period, parameters): - tax_band = person("tax_band", period) - tax_bands = tax_band.possible_values - amounts = parameters( - period - ).gov.hmrc.income_tax.allowances.personal_savings_allowance + income_tax = parameters(period).gov.hmrc.income_tax + amounts = income_tax.allowances.personal_savings_allowance + thresholds = income_tax.rates.uk.thresholds + taxable_income = person("adjusted_net_income", period) - person( + "allowances", period + ) + # thresholds[1] is the basic rate limit and thresholds[2] the higher + # rate limit. Income up to a limit is charged at the lower rate and + # only income above it at the higher one (ITA 2007 s. 10(2)-(3)), so + # taxable income exactly at a limit has none in the band above. return select( - [ - tax_band == tax_bands.ADDITIONAL, - tax_band == tax_bands.HIGHER, - ( - (tax_band == tax_bands.STARTER) - | (tax_band == tax_bands.BASIC) - | (tax_band == tax_bands.INTERMEDIATE) - ), - tax_band == tax_bands.NONE, - ], - [amounts.additional, amounts.higher, amounts.basic, amounts.basic], + [taxable_income > thresholds[2], taxable_income > thresholds[1]], + [amounts.additional, amounts.higher], + default=amounts.basic, ) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py index 30f407bf45..ce8be05b12 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bases/taxable_miscellaneous_income.py @@ -5,11 +5,31 @@ class taxable_miscellaneous_income(Variable): value_type = float entity = Person label = "Amount of miscellaneous income that is taxable" - definition_period = YEAR - reference = dict( - title="Income Tax Act (Trading and Other Income) 2005, s. 574", - href="https://www.legislation.gov.uk/ukpga/2005/5/section/574", + documentation = ( + "Miscellaneous income charged to income tax. On a claim, a loss on a " + "miscellaneous transaction is relieved only against relevant " + "miscellaneous income: income under the same section 1016 provision as " + "the loss (ITA 2007 s. 152(4)-(5)). It is never set against other " + "income, so a net loss counts as nil here. The model holds one net " + "figure per person, so it assumes the items netted inside " + "miscellaneous_income can be relieved against each other; that can " + "undertax a person with a gain under one provision and a loss under " + "another. The input also covers income from sources outside section " + "1016, some of which carry no loss relief at all. Carrying a loss " + "forward against later years' miscellaneous income is not modelled." ) + definition_period = YEAR + reference = [ + dict( + title="Income Tax (Trading and Other Income) Act 2005 s. 574", + href="https://www.legislation.gov.uk/ukpga/2005/5/section/574", + ), + dict( + title="Income Tax Act 2007 s. 152", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/152", + ), + ] unit = GBP - adds = ["miscellaneous_income"] + def formula(person, period, parameters): + return max_(0, person("miscellaneous_income", period)) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_income_after_allowances.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_income_after_allowances.py new file mode 100644 index 0000000000..0bc9405495 --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_income_after_allowances.py @@ -0,0 +1,29 @@ +from policyengine_uk.model_api import * + + +class non_savings_income_after_allowances(Variable): + value_type = float + entity = Person + label = "Non-savings, non-dividend income for Income Tax after allowances" + documentation = ( + "Adjusted net income less savings and dividend income and allowances, " + "floored at nil: the income below savings and dividends (ITA 2007 s. " + "16) once the allowances, which go to it first, are taken off. " + "property_income_tax and taxable_property_income_after_allowances " + "both read it." + ) + definition_period = YEAR + reference = dict( + title="Income Tax Act 2007, s. 16", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/16", + ) + unit = GBP + + def formula(person, period, parameters): + return max_( + 0, + person("adjusted_net_income", period) + - person("taxable_savings_interest_income", period) + - person("taxable_dividend_income", period) + - person("allowances", period), + ) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py new file mode 100644 index 0000000000..8e1d2253a4 --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/non_savings_taxable_income.py @@ -0,0 +1,36 @@ +from policyengine_uk.model_api import * + + +class non_savings_taxable_income(Variable): + value_type = float + entity = Person + label = "Taxable income other than savings and dividend income" + documentation = ( + "The part of taxable income below savings and dividend income, which " + "ITA 2007 s. 16 treats as the highest part of total income: " + "earned_taxable_income plus property income after allowances. " + "earned_taxable_income leaves property income out because " + "property_income_tax taxes it separately, but property income still " + "fills the bands before savings and dividends do (from 2027-28, " + "s. 16A places it immediately below savings income). With no Marriage " + "Allowance it equals non_savings_income_after_allowances; " + "earned_taxable_income also subtracts Marriage Allowance, which the " + "law gives as a tax reduction (#1947)." + ) + definition_period = YEAR + reference = [ + dict( + title="Income Tax Act 2007, s. 16", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/16", + ), + dict( + title="Income Tax Act 2007, s. 16A (from 2027-28)", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/16A", + ), + ] + unit = GBP + + def formula(person, period, parameters): + return person("earned_taxable_income", period) + person( + "taxable_property_income_after_allowances", period + ) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py new file mode 100644 index 0000000000..b38aabcf36 --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_earned_income/taxable_property_income_after_allowances.py @@ -0,0 +1,36 @@ +from policyengine_uk.model_api import * + + +class taxable_property_income_after_allowances(Variable): + value_type = float + entity = Person + label = "Property income for Income Tax after allowances" + documentation = ( + "Taxable property income less the part of the allowances it absorbs. " + "Allowances go to non-savings, non-dividend income first, and " + "property income is stacked above the rest of it, so it takes " + "whatever allowances the other non-savings income leaves. From " + "2027-28 that order is statutory: s. 25(3A) sets allowances against " + "income other than property, savings and dividend income first, and " + "s. 16A places property income immediately below savings income." + ) + definition_period = YEAR + reference = [ + dict( + title="Income Tax Act 2007, s. 25(3A) (from 2027-28)", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/25", + ), + dict( + title="Income Tax Act 2007, s. 16A (from 2027-28)", + href="https://www.legislation.gov.uk/ukpga/2007/3/section/16A", + ), + ] + unit = GBP + + def formula(person, period, parameters): + non_savings_after_allowances = person( + "non_savings_income_after_allowances", period + ) + return min_( + person("taxable_property_income", period), non_savings_after_allowances + ) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/add_rate_savings_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/add_rate_savings_income.py index 610b9481b2..979d1c254d 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/add_rate_savings_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/add_rate_savings_income.py @@ -14,7 +14,9 @@ class add_rate_savings_income(Variable): def formula(person, period, parameters): thresholds = parameters(period).gov.hmrc.income_tax.rates.uk.thresholds - other_income = person("earned_taxable_income", period) + # Savings sit above all non-savings income, property income + # included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) savings_after_allowances = max_( 0, person("taxable_savings_interest_income", period) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income.py index cea212f46d..d4aa229abb 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income.py @@ -14,7 +14,9 @@ class basic_rate_savings_income(Variable): def formula(person, period, parameters): thresholds = parameters(period).gov.hmrc.income_tax.rates.uk.thresholds - other_income = person("earned_taxable_income", period) + # Savings sit above all non-savings income, property income + # included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) savings_after_allowances = max_( 0, person("taxable_savings_interest_income", period) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income_pre_starter.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income_pre_starter.py index a3f99e40ce..214b51cfcf 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income_pre_starter.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/basic_rate_savings_income_pre_starter.py @@ -19,7 +19,9 @@ def formula(person, period, parameters): person("taxable_savings_interest_income", period) - person("received_allowances_savings_income", period), ) - other_income = person("earned_taxable_income", period) + # Savings sit above all non-savings income, property income + # included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) basic_rate_amount_with_savings = clip( savings_income + other_income, thresholds[0], diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/higher_rate_savings_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/higher_rate_savings_income.py index cfb7da1a31..96f845b5b3 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/higher_rate_savings_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/higher_rate_savings_income.py @@ -14,7 +14,9 @@ class higher_rate_savings_income(Variable): def formula(person, period, parameters): thresholds = parameters(period).gov.hmrc.income_tax.rates.uk.thresholds - other_income = person("earned_taxable_income", period) + # Savings sit above all non-savings income, property income + # included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) savings_after_allowances = max_( 0, person("taxable_savings_interest_income", period) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py index 354b3908c8..94e38d7101 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/bracketized_savings_income/savings_starter_rate_income.py @@ -37,7 +37,11 @@ def formula(person, period, parameters): - person("received_allowances_savings_income", period), ), ) - earned_taxable_income = person("earned_taxable_income", period) + # The taper reads all non-savings income, property income included + # (ITA 2007 s. 16). Where the taper starts is a separate, known defect: + # s. 12 shrinks the band from the first pound of taxable non-savings + # income, not from the personal allowance plus the band (#1799). + earned_taxable_income = person("non_savings_taxable_income", period) removed_amount = max_( 0, diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/charges/child_benefit_hitc.py b/policyengine_uk/variables/gov/hmrc/income_tax/charges/child_benefit_hitc.py index b5add0442d..a18a822f9e 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/charges/child_benefit_hitc.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/charges/child_benefit_hitc.py @@ -1,21 +1,39 @@ from policyengine_uk.model_api import * -from policyengine_uk.utils.child_benefit import child_benefit_charge_share +from policyengine_uk.utils.child_benefit import ( + child_benefit_charge_percentage, + round_down_to_pounds, +) class CB_HITC(Variable): value_type = float entity = Person label = "Child Benefit High-Income Tax Charge" + documentation = ( + "The appropriate percentage of the Child Benefit received, charged on " + "the partner with the higher adjusted net income. Section 681C(3) " + "rounds down the Child Benefit total and the charge to whole pounds, " + "and the percentage to a whole number." + ) definition_period = YEAR - reference = "https://www.legislation.gov.uk/ukpga/2003/1/part/10/chapter/8" + reference = [ + dict( + title="Income Tax (Earnings and Pensions) Act 2003 s. 681C", + href="https://www.legislation.gov.uk/ukpga/2003/1/section/681C", + ), + dict( + title="Income Tax (Earnings and Pensions) Act 2003 Part 10 Chapter 8", + href="https://www.legislation.gov.uk/ukpga/2003/1/part/10/chapter/8", + ), + ] unit = GBP defined_for = "is_higher_earner" def formula(person, period, parameters): - CB_received = person.benunit("child_benefit", period) + CB_received = round_down_to_pounds(person.benunit("child_benefit", period)) hitc = parameters(period).gov.hmrc.income_tax.charges.CB_HITC income = person("adjusted_net_income", period) - percentage = child_benefit_charge_share( + percentage = child_benefit_charge_percentage( income, hitc.phase_out_start, hitc.phase_out_end ) - return percentage * CB_received + return round_down_to_pounds(percentage * CB_received) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/liability/dividend_income_tax.py b/policyengine_uk/variables/gov/hmrc/income_tax/liability/dividend_income_tax.py index e812a8f769..e9858f4184 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/liability/dividend_income_tax.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/liability/dividend_income_tax.py @@ -14,7 +14,9 @@ class dividend_income_tax(Variable): def formula(person, period, parameters): rates = parameters(period).gov.hmrc.income_tax.rates - other_income = person("earned_taxable_income", period) + max_( + # Dividends are the highest part of income, above savings and all + # non-savings income, property income included (ITA 2007 s. 16). + other_income = person("non_savings_taxable_income", period) + max_( 0, person("taxable_savings_interest_income", period) - person("received_allowances_savings_income", period), diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py b/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py index 0fff929eac..fad31dd093 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/liability/property_income_tax.py @@ -34,15 +34,12 @@ def formula(person, period, parameters): determining which tax band it falls into. """ rates = parameters(period).gov.hmrc.income_tax.rates - taxable_property = person("taxable_property_income", period) - non_savings_after_allowances = max_( - 0, - person("adjusted_net_income", period) - - person("taxable_savings_interest_income", period) - - person("taxable_dividend_income", period) - - person("allowances", period), + non_savings_after_allowances = person( + "non_savings_income_after_allowances", period + ) + property_after_allowances = person( + "taxable_property_income_after_allowances", period ) - property_after_allowances = min_(taxable_property, non_savings_after_allowances) other_earned_taxable = max_( 0, non_savings_after_allowances - property_after_allowances ) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/taxed_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/taxed_income.py index 3729ad0428..0a937cb36d 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/taxed_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/taxed_income.py @@ -12,8 +12,11 @@ class taxed_income(Variable): href="https://www.legislation.gov.uk/ukpga/2007/3/section/23", ) + # Property income is taxed income too: non_savings_taxable_income adds it + # to earned_taxable_income, which leaves it out because + # property_income_tax taxes it separately. adds = [ - "earned_taxable_income", + "non_savings_taxable_income", "taxed_savings_income", "taxed_dividend_income", ] diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/total_income.py b/policyengine_uk/variables/gov/hmrc/income_tax/total_income.py index f13a71fb41..0503c8447c 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/total_income.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/total_income.py @@ -20,5 +20,6 @@ class total_income(Variable): "property_income", "savings_interest_income", "dividend_income", - "miscellaneous_income", + # A miscellaneous loss is never part of total income (ITA 2007 s. 152(4)). + "taxable_miscellaneous_income", ] diff --git a/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py b/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py index 4c5e80aa56..f86e87520d 100644 --- a/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py +++ b/policyengine_uk/variables/household/demographic/is_carer_for_benefits.py @@ -20,15 +20,31 @@ class is_carer_for_benefits(Variable): "person cared for receives a qualifying disability benefit (part of " "the Carer's Allowance conditions referenced by regulation 30), so a " "survey hours field counts caring for " - "anyone and a dataset's take-up input carries that simplification." + "anyone and a dataset's take-up input carries that simplification. " + "A reported Carer's Allowance award the person would claim also " + "counts, so a carer whose earnings put them over the Carer's Allowance " + "earnings limit keeps the Universal Credit carer element, which does " + "not apply that limit (regulation 30(1)(a)(i))." ) reference = "https://www.legislation.gov.uk/uksi/2013/376/regulation/30" def formula(person, period, parameters): min_hours = parameters(period).gov.dwp.carers_allowance.min_hours meets_hours = person("care_hours", period) >= min_hours + # A reported Carer's Allowance award the person would claim is + # evidence of the caring itself, where no hours are recorded. It still + # counts when the earnings limit (which the Universal Credit carer + # element does not apply) stops the allowance being paid. Requiring + # would_claim is a modelling convention, not the law: regulation 30(2) + # applies whether or not a claim is made. It keeps an award the person + # would not claim counting no more than it did before; in the data, + # would_claim follows reported receipt. + reports_carers_allowance = ( + person("carers_allowance_reported", period) > 0 + ) & person("would_claim_carers_allowance", period) return ( person("receives_carer_benefit", period) | person("is_entitled_to_carer_benefit", period) | meets_hours + | reports_carers_allowance )