Set income_not_otherwise_charged to FRS odd-job income

changelog.d/income-not-otherwise-charged.added.md

@@ -0,0 +1 @@
+Set `income_not_otherwise_charged` to FRS odd-job income, as a proxy for income charged under ITTOIA 2005 Part 5 Chapter 8, so policyengine-uk's trading allowance relieves it. The other parts of `miscellaneous_income` (royalties, a sleeping partner's share, overseas pensions, a child's spare-time earnings, trust income and an organisation's allowance) stay outside it, and the SPI-synthetic copy holds none.

policyengine_uk_data/datasets/frs.py

@@ -352,6 +352,66 @@ def frs_property_income(person: pd.DataFrame, household: pd.DataFrame) -> np.nda
     ).values
 
 
+MISC_INCOME_FIELDS = [
+    "allpay2",
+    "royyr2",
+    "royyr3",
+    "royyr4",
+    "chamtern",
+    "chamttst",
+]
+
+
+def frs_miscellaneous_income(
+    person: pd.DataFrame, oddjob: pd.DataFrame
+) -> tuple[np.ndarray, np.ndarray]:
+    """Annual miscellaneous income and the income not otherwise charged in it.
+
+    Returns ``(miscellaneous_income, income_not_otherwise_charged)``, aligned
+    with ``person.person_id``. The second is part of the first, so it never
+    exceeds it.
+
+    Miscellaneous income adds odd-job income to six amounts on the person
+    tables, whose sum is floored at zero: ALLPAY2 (allowance from an
+    organisation), ROYYR2 (royalties), ROYYR3 (income as a sleeping
+    partner), ROYYR4 (overseas pension), and from the child table CHAMTERN
+    (earnings from a spare time job) and CHAMTTST (income from a trust).
+
+    Odd-job income comes from the ODDJOB table. The FRS asks (OddJob) about
+    money the person received in the last 4 weeks from babysitting, acting as
+    a mail order agent, or an odd job, occasional work or professional advice
+    (ODDTYPE 1 to 3), that the employment and self-employment questions have
+    not covered. The FRS defines an odd job as one done once only, without
+    regularity or continuity. OJAMT is the amount the person received and
+    OJNOW whether they did the work in the last 7 days. Jobs from the last 7
+    days count, as weekly amounts (#640). Negative OJAMT values are FRS
+    missing-value codes (-1 to -9), not amounts, so they count as zero.
+
+    Odd-job income stands in for the income not otherwise charged. The FRS
+    records it outside the employment and self-employment questions, and
+    casual income that no other provision charges falls under ITTOIA 2005
+    Part 5 Chapter 8 (s. 687), which the trading allowance relieves
+    (s. 783AB). The survey does not say how HMRC would treat each job: casual
+    work that amounts to a trade takes the same allowance as trade
+    receipts, and Chapter 8 then does not charge it. The six other amounts
+    are not Chapter 8 income: royalties fall under Part 5 Chapter 2, a
+    sleeping partner's share is partnership trading income, an overseas
+    pension is pension income (ITEPA 2003 Part 9), a child's spare time job
+    is a job, trust income is an annual payment (Part 5 Chapter 7) or keeps
+    its source, and the organisation's allowance is a transfer, which
+    private_transfer_income also counts (#627).
+    """
+    odd_job_income = sum_to_entity(
+        oddjob.ojamt.clip(lower=0) * (oddjob.ojnow == 1),
+        oddjob.person_id,
+        person.person_id,
+    )
+    miscellaneous_income = (
+        odd_job_income + sum_from_positive_fields(person, MISC_INCOME_FIELDS)
+    ) * WEEKS_IN_YEAR
+    return miscellaneous_income, odd_job_income * WEEKS_IN_YEAR
+
+
 def derive_is_in_non_advanced_education(
     current_education,
     is_apprentice=None,
@@ -1376,22 +1436,10 @@ def determine_education_level(fted_val, typeed2_val, age_val):
         * WEEKS_IN_YEAR
     )
 
-    odd_job_income = sum_to_entity(
-        oddjob.ojamt * (oddjob.ojnow == 1), oddjob.person_id, person.person_id
-    )
-
-    MISC_INCOME_FIELDS = [
-        "allpay2",
-        "royyr2",
-        "royyr3",
-        "royyr4",
-        "chamtern",
-        "chamttst",
-    ]
-
-    pe_person["miscellaneous_income"] = (
-        odd_job_income + sum_from_positive_fields(person, MISC_INCOME_FIELDS)
-    ) * WEEKS_IN_YEAR
+    (
+        pe_person["miscellaneous_income"],
+        pe_person["income_not_otherwise_charged"],
+    ) = frs_miscellaneous_income(person, oddjob)
 
     PRIVATE_TRANSFER_INCOME_FIELDS = [
         "apamt",

policyengine_uk_data/datasets/imputations/income.py

@@ -474,6 +474,26 @@ def clear_frs_reported_capital(dataset: UKSingleYearDataset) -> UKSingleYearData
     return dataset
 
 
+def clear_income_not_otherwise_charged(
+    dataset: UKSingleYearDataset,
+) -> UKSingleYearDataset:
+    """Set ``income_not_otherwise_charged`` to zero.
+
+    The build applies it to the SPI-synthetic copy. Its trading profits are
+    SPI PROFITS, which ``datasets/spi.py`` takes to be after the trading
+    allowance, while its miscellaneous income stays its FRS donor's (neither
+    imputation stage replaces it). policyengine-uk gives a person's trades the
+    allowance first and their income not otherwise charged what is left, so
+    on these rows the donor's odd-job income would take an allowance the SPI
+    profit may already have used. At zero, policyengine-uk taxes the copy's
+    miscellaneous income in full, as it did before it applied the allowance
+    to that income.
+    """
+    if "income_not_otherwise_charged" in dataset.person.columns:
+        dataset.person["income_not_otherwise_charged"] = 0.0
+    return dataset
+
+
 def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
     """
     Impute detailed income components using trained model.
@@ -503,6 +523,7 @@ def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
     zero_weight_copy.household.household_weight = 0
     zero_weight_copy.household["household_is_spi_synthetic"] = True
     zero_weight_copy = clear_frs_reported_capital(zero_weight_copy)
+    zero_weight_copy = clear_income_not_otherwise_charged(zero_weight_copy)
     zero_weight_copy = subsample_dataset(zero_weight_copy, 10_000)
 
     model = create_income_model()

policyengine_uk_data/tests/record_safe.py

@@ -0,0 +1,34 @@
+"""Reduce licensed microdata to counts without letting a failure print a record.
+
+A data test that fails can print the values it checks: pytest shows the
+operands of a failing assert, ``-l`` shows each frame's locals, and pandas and
+numpy quote the offending value in some error messages ("could not convert
+string to float: ..."). Tests on the FRS reduce a dataset to counts through
+``counts_or_fail`` and assert only on what it returns.
+"""
+
+import pytest
+
+
+def counts_or_fail(count, dataset, *args):
+    """``count(dataset, *args)``, or a test failure that quotes no record.
+
+    An error inside ``count`` becomes a failure that names only ``count`` and
+    the error's type, and the error's own traceback goes with it. The function
+    raises the failure after the ``except`` block, so it chains no exception.
+    ``pytrace=False`` suppresses pytest's usual traceback; ``--full-trace``
+    overrides that and shows the frames down to this function, whose only
+    locals are ``count``, ``dataset``, ``args`` and the type's name.
+    Warnings that ``-W error`` turns into exceptions take the same path.
+    ``dataset`` and ``args`` must not show records in their reprs (a
+    UKSingleYearDataset shows only its class and address).
+    """
+    try:
+        return count(dataset, *args)
+    except Exception as error:
+        error_type = type(error).__name__
+    raise pytest.fail.Exception(
+        f"{count.__name__} raised {error_type}; its message is withheld "
+        "because it can quote a record",
+        pytrace=False,
+    )