Calibrate GB universal credit to one OBR total; reach the UC payment top band

changelog.d/obr-uc-welfare-cap-targets.fixed.md

@@ -0,0 +1 @@
+Calibrate universal credit to one OBR total for Great Britain, the sum of EFO table 4.9's rows inside and outside the welfare cap. The outside-the-cap row is DWP's UC equivalent of JSA (the Intensive Work Search group), not UC for households the benefit cap leaves alone: computed that way, it asked for £12.9bn of nearly all UC (estimate £72bn, +460%), and the inside-the-cap row was compared with all UK UC. Also count the open top band of the DWP UC payment distribution ("£2500.01 or over" a month), which parsed to missing bounds so its four targets (1.4k to 83k households) could never be met, and make the bands meet without gaps, so awards of exactly a band's top, or a penny a month after deductions, land in their Stat-Xplore band.

policyengine_uk_data/targets/build_loss_matrix.py

@@ -52,7 +52,6 @@
     compute_two_child_limit,
     compute_uc_by_children,
     compute_uc_by_family_type,
-    compute_uc_outside_cap,
     compute_uc_payment_dist,
     compute_uk_population,
     compute_vehicles,
@@ -113,7 +112,7 @@ def create_target_matrix(
             col = _compute_column(target, ctx, year)
             if col is None:
                 continue
-            df[target.name] = col
+            df[target.name] = restrict_to_countries(col, ctx.country, target.countries)
             target_names.append(target.name)
             target_values.append(val)
         except Exception as e:
@@ -122,6 +121,18 @@ def create_target_matrix(
     return df, pd.Series(target_values, index=target_names)
 
 
+def restrict_to_countries(column, household_country, countries):
+    """Zero a household column outside the countries a target covers.
+
+    ``countries`` of None means the target covers the whole UK, and the
+    column is returned unchanged.
+    """
+    if countries is None:
+        return column
+    in_scope = np.isin(np.asarray(household_country), countries)
+    return np.asarray(column, dtype=float) * in_scope
+
+
 def _resolve_value(target: Target, year: int) -> float | None:
     """Get the target value for a year, falling back to nearest year.
 
@@ -392,10 +403,6 @@ def _compute_column(target: Target, ctx: _SimContext, year: int) -> np.ndarray |
     ):
         return compute_ss_ni_relief(target, ctx)
 
-    # UC outside benefit cap
-    if name == "obr/universal_credit_outside_cap":
-        return compute_uc_outside_cap(target, ctx)
-
     # Two-child limit
     if "two_child_limit" in name:
         return compute_two_child_limit(target, ctx)

policyengine_uk_data/targets/compute/__init__.py

@@ -7,7 +7,6 @@
     compute_two_child_limit,
     compute_uc_by_children,
     compute_uc_by_family_type,
-    compute_uc_outside_cap,
     compute_uc_payment_dist,
 )
 from policyengine_uk_data.targets.compute.council_tax import (
@@ -76,7 +75,6 @@
     "compute_two_child_limit",
     "compute_uc_by_children",
     "compute_uc_by_family_type",
-    "compute_uc_outside_cap",
     "compute_uc_payment_dist",
     "compute_uk_population",
     "compute_vehicles",

policyengine_uk_data/targets/compute/benefits.py

@@ -82,7 +82,14 @@ def ft_hh(value):
 
 
 def compute_uc_payment_dist(target, ctx) -> np.ndarray:
-    """Compute UC payment distribution band x family type."""
+    """Compute UC payment distribution band x family type.
+
+    Stat-Xplore's monthly award is the UC due after deductions (its "Monthly
+    Award Amount (payment bands)" metadata), as universal_credit is. Bands
+    are (lower, upper] annual amounts (see
+    utils.uc_data.parse_monthly_award_band), so consecutive bands meet and
+    the open top band has an infinite upper bound.
+    """
     name = target.name.removeprefix("dwp/uc_payment_dist/")
     idx = name.index("_annual_payment_")
     family_type = name[:idx]
@@ -93,22 +100,11 @@ def compute_uc_payment_dist(target, ctx) -> np.ndarray:
     uc_family_type = ctx.sim.calculate("family_type", map_to="benunit").values
 
     in_band = (
-        (uc_payments >= lower) & (uc_payments < upper) & (uc_family_type == family_type)
+        (uc_payments > lower) & (uc_payments <= upper) & (uc_family_type == family_type)
     )
     return ctx.household_from_family(in_band)
 
 
-def compute_uc_outside_cap(target, ctx) -> np.ndarray:
-    """Compute OBR UC outside benefit cap."""
-    uc = ctx.sim.calculate("universal_credit")
-    uc_hh = ctx.household_from_family(uc)
-    cap_reduction = ctx.sim.calculate(
-        "benefit_cap_reduction", map_to="household"
-    ).values
-    not_capped = cap_reduction == 0
-    return uc_hh * not_capped
-
-
 def compute_two_child_limit(target, ctx) -> np.ndarray | None:
     """Compute two-child limit targets."""
     name = target.name

policyengine_uk_data/targets/schema.py

@@ -19,6 +19,11 @@ class Unit(str, Enum):
     RATE = "rate"
 
 
+# DWP statistics cover Great Britain: benefits for Northern Ireland residents
+# are the Northern Ireland Executive's responsibility.
+GREAT_BRITAIN = ("ENGLAND", "SCOTLAND", "WALES")
+
+
 class Target(BaseModel):
     """A single calibration target from an official statistical source.
 
@@ -41,6 +46,10 @@ class Target(BaseModel):
     is_count: bool = False
     reference_url: str | None = None
     forecast_vintage: str | None = None
+    # Countries a national source covers when that is less than the UK, as
+    # values of the model's `country` variable. The loss matrix column only
+    # counts households in these countries. None means the whole UK.
+    countries: tuple[str, ...] | None = None
 
     # For targets needing custom simulation logic (UC splits,
     # counterfactuals). Excluded from serialisation.

policyengine_uk_data/targets/sources/obr.py

@@ -17,7 +17,7 @@
 import openpyxl
 import requests
 
-from policyengine_uk_data.targets.schema import Target, Unit
+from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit
 from policyengine_uk_data.targets.sources._common import (
     HEADERS,
     load_config,
@@ -455,6 +455,32 @@ def _parse_nics(wb: openpyxl.Workbook) -> list[Target]:
     return targets
 
 
+def _universal_credit_rows(ws) -> tuple[int, int]:
+    """Table 4.9's universal credit rows inside and outside the welfare cap.
+
+    Each section has exactly one row starting "Universal credit"; the
+    outside-the-cap section starts at the row headed "Welfare spending
+    outside the welfare cap".
+    """
+    max_row = ws.max_row
+    boundary = _find_row(
+        ws, "Welfare spending outside the welfare cap", max_row=max_row
+    )
+    rows = [
+        row
+        for row in range(1, max_row + 1)
+        if str(ws[f"B{row}"].value or "").strip().startswith("Universal credit")
+    ]
+    inside = [row for row in rows if row < boundary]
+    outside = [row for row in rows if row > boundary]
+    if len(inside) != 1 or len(outside) != 1:
+        raise ValueError(
+            f"expected one universal credit row each side of row {boundary}, "
+            f"found rows {rows}"
+        )
+    return inside[0], outside[0]
+
+
 def _parse_welfare(wb: openpyxl.Workbook) -> list[Target]:
     """Parse Table 4.9 (welfare spending) from expenditure xlsx."""
     config = load_config()
@@ -506,10 +532,6 @@ def read_49(row_num: int) -> dict[int, float]:
             "Winter fuel payment",
             "winter_fuel_allowance",
         ),
-        "universal_credit_in_cap": (
-            "Universal credit",
-            "universal_credit",
-        ),
         "child_benefit": ("Child benefit", "child_benefit"),
         "state_pension": ("State pension", "state_pension"),
         "jobseekers_allowance": (
@@ -539,30 +561,41 @@ def read_49(row_num: int) -> dict[int, float]:
         except ValueError:
             logger.warning("OBR welfare: row '%s' not found", label)
 
-    # Universal credit outside cap (row 43) is jobseekers UC
+    # Universal credit has two rows: spending inside the welfare cap (row 18
+    # of the March 2026 table) and outside it (row 43). The welfare cap is the
+    # Charter for Budget Responsibility's limit on welfare spending, not the
+    # household benefit cap. It excludes the State Pension and the payments
+    # most sensitive to the economic cycle: JSA, associated Housing Benefit
+    # and their UC equivalent. DWP's benefit expenditure and caseload tables
+    # (Spring Forecast 2026) give the same £12.876bn for 2025-26 and define
+    # it (Notes) as "the total expenditure ... directed to those in the
+    # Intensive Work Search group"; EFO March 2026 para 4.24 n.20 uses the
+    # same regime. policyengine-uk has no UC conditionality regime. A proxy
+    # built from the legal tests and FRS employment status went from 5% under
+    # to 27% over DWP's 2025-26 figure when one employment-status code was
+    # mapped correctly, too fragile to calibrate to, so the two rows are
+    # targeted as one total. Both sit under "DWP social security",
+    # which covers Great Britain: Northern Ireland's UC is in the "NI social
+    # security" rows.
     try:
-        # UC outside cap = predominantly JSA-conditionality UC
-        uc_outside_row = _find_row(ws, "Universal credit", col="B", max_row=55)
-        # Find the second UC row (outside cap section)
-        for row in range(uc_outside_row + 1, 55):
-            cell_val = ws[f"B{row}"].value
-            if cell_val and str(cell_val).strip().startswith("Universal credit"):
-                values = read_49(row)
-                if values:
-                    targets.append(
-                        Target(
-                            name="obr/universal_credit_outside_cap",
-                            variable="universal_credit",
-                            source="obr",
-                            unit=Unit.GBP,
-                            values=values,
-                            reference_url=ref,
-                            forecast_vintage=vintage,
-                        )
-                    )
-                break
-    except ValueError:
-        logger.warning("OBR welfare: UC outside cap not found")
+        inside_row, outside_row = _universal_credit_rows(ws)
+        inside, outside = read_49(inside_row), read_49(outside_row)
+        if not inside or inside.keys() != outside.keys():
+            raise ValueError("the two universal credit rows cover different years")
+        targets.append(
+            Target(
+                name="obr/universal_credit",
+                variable="universal_credit",
+                source="obr",
+                unit=Unit.GBP,
+                values={year: inside[year] + outside[year] for year in inside},
+                countries=GREAT_BRITAIN,
+                reference_url=ref,
+                forecast_vintage=vintage,
+            )
+        )
+    except ValueError as e:
+        logger.warning("OBR welfare: universal credit not parsed: %s", e)
 
     return targets