Use Rent Service Scotland's own lists of rents for Scottish percentile ratios

changelog.d/2036.fixed.md

@@ -0,0 +1 @@
+- Replaced the Scottish lists of rents, which were copies of English BRMAs' lists, with Rent Service Scotland's own lists released under FOI 202200303624, so percentile reforms scale Scottish LHA rates by Scotland's rent distributions.

policyengine_uk/parameters/gov/dwp/LHA/README.md

@@ -1 +1,9 @@
 # Local Housing Allowance
+
+## Lists of rents
+
+`lha_list_of_rents.csv.gz` holds one row per rent in the lists behind the April 2019 and April 2020 determinations (`year`), with its BRMA and LHA category. The model uses it only to turn the published 30th percentile into another percentile when a reform changes `percentile`.
+
+- **England:** the Valuation Office Agency's published lists.
+- **Scotland:** Rent Service Scotland's records, released by the Scottish Government under FOI 202200303624. The list for April Y is the sheet for the year to September Y-1. `scripts/build_scottish_list_of_rents.py` rebuilds these rows from the pinned workbook. In 522 of 540 cells (April 2017-2022), those sheets reproduce the published 30th percentiles to the penny.
+- **Wales and Northern Ireland:** copies of English BRMAs' lists, each matched on the nation's April 2019 rate (issue #2036). Their percentile reforms use the median English ratio for the category instead.

policyengine_uk/tests/test_lha_published_rates.py

@@ -373,21 +373,74 @@ def test_freezing_the_2024_reset_holds_the_2020_rates():
     assert held == pytest.approx(187.56, abs=0.001)
 
 
-def test_devolved_percentile_ratios_use_the_english_median():
-    """Welsh, Scottish and NI lists of rents in the file copy English ones.
+def test_welsh_and_ni_percentile_ratios_use_the_english_median():
+    """The Welsh and NI lists of rents in the file copy English ones.
 
     So a percentile reform scales their published rates by the median English
     ratio for the category, not by a copied list's shape.
     """
-    from policyengine_uk.utils.lha import _percentile_ratios, _sorted_list_of_rents
+    from policyengine_uk.utils.lha import (
+        NON_ENGLISH_REGIONS,
+        _percentile_ratios,
+        _sorted_list_of_rents,
+        statutory_percentile,
+    )
 
-    lists, devolved = _sorted_list_of_rents()
-    assert {"LOTHIAN", "CARDIFF", "BELFAST"} <= devolved
-    assert not devolved & {"MAIDSTONE", "CENTRAL_LONDON"}
+    lists, regions = _sorted_list_of_rents()
     ratios = _percentile_ratios(0.5)
     brmas = published_rates().brmas
-    english = [i for i, b in enumerate(brmas) if b not in devolved]
-    for b in ("LOTHIAN", "CARDIFF", "BELFAST"):
-        row = ratios[brmas.get_loc(b)]
-        np.testing.assert_allclose(row, np.median(ratios[english], axis=0))
+    english = {category: [] for category in "ABCDE"}
+    for (brma, category), rents in lists.items():
+        if regions[brma] not in NON_ENGLISH_REGIONS:
+            english[category].append(
+                statutory_percentile(rents, 0.5) / statutory_percentile(rents, 0.3)
+            )
+    median = [np.median(english[category]) for category in "ABCDE"]
+    for b in ("CARDIFF", "BELFAST"):
+        np.testing.assert_allclose(ratios[brmas.get_loc(b)], median)
     assert (ratios >= 1).all()
+
+
+def test_scottish_percentile_ratios_use_scotlands_own_lists():
+    """Ratios from Rent Service Scotland's April 2020 lists (FOI 202200303624)."""
+    from policyengine_uk.utils.lha import _percentile_ratios
+
+    ratios = _percentile_ratios(0.5)
+    brmas = published_rates().brmas
+    assert ratios[brmas.get_loc("GREATER_GLASGOW"), 3] == pytest.approx(1.267, abs=5e-4)
+    assert ratios[brmas.get_loc("LOTHIAN"), 2] == pytest.approx(1.091, abs=5e-4)
+
+
+def test_scottish_lists_are_rent_service_scotlands_own():
+    """Scotland's lists (FOI 202200303624) reproduce the published 30th
+    percentiles and copy no English list, unlike the rows they replaced."""
+    from policyengine_uk.utils.lha import (
+        LIST_OF_RENTS_PATH,
+        NON_ENGLISH_REGIONS,
+        round_half_up,
+        statutory_percentile,
+    )
+
+    rents = pd.read_csv(LIST_OF_RENTS_PATH)
+    published = pd.read_csv(PUBLISHED_RATES_PATH).set_index(
+        ["year", "brma", "lha_category"]
+    )
+    blocks = {
+        key: tuple(np.sort(group.weekly_rent.to_numpy()))
+        for key, group in rents.groupby(["region", "year", "brma", "lha_category"])
+    }
+    english = {
+        rents
+        for (region, *_), rents in blocks.items()
+        if region not in NON_ENGLISH_REGIONS
+    }
+    scottish = {k: v for k, v in blocks.items() if k[0] == "SCOTLAND"}
+    assert len(scottish) == 2 * 18 * 5
+    assert not english & set(scottish.values())
+    exact = sum(
+        float(round_half_up(statutory_percentile(np.array(v), 0.3)))
+        == pytest.approx(published.percentile_30[(year, brma, category)], abs=0.001)
+        for (_, year, brma, category), v in scottish.items()
+    )
+    # 87 of 90 cells for April 2019 and 86 of 90 for April 2020 to the penny.
+    assert exact >= 170

policyengine_uk/utils/build_scottish_list_of_rents.py

@@ -0,0 +1,96 @@
+"""Put Scotland's own lists of rents into ``lha_list_of_rents.csv.gz``.
+
+The Scottish rows that came with the file were copies of English BRMAs' lists.
+The Scottish Government released Rent Service Scotland's market evidence, one
+row per rent, under FOI 202200303624. Each worksheet holds the rents collected
+in the twelve months to September of its year, which set the next April's
+determination, so sheet ``Y - 1`` is the list for April ``Y``. This replaces all
+Scottish rows with sheets 2018 and 2019 as the lists for 2019 and 2020, keeping
+only each rent, its BRMA and its LHA category, and leaves every other row as it
+was.
+
+Spreadsheet readers are not runtime dependencies, so run the build with them:
+
+    uv run --with openpyxl python scripts/build_scottish_list_of_rents.py \
+        --cache ~/.cache/lha
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import pandas as pd
+
+from policyengine_uk.utils.build_lha_published_rates import fetch, scottish_brma
+from policyengine_uk.utils.lha import CATEGORIES, LIST_OF_RENTS_PATH
+
+SOURCE = pd.Series(
+    {
+        "id": "scot-foi-202200303624-brma-2016-2021",
+        "file": "FOI-202200303624-Information-Released-BRMA-2016-2021.xlsx",
+        "url": (
+            "https://www.gov.scot/binaries/content/documents/govscot/publications/"
+            "foi-eir-release/2023/08/foi-202200303624/documents/"
+            "foi-202200303624---information-released---brma-2016-2021/"
+            "foi-202200303624---information-released---brma-2016-2021/"
+            "govscot%3Adocument/FOI%2B202200303624%2B-%2BInformation%2BReleased"
+            "%2B-%2BBRMA%2B2016-2021.xlsx?download=true"
+        ),
+        "landing_page": "https://www.gov.scot/publications/foi-202200303624/",
+        "sha256": "dba86fa1cce920c8f646460bb13e1d99e16927d57fdf290e874f9631e804ed06",
+    }
+)
+YEARS = (2019, 2020)  # April determinations; sheet = year - 1.
+
+
+def scottish_lists(path: Path) -> pd.DataFrame:
+    frames = []
+    for year in YEARS:
+        sheet = pd.read_excel(path, sheet_name=str(year - 1)).dropna(how="all")
+        # Sheet 2019 ends with two rents that name no BRMA; they cannot be placed.
+        assert sheet["BRMA"].isna().sum() == (2 if year == 2020 else 0)
+        sheet = sheet[sheet["BRMA"].notna()]
+        assert (sheet["FREQUENCY"] == "Weekly").all()
+        assert (
+            sheet["YEAR ENDING SEPTEMBER"] == f"{year - 2}/{str(year - 1)[2:]}"
+        ).all()
+        category = sheet["LHA TYPE"].str.strip().str.removeprefix("Cat ")
+        assert category.isin(CATEGORIES).all()
+        frames.append(
+            pd.DataFrame(
+                {
+                    # Rents are whole pennies; rounding only guards the float.
+                    "weekly_rent": sheet["NET RENT"].astype(float).round(2),
+                    "year": year,
+                    "brma": sheet["BRMA"].map(scottish_brma),
+                    "lha_category": category,
+                    "region": "SCOTLAND",
+                }
+            )
+        )
+    lists = pd.concat(frames, ignore_index=True)
+    return lists.sort_values(["year", "brma", "lha_category", "weekly_rent"])
+
+
+def build(cache: Path) -> pd.DataFrame:
+    cache.mkdir(parents=True, exist_ok=True)
+    rents = pd.read_csv(LIST_OF_RENTS_PATH)
+    scotland = scottish_lists(fetch(SOURCE, cache))
+    assert scotland.brma.nunique() == 18
+    rents = pd.concat([rents[rents.region != "SCOTLAND"], scotland], ignore_index=True)
+    return rents
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--cache", type=Path, default=Path.home() / ".cache" / "lha")
+    args = parser.parse_args(argv)
+    rents = build(args.cache)
+    rents.to_csv(
+        LIST_OF_RENTS_PATH,
+        index=False,
+        compression={"method": "gzip", "mtime": 0},
+    )
+    print(f"Wrote {len(rents):,} rents to {LIST_OF_RENTS_PATH}")
+    return 0

policyengine_uk/utils/lha.py

@@ -131,26 +131,28 @@ def published_rates() -> PublishedRates:
     return PublishedRates(pd.read_csv(PUBLISHED_RATES_PATH))
 
 
-DEVOLVED_REGIONS = ("WALES", "SCOTLAND", "NORTHERN_IRELAND")
+NON_ENGLISH_REGIONS = ("WALES", "SCOTLAND", "NORTHERN_IRELAND")
+# The Welsh and Northern Irish lists in the file are copies of English BRMAs'
+# lists, often of another category, so they do not describe their own areas.
+# The Scottish lists are Rent Service Scotland's own (FOI 202200303624; see
+# ``utils/build_scottish_list_of_rents.py``).
+COPIED_LIST_REGIONS = ("WALES", "NORTHERN_IRELAND")
 
 
 @lru_cache(maxsize=1)
-def _sorted_list_of_rents() -> tuple[dict, set]:
-    """The latest list of rents, sorted, keyed by (BRMA, category).
+def _sorted_list_of_rents() -> tuple[dict, dict]:
+    """Each region's latest list of rents, sorted, keyed by (BRMA, category).
 
-    Returns the lists and the set of BRMAs outside England. Every Welsh,
-    Scottish and Northern Irish list in the file is a copy of an English
-    BRMA's list, often of another category, so those lists do not describe
-    their own areas.
+    Returns the lists and each BRMA's region.
     """
     rents = pd.read_csv(LIST_OF_RENTS_PATH)
-    rents = rents[rents.year == rents.year.max()]
+    rents = rents[rents.year == rents.groupby("region").year.transform("max")]
     lists = {
         key: np.sort(group.weekly_rent.to_numpy(dtype=float))
         for key, group in rents.groupby(["brma", "lha_category"])
     }
-    devolved = set(rents.brma[rents.region.isin(DEVOLVED_REGIONS)])
-    return lists, devolved
+    regions = rents.groupby("brma").region.first().to_dict()
+    return lists, regions
 
 
 def statutory_percentile(rents: np.ndarray, percentile: float) -> float:
@@ -175,25 +177,27 @@ def _percentile_ratios(percentile: float) -> np.ndarray:
 
     The published tables give only the 30th percentile, so another percentile
     is reached by scaling it by this ratio from the model's list of rents.
-    Only the English lists describe their own areas (see
-    ``_sorted_list_of_rents``), so BRMAs in Wales, Scotland and Northern
-    Ireland take the median English ratio for the same category. Cells with
-    no list keep a ratio of one.
+    English and Scottish BRMAs use their own lists (April 2020 for both).
+    The Welsh and Northern Irish lists are copies of English ones (see
+    ``_sorted_list_of_rents``), so those BRMAs take the median English ratio
+    for the same category. Cells with no list keep a ratio of one.
     """
     rates = published_rates()
-    lists, devolved = _sorted_list_of_rents()
+    lists, regions = _sorted_list_of_rents()
+    copied = {b for b, r in regions.items() if r in COPIED_LIST_REGIONS}
     ratios = np.ones((len(rates.brmas), len(CATEGORIES)))
     english = {category: [] for category in CATEGORIES}
     for (brma, category), rents in lists.items():
         position = rates.brmas.get_indexer([brma])[0]
-        if position < 0 or category not in CATEGORIES or brma in devolved:
+        if position < 0 or category not in CATEGORIES or brma in copied:
             continue
         base = statutory_percentile(rents, PUBLISHED_PERCENTILE)
         if base > 0:
             ratio = statutory_percentile(rents, percentile) / base
             ratios[position, CATEGORIES.index(category)] = ratio
-            english[category].append(ratio)
-    for brma in devolved:
+            if regions[brma] not in NON_ENGLISH_REGIONS:
+                english[category].append(ratio)
+    for brma in copied:
         position = rates.brmas.get_indexer([brma])[0]
         if position < 0:
             continue

scripts/build_scottish_list_of_rents.py

@@ -0,0 +1,10 @@
+from pathlib import Path
+import sys
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from policyengine_uk.utils.build_scottish_list_of_rents import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())