Require PolicyEngine 6.3.4 for regional simulations

changelog.d/update-policyengine-bundle-6.3.4.changed.md

@@ -0,0 +1 @@
+Update the PolicyEngine bundle to 6.3.4 so state and congressional-district simulations use the certified ACS local-area dataset. 6.3.4 adds backward compatibility for that legacy source: ACS group-quarters units are outside the SPM measurement, its ASEC and PUF records take the certified SPM independence role, and a poverty group with nobody measured reports a null rate instead of 0, which economic impact results now pass through. Adopt the bundle's UK model 2.126.12 and certified datasets. Import multi-region membership filters into the existing catalog text column without changing the database schema.

docker/Dockerfile

@@ -1,4 +1,4 @@
 FROM python:3.12
 # Match the API bundle in pyproject.toml; the bundle updater changes both pins.
 # The models extra pins the complete certified runtime package set.
-RUN pip install "policyengine[models]==6.2.1" ipython
+RUN pip install "policyengine[models]==6.3.4" ipython

policyengine_api/data/v2/catalog/extraction.py

@@ -7,6 +7,7 @@
 from datetime import date, datetime, timedelta, timezone
 from enum import Enum
 from importlib import metadata as importlib_metadata
+import json
 import math
 import re
 from typing import Any
@@ -92,6 +93,40 @@ def _optional_text(
     return value
 
 
+def _serialize_region_filter_value(value: object, *, region_code: str) -> str:
+    """Keep scalar text or encode flat membership values in the existing column.
+
+    Since PolicyEngine 6.2.7, the England filter names all nine English regions. The
+    catalog's existing text column can retain that complete list as a JSON
+    array without changing its schema or the representation of scalar filters.
+    """
+    scalar_types = (str, int, float, bool)
+    if isinstance(value, scalar_types):
+        serialized = str(value)
+    elif (
+        isinstance(value, (list, tuple))
+        and value
+        and all(isinstance(item, scalar_types) for item in value)
+    ):
+        try:
+            serialized = json.dumps(
+                value, ensure_ascii=False, separators=(",", ":"), allow_nan=False
+            )
+        except ValueError as error:
+            raise CatalogExtractionError(
+                f"region {region_code!r} has an unsupported filter value"
+            ) from error
+    else:
+        raise CatalogExtractionError(
+            f"region {region_code!r} has an unsupported filter value"
+        )
+    if len(serialized) > 255:
+        raise CatalogExtractionError(
+            f"region {region_code!r} filter value exceeds 255 characters"
+        )
+    return serialized
+
+
 def _type_name(value: object) -> str | None:
     if value is None:
         return None
@@ -592,11 +627,9 @@ def _normalize_country(
                 maximum=128,
             )
             filter_value_source = getattr(strategy, "variable_value", None)
-            if not isinstance(filter_value_source, (str, int, float, bool)):
-                raise CatalogExtractionError(
-                    f"region {code!r} has an unsupported filter value"
-                )
-            filter_value = str(filter_value_source)
+            filter_value = _serialize_region_filter_value(
+                filter_value_source, region_code=code
+            )
             additional_filters = getattr(strategy, "additional_filters", {})
             if additional_filters:
                 raise CatalogExtractionError(

pyproject.toml

@@ -47,7 +47,7 @@ dependencies = [
     # versions certified by this PolicyEngine bundle. Cloud Run installs the
     # complete frozen environment from uv.lock; the generic image installs
     # this requirement directly.
-    "policyengine[models]==6.2.1",
+    "policyengine[models]==6.3.4",
     "pydantic",
     "pymysql",
     "python-dotenv",

tests/integration/test_v2_catalog_installed.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from importlib import metadata as importlib_metadata
+import json
 import os
 
 import pytest
@@ -36,14 +37,19 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None:
         importlib_metadata.version(name) == expected
         for name, expected in expected_dependencies
     )
+    # Counts for the 6.3.4 bundle's models: US 2.2.1 and UK 2.126.12. Measured
+    # with the published 6.3.1 bundle, which installs the same model versions;
+    # 6.3.4 changes no model package. Keep this exact assertion so later
+    # package changes require explicit review.
     assert catalog.entity_counts() == {
         "models": 2,
         "model_versions": 2,
-        "variables": 7_104,
-        "parameter_nodes": 29_170,
-        "parameters": 103_853,
-        "parameter_values": 1_193_400,
-        "datasets": 2,
+        "variables": 7_294,
+        "parameter_nodes": 29_300,
+        "parameters": 104_158,
+        "parameter_values": 1_194_256,
+        # US national and ACS-local datasets, plus the UK default dataset.
+        "datasets": 3,
         "regions": 826,
     }
 
@@ -68,16 +74,50 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None:
         assert total_values == country.entity_counts()["parameter_values"]
 
     assert {dataset.name for dataset in catalog.country("us").datasets} == {
-        "populace_us_2024"
+        "populace_us_2024",
+        "populace_us_2024_acs_local",
     }
     assert {dataset.name for dataset in catalog.country("uk").datasets} == {
         "enhanced_frs_2024_25"
     }
+    england = next(
+        region
+        for region in catalog.country("uk").regions
+        if region.code == "country/england"
+    )
+    assert england.filter_field == "region"
+    assert england.filter_strategy == "row_filter"
+    assert england.filter_value is not None
+    assert json.loads(england.filter_value) == [
+        "NORTH_EAST",
+        "NORTH_WEST",
+        "YORKSHIRE",
+        "EAST_MIDLANDS",
+        "WEST_MIDLANDS",
+        "EAST_OF_ENGLAND",
+        "LONDON",
+        "SOUTH_EAST",
+        "SOUTH_WEST",
+    ]
+    us_catalog = catalog.country("us")
+    us_dataset_names = {dataset.id: dataset.name for dataset in us_catalog.datasets}
+    expected_dataset_by_region_type = {
+        "national": "populace_us_2024",
+        "place": "populace_us_2024",
+        "state": "populace_us_2024_acs_local",
+        "congressional_district": "populace_us_2024_acs_local",
+    }
+    for region in us_catalog.regions:
+        assert (
+            us_dataset_names[region.default_dataset_id]
+            == (expected_dataset_by_region_type[region.region_type])
+        )
+
+    # State and district records now have explicit certified dataset paths;
+    # only places still use the national dataset when no path is provided.
     assert [
         (summary.region_type, summary.count)
-        for summary in catalog.country("us").fallback_summaries
+        for summary in us_catalog.fallback_summaries
     ] == [
-        ("congressional_district", 436),
         ("place", 333),
-        ("state", 51),
     ]

tests/unit/services/test_economy_service.py

@@ -331,6 +331,53 @@ def test__given_computing_impact_with_succeeded_execution__returns_completed_res
             )
             mock_reform_impacts_service.set_complete_reform_impact.assert_called_once()
 
+        def test__given_succeeded_execution_with_unmeasured_poverty_group__keeps_null_rate(
+            self,
+            economy_service,
+            base_params,
+            mock_country_package_versions,
+            mock_policyengine_version,
+            mock_policy_service,
+            mock_reform_impacts_service,
+            mock_simulation_entrypoint,
+            mock_logger,
+            mock_datetime,
+            mock_submission_claim_id,
+        ):
+            # The simulation API reports a null rate for a poverty group with
+            # nobody inside the poverty measurement (PovertyRateValue), for
+            # example one made up entirely of ACS group-quarters residents.
+            # That is not a measured rate of zero, so it must pass through.
+            unmeasured = {"baseline": None, "reform": None}
+            measured = {"baseline": 0.12, "reform": 0.1}
+            poverty_by_age = {
+                "child": measured,
+                "adult": measured,
+                "senior": unmeasured,
+                "all": measured,
+            }
+            impact = {
+                **MOCK_REFORM_IMPACT_DATA,
+                "poverty": {"poverty": poverty_by_age, "deep_poverty": poverty_by_age},
+            }
+            computing_impact = create_mock_reform_impact(status="computing")
+            mock_reform_impacts_service.get_all_reform_impacts_by_options_hash_prefix.return_value = [
+                computing_impact
+            ]
+            mock_simulation_entrypoint.get_execution_status.return_value = "complete"
+            mock_simulation_entrypoint.get_execution_result.return_value = impact
+
+            result = economy_service.get_economic_impact(**base_params)
+
+            assert result.status == ImpactStatus.OK
+            assert result.data["poverty"]["poverty"]["senior"] == unmeasured
+            assert result.data["poverty"]["poverty"]["child"] == measured
+            stored = mock_reform_impacts_service.set_complete_reform_impact.call_args
+            assert (
+                stored.kwargs["reform_impact_json"]["poverty"]["deep_poverty"]["senior"]
+                == unmeasured
+            )
+
         def test__given_computing_impact_with_failed_execution__returns_error_result(
             self,
             economy_service,

tests/unit/test_constants.py

@@ -237,3 +237,21 @@ def test__uses_policyengine_bundle_versions_for_us_uk_and_core(self):
         assert POLICYENGINE_CORE_VERSION == expected_versions["core"]
         assert COUNTRY_PACKAGE_VERSIONS["us"] == expected_versions["us"]
         assert COUNTRY_PACKAGE_VERSIONS["uk"] == expected_versions["uk"]
+
+    def test__us_bundle_certifies_acs_local_regional_dataset(self):
+        manifest = json.loads(get_py_manifest().read_text(encoding="utf-8"))
+        us_release = manifest["data_releases"]["us"]
+        region_datasets = us_release["region_datasets"]
+
+        assert us_release["default_dataset"] == "populace_us_2024"
+        assert (
+            region_datasets["state"]["path_template"] == "populace_us_2024_acs_local.h5"
+        )
+        assert (
+            region_datasets["congressional_district"]["path_template"]
+            == "populace_us_2024_acs_local.h5"
+        )
+        assert "populace_us_2024_acs_local" in us_release["datasets"]
+        assert "populace_us_2024_acs_local" not in manifest.get(
+            "dataset_overlays", {}
+        ).get("us", {})

tests/unit/v2/test_catalog_extraction.py

@@ -5,6 +5,7 @@
 import ast
 from datetime import datetime, timezone
 from importlib.metadata import PackageNotFoundError
+import json
 from pathlib import Path
 from types import SimpleNamespace
 
@@ -118,6 +119,66 @@ def item(self):
     assert normalize_json_value(float("-inf")) == "-Infinity"
 
 
+@pytest.mark.parametrize("value", [("NORTH_EAST", "LONDON"), ["NORTH_EAST", "LONDON"]])
+def test_serializes_region_membership_filters_in_the_existing_text_field(
+    value: object,
+) -> None:
+    models = source_models()
+    models["uk"].region_registry[1].scoping_strategy.variable_value = value
+
+    catalog = extract_catalog(
+        bundle=bundle(),
+        policyengine_version=POLICYENGINE_VERSION,
+        models=models,
+        installed_version=installed_version,
+    )
+    england = next(
+        region
+        for region in catalog.country("uk").regions
+        if region.code == "country/england"
+    )
+
+    assert isinstance(england.filter_value, str)
+    assert json.loads(england.filter_value) == ["NORTH_EAST", "LONDON"]
+
+
+@pytest.mark.parametrize("value", ["ENGLAND", 6, True, 1.5])
+def test_preserves_existing_scalar_region_filter_serialization(value: object) -> None:
+    models = source_models()
+    models["uk"].region_registry[1].scoping_strategy.variable_value = value
+
+    catalog = extract_catalog(
+        bundle=bundle(),
+        policyengine_version=POLICYENGINE_VERSION,
+        models=models,
+        installed_version=installed_version,
+    )
+    england = next(
+        region
+        for region in catalog.country("uk").regions
+        if region.code == "country/england"
+    )
+
+    assert england.filter_value == str(value)
+
+
+@pytest.mark.parametrize(
+    "value",
+    [[], [["LONDON"]], [None], [object()], [float("nan")], ["x" * 256], "x" * 256],
+)
+def test_rejects_invalid_or_oversized_region_membership_values(value: object) -> None:
+    models = source_models()
+    models["uk"].region_registry[1].scoping_strategy.variable_value = value
+
+    with pytest.raises(CatalogExtractionError, match="filter value"):
+        extract_catalog(
+            bundle=bundle(),
+            policyengine_version=POLICYENGINE_VERSION,
+            models=models,
+            installed_version=installed_version,
+        )
+
+
 @pytest.mark.parametrize("value", [float("nan"), object(), {1: "x"}])
 def test_rejects_non_json_values(value: object) -> None:
     with pytest.raises(CatalogExtractionError):