Update PolicyEngine bundle to 6.3.4

changelog.d/uk-catalog-membership-filters.fixed.md

@@ -0,0 +1 @@
+Store list-valued UK region filters, such as England's nine English regions, as a JSON array in the catalog's existing filter text, so catalog extraction no longer fails on PolicyEngine 6.3.4.

changelog.d/update-policyengine-bundle-6.3.4.changed.md

@@ -0,0 +1 @@
+Update the PolicyEngine bundle to 6.3.4.

docker/Dockerfile

@@ -1,4 +1,4 @@
 FROM python:3.12
 # Match the API bundle in pyproject.toml; the bundle updater changes both pins.
 # The models extra pins the complete certified runtime package set.
-RUN pip install "policyengine[models]==6.2.1" ipython
+RUN pip install "policyengine[models]==6.3.4" ipython

policyengine_api/data/v2/catalog/extraction.py

@@ -7,6 +7,7 @@
 from datetime import date, datetime, timedelta, timezone
 from enum import Enum
 from importlib import metadata as importlib_metadata
+import json
 import math
 import re
 from typing import Any
@@ -92,6 +93,40 @@ def _optional_text(
     return value
 
 
+def _serialize_region_filter_value(value: object, *, region_code: str) -> str:
+    """Keep scalar text or encode flat membership values in the existing column.
+
+    Since PolicyEngine 6.2.7, the England filter names all nine English regions. The
+    catalog's existing text column can retain that complete list as a JSON
+    array without changing its schema or the representation of scalar filters.
+    """
+    scalar_types = (str, int, float, bool)
+    if isinstance(value, scalar_types):
+        serialized = str(value)
+    elif (
+        isinstance(value, (list, tuple))
+        and value
+        and all(isinstance(item, scalar_types) for item in value)
+    ):
+        try:
+            serialized = json.dumps(
+                value, ensure_ascii=False, separators=(",", ":"), allow_nan=False
+            )
+        except ValueError as error:
+            raise CatalogExtractionError(
+                f"region {region_code!r} has an unsupported filter value"
+            ) from error
+    else:
+        raise CatalogExtractionError(
+            f"region {region_code!r} has an unsupported filter value"
+        )
+    if len(serialized) > 255:
+        raise CatalogExtractionError(
+            f"region {region_code!r} filter value exceeds 255 characters"
+        )
+    return serialized
+
+
 def _type_name(value: object) -> str | None:
     if value is None:
         return None
@@ -592,11 +627,9 @@ def _normalize_country(
                 maximum=128,
             )
             filter_value_source = getattr(strategy, "variable_value", None)
-            if not isinstance(filter_value_source, (str, int, float, bool)):
-                raise CatalogExtractionError(
-                    f"region {code!r} has an unsupported filter value"
-                )
-            filter_value = str(filter_value_source)
+            filter_value = _serialize_region_filter_value(
+                filter_value_source, region_code=code
+            )
             additional_filters = getattr(strategy, "additional_filters", {})
             if additional_filters:
                 raise CatalogExtractionError(

pyproject.toml

@@ -47,7 +47,7 @@ dependencies = [
     # versions certified by this PolicyEngine bundle. Cloud Run installs the
     # complete frozen environment from uv.lock; the generic image installs
     # this requirement directly.
-    "policyengine[models]==6.2.1",
+    "policyengine[models]==6.3.4",
     "pydantic",
     "pymysql",
     "python-dotenv",

tests/integration/test_v2_catalog_installed.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from importlib import metadata as importlib_metadata
+import json
 import os
 
 import pytest
@@ -36,13 +37,15 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None:
         importlib_metadata.version(name) == expected
         for name, expected in expected_dependencies
     )
+    # Counts for the 6.3.4 bundle's models: US 2.2.1 and UK 2.126.12. Keep
+    # this exact assertion so later package changes require explicit review.
     assert catalog.entity_counts() == {
         "models": 2,
         "model_versions": 2,
-        "variables": 7_104,
-        "parameter_nodes": 29_170,
-        "parameters": 103_853,
-        "parameter_values": 1_193_400,
+        "variables": 7_294,
+        "parameter_nodes": 29_300,
+        "parameters": 104_158,
+        "parameter_values": 1_194_256,
         "datasets": 2,
         "regions": 826,
     }
@@ -73,11 +76,37 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None:
     assert {dataset.name for dataset in catalog.country("uk").datasets} == {
         "enhanced_frs_2024_25"
     }
+    england = next(
+        region
+        for region in catalog.country("uk").regions
+        if region.code == "country/england"
+    )
+    assert england.filter_field == "region"
+    assert england.filter_strategy == "row_filter"
+    assert england.filter_value is not None
+    assert json.loads(england.filter_value) == [
+        "NORTH_EAST",
+        "NORTH_WEST",
+        "YORKSHIRE",
+        "EAST_MIDLANDS",
+        "WEST_MIDLANDS",
+        "EAST_OF_ENGLAND",
+        "LONDON",
+        "SOUTH_EAST",
+        "SOUTH_WEST",
+    ]
+
+    # 6.3.4 points US state and congressional-district regions at the national
+    # dataset by explicit path (PolicyEngine/policyengine.py#604), so every US
+    # region uses it and only places still fall back to it.
+    us_catalog = catalog.country("us")
+    us_dataset_names = {dataset.id: dataset.name for dataset in us_catalog.datasets}
+    assert {
+        us_dataset_names[region.default_dataset_id] for region in us_catalog.regions
+    } == {"populace_us_2024"}
     assert [
         (summary.region_type, summary.count)
-        for summary in catalog.country("us").fallback_summaries
+        for summary in us_catalog.fallback_summaries
     ] == [
-        ("congressional_district", 436),
         ("place", 333),
-        ("state", 51),
     ]

tests/unit/v2/test_catalog_extraction.py

@@ -5,6 +5,7 @@
 import ast
 from datetime import datetime, timezone
 from importlib.metadata import PackageNotFoundError
+import json
 from pathlib import Path
 from types import SimpleNamespace
 
@@ -118,6 +119,66 @@ def item(self):
     assert normalize_json_value(float("-inf")) == "-Infinity"
 
 
+@pytest.mark.parametrize("value", [("NORTH_EAST", "LONDON"), ["NORTH_EAST", "LONDON"]])
+def test_serializes_region_membership_filters_in_the_existing_text_field(
+    value: object,
+) -> None:
+    models = source_models()
+    models["uk"].region_registry[1].scoping_strategy.variable_value = value
+
+    catalog = extract_catalog(
+        bundle=bundle(),
+        policyengine_version=POLICYENGINE_VERSION,
+        models=models,
+        installed_version=installed_version,
+    )
+    england = next(
+        region
+        for region in catalog.country("uk").regions
+        if region.code == "country/england"
+    )
+
+    assert isinstance(england.filter_value, str)
+    assert json.loads(england.filter_value) == ["NORTH_EAST", "LONDON"]
+
+
+@pytest.mark.parametrize("value", ["ENGLAND", 6, True, 1.5])
+def test_preserves_existing_scalar_region_filter_serialization(value: object) -> None:
+    models = source_models()
+    models["uk"].region_registry[1].scoping_strategy.variable_value = value
+
+    catalog = extract_catalog(
+        bundle=bundle(),
+        policyengine_version=POLICYENGINE_VERSION,
+        models=models,
+        installed_version=installed_version,
+    )
+    england = next(
+        region
+        for region in catalog.country("uk").regions
+        if region.code == "country/england"
+    )
+
+    assert england.filter_value == str(value)
+
+
+@pytest.mark.parametrize(
+    "value",
+    [[], [["LONDON"]], [None], [object()], [float("nan")], ["x" * 256], "x" * 256],
+)
+def test_rejects_invalid_or_oversized_region_membership_values(value: object) -> None:
+    models = source_models()
+    models["uk"].region_registry[1].scoping_strategy.variable_value = value
+
+    with pytest.raises(CatalogExtractionError, match="filter value"):
+        extract_catalog(
+            bundle=bundle(),
+            policyengine_version=POLICYENGINE_VERSION,
+            models=models,
+            installed_version=installed_version,
+        )
+
+
 @pytest.mark.parametrize("value", [float("nan"), object(), {1: "x"}])
 def test_rejects_non_json_values(value: object) -> None:
     with pytest.raises(CatalogExtractionError):

tests/unit/v2/test_stage12_comparison_models.py

@@ -65,6 +65,8 @@ def _report(**changes: object) -> Stage12ComparisonReport:
         "data_package_name": "policyengine-us-data",
         "data_package_version": "1.0.0",
         "data_artifact_revision": "revision",
+        "created_at": NOW,
+        "updated_at": NOW,
         "retention_expires_at": NOW + timedelta(days=30),
     }
     fields.update(changes)
@@ -80,6 +82,8 @@ def _simulation(**changes: object) -> Stage12ComparisonSimulation:
         "modal_application": "policyengine-v2-worker-5-2-0",
         "simulation_callable": "run_simulation",
         "version_manifest_sha256": DIGEST_A,
+        "created_at": NOW,
+        "updated_at": NOW,
         "retention_expires_at": NOW + timedelta(days=30),
     }
     fields.update(changes)