diff --git a/changelog.d/uk-catalog-membership-filters.fixed.md b/changelog.d/uk-catalog-membership-filters.fixed.md new file mode 100644 index 000000000..fbfd0cf8b --- /dev/null +++ b/changelog.d/uk-catalog-membership-filters.fixed.md @@ -0,0 +1 @@ +Store list-valued UK region filters, such as England's nine English regions, as a JSON array in the catalog's existing filter text, so catalog extraction no longer fails on PolicyEngine 6.3.4. diff --git a/changelog.d/update-policyengine-bundle-6.3.4.changed.md b/changelog.d/update-policyengine-bundle-6.3.4.changed.md new file mode 100644 index 000000000..05f0666c9 --- /dev/null +++ b/changelog.d/update-policyengine-bundle-6.3.4.changed.md @@ -0,0 +1 @@ +Update the PolicyEngine bundle to 6.3.4. diff --git a/docker/Dockerfile b/docker/Dockerfile index 83b3cabe4..73fa86ec2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ FROM python:3.12 # Match the API bundle in pyproject.toml; the bundle updater changes both pins. # The models extra pins the complete certified runtime package set. -RUN pip install "policyengine[models]==6.2.1" ipython +RUN pip install "policyengine[models]==6.3.4" ipython diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 7d2302920..c8bedeeed 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -7,6 +7,7 @@ from datetime import date, datetime, timedelta, timezone from enum import Enum from importlib import metadata as importlib_metadata +import json import math import re from typing import Any @@ -92,6 +93,40 @@ def _optional_text( return value +def _serialize_region_filter_value(value: object, *, region_code: str) -> str: + """Keep scalar text or encode flat membership values in the existing column. + + Since PolicyEngine 6.2.7, the England filter names all nine English regions. The + catalog's existing text column can retain that complete list as a JSON + array without changing its schema or the representation of scalar filters. + """ + scalar_types = (str, int, float, bool) + if isinstance(value, scalar_types): + serialized = str(value) + elif ( + isinstance(value, (list, tuple)) + and value + and all(isinstance(item, scalar_types) for item in value) + ): + try: + serialized = json.dumps( + value, ensure_ascii=False, separators=(",", ":"), allow_nan=False + ) + except ValueError as error: + raise CatalogExtractionError( + f"region {region_code!r} has an unsupported filter value" + ) from error + else: + raise CatalogExtractionError( + f"region {region_code!r} has an unsupported filter value" + ) + if len(serialized) > 255: + raise CatalogExtractionError( + f"region {region_code!r} filter value exceeds 255 characters" + ) + return serialized + + def _type_name(value: object) -> str | None: if value is None: return None @@ -592,11 +627,9 @@ def _normalize_country( maximum=128, ) filter_value_source = getattr(strategy, "variable_value", None) - if not isinstance(filter_value_source, (str, int, float, bool)): - raise CatalogExtractionError( - f"region {code!r} has an unsupported filter value" - ) - filter_value = str(filter_value_source) + filter_value = _serialize_region_filter_value( + filter_value_source, region_code=code + ) additional_filters = getattr(strategy, "additional_filters", {}) if additional_filters: raise CatalogExtractionError( diff --git a/pyproject.toml b/pyproject.toml index aa855f79f..9fb3d6af2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dependencies = [ # versions certified by this PolicyEngine bundle. Cloud Run installs the # complete frozen environment from uv.lock; the generic image installs # this requirement directly. - "policyengine[models]==6.2.1", + "policyengine[models]==6.3.4", "pydantic", "pymysql", "python-dotenv", diff --git a/tests/integration/test_v2_catalog_installed.py b/tests/integration/test_v2_catalog_installed.py index 88718b122..06f7e79e2 100644 --- a/tests/integration/test_v2_catalog_installed.py +++ b/tests/integration/test_v2_catalog_installed.py @@ -3,6 +3,7 @@ from __future__ import annotations from importlib import metadata as importlib_metadata +import json import os import pytest @@ -36,13 +37,15 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: importlib_metadata.version(name) == expected for name, expected in expected_dependencies ) + # Counts for the 6.3.4 bundle's models: US 2.2.1 and UK 2.126.12. Keep + # this exact assertion so later package changes require explicit review. assert catalog.entity_counts() == { "models": 2, "model_versions": 2, - "variables": 7_104, - "parameter_nodes": 29_170, - "parameters": 103_853, - "parameter_values": 1_193_400, + "variables": 7_294, + "parameter_nodes": 29_300, + "parameters": 104_158, + "parameter_values": 1_194_256, "datasets": 2, "regions": 826, } @@ -73,11 +76,37 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: assert {dataset.name for dataset in catalog.country("uk").datasets} == { "enhanced_frs_2024_25" } + england = next( + region + for region in catalog.country("uk").regions + if region.code == "country/england" + ) + assert england.filter_field == "region" + assert england.filter_strategy == "row_filter" + assert england.filter_value is not None + assert json.loads(england.filter_value) == [ + "NORTH_EAST", + "NORTH_WEST", + "YORKSHIRE", + "EAST_MIDLANDS", + "WEST_MIDLANDS", + "EAST_OF_ENGLAND", + "LONDON", + "SOUTH_EAST", + "SOUTH_WEST", + ] + + # 6.3.4 points US state and congressional-district regions at the national + # dataset by explicit path (PolicyEngine/policyengine.py#604), so every US + # region uses it and only places still fall back to it. + us_catalog = catalog.country("us") + us_dataset_names = {dataset.id: dataset.name for dataset in us_catalog.datasets} + assert { + us_dataset_names[region.default_dataset_id] for region in us_catalog.regions + } == {"populace_us_2024"} assert [ (summary.region_type, summary.count) - for summary in catalog.country("us").fallback_summaries + for summary in us_catalog.fallback_summaries ] == [ - ("congressional_district", 436), ("place", 333), - ("state", 51), ] diff --git a/tests/unit/v2/test_catalog_extraction.py b/tests/unit/v2/test_catalog_extraction.py index 4e07ede36..cb43c4508 100644 --- a/tests/unit/v2/test_catalog_extraction.py +++ b/tests/unit/v2/test_catalog_extraction.py @@ -5,6 +5,7 @@ import ast from datetime import datetime, timezone from importlib.metadata import PackageNotFoundError +import json from pathlib import Path from types import SimpleNamespace @@ -118,6 +119,66 @@ def item(self): assert normalize_json_value(float("-inf")) == "-Infinity" +@pytest.mark.parametrize("value", [("NORTH_EAST", "LONDON"), ["NORTH_EAST", "LONDON"]]) +def test_serializes_region_membership_filters_in_the_existing_text_field( + value: object, +) -> None: + models = source_models() + models["uk"].region_registry[1].scoping_strategy.variable_value = value + + catalog = extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + england = next( + region + for region in catalog.country("uk").regions + if region.code == "country/england" + ) + + assert isinstance(england.filter_value, str) + assert json.loads(england.filter_value) == ["NORTH_EAST", "LONDON"] + + +@pytest.mark.parametrize("value", ["ENGLAND", 6, True, 1.5]) +def test_preserves_existing_scalar_region_filter_serialization(value: object) -> None: + models = source_models() + models["uk"].region_registry[1].scoping_strategy.variable_value = value + + catalog = extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + england = next( + region + for region in catalog.country("uk").regions + if region.code == "country/england" + ) + + assert england.filter_value == str(value) + + +@pytest.mark.parametrize( + "value", + [[], [["LONDON"]], [None], [object()], [float("nan")], ["x" * 256], "x" * 256], +) +def test_rejects_invalid_or_oversized_region_membership_values(value: object) -> None: + models = source_models() + models["uk"].region_registry[1].scoping_strategy.variable_value = value + + with pytest.raises(CatalogExtractionError, match="filter value"): + extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + @pytest.mark.parametrize("value", [float("nan"), object(), {1: "x"}]) def test_rejects_non_json_values(value: object) -> None: with pytest.raises(CatalogExtractionError): diff --git a/tests/unit/v2/test_stage12_comparison_models.py b/tests/unit/v2/test_stage12_comparison_models.py index 00a372454..21c264981 100644 --- a/tests/unit/v2/test_stage12_comparison_models.py +++ b/tests/unit/v2/test_stage12_comparison_models.py @@ -65,6 +65,8 @@ def _report(**changes: object) -> Stage12ComparisonReport: "data_package_name": "policyengine-us-data", "data_package_version": "1.0.0", "data_artifact_revision": "revision", + "created_at": NOW, + "updated_at": NOW, "retention_expires_at": NOW + timedelta(days=30), } fields.update(changes) @@ -80,6 +82,8 @@ def _simulation(**changes: object) -> Stage12ComparisonSimulation: "modal_application": "policyengine-v2-worker-5-2-0", "simulation_callable": "run_simulation", "version_manifest_sha256": DIGEST_A, + "created_at": NOW, + "updated_at": NOW, "retention_expires_at": NOW + timedelta(days=30), } fields.update(changes) diff --git a/uv.lock b/uv.lock index 80711e244..aa0a2b03b 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", @@ -2743,7 +2743,7 @@ wheels = [ [[package]] name = "policyengine" -version = "6.2.1" +version = "6.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "h5py" }, @@ -2755,9 +2755,9 @@ dependencies = [ { name = "pydantic" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/67/f2/3a75eb571fa3eda2e9f043c145cadae9eab4df6959fc447e767f92885d88/policyengine-6.2.1.tar.gz", hash = "sha256:0f2e51be23229ab5d8a8f80fb7b716fe4d3780806e2e16dfcda0362983beac56", size = 772927, upload-time = "2026-09-30T22:19:57.846Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/e3/4cf7cbaec22b8669828030614e59e37ba7d5ba6164cc1b91d8ce461190a2/policyengine-6.3.4.tar.gz", hash = "sha256:a3c4859ad1652955fea41e79b59211e3f1c8068fdf6daefbb993158b18cf4174", size = 872628, upload-time = "2026-10-13T20:25:14.62Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/8b/4b216e42d150852e51c01ea654455496e1e1197c008e72e70f29392dec8c/policyengine-6.2.1-py3-none-any.whl", hash = "sha256:d07ff400da096be4b43843bd333ea0b7f42b06f5eea584e72c78f75a16fe3cdb", size = 252406, upload-time = "2026-09-30T22:19:56.61Z" }, + { url = "https://files.pythonhosted.org/packages/f0/c4/168a31c826b6d50dd132bb28d38bb2a646a5b60557735f6bdf54618a341e/policyengine-6.3.4-py3-none-any.whl", hash = "sha256:7362e8feafcc48ab3d41c0d6f2d8b0e2343da9922835e69f180c656911c467eb", size = 306287, upload-time = "2026-10-13T20:25:12.422Z" }, ] [package.optional-dependencies] @@ -2844,7 +2844,7 @@ requires-dist = [ { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.15,<2" }, { name = "openai" }, { name = "packaging", specifier = ">=24,<27" }, - { name = "policyengine", extras = ["models"], specifier = "==6.2.1" }, + { name = "policyengine", extras = ["models"], specifier = "==6.3.4" }, { name = "policyengine-canada", specifier = "==0.96.3" }, { name = "policyengine-il", specifier = "==0.1.0" }, { name = "policyengine-ng", specifier = "==0.5.1" }, @@ -2898,7 +2898,7 @@ wheels = [ [[package]] name = "policyengine-core" -version = "3.32.10" +version = "3.33.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -2918,9 +2918,9 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d6/cd/9fc967c5b787baa5f9d83461773ba0982a8e22ae83c40155a7ee55edd11a/policyengine_core-3.32.10.tar.gz", hash = "sha256:0f51e8abfa8f172c69d03ba8f4304632a348fc6dca780e41021b2fd0cec04fde", size = 427479, upload-time = "2026-09-29T17:56:43.582Z" } +sdist = { url = "https://files.pythonhosted.org/packages/90/d7/70936abfab724399ea3d8db04bbd5808339ee1534819bf25ab46d00c8d32/policyengine_core-3.33.0.tar.gz", hash = "sha256:5c923828dc84a7a1ff8f6f407f7a704fe067246dd277ede04c15f409804b9c19", size = 674785, upload-time = "2026-10-10T09:43:12.12Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/50/291d73da16f39a3c4f05dffb6d40b1142fc400ec39b5a2ca7481e6fe9d90/policyengine_core-3.32.10-py3-none-any.whl", hash = "sha256:124afc555fe7c848ae8237b820fed0b6ea0ea1280e8850ad8bf356937ee10e5b", size = 247410, upload-time = "2026-09-29T17:56:42.218Z" }, + { url = "https://files.pythonhosted.org/packages/6d/02/190e06717785ff974120a9136c6c5408a00beaa8cddd792f557a6bf974c4/policyengine_core-3.33.0-py3-none-any.whl", hash = "sha256:2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789", size = 299495, upload-time = "2026-10-10T09:43:10.318Z" }, ] [[package]] @@ -2977,7 +2977,7 @@ otlp-grpc = [ [[package]] name = "policyengine-uk" -version = "2.102.3" +version = "2.126.12" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -2985,9 +2985,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a7/49/1db19a439463f1304e93521debf40794f7b3483d37356c6d99c346112d0f/policyengine_uk-2.102.3.tar.gz", hash = "sha256:cdb0900e78818b46fdb79b2a8299bce6c7d6c14692e26cf38367781beea607be", size = 1314347, upload-time = "2026-09-28T19:50:44.714Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/c9/5ad8f2c5833742db4873098f2689cd45cf7bc7f3d79755d5e13b98dd8ab4/policyengine_uk-2.126.12.tar.gz", hash = "sha256:86ac5ac83f7a22b6ef69ca701b0a87017c4a143f2f34937c74d5311786ed34b4", size = 2277744, upload-time = "2026-10-12T04:31:37.012Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/37/e6/e8d9069d498f1510a1380f24a78e957902d01f00b10248c146d6710d5b37/policyengine_uk-2.102.3-py3-none-any.whl", hash = "sha256:f02a76712a3edf9983c713c1db42e9653a6e974e4ce335dd4b5676bbc2bcd636", size = 2189605, upload-time = "2026-09-28T19:50:42.472Z" }, + { url = "https://files.pythonhosted.org/packages/4f/3f/0a611e9b25beb75db3f2f521d359eabad0d1ab4e50194b866663355591ff/policyengine_uk-2.126.12-py3-none-any.whl", hash = "sha256:4aa6d7ba411de0009d460efdc4e4992f2e8c89cbf8ec419024b1bce89d407ace", size = 3524817, upload-time = "2026-10-12T04:31:34.999Z" }, ] [[package]]