From 43b43f6166ef15b51c1b88255f52263a9ca5db0e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 26 Apr 2026 10:23:15 -0400 Subject: [PATCH] Deploy Modal worker for API v4 migration --- .../pyproject.toml | 4 +-- .../src/modal/gateway/endpoints.py | 1 + .../src/modal/gateway/models.py | 13 ++++--- .../tests/gateway/test_endpoints.py | 34 +++++++++++++++++++ projects/policyengine-api-simulation/uv.lock | 20 ++++++----- 5 files changed, 54 insertions(+), 18 deletions(-) diff --git a/projects/policyengine-api-simulation/pyproject.toml b/projects/policyengine-api-simulation/pyproject.toml index 9651569b2..a70e04b35 100644 --- a/projects/policyengine-api-simulation/pyproject.toml +++ b/projects/policyengine-api-simulation/pyproject.toml @@ -18,8 +18,8 @@ dependencies = [ "policyengine-fastapi", "policyengine==0.13.0", "policyengine-core>=3.23.5", - "policyengine-uk>=2.22.8", - "policyengine-us>=1.370.2", + "policyengine-uk==2.88.0", + "policyengine-us==1.653.3", "tables>=3.10.2", "modal>=0.73.0", "logfire>=3.0.0", diff --git a/projects/policyengine-api-simulation/src/modal/gateway/endpoints.py b/projects/policyengine-api-simulation/src/modal/gateway/endpoints.py index 7bba88dae..a4716f147 100644 --- a/projects/policyengine-api-simulation/src/modal/gateway/endpoints.py +++ b/projects/policyengine-api-simulation/src/modal/gateway/endpoints.py @@ -73,6 +73,7 @@ def _build_policyengine_bundle( resolved_dataset = None return PolicyEngineBundle( model_version=resolved_version, + data_version=payload.get("data_version"), dataset=resolved_dataset, ) diff --git a/projects/policyengine-api-simulation/src/modal/gateway/models.py b/projects/policyengine-api-simulation/src/modal/gateway/models.py index 0456d7026..a94e98e51 100644 --- a/projects/policyengine-api-simulation/src/modal/gateway/models.py +++ b/projects/policyengine-api-simulation/src/modal/gateway/models.py @@ -20,7 +20,7 @@ MAX_GATEWAY_REQUEST_BYTES = 262_144 -INTERNAL_PASSTHROUGH_FIELDS = frozenset({"_metadata"}) +INTERNAL_PASSTHROUGH_FIELDS = frozenset({"_metadata", "_runtime_bundle"}) def _move_internal_telemetry_alias(value): @@ -35,12 +35,11 @@ def _move_internal_telemetry_alias(value): def _strip_internal_passthrough_fields(value): """Drop internal-only fields the gateway adds to payloads in flight. - When the parent batch entrypoint forwards a payload to the worker we - attach ``_metadata`` describing resolved routing. That enrichment is - consumed by :mod:`src.modal.budget_window_context`, not by the request - model itself. We strip it before strict validation so ``extra="forbid"`` - keeps catching unknown fields from *callers* without breaking the - internal round-trip. + Parent batch entrypoints attach ``_metadata`` describing resolved routing, + and the v4 API attaches ``_runtime_bundle`` describing provenance that the + gateway returns separately. Strip those fields before strict validation so + ``extra="forbid"`` keeps catching unknown caller fields without breaking + internal round-trips. """ if not isinstance(value, dict): diff --git a/projects/policyengine-api-simulation/tests/gateway/test_endpoints.py b/projects/policyengine-api-simulation/tests/gateway/test_endpoints.py index c0a26e8b6..1dd3715ff 100644 --- a/projects/policyengine-api-simulation/tests/gateway/test_endpoints.py +++ b/projects/policyengine-api-simulation/tests/gateway/test_endpoints.py @@ -311,6 +311,40 @@ def test__given_submission_with_uk_alias_data__then_bundle_dataset_is_versioned_ == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.3" ) + def test__given_submission_with_runtime_bundle__then_accepts_internal_provenance( + self, mock_modal, client: TestClient + ): + mock_modal["dicts"]["simulation-api-us-versions"] = { + "latest": "1.500.0", + "1.500.0": "policyengine-simulation-us1-500-0-uk2-66-0", + } + + request_body = { + "country": "us", + "scope": "macro", + "reform": {}, + "data": "enhanced_cps_2024", + "data_version": "1.78.2", + "_runtime_bundle": { + "model_version": "1.500.0", + "data_version": "1.78.2", + }, + "_metadata": {"process_id": "process-123"}, + } + + response = client.post("/simulate/economy/comparison", json=request_body) + + assert response.status_code == 200 + data = response.json() + assert data["policyengine_bundle"] == { + "model_version": "1.500.0", + "data_version": "1.78.2", + "dataset": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@1.77.0", + } + assert mock_modal["func"].last_payload["data_version"] == "1.78.2" + assert "_runtime_bundle" not in mock_modal["func"].last_payload + assert "_metadata" not in mock_modal["func"].last_payload + def test__given_submission_with_unknown_alias_data__then_bundle_dataset_is_preserved( self, mock_modal, client: TestClient ): diff --git a/projects/policyengine-api-simulation/uv.lock b/projects/policyengine-api-simulation/uv.lock index 7adc8ed44..1039bd05b 100644 --- a/projects/policyengine-api-simulation/uv.lock +++ b/projects/policyengine-api-simulation/uv.lock @@ -746,7 +746,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" }, { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" }, { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" }, + { url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" }, { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" }, + { url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" }, { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" }, { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" }, { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" }, @@ -1641,7 +1643,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -1802,8 +1804,8 @@ requires-dist = [ { name = "policyengine", specifier = "==0.13.0" }, { name = "policyengine-core", specifier = ">=3.23.5" }, { name = "policyengine-fastapi", editable = "../../libs/policyengine-fastapi" }, - { name = "policyengine-uk", specifier = ">=2.22.8" }, - { name = "policyengine-us", specifier = ">=1.370.2" }, + { name = "policyengine-uk", specifier = "==2.88.0" }, + { name = "policyengine-us", specifier = "==1.653.3" }, { name = "pydantic-settings", specifier = ">=2.7.1,<3.0.0" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, @@ -1815,7 +1817,7 @@ provides-extras = ["test", "build"] [[package]] name = "policyengine-uk" -version = "2.88.9" +version = "2.88.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1823,14 +1825,14 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/4b/9c19b51722cd0e020c6e55ecdf15e82f783a7002d78e2e3f361b4e615473/policyengine_uk-2.88.9.tar.gz", hash = "sha256:de204a6ab94780b2217f33ef53e8b89c1396ae4b9b1147a99a703952c4fc943c", size = 1172698, upload-time = "2026-04-20T11:26:03.248Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/cf/749dea25c17210b5dc40098363e0b6a60b7fc5feb69ff77c74b88deb5cde/policyengine_uk-2.88.0.tar.gz", hash = "sha256:d157c7336b7aa3a321f317af1a4f111d7b857451ff43f4998abdc5a8c893e989", size = 1166666, upload-time = "2026-04-17T19:22:55.418Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/7f/54b248a43919fcd81f035d33e3dc2938293db58455d37e11eb7aad8e84d5/policyengine_uk-2.88.9-py3-none-any.whl", hash = "sha256:10110d002b0aedf2c2b3610a81946e077baff4f46e095730645904dea9dd5cab", size = 1876894, upload-time = "2026-04-20T11:26:01.379Z" }, + { url = "https://files.pythonhosted.org/packages/23/7e/8a2a42eac1da63730a865964aa17e7fd4420ce4db4c80001c1b5ca6011e8/policyengine_uk-2.88.0-py3-none-any.whl", hash = "sha256:46a3ba443b43ec810c5efaccd4645edb63c8dc90ef5acf9b0cdf5ace86b9334d", size = 1867764, upload-time = "2026-04-17T19:22:53.244Z" }, ] [[package]] name = "policyengine-us" -version = "1.666.1" +version = "1.653.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1840,9 +1842,9 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7c/ed/779ae0f296a5652b1a27c0ff9a210d0907cb28887b693fe390e4816bcadb/policyengine_us-1.666.1.tar.gz", hash = "sha256:796fa45584a7e5795595fc31688a2df9841c35b0811dfe83dfa6df250140a2f4", size = 9309724, upload-time = "2026-04-23T22:36:52.07Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/60/5b736fa238559857fbf29168933c809eaada9abf006d26910b7958f5748e/policyengine_us-1.653.3.tar.gz", hash = "sha256:8a5c33997b7aefa2061d0dafce837b130e8ebdb0b9f83ae8c236f80cbf1805d6", size = 9180339, upload-time = "2026-04-18T12:06:45.764Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/c1/aa62214930a4ce1c0bc0906f10b19dc5d67128cfedc1b96dbd4330c113b3/policyengine_us-1.666.1-py3-none-any.whl", hash = "sha256:33ef9c7f0c72bd4d17bca922d4ab4f65b42edc4ab41c950137938880ed6de13b", size = 9589718, upload-time = "2026-04-23T22:36:48.87Z" }, + { url = "https://files.pythonhosted.org/packages/02/07/25f39a2bfa1ff210cd8e78826c47c03b9040a98a83f4eed59c434c1ed862/policyengine_us-1.653.3-py3-none-any.whl", hash = "sha256:67a49b98d85c060b24d547a569e91a6703c0fc9c41299c1c67f4ecfac75c67c6", size = 9445650, upload-time = "2026-04-18T12:06:43.163Z" }, ] [[package]]