Pin .py 6.4.1 instead of the deleted policyengine-us 1.764.6

backend/requirements.txt

@@ -1,9 +1,9 @@
-# Match the reviewed Build P certificate and immutable data identity in
-# frontend/scripts/hosted_release.json. Upgrade all model/data pins together.
-policyengine-us==1.764.6
-policyengine-core==3.26.11
-# This country wheel imports legacy SPM APIs and has no upper dependency bound.
-spm-calculator==0.3.1
+# .py (the PyPI package `policyengine`) is the hosted runtime's model identity.
+# Upgrade by bumping .py; the dataset in frontend/scripts/hosted_release.json must
+# be one that version of .py certifies. Never pin policyengine-us,
+# policyengine-core or spm-calculator here: their versions come from .py's `us`
+# extra and are checked against frontend/scripts/hosted_release.json.
+policyengine[us]==6.4.1
 huggingface_hub>=0.25,<1.0
 numpy>=1.26,<3.0
 fastapi==0.141.1

docs/integrations/microcosm-chronicle-gap-review.md

@@ -159,9 +159,11 @@ These remain deliberately unscored:
 > Deprecated upstream identifier: the pinned Microcosm release ID below still
 > uses its current `populace-us-*` prefix.
 
-- Pinned Microcosm release:
+- Microcosm release audited:
   `populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z`.
-- Pinned PolicyEngine-US version: `1.764.6`.
+- PolicyEngine-US version audited: `1.764.6`. The harness now pins
+  `policyengine[us]==6.4.1` and its certified source-enrichment of this release,
+  `populace-us-2024-spm-20260915`; this audit has not been re-run on that pair.
 - Reviewed Microcosm target inclusions and exclusions:
   `packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json`
   at commit `cae8640f9e65e274aea65c7916cb37b956978e32`.

docs/integrations/microcosm-policyengine-us.md

@@ -3,9 +3,10 @@
 > Deprecated upstream identifiers: the current Microcosm HF release, files,
 > diagnostics metadata, and Chronicle fact keys still use `populace`/`ledger`.
 
-Status: **adapter implemented and verified against the pinned release**
+Status: **adapter implemented; the full evaluation was verified against the Build P
+release with policyengine-us 1.764.6 and has not yet been re-run on the pair below**
 
-Reviewed Microcosm release: `populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z`
+Reviewed Microcosm release: `populace-us-2024-spm-20260915`
 
 Chronicle snapshot: `chronicle-7917ea815df710fb20db076b`
 
@@ -15,11 +16,16 @@ This is a model/dataset pairing, not a raw-dataset adapter:
 
 - Dataset: Microcosm US, published at
   `policyengine/populace-us`, file `populace_us_2024.h5`, SHA-256
-  `48b9d479fb4fd1c3537f9383ce4697d130b6f618658409d74f6233c43b994c7e`.
-- Dataset build: Microcosm commit
-  `cae8640f9e65e274aea65c7916cb37b956978e32`.
-- Model: `policyengine-us==1.764.6` (with `policyengine-core==3.26.11` in the
-  release manifest).
+  `6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84`.
+- Dataset build: the US release .py 6.4.1 certifies. It is a source-enrichment of
+  `populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z` (Microcosm
+  commit `cae8640f9e65e274aea65c7916cb37b956978e32`): the same tables, weights and
+  calibration diagnostics plus `is_spm_independent_minor_role`, added at Microcosm
+  commit `743683b0e121378e93c9b426cd9c79d41d29595b`.
+- Model: `policyengine[us]==6.4.1`, the only declared model dependency. Its `us`
+  extra pins `policyengine-us==2.2.1`, `policyengine-core==3.33.0` and
+  `spm-calculator==1.0.0`. Upgrade by bumping .py, to a Microcosm release that
+  version certifies; the hosted variable runtime uses the same pair.
 - Population period: calendar year 2024.
 - Policy calculation period: tax year 2024.
 - Weight entities: person, household, and tax unit.

evaluation_harness/adapters/microcosm.py

@@ -21,6 +21,7 @@ class MicrocosmRelease:
     release_id: str
     dataset_filename: str
     dataset_sha256: str
+    # The model requirement, such as "policyengine[us]==6.4.1".
     model_version: str
     repository: str = MICROCOSM_REPOSITORY
     calibration_diagnostics_filename: str | None = None
@@ -37,15 +38,16 @@ def file_download_url(self, filename: str) -> str:
         )
 
 
+# The release .py 6.4.1 certifies, shared with the hosted variable runtime
+# (frontend/scripts/hosted_release.json). It is a source-enrichment of the Build P
+# release and keeps Build P's calibration diagnostics.
 MICROCOSM_RELEASE = MicrocosmRelease(
-    release_id="populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z",
+    release_id="populace-us-2024-spm-20260915",
     dataset_filename="populace_us_2024.h5",
-    dataset_sha256="48b9d479fb4fd1c3537f9383ce4697d130b6f618658409d74f6233c43b994c7e",
-    model_version="1.764.6",
+    dataset_sha256="6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84",
+    model_version="policyengine[us]==6.4.1",
     calibration_diagnostics_filename=(
-        "releases/"
-        "populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z/"
-        "calibration_diagnostics.json"
+        "releases/populace-us-2024-spm-20260915/calibration_diagnostics.json"
     ),
     calibration_diagnostics_sha256=(
         "870449b44e86b13b25bcea1a57f0e7af37f4d4db18be815eea3acdf9fe6eb40e"
@@ -709,6 +711,6 @@ def prepare(self, group: RunGroup) -> ArrayBundle:
         return ArrayBundle(
             arrays=arrays,
             dataset_version=self.release.release_id,
-            model_version=f"policyengine-us=={self.release.model_version}",
+            model_version=self.release.model_version,
             domain_masks=masks,
         )

frontend/HOSTED_RUNTIME.md

@@ -2,14 +2,62 @@
 
 The Next.js dashboard proxies variable requests to a private Modal service. The service
 executes `scripts/microcosm_variable_core.py`, also used by the local command-line tool,
-and verifies the reviewed model tuple, both release manifests, and the H5 bytes before
-constructing a native PolicyEngine microsimulation. Weighted results use MicroSeries.
-The root evaluation environment and its recorded outputs remain separate.
+and verifies the installed .py version and the model versions it pins, both release
+manifests, the data certification and the H5 bytes before constructing a native
+PolicyEngine microsimulation. Weighted results use MicroSeries. The root evaluation
+environment and its recorded outputs remain separate.
 
 The runtime uses Python 3.12 and the hashed `../backend/requirements.lock`. Regenerate
 that lock with `uv pip compile backend/requirements.txt --python-version 3.12
 --generate-hashes --output-file backend/requirements.lock` from the repository root.
-Upgrade the country model, Core, SPM and `scripts/hosted_release.json` together.
+
+## Model and data identity
+
+.py (the PyPI package `policyengine`) is the model identity, and
+`backend/requirements.txt` declares only `policyengine[us]==VERSION`. **Upgrade by
+bumping .py; the dataset must be one that version of .py certifies.** Never pin
+`policyengine-us`, `policyengine-core` or `spm-calculator` directly; a test fails if a
+bare pin reappears. `scripts/hosted_release.json` records the certifying .py version
+in `certifying_package` and the reviewed data identity: repository, immutable commit
+`hf_revision`, release ID, H5 filename and SHA-256, data year, and the canonical
+digests of both release manifests.
+
+`scripts/hosted_release.py` enforces the identity in two places:
+
+- `validate_runtime` requires the installed .py to be the reviewed version, derives
+  the exact `policyengine-us`, `policyengine-core` and `spm-calculator` versions from
+  that .py's `us` extra, requires the `packages` block to equal them, and requires the
+  installed versions to equal them exactly. It runs before any manifest or H5 download.
+- `validate_manifests` checks both manifest digests and the release's own identity
+  fields, then requires the installed .py's bundle manifest
+  (`policyengine/data/bundle/manifest.json`) to certify this release: its release ID,
+  revision, dataset URI and SHA-256, manifest paths, model version, and the digest of
+  every release file it records, including the parent release and build manifests.
+
+The release's own built-with Core version and its build manifest's `runtime` block are
+not the identity: .py is. For the same reason the build manifest's exported-household
+count is checked by `verify_hosted_runtime.py --dataset-file` only when the
+.py-certified build records one.
+
+The reviewed release is `populace-us-2024-spm-20260915`, the US dataset .py 6.4.1
+certifies. It is a source-enrichment release of
+`populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z`: the same tables
+and calibration plus `is_spm_independent_minor_role`, which `policyengine-us` 2.x
+requires to calculate SPM thresholds. Its parent manifests are the Build P manifests
+previously reviewed here.
+
+To upgrade:
+
+1. Bump `policyengine[us]` in `backend/requirements.txt` and regenerate the lock.
+2. Read the new .py's `data_releases.us` bundle entry, and set `hosted_release.json`'s
+   data identity and `packages` to what that .py certifies and pins.
+3. Refresh the manifest fixtures under `tests/fixtures/`.
+4. Regenerate `public/variable-catalog.json` in the hosted environment with
+   `python frontend/scripts/generate_variable_catalog.py`. It builds the catalog from a
+   simulation over the reviewed H5, verified by SHA-256 (`--dataset-file` reuses a
+   local copy), never from the country package's default dataset, and records the
+   release and package versions it used.
+5. Qualify the deployment with the checks below.
 
 ## Automated production deployment
 
@@ -80,14 +128,20 @@ serializes requests (`max_inputs=1`, one container). It scales to zero. Its work
 holds the normal HF download; there is no Vercel RAM-backed HDF5 workaround. Observe the
 actual deployed allocation and response resource measurements rather than assuming the
 requested allocation is sufficient.
-The identical local SPM reference peaked at approximately 12 GB; an 8 GiB hosted
-allocation did not complete the calculation. The required checks below must still pass.
+Under policyengine-us 1.764.6 the local SPM reference peaked at approximately 12 GB, and
+an 8 GiB hosted allocation did not complete the calculation. Under .py 6.4.1
+(policyengine-us 2.2.1, spm-calculator 1.0.0), a local national `spm_unit_spm_threshold`
+calculation, then `ca_income_tax`, then the national threshold again, peaked at about
+1 GB resident memory (macOS, 2026-10-16). Keep the 16 GiB allocation until a hosted
+measurement under the new tuple is recorded. The required checks below must still pass.
 The service loads only the reviewed 2024 input. It validates the native dataset year and
 wraps that `USSingleYearDataset` in a one-entry `USMultiYearDataset` before constructing
 the simulation. This compatibility step prevents the pinned country package from
-automatically allocating copies for later economic-assumption years. Remove the wrapper
-only after upgrading to a country-package version that supports an explicit extension
-end year, and qualify that upgrade with the same calculation checks.
+automatically allocating copies for later economic-assumption years: policyengine-us
+2.2.1's `Microsimulation` still extends a single-year dataset through every
+economic-assumption year and does not let the caller set the end year. Remove the
+wrapper only after upgrading to a country-package version that does, and qualify that
+upgrade with the same calculation checks.
 Only one native simulation stays resident: changing between national and state scopes
 evicts the preceding simulation before loading the next input tables. Same-scope requests
 can reuse it; mixed variable lists may rebuild when their scopes change.
@@ -118,7 +172,7 @@ explicitly. Without a remote URL, local calculations still use the shared Python
    selection and answers 409. A 200 metadata receipt therefore also records that this
    deployment is not carrying a stale override.
 2. On a fresh backend deployment, run a genuine `spm_unit_spm_threshold` lookup for 2024
-   on the pinned BuildP release through the exact Next preview. Require JSON 200, verified
+   on the pinned release through the exact Next preview. Require JSON 200, verified
    H5 SHA-256, exact model/source identity, `execution.simulation_cache_hits.national=false`
    and a non-null weighted result. Require comparison with a separately executed reference
    using the identical source, package tuple and immutable H5. An older Core version's

frontend/scripts/generate_variable_catalog.py

@@ -1,44 +1,90 @@
 """Generate a static catalog of PolicyEngine-US variables for the lookup page.
 
-Writes frontend/public/variable-catalog.json — a list of
-{name, label, entity, unit} for every variable in the model, so the variable
-lookup page can offer search/select without a per-keystroke API round-trip.
+Writes frontend/public/variable-catalog.json: the model and data identity it was
+built from, plus {name, label, entity, unit} for every variable in the model, so
+the variable lookup page can offer search/select without a per-keystroke API
+round-trip.
 
-Run from the frontend dir:  python scripts/generate_variable_catalog.py
-Refresh whenever policyengine-us is upgraded.
+The catalog is built from a simulation over the reviewed hosted dataset under the
+pinned .py, never from the country package's default dataset. Run it from the
+repository root in the hosted runtime environment:
+    python frontend/scripts/generate_variable_catalog.py [--dataset-file H5]
+Without --dataset-file, the reviewed manifests and H5 are downloaded at the
+pinned revision. Either way, the H5 bytes must match the reviewed SHA-256.
+Regenerate it whenever .py is upgraded.
 """
 
+import argparse
 import json
-import os
-
-from policyengine_us import Microsimulation
-
-
-def main() -> None:
-    sim = Microsimulation()
-    variables = sim.tax_benefit_system.variables
-    catalog = []
-    for name, variable in sorted(variables.items()):
-        catalog.append(
-            {
-                "name": name,
-                "label": getattr(variable, "label", None) or None,
-                "entity": getattr(getattr(variable, "entity", None), "key", None),
-                "unit": getattr(variable, "unit", None),
-            }
-        )
-
-    here = os.path.dirname(os.path.abspath(__file__))
-    out_dir = os.path.join(here, "..", "public")
-    os.makedirs(out_dir, exist_ok=True)
-    out_path = os.path.join(out_dir, "variable-catalog.json")
-    with open(out_path, "w") as handle:
-        json.dump(
-            {"count": len(catalog), "variables": catalog},
-            handle,
-            separators=(",", ":"),
-        )
-    print(f"Wrote {len(catalog)} variables to {os.path.normpath(out_path)}")
+import sys
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from scripts.hosted_release import (
+    download_certification,
+    reviewed_release,
+    validate_runtime,
+    verify_h5,
+)
+from scripts.microcosm_variable_core import single_year_simulation_dataset
+
+FRONTEND = Path(__file__).resolve().parents[1]
+DEFAULT_OUTPUT = FRONTEND / "public" / "variable-catalog.json"
+
+
+def reviewed_dataset_path(config: dict[str, Any], dataset_file: Path | None) -> str:
+    """Return a local copy of the reviewed H5, downloading it only when needed."""
+    if dataset_file is not None:
+        return str(dataset_file)
+    from huggingface_hub import hf_hub_download
+
+    download_certification(config, hf_hub_download)
+    return hf_hub_download(
+        repo_id=config["repo"],
+        filename=config["filename"],
+        revision=config["hf_revision"],
+        repo_type="dataset",
+    )
+
+
+def build_catalog(config: dict[str, Any], dataset_path: str) -> dict[str, Any]:
+    """Describe every variable of a simulation over the verified reviewed H5."""
+    verify_h5(dataset_path, config)
+    from policyengine_us import Microsimulation
+
+    simulation = Microsimulation(dataset=single_year_simulation_dataset(dataset_path))
+    variables = simulation.tax_benefit_system.variables
+    catalog = [
+        {
+            "name": name,
+            "label": getattr(variable, "label", None) or None,
+            "entity": getattr(getattr(variable, "entity", None), "key", None),
+            "unit": getattr(variable, "unit", None),
+        }
+        for name, variable in sorted(variables.items())
+    ]
+    return {
+        "count": len(catalog),
+        "release_id": config["release_id"],
+        "packages": config["packages"],
+        "variables": catalog,
+    }
+
+
+def main(arguments: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--dataset-file", type=Path)
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+    options = parser.parse_args(arguments)
+    config = reviewed_release()
+    validate_runtime(config)
+    dataset_path = reviewed_dataset_path(config, options.dataset_file)
+    catalog = build_catalog(config, dataset_path)
+    options.output.parent.mkdir(parents=True, exist_ok=True)
+    options.output.write_text(json.dumps(catalog, separators=(",", ":")))
+    print(f"Wrote {catalog['count']} variables to {options.output}")
 
 
 if __name__ == "__main__":

frontend/scripts/hosted_release.json

@@ -1,23 +1,29 @@
 {
-  "schema_version": 1,
+  "schema_version": 2,
   "repo": "policyengine/populace-us",
-  "hf_revision": "f09f2f3b9fa8409642dc0c7fc9c8f7516ae0e3c5",
-  "release_id": "populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z",
+  "hf_revision": "8ab57ffc2ca41d8af631ff62ebbce95e966299f3",
+  "release_id": "populace-us-2024-spm-20260915",
   "filename": "populace_us_2024.h5",
-  "sha256": "48b9d479fb4fd1c3537f9383ce4697d130b6f618658409d74f6233c43b994c7e",
+  "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84",
   "data_year": 2024,
   "release_manifest": {
-    "path": "releases/populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z/release_manifest.json",
-    "json_sha256": "ecda19d971fad6d7ffcbb9ad1cbcb7825e1cdfa022f6da1eb89fcc96eb1250b0"
+    "path": "releases/populace-us-2024-spm-20260915/release_manifest.json",
+    "json_sha256": "73f6a102b7721e188af3a39e1d6f22cd31750ce7c44bd1e230b8ac8e5bd788e8"
   },
   "build_manifest": {
-    "path": "releases/populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z/build_manifest.json",
-    "json_sha256": "f503fc887efcf3ae5b36741f395e91d46f7e870f7f5d1254d3b7f79a4f3e655c"
+    "path": "releases/populace-us-2024-spm-20260915/build_manifest.json",
+    "json_sha256": "9db2a76cd9761ab2fcdb0c929613f9637ef9532eefeea39ec5fa93f7785648b1"
+  },
+  "certifying_package": {
+    "name": "policyengine",
+    "version": "6.4.1",
+    "extra": "us"
   },
   "packages": {
-    "policyengine-us": "1.764.6",
-    "policyengine-core": "3.26.11",
-    "spm-calculator": "0.3.1"
+    "policyengine": "6.4.1",
+    "policyengine-us": "2.2.1",
+    "policyengine-core": "3.33.0",
+    "spm-calculator": "1.0.0"
   },
   "manifest_digest_algorithm": "sha256-json-sort-keys-compact-ascii-escaped-utf8"
 }