Preserve nullable SPM summaries and JSON

changelog.d/518.fixed.md

@@ -0,0 +1,3 @@
+Return null poverty rates and aggregate means when the eligible weighted denominator is empty, preserve nullable poverty rates in DataFrame record exports, and convert nonfinite US household results to JSON null.
+
+Reject undefined upward and group-to-group mapping of nullable US SPM statuses in poverty and aggregate outputs, including demographic filter paths, while preserving native-entity summaries and projection to people.

docs/outputs.md

@@ -179,6 +179,12 @@ rates.dataframe
 
 Call it once per simulation for a baseline-vs-reform comparison. Age / gender / race breakdowns: `calculate_us_poverty_by_age`, `_by_gender`, `_by_race`. UK counterparts: `calculate_uk_poverty_rates`, `_by_age`, `_by_gender`.
 
+Missing poverty statuses are excluded from `total_population`, which reports the eligible weighted population after demographic filters. If that denominator is zero, `rate` is `None`; an eligible population with no poor members has a measured rate of `0.0`. The collection's `rate` column uses Pandas `Float64`, so `rates.dataframe.to_dict("records")` retains missing rates as JSON `null` and supports `json.dumps(..., allow_nan=False)`.
+
+Generic `Aggregate` and `ChangeAggregate` means also return `None` when their eligible weighted denominator is zero. US household results convert nonfinite country-model values to `None`, including missing SPM statuses and values returned along household-calculator axes.
+
+The five US SPM status variables (`in_poverty`, `in_deep_poverty`, `person_in_poverty`, `spm_unit_is_in_spm_poverty`, and `spm_unit_is_in_deep_spm_poverty`) support summaries at their native entity and projection to people. `Poverty`, `Aggregate`, and `ChangeAggregate` reject upward or group-to-group mapping of these statuses, including when used as filters: combining status flags across entities requires an explicit measurement-universe rule. Generic amount mapping retains its existing behavior.
+
 ## Inequality
 
 Gini, top-10 share, top-1 share, bottom-50 share — for one simulation.

src/policyengine/outputs/_poverty_status.py

@@ -0,0 +1,28 @@
+"""Entity-mapping limits for nullable US poverty status indicators."""
+
+NULLABLE_SPM_POVERTY_VARIABLES = frozenset(
+    {
+        "in_poverty",
+        "in_deep_poverty",
+        "person_in_poverty",
+        "spm_unit_is_in_spm_poverty",
+        "spm_unit_is_in_deep_spm_poverty",
+    }
+)
+
+
+def validate_poverty_status_mapping(
+    variable: str, source_entity: str, target_entity: str, context: str
+) -> None:
+    """Reject undefined status aggregation before a generic mapper loses NaN."""
+    if (
+        variable in NULLABLE_SPM_POVERTY_VARIABLES
+        and source_entity != target_entity
+        and target_entity != "person"
+    ):
+        raise ValueError(
+            f"{context}: nullable SPM poverty status '{variable}' cannot be mapped "
+            f"from '{source_entity}' to '{target_entity}'. Aggregating statuses "
+            "across entities has no defined measurement universe; use its native "
+            "entity or project it to 'person'."
+        )

src/policyengine/outputs/aggregate.py

@@ -3,6 +3,7 @@
 from typing import Any, Optional
 
 from policyengine.core import Output, Simulation, Variable
+from policyengine.outputs._poverty_status import validate_poverty_status_mapping
 
 
 class AggregateType(str, Enum):
@@ -120,6 +121,9 @@ def run(self):
 
         # Get the target entity data
         target_entity = self.entity or var_obj.entity
+        validate_poverty_status_mapping(
+            self.variable, var_obj.entity, target_entity, "Aggregate.variable"
+        )
         data = get_output_entity_data(
             self.simulation, target_entity, "Aggregate.entity"
         )
@@ -155,6 +159,12 @@ def run(self):
             filter_var_obj = get_aggregate_variable(
                 self.simulation, self.filter_variable, "Aggregate.filter_variable"
             )
+            validate_poverty_status_mapping(
+                self.filter_variable,
+                filter_var_obj.entity,
+                target_entity,
+                "Aggregate.filter_variable",
+            )
 
             if filter_var_obj.entity != target_entity:
                 filter_source_data = get_output_entity_data(
@@ -207,6 +217,6 @@ def run(self):
         if self.aggregate_type == AggregateType.SUM:
             self.result = series.sum()
         elif self.aggregate_type == AggregateType.MEAN:
-            self.result = series.mean()
+            self.result = series.mean() if series.count() > 0 else None
         elif self.aggregate_type == AggregateType.COUNT:
             self.result = series.count()

src/policyengine/outputs/change_aggregate.py

@@ -2,6 +2,7 @@
 from typing import Any, Optional
 
 from policyengine.core import Output, Simulation
+from policyengine.outputs._poverty_status import validate_poverty_status_mapping
 from policyengine.outputs.aggregate import (
     get_aggregate_variable,
     get_output_entity_data,
@@ -70,6 +71,9 @@ def run(self):
 
         # Get the target entity data
         target_entity = self.entity or var_obj.entity
+        validate_poverty_status_mapping(
+            self.variable, var_obj.entity, target_entity, "ChangeAggregate.variable"
+        )
         baseline_data = get_output_entity_data(
             self.baseline_simulation,
             target_entity,
@@ -175,6 +179,12 @@ def run(self):
                 self.filter_variable,
                 "ChangeAggregate.filter_variable",
             )
+            validate_poverty_status_mapping(
+                self.filter_variable,
+                filter_var_obj.entity,
+                target_entity,
+                "ChangeAggregate.filter_variable",
+            )
 
             if filter_var_obj.entity != target_entity:
                 filter_source_data = get_output_entity_data(
@@ -234,4 +244,6 @@ def run(self):
         elif self.aggregate_type == ChangeAggregateType.SUM:
             self.result = filtered_change.sum()
         elif self.aggregate_type == ChangeAggregateType.MEAN:
-            self.result = filtered_change.mean()
+            self.result = (
+                filtered_change.mean() if filtered_change.count() > 0 else None
+            )

src/policyengine/outputs/poverty.py

@@ -7,6 +7,7 @@
 from pydantic import ConfigDict
 
 from policyengine.core import Output, OutputCollection, Simulation
+from policyengine.outputs._poverty_status import validate_poverty_status_mapping
 
 
 class UKPovertyType(str, Enum):
@@ -44,7 +45,8 @@ class Poverty(Output):
 
     This is a single-simulation output type that calculates poverty
     headcount and rate for a given poverty measure, optionally filtered
-    by demographic variables.
+    by demographic variables. Missing statuses are excluded from the
+    eligible population; a zero eligible denominator produces a null rate.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -77,6 +79,12 @@ def run(self):
 
         # Get target entity data
         target_entity = self.entity
+        validate_poverty_status_mapping(
+            self.poverty_variable,
+            poverty_var_obj.entity,
+            target_entity,
+            "Poverty.poverty_variable",
+        )
         data = getattr(self.simulation.output_dataset.data, target_entity)
 
         # Map poverty variable to target entity if needed
@@ -95,6 +103,12 @@ def run(self):
             filter_var_obj = self.simulation.tax_benefit_model_version.get_variable(
                 self.filter_variable
             )
+            validate_poverty_status_mapping(
+                self.filter_variable,
+                filter_var_obj.entity,
+                target_entity,
+                "Poverty.filter_variable",
+            )
 
             if filter_var_obj.entity != target_entity:
                 filter_mapped = self.simulation.output_dataset.data.map_to_entity(
@@ -118,11 +132,14 @@ def run(self):
             # Apply mask
             poverty_series = poverty_series[mask]
 
-        # Calculate results using weighted counts
+        # Missing statuses are outside the measure's eligible population.
+        # MicroSeries.count() excludes them while preserving survey weights.
         self.headcount = float((poverty_series == True).sum())  # noqa: E712
         self.total_population = float(poverty_series.count())
         self.rate = (
-            self.headcount / self.total_population if self.total_population > 0 else 0.0
+            self.headcount / self.total_population
+            if self.total_population > 0
+            else None
         )
 
 
@@ -179,6 +196,9 @@ def calculate_uk_poverty_rates(
         ]
     )
 
+    # Preserve absent rates as JSON null in record exports.
+    df["rate"] = df["rate"].astype("Float64")
+
     return OutputCollection(outputs=results, dataframe=df)
 
 
@@ -235,6 +255,9 @@ def calculate_us_poverty_rates(
         ]
     )
 
+    # Preserve absent rates as JSON null in record exports.
+    df["rate"] = df["rate"].astype("Float64")
+
     return OutputCollection(outputs=results, dataframe=df)
 
 
@@ -300,6 +323,9 @@ def calculate_uk_poverty_by_age(
         ]
     )
 
+    # Preserve absent rates as JSON null in record exports.
+    df["rate"] = df["rate"].astype("Float64")
+
     return OutputCollection(outputs=results, dataframe=df)
 
 
@@ -339,6 +365,9 @@ def calculate_us_poverty_by_age(
         ]
     )
 
+    # Preserve absent rates as JSON null in record exports.
+    df["rate"] = df["rate"].astype("Float64")
+
     return OutputCollection(outputs=results, dataframe=df)
 
 
@@ -378,6 +407,9 @@ def calculate_uk_poverty_by_gender(
         ]
     )
 
+    # Preserve absent rates as JSON null in record exports.
+    df["rate"] = df["rate"].astype("Float64")
+
     return OutputCollection(outputs=results, dataframe=df)
 
 
@@ -417,6 +449,9 @@ def calculate_us_poverty_by_gender(
         ]
     )
 
+    # Preserve absent rates as JSON null in record exports.
+    df["rate"] = df["rate"].astype("Float64")
+
     return OutputCollection(outputs=results, dataframe=df)
 
 
@@ -459,4 +494,7 @@ def calculate_us_poverty_by_race(
         ]
     )
 
+    # Preserve absent rates as JSON null in record exports.
+    df["rate"] = df["rate"].astype("Float64")
+
     return OutputCollection(outputs=results, dataframe=df)

src/policyengine/tax_benefit_models/us/household.py

@@ -38,6 +38,7 @@
 from __future__ import annotations
 
 from collections.abc import Mapping
+from math import isfinite
 from typing import Any, Optional
 
 from policyengine.tax_benefit_models.common import (
@@ -90,7 +91,8 @@ def _default_output_columns(
 
 def _safe_convert(value: Any) -> Any:
     try:
-        return float(value)
+        number = float(value)
+        return number if isfinite(number) else None
     except (ValueError, TypeError):
         return str(value) if value is not None else None