Carry ACS margins of error through cleaning, with reliability labels

backend/Data/warehouse.schema.json

@@ -382,6 +382,8 @@
       "geo_type": "VARCHAR",
       "geoid": "VARCHAR",
       "median_age": "DOUBLE",
+      "median_age_moe": "DOUBLE",
+      "median_age_reliability": "VARCHAR",
       "name": "VARCHAR",
       "year": "VARCHAR"
     },
@@ -419,6 +421,8 @@
       "geo_type": "VARCHAR",
       "geoid": "VARCHAR",
       "median_household_income": "DOUBLE",
+      "median_household_income_moe": "DOUBLE",
+      "median_household_income_reliability": "VARCHAR",
       "name": "VARCHAR",
       "year": "VARCHAR"
     },
@@ -429,6 +433,8 @@
       "geoid": "VARCHAR",
       "name": "VARCHAR",
       "per_capita_income": "DOUBLE",
+      "per_capita_income_moe": "DOUBLE",
+      "per_capita_income_reliability": "VARCHAR",
       "year": "VARCHAR"
     },
     "acs5Housing_housingUnits_timeseries": {
@@ -438,6 +444,8 @@
       "geoid": "VARCHAR",
       "name": "VARCHAR",
       "total_housing_units": "DOUBLE",
+      "total_housing_units_moe": "DOUBLE",
+      "total_housing_units_reliability": "VARCHAR",
       "year": "VARCHAR"
     },
     "acs5Housing_incomeBurden_timeseries": {
@@ -457,6 +465,8 @@
       "geo_type": "VARCHAR",
       "geoid": "VARCHAR",
       "median_home_value": "DOUBLE",
+      "median_home_value_moe": "DOUBLE",
+      "median_home_value_reliability": "VARCHAR",
       "name": "VARCHAR",
       "year": "VARCHAR"
     },
@@ -468,6 +478,8 @@
       "geoid": "VARCHAR",
       "name": "VARCHAR",
       "percent": "DOUBLE",
+      "percent_moe": "DOUBLE",
+      "percent_reliability": "VARCHAR",
       "year": "VARCHAR"
     },
     "acs5_demographics_tidy": {
@@ -477,8 +489,12 @@
       "geoid": "VARCHAR",
       "name": "VARCHAR",
       "percent": "DOUBLE",
+      "percent_moe": "DOUBLE",
+      "percent_reliability": "VARCHAR",
       "section": "VARCHAR",
       "value": "DOUBLE",
+      "value_moe": "DOUBLE",
+      "value_reliability": "VARCHAR",
       "variable": "VARCHAR",
       "year": "VARCHAR"
     },
@@ -488,7 +504,9 @@
       "geo_type": "VARCHAR",
       "geoid": "VARCHAR",
       "measure": "VARCHAR",
+      "moe": "DOUBLE",
       "name": "VARCHAR",
+      "reliability": "VARCHAR",
       "source_label": "VARCHAR",
       "subcategory": "VARCHAR",
       "table": "VARCHAR",
@@ -504,8 +522,12 @@
       "geoid": "VARCHAR",
       "name": "VARCHAR",
       "percent": "DOUBLE",
+      "percent_moe": "DOUBLE",
+      "percent_reliability": "VARCHAR",
       "section": "VARCHAR",
       "value": "DOUBLE",
+      "value_moe": "DOUBLE",
+      "value_reliability": "VARCHAR",
       "variable": "VARCHAR",
       "year": "VARCHAR"
     },
@@ -516,8 +538,12 @@
       "geoid": "VARCHAR",
       "name": "VARCHAR",
       "percent": "DOUBLE",
+      "percent_moe": "DOUBLE",
+      "percent_reliability": "VARCHAR",
       "section": "VARCHAR",
       "value": "BIGINT",
+      "value_moe": "DOUBLE",
+      "value_reliability": "VARCHAR",
       "variable": "VARCHAR",
       "year": "VARCHAR"
     },
@@ -528,8 +554,12 @@
       "geoid": "VARCHAR",
       "name": "VARCHAR",
       "percent": "DOUBLE",
+      "percent_moe": "DOUBLE",
+      "percent_reliability": "VARCHAR",
       "section": "VARCHAR",
       "value": "DOUBLE",
+      "value_moe": "DOUBLE",
+      "value_reliability": "VARCHAR",
       "variable": "VARCHAR",
       "year": "VARCHAR"
     },
@@ -540,8 +570,12 @@
       "geoid": "VARCHAR",
       "name": "VARCHAR",
       "percent": "DOUBLE",
+      "percent_moe": "DOUBLE",
+      "percent_reliability": "VARCHAR",
       "section": "VARCHAR",
       "value": "DOUBLE",
+      "value_moe": "DOUBLE",
+      "value_reliability": "VARCHAR",
       "variable": "VARCHAR",
       "year": "VARCHAR"
     },
@@ -553,6 +587,7 @@
       "name": "VARCHAR",
       "percent": "DOUBLE",
       "percent_moe": "DOUBLE",
+      "percent_reliability": "VARCHAR",
       "section": "VARCHAR",
       "universe": "HUGEINT",
       "value": "HUGEINT",
@@ -655,7 +690,9 @@
       "geo_type": "VARCHAR",
       "geoid": "VARCHAR",
       "measure": "VARCHAR",
+      "moe": "DOUBLE",
       "name": "VARCHAR",
+      "reliability": "VARCHAR",
       "source_label": "VARCHAR",
       "subcategory": "VARCHAR",
       "table": "VARCHAR",
@@ -670,7 +707,9 @@
       "geo_type": "VARCHAR",
       "geoid": "VARCHAR",
       "measure": "VARCHAR",
+      "moe": "DOUBLE",
       "name": "VARCHAR",
+      "reliability": "VARCHAR",
       "source_label": "VARCHAR",
       "subcategory": "VARCHAR",
       "table": "VARCHAR",
@@ -685,7 +724,9 @@
       "geo_type": "VARCHAR",
       "geoid": "VARCHAR",
       "measure": "VARCHAR",
+      "moe": "DOUBLE",
       "name": "VARCHAR",
+      "reliability": "VARCHAR",
       "source_label": "VARCHAR",
       "subcategory": "VARCHAR",
       "table": "VARCHAR",
@@ -700,7 +741,9 @@
       "geo_type": "VARCHAR",
       "geoid": "VARCHAR",
       "measure": "VARCHAR",
+      "moe": "DOUBLE",
       "name": "VARCHAR",
+      "reliability": "VARCHAR",
       "source_label": "VARCHAR",
       "subcategory": "VARCHAR",
       "table": "VARCHAR",

backend/data_cleaning/clean_acs5.py

@@ -1,6 +1,7 @@
 import duckdb
 
 from data_cleaning.geo_lookup import inline_sql_unavailable
+from data_cleaning.margins import reliability_sql
 
 DP_TABLES = {
     "DP02": ("acs5_social", "dp_social"),
@@ -36,6 +37,26 @@
 }
 
 
+def published_moe_sql(moe: str) -> str:
+    """
+    A raw DP table's published margin of error as a number. -555555555 means
+    the estimate is controlled, so its margin is zero; other annotation codes
+    (negative, "(X)") are missing.
+
+    Args:
+        moe: SQL expression for the raw MOE column
+
+    Returns:
+        str: A SQL CASE expression
+    """
+    return f"""CASE
+        WHEN TRY_CAST({moe} AS DOUBLE) = -555555555 THEN 0
+        WHEN TRY_CAST({moe} AS DOUBLE) IN ({inline_sql_unavailable(DP_UNAVAILABLE)})
+            THEN NULL
+        WHEN TRY_CAST({moe} AS DOUBLE) >= 0 THEN TRY_CAST({moe} AS DOUBLE)
+    END"""
+
+
 def _dp_select_sql(dp: str, raw_table_name: str) -> str:
     """
     Build the SELECT query for a single census DP table, joined against `vt_town_lines`
@@ -63,6 +84,8 @@ def _dp_select_sql(dp: str, raw_table_name: str) -> str:
             g.variable,
             g.measure,
             g.value,
+            g.moe,
+            {reliability_sql("g.value", "g.moe")} AS reliability,
             g."table",
             g.geo_type,
             g.geoid
@@ -81,6 +104,9 @@ def _dp_select_sql(dp: str, raw_table_name: str) -> str:
                     WHEN CAST(n.Value AS DOUBLE) IN ({inline_sql_unavailable(DP_UNAVAILABLE)}) THEN NULL
                     ELSE CAST(n.Value AS DOUBLE)
                 END AS "value",
+                -- The published margin of error (M for an estimate, PM for a
+                -- percent).
+                {published_moe_sql("n.MOE")} AS moe,
                 '{dp}' AS "table",
                 n.geo_type_norm AS geo_type,
                 CASE

backend/data_cleaning/clean_acs5_tidy.py

@@ -9,14 +9,18 @@
     `snapshot` table that combines a few key indicators from them.
 
     Each output is in tidy format: year, geoid, name, geo_type, county, county_fips,
-    section, variable, value, percent.
+    section, variable, value, value_moe, value_reliability, percent,
+    percent_moe, percent_reliability. The *_moe columns are 90% margins of
+    error (percent_moe in percentage points), computed at collection (see
+    data_collection/base.py); *_reliability labels them (see margins.py).
 **Run with**:
 python run_data_cleaning.py clean_acs5_tidy
 """
 
 import duckdb
 
 from data_cleaning.geo_lookup import acs_geo_sql, build_geo_lookups, write_table
+from data_cleaning.margins import reliability_sql
 
 # RAW table name -> CLEANED table name
 TIDY_TABLES = {
@@ -57,7 +61,20 @@ def tidy_sql(base: str) -> str:
             section AS section,
             variable AS variable,
             value AS value,
-            percent AS percent
+            value_moe AS value_moe,
+            {reliability_sql("value", "value_moe")} AS value_reliability,
+            percent AS percent,
+            percent_moe AS percent_moe,
+            -- Collection rounds percent_moe to 0.1 point, so a small share's
+            -- real margin can round to 0, which reliability_sql reads as a
+            -- controlled estimate. Unless the count is controlled too, label
+            -- the share by its count: exact when the total is controlled, and
+            -- otherwise never rosier than the share's own label.
+            CASE
+                WHEN percent_moe = 0 AND value_moe > 0
+                    THEN {reliability_sql("value", "value_moe")}
+                ELSE {reliability_sql("percent", "percent_moe")}
+            END AS percent_reliability
         FROM ({acs_geo_sql(base)})
         """