From e369f2da9411dac99e09078506139b3eceb1bb65 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Thu, 29 Oct 2026 12:10:33 -0400 Subject: [PATCH 1/5] Treat an uncomputable tract margin of error as missing, not zero Census code -222222222 means the margin could not be computed because there were too few sample cases. The tract cleaner mapped it to 0, which gave the least reliable estimates the tightest intervals. Only -555555555 (a controlled estimate) has a zero margin. --- backend/data_cleaning/clean_acs5_tract.py | 7 ++++--- backend/tests/test_acs5_tract.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/backend/data_cleaning/clean_acs5_tract.py b/backend/data_cleaning/clean_acs5_tract.py index 423431c3..a24818b7 100644 --- a/backend/data_cleaning/clean_acs5_tract.py +++ b/backend/data_cleaning/clean_acs5_tract.py @@ -96,11 +96,12 @@ def tract_sql() -> str: SELECT year, NAME, state, county, tract, code, CASE WHEN estimate >= 0 THEN estimate END AS est, - -- -555555555 / -222222222: the estimate is controlled, so its - -- margin of error is zero; other negative codes are missing. + -- -555555555: the estimate is controlled, so its margin of + -- error is zero. Other negative codes are missing, e.g. + -- -222222222: too few sample cases to compute a margin. CASE WHEN moe >= 0 THEN moe - WHEN moe IN (-555555555, -222222222) THEN 0 + WHEN moe = -555555555 THEN 0 END AS moe FROM lake.RAW.acs5_tract ), diff --git a/backend/tests/test_acs5_tract.py b/backend/tests/test_acs5_tract.py index 6e6890de..18ee8290 100644 --- a/backend/tests/test_acs5_tract.py +++ b/backend/tests/test_acs5_tract.py @@ -110,6 +110,24 @@ def test_uninsured_split_by_poverty_is_a_crosstab(lake): assert len(uninsured(BELOW_138 + AT_OR_ABOVE_138)) == 15 +def test_controlled_margin_counts_as_zero(lake): + # -555555555: the universe is controlled, so it has no sampling error. + df = _clean(lake, {"B17001_001": (1000, -555555555), "B17001_002": (150, 50)}) + row = df[df.variable == "Below poverty level"].iloc[0] + + assert row.percent_moe == round(100 * 50 / 1000, 1) + + +def test_uncomputable_margin_makes_the_margin_null(lake): + # -222222222: too few sample cases to compute a margin. The estimate + # stands, but its margin is unknown, not zero. + df = _clean(lake, {"B17001_001": (1000, 100), "B17001_002": (0, -222222222)}) + row = df[df.variable == "Below poverty level"].iloc[0] + + assert row.percent == 0.0 + assert pd.isna(row.percent_moe) + + def test_suppressed_component_makes_the_indicator_null(lake): df = _clean( lake, {"B17001_001": (1000, 100), "B17001_002": (-666666666, -222222222)} From 30265ee4cb24e0aaafe9d7e191db0946bda8fa83 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Thu, 29 Oct 2026 12:35:26 -0400 Subject: [PATCH 2/5] Apply the Census zero-estimate and 100%-share rules to tract margins Sums of many zero estimates overstated the margin: a zero estimate's margin is modeled, and Census guidance counts only the largest of them in a sum. Uninsured shares sum 15 C27016 cells, most of them zero, and the widest tract margin was 215.8 points. Separately, a 100% share made the proportion formula's term zero and claimed no sampling error at all; the ratio form now covers that case, as it already did a negative term. --- backend/data_cleaning/clean_acs5_tract.py | 24 ++++++++++++++++------- backend/tests/test_acs5_tract.py | 22 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/backend/data_cleaning/clean_acs5_tract.py b/backend/data_cleaning/clean_acs5_tract.py index a24818b7..ebe9aa92 100644 --- a/backend/data_cleaning/clean_acs5_tract.py +++ b/backend/data_cleaning/clean_acs5_tract.py @@ -5,7 +5,7 @@ Each indicator is a share (count / universe) with a 90% margin of error, derived with the Census Bureau's formulas for sums (root sum of squared - MOEs) and proportions. Census sentinels (negative estimates) and any + MOEs, counting only the largest among zero estimates) and proportions. Census sentinels (negative estimates) and any missing component make the indicator NULL instead of a partial sum. Writes lake.CLEANED.acs5_tract_tidy: year, geoid, county_fips, county, @@ -64,6 +64,17 @@ def _in_list(codes: list[str]) -> str: return ", ".join(f"'{c}'" for c in codes) +def _sum_moe(codes: list[str]) -> str: + """Margin of error of the sum of `codes`: root sum of squares, counting + only the largest margin among zero estimates, whose margins are modeled + (Census guidance; matches data_collection/base.py).""" + codes = _in_list(codes) + return f"""SQRT( + COALESCE(SUM(moe * moe) FILTER (WHERE code IN ({codes}) AND est <> 0), 0) + + COALESCE(MAX(moe * moe) FILTER (WHERE code IN ({codes}) AND est = 0), 0) + )""" + + def indicator_sql(section: str, variable: str, num: list[str], den: list[str]) -> str: """One row per tract and year for a single count / universe indicator.""" return f"""--sql @@ -74,13 +85,11 @@ def indicator_sql(section: str, variable: str, num: list[str], den: list[str]) - SUM(est) FILTER (WHERE code IN ({_in_list(num)})) AS num, COUNT(est) FILTER (WHERE code IN ({_in_list(num)})) = {len(num)} AS num_ok, - SQRT(SUM(moe * moe) FILTER (WHERE code IN ({_in_list(num)}))) - AS num_moe, + {_sum_moe(num)} AS num_moe, SUM(est) FILTER (WHERE code IN ({_in_list(den)})) AS den, COUNT(est) FILTER (WHERE code IN ({_in_list(den)})) = {len(den)} AS den_ok, - SQRT(SUM(moe * moe) FILTER (WHERE code IN ({_in_list(den)}))) - AS den_moe, + {_sum_moe(den)} AS den_moe, COUNT(moe) FILTER (WHERE code IN ({_in_list(num + den)})) = {len(set(num + den))} AS moe_ok FROM raw @@ -132,11 +141,12 @@ def tract_sql() -> str: CASE WHEN r.den_ok THEN r.den END AS universe, ROUND(100 * r.p, 1) AS percent, -- Census proportion MOE; the + form when the difference goes - -- negative, per the ACS handbook. + -- negative, per the ACS handbook, or is zero (a 100% share, + -- which would otherwise claim no sampling error). CASE WHEN r.p IS NOT NULL AND r.moe_ok THEN ROUND( 100 * SQRT( CASE - WHEN r.num_moe ^ 2 >= r.p ^ 2 * r.den_moe ^ 2 + WHEN r.num_moe ^ 2 > r.p ^ 2 * r.den_moe ^ 2 THEN r.num_moe ^ 2 - r.p ^ 2 * r.den_moe ^ 2 ELSE r.num_moe ^ 2 + r.p ^ 2 * r.den_moe ^ 2 END diff --git a/backend/tests/test_acs5_tract.py b/backend/tests/test_acs5_tract.py index 18ee8290..e9f3452c 100644 --- a/backend/tests/test_acs5_tract.py +++ b/backend/tests/test_acs5_tract.py @@ -128,6 +128,28 @@ def test_uncomputable_margin_makes_the_margin_null(lake): assert pd.isna(row.percent_moe) +def test_zero_estimates_count_only_their_largest_margin(lake): + # Of the 15 uninsured codes, 5 are 8 and 10 are zero. One zero code has a + # margin of 25 and the rest 10, so the sum's margin is + # sqrt(10^2 * 5 + 25^2): every nonzero margin, plus the largest zero one. + estimates = _c27016_fixture(uninsured_count=8) + zero = next(c for c in uninsured(BELOW_138) if estimates[c][0] == 0) + estimates[zero] = (0, 25) + df = _clean(lake, estimates).set_index("variable") + num_moe = math.sqrt(10**2 * 5 + 25**2) + p = 40 / 500 + expected = 100 * math.sqrt(num_moe**2 - p**2 * 40**2) / 500 + assert df.loc["No health insurance", "percent_moe"] == round(expected, 1) + + +def test_full_share_margin_is_not_zero(lake): + df = _clean(lake, {"B17001_001": (40, 12), "B17001_002": (40, 12)}) + row = df[df.variable == "Below poverty level"].iloc[0] + + assert row.percent == 100.0 + assert row.percent_moe == round(100 * math.sqrt(12**2 + 12**2) / 40, 1) + + def test_suppressed_component_makes_the_indicator_null(lake): df = _clean( lake, {"B17001_001": (1000, 100), "B17001_002": (-666666666, -222222222)} From d12d7a058a196a235229d56cd95423716d3101f7 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Thu, 29 Oct 2026 12:40:18 -0400 Subject: [PATCH 3/5] Carry ACS margins of error through cleaning, with reliability labels The cleaned tidy, data profile, series and tract tables now keep the margins collected in the RAW tables, and each margin gets a high, medium or low reliability label from its coefficient of variation (data_cleaning/margins.py). The Data Chat catalog lists the new columns and explains how to read them. warehouse.schema.json is regenerated from a scratch build of every ACS table (2009-2024). --- backend/Data/warehouse.schema.json | 43 ++++++ backend/data_cleaning/clean_acs5.py | 12 ++ backend/data_cleaning/clean_acs5_tidy.py | 21 ++- .../data_cleaning/clean_acs5_timeseries.py | 56 +++++-- backend/data_cleaning/clean_acs5_tract.py | 15 +- backend/data_cleaning/margins.py | 41 ++++++ backend/data_tools/catalog.py | 139 +++++++++++++++--- backend/data_tools/service.py | 11 +- backend/tests/test_acs5_tidy_moe.py | 116 +++++++++++++++ backend/tests/test_acs5_tract.py | 12 ++ backend/tests/test_clean_acs5.py | 39 ++++- backend/tests/test_margins.py | 24 +++ backend/tests/test_mcp_catalog.py | 27 +++- backend/tests/test_mcp_issue_fixes.py | 45 ++++++ 14 files changed, 554 insertions(+), 47 deletions(-) create mode 100644 backend/data_cleaning/margins.py create mode 100644 backend/tests/test_acs5_tidy_moe.py create mode 100644 backend/tests/test_margins.py diff --git a/backend/Data/warehouse.schema.json b/backend/Data/warehouse.schema.json index fb7385de..86ce1485 100644 --- a/backend/Data/warehouse.schema.json +++ b/backend/Data/warehouse.schema.json @@ -382,6 +382,8 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "median_age": "DOUBLE", + "median_age_moe": "DOUBLE", + "median_age_reliability": "VARCHAR", "name": "VARCHAR", "year": "VARCHAR" }, @@ -419,6 +421,8 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "median_household_income": "DOUBLE", + "median_household_income_moe": "DOUBLE", + "median_household_income_reliability": "VARCHAR", "name": "VARCHAR", "year": "VARCHAR" }, @@ -429,6 +433,8 @@ "geoid": "VARCHAR", "name": "VARCHAR", "per_capita_income": "DOUBLE", + "per_capita_income_moe": "DOUBLE", + "per_capita_income_reliability": "VARCHAR", "year": "VARCHAR" }, "acs5Housing_housingUnits_timeseries": { @@ -438,6 +444,8 @@ "geoid": "VARCHAR", "name": "VARCHAR", "total_housing_units": "DOUBLE", + "total_housing_units_moe": "DOUBLE", + "total_housing_units_reliability": "VARCHAR", "year": "VARCHAR" }, "acs5Housing_incomeBurden_timeseries": { @@ -457,6 +465,8 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "median_home_value": "DOUBLE", + "median_home_value_moe": "DOUBLE", + "median_home_value_reliability": "VARCHAR", "name": "VARCHAR", "year": "VARCHAR" }, @@ -468,6 +478,8 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "year": "VARCHAR" }, "acs5_demographics_tidy": { @@ -477,8 +489,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "DOUBLE", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -488,7 +504,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", @@ -504,8 +522,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "DOUBLE", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -516,8 +538,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "BIGINT", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -528,8 +554,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "DOUBLE", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -540,8 +570,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "DOUBLE", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -553,6 +587,7 @@ "name": "VARCHAR", "percent": "DOUBLE", "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "universe": "HUGEINT", "value": "HUGEINT", @@ -655,7 +690,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", @@ -670,7 +707,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", @@ -685,7 +724,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", @@ -700,7 +741,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", diff --git a/backend/data_cleaning/clean_acs5.py b/backend/data_cleaning/clean_acs5.py index 058028a1..d4218d3f 100644 --- a/backend/data_cleaning/clean_acs5.py +++ b/backend/data_cleaning/clean_acs5.py @@ -1,6 +1,7 @@ import duckdb from data_cleaning.geo_lookup import inline_sql_unavailable +from data_cleaning.margins import reliability_sql DP_TABLES = { "DP02": ("acs5_social", "dp_social"), @@ -63,6 +64,8 @@ def _dp_select_sql(dp: str, raw_table_name: str) -> str: g.variable, g.measure, g.value, + g.moe, + {reliability_sql("g.value", "g.moe")} AS reliability, g."table", g.geo_type, g.geoid @@ -81,6 +84,15 @@ def _dp_select_sql(dp: str, raw_table_name: str) -> str: WHEN CAST(n.Value AS DOUBLE) IN ({inline_sql_unavailable(DP_UNAVAILABLE)}) THEN NULL ELSE CAST(n.Value AS DOUBLE) END AS "value", + -- The published margin of error (M for an estimate, PM for a + -- percent). -555555555: the estimate is controlled, so its + -- margin is zero; other codes (negative, "(X)") are missing. + CASE + WHEN TRY_CAST(n.MOE AS DOUBLE) = -555555555 THEN 0 + WHEN TRY_CAST(n.MOE AS DOUBLE) IN ({inline_sql_unavailable(DP_UNAVAILABLE)}) + THEN NULL + WHEN TRY_CAST(n.MOE AS DOUBLE) >= 0 THEN TRY_CAST(n.MOE AS DOUBLE) + END AS moe, '{dp}' AS "table", n.geo_type_norm AS geo_type, CASE diff --git a/backend/data_cleaning/clean_acs5_tidy.py b/backend/data_cleaning/clean_acs5_tidy.py index 67963df0..2f324707 100644 --- a/backend/data_cleaning/clean_acs5_tidy.py +++ b/backend/data_cleaning/clean_acs5_tidy.py @@ -9,7 +9,10 @@ `snapshot` table that combines a few key indicators from them. Each output is in tidy format: year, geoid, name, geo_type, county, county_fips, - section, variable, value, percent. + section, variable, value, value_moe, value_reliability, percent, + percent_moe, percent_reliability. The *_moe columns are 90% margins of + error (percent_moe in percentage points), computed at collection (see + data_collection/base.py); *_reliability labels them (see margins.py). **Run with**: python run_data_cleaning.py clean_acs5_tidy """ @@ -17,6 +20,7 @@ import duckdb from data_cleaning.geo_lookup import acs_geo_sql, build_geo_lookups, write_table +from data_cleaning.margins import reliability_sql # RAW table name -> CLEANED table name TIDY_TABLES = { @@ -57,7 +61,20 @@ def tidy_sql(base: str) -> str: section AS section, variable AS variable, value AS value, - percent AS percent + value_moe AS value_moe, + {reliability_sql("value", "value_moe")} AS value_reliability, + percent AS percent, + percent_moe AS percent_moe, + -- Collection rounds percent_moe to 0.1 point, so a small share's + -- real margin can round to 0, which reliability_sql reads as a + -- controlled estimate. Unless the count is controlled too, label + -- the share by its count: exact when the total is controlled, and + -- otherwise never rosier than the share's own label. + CASE + WHEN percent_moe = 0 AND value_moe > 0 + THEN {reliability_sql("value", "value_moe")} + ELSE {reliability_sql("percent", "percent_moe")} + END AS percent_reliability FROM ({acs_geo_sql(base)}) """ diff --git a/backend/data_cleaning/clean_acs5_timeseries.py b/backend/data_cleaning/clean_acs5_timeseries.py index 9bf81d7e..8be9674a 100644 --- a/backend/data_cleaning/clean_acs5_timeseries.py +++ b/backend/data_cleaning/clean_acs5_timeseries.py @@ -31,6 +31,7 @@ build_geo_lookups, write_table, ) +from data_cleaning.margins import reliability_sql # Raw identifier columns every base query must expose (see `acs_geo_sql`) KEYS = "year, NAME, geo_type, state, county" @@ -99,12 +100,18 @@ def in_list(values: list[str]) -> str: return ", ".join(f"'{v}'" for v in values) +def with_moe(column: str) -> tuple[str, str, str]: + """A measure column with its margin of error and reliability columns.""" + return (column, f"{column}_moe", f"{column}_reliability") + + def single_variable( source: str, variables: list[str], out_col: str, value_col: str = "Value", keep_variable: bool = False, + moe_col: str | None = None, ) -> str: """ Filters a RAW lake table to Key, variable (if true), and value columns @@ -115,6 +122,9 @@ def single_variable( out_col: value_col: Column name containing the tracked value over time. keep_variable: (T/F) if you want to keep the variable column. Best for timeseries tables tracking multiple variables. + moe_col: Column holding value_col's 90% margin of error, written as + `{out_col}_moe` with its `{out_col}_reliability` label (see + :func:`with_moe`). None leaves both out. Returns: str: A SQL query which filters the source table to contain the time series variable(s) @@ -122,8 +132,15 @@ def single_variable( """ variable = ", Variable" if keep_variable else "" + moe = ( + f", {numeric(moe_col)} AS {out_col}_moe, " + f"{reliability_sql(numeric(value_col), numeric(moe_col))} " + f"AS {out_col}_reliability" + if moe_col + else "" + ) return f"""--sql - SELECT {KEYS}{variable}, {numeric(value_col)} AS {out_col} + SELECT {KEYS}{variable}, {numeric(value_col)} AS {out_col}{moe} FROM lake.RAW.{source} WHERE Variable IN ({in_list(variables)}) """ @@ -318,34 +335,48 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: # Median age "median_age": TimeSeries( table="acs5Demographics_medianAge_timeseries", - base=single_variable("demographics", ["Median Age"], "median_age"), - columns=("median_age",), + base=single_variable( + "demographics", ["Median Age"], "median_age", moe_col="Value_MOE" + ), + columns=with_moe("median_age"), ), # Median Household Income "median_hh_income": TimeSeries( table="acs5Economics_medianHouseholdIncome_timeseries", base=single_variable( - "economic", ["Median Household Income"], "median_household_income" + "economic", + ["Median Household Income"], + "median_household_income", + moe_col="Value_MOE", ), - columns=("median_household_income",), + columns=with_moe("median_household_income"), ), # Median Home Value "median_home_value": TimeSeries( table="acs5Housing_medianHomeValue_timeseries", - base=single_variable("housing", ["Median Home Value"], "median_home_value"), - columns=("median_home_value",), + base=single_variable( + "housing", ["Median Home Value"], "median_home_value", moe_col="Value_MOE" + ), + columns=with_moe("median_home_value"), ), # Per Capita Income "per_capita_income": TimeSeries( table="acs5Economics_perCapitaIncome_timeseries", - base=single_variable("economic", ["Per Capita Income"], "per_capita_income"), - columns=("per_capita_income",), + base=single_variable( + "economic", ["Per Capita Income"], "per_capita_income", moe_col="Value_MOE" + ), + columns=with_moe("per_capita_income"), ), # Total Housing Units "total_housing_units": TimeSeries( table="acs5Housing_housingUnits_timeseries", - base=single_variable("housing", ["Total Housing Units"], "total_housing_units"), - columns=("total_housing_units",), + base=single_variable( + "housing", + ["Total Housing Units"], + "total_housing_units", + moe_col="Value_MOE", + ), + columns=with_moe("total_housing_units"), ), # Vacancy Rates "vacancy_rate": TimeSeries( @@ -356,8 +387,9 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: "percent", value_col="Percent", keep_variable=True, + moe_col="Percent_MOE", ), - columns=("Variable", "percent"), + columns=("Variable", *with_moe("percent")), ), # Population Change (%) "population_change": TimeSeries( diff --git a/backend/data_cleaning/clean_acs5_tract.py b/backend/data_cleaning/clean_acs5_tract.py index ebe9aa92..a5d88f9d 100644 --- a/backend/data_cleaning/clean_acs5_tract.py +++ b/backend/data_cleaning/clean_acs5_tract.py @@ -10,12 +10,13 @@ Writes lake.CLEANED.acs5_tract_tidy: year, geoid, county_fips, county, name, geo_type, section, variable, value (count), universe, percent, - percent_moe. + percent_moe, percent_reliability (see data_cleaning/margins.py). """ import duckdb from data_cleaning.geo_lookup import build_geo_lookups, write_table +from data_cleaning.margins import reliability_sql TABLE_NAME = "acs5_tract_tidy" @@ -159,6 +160,16 @@ def tract_sql() -> str: """ +def reliable_tract_sql() -> str: + """:func:`tract_sql` with a reliability label for each percent.""" + return f"""--sql + SELECT + *, + {reliability_sql("percent", "percent_moe")} AS percent_reliability + FROM ({tract_sql()}) + """ + + def main(con: duckdb.DuckDBPyConnection): build_geo_lookups(con) - write_table(con, TABLE_NAME, tract_sql()) + write_table(con, TABLE_NAME, reliable_tract_sql()) diff --git a/backend/data_cleaning/margins.py b/backend/data_cleaning/margins.py new file mode 100644 index 00000000..724cf648 --- /dev/null +++ b/backend/data_cleaning/margins.py @@ -0,0 +1,41 @@ +""" +**Description**: + Shared SQL for ACS margins of error in the cleaning step. + + Each margin of error column (`_moe`, or `moe` in the data + profiles) gets a reliability label beside it, from the coefficient of + variation CV = standard error / estimate, where the standard error of an + ACS 90% margin is MOE / 1.645. The Census Bureau sets no fixed cut + points; CV under 15% (high) and over 30% (low) is a common convention + among ACS data users. +""" + +Z_90 = 1.645 +HIGH_CV = 0.15 # below this, high reliability +LOW_CV = 0.30 # above this, low reliability + + +def reliability_sql(estimate: str, moe: str) -> str: + """ + 'high', 'medium' or 'low' reliability for an estimate and its 90% margin. + + A zero estimate with a nonzero margin is 'low': its CV is undefined, and + the true value may well not be zero. A zero margin (a controlled + estimate) is 'high'. Null when either input is null. + + Args: + estimate: SQL expression for the estimate + moe: SQL expression for its margin of error + + Returns: + str: A SQL CASE expression + """ + cv = f"{moe} / {Z_90} / ABS({estimate})" + return f"""CASE + WHEN {estimate} IS NULL OR {moe} IS NULL THEN NULL + WHEN {moe} = 0 THEN 'high' + WHEN {estimate} = 0 THEN 'low' + WHEN {cv} < {HIGH_CV} THEN 'high' + WHEN {cv} <= {LOW_CV} THEN 'medium' + ELSE 'low' + END""" diff --git a/backend/data_tools/catalog.py b/backend/data_tools/catalog.py index 05e92606..775e1df9 100644 --- a/backend/data_tools/catalog.py +++ b/backend/data_tools/catalog.py @@ -40,19 +40,41 @@ def __post_init__(self) -> None: ACS_URL = "https://www.census.gov/programs-surveys/acs/data.html" +_ACS_PERIOD = ( + "ACS five-year estimates: year is the final year of the five-year period, " + "not a single-year count. Adjacent periods overlap." +) +_ACS_NULLS = ( + "Unavailable Census values and non-finite numbers are returned as null; " + "null does not mean zero. Counts, percentages, and medians are separate measures." +) ACS_CAVEATS = ( - ( - "ACS five-year estimates: year is the final year of the five-year period, " - "not a single-year count. Adjacent periods overlap." - ), + _ACS_PERIOD, ( "Margins of error are not included in these cleaned tables. Differences " "alone do not establish statistical significance." ), + _ACS_NULLS, +) +# For tables that carry *_moe columns. +ACS_MOE_CAVEATS = ( + _ACS_PERIOD, ( - "Unavailable Census values and non-finite numbers are returned as null; " - "null does not mean zero. Counts, percentages, and medians are separate measures." + "Columns ending in _moe are 90% margins of error: the estimate plus or " + "minus its margin is the 90% confidence interval; percent_moe is in " + "percentage points. Margins of sums and percentages are Census " + "approximations. For estimates from periods that do not overlap (such as " + "2018 and 2023), or from separate places in the same period, a difference " + "larger than the square root of the sum of their squared margins is " + "significant at 90% confidence. That test does not apply to adjacent " + "years, whose five-year periods share four years, or to a place and an " + "area that contains it. A null margin means the Census could not compute one. Each margin has a " + "reliability label (high, medium, low) from the coefficient of variation, " + "CV = (moe / 1.645) / estimate: under 15% high, over 30% low; a zero " + "estimate with a nonzero margin is low. Treat low-reliability estimates " + "as rough, especially for small towns." ), + _ACS_NULLS, ) ZONING_URL = "https://geodata.vermont.gov/datasets/VCGI::vt-zoning-areas/about" ZONING_CAVEATS = ( @@ -96,13 +118,18 @@ def _acs_tidy(key: str, label: str, description: str, table: str) -> Dataset: table, ACS_URL, "U.S. Census Bureau, American Community Survey five-year estimates", - ACS_CAVEATS, + ACS_MOE_CAVEATS, "tidy", "year", "name", "geo_type", variable_columns=("section", "variable"), - value_columns={"value": "varies by variable", "percent": "percent"}, + value_columns={ + "value": "varies by variable", + "value_moe": "varies by variable", + "percent": "percent", + "percent_moe": "percentage points", + }, filter_columns=( "year", "name", @@ -112,6 +139,8 @@ def _acs_tidy(key: str, label: str, description: str, table: str) -> Dataset: "geoid", "section", "variable", + "value_reliability", + "percent_reliability", ), ) @@ -131,6 +160,9 @@ def _acs_series( # ACS series carry the standard geoid/county_fips identifiers; `county` is # False for tables built with `with_county=False` in clean_acs5_timeseries. identifiers = ("geoid", "county_fips") + (("county",) if county else ()) + # Series with _moe columns also carry _reliability. + margins = [c.removesuffix("_moe") for c in values if c.endswith("_moe")] + acs_caveats = ACS_MOE_CAVEATS if margins else ACS_CAVEATS return Dataset( f"acs5_ts_{key}", label, @@ -145,7 +177,7 @@ def _acs_series( ( ("Historic boundaries and coverage can change between censuses.",) if historic - else ACS_CAVEATS + else acs_caveats ) + caveats, "tidy" if dimensions else "wide", @@ -157,7 +189,8 @@ def _acs_series( value_columns=values, filter_columns=("year", "name", "geo_type") + dimensions - + (("geoid", "county_fips", "County") if historic else identifiers), + + (("geoid", "county_fips", "County") if historic else identifiers) + + tuple(f"{measure}_reliability" for measure in margins), ) @@ -388,7 +421,7 @@ def _zoning_numeric(column: str) -> bool: "median_age", "Median age", "acs5Demographics_medianAge_timeseries", - {"median_age": "years"}, + {"median_age": "years", "median_age_moe": "years"}, ), _acs_series( "age_dependency_ratio", @@ -400,13 +433,16 @@ def _zoning_numeric(column: str) -> bool: "median_home_value", "Median home value", "acs5Housing_medianHomeValue_timeseries", - {"median_home_value": "USD (year-specific dollars)"}, + { + "median_home_value": "USD (year-specific dollars)", + "median_home_value_moe": "USD (year-specific dollars)", + }, ), _acs_series( "vacancy_rates", "Vacancy rates", "acs5Housing_vacancyRates_timeseries", - {"percent": "percent"}, + {"percent": "percent", "percent_moe": "percentage points"}, dimensions=("Variable",), ), _acs_series( @@ -427,13 +463,19 @@ def _zoning_numeric(column: str) -> bool: "household_income", "Median household income", "acs5Economics_medianHouseholdIncome_timeseries", - {"median_household_income": "USD (year-specific dollars)"}, + { + "median_household_income": "USD (year-specific dollars)", + "median_household_income_moe": "USD (year-specific dollars)", + }, ), _acs_series( "per_capita_income", "Per capita income", "acs5Economics_perCapitaIncome_timeseries", - {"per_capita_income": "USD (year-specific dollars)"}, + { + "per_capita_income": "USD (year-specific dollars)", + "per_capita_income_moe": "USD (year-specific dollars)", + }, ), _acs_series( "median_earnings", @@ -456,7 +498,10 @@ def _zoning_numeric(column: str) -> bool: "housing_units", "Housing units", "acs5Housing_housingUnits_timeseries", - {"total_housing_units": "housing units"}, + { + "total_housing_units": "housing units", + "total_housing_units_moe": "housing units", + }, ), Dataset( "acs5_dp", @@ -466,7 +511,7 @@ def _zoning_numeric(column: str) -> bool: "acs5_dp_combined_tidy", ACS_URL, "U.S. Census Bureau, ACS data profiles", - ACS_CAVEATS + ACS_MOE_CAVEATS + ( ( "Estimate and percentage labels vary across source years. " @@ -490,7 +535,11 @@ def _zoning_numeric(column: str) -> bool: "year", "name", variable_columns=("table", "category", "subcategory", "variable", "measure"), - value_columns={"value": "varies by variable and measure"}, + # moe is value's published 90% margin of error; reliability labels it. + value_columns={ + "value": "varies by variable and measure", + "moe": "same as value", + }, filter_columns=( "name", "geo_type", @@ -504,6 +553,7 @@ def _zoning_numeric(column: str) -> bool: "variable_code", "source_label", "year", + "reliability", ), ), Dataset( @@ -959,6 +1009,14 @@ def dataset_lineage(dataset: Dataset) -> dict[str, Any]: "code_reference": f"backend/data_collection/{source}.py", } } + if f"{output}_moe" in dataset.value_columns: + result["source_columns"][f"{output}_moe"] = { + "raw_table": f"RAW.{source}", + "raw_column": "Value_MOE", + "selectors": {"Variable": variable}, + "census_codes": [code[:-1] + "M"], + "code_reference": f"backend/data_collection/{source}.py", + } result["transformations"].insert(0, f"backend/data_collection/{source}.py") if dataset.id == "acs5_ts_median_home_value": result["source_code_note"] += ( @@ -1020,7 +1078,18 @@ def dataset_lineage(dataset: Dataset) -> dict[str, Any]: "raw_column": "Value", "census_codes": [], "selector_columns": list(dataset.variable_columns), - } + }, + "moe": { + "raw_tables": [ + "RAW.acs5_social", + "RAW.acs5_economic", + "RAW.acs5_housing", + "RAW.acs5_demographic", + ], + "raw_column": "MOE", + "census_codes": [], + "selector_columns": list(dataset.variable_columns), + }, } return result @@ -1317,12 +1386,32 @@ def decode_variable_id(dataset: Dataset, variable_id: str) -> dict[str, Any]: return selectors +def margin_column(dataset: Dataset, measure: str) -> str | None: + """The column holding `measure`'s 90% margin of error, if it has one.""" + columns = dataset.value_columns or {} + if f"{measure}_moe" in columns: + return f"{measure}_moe" + # Data profile rows keep their published margin as `moe` beside `value`. + if dataset.kind == "dp" and measure == "value" and "moe" in columns: + return "moe" + return None + + +def with_margins(dataset: Dataset, measures: list[str]) -> list[str]: + """`measures` with each one's margin right after it. A margin means nothing + without its estimate, so the two always travel together.""" + paired = [] + for measure in measures: + paired += [measure, margin_column(dataset, measure)] + return list(dict.fromkeys(m for m in paired if m)) + + def variable_units(dataset: Dataset, selectors: dict[str, Any]) -> dict[str, str]: """Resolve row-specific units without treating every ACS value as a count.""" units = dict(dataset.value_columns or {}) column = selectors.get("$column") if column: - return {column: units[column]} + return {c: units[c] for c in with_margins(dataset, [column])} variable = str(selectors.get("variable", "")) if dataset.id in { "acs5_demographics", @@ -1383,6 +1472,7 @@ def variable_units(dataset: Dataset, selectors: dict[str, Any]) -> dict[str, str "Median Age": "years", } units["value"] = value_units.get(variable, "source-defined; unit unavailable") + units["value_moe"] = units["value"] if dataset.kind == "dp": measure = str(selectors.get("measure", "")).casefold() subcategory = str(selectors.get("subcategory", "")).strip().casefold() @@ -1400,6 +1490,10 @@ def variable_units(dataset: Dataset, selectors: dict[str, Any]) -> dict[str, str units["value"] = "percent" else: units["value"] = "source-defined estimate; inspect variable label" + # A percent's margin (the PM code) is in percentage points. + units["moe"] = ( + "percentage points" if units["value"] == "percent" else units["value"] + ) return units @@ -1450,10 +1544,13 @@ def search_variables( for row in rows ] else: + # A margin column comes back with its estimate, not as its own variable. + margins = {margin_column(dataset, c) for c in dataset.value_columns} selectors_list = [ {"$column": column} for column in dataset.value_columns - if all( + if column not in margins + and all( token in " ".join(re.findall(r"[a-z0-9]+", column.casefold())) for token in tokens ) diff --git a/backend/data_tools/service.py b/backend/data_tools/service.py index fd0083e5..a8cf3164 100644 --- a/backend/data_tools/service.py +++ b/backend/data_tools/service.py @@ -389,15 +389,22 @@ def _where(self, ds, request, locations, column_types=None, location_clause=None raise DataToolError( f"Unsupported measures. Allowed: {', '.join(ds.value_columns)}" ) + margins = set(catalog.with_margins(ds, selected_columns)) - set( + selected_columns + ) if ( request.measures and selected_columns - and set(request.measures) != set(selected_columns) + and set(request.measures) - margins != set(selected_columns) ): raise DataToolError( "measures must match the selected wide-column variable IDs" ) - return " AND ".join(clauses) or "TRUE", params, list(dict.fromkeys(measures)) + return ( + " AND ".join(clauses) or "TRUE", + params, + catalog.with_margins(ds, measures), + ) def _query(self, conn, request, tool, version): ds = catalog.get_dataset(request.dataset_id) diff --git a/backend/tests/test_acs5_tidy_moe.py b/backend/tests/test_acs5_tidy_moe.py new file mode 100644 index 00000000..507a1299 --- /dev/null +++ b/backend/tests/test_acs5_tidy_moe.py @@ -0,0 +1,116 @@ +""" +Margins of error and their reliability labels through the ACS B-table +cleaners: RAW tidy rows (as written by data_collection/base.py) to the CLEANED +tidy tables and single-variable series. +""" + +import duckdb +import pandas as pd +import pytest + +from data_cleaning import clean_acs5_tidy, clean_acs5_timeseries + +TOWN = (2023, "county_subdivision", "Middlebury town, Addison County, Vermont") +ROWS = { + "demographics": [ + ("Age/Sex", "Median Age", 38.2, 1.9, None, None), + ("Age/Sex", "Population (ACS)", 2000.0, 0.0, 100.0, 0.0), + ("Race", "Asian", 0.0, 20.0, 0.0, 0.0), + ("Race", "Native Hawaiian and Other Pacific Islander", 150.0, 80.0, 0.0, 0.0), + ], + "economic": [ + ("Income", "Median Household Income", 68854.0, 4120.0, None, None), + ("Labor Force", "Unemployment Rate", 120.0, 45.0, 3.1, 1.2), + ], + "education": [("Education", "Bachelor's Degree", 1500.0, 210.0, 30.0, 4.0)], + "housing": [ + ("Housing", "Median Home Value", 361200.0, 8938.0, None, None), + ("Housing", "Rental Vacancy Rate", 25.0, 18.0, 2.6, 1.9), + ], +} + + +@pytest.fixture +def lake(): + con = duckdb.connect() + con.execute("ATTACH ':memory:' AS lake") + con.execute("CREATE SCHEMA lake.RAW") + con.execute("CREATE SCHEMA lake.CLEANED") + con.execute( + """ + CREATE TABLE lake.RAW.vt_town_lines AS + SELECT 5000144350 AS GEOID, + 'Middlebury town, Addison County, Vermont' AS NAME + """ + ) + for table, rows in ROWS.items(): + raw = pd.DataFrame( + [(*TOWN, "50", "001", *row) for row in rows], + columns=["year", "geo_type", "NAME", "state", "county", "Section", + "Variable", "Value", "Value_MOE", "Percent", "Percent_MOE"], + ) # fmt: skip + con.register("raw_df", raw) + con.execute(f"CREATE TABLE lake.RAW.{table} AS SELECT * FROM raw_df") + con.unregister("raw_df") + return con + + +def test_tidy_tables_keep_both_margins(lake): + clean_acs5_tidy.main(lake) + row = lake.execute( + """ + SELECT geoid, value, value_moe, value_reliability, + percent, percent_moe, percent_reliability + FROM lake.CLEANED.acs5_economics_tidy WHERE variable = 'Unemployment Rate' + """ + ).fetchone() + # CV = (45 / 1.645) / 120 = 23% and (1.2 / 1.645) / 3.1 = 24%: medium + assert row == ("5000144350", 120.0, 45.0, "medium", 3.1, 1.2, "medium") + + +def test_share_whose_margin_rounds_to_zero_is_labeled_by_its_count(lake): + # Collection rounds percent_moe to 0.1 point, so a tiny share's real + # margin reads 0.0. Only a controlled count makes that 'high'. + clean_acs5_tidy.main(lake) + rows = lake.execute( + """ + SELECT variable, value_reliability, percent_reliability + FROM lake.CLEANED.acs5_demographics_tidy + WHERE percent_moe = 0 ORDER BY variable + """ + ).fetchall() + assert rows == [ + ("Asian", "low", "low"), # zero count with a nonzero margin + ("Native Hawaiian and Other Pacific Islander", "low", "low"), # CV 32% + ("Population (ACS)", "high", "high"), # controlled: no sampling error + ] + + +def test_snapshot_keeps_margins(lake): + clean_acs5_tidy.main(lake) + moe = lake.execute( + """ + SELECT value_moe FROM lake.CLEANED.acs5_snapshot_indicators_tidy + WHERE variable = 'Median Home Value' + """ + ).fetchone() + assert moe == (8938.0,) + + +def test_single_variable_series_carry_their_margin(lake): + clean_acs5_timeseries.main(lake, ["median_hh_income", "vacancy_rate"]) + income = lake.execute( + """ + SELECT median_household_income, median_household_income_moe, + median_household_income_reliability + FROM lake.CLEANED.acs5Economics_medianHouseholdIncome_timeseries + """ + ).fetchone() + vacancy = lake.execute( + """ + SELECT Variable, percent, percent_moe, percent_reliability + FROM lake.CLEANED.acs5Housing_vacancyRates_timeseries + """ + ).fetchone() + assert income == (68854.0, 4120.0, "high") # CV 4% + assert vacancy == ("Rental Vacancy Rate", 2.6, 1.9, "low") # CV 44% diff --git a/backend/tests/test_acs5_tract.py b/backend/tests/test_acs5_tract.py index e9f3452c..c77d81f5 100644 --- a/backend/tests/test_acs5_tract.py +++ b/backend/tests/test_acs5_tract.py @@ -150,6 +150,18 @@ def test_full_share_margin_is_not_zero(lake): assert row.percent_moe == round(100 * math.sqrt(12**2 + 12**2) / 40, 1) +def test_percent_reliability_follows_the_coefficient_of_variation(lake): + # CV = (MOE / 1.645) / estimate: 1.2 / 1.645 / 15 = 5% -> high + high = _clean(lake, {"B17001_001": (10000, 100), "B17001_002": (1500, 120)}) + # 4.8 / 1.645 / 15 = 19% -> medium; 9.5 / 1.645 / 15 = 39% -> low + medium = _clean(lake, {"B17001_001": (1000, 100), "B17001_002": (150, 50)}) + low = _clean(lake, {"B17001_001": (200, 40), "B17001_002": (30, 20)}) + + assert high.percent_reliability.iloc[0] == "high" + assert medium.percent_reliability.iloc[0] == "medium" + assert low.percent_reliability.iloc[0] == "low" + + def test_suppressed_component_makes_the_indicator_null(lake): df = _clean( lake, {"B17001_001": (1000, 100), "B17001_002": (-666666666, -222222222)} diff --git a/backend/tests/test_clean_acs5.py b/backend/tests/test_clean_acs5.py index 4ebf0160..3fc854e6 100644 --- a/backend/tests/test_clean_acs5.py +++ b/backend/tests/test_clean_acs5.py @@ -28,7 +28,7 @@ _RAW_SCHEMA = """ year BIGINT, NAME VARCHAR, Variable_Code VARCHAR, Source_Label VARCHAR, Category VARCHAR, Subcategory VARCHAR, Variable VARCHAR, Measure VARCHAR, - Value VARCHAR, geo_type VARCHAR, state VARCHAR, county VARCHAR + Value VARCHAR, geo_type VARCHAR, state VARCHAR, county VARCHAR, MOE VARCHAR """ # (year, Variable_Code, Value) for Burlington's colliding DP04 SMOC medians. @@ -66,9 +66,10 @@ def _housing_rows(rows): {year}, '{BURLINGTON}', '{code}', 'Estimate!!SELECTED MONTHLY OWNER COSTS (SMOC)!!Median (dollars)', 'SELECTED MONTHLY OWNER COSTS (SMOC)', 'Median (dollars)', 'Total', - 'Estimate', '{value}', 'county_subdivision', '50', '007' + 'Estimate', '{value}', 'county_subdivision', '50', '007', + {f"'{moe[0]}'" if moe else "NULL"} )""" - for year, code, value in rows + for year, code, value, *moe in rows ) return f""" CREATE TABLE lake.RAW.acs5_housing ({_RAW_SCHEMA}); @@ -188,3 +189,35 @@ def test_dp_value_col(): assert df["value"].dtype.kind == "f" assert df["value"].min() >= 0 assert df["value"].max() < 666666666 + + +def test_dp_margins_are_numeric_and_labeled(): + con = _base_con() + con.execute( + _housing_rows( + [ + (2023, "DP04_0100E", "1800", "90"), # CV 3% -> high + (2023, "DP04_0101E", "40", "-555555555"), # controlled -> 0 + (2023, "DP04_0102E", "0", "-222222222"), # not computable + (2023, "DP04_0103E", "-666666666", "-222222222"), + (2023, "DP04_0104E", "500", "(X)"), + (2023, "DP04_0105E", "100", "60"), # CV 36% -> low + ] + ) + ) + build_dp_combined(con) + + rows = con.execute( + """ + SELECT variable_code, value, moe, reliability + FROM lake.CLEANED.acs5_dp_combined_tidy ORDER BY variable_code + """ + ).fetchall() + assert rows == [ + ("DP04_0100E", 1800.0, 90.0, "high"), + ("DP04_0101E", 40.0, 0.0, "high"), + ("DP04_0102E", 0.0, None, None), + ("DP04_0103E", None, None, None), + ("DP04_0104E", 500.0, None, None), + ("DP04_0105E", 100.0, 60.0, "low"), + ] diff --git a/backend/tests/test_margins.py b/backend/tests/test_margins.py new file mode 100644 index 00000000..297f84af --- /dev/null +++ b/backend/tests/test_margins.py @@ -0,0 +1,24 @@ +"""Tests for data_cleaning/margins.py: reliability labels from 90% margins.""" + +import duckdb +import pytest + +from data_cleaning.margins import reliability_sql + + +@pytest.mark.parametrize( + ("estimate", "moe", "label"), + [ + (100, 24, "high"), # CV 14.6% + (100, 25, "medium"), # CV 15.2% + (100, 49, "medium"), # CV 29.8% + (100, 50, "low"), # CV 30.4% + (40, 0, "high"), # controlled: no sampling error + (0, 12, "low"), # CV undefined; the true value may not be zero + (None, 12, None), + (100, None, None), + ], +) +def test_reliability_follows_the_coefficient_of_variation(estimate, moe, label): + sql = f"SELECT {reliability_sql('e', 'm')} FROM (SELECT ?::DOUBLE e, ?::DOUBLE m)" + assert duckdb.execute(sql, [estimate, moe]).fetchone() == (label,) diff --git a/backend/tests/test_mcp_catalog.py b/backend/tests/test_mcp_catalog.py index 11540b14..d093c8a0 100644 --- a/backend/tests/test_mcp_catalog.py +++ b/backend/tests/test_mcp_catalog.py @@ -139,7 +139,12 @@ def test_variable_search_deduplicates_years_and_roundtrips(warehouse): "section": "Age/Sex", "variable": "Population (ACS)", } - assert rows[0]["value_columns"] == {"value": "people", "percent": "percent"} + assert rows[0]["value_columns"] == { + "value": "people", + "value_moe": "people", + "percent": "percent", + "percent_moe": "percentage points", + } dataset = get_dataset("acs5_demographics") assert decode_variable_id(dataset, rows[0]["variable_id"]) == rows[0]["selectors"] assert ( @@ -153,9 +158,13 @@ def test_variable_search_deduplicates_years_and_roundtrips(warehouse): def test_wide_metric_variable_id_and_units(warehouse): - row = search_variables(warehouse, "acs5_ts_median_age")[0] + # The margin is not a variable of its own: it comes back with the estimate. + [row] = search_variables(warehouse, "acs5_ts_median_age") assert row["selectors"] == {"$column": "median_age"} - assert row["measures"] == [{"name": "median_age", "unit": "years"}] + assert row["measures"] == [ + {"name": "median_age", "unit": "years"}, + {"name": "median_age_moe", "unit": "years"}, + ] assert decode_variable_id( get_dataset("acs5_ts_median_age"), row["variable_id"] ) == {"$column": "median_age"} @@ -309,7 +318,8 @@ def test_missing_dataset_and_unknown_geography_fail_clearly(warehouse): def test_units_preserve_percentages_and_source_specific_estimates(): dataset = get_dataset("acs5_dp") assert variable_units(dataset, {"measure": "Percent Estimate"}) == { - "value": "percent" + "value": "percent", + "moe": "percentage points", } assert "source-defined" in variable_units(dataset, {"measure": "Estimate"})["value"] assert get_dataset("zoning_bylaws").value_columns["f1f_min_lot_size"] == "acres" @@ -318,11 +328,18 @@ def test_units_preserve_percentages_and_source_specific_estimates(): economics = get_dataset("acs5_economics") assert variable_units( economics, {"variable": "Labor Force Participation Rate (16+)"} - ) == {"value": "people", "percent": "percent"} + ) == { + "value": "people", + "value_moe": "people", + "percent": "percent", + "percent_moe": "percentage points", + } housing = get_dataset("acs5_housing") assert variable_units(housing, {"variable": "Rental Vacancy Rate"}) == { "value": "housing units", + "value_moe": "housing units", "percent": "percent", + "percent_moe": "percentage points", } assert ( "unavailable" diff --git a/backend/tests/test_mcp_issue_fixes.py b/backend/tests/test_mcp_issue_fixes.py index 2c973e7d..98fa240f 100644 --- a/backend/tests/test_mcp_issue_fixes.py +++ b/backend/tests/test_mcp_issue_fixes.py @@ -296,3 +296,48 @@ async def run(): ) asyncio.run(run()) + + +@pytest.fixture +def margins_service(tmp_path): + warehouse = tmp_path / "margins.duckdb" + with duckdb.connect(str(warehouse)) as conn: + conn.execute(""" + CREATE TABLE acs5Demographics_medianAge_timeseries( + year VARCHAR, geoid VARCHAR, county_fips VARCHAR, county VARCHAR, + name VARCHAR, geo_type VARCHAR, median_age DOUBLE, + median_age_moe DOUBLE, median_age_reliability VARCHAR + ); + INSERT INTO acs5Demographics_medianAge_timeseries VALUES + ('2023', '50', NULL, NULL, 'Vermont', 'state', 43.2, 0.2, 'high'); + CREATE TABLE acs5_demographics_tidy( + year VARCHAR, geoid VARCHAR, name VARCHAR, geo_type VARCHAR, + county VARCHAR, county_fips VARCHAR, section VARCHAR, + variable VARCHAR, value DOUBLE, value_moe DOUBLE, + value_reliability VARCHAR, percent DOUBLE, percent_moe DOUBLE, + percent_reliability VARCHAR + ); + INSERT INTO acs5_demographics_tidy VALUES + ('2023', '50', 'Vermont', 'state', NULL, NULL, 'Age/Sex', + 'Under 18', 117177, 1220, 'high', 18.2, 0.2, 'high'); + """) + return DataToolService(warehouse) + + +def test_margins_come_back_with_their_estimates(margins_service): + [variable] = margins_service.call( + "search_variables", {"dataset_id": "acs5_ts_median_age"} + )["variables"] + series = margins_service.call( + "query_data", + {"dataset_id": "acs5_ts_median_age", "variable_ids": [variable["variable_id"]]}, + ) + assert series["rows"][0]["median_age"] == 43.2 + assert series["rows"][0]["median_age_moe"] == 0.2 + assert series["units"] == {"median_age": "years", "median_age_moe": "years"} + + tidy = margins_service.call( + "query_data", {"dataset_id": "acs5_demographics", "measures": ["percent"]} + ) + assert "value" not in tidy["columns"] + assert (tidy["rows"][0]["percent"], tidy["rows"][0]["percent_moe"]) == (18.2, 0.2) From a25bfb250df5afe7149ec44f807624ad917258c4 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Mon, 2 Nov 2026 23:01:37 -0500 Subject: [PATCH 4/5] Give the age dependency ratio a margin of error Add SQL helpers for the margin of a sum of estimates (root sum of squares) and of a ratio, and use them for the age dependency ratio, the one derived series the cleaning step computes. --- .../data_cleaning/clean_acs5_timeseries.py | 76 ++++++++++++------- backend/data_cleaning/margins.py | 42 ++++++++++ backend/tests/test_acs5_tidy_moe.py | 28 +++++++ backend/tests/test_margins.py | 36 ++++++++- 4 files changed, 152 insertions(+), 30 deletions(-) diff --git a/backend/data_cleaning/clean_acs5_timeseries.py b/backend/data_cleaning/clean_acs5_timeseries.py index 8be9674a..ca59a7cb 100644 --- a/backend/data_cleaning/clean_acs5_timeseries.py +++ b/backend/data_cleaning/clean_acs5_timeseries.py @@ -31,7 +31,7 @@ build_geo_lookups, write_table, ) -from data_cleaning.margins import reliability_sql +from data_cleaning.margins import ratio_moe_sql, reliability_sql, sum_moe_sql # Raw identifier columns every base query must expose (see `acs_geo_sql`) KEYS = "year, NAME, geo_type, state, county" @@ -184,45 +184,65 @@ def population_change_sql() -> str: def age_dependency_ratio_sql() -> str: """ - SQL calculating age dependency ratio: (`Under 18` + `65 and over`) / (`18 to 64`) * 100. + SQL calculating age dependency ratio: (`Under 18` + `65 and over`) / (`18 to 64`) * 100, + with its 90% margin of error and reliability label. + + The dependents and the working-age population don't overlap, so the margin + takes the ratio form, from the two sums' margins. Returns: str: A SQL query that creates the age dependency ratio table """ - ages = [ - "Under 18", - "18 to 24", - "25 to 34", - "35 to 44", - "45 to 54", - "55 to 64", - "65 to 74", - "75 Plus", - ] + dependents = ["Under 18", "65 to 74", "75 Plus"] + working = ["18 to 24", "25 to 34", "35 to 44", "45 to 54", "55 to 64"] + ages = dependents + working wide = ",\n".join( - f"MAX(CASE WHEN Variable = '{a}' THEN v END) AS \"{a}\"" for a in ages + f"MAX(CASE WHEN Variable = '{a}' THEN {col} END) AS \"{a}{suffix}\"" + for a in ages + for col, suffix in [("v", ""), ("m", " MOE")] ) - working = " + ".join(f'"{a}"' for a in ages[1:6]) + + def total(group: list[str]) -> str: + return " + ".join(f'"{a}"' for a in group) + + def total_moe(group: list[str]) -> str: + return sum_moe_sql([(f'"{a}"', f'"{a} MOE"') for a in group]) return f"""--sql SELECT {KEYS}, - ROUND( - CASE - WHEN ({working}) > 0 - THEN ("Under 18" + "65 to 74" + "75 Plus") - / ({working}) * 100 - END, - 1 - ) AS age_dependency_ratio + age_dependency_ratio, + age_dependency_ratio_moe, + {reliability_sql("age_dependency_ratio", "age_dependency_ratio_moe")} + AS age_dependency_ratio_reliability FROM ( - SELECT {KEYS}, {wide} + SELECT + {KEYS}, + ROUND(100 * ratio, 1) AS age_dependency_ratio, + ROUND( + 100 * {ratio_moe_sql("ratio", "num_moe", "den", "den_moe")}, 1 + ) AS age_dependency_ratio_moe FROM ( - SELECT {KEYS}, Variable, {numeric("Value")} AS v - FROM lake.RAW.demographics - WHERE Variable IN ({in_list(ages)}) + SELECT + {KEYS}, + ({total(dependents)}) / NULLIF({total(working)}, 0) AS ratio, + {total_moe(dependents)} AS num_moe, + NULLIF({total(working)}, 0) AS den, + {total_moe(working)} AS den_moe + FROM ( + SELECT {KEYS}, {wide} + FROM ( + SELECT + {KEYS}, + Variable, + {numeric("Value")} AS v, + {numeric("Value_MOE")} AS m + FROM lake.RAW.demographics + WHERE Variable IN ({in_list(ages)}) + ) + GROUP BY {KEYS} + ) ) - GROUP BY {KEYS} ) """ @@ -401,7 +421,7 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: "age_dependency_ratio": TimeSeries( table="acs5Demographics_ageDependencyRatio_timeseries", base=age_dependency_ratio_sql(), - columns=("age_dependency_ratio",), + columns=with_moe("age_dependency_ratio"), ), # Health Insurance Coverage (Public, Private, None) "health_insurance": TimeSeries( diff --git a/backend/data_cleaning/margins.py b/backend/data_cleaning/margins.py index 724cf648..d7eca3c0 100644 --- a/backend/data_cleaning/margins.py +++ b/backend/data_cleaning/margins.py @@ -39,3 +39,45 @@ def reliability_sql(estimate: str, moe: str) -> str: WHEN {cv} <= {LOW_CV} THEN 'medium' ELSE 'low' END""" + + +def sum_moe_sql(terms: list[tuple[str, str]]) -> str: + """ + 90% margin of a sum of estimates: the root sum of their squared margins + (ACS handbook). As in data_collection.base.sum_moe, only the largest + margin among zero estimates counts, since summing many modeled zero + margins overstates the error. Null when any margin is null. + + Args: + terms: (estimate, margin of error) SQL expression pairs + + Returns: + str: A SQL expression + """ + nonzero = " + ".join( + f"CASE WHEN {e} = 0 THEN 0 ELSE ({m}) ^ 2 END" for e, m in terms + ) + zero = ", ".join(f"CASE WHEN {e} = 0 THEN ({m}) ^ 2 ELSE 0 END" for e, m in terms) + missing = " OR ".join(f"{m} IS NULL" for _, m in terms) + return f"""CASE + WHEN {missing} THEN NULL + ELSE SQRT({nonzero} + GREATEST({zero})) + END""" + + +def ratio_moe_sql(ratio: str, num_moe: str, den: str, den_moe: str) -> str: + """ + 90% margin of the ratio num / den, where num is not a subset of den: + sqrt(num_moe^2 + ratio^2 * den_moe^2) / den (ACS handbook). A subset + takes the proportion form instead (data_collection.base.proportion_moe). + + Args: + ratio: SQL expression for num / den + num_moe: SQL expression for the numerator's margin of error + den: SQL expression for the denominator + den_moe: SQL expression for the denominator's margin of error + + Returns: + str: A SQL expression + """ + return f"SQRT(({num_moe}) ^ 2 + ({ratio}) ^ 2 * ({den_moe}) ^ 2) / ({den})" diff --git a/backend/tests/test_acs5_tidy_moe.py b/backend/tests/test_acs5_tidy_moe.py index 507a1299..483fe2b0 100644 --- a/backend/tests/test_acs5_tidy_moe.py +++ b/backend/tests/test_acs5_tidy_moe.py @@ -4,6 +4,8 @@ tidy tables and single-variable series. """ +import math + import duckdb import pandas as pd import pytest @@ -15,6 +17,16 @@ "demographics": [ ("Age/Sex", "Median Age", 38.2, 1.9, None, None), ("Age/Sex", "Population (ACS)", 2000.0, 0.0, 100.0, 0.0), + # Age bands for the age dependency ratio: 500 dependents to 1,000 of + # working age. + ("Age/Sex", "Under 18", 300.0, 40.0, 15.0, 2.0), + ("Age/Sex", "65 to 74", 200.0, 30.0, 10.0, 1.5), + ("Age/Sex", "75 Plus", 0.0, 15.0, 0.0, 0.8), + ("Age/Sex", "18 to 24", 100.0, 20.0, 5.0, 1.0), + ("Age/Sex", "25 to 34", 200.0, 30.0, 10.0, 1.5), + ("Age/Sex", "35 to 44", 200.0, 30.0, 10.0, 1.5), + ("Age/Sex", "45 to 54", 250.0, 40.0, 12.5, 2.0), + ("Age/Sex", "55 to 64", 250.0, 40.0, 12.5, 2.0), ("Race", "Asian", 0.0, 20.0, 0.0, 0.0), ("Race", "Native Hawaiian and Other Pacific Islander", 150.0, 80.0, 0.0, 0.0), ], @@ -114,3 +126,19 @@ def test_single_variable_series_carry_their_margin(lake): ).fetchone() assert income == (68854.0, 4120.0, "high") # CV 4% assert vacancy == ("Rental Vacancy Rate", 2.6, 1.9, "low") # CV 44% + + +def test_age_dependency_ratio_carries_a_ratio_margin(lake): + clean_acs5_timeseries.main(lake, ["age_dependency_ratio"]) + row = lake.execute( + """ + SELECT age_dependency_ratio, age_dependency_ratio_moe, + age_dependency_ratio_reliability + FROM lake.CLEANED.acs5Demographics_ageDependencyRatio_timeseries + """ + ).fetchone() + num_moe = math.sqrt(40**2 + 30**2 + 15**2) + den_moe = math.sqrt(20**2 + 30**2 + 30**2 + 40**2 + 40**2) + # sqrt(num_moe^2 + R^2 * den_moe^2) / den, with R = 500 / 1000 + moe = round(100 * math.sqrt(num_moe**2 + 0.5**2 * den_moe**2) / 1000, 1) + assert row == (50.0, moe, "high") # CV 8% diff --git a/backend/tests/test_margins.py b/backend/tests/test_margins.py index 297f84af..732aad47 100644 --- a/backend/tests/test_margins.py +++ b/backend/tests/test_margins.py @@ -1,9 +1,11 @@ -"""Tests for data_cleaning/margins.py: reliability labels from 90% margins.""" +"""Tests for data_cleaning/margins.py: reliability labels and combined 90% margins.""" + +import math import duckdb import pytest -from data_cleaning.margins import reliability_sql +from data_cleaning.margins import ratio_moe_sql, reliability_sql, sum_moe_sql @pytest.mark.parametrize( @@ -22,3 +24,33 @@ def test_reliability_follows_the_coefficient_of_variation(estimate, moe, label): sql = f"SELECT {reliability_sql('e', 'm')} FROM (SELECT ?::DOUBLE e, ?::DOUBLE m)" assert duckdb.execute(sql, [estimate, moe]).fetchone() == (label,) + + +def summed_margin(*terms): + pairs = [(f"e{i}", f"m{i}") for i in range(len(terms))] + cols = ", ".join(f"?::DOUBLE {e}, ?::DOUBLE {m}" for e, m in pairs) + sql = f"SELECT {sum_moe_sql(pairs)} FROM (SELECT {cols})" + return duckdb.execute(sql, [x for t in terms for x in t]).fetchone()[0] + + +def test_summed_margin_is_the_root_sum_of_squares(): + assert summed_margin((100, 30), (50, 40)) == pytest.approx(50.0) + + +def test_summed_margin_counts_only_the_largest_zero_estimate_margin(): + expected = math.sqrt(30**2 + 25**2) + assert summed_margin((100, 30), (0, 20), (0, 25)) == pytest.approx(expected) + + +def test_summed_margin_of_only_zero_estimates_is_their_largest(): + assert summed_margin((0, 20), (0, 25)) == pytest.approx(25.0) + + +def test_summed_margin_is_null_when_any_margin_is(): + assert summed_margin((100, 30), (0, None)) is None + + +def test_ratio_margin(): + sql = f"SELECT {ratio_moe_sql('0.5', '40', '1000', '60')}" + # sqrt(40^2 + 0.5^2 * 60^2) / 1000 + assert duckdb.execute(sql).fetchone()[0] == pytest.approx(0.05) From 889e63711248c6eb014e23b54991d72a38a17548 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Mon, 2 Nov 2026 23:17:20 -0500 Subject: [PATCH 5/5] Carry margins through the cost burden, health insurance and earnings series The DP03 series pass their published margins through. Housing cost burden sums bracket and tenure margins by root sum of squares (an aggregate form of sum_moe_sql) and gives Percent the proportion form. --- backend/data_cleaning/clean_acs5.py | 30 +++-- .../data_cleaning/clean_acs5_timeseries.py | 105 ++++++++++++++---- backend/data_cleaning/margins.py | 45 ++++++++ backend/tests/test_acs5_tidy_moe.py | 33 ++++++ backend/tests/test_housing_cost_burden.py | 78 +++++++++++-- 5 files changed, 253 insertions(+), 38 deletions(-) diff --git a/backend/data_cleaning/clean_acs5.py b/backend/data_cleaning/clean_acs5.py index d4218d3f..8baa1efb 100644 --- a/backend/data_cleaning/clean_acs5.py +++ b/backend/data_cleaning/clean_acs5.py @@ -37,6 +37,26 @@ } +def published_moe_sql(moe: str) -> str: + """ + A raw DP table's published margin of error as a number. -555555555 means + the estimate is controlled, so its margin is zero; other annotation codes + (negative, "(X)") are missing. + + Args: + moe: SQL expression for the raw MOE column + + Returns: + str: A SQL CASE expression + """ + return f"""CASE + WHEN TRY_CAST({moe} AS DOUBLE) = -555555555 THEN 0 + WHEN TRY_CAST({moe} AS DOUBLE) IN ({inline_sql_unavailable(DP_UNAVAILABLE)}) + THEN NULL + WHEN TRY_CAST({moe} AS DOUBLE) >= 0 THEN TRY_CAST({moe} AS DOUBLE) + END""" + + def _dp_select_sql(dp: str, raw_table_name: str) -> str: """ Build the SELECT query for a single census DP table, joined against `vt_town_lines` @@ -85,14 +105,8 @@ def _dp_select_sql(dp: str, raw_table_name: str) -> str: ELSE CAST(n.Value AS DOUBLE) END AS "value", -- The published margin of error (M for an estimate, PM for a - -- percent). -555555555: the estimate is controlled, so its - -- margin is zero; other codes (negative, "(X)") are missing. - CASE - WHEN TRY_CAST(n.MOE AS DOUBLE) = -555555555 THEN 0 - WHEN TRY_CAST(n.MOE AS DOUBLE) IN ({inline_sql_unavailable(DP_UNAVAILABLE)}) - THEN NULL - WHEN TRY_CAST(n.MOE AS DOUBLE) >= 0 THEN TRY_CAST(n.MOE AS DOUBLE) - END AS moe, + -- percent). + {published_moe_sql("n.MOE")} AS moe, '{dp}' AS "table", n.geo_type_norm AS geo_type, CASE diff --git a/backend/data_cleaning/clean_acs5_timeseries.py b/backend/data_cleaning/clean_acs5_timeseries.py index ca59a7cb..fb7288d0 100644 --- a/backend/data_cleaning/clean_acs5_timeseries.py +++ b/backend/data_cleaning/clean_acs5_timeseries.py @@ -25,13 +25,20 @@ import duckdb +from data_cleaning.clean_acs5 import published_moe_sql from data_cleaning.geo_lookup import ( UNAVAILABLE, acs_geo_sql, build_geo_lookups, write_table, ) -from data_cleaning.margins import ratio_moe_sql, reliability_sql, sum_moe_sql +from data_cleaning.margins import ( + proportion_moe_sql, + ratio_moe_sql, + reliability_sql, + sum_moe_agg_sql, + sum_moe_sql, +) # Raw identifier columns every base query must expose (see `acs_geo_sql`) KEYS = "year, NAME, geo_type, state, county" @@ -105,6 +112,21 @@ def with_moe(column: str) -> tuple[str, str, str]: return (column, f"{column}_moe", f"{column}_reliability") +def published_value_moe() -> str: + """ + SELECT columns for a raw DP row's published margin of `Value`, as + `Value_MOE` and `Value_Reliability`. + """ + moe = published_moe_sql("MOE") + return ( + f'{moe} AS "Value_MOE", ' + f'{reliability_sql(numeric("Value"), moe)} AS "Value_Reliability"' + ) + + +VALUE_WITH_MOE = ("Value", "Value_MOE", "Value_Reliability") + + def single_variable( source: str, variables: list[str], @@ -265,6 +287,9 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: households" is NULL unless all three tenures are complete. Percent is NULL when the universe is 0 (no households of that tenure). + Summed counts take the root sum of their published margins, and Percent + the proportion form, since burdened households are a subset of Total. + Starts in 2013: earlier DP04 layouts differ, and 2010-2012 do not separate owners with and without a mortgage. @@ -273,12 +298,15 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: Returns: str: SELECT of KEYS, Variable, Value (burdened households), - Total (universe), and Percent. + Total (universe), and Percent, each with its 90% margin of error + (`_MOE`) and reliability label (`_Reliability`). """ tenure_cases = "\n".join( f"WHEN '{subcategory}' THEN '{label}'" for subcategory, label in BURDEN_TENURES.items() ) + brackets = "Variable <> 'Total'" + percent_moe = proportion_moe_sql("p", '"Value_MOE"', "Total", "Total_MOE") return f"""--sql WITH src AS ( SELECT @@ -288,7 +316,8 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: CASE WHEN TRY_CAST(Value AS DOUBLE) >= 0 THEN TRY_CAST(Value AS DOUBLE) - END AS v + END AS v, + {published_moe_sql("MOE")} AS m FROM {source} WHERE year >= {BURDEN_FIRST_YEAR} AND Measure = 'Estimate' @@ -300,10 +329,12 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: SELECT {KEYS}, tenure, - SUM(v) FILTER (WHERE Variable <> 'Total') AS burdened, - COUNT(v) FILTER (WHERE Variable <> 'Total') AS n_brackets, - COUNT(*) FILTER (WHERE Variable <> 'Total') AS n_bracket_rows, + SUM(v) FILTER (WHERE {brackets}) AS burdened, + {sum_moe_agg_sql("v", "m", brackets)} AS burdened_moe, + COUNT(v) FILTER (WHERE {brackets}) AS n_brackets, + COUNT(*) FILTER (WHERE {brackets}) AS n_bracket_rows, MAX(v) FILTER (WHERE Variable = 'Total') AS total, + MAX(m) FILTER (WHERE Variable = 'Total') AS total_moe, COUNT(v) FILTER (WHERE Variable = 'Total') AS n_totals, COUNT(*) FILTER (WHERE Variable = 'Total') AS n_total_rows FROM src @@ -317,7 +348,13 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: CASE WHEN n_brackets = 2 AND n_bracket_rows = 2 THEN burdened END AS "Value", - CASE WHEN n_totals = 1 AND n_total_rows = 1 THEN total END AS Total + CASE + WHEN n_brackets = 2 AND n_bracket_rows = 2 THEN burdened_moe + END AS "Value_MOE", + CASE WHEN n_totals = 1 AND n_total_rows = 1 THEN total END AS Total, + CASE + WHEN n_totals = 1 AND n_total_rows = 1 THEN total_moe + END AS Total_MOE FROM by_tenure ), @@ -326,26 +363,43 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: {KEYS}, '{BURDEN_ALL_HOUSEHOLDS}' AS Variable, CASE WHEN COUNT("Value") = 3 THEN SUM("Value") END AS "Value", - CASE WHEN COUNT(Total) = 3 THEN SUM(Total) END AS Total + CASE + WHEN COUNT("Value") = 3 + THEN {sum_moe_agg_sql('"Value"', '"Value_MOE"')} + END AS "Value_MOE", + CASE WHEN COUNT(Total) = 3 THEN SUM(Total) END AS Total, + CASE + WHEN COUNT(Total) = 3 THEN {sum_moe_agg_sql("Total", "Total_MOE")} + END AS Total_MOE FROM tenure_rows GROUP BY {KEYS} ), combined AS ( - SELECT * FROM tenure_rows - UNION ALL - SELECT * FROM all_rows + SELECT + *, + CASE WHEN "Value" IS NOT NULL AND Total > 0 THEN "Value" / Total END AS p + FROM ( + SELECT * FROM tenure_rows + UNION ALL + SELECT * FROM all_rows + ) ) + -- Reliability comes from the unrounded figures: a small share's + -- margin can round to 0.0, which would read as a controlled estimate. SELECT {KEYS}, Variable, "Value", + ROUND("Value_MOE") AS "Value_MOE", + {reliability_sql('"Value"', '"Value_MOE"')} AS "Value_Reliability", Total, - CASE - WHEN "Value" IS NOT NULL AND Total > 0 - THEN ROUND(100 * "Value" / Total, 1) - END AS Percent + ROUND(Total_MOE) AS Total_MOE, + {reliability_sql("Total", "Total_MOE")} AS Total_Reliability, + ROUND(100 * p, 1) AS Percent, + ROUND(100 * {percent_moe}, 1) AS Percent_MOE, + {reliability_sql("p", percent_moe)} AS Percent_Reliability FROM combined """ @@ -427,7 +481,8 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: "health_insurance": TimeSeries( table="acs5Economics_healthInsurance_timeseries", base=f"""--sql - SELECT {KEYS}, Variable, {numeric("Value")} AS "Value" + SELECT {KEYS}, Variable, {numeric("Value")} AS "Value", + {published_value_moe()} FROM lake.RAW.acs5_economic WHERE Category LIKE '%INSURANCE%' AND Subcategory = 'Civilian noninstitutionalized population' @@ -438,14 +493,15 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: ) AND Measure = 'Estimate' """, - columns=("Variable", "Value"), + columns=("Variable", *VALUE_WITH_MOE), with_county=False, ), # Median Earnings (Male, Female, All Workers) "median_earnings": TimeSeries( table="acs5Economics_medianEarnings_timeseries", base=f"""--sql - SELECT {KEYS}, Subcategory AS Variable, {numeric("Value")} AS "Value" + SELECT {KEYS}, Subcategory AS Variable, {numeric("Value")} AS "Value", + {published_value_moe()} FROM lake.RAW.acs5_economic WHERE Category LIKE '%INCOME AND BENEFITS%' AND Subcategory IN ( @@ -456,14 +512,23 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: AND Variable = 'Total' AND Measure = 'Estimate' """, - columns=("Variable", "Value"), + columns=("Variable", *VALUE_WITH_MOE), with_county=False, ), # Housing Cost Burden (30%+ of income on housing), by tenure "housing_cost_burden": TimeSeries( table="acs5Housing_incomeBurden_timeseries", base=housing_cost_burden_sql(), - columns=("Variable", "Value", "Total", "Percent"), + columns=( + "Variable", + *VALUE_WITH_MOE, + "Total", + "Total_MOE", + "Total_Reliability", + "Percent", + "Percent_MOE", + "Percent_Reliability", + ), with_county=False, ), } diff --git a/backend/data_cleaning/margins.py b/backend/data_cleaning/margins.py index d7eca3c0..e4c6c52e 100644 --- a/backend/data_cleaning/margins.py +++ b/backend/data_cleaning/margins.py @@ -81,3 +81,48 @@ def ratio_moe_sql(ratio: str, num_moe: str, den: str, den_moe: str) -> str: str: A SQL expression """ return f"SQRT(({num_moe}) ^ 2 + ({ratio}) ^ 2 * ({den_moe}) ^ 2) / ({den})" + + +def sum_moe_agg_sql(estimate: str, moe: str, where: str = "TRUE") -> str: + """ + :func:`sum_moe_sql` as an aggregate: the margin of the SUM of `estimate` + over a group's rows (those matching `where`). Null when any of their + margins is null. + + Args: + estimate: SQL expression for each row's estimate + moe: SQL expression for its margin of error + where: SQL condition picking the rows summed + + Returns: + str: A SQL aggregate expression + """ + return f"""CASE + WHEN COUNT(*) FILTER (WHERE ({where}) AND {moe} IS NULL) > 0 THEN NULL + ELSE SQRT( + COALESCE(SUM(({moe}) ^ 2) FILTER (WHERE ({where}) AND {estimate} <> 0), 0) + + COALESCE(MAX(({moe}) ^ 2) FILTER (WHERE ({where}) AND {estimate} = 0), 0) + ) + END""" + + +def proportion_moe_sql(p: str, num_moe: str, den: str, den_moe: str) -> str: + """ + 90% margin of the proportion p = num / den, where num is a subset of den, + as data_collection.base.proportion_moe: sqrt(num_moe^2 - p^2 * den_moe^2) + / den, or the ratio form (+) when the term under the root isn't positive. + + Args: + p: SQL expression for num / den + num_moe: SQL expression for the numerator's margin of error + den: SQL expression for the denominator + den_moe: SQL expression for the denominator's margin of error + + Returns: + str: A SQL expression + """ + under = f"({num_moe}) ^ 2 - ({p}) ^ 2 * ({den_moe}) ^ 2" + return f"""CASE + WHEN {under} > 0 THEN SQRT({under}) / ({den}) + ELSE {ratio_moe_sql(p, num_moe, den, den_moe)} + END""" diff --git a/backend/tests/test_acs5_tidy_moe.py b/backend/tests/test_acs5_tidy_moe.py index 483fe2b0..81848904 100644 --- a/backend/tests/test_acs5_tidy_moe.py +++ b/backend/tests/test_acs5_tidy_moe.py @@ -142,3 +142,36 @@ def test_age_dependency_ratio_carries_a_ratio_margin(lake): # sqrt(num_moe^2 + R^2 * den_moe^2) / den, with R = 500 / 1000 moe = round(100 * math.sqrt(num_moe**2 + 0.5**2 * den_moe**2) / 1000, 1) assert row == (50.0, moe, "high") # CV 8% + + +def test_dp_series_carry_their_published_margins(lake): + raw = pd.DataFrame( + [ + (*TOWN, "50", "001", "SELECTED ECONOMIC CHARACTERISTICS: HEALTH INSURANCE", + "Civilian noninstitutionalized population", "No health insurance coverage", + "Estimate", "300", "45"), + (*TOWN, "50", "001", "INCOME AND BENEFITS", + "Median earnings for workers (dollars)", "Total", "Estimate", "41250", + "-555555555"), + ], + columns=["year", "geo_type", "NAME", "state", "county", "Category", + "Subcategory", "Variable", "Measure", "Value", "MOE"], + ) # fmt: skip + lake.register("raw_df", raw) + lake.execute("CREATE TABLE lake.RAW.acs5_economic AS SELECT * FROM raw_df") + clean_acs5_timeseries.main(lake, ["health_insurance", "median_earnings"]) + + uninsured = lake.execute( + """ + SELECT "Value", "Value_MOE", "Value_Reliability" + FROM lake.CLEANED.acs5Economics_healthInsurance_timeseries + """ + ).fetchone() + earnings = lake.execute( + """ + SELECT "Value", "Value_MOE", "Value_Reliability" + FROM lake.CLEANED.acs5Economics_medianEarnings_timeseries + """ + ).fetchone() + assert uninsured == (300.0, 45.0, "high") # CV 9% + assert earnings == (41250.0, 0.0, "high") # controlled diff --git a/backend/tests/test_housing_cost_burden.py b/backend/tests/test_housing_cost_burden.py index 5847c0dd..60f4b355 100644 --- a/backend/tests/test_housing_cost_burden.py +++ b/backend/tests/test_housing_cost_burden.py @@ -4,6 +4,8 @@ query, before the shared geography standardization is applied. """ +import math + import duckdb import pandas as pd import pytest @@ -21,15 +23,21 @@ GEO = ("county_subdivision", "50", "007") -def rows(year, name, subcategory, low, high, total, measure="Estimate"): - """DP04 rows for one tenure: the two burden brackets and the total.""" +def rows( + year, name, subcategory, low, high, total, measure="Estimate", moes=("4", "6", "9") +): + """DP04 rows for one tenure: the two burden brackets and the total, with + their published margins.""" return [ - (year, name, *GEO, subcategory, variable, measure, value) - for variable, value in [ - ("30.0 to 34.9 percent", low), - ("35.0 percent or more", high), - ("Total", total), - ] + (year, name, *GEO, subcategory, variable, measure, value, moe) + for (variable, value), moe in zip( + [ + ("30.0 to 34.9 percent", low), + ("35.0 percent or more", high), + ("Total", total), + ], + moes, + ) ] @@ -48,11 +56,11 @@ def run(raw_rows): CREATE TABLE raw ( year BIGINT, NAME VARCHAR, geo_type VARCHAR, state VARCHAR, county VARCHAR, Subcategory VARCHAR, Variable VARCHAR, - Measure VARCHAR, Value VARCHAR + Measure VARCHAR, Value VARCHAR, MOE VARCHAR ) """ ) - con.executemany("INSERT INTO raw VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", raw_rows) + con.executemany("INSERT INTO raw VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", raw_rows) df = con.execute(housing_cost_burden_sql(source="raw")).df() return df.set_index(["year", "NAME", "Variable"]) @@ -86,6 +94,7 @@ def test_percent_estimate_years_are_included(): "35.0 percent or more", "Percent Estimate", "30.0", + "5.0", ), ] df = run(raw) @@ -105,6 +114,7 @@ def test_percent_rows_and_early_years_are_ignored(): "35.0 percent or more", "Percent", "99.9", + "5.0", ), ] + complete_place(2012, "A") @@ -188,3 +198,51 @@ def test_rates_are_rounded_to_one_decimal(): df = run(raw) assert value(df, 2020, "A", "Renters") == 33.3 + + +def test_margins_by_tenure_and_all_households(): + raw = ( + rows(2020, "A", RENT, "10", "30", "100", moes=("6", "8", "12")) + + rows(2020, "A", MORTGAGE, "5", "15", "80", moes=("4", "7", "-555555555")) + + rows(2020, "A", NO_MORTGAGE, "2", "3", "50", moes=("3", "3", "9")) + ) + df = run(raw) + + # 40 renters of 100: the brackets' margins summed, then the proportion + # form, sqrt(10^2 - 0.4^2 * 12^2) / 100. + assert value(df, 2020, "A", "Renters", "Value_MOE") == 10 + assert value(df, 2020, "A", "Renters", "Percent_MOE") == round( + 100 * math.sqrt(10**2 - 0.4**2 * 12**2) / 100, 1 + ) + # A controlled total has no margin. + assert value(df, 2020, "A", "Owners with a mortgage", "Total_MOE") == 0 + assert value(df, 2020, "A", "Owners with a mortgage", "Total_Reliability") == ( + "high" + ) + + value_moe = math.sqrt(6**2 + 8**2 + 4**2 + 7**2 + 3**2 + 3**2) + total_moe = math.sqrt(12**2 + 0**2 + 9**2) + assert value(df, 2020, "A", ALL_HOUSEHOLDS, "Value_MOE") == round(value_moe) + assert value(df, 2020, "A", ALL_HOUSEHOLDS, "Total_MOE") == round(total_moe) + p = 65 / 230 + assert value(df, 2020, "A", ALL_HOUSEHOLDS, "Percent_MOE") == round( + 100 * math.sqrt(value_moe**2 - p**2 * total_moe**2) / 230, 1 + ) + + +def test_zero_bracket_margins_count_once(): + # Both brackets are zero: only the larger modeled margin counts. + raw = rows(2020, "A", RENT, "0", "0", "100", moes=("12", "16", "20")) + df = run(raw) + + assert value(df, 2020, "A", "Renters", "Value_MOE") == 16 + assert value(df, 2020, "A", "Renters", "Percent_Reliability") == "low" + + +def test_missing_margin_leaves_the_margin_missing(): + raw = rows(2020, "A", RENT, "10", "30", "100", moes=("-222222222", "8", "12")) + df = run(raw) + + assert value(df, 2020, "A", "Renters") == 40.0 + assert value(df, 2020, "A", "Renters", "Value_MOE") is None + assert value(df, 2020, "A", "Renters", "Percent_MOE") is None