diff --git a/backend/Data/warehouse.schema.json b/backend/Data/warehouse.schema.json index fb7385de..86ce1485 100644 --- a/backend/Data/warehouse.schema.json +++ b/backend/Data/warehouse.schema.json @@ -382,6 +382,8 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "median_age": "DOUBLE", + "median_age_moe": "DOUBLE", + "median_age_reliability": "VARCHAR", "name": "VARCHAR", "year": "VARCHAR" }, @@ -419,6 +421,8 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "median_household_income": "DOUBLE", + "median_household_income_moe": "DOUBLE", + "median_household_income_reliability": "VARCHAR", "name": "VARCHAR", "year": "VARCHAR" }, @@ -429,6 +433,8 @@ "geoid": "VARCHAR", "name": "VARCHAR", "per_capita_income": "DOUBLE", + "per_capita_income_moe": "DOUBLE", + "per_capita_income_reliability": "VARCHAR", "year": "VARCHAR" }, "acs5Housing_housingUnits_timeseries": { @@ -438,6 +444,8 @@ "geoid": "VARCHAR", "name": "VARCHAR", "total_housing_units": "DOUBLE", + "total_housing_units_moe": "DOUBLE", + "total_housing_units_reliability": "VARCHAR", "year": "VARCHAR" }, "acs5Housing_incomeBurden_timeseries": { @@ -457,6 +465,8 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "median_home_value": "DOUBLE", + "median_home_value_moe": "DOUBLE", + "median_home_value_reliability": "VARCHAR", "name": "VARCHAR", "year": "VARCHAR" }, @@ -468,6 +478,8 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "year": "VARCHAR" }, "acs5_demographics_tidy": { @@ -477,8 +489,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "DOUBLE", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -488,7 +504,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", @@ -504,8 +522,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "DOUBLE", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -516,8 +538,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "BIGINT", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -528,8 +554,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "DOUBLE", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -540,8 +570,12 @@ "geoid": "VARCHAR", "name": "VARCHAR", "percent": "DOUBLE", + "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "value": "DOUBLE", + "value_moe": "DOUBLE", + "value_reliability": "VARCHAR", "variable": "VARCHAR", "year": "VARCHAR" }, @@ -553,6 +587,7 @@ "name": "VARCHAR", "percent": "DOUBLE", "percent_moe": "DOUBLE", + "percent_reliability": "VARCHAR", "section": "VARCHAR", "universe": "HUGEINT", "value": "HUGEINT", @@ -655,7 +690,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", @@ -670,7 +707,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", @@ -685,7 +724,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", @@ -700,7 +741,9 @@ "geo_type": "VARCHAR", "geoid": "VARCHAR", "measure": "VARCHAR", + "moe": "DOUBLE", "name": "VARCHAR", + "reliability": "VARCHAR", "source_label": "VARCHAR", "subcategory": "VARCHAR", "table": "VARCHAR", diff --git a/backend/data_cleaning/clean_acs5.py b/backend/data_cleaning/clean_acs5.py index 058028a1..8baa1efb 100644 --- a/backend/data_cleaning/clean_acs5.py +++ b/backend/data_cleaning/clean_acs5.py @@ -1,6 +1,7 @@ import duckdb from data_cleaning.geo_lookup import inline_sql_unavailable +from data_cleaning.margins import reliability_sql DP_TABLES = { "DP02": ("acs5_social", "dp_social"), @@ -36,6 +37,26 @@ } +def published_moe_sql(moe: str) -> str: + """ + A raw DP table's published margin of error as a number. -555555555 means + the estimate is controlled, so its margin is zero; other annotation codes + (negative, "(X)") are missing. + + Args: + moe: SQL expression for the raw MOE column + + Returns: + str: A SQL CASE expression + """ + return f"""CASE + WHEN TRY_CAST({moe} AS DOUBLE) = -555555555 THEN 0 + WHEN TRY_CAST({moe} AS DOUBLE) IN ({inline_sql_unavailable(DP_UNAVAILABLE)}) + THEN NULL + WHEN TRY_CAST({moe} AS DOUBLE) >= 0 THEN TRY_CAST({moe} AS DOUBLE) + END""" + + def _dp_select_sql(dp: str, raw_table_name: str) -> str: """ Build the SELECT query for a single census DP table, joined against `vt_town_lines` @@ -63,6 +84,8 @@ def _dp_select_sql(dp: str, raw_table_name: str) -> str: g.variable, g.measure, g.value, + g.moe, + {reliability_sql("g.value", "g.moe")} AS reliability, g."table", g.geo_type, g.geoid @@ -81,6 +104,9 @@ def _dp_select_sql(dp: str, raw_table_name: str) -> str: WHEN CAST(n.Value AS DOUBLE) IN ({inline_sql_unavailable(DP_UNAVAILABLE)}) THEN NULL ELSE CAST(n.Value AS DOUBLE) END AS "value", + -- The published margin of error (M for an estimate, PM for a + -- percent). + {published_moe_sql("n.MOE")} AS moe, '{dp}' AS "table", n.geo_type_norm AS geo_type, CASE diff --git a/backend/data_cleaning/clean_acs5_tidy.py b/backend/data_cleaning/clean_acs5_tidy.py index 67963df0..2f324707 100644 --- a/backend/data_cleaning/clean_acs5_tidy.py +++ b/backend/data_cleaning/clean_acs5_tidy.py @@ -9,7 +9,10 @@ `snapshot` table that combines a few key indicators from them. Each output is in tidy format: year, geoid, name, geo_type, county, county_fips, - section, variable, value, percent. + section, variable, value, value_moe, value_reliability, percent, + percent_moe, percent_reliability. The *_moe columns are 90% margins of + error (percent_moe in percentage points), computed at collection (see + data_collection/base.py); *_reliability labels them (see margins.py). **Run with**: python run_data_cleaning.py clean_acs5_tidy """ @@ -17,6 +20,7 @@ import duckdb from data_cleaning.geo_lookup import acs_geo_sql, build_geo_lookups, write_table +from data_cleaning.margins import reliability_sql # RAW table name -> CLEANED table name TIDY_TABLES = { @@ -57,7 +61,20 @@ def tidy_sql(base: str) -> str: section AS section, variable AS variable, value AS value, - percent AS percent + value_moe AS value_moe, + {reliability_sql("value", "value_moe")} AS value_reliability, + percent AS percent, + percent_moe AS percent_moe, + -- Collection rounds percent_moe to 0.1 point, so a small share's + -- real margin can round to 0, which reliability_sql reads as a + -- controlled estimate. Unless the count is controlled too, label + -- the share by its count: exact when the total is controlled, and + -- otherwise never rosier than the share's own label. + CASE + WHEN percent_moe = 0 AND value_moe > 0 + THEN {reliability_sql("value", "value_moe")} + ELSE {reliability_sql("percent", "percent_moe")} + END AS percent_reliability FROM ({acs_geo_sql(base)}) """ diff --git a/backend/data_cleaning/clean_acs5_timeseries.py b/backend/data_cleaning/clean_acs5_timeseries.py index 9bf81d7e..fb7288d0 100644 --- a/backend/data_cleaning/clean_acs5_timeseries.py +++ b/backend/data_cleaning/clean_acs5_timeseries.py @@ -25,12 +25,20 @@ import duckdb +from data_cleaning.clean_acs5 import published_moe_sql from data_cleaning.geo_lookup import ( UNAVAILABLE, acs_geo_sql, build_geo_lookups, write_table, ) +from data_cleaning.margins import ( + proportion_moe_sql, + ratio_moe_sql, + reliability_sql, + sum_moe_agg_sql, + sum_moe_sql, +) # Raw identifier columns every base query must expose (see `acs_geo_sql`) KEYS = "year, NAME, geo_type, state, county" @@ -99,12 +107,33 @@ def in_list(values: list[str]) -> str: return ", ".join(f"'{v}'" for v in values) +def with_moe(column: str) -> tuple[str, str, str]: + """A measure column with its margin of error and reliability columns.""" + return (column, f"{column}_moe", f"{column}_reliability") + + +def published_value_moe() -> str: + """ + SELECT columns for a raw DP row's published margin of `Value`, as + `Value_MOE` and `Value_Reliability`. + """ + moe = published_moe_sql("MOE") + return ( + f'{moe} AS "Value_MOE", ' + f'{reliability_sql(numeric("Value"), moe)} AS "Value_Reliability"' + ) + + +VALUE_WITH_MOE = ("Value", "Value_MOE", "Value_Reliability") + + def single_variable( source: str, variables: list[str], out_col: str, value_col: str = "Value", keep_variable: bool = False, + moe_col: str | None = None, ) -> str: """ Filters a RAW lake table to Key, variable (if true), and value columns @@ -115,6 +144,9 @@ def single_variable( out_col: value_col: Column name containing the tracked value over time. keep_variable: (T/F) if you want to keep the variable column. Best for timeseries tables tracking multiple variables. + moe_col: Column holding value_col's 90% margin of error, written as + `{out_col}_moe` with its `{out_col}_reliability` label (see + :func:`with_moe`). None leaves both out. Returns: str: A SQL query which filters the source table to contain the time series variable(s) @@ -122,8 +154,15 @@ def single_variable( """ variable = ", Variable" if keep_variable else "" + moe = ( + f", {numeric(moe_col)} AS {out_col}_moe, " + f"{reliability_sql(numeric(value_col), numeric(moe_col))} " + f"AS {out_col}_reliability" + if moe_col + else "" + ) return f"""--sql - SELECT {KEYS}{variable}, {numeric(value_col)} AS {out_col} + SELECT {KEYS}{variable}, {numeric(value_col)} AS {out_col}{moe} FROM lake.RAW.{source} WHERE Variable IN ({in_list(variables)}) """ @@ -167,45 +206,65 @@ def population_change_sql() -> str: def age_dependency_ratio_sql() -> str: """ - SQL calculating age dependency ratio: (`Under 18` + `65 and over`) / (`18 to 64`) * 100. + SQL calculating age dependency ratio: (`Under 18` + `65 and over`) / (`18 to 64`) * 100, + with its 90% margin of error and reliability label. + + The dependents and the working-age population don't overlap, so the margin + takes the ratio form, from the two sums' margins. Returns: str: A SQL query that creates the age dependency ratio table """ - ages = [ - "Under 18", - "18 to 24", - "25 to 34", - "35 to 44", - "45 to 54", - "55 to 64", - "65 to 74", - "75 Plus", - ] + dependents = ["Under 18", "65 to 74", "75 Plus"] + working = ["18 to 24", "25 to 34", "35 to 44", "45 to 54", "55 to 64"] + ages = dependents + working wide = ",\n".join( - f"MAX(CASE WHEN Variable = '{a}' THEN v END) AS \"{a}\"" for a in ages + f"MAX(CASE WHEN Variable = '{a}' THEN {col} END) AS \"{a}{suffix}\"" + for a in ages + for col, suffix in [("v", ""), ("m", " MOE")] ) - working = " + ".join(f'"{a}"' for a in ages[1:6]) + + def total(group: list[str]) -> str: + return " + ".join(f'"{a}"' for a in group) + + def total_moe(group: list[str]) -> str: + return sum_moe_sql([(f'"{a}"', f'"{a} MOE"') for a in group]) return f"""--sql SELECT {KEYS}, - ROUND( - CASE - WHEN ({working}) > 0 - THEN ("Under 18" + "65 to 74" + "75 Plus") - / ({working}) * 100 - END, - 1 - ) AS age_dependency_ratio + age_dependency_ratio, + age_dependency_ratio_moe, + {reliability_sql("age_dependency_ratio", "age_dependency_ratio_moe")} + AS age_dependency_ratio_reliability FROM ( - SELECT {KEYS}, {wide} + SELECT + {KEYS}, + ROUND(100 * ratio, 1) AS age_dependency_ratio, + ROUND( + 100 * {ratio_moe_sql("ratio", "num_moe", "den", "den_moe")}, 1 + ) AS age_dependency_ratio_moe FROM ( - SELECT {KEYS}, Variable, {numeric("Value")} AS v - FROM lake.RAW.demographics - WHERE Variable IN ({in_list(ages)}) + SELECT + {KEYS}, + ({total(dependents)}) / NULLIF({total(working)}, 0) AS ratio, + {total_moe(dependents)} AS num_moe, + NULLIF({total(working)}, 0) AS den, + {total_moe(working)} AS den_moe + FROM ( + SELECT {KEYS}, {wide} + FROM ( + SELECT + {KEYS}, + Variable, + {numeric("Value")} AS v, + {numeric("Value_MOE")} AS m + FROM lake.RAW.demographics + WHERE Variable IN ({in_list(ages)}) + ) + GROUP BY {KEYS} + ) ) - GROUP BY {KEYS} ) """ @@ -228,6 +287,9 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: households" is NULL unless all three tenures are complete. Percent is NULL when the universe is 0 (no households of that tenure). + Summed counts take the root sum of their published margins, and Percent + the proportion form, since burdened households are a subset of Total. + Starts in 2013: earlier DP04 layouts differ, and 2010-2012 do not separate owners with and without a mortgage. @@ -236,12 +298,15 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: Returns: str: SELECT of KEYS, Variable, Value (burdened households), - Total (universe), and Percent. + Total (universe), and Percent, each with its 90% margin of error + (`_MOE`) and reliability label (`_Reliability`). """ tenure_cases = "\n".join( f"WHEN '{subcategory}' THEN '{label}'" for subcategory, label in BURDEN_TENURES.items() ) + brackets = "Variable <> 'Total'" + percent_moe = proportion_moe_sql("p", '"Value_MOE"', "Total", "Total_MOE") return f"""--sql WITH src AS ( SELECT @@ -251,7 +316,8 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: CASE WHEN TRY_CAST(Value AS DOUBLE) >= 0 THEN TRY_CAST(Value AS DOUBLE) - END AS v + END AS v, + {published_moe_sql("MOE")} AS m FROM {source} WHERE year >= {BURDEN_FIRST_YEAR} AND Measure = 'Estimate' @@ -263,10 +329,12 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: SELECT {KEYS}, tenure, - SUM(v) FILTER (WHERE Variable <> 'Total') AS burdened, - COUNT(v) FILTER (WHERE Variable <> 'Total') AS n_brackets, - COUNT(*) FILTER (WHERE Variable <> 'Total') AS n_bracket_rows, + SUM(v) FILTER (WHERE {brackets}) AS burdened, + {sum_moe_agg_sql("v", "m", brackets)} AS burdened_moe, + COUNT(v) FILTER (WHERE {brackets}) AS n_brackets, + COUNT(*) FILTER (WHERE {brackets}) AS n_bracket_rows, MAX(v) FILTER (WHERE Variable = 'Total') AS total, + MAX(m) FILTER (WHERE Variable = 'Total') AS total_moe, COUNT(v) FILTER (WHERE Variable = 'Total') AS n_totals, COUNT(*) FILTER (WHERE Variable = 'Total') AS n_total_rows FROM src @@ -280,7 +348,13 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: CASE WHEN n_brackets = 2 AND n_bracket_rows = 2 THEN burdened END AS "Value", - CASE WHEN n_totals = 1 AND n_total_rows = 1 THEN total END AS Total + CASE + WHEN n_brackets = 2 AND n_bracket_rows = 2 THEN burdened_moe + END AS "Value_MOE", + CASE WHEN n_totals = 1 AND n_total_rows = 1 THEN total END AS Total, + CASE + WHEN n_totals = 1 AND n_total_rows = 1 THEN total_moe + END AS Total_MOE FROM by_tenure ), @@ -289,26 +363,43 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: {KEYS}, '{BURDEN_ALL_HOUSEHOLDS}' AS Variable, CASE WHEN COUNT("Value") = 3 THEN SUM("Value") END AS "Value", - CASE WHEN COUNT(Total) = 3 THEN SUM(Total) END AS Total + CASE + WHEN COUNT("Value") = 3 + THEN {sum_moe_agg_sql('"Value"', '"Value_MOE"')} + END AS "Value_MOE", + CASE WHEN COUNT(Total) = 3 THEN SUM(Total) END AS Total, + CASE + WHEN COUNT(Total) = 3 THEN {sum_moe_agg_sql("Total", "Total_MOE")} + END AS Total_MOE FROM tenure_rows GROUP BY {KEYS} ), combined AS ( - SELECT * FROM tenure_rows - UNION ALL - SELECT * FROM all_rows + SELECT + *, + CASE WHEN "Value" IS NOT NULL AND Total > 0 THEN "Value" / Total END AS p + FROM ( + SELECT * FROM tenure_rows + UNION ALL + SELECT * FROM all_rows + ) ) + -- Reliability comes from the unrounded figures: a small share's + -- margin can round to 0.0, which would read as a controlled estimate. SELECT {KEYS}, Variable, "Value", + ROUND("Value_MOE") AS "Value_MOE", + {reliability_sql('"Value"', '"Value_MOE"')} AS "Value_Reliability", Total, - CASE - WHEN "Value" IS NOT NULL AND Total > 0 - THEN ROUND(100 * "Value" / Total, 1) - END AS Percent + ROUND(Total_MOE) AS Total_MOE, + {reliability_sql("Total", "Total_MOE")} AS Total_Reliability, + ROUND(100 * p, 1) AS Percent, + ROUND(100 * {percent_moe}, 1) AS Percent_MOE, + {reliability_sql("p", percent_moe)} AS Percent_Reliability FROM combined """ @@ -318,34 +409,48 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: # Median age "median_age": TimeSeries( table="acs5Demographics_medianAge_timeseries", - base=single_variable("demographics", ["Median Age"], "median_age"), - columns=("median_age",), + base=single_variable( + "demographics", ["Median Age"], "median_age", moe_col="Value_MOE" + ), + columns=with_moe("median_age"), ), # Median Household Income "median_hh_income": TimeSeries( table="acs5Economics_medianHouseholdIncome_timeseries", base=single_variable( - "economic", ["Median Household Income"], "median_household_income" + "economic", + ["Median Household Income"], + "median_household_income", + moe_col="Value_MOE", ), - columns=("median_household_income",), + columns=with_moe("median_household_income"), ), # Median Home Value "median_home_value": TimeSeries( table="acs5Housing_medianHomeValue_timeseries", - base=single_variable("housing", ["Median Home Value"], "median_home_value"), - columns=("median_home_value",), + base=single_variable( + "housing", ["Median Home Value"], "median_home_value", moe_col="Value_MOE" + ), + columns=with_moe("median_home_value"), ), # Per Capita Income "per_capita_income": TimeSeries( table="acs5Economics_perCapitaIncome_timeseries", - base=single_variable("economic", ["Per Capita Income"], "per_capita_income"), - columns=("per_capita_income",), + base=single_variable( + "economic", ["Per Capita Income"], "per_capita_income", moe_col="Value_MOE" + ), + columns=with_moe("per_capita_income"), ), # Total Housing Units "total_housing_units": TimeSeries( table="acs5Housing_housingUnits_timeseries", - base=single_variable("housing", ["Total Housing Units"], "total_housing_units"), - columns=("total_housing_units",), + base=single_variable( + "housing", + ["Total Housing Units"], + "total_housing_units", + moe_col="Value_MOE", + ), + columns=with_moe("total_housing_units"), ), # Vacancy Rates "vacancy_rate": TimeSeries( @@ -356,8 +461,9 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: "percent", value_col="Percent", keep_variable=True, + moe_col="Percent_MOE", ), - columns=("Variable", "percent"), + columns=("Variable", *with_moe("percent")), ), # Population Change (%) "population_change": TimeSeries( @@ -369,13 +475,14 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: "age_dependency_ratio": TimeSeries( table="acs5Demographics_ageDependencyRatio_timeseries", base=age_dependency_ratio_sql(), - columns=("age_dependency_ratio",), + columns=with_moe("age_dependency_ratio"), ), # Health Insurance Coverage (Public, Private, None) "health_insurance": TimeSeries( table="acs5Economics_healthInsurance_timeseries", base=f"""--sql - SELECT {KEYS}, Variable, {numeric("Value")} AS "Value" + SELECT {KEYS}, Variable, {numeric("Value")} AS "Value", + {published_value_moe()} FROM lake.RAW.acs5_economic WHERE Category LIKE '%INSURANCE%' AND Subcategory = 'Civilian noninstitutionalized population' @@ -386,14 +493,15 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: ) AND Measure = 'Estimate' """, - columns=("Variable", "Value"), + columns=("Variable", *VALUE_WITH_MOE), with_county=False, ), # Median Earnings (Male, Female, All Workers) "median_earnings": TimeSeries( table="acs5Economics_medianEarnings_timeseries", base=f"""--sql - SELECT {KEYS}, Subcategory AS Variable, {numeric("Value")} AS "Value" + SELECT {KEYS}, Subcategory AS Variable, {numeric("Value")} AS "Value", + {published_value_moe()} FROM lake.RAW.acs5_economic WHERE Category LIKE '%INCOME AND BENEFITS%' AND Subcategory IN ( @@ -404,14 +512,23 @@ def housing_cost_burden_sql(source: str = "lake.RAW.acs5_housing") -> str: AND Variable = 'Total' AND Measure = 'Estimate' """, - columns=("Variable", "Value"), + columns=("Variable", *VALUE_WITH_MOE), with_county=False, ), # Housing Cost Burden (30%+ of income on housing), by tenure "housing_cost_burden": TimeSeries( table="acs5Housing_incomeBurden_timeseries", base=housing_cost_burden_sql(), - columns=("Variable", "Value", "Total", "Percent"), + columns=( + "Variable", + *VALUE_WITH_MOE, + "Total", + "Total_MOE", + "Total_Reliability", + "Percent", + "Percent_MOE", + "Percent_Reliability", + ), with_county=False, ), } diff --git a/backend/data_cleaning/clean_acs5_tract.py b/backend/data_cleaning/clean_acs5_tract.py index 423431c3..a5d88f9d 100644 --- a/backend/data_cleaning/clean_acs5_tract.py +++ b/backend/data_cleaning/clean_acs5_tract.py @@ -5,17 +5,18 @@ Each indicator is a share (count / universe) with a 90% margin of error, derived with the Census Bureau's formulas for sums (root sum of squared - MOEs) and proportions. Census sentinels (negative estimates) and any + MOEs, counting only the largest among zero estimates) and proportions. Census sentinels (negative estimates) and any missing component make the indicator NULL instead of a partial sum. Writes lake.CLEANED.acs5_tract_tidy: year, geoid, county_fips, county, name, geo_type, section, variable, value (count), universe, percent, - percent_moe. + percent_moe, percent_reliability (see data_cleaning/margins.py). """ import duckdb from data_cleaning.geo_lookup import build_geo_lookups, write_table +from data_cleaning.margins import reliability_sql TABLE_NAME = "acs5_tract_tidy" @@ -64,6 +65,17 @@ def _in_list(codes: list[str]) -> str: return ", ".join(f"'{c}'" for c in codes) +def _sum_moe(codes: list[str]) -> str: + """Margin of error of the sum of `codes`: root sum of squares, counting + only the largest margin among zero estimates, whose margins are modeled + (Census guidance; matches data_collection/base.py).""" + codes = _in_list(codes) + return f"""SQRT( + COALESCE(SUM(moe * moe) FILTER (WHERE code IN ({codes}) AND est <> 0), 0) + + COALESCE(MAX(moe * moe) FILTER (WHERE code IN ({codes}) AND est = 0), 0) + )""" + + def indicator_sql(section: str, variable: str, num: list[str], den: list[str]) -> str: """One row per tract and year for a single count / universe indicator.""" return f"""--sql @@ -74,13 +86,11 @@ def indicator_sql(section: str, variable: str, num: list[str], den: list[str]) - SUM(est) FILTER (WHERE code IN ({_in_list(num)})) AS num, COUNT(est) FILTER (WHERE code IN ({_in_list(num)})) = {len(num)} AS num_ok, - SQRT(SUM(moe * moe) FILTER (WHERE code IN ({_in_list(num)}))) - AS num_moe, + {_sum_moe(num)} AS num_moe, SUM(est) FILTER (WHERE code IN ({_in_list(den)})) AS den, COUNT(est) FILTER (WHERE code IN ({_in_list(den)})) = {len(den)} AS den_ok, - SQRT(SUM(moe * moe) FILTER (WHERE code IN ({_in_list(den)}))) - AS den_moe, + {_sum_moe(den)} AS den_moe, COUNT(moe) FILTER (WHERE code IN ({_in_list(num + den)})) = {len(set(num + den))} AS moe_ok FROM raw @@ -96,11 +106,12 @@ def tract_sql() -> str: SELECT year, NAME, state, county, tract, code, CASE WHEN estimate >= 0 THEN estimate END AS est, - -- -555555555 / -222222222: the estimate is controlled, so its - -- margin of error is zero; other negative codes are missing. + -- -555555555: the estimate is controlled, so its margin of + -- error is zero. Other negative codes are missing, e.g. + -- -222222222: too few sample cases to compute a margin. CASE WHEN moe >= 0 THEN moe - WHEN moe IN (-555555555, -222222222) THEN 0 + WHEN moe = -555555555 THEN 0 END AS moe FROM lake.RAW.acs5_tract ), @@ -131,11 +142,12 @@ def tract_sql() -> str: CASE WHEN r.den_ok THEN r.den END AS universe, ROUND(100 * r.p, 1) AS percent, -- Census proportion MOE; the + form when the difference goes - -- negative, per the ACS handbook. + -- negative, per the ACS handbook, or is zero (a 100% share, + -- which would otherwise claim no sampling error). CASE WHEN r.p IS NOT NULL AND r.moe_ok THEN ROUND( 100 * SQRT( CASE - WHEN r.num_moe ^ 2 >= r.p ^ 2 * r.den_moe ^ 2 + WHEN r.num_moe ^ 2 > r.p ^ 2 * r.den_moe ^ 2 THEN r.num_moe ^ 2 - r.p ^ 2 * r.den_moe ^ 2 ELSE r.num_moe ^ 2 + r.p ^ 2 * r.den_moe ^ 2 END @@ -148,6 +160,16 @@ def tract_sql() -> str: """ +def reliable_tract_sql() -> str: + """:func:`tract_sql` with a reliability label for each percent.""" + return f"""--sql + SELECT + *, + {reliability_sql("percent", "percent_moe")} AS percent_reliability + FROM ({tract_sql()}) + """ + + def main(con: duckdb.DuckDBPyConnection): build_geo_lookups(con) - write_table(con, TABLE_NAME, tract_sql()) + write_table(con, TABLE_NAME, reliable_tract_sql()) diff --git a/backend/data_cleaning/margins.py b/backend/data_cleaning/margins.py new file mode 100644 index 00000000..e4c6c52e --- /dev/null +++ b/backend/data_cleaning/margins.py @@ -0,0 +1,128 @@ +""" +**Description**: + Shared SQL for ACS margins of error in the cleaning step. + + Each margin of error column (`_moe`, or `moe` in the data + profiles) gets a reliability label beside it, from the coefficient of + variation CV = standard error / estimate, where the standard error of an + ACS 90% margin is MOE / 1.645. The Census Bureau sets no fixed cut + points; CV under 15% (high) and over 30% (low) is a common convention + among ACS data users. +""" + +Z_90 = 1.645 +HIGH_CV = 0.15 # below this, high reliability +LOW_CV = 0.30 # above this, low reliability + + +def reliability_sql(estimate: str, moe: str) -> str: + """ + 'high', 'medium' or 'low' reliability for an estimate and its 90% margin. + + A zero estimate with a nonzero margin is 'low': its CV is undefined, and + the true value may well not be zero. A zero margin (a controlled + estimate) is 'high'. Null when either input is null. + + Args: + estimate: SQL expression for the estimate + moe: SQL expression for its margin of error + + Returns: + str: A SQL CASE expression + """ + cv = f"{moe} / {Z_90} / ABS({estimate})" + return f"""CASE + WHEN {estimate} IS NULL OR {moe} IS NULL THEN NULL + WHEN {moe} = 0 THEN 'high' + WHEN {estimate} = 0 THEN 'low' + WHEN {cv} < {HIGH_CV} THEN 'high' + WHEN {cv} <= {LOW_CV} THEN 'medium' + ELSE 'low' + END""" + + +def sum_moe_sql(terms: list[tuple[str, str]]) -> str: + """ + 90% margin of a sum of estimates: the root sum of their squared margins + (ACS handbook). As in data_collection.base.sum_moe, only the largest + margin among zero estimates counts, since summing many modeled zero + margins overstates the error. Null when any margin is null. + + Args: + terms: (estimate, margin of error) SQL expression pairs + + Returns: + str: A SQL expression + """ + nonzero = " + ".join( + f"CASE WHEN {e} = 0 THEN 0 ELSE ({m}) ^ 2 END" for e, m in terms + ) + zero = ", ".join(f"CASE WHEN {e} = 0 THEN ({m}) ^ 2 ELSE 0 END" for e, m in terms) + missing = " OR ".join(f"{m} IS NULL" for _, m in terms) + return f"""CASE + WHEN {missing} THEN NULL + ELSE SQRT({nonzero} + GREATEST({zero})) + END""" + + +def ratio_moe_sql(ratio: str, num_moe: str, den: str, den_moe: str) -> str: + """ + 90% margin of the ratio num / den, where num is not a subset of den: + sqrt(num_moe^2 + ratio^2 * den_moe^2) / den (ACS handbook). A subset + takes the proportion form instead (data_collection.base.proportion_moe). + + Args: + ratio: SQL expression for num / den + num_moe: SQL expression for the numerator's margin of error + den: SQL expression for the denominator + den_moe: SQL expression for the denominator's margin of error + + Returns: + str: A SQL expression + """ + return f"SQRT(({num_moe}) ^ 2 + ({ratio}) ^ 2 * ({den_moe}) ^ 2) / ({den})" + + +def sum_moe_agg_sql(estimate: str, moe: str, where: str = "TRUE") -> str: + """ + :func:`sum_moe_sql` as an aggregate: the margin of the SUM of `estimate` + over a group's rows (those matching `where`). Null when any of their + margins is null. + + Args: + estimate: SQL expression for each row's estimate + moe: SQL expression for its margin of error + where: SQL condition picking the rows summed + + Returns: + str: A SQL aggregate expression + """ + return f"""CASE + WHEN COUNT(*) FILTER (WHERE ({where}) AND {moe} IS NULL) > 0 THEN NULL + ELSE SQRT( + COALESCE(SUM(({moe}) ^ 2) FILTER (WHERE ({where}) AND {estimate} <> 0), 0) + + COALESCE(MAX(({moe}) ^ 2) FILTER (WHERE ({where}) AND {estimate} = 0), 0) + ) + END""" + + +def proportion_moe_sql(p: str, num_moe: str, den: str, den_moe: str) -> str: + """ + 90% margin of the proportion p = num / den, where num is a subset of den, + as data_collection.base.proportion_moe: sqrt(num_moe^2 - p^2 * den_moe^2) + / den, or the ratio form (+) when the term under the root isn't positive. + + Args: + p: SQL expression for num / den + num_moe: SQL expression for the numerator's margin of error + den: SQL expression for the denominator + den_moe: SQL expression for the denominator's margin of error + + Returns: + str: A SQL expression + """ + under = f"({num_moe}) ^ 2 - ({p}) ^ 2 * ({den_moe}) ^ 2" + return f"""CASE + WHEN {under} > 0 THEN SQRT({under}) / ({den}) + ELSE {ratio_moe_sql(p, num_moe, den, den_moe)} + END""" diff --git a/backend/data_tools/catalog.py b/backend/data_tools/catalog.py index 05e92606..775e1df9 100644 --- a/backend/data_tools/catalog.py +++ b/backend/data_tools/catalog.py @@ -40,19 +40,41 @@ def __post_init__(self) -> None: ACS_URL = "https://www.census.gov/programs-surveys/acs/data.html" +_ACS_PERIOD = ( + "ACS five-year estimates: year is the final year of the five-year period, " + "not a single-year count. Adjacent periods overlap." +) +_ACS_NULLS = ( + "Unavailable Census values and non-finite numbers are returned as null; " + "null does not mean zero. Counts, percentages, and medians are separate measures." +) ACS_CAVEATS = ( - ( - "ACS five-year estimates: year is the final year of the five-year period, " - "not a single-year count. Adjacent periods overlap." - ), + _ACS_PERIOD, ( "Margins of error are not included in these cleaned tables. Differences " "alone do not establish statistical significance." ), + _ACS_NULLS, +) +# For tables that carry *_moe columns. +ACS_MOE_CAVEATS = ( + _ACS_PERIOD, ( - "Unavailable Census values and non-finite numbers are returned as null; " - "null does not mean zero. Counts, percentages, and medians are separate measures." + "Columns ending in _moe are 90% margins of error: the estimate plus or " + "minus its margin is the 90% confidence interval; percent_moe is in " + "percentage points. Margins of sums and percentages are Census " + "approximations. For estimates from periods that do not overlap (such as " + "2018 and 2023), or from separate places in the same period, a difference " + "larger than the square root of the sum of their squared margins is " + "significant at 90% confidence. That test does not apply to adjacent " + "years, whose five-year periods share four years, or to a place and an " + "area that contains it. A null margin means the Census could not compute one. Each margin has a " + "reliability label (high, medium, low) from the coefficient of variation, " + "CV = (moe / 1.645) / estimate: under 15% high, over 30% low; a zero " + "estimate with a nonzero margin is low. Treat low-reliability estimates " + "as rough, especially for small towns." ), + _ACS_NULLS, ) ZONING_URL = "https://geodata.vermont.gov/datasets/VCGI::vt-zoning-areas/about" ZONING_CAVEATS = ( @@ -96,13 +118,18 @@ def _acs_tidy(key: str, label: str, description: str, table: str) -> Dataset: table, ACS_URL, "U.S. Census Bureau, American Community Survey five-year estimates", - ACS_CAVEATS, + ACS_MOE_CAVEATS, "tidy", "year", "name", "geo_type", variable_columns=("section", "variable"), - value_columns={"value": "varies by variable", "percent": "percent"}, + value_columns={ + "value": "varies by variable", + "value_moe": "varies by variable", + "percent": "percent", + "percent_moe": "percentage points", + }, filter_columns=( "year", "name", @@ -112,6 +139,8 @@ def _acs_tidy(key: str, label: str, description: str, table: str) -> Dataset: "geoid", "section", "variable", + "value_reliability", + "percent_reliability", ), ) @@ -131,6 +160,9 @@ def _acs_series( # ACS series carry the standard geoid/county_fips identifiers; `county` is # False for tables built with `with_county=False` in clean_acs5_timeseries. identifiers = ("geoid", "county_fips") + (("county",) if county else ()) + # Series with _moe columns also carry _reliability. + margins = [c.removesuffix("_moe") for c in values if c.endswith("_moe")] + acs_caveats = ACS_MOE_CAVEATS if margins else ACS_CAVEATS return Dataset( f"acs5_ts_{key}", label, @@ -145,7 +177,7 @@ def _acs_series( ( ("Historic boundaries and coverage can change between censuses.",) if historic - else ACS_CAVEATS + else acs_caveats ) + caveats, "tidy" if dimensions else "wide", @@ -157,7 +189,8 @@ def _acs_series( value_columns=values, filter_columns=("year", "name", "geo_type") + dimensions - + (("geoid", "county_fips", "County") if historic else identifiers), + + (("geoid", "county_fips", "County") if historic else identifiers) + + tuple(f"{measure}_reliability" for measure in margins), ) @@ -388,7 +421,7 @@ def _zoning_numeric(column: str) -> bool: "median_age", "Median age", "acs5Demographics_medianAge_timeseries", - {"median_age": "years"}, + {"median_age": "years", "median_age_moe": "years"}, ), _acs_series( "age_dependency_ratio", @@ -400,13 +433,16 @@ def _zoning_numeric(column: str) -> bool: "median_home_value", "Median home value", "acs5Housing_medianHomeValue_timeseries", - {"median_home_value": "USD (year-specific dollars)"}, + { + "median_home_value": "USD (year-specific dollars)", + "median_home_value_moe": "USD (year-specific dollars)", + }, ), _acs_series( "vacancy_rates", "Vacancy rates", "acs5Housing_vacancyRates_timeseries", - {"percent": "percent"}, + {"percent": "percent", "percent_moe": "percentage points"}, dimensions=("Variable",), ), _acs_series( @@ -427,13 +463,19 @@ def _zoning_numeric(column: str) -> bool: "household_income", "Median household income", "acs5Economics_medianHouseholdIncome_timeseries", - {"median_household_income": "USD (year-specific dollars)"}, + { + "median_household_income": "USD (year-specific dollars)", + "median_household_income_moe": "USD (year-specific dollars)", + }, ), _acs_series( "per_capita_income", "Per capita income", "acs5Economics_perCapitaIncome_timeseries", - {"per_capita_income": "USD (year-specific dollars)"}, + { + "per_capita_income": "USD (year-specific dollars)", + "per_capita_income_moe": "USD (year-specific dollars)", + }, ), _acs_series( "median_earnings", @@ -456,7 +498,10 @@ def _zoning_numeric(column: str) -> bool: "housing_units", "Housing units", "acs5Housing_housingUnits_timeseries", - {"total_housing_units": "housing units"}, + { + "total_housing_units": "housing units", + "total_housing_units_moe": "housing units", + }, ), Dataset( "acs5_dp", @@ -466,7 +511,7 @@ def _zoning_numeric(column: str) -> bool: "acs5_dp_combined_tidy", ACS_URL, "U.S. Census Bureau, ACS data profiles", - ACS_CAVEATS + ACS_MOE_CAVEATS + ( ( "Estimate and percentage labels vary across source years. " @@ -490,7 +535,11 @@ def _zoning_numeric(column: str) -> bool: "year", "name", variable_columns=("table", "category", "subcategory", "variable", "measure"), - value_columns={"value": "varies by variable and measure"}, + # moe is value's published 90% margin of error; reliability labels it. + value_columns={ + "value": "varies by variable and measure", + "moe": "same as value", + }, filter_columns=( "name", "geo_type", @@ -504,6 +553,7 @@ def _zoning_numeric(column: str) -> bool: "variable_code", "source_label", "year", + "reliability", ), ), Dataset( @@ -959,6 +1009,14 @@ def dataset_lineage(dataset: Dataset) -> dict[str, Any]: "code_reference": f"backend/data_collection/{source}.py", } } + if f"{output}_moe" in dataset.value_columns: + result["source_columns"][f"{output}_moe"] = { + "raw_table": f"RAW.{source}", + "raw_column": "Value_MOE", + "selectors": {"Variable": variable}, + "census_codes": [code[:-1] + "M"], + "code_reference": f"backend/data_collection/{source}.py", + } result["transformations"].insert(0, f"backend/data_collection/{source}.py") if dataset.id == "acs5_ts_median_home_value": result["source_code_note"] += ( @@ -1020,7 +1078,18 @@ def dataset_lineage(dataset: Dataset) -> dict[str, Any]: "raw_column": "Value", "census_codes": [], "selector_columns": list(dataset.variable_columns), - } + }, + "moe": { + "raw_tables": [ + "RAW.acs5_social", + "RAW.acs5_economic", + "RAW.acs5_housing", + "RAW.acs5_demographic", + ], + "raw_column": "MOE", + "census_codes": [], + "selector_columns": list(dataset.variable_columns), + }, } return result @@ -1317,12 +1386,32 @@ def decode_variable_id(dataset: Dataset, variable_id: str) -> dict[str, Any]: return selectors +def margin_column(dataset: Dataset, measure: str) -> str | None: + """The column holding `measure`'s 90% margin of error, if it has one.""" + columns = dataset.value_columns or {} + if f"{measure}_moe" in columns: + return f"{measure}_moe" + # Data profile rows keep their published margin as `moe` beside `value`. + if dataset.kind == "dp" and measure == "value" and "moe" in columns: + return "moe" + return None + + +def with_margins(dataset: Dataset, measures: list[str]) -> list[str]: + """`measures` with each one's margin right after it. A margin means nothing + without its estimate, so the two always travel together.""" + paired = [] + for measure in measures: + paired += [measure, margin_column(dataset, measure)] + return list(dict.fromkeys(m for m in paired if m)) + + def variable_units(dataset: Dataset, selectors: dict[str, Any]) -> dict[str, str]: """Resolve row-specific units without treating every ACS value as a count.""" units = dict(dataset.value_columns or {}) column = selectors.get("$column") if column: - return {column: units[column]} + return {c: units[c] for c in with_margins(dataset, [column])} variable = str(selectors.get("variable", "")) if dataset.id in { "acs5_demographics", @@ -1383,6 +1472,7 @@ def variable_units(dataset: Dataset, selectors: dict[str, Any]) -> dict[str, str "Median Age": "years", } units["value"] = value_units.get(variable, "source-defined; unit unavailable") + units["value_moe"] = units["value"] if dataset.kind == "dp": measure = str(selectors.get("measure", "")).casefold() subcategory = str(selectors.get("subcategory", "")).strip().casefold() @@ -1400,6 +1490,10 @@ def variable_units(dataset: Dataset, selectors: dict[str, Any]) -> dict[str, str units["value"] = "percent" else: units["value"] = "source-defined estimate; inspect variable label" + # A percent's margin (the PM code) is in percentage points. + units["moe"] = ( + "percentage points" if units["value"] == "percent" else units["value"] + ) return units @@ -1450,10 +1544,13 @@ def search_variables( for row in rows ] else: + # A margin column comes back with its estimate, not as its own variable. + margins = {margin_column(dataset, c) for c in dataset.value_columns} selectors_list = [ {"$column": column} for column in dataset.value_columns - if all( + if column not in margins + and all( token in " ".join(re.findall(r"[a-z0-9]+", column.casefold())) for token in tokens ) diff --git a/backend/data_tools/service.py b/backend/data_tools/service.py index fd0083e5..a8cf3164 100644 --- a/backend/data_tools/service.py +++ b/backend/data_tools/service.py @@ -389,15 +389,22 @@ def _where(self, ds, request, locations, column_types=None, location_clause=None raise DataToolError( f"Unsupported measures. Allowed: {', '.join(ds.value_columns)}" ) + margins = set(catalog.with_margins(ds, selected_columns)) - set( + selected_columns + ) if ( request.measures and selected_columns - and set(request.measures) != set(selected_columns) + and set(request.measures) - margins != set(selected_columns) ): raise DataToolError( "measures must match the selected wide-column variable IDs" ) - return " AND ".join(clauses) or "TRUE", params, list(dict.fromkeys(measures)) + return ( + " AND ".join(clauses) or "TRUE", + params, + catalog.with_margins(ds, measures), + ) def _query(self, conn, request, tool, version): ds = catalog.get_dataset(request.dataset_id) diff --git a/backend/tests/test_acs5_tidy_moe.py b/backend/tests/test_acs5_tidy_moe.py new file mode 100644 index 00000000..81848904 --- /dev/null +++ b/backend/tests/test_acs5_tidy_moe.py @@ -0,0 +1,177 @@ +""" +Margins of error and their reliability labels through the ACS B-table +cleaners: RAW tidy rows (as written by data_collection/base.py) to the CLEANED +tidy tables and single-variable series. +""" + +import math + +import duckdb +import pandas as pd +import pytest + +from data_cleaning import clean_acs5_tidy, clean_acs5_timeseries + +TOWN = (2023, "county_subdivision", "Middlebury town, Addison County, Vermont") +ROWS = { + "demographics": [ + ("Age/Sex", "Median Age", 38.2, 1.9, None, None), + ("Age/Sex", "Population (ACS)", 2000.0, 0.0, 100.0, 0.0), + # Age bands for the age dependency ratio: 500 dependents to 1,000 of + # working age. + ("Age/Sex", "Under 18", 300.0, 40.0, 15.0, 2.0), + ("Age/Sex", "65 to 74", 200.0, 30.0, 10.0, 1.5), + ("Age/Sex", "75 Plus", 0.0, 15.0, 0.0, 0.8), + ("Age/Sex", "18 to 24", 100.0, 20.0, 5.0, 1.0), + ("Age/Sex", "25 to 34", 200.0, 30.0, 10.0, 1.5), + ("Age/Sex", "35 to 44", 200.0, 30.0, 10.0, 1.5), + ("Age/Sex", "45 to 54", 250.0, 40.0, 12.5, 2.0), + ("Age/Sex", "55 to 64", 250.0, 40.0, 12.5, 2.0), + ("Race", "Asian", 0.0, 20.0, 0.0, 0.0), + ("Race", "Native Hawaiian and Other Pacific Islander", 150.0, 80.0, 0.0, 0.0), + ], + "economic": [ + ("Income", "Median Household Income", 68854.0, 4120.0, None, None), + ("Labor Force", "Unemployment Rate", 120.0, 45.0, 3.1, 1.2), + ], + "education": [("Education", "Bachelor's Degree", 1500.0, 210.0, 30.0, 4.0)], + "housing": [ + ("Housing", "Median Home Value", 361200.0, 8938.0, None, None), + ("Housing", "Rental Vacancy Rate", 25.0, 18.0, 2.6, 1.9), + ], +} + + +@pytest.fixture +def lake(): + con = duckdb.connect() + con.execute("ATTACH ':memory:' AS lake") + con.execute("CREATE SCHEMA lake.RAW") + con.execute("CREATE SCHEMA lake.CLEANED") + con.execute( + """ + CREATE TABLE lake.RAW.vt_town_lines AS + SELECT 5000144350 AS GEOID, + 'Middlebury town, Addison County, Vermont' AS NAME + """ + ) + for table, rows in ROWS.items(): + raw = pd.DataFrame( + [(*TOWN, "50", "001", *row) for row in rows], + columns=["year", "geo_type", "NAME", "state", "county", "Section", + "Variable", "Value", "Value_MOE", "Percent", "Percent_MOE"], + ) # fmt: skip + con.register("raw_df", raw) + con.execute(f"CREATE TABLE lake.RAW.{table} AS SELECT * FROM raw_df") + con.unregister("raw_df") + return con + + +def test_tidy_tables_keep_both_margins(lake): + clean_acs5_tidy.main(lake) + row = lake.execute( + """ + SELECT geoid, value, value_moe, value_reliability, + percent, percent_moe, percent_reliability + FROM lake.CLEANED.acs5_economics_tidy WHERE variable = 'Unemployment Rate' + """ + ).fetchone() + # CV = (45 / 1.645) / 120 = 23% and (1.2 / 1.645) / 3.1 = 24%: medium + assert row == ("5000144350", 120.0, 45.0, "medium", 3.1, 1.2, "medium") + + +def test_share_whose_margin_rounds_to_zero_is_labeled_by_its_count(lake): + # Collection rounds percent_moe to 0.1 point, so a tiny share's real + # margin reads 0.0. Only a controlled count makes that 'high'. + clean_acs5_tidy.main(lake) + rows = lake.execute( + """ + SELECT variable, value_reliability, percent_reliability + FROM lake.CLEANED.acs5_demographics_tidy + WHERE percent_moe = 0 ORDER BY variable + """ + ).fetchall() + assert rows == [ + ("Asian", "low", "low"), # zero count with a nonzero margin + ("Native Hawaiian and Other Pacific Islander", "low", "low"), # CV 32% + ("Population (ACS)", "high", "high"), # controlled: no sampling error + ] + + +def test_snapshot_keeps_margins(lake): + clean_acs5_tidy.main(lake) + moe = lake.execute( + """ + SELECT value_moe FROM lake.CLEANED.acs5_snapshot_indicators_tidy + WHERE variable = 'Median Home Value' + """ + ).fetchone() + assert moe == (8938.0,) + + +def test_single_variable_series_carry_their_margin(lake): + clean_acs5_timeseries.main(lake, ["median_hh_income", "vacancy_rate"]) + income = lake.execute( + """ + SELECT median_household_income, median_household_income_moe, + median_household_income_reliability + FROM lake.CLEANED.acs5Economics_medianHouseholdIncome_timeseries + """ + ).fetchone() + vacancy = lake.execute( + """ + SELECT Variable, percent, percent_moe, percent_reliability + FROM lake.CLEANED.acs5Housing_vacancyRates_timeseries + """ + ).fetchone() + assert income == (68854.0, 4120.0, "high") # CV 4% + assert vacancy == ("Rental Vacancy Rate", 2.6, 1.9, "low") # CV 44% + + +def test_age_dependency_ratio_carries_a_ratio_margin(lake): + clean_acs5_timeseries.main(lake, ["age_dependency_ratio"]) + row = lake.execute( + """ + SELECT age_dependency_ratio, age_dependency_ratio_moe, + age_dependency_ratio_reliability + FROM lake.CLEANED.acs5Demographics_ageDependencyRatio_timeseries + """ + ).fetchone() + num_moe = math.sqrt(40**2 + 30**2 + 15**2) + den_moe = math.sqrt(20**2 + 30**2 + 30**2 + 40**2 + 40**2) + # sqrt(num_moe^2 + R^2 * den_moe^2) / den, with R = 500 / 1000 + moe = round(100 * math.sqrt(num_moe**2 + 0.5**2 * den_moe**2) / 1000, 1) + assert row == (50.0, moe, "high") # CV 8% + + +def test_dp_series_carry_their_published_margins(lake): + raw = pd.DataFrame( + [ + (*TOWN, "50", "001", "SELECTED ECONOMIC CHARACTERISTICS: HEALTH INSURANCE", + "Civilian noninstitutionalized population", "No health insurance coverage", + "Estimate", "300", "45"), + (*TOWN, "50", "001", "INCOME AND BENEFITS", + "Median earnings for workers (dollars)", "Total", "Estimate", "41250", + "-555555555"), + ], + columns=["year", "geo_type", "NAME", "state", "county", "Category", + "Subcategory", "Variable", "Measure", "Value", "MOE"], + ) # fmt: skip + lake.register("raw_df", raw) + lake.execute("CREATE TABLE lake.RAW.acs5_economic AS SELECT * FROM raw_df") + clean_acs5_timeseries.main(lake, ["health_insurance", "median_earnings"]) + + uninsured = lake.execute( + """ + SELECT "Value", "Value_MOE", "Value_Reliability" + FROM lake.CLEANED.acs5Economics_healthInsurance_timeseries + """ + ).fetchone() + earnings = lake.execute( + """ + SELECT "Value", "Value_MOE", "Value_Reliability" + FROM lake.CLEANED.acs5Economics_medianEarnings_timeseries + """ + ).fetchone() + assert uninsured == (300.0, 45.0, "high") # CV 9% + assert earnings == (41250.0, 0.0, "high") # controlled diff --git a/backend/tests/test_acs5_tract.py b/backend/tests/test_acs5_tract.py index 6e6890de..c77d81f5 100644 --- a/backend/tests/test_acs5_tract.py +++ b/backend/tests/test_acs5_tract.py @@ -110,6 +110,58 @@ def test_uninsured_split_by_poverty_is_a_crosstab(lake): assert len(uninsured(BELOW_138 + AT_OR_ABOVE_138)) == 15 +def test_controlled_margin_counts_as_zero(lake): + # -555555555: the universe is controlled, so it has no sampling error. + df = _clean(lake, {"B17001_001": (1000, -555555555), "B17001_002": (150, 50)}) + row = df[df.variable == "Below poverty level"].iloc[0] + + assert row.percent_moe == round(100 * 50 / 1000, 1) + + +def test_uncomputable_margin_makes_the_margin_null(lake): + # -222222222: too few sample cases to compute a margin. The estimate + # stands, but its margin is unknown, not zero. + df = _clean(lake, {"B17001_001": (1000, 100), "B17001_002": (0, -222222222)}) + row = df[df.variable == "Below poverty level"].iloc[0] + + assert row.percent == 0.0 + assert pd.isna(row.percent_moe) + + +def test_zero_estimates_count_only_their_largest_margin(lake): + # Of the 15 uninsured codes, 5 are 8 and 10 are zero. One zero code has a + # margin of 25 and the rest 10, so the sum's margin is + # sqrt(10^2 * 5 + 25^2): every nonzero margin, plus the largest zero one. + estimates = _c27016_fixture(uninsured_count=8) + zero = next(c for c in uninsured(BELOW_138) if estimates[c][0] == 0) + estimates[zero] = (0, 25) + df = _clean(lake, estimates).set_index("variable") + num_moe = math.sqrt(10**2 * 5 + 25**2) + p = 40 / 500 + expected = 100 * math.sqrt(num_moe**2 - p**2 * 40**2) / 500 + assert df.loc["No health insurance", "percent_moe"] == round(expected, 1) + + +def test_full_share_margin_is_not_zero(lake): + df = _clean(lake, {"B17001_001": (40, 12), "B17001_002": (40, 12)}) + row = df[df.variable == "Below poverty level"].iloc[0] + + assert row.percent == 100.0 + assert row.percent_moe == round(100 * math.sqrt(12**2 + 12**2) / 40, 1) + + +def test_percent_reliability_follows_the_coefficient_of_variation(lake): + # CV = (MOE / 1.645) / estimate: 1.2 / 1.645 / 15 = 5% -> high + high = _clean(lake, {"B17001_001": (10000, 100), "B17001_002": (1500, 120)}) + # 4.8 / 1.645 / 15 = 19% -> medium; 9.5 / 1.645 / 15 = 39% -> low + medium = _clean(lake, {"B17001_001": (1000, 100), "B17001_002": (150, 50)}) + low = _clean(lake, {"B17001_001": (200, 40), "B17001_002": (30, 20)}) + + assert high.percent_reliability.iloc[0] == "high" + assert medium.percent_reliability.iloc[0] == "medium" + assert low.percent_reliability.iloc[0] == "low" + + def test_suppressed_component_makes_the_indicator_null(lake): df = _clean( lake, {"B17001_001": (1000, 100), "B17001_002": (-666666666, -222222222)} diff --git a/backend/tests/test_clean_acs5.py b/backend/tests/test_clean_acs5.py index 4ebf0160..3fc854e6 100644 --- a/backend/tests/test_clean_acs5.py +++ b/backend/tests/test_clean_acs5.py @@ -28,7 +28,7 @@ _RAW_SCHEMA = """ year BIGINT, NAME VARCHAR, Variable_Code VARCHAR, Source_Label VARCHAR, Category VARCHAR, Subcategory VARCHAR, Variable VARCHAR, Measure VARCHAR, - Value VARCHAR, geo_type VARCHAR, state VARCHAR, county VARCHAR + Value VARCHAR, geo_type VARCHAR, state VARCHAR, county VARCHAR, MOE VARCHAR """ # (year, Variable_Code, Value) for Burlington's colliding DP04 SMOC medians. @@ -66,9 +66,10 @@ def _housing_rows(rows): {year}, '{BURLINGTON}', '{code}', 'Estimate!!SELECTED MONTHLY OWNER COSTS (SMOC)!!Median (dollars)', 'SELECTED MONTHLY OWNER COSTS (SMOC)', 'Median (dollars)', 'Total', - 'Estimate', '{value}', 'county_subdivision', '50', '007' + 'Estimate', '{value}', 'county_subdivision', '50', '007', + {f"'{moe[0]}'" if moe else "NULL"} )""" - for year, code, value in rows + for year, code, value, *moe in rows ) return f""" CREATE TABLE lake.RAW.acs5_housing ({_RAW_SCHEMA}); @@ -188,3 +189,35 @@ def test_dp_value_col(): assert df["value"].dtype.kind == "f" assert df["value"].min() >= 0 assert df["value"].max() < 666666666 + + +def test_dp_margins_are_numeric_and_labeled(): + con = _base_con() + con.execute( + _housing_rows( + [ + (2023, "DP04_0100E", "1800", "90"), # CV 3% -> high + (2023, "DP04_0101E", "40", "-555555555"), # controlled -> 0 + (2023, "DP04_0102E", "0", "-222222222"), # not computable + (2023, "DP04_0103E", "-666666666", "-222222222"), + (2023, "DP04_0104E", "500", "(X)"), + (2023, "DP04_0105E", "100", "60"), # CV 36% -> low + ] + ) + ) + build_dp_combined(con) + + rows = con.execute( + """ + SELECT variable_code, value, moe, reliability + FROM lake.CLEANED.acs5_dp_combined_tidy ORDER BY variable_code + """ + ).fetchall() + assert rows == [ + ("DP04_0100E", 1800.0, 90.0, "high"), + ("DP04_0101E", 40.0, 0.0, "high"), + ("DP04_0102E", 0.0, None, None), + ("DP04_0103E", None, None, None), + ("DP04_0104E", 500.0, None, None), + ("DP04_0105E", 100.0, 60.0, "low"), + ] diff --git a/backend/tests/test_housing_cost_burden.py b/backend/tests/test_housing_cost_burden.py index 5847c0dd..60f4b355 100644 --- a/backend/tests/test_housing_cost_burden.py +++ b/backend/tests/test_housing_cost_burden.py @@ -4,6 +4,8 @@ query, before the shared geography standardization is applied. """ +import math + import duckdb import pandas as pd import pytest @@ -21,15 +23,21 @@ GEO = ("county_subdivision", "50", "007") -def rows(year, name, subcategory, low, high, total, measure="Estimate"): - """DP04 rows for one tenure: the two burden brackets and the total.""" +def rows( + year, name, subcategory, low, high, total, measure="Estimate", moes=("4", "6", "9") +): + """DP04 rows for one tenure: the two burden brackets and the total, with + their published margins.""" return [ - (year, name, *GEO, subcategory, variable, measure, value) - for variable, value in [ - ("30.0 to 34.9 percent", low), - ("35.0 percent or more", high), - ("Total", total), - ] + (year, name, *GEO, subcategory, variable, measure, value, moe) + for (variable, value), moe in zip( + [ + ("30.0 to 34.9 percent", low), + ("35.0 percent or more", high), + ("Total", total), + ], + moes, + ) ] @@ -48,11 +56,11 @@ def run(raw_rows): CREATE TABLE raw ( year BIGINT, NAME VARCHAR, geo_type VARCHAR, state VARCHAR, county VARCHAR, Subcategory VARCHAR, Variable VARCHAR, - Measure VARCHAR, Value VARCHAR + Measure VARCHAR, Value VARCHAR, MOE VARCHAR ) """ ) - con.executemany("INSERT INTO raw VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", raw_rows) + con.executemany("INSERT INTO raw VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", raw_rows) df = con.execute(housing_cost_burden_sql(source="raw")).df() return df.set_index(["year", "NAME", "Variable"]) @@ -86,6 +94,7 @@ def test_percent_estimate_years_are_included(): "35.0 percent or more", "Percent Estimate", "30.0", + "5.0", ), ] df = run(raw) @@ -105,6 +114,7 @@ def test_percent_rows_and_early_years_are_ignored(): "35.0 percent or more", "Percent", "99.9", + "5.0", ), ] + complete_place(2012, "A") @@ -188,3 +198,51 @@ def test_rates_are_rounded_to_one_decimal(): df = run(raw) assert value(df, 2020, "A", "Renters") == 33.3 + + +def test_margins_by_tenure_and_all_households(): + raw = ( + rows(2020, "A", RENT, "10", "30", "100", moes=("6", "8", "12")) + + rows(2020, "A", MORTGAGE, "5", "15", "80", moes=("4", "7", "-555555555")) + + rows(2020, "A", NO_MORTGAGE, "2", "3", "50", moes=("3", "3", "9")) + ) + df = run(raw) + + # 40 renters of 100: the brackets' margins summed, then the proportion + # form, sqrt(10^2 - 0.4^2 * 12^2) / 100. + assert value(df, 2020, "A", "Renters", "Value_MOE") == 10 + assert value(df, 2020, "A", "Renters", "Percent_MOE") == round( + 100 * math.sqrt(10**2 - 0.4**2 * 12**2) / 100, 1 + ) + # A controlled total has no margin. + assert value(df, 2020, "A", "Owners with a mortgage", "Total_MOE") == 0 + assert value(df, 2020, "A", "Owners with a mortgage", "Total_Reliability") == ( + "high" + ) + + value_moe = math.sqrt(6**2 + 8**2 + 4**2 + 7**2 + 3**2 + 3**2) + total_moe = math.sqrt(12**2 + 0**2 + 9**2) + assert value(df, 2020, "A", ALL_HOUSEHOLDS, "Value_MOE") == round(value_moe) + assert value(df, 2020, "A", ALL_HOUSEHOLDS, "Total_MOE") == round(total_moe) + p = 65 / 230 + assert value(df, 2020, "A", ALL_HOUSEHOLDS, "Percent_MOE") == round( + 100 * math.sqrt(value_moe**2 - p**2 * total_moe**2) / 230, 1 + ) + + +def test_zero_bracket_margins_count_once(): + # Both brackets are zero: only the larger modeled margin counts. + raw = rows(2020, "A", RENT, "0", "0", "100", moes=("12", "16", "20")) + df = run(raw) + + assert value(df, 2020, "A", "Renters", "Value_MOE") == 16 + assert value(df, 2020, "A", "Renters", "Percent_Reliability") == "low" + + +def test_missing_margin_leaves_the_margin_missing(): + raw = rows(2020, "A", RENT, "10", "30", "100", moes=("-222222222", "8", "12")) + df = run(raw) + + assert value(df, 2020, "A", "Renters") == 40.0 + assert value(df, 2020, "A", "Renters", "Value_MOE") is None + assert value(df, 2020, "A", "Renters", "Percent_MOE") is None diff --git a/backend/tests/test_margins.py b/backend/tests/test_margins.py new file mode 100644 index 00000000..732aad47 --- /dev/null +++ b/backend/tests/test_margins.py @@ -0,0 +1,56 @@ +"""Tests for data_cleaning/margins.py: reliability labels and combined 90% margins.""" + +import math + +import duckdb +import pytest + +from data_cleaning.margins import ratio_moe_sql, reliability_sql, sum_moe_sql + + +@pytest.mark.parametrize( + ("estimate", "moe", "label"), + [ + (100, 24, "high"), # CV 14.6% + (100, 25, "medium"), # CV 15.2% + (100, 49, "medium"), # CV 29.8% + (100, 50, "low"), # CV 30.4% + (40, 0, "high"), # controlled: no sampling error + (0, 12, "low"), # CV undefined; the true value may not be zero + (None, 12, None), + (100, None, None), + ], +) +def test_reliability_follows_the_coefficient_of_variation(estimate, moe, label): + sql = f"SELECT {reliability_sql('e', 'm')} FROM (SELECT ?::DOUBLE e, ?::DOUBLE m)" + assert duckdb.execute(sql, [estimate, moe]).fetchone() == (label,) + + +def summed_margin(*terms): + pairs = [(f"e{i}", f"m{i}") for i in range(len(terms))] + cols = ", ".join(f"?::DOUBLE {e}, ?::DOUBLE {m}" for e, m in pairs) + sql = f"SELECT {sum_moe_sql(pairs)} FROM (SELECT {cols})" + return duckdb.execute(sql, [x for t in terms for x in t]).fetchone()[0] + + +def test_summed_margin_is_the_root_sum_of_squares(): + assert summed_margin((100, 30), (50, 40)) == pytest.approx(50.0) + + +def test_summed_margin_counts_only_the_largest_zero_estimate_margin(): + expected = math.sqrt(30**2 + 25**2) + assert summed_margin((100, 30), (0, 20), (0, 25)) == pytest.approx(expected) + + +def test_summed_margin_of_only_zero_estimates_is_their_largest(): + assert summed_margin((0, 20), (0, 25)) == pytest.approx(25.0) + + +def test_summed_margin_is_null_when_any_margin_is(): + assert summed_margin((100, 30), (0, None)) is None + + +def test_ratio_margin(): + sql = f"SELECT {ratio_moe_sql('0.5', '40', '1000', '60')}" + # sqrt(40^2 + 0.5^2 * 60^2) / 1000 + assert duckdb.execute(sql).fetchone()[0] == pytest.approx(0.05) diff --git a/backend/tests/test_mcp_catalog.py b/backend/tests/test_mcp_catalog.py index 11540b14..d093c8a0 100644 --- a/backend/tests/test_mcp_catalog.py +++ b/backend/tests/test_mcp_catalog.py @@ -139,7 +139,12 @@ def test_variable_search_deduplicates_years_and_roundtrips(warehouse): "section": "Age/Sex", "variable": "Population (ACS)", } - assert rows[0]["value_columns"] == {"value": "people", "percent": "percent"} + assert rows[0]["value_columns"] == { + "value": "people", + "value_moe": "people", + "percent": "percent", + "percent_moe": "percentage points", + } dataset = get_dataset("acs5_demographics") assert decode_variable_id(dataset, rows[0]["variable_id"]) == rows[0]["selectors"] assert ( @@ -153,9 +158,13 @@ def test_variable_search_deduplicates_years_and_roundtrips(warehouse): def test_wide_metric_variable_id_and_units(warehouse): - row = search_variables(warehouse, "acs5_ts_median_age")[0] + # The margin is not a variable of its own: it comes back with the estimate. + [row] = search_variables(warehouse, "acs5_ts_median_age") assert row["selectors"] == {"$column": "median_age"} - assert row["measures"] == [{"name": "median_age", "unit": "years"}] + assert row["measures"] == [ + {"name": "median_age", "unit": "years"}, + {"name": "median_age_moe", "unit": "years"}, + ] assert decode_variable_id( get_dataset("acs5_ts_median_age"), row["variable_id"] ) == {"$column": "median_age"} @@ -309,7 +318,8 @@ def test_missing_dataset_and_unknown_geography_fail_clearly(warehouse): def test_units_preserve_percentages_and_source_specific_estimates(): dataset = get_dataset("acs5_dp") assert variable_units(dataset, {"measure": "Percent Estimate"}) == { - "value": "percent" + "value": "percent", + "moe": "percentage points", } assert "source-defined" in variable_units(dataset, {"measure": "Estimate"})["value"] assert get_dataset("zoning_bylaws").value_columns["f1f_min_lot_size"] == "acres" @@ -318,11 +328,18 @@ def test_units_preserve_percentages_and_source_specific_estimates(): economics = get_dataset("acs5_economics") assert variable_units( economics, {"variable": "Labor Force Participation Rate (16+)"} - ) == {"value": "people", "percent": "percent"} + ) == { + "value": "people", + "value_moe": "people", + "percent": "percent", + "percent_moe": "percentage points", + } housing = get_dataset("acs5_housing") assert variable_units(housing, {"variable": "Rental Vacancy Rate"}) == { "value": "housing units", + "value_moe": "housing units", "percent": "percent", + "percent_moe": "percentage points", } assert ( "unavailable" diff --git a/backend/tests/test_mcp_issue_fixes.py b/backend/tests/test_mcp_issue_fixes.py index 2c973e7d..98fa240f 100644 --- a/backend/tests/test_mcp_issue_fixes.py +++ b/backend/tests/test_mcp_issue_fixes.py @@ -296,3 +296,48 @@ async def run(): ) asyncio.run(run()) + + +@pytest.fixture +def margins_service(tmp_path): + warehouse = tmp_path / "margins.duckdb" + with duckdb.connect(str(warehouse)) as conn: + conn.execute(""" + CREATE TABLE acs5Demographics_medianAge_timeseries( + year VARCHAR, geoid VARCHAR, county_fips VARCHAR, county VARCHAR, + name VARCHAR, geo_type VARCHAR, median_age DOUBLE, + median_age_moe DOUBLE, median_age_reliability VARCHAR + ); + INSERT INTO acs5Demographics_medianAge_timeseries VALUES + ('2023', '50', NULL, NULL, 'Vermont', 'state', 43.2, 0.2, 'high'); + CREATE TABLE acs5_demographics_tidy( + year VARCHAR, geoid VARCHAR, name VARCHAR, geo_type VARCHAR, + county VARCHAR, county_fips VARCHAR, section VARCHAR, + variable VARCHAR, value DOUBLE, value_moe DOUBLE, + value_reliability VARCHAR, percent DOUBLE, percent_moe DOUBLE, + percent_reliability VARCHAR + ); + INSERT INTO acs5_demographics_tidy VALUES + ('2023', '50', 'Vermont', 'state', NULL, NULL, 'Age/Sex', + 'Under 18', 117177, 1220, 'high', 18.2, 0.2, 'high'); + """) + return DataToolService(warehouse) + + +def test_margins_come_back_with_their_estimates(margins_service): + [variable] = margins_service.call( + "search_variables", {"dataset_id": "acs5_ts_median_age"} + )["variables"] + series = margins_service.call( + "query_data", + {"dataset_id": "acs5_ts_median_age", "variable_ids": [variable["variable_id"]]}, + ) + assert series["rows"][0]["median_age"] == 43.2 + assert series["rows"][0]["median_age_moe"] == 0.2 + assert series["units"] == {"median_age": "years", "median_age_moe": "years"} + + tidy = margins_service.call( + "query_data", {"dataset_id": "acs5_demographics", "measures": ["percent"]} + ) + assert "value" not in tidy["columns"] + assert (tidy["rows"][0]["percent"], tidy["rows"][0]["percent_moe"]) == (18.2, 0.2)