diff --git a/backend/.sqlfluff b/backend/.sqlfluff index f3740201..04036b76 100644 --- a/backend/.sqlfluff +++ b/backend/.sqlfluff @@ -38,3 +38,4 @@ rule_string = CAST("Residential_Min_Lot" AS VARCHAR) AS residential_min_lot indicators = 'ASTHMA', 'DIABETES' path = dummy.csv min_acres = 10 +margins = True diff --git a/backend/api/routes/post_routes/post_acs5_db.py b/backend/api/routes/post_routes/post_acs5_db.py index cf78f00a..378e3a37 100644 --- a/backend/api/routes/post_routes/post_acs5_db.py +++ b/backend/api/routes/post_routes/post_acs5_db.py @@ -448,12 +448,14 @@ def dp_combined_tree(): # (see query/dp_variable_names.py). DP_IDENTITY_COLUMNS = ("variable_code", "source_label") +# Each value's 90% margin of error and reliability label, under the names the +# frontend reads beside Value. A warehouse built before margins were collected +# has neither, and responses then leave them out. +DP_MARGIN_COLUMNS = {"moe": "Value_MOE", "reliability": "Value_Reliability"} -def _dp_identity_columns() -> list[str]: - present = { - row[0] for row in DB.execute("DESCRIBE acs5_dp_combined_tidy").fetchall() - } - return [c for c in DP_IDENTITY_COLUMNS if c in present] + +def _dp_columns() -> set[str]: + return {row[0] for row in DB.execute("DESCRIBE acs5_dp_combined_tidy").fetchall()} @router.post("/load/acs5-db/dp-combined/series") @@ -465,8 +467,12 @@ def dp_combined_series(request: DPSeriesRequest): in a stable order so each keeps its place across years: by variable code when the warehouse has it, otherwise by load order. """ - identity = _dp_identity_columns() + present = _dp_columns() + identity = [c for c in DP_IDENTITY_COLUMNS if c in present] extra = "".join(f", {c}" for c in identity) + extra += "".join( + f", {c} AS {name}" for c, name in DP_MARGIN_COLUMNS.items() if c in present + ) tiebreak = "variable_code" if "variable_code" in identity else "rowid" rows = DB.execute( f"""--sql @@ -512,11 +518,12 @@ def county_comparison_indicators(): @router.post("/load/acs5-db/county-comparison") def county_comparison(request: CountyComparisonRequest): """Return {place, geo_type, year, value} for each county, Vermont and the - United States, every year the indicator has data.""" + United States, every year the indicator has data, with value_MOE and + value_Reliability where the warehouse has margins.""" indicator = INDICATORS_BY_KEY.get(request.indicator) if indicator is None or not indicator.available: raise HTTPException(404, f"Unknown indicator: {request.indicator}") - sql, params = comparison_query(indicator) + sql, params = comparison_query(indicator, margins="moe" in _dp_columns()) rows = DB.execute(sql, params).df() return make_response( data=rows, diff --git a/backend/query/acs5.py b/backend/query/acs5.py index f47e7aab..245bb811 100644 --- a/backend/query/acs5.py +++ b/backend/query/acs5.py @@ -126,6 +126,8 @@ # The cleaned timeseries tables store their measure columns in snake_case, but # the frontend charts read these Title_Case field names, so responses keep them. +# Where a series has a 90% margin of error, it follows as `_MOE`, with +# its reliability label (high, medium or low) as `_Reliability`. TIMESERIES_FIELD_NAMES = { "median_age": "Median_Age", "age_dependency_ratio": "Age_Dependency_Ratio", @@ -135,6 +137,26 @@ "median_home_value": "Median_Home_Value", "percent": "Percent", } +TIMESERIES_FIELD_NAMES |= { + f"{column}_{suffix.lower()}": f"{field}_{suffix}" + for column, field in TIMESERIES_FIELD_NAMES.items() + for suffix in ("MOE", "Reliability") +} + +# The tidy tables' columns the tidy endpoints return, under the Title Case +# names the frontend reads. A warehouse built before margins were collected +# has no margin columns; its responses just leave them out. +TIDY_FIELD_NAMES = { + "year": "year", + "section": "Section", + "variable": "Variable", + "value": "Value", + "value_moe": "Value_MOE", + "value_reliability": "Value_Reliability", + "percent": "Percent", + "percent_moe": "Percent_MOE", + "percent_reliability": "Percent_Reliability", +} def _acs5_source( @@ -172,6 +194,9 @@ def get_acs5_tidy(dataset: str, filters: dict | None = None) -> pd.DataFrame: sql, params = sql_filter_block(sql_path / "acs5_tidy.sql", [source]) result = DB.execute(sql, params).df() + result = result[[c for c in TIDY_FIELD_NAMES if c in result.columns]].rename( + columns=TIDY_FIELD_NAMES + ) if result.empty: logger.error( diff --git a/backend/query/comparison.py b/backend/query/comparison.py index b7746f4c..2aba0e5c 100644 --- a/backend/query/comparison.py +++ b/backend/query/comparison.py @@ -30,6 +30,8 @@ logger = logging.getLogger(__name__) sql_dir = Path(__file__).resolve().parent / "sql" +# A value's 90% margin of error and reliability label, where the source has them +MARGIN_COLUMNS = ("moe", "reliability") MIN_COMPLETE_ROWS = 3 # fewest geographies a composite index can be fit from # Census ACS sentinel for a suppressed/not-applicable estimate (e.g. median age # in a town too small to estimate). Never a real value, so drop it up front. @@ -223,6 +225,14 @@ def dataset_for_table(table: str) -> str: raise ValueError(f"unregistered comparison table: {table}") from e +def _has_margins(table: str) -> bool: + """Whether `table` carries ACS margins of error. A warehouse built before + margins were collected has none, and the ACS templates then leave their + `moe`/`reliability` columns out (CDC PLACES never has them).""" + columns = {row[0] for row in DB.execute(f"DESCRIBE {table}").fetchall()} + return "percent_moe" in columns + + def _fetch(cfg: dict, extra_filters: dict | None = None) -> pd.DataFrame: filters = dict(extra_filters or {}) if "geo_type" in cfg: @@ -230,7 +240,9 @@ def _fetch(cfg: dict, extra_filters: dict | None = None) -> pd.DataFrame: filters["year"] = [_latest_year(cfg["table"])] source = FilterSource(filter_table=cfg["table"], filters=filters) - sql, params = sql_filter_block(cfg["sql"], [source]) + sql, params = sql_filter_block( + cfg["sql"], [source], margins=_has_margins(cfg["table"]) + ) df = DB.execute(sql, params).df() df = df.rename( @@ -241,6 +253,9 @@ def _fetch(cfg: dict, extra_filters: dict | None = None) -> pd.DataFrame: cfg["name_col"]: "name", } ) + for column in MARGIN_COLUMNS: + if column not in df.columns: + df[column] = None return df[df["data_value"] != ACS_SENTINEL] @@ -261,7 +276,8 @@ def _single_variable( filters = dict(extra_filters or {}) filters[cfg["var_col"]] = [var] df = _fetch(cfg, filters) - sub = df[df["measure"] == var][["geoid", "geometry", "data_value", "name"]].copy() + columns = ["geoid", "geometry", "data_value", "name", *MARGIN_COLUMNS] + sub = df[df["measure"] == var][columns].copy() # Rows whose Jurisdiction/geo-id failed to match a geometry (a handful of # unparsed or non-geographic entries in the source tidy table) carry no # geoid at all. Left in, two such rows from either side of a comparison @@ -299,7 +315,8 @@ def compare_variables( its Crude and Age-adjusted rows, doubling every geography on the map). """ d1 = _single_variable(dataset1, level, var1, filters1) - d2 = _single_variable(dataset2, level, var2, filters2)[["geoid", "data_value"]] + d2 = _single_variable(dataset2, level, var2, filters2) + d2 = d2[["geoid", "data_value", *MARGIN_COLUMNS]] d1["bin"], edges_x = _bin3(d1["data_value"]) d2["bin"], edges_y = _bin3(d2["data_value"]) @@ -321,6 +338,20 @@ def compare_variables( var1: round(r.data_value_1, 2), var2: round(r.data_value_2, 2), } + margins = { + var: { + "moe": round(moe, 2), + "reliability": reliability if pd.notna(reliability) else None, + } + for var, moe, reliability in [ + (var1, r.moe_1, r.reliability_1), + (var2, r.moe_2, r.reliability_2), + ] + if pd.notna(moe) + } + if margins: + # Read by the map tooltip, which shows each value with its margin. + tooltip["__margins__"] = margins features.append( { "type": "Feature", diff --git a/backend/query/county_comparison.py b/backend/query/county_comparison.py index 2886a6e9..94d958af 100644 --- a/backend/query/county_comparison.py +++ b/backend/query/county_comparison.py @@ -19,11 +19,12 @@ GEO_TYPES = ("county", "state", "national") -def comparison_query(indicator: Indicator) -> tuple[str, list]: +def comparison_query(indicator: Indicator, margins: bool = False) -> tuple[str, list]: """Render the comparison query for one indicator; returns (sql, params). Rows are {place, geo_type, year, value}, where `place` is the short display - name ("Essex", "Vermont", "United States"). Values travel as $N + name ("Essex", "Vermont", "United States"), plus value_MOE and + value_Reliability when `margins` (the table has them). Values travel as $N placeholders: execute with `DB.execute(sql, params)`. """ params: list = [] @@ -46,4 +47,7 @@ def comparison_query(indicator: Indicator) -> tuple[str, list]: # with nulls, as query/acs5.py does. clauses.append("value >= 0") where_string = "WHERE " + "\n AND ".join(clauses) - return render_sql(SQL_PATH, table=SOURCE_TABLE, where_string=where_string), params + sql = render_sql( + SQL_PATH, table=SOURCE_TABLE, where_string=where_string, margins=margins + ) + return sql, params diff --git a/backend/query/sql/acs5/acs5_county_compare.sql b/backend/query/sql/acs5/acs5_county_compare.sql index 7e75262d..65a5098e 100644 --- a/backend/query/sql/acs5/acs5_county_compare.sql +++ b/backend/query/sql/acs5/acs5_county_compare.sql @@ -7,6 +7,17 @@ SELECT -- present; count-only variables (Population, Median Age, ...) have no -- Percent and fall back to Value unchanged. COALESCE(a.percent, a.value) AS value, + {% if margins %} + -- The margin and reliability label of whichever figure `value` is. A + -- warehouse built before margins were collected has neither column. + CASE + WHEN a.percent IS NOT NULL THEN a.percent_moe ELSE a.value_moe + END AS moe, + CASE + WHEN a.percent IS NOT NULL THEN a.percent_reliability + ELSE a.value_reliability + END AS reliability, + {% endif %} c.geoid, c.county, ST_ASGEOJSON(ST_GeomFromWKB(c.geometry)) AS geometry diff --git a/backend/query/sql/acs5/acs5_dp_county_comparison.sql b/backend/query/sql/acs5/acs5_dp_county_comparison.sql index b0836937..0b28a85f 100644 --- a/backend/query/sql/acs5/acs5_dp_county_comparison.sql +++ b/backend/query/sql/acs5/acs5_dp_county_comparison.sql @@ -9,7 +9,11 @@ SELECT REGEXP_REPLACE(name, ' County, Vermont$', '') AS place, geo_type, CAST(year AS INTEGER) AS year, - CAST(value AS DOUBLE) AS value + CAST(value AS DOUBLE) AS value{% if margins %}, + -- Named for the frontend, which reads a field's margin as _MOE. + moe AS value_MOE, + reliability AS value_Reliability +{% endif %} FROM {{ table }} {{ where_string }} ORDER BY year, place diff --git a/backend/query/sql/acs5/acs5_tidy.sql b/backend/query/sql/acs5/acs5_tidy.sql index 4154c65a..77a4fb34 100644 --- a/backend/query/sql/acs5/acs5_tidy.sql +++ b/backend/query/sql/acs5/acs5_tidy.sql @@ -1,11 +1,6 @@ --- The response keeps the Title Case field names the frontend reads; the tidy --- tables store them lowercase, and DuckDB would otherwise echo that casing. -SELECT - year, - section AS Section, - variable AS Variable, - value AS Value, - percent AS Percent +-- Every column, so the margin columns come along where the warehouse has +-- them; get_acs5_tidy keeps and renames the fields the frontend reads. +SELECT * FROM {{ table }} {{ where_string }} -ORDER BY year, Section, Variable +ORDER BY year, section, variable diff --git a/backend/query/sql/acs5/acs5_town_compare.sql b/backend/query/sql/acs5/acs5_town_compare.sql index eb727fc0..5680633b 100644 --- a/backend/query/sql/acs5/acs5_town_compare.sql +++ b/backend/query/sql/acs5/acs5_town_compare.sql @@ -3,6 +3,17 @@ SELECT -- see acs5_county_compare.sql: prefer the share-of-total (Percent) over -- the raw count (Value) when the variable has one. COALESCE(a.percent, a.value) AS value, + {% if margins %} + -- see acs5_county_compare.sql: the margin of whichever figure + -- `value` is. + CASE + WHEN a.percent IS NOT NULL THEN a.percent_moe ELSE a.value_moe + END AS moe, + CASE + WHEN a.percent IS NOT NULL THEN a.percent_reliability + ELSE a.value_reliability + END AS reliability, + {% endif %} t.FIPS_ID, t.TOWN_NAME, ST_ASGEOJSON(ST_GeomFromWKB(t.geometry)) AS geometry diff --git a/backend/query/sql/acs5/acs5_tract_compare.sql b/backend/query/sql/acs5/acs5_tract_compare.sql index b34abcf9..881829bd 100644 --- a/backend/query/sql/acs5/acs5_tract_compare.sql +++ b/backend/query/sql/acs5/acs5_tract_compare.sql @@ -3,6 +3,12 @@ SELECT a.variable, COALESCE(a.percent, a.value) AS value, + {% if margins %} + -- The share's margin and reliability label, where the warehouse has + -- them. + a.percent_moe AS moe, + a.percent_reliability AS reliability, + {% endif %} a.geoid, a.name, ST_ASGEOJSON(ST_GEOMFROMWKB(t.geometry)) AS geometry diff --git a/backend/query/sql_render.py b/backend/query/sql_render.py index 970ff751..417c7952 100644 --- a/backend/query/sql_render.py +++ b/backend/query/sql_render.py @@ -129,18 +129,21 @@ def compile_filters(sources: list[FilterSource], params: list) -> tuple[str, str return cte_block, join_block -def sql_filter_block(sql_path: Path, sources: list[FilterSource]) -> tuple[str, list]: +def sql_filter_block( + sql_path: Path, sources: list[FilterSource], **extra +) -> tuple[str, list]: """Render a query template with compiled filters; returns (sql, params). Only the fragments the template actually references are compiled, so `params` contains exactly the values behind the $N placeholders present in the SQL. + `extra` is passed to the template as is (e.g. a flag for optional columns). Execute with `DB.execute(sql, params)`. """ source_text = sql_path.read_text() used = meta.find_undeclared_variables(JINJA_ENV.parse(source_text)) params: list = [] - context: dict = {} + context: dict = dict(extra) if used & {"cte_filter_block", "join_filter_block"}: cte_block, join_block = compile_filters(sources, params) context["cte_filter_block"] = cte_block diff --git a/backend/tests/test_acs5_tidy_query.py b/backend/tests/test_acs5_tidy_query.py new file mode 100644 index 00000000..e3a35fdf --- /dev/null +++ b/backend/tests/test_acs5_tidy_query.py @@ -0,0 +1,69 @@ +""" +The tidy ACS endpoints' query, run against a small in-memory tidy table +instead of the warehouse: Title Case fields, with each estimate's margin of +error where the warehouse has one. +""" + +import importlib + +import duckdb +import pytest + +from query import production_db + +ESTIMATES = ( + "year BIGINT, geoid VARCHAR, name VARCHAR, section VARCHAR, " + "variable VARCHAR, value DOUBLE, percent DOUBLE" +) +MARGINS = ( + "value_moe DOUBLE, value_reliability VARCHAR, " + "percent_moe DOUBLE, percent_reliability VARCHAR" +) +ROW = (2023, "5000144350", "Middlebury town", "Race", "Asian", 150.0, 1.8) + + +@pytest.fixture +def warehouse(monkeypatch): + conn = duckdb.connect() + monkeypatch.setattr(production_db, "get_db", lambda: conn) + module = importlib.import_module("query.acs5") + monkeypatch.setattr(module, "DB", conn) + return module, conn + + +def test_margins_come_back_with_their_estimates(warehouse): + acs5, conn = warehouse + conn.execute(f"CREATE TABLE acs5_demographics_tidy ({ESTIMATES}, {MARGINS})") + conn.execute( + "INSERT INTO acs5_demographics_tidy VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [*ROW, 80.0, "low", 1.0, "low"], + ) + df = acs5.get_acs5_tidy("demographics", {"Location": "Middlebury town"}) + + assert df.to_dict("records") == [ + { + "year": 2023, + "Section": "Race", + "Variable": "Asian", + "Value": 150.0, + "Value_MOE": 80.0, + "Value_Reliability": "low", + "Percent": 1.8, + "Percent_MOE": 1.0, + "Percent_Reliability": "low", + } + ] + + +def test_warehouse_without_margins_still_serves_its_estimates(warehouse): + # Built before margins were collected: the endpoints must not fail on the + # missing columns while the ETL catches up. + acs5, conn = warehouse + conn.execute(f"CREATE TABLE acs5_demographics_tidy ({ESTIMATES})") + conn.execute( + "INSERT INTO acs5_demographics_tidy VALUES (?, ?, ?, ?, ?, ?, ?)", list(ROW) + ) + df = acs5.get_acs5_tidy("demographics", {"Location": "Middlebury town"}) + + assert list(df.columns) == ["year", "Section", "Variable", "Value", "Percent"] + assert df.iloc[0].Value == 150.0 diff --git a/backend/tests/test_comparison_margins.py b/backend/tests/test_comparison_margins.py new file mode 100644 index 00000000..f2ed1b45 --- /dev/null +++ b/backend/tests/test_comparison_margins.py @@ -0,0 +1,114 @@ +""" +Margins of error on the variable explorer's comparison map +(query.comparison.compare_variables), run against small in-memory tidy tables +instead of the warehouse: each town's tooltip carries the margin and +reliability label of both values, and a warehouse built before margins were +collected still serves its map without them. +""" + +import duckdb +import pytest + +from query import comparison + +TOWNS = [ + ("5000100001", "ADDISON"), + ("5000100002", "BRISTOL"), + ("5000100003", "LINCOLN"), +] + +# (geoid, value, percent, value_moe, percent_moe, value_reliability, +# percent_reliability) per variable. Unemployment Rate is a count with its +# rate in percent, so the map shows the rate's margin; Median Age has no +# percent, so it shows the count's. +UNEMPLOYMENT = [ + ("5000100001", 40.0, 3.1, 15.0, 1.2, "medium", "medium"), + ("5000100002", 25.0, 2.0, 20.0, 1.5, "low", "low"), + ("5000100003", 60.0, 4.4, 18.0, 1.0, "high", "high"), +] +MEDIAN_AGE = [ + ("5000100001", 41.2, None, 2.1, None, "high", None), + ("5000100002", 38.5, None, 3.4, None, "high", None), + ("5000100003", 45.0, None, 6.0, None, "high", None), +] + + +def _db(margins: bool) -> duckdb.DuckDBPyConnection: + con = duckdb.connect() + # The comparison queries build GeoJSON with the spatial extension, which a + # fresh environment (CI) doesn't have yet; installing is a no-op once it's + # there. + try: + con.install_extension("spatial") + con.load_extension("spatial") + except duckdb.Error as e: + pytest.skip(f"DuckDB spatial extension unavailable: {e}") + con.execute( + """ + CREATE TABLE vt_town_lines_geom AS + SELECT * FROM (VALUES + ('5000100001', 'ADDISON', ST_AsWKB(ST_Point(0, 0))), + ('5000100002', 'BRISTOL', ST_AsWKB(ST_Point(1, 0))), + ('5000100003', 'LINCOLN', ST_AsWKB(ST_Point(2, 0))) + ) AS t(FIPS_ID, TOWN_NAME, geometry) + """ + ) + margin_columns = ( + ", value_moe DOUBLE, percent_moe DOUBLE, value_reliability VARCHAR," + " percent_reliability VARCHAR" + if margins + else "" + ) + for table, variable, rows in [ + ("acs5_economics_tidy", "Unemployment Rate", UNEMPLOYMENT), + ("acs5_demographics_tidy", "Median Age", MEDIAN_AGE), + ]: + con.execute( + f""" + CREATE TABLE {table} ( + year VARCHAR, geo_type VARCHAR, geoid VARCHAR, variable VARCHAR, + value DOUBLE, percent DOUBLE{margin_columns} + ) + """ + ) + for row in rows: + values = ["2023", "town", row[0], variable, *row[1:3]] + if margins: + values += row[3:] + con.execute( + f"INSERT INTO {table} VALUES ({', '.join('?' * len(values))})", + values, + ) + return con + + +def _tooltips(monkeypatch, margins: bool) -> dict[str, dict]: + monkeypatch.setattr(comparison, "DB", _db(margins)) + geojson, _legend = comparison.compare_variables( + "economics", "town", "Unemployment Rate", "demographics", "Median Age" + ) + return { + f["properties"]["tooltip"]["__title__"]: f["properties"]["tooltip"] + for f in geojson["features"] + } + + +def test_tooltips_carry_each_values_margin(monkeypatch): + bristol = _tooltips(monkeypatch, margins=True)["Bristol"] + + assert bristol["Unemployment Rate"] == 2.0 + assert bristol["__margins__"] == { + "Unemployment Rate": {"moe": 1.5, "reliability": "low"}, + "Median Age": {"moe": 3.4, "reliability": "high"}, + } + + +def test_warehouse_without_margins_still_serves_its_map(monkeypatch): + tooltips = _tooltips(monkeypatch, margins=False) + + assert tooltips["Bristol"] == { + "__title__": "Bristol", + "Unemployment Rate": 2.0, + "Median Age": 38.5, + } + assert len(tooltips) == len(TOWNS) diff --git a/backend/tests/test_county_comparison.py b/backend/tests/test_county_comparison.py index a059567e..2c31e15a 100644 --- a/backend/tests/test_county_comparison.py +++ b/backend/tests/test_county_comparison.py @@ -349,3 +349,38 @@ def test_warehouse_has_16_places_every_year(monkeypatch, indicator): first = 2013 if INDICATORS_BY_KEY[indicator].topic == "Disability" else 2010 _expect(min(per_year), first, f"{indicator}: first year") _expect(max(per_year), 2024, f"{indicator}: last year") + + +def test_margins_come_with_their_values(monkeypatch): + conn = duckdb.connect() + conn.execute( + """ + CREATE TABLE acs5_dp_combined_tidy ( + year BIGINT, name VARCHAR, geo_type VARCHAR, "table" VARCHAR, + category VARCHAR, subcategory VARCHAR, variable VARCHAR, + measure VARCHAR, value DOUBLE, moe DOUBLE, reliability VARCHAR + ) + """ + ) + conn.execute( + "INSERT INTO acs5_dp_combined_tidy VALUES " + "(2020, ?, 'county', 'DP02', ?, ?, 'Bachelor''s degree or higher', " + "'Percent', 18.4, 2.6, 'high')", + [ESSEX, EDU, ADULTS], + ) + response = _client(monkeypatch, conn).post( + "/load/acs5-db/county-comparison", json={"indicator": "edu_bachelors_or_higher"} + ) + (row,) = response.json()["data"] + assert (row["value"], row["value_MOE"], row["value_Reliability"]) == ( + 18.4, + 2.6, + "high", + ) + + +def test_warehouse_without_margins_leaves_them_out(monkeypatch): + response = _client(monkeypatch).post( + "/load/acs5-db/county-comparison", json={"indicator": "edu_bachelors_or_higher"} + ) + assert all("value_MOE" not in row for row in response.json()["data"]) diff --git a/backend/tests/test_dp_series_route.py b/backend/tests/test_dp_series_route.py index 437d8064..4a6aed2e 100644 --- a/backend/tests/test_dp_series_route.py +++ b/backend/tests/test_dp_series_route.py @@ -29,6 +29,9 @@ (2012, "DP04_0100E", "Housing units with a mortgage", "1764"), ] +# Published margins, by variable code +MOES = {"DP04_0107E": 61.0, "DP04_0100E": 88.0} + REQUEST = { "name": BURLINGTON, "table": "DP04", @@ -41,9 +44,11 @@ } -def _client(monkeypatch, with_identity: bool) -> TestClient: +def _client(monkeypatch, with_identity: bool, with_margins=False) -> TestClient: conn = duckdb.connect() identity = ", variable_code VARCHAR, source_label VARCHAR" if with_identity else "" + if with_margins: + identity += ", moe DOUBLE, reliability VARCHAR" conn.execute( f""" CREATE TABLE acs5_dp_combined_tidy ( @@ -58,6 +63,8 @@ def _client(monkeypatch, with_identity: bool) -> TestClient: base += ["Estimate", value] if with_identity: base += [code, LABEL.format(part)] + if with_margins: + base += [MOES[code], "high"] conn.execute( f"INSERT INTO acs5_dp_combined_tidy VALUES ({', '.join('?' * len(base))})", base, @@ -129,3 +136,19 @@ def test_without_identity_columns_rows_keep_load_order(monkeypatch): assert "variable_code" not in data[0] # Same position each year, so the chart's nth line stays one observation. assert [r["Value"] for r in data] == [699, 1731, 737, 1764] + + +@pytest.mark.parametrize("with_margins", [True, False]) +def test_margins_come_with_their_values(monkeypatch, with_margins): + rows = _client(monkeypatch, True, with_margins).post( + "/load/acs5-db/dp-combined/series", json=REQUEST + ) + first = rows.json()["data"][0] + if with_margins: + assert (first["Value"], first["Value_MOE"], first["Value_Reliability"]) == ( + 1731, + 88.0, + "high", + ) + else: + assert "Value_MOE" not in first and "Value_Reliability" not in first diff --git a/backend/tests/test_sql_render.py b/backend/tests/test_sql_render.py index 34851b63..bffc812a 100644 --- a/backend/tests/test_sql_render.py +++ b/backend/tests/test_sql_render.py @@ -291,10 +291,13 @@ def test_templates_pass_sqlfluff(): assert violations == [], [str(v) for v in violations] +@pytest.mark.parametrize("margins", [True, False]) @pytest.mark.parametrize("template", sorted(QUERY_TEMPLATE_SOURCES)) -def test_rendered_query_sql_parses(template): +def test_rendered_query_sql_parses(template, margins): + # `margins` adds the ACS templates' optional margin columns; the rest + # ignore it. sql, _params = sql_filter_block( - BACKEND / template, QUERY_TEMPLATE_SOURCES[template] + BACKEND / template, QUERY_TEMPLATE_SOURCES[template], margins=margins ) _assert_parses(sql, template) diff --git a/frontend/src/app/data-comparison/county-comparison/page.tsx b/frontend/src/app/data-comparison/county-comparison/page.tsx index 11d13862..d7fc4cc4 100644 --- a/frontend/src/app/data-comparison/county-comparison/page.tsx +++ b/frontend/src/app/data-comparison/county-comparison/page.tsx @@ -45,6 +45,10 @@ interface ComparisonRow { geo_type: 'county' | 'state' | 'national'; year: number; value: number; + /** The value's 90% margin of error and reliability label, where the + * warehouse has them. */ + value_MOE?: number | null; + value_Reliability?: string | null; } interface ComparisonMeta { @@ -215,6 +219,8 @@ export default function CountyComparisonPage() { .map((r) => ({ place: r.place, value: r.value, + value_MOE: r.value_MOE ?? null, + value_Reliability: r.value_Reliability ?? null, color: barColor(r, highlighted), })), [rows, year, highlighted], diff --git a/frontend/src/app/data-comparison/dp-explorer/page.tsx b/frontend/src/app/data-comparison/dp-explorer/page.tsx index a6c1d2e1..2e6d9e12 100644 --- a/frontend/src/app/data-comparison/dp-explorer/page.tsx +++ b/frontend/src/app/data-comparison/dp-explorer/page.tsx @@ -38,6 +38,7 @@ import { BASE_API_URL } from '@/config'; import { ChartStack } from '@/components/Charts'; import { createChartItem } from '@/utils/itemFactory'; import { identityField, identityNames } from '@/components/Charts/seriesLines'; +import { useShowMargins } from '@/components/Charts/margins'; import county_town_names from '@/data/county_town_names.json'; import { DataRow } from '@/types/cachedCharts'; import { COLORS, FONTS } from '@/app/theme'; @@ -224,12 +225,18 @@ function ValueCard({ location, valueLabel, value, + margin, + lowReliability, accent, }: { location: string; /** Which of several values this is (see ValueRow). */ valueLabel?: string | null; value: string; + /** The value's margin of error, formatted, when it has one and margins + * are shown. */ + margin?: string | null; + lowReliability?: boolean; accent: string; }) { return ( @@ -266,6 +273,13 @@ function ValueCard({ > {value} + {(margin || lowReliability) && ( + + {[margin && `± ${margin}`, lowReliability && 'low reliability'] + .filter(Boolean) + .join(' · ')} + + )} ); } @@ -349,6 +363,7 @@ function StatusPanel({ export default function DPExplorerPage() { const { myLocation, comparison, yearMax } = useProfile(); + const showMargins = useShowMargins(); // --------------------------------------------------------------------------- // Side A defaults from profile myLocation @@ -635,6 +650,12 @@ export default function DPExplorerPage() { const fmtVal = (v: number | null) => v != null ? (isPercent ? `${v}%` : Number(v).toLocaleString()) : '—'; const valueRows = pairValues(pointsA, pointsB); + const fmtMoe = (row?: DataRow) => { + const m = row?.Value_MOE; + if (!showMargins || typeof m !== 'number') return null; + return isPercent ? `${m}%` : m.toLocaleString(); + }; + const isLow = (row?: DataRow) => row?.Value_Reliability === 'low'; // --------------------------------------------------------------------------- // Render @@ -954,6 +975,8 @@ export default function DPExplorerPage() { location={makeLabel(sideA)} valueLabel={row.label} value={fmtVal(a)} + margin={a != null ? fmtMoe(row.a) : null} + lowReliability={isLow(row.a)} accent={COLORS.spruce} /> @@ -970,6 +993,8 @@ export default function DPExplorerPage() { location={makeLabel(sideB)} valueLabel={row.label} value={fmtVal(b)} + margin={b != null ? fmtMoe(row.b) : null} + lowReliability={isLow(row.b)} accent={COLORS.amber} /> diff --git a/frontend/src/components/Charts/Bar.tsx b/frontend/src/components/Charts/Bar.tsx index 39e70f06..9a20cf5b 100644 --- a/frontend/src/components/Charts/Bar.tsx +++ b/frontend/src/components/Charts/Bar.tsx @@ -4,6 +4,7 @@ import { useMemo, useEffect } from 'react'; import { BarChart, Bar, + ErrorBar, XAxis, YAxis, CartesianGrid, @@ -28,11 +29,26 @@ import { ChartEvent, LegendItem, LegendElement, + Plugin, } from 'chart.js'; ChartJS.register(CategoryScale, LinearScale, BarElement, TooltipJS, LegendJS); import { ChartItem, DataRow } from '@/types/cachedCharts'; import { usePdfMode } from '@/contexts/PdfModeContext'; +import { + MarginNote, + anyLowReliability, + hasMargins, + interval, + intervalAt, + moeField, + moeKey, + plotValue, + reliabilityField, + reliabilityKey, + useShowMargins, + withMargin, +} from './margins'; import { Box, Group, @@ -53,6 +69,68 @@ const d3Schemes = d3 as unknown as Record; const EMPTY_ROWS: DataRow[] = []; const stableRows = (rows?: DataRow[]) => (rows?.length ? rows : EMPTY_ROWS); +// A bar dataset may carry `intervals`, each bar's margin-of-error range (see +// margins.tsx); this plugin draws them as capped whiskers over the bars. +type MarginDataset = { intervals?: ([number, number] | null)[] }; + +const marginWhiskers: Plugin<'bar'> = { + id: 'marginWhiskers', + // Chart.js scales the y-axis to the bars alone. Raise it to the tallest + // visible whisker here, on every layout, so hiding a series in the legend + // drops its whiskers' headroom too. + afterDataLimits(chartJS, { scale }) { + if (scale.id !== 'y') return; + chartJS.data.datasets.forEach((dataset, di) => { + const { intervals } = dataset as MarginDataset; + if (!intervals || !chartJS.isDatasetVisible(di)) return; + for (const range of intervals) + if (range && range[1] > scale.max) scale.max = range[1]; + }); + }, + afterDatasetsDraw(chartJS) { + const { ctx } = chartJS; + const y = chartJS.scales.y; + chartJS.data.datasets.forEach((dataset, di) => { + const { intervals } = dataset as MarginDataset; + if (!intervals || !chartJS.isDatasetVisible(di)) return; + ctx.save(); + ctx.strokeStyle = '#333'; + ctx.lineWidth = 1.5; + chartJS.getDatasetMeta(di).data.forEach((bar, i) => { + const range = intervals[i]; + if (!range) return; + const top = y.getPixelForValue(range[1]); + const bottom = y.getPixelForValue(range[0]); + const { width } = bar.getProps(['width'], true) as { width: number }; + const cap = Math.min(6, width / 4); + ctx.beginPath(); + ctx.moveTo(bar.x, top); + ctx.lineTo(bar.x, bottom); + ctx.moveTo(bar.x - cap, top); + ctx.lineTo(bar.x + cap, top); + ctx.moveTo(bar.x - cap, bottom); + ctx.lineTo(bar.x + cap, bottom); + ctx.stroke(); + }); + ctx.restore(); + }); + }, +}; +const MARGIN_PLUGINS = [marginWhiskers]; + +/** A row's margin as Recharts ErrorBar offsets below and above its value. */ +const errorOffsets = ( + row: DataRow | undefined, + field: string, + percent: boolean, +) => { + const value = row?.[field]; + const moe = row?.[moeField(field)]; + if (typeof value !== 'number' || typeof moe !== 'number') return null; + const [low, high] = interval(value, moe, percent); + return [value - low, high - value]; +}; + const SamePerXBarChart = ({ chart, view, @@ -249,14 +327,22 @@ const CompareDiffPerXBarChartSVG = ({ colors = filteredData.map((_, i) => colorScale(i.toString())); } + const percent = chart.chartParams.percentFormat ?? false; + const margins = + hasMargins(filteredData, chart.yField) || + hasMargins(filteredCompareData, chart.yField); + const show = useShowMargins(); + // Merge primary + compare into one array for grouped bars const merged = filteredData.map((entry, i) => ({ [chart.xField]: entry[chart.xField], primary: entry[chart.yField], compare: filteredCompareData[i]?.[chart.yField] ?? null, + primaryError: errorOffsets(entry, chart.yField, percent), + compareError: errorOffsets(filteredCompareData[i], chart.yField, percent), })); - return ( + const plot = ( ( ))} + {margins && show && ( + + )} + + + {margins && show && ( + + )} - ); + if (!margins || !show) return plot; + + return ( + + + {plot} + + ); }; const CompareDiffPerXBarChart = ({ @@ -322,19 +423,22 @@ const CompareDiffPerXBarChart = ({ return cmp.length ? alignByX(filteredData, cmp, chart.xField) : cmp; }, [compareRows, filteredData, includeCategories, chart.xField]); + const margins = + hasMargins(filteredData, chart.yField) || + hasMargins(filteredCompareData, chart.yField); + const show = useShowMargins(); + // Derive exact plottable rows and report back to ChartCard for TableView const plotData = useMemo(() => { return filteredData.map((entry, i) => { const cmpEntry = filteredCompareData[i]; - return { - [chart.xField]: entry[chart.xField], - [chart.yField]: entry[chart.yField], - ...(cmpEntry - ? { [`${chart.yField} (cmp)`]: cmpEntry[chart.yField] } - : {}), - }; + const pt: DataRow = { [chart.xField]: entry[chart.xField] }; + plotValue(pt, chart.yField, entry, chart.yField, margins); + if (cmpEntry) + plotValue(pt, `${chart.yField} (cmp)`, cmpEntry, chart.yField, margins); + return pt; }); - }, [filteredData, filteredCompareData, chart.xField, chart.yField]); + }, [filteredData, filteredCompareData, chart.xField, chart.yField, margins]); useEffect(() => { onPlotData?.(plotData); @@ -346,6 +450,7 @@ const CompareDiffPerXBarChart = ({ const primaryLabel = chart.chartParams?.legendLabels?.[0] ?? yField; const compareLabel = chart.chartParams?.legendLabels?.[1] ?? `${yField} (compare)`; + const percentFormat = chart.chartParams?.percentFormat ?? false; const data = useMemo(() => { let colors: string[]; @@ -365,11 +470,21 @@ const CompareDiffPerXBarChart = ({ label: primaryLabel, data: filteredData.map((entry) => entry[yField] as number), backgroundColor: colors, + intervals: + margins && show + ? plotData.map((pt) => intervalAt(pt, yField, percentFormat)) + : undefined, }, { label: compareLabel, data: filteredCompareData.map((entry) => entry[yField] as number), backgroundColor: filteredCompareData.map(() => '#D3D3D3'), + intervals: + margins && show + ? plotData.map((pt) => + intervalAt(pt, `${yField} (cmp)`, percentFormat), + ) + : undefined, }, ], }; @@ -383,12 +498,17 @@ const CompareDiffPerXBarChart = ({ colorScheme, primaryLabel, compareLabel, + margins, + show, + plotData, + percentFormat, ]); - const percentFormat = chart.chartParams?.percentFormat ?? false; const yLabel = chart.chartParams?.yLabel; - const options = useMemo( - () => ({ + const options = useMemo(() => { + const format = (v: number) => + percentFormat ? `${v}%` : v.toLocaleString(); + return { responsive: true, maintainAspectRatio: false, animation: view === 'gallery' ? (false as const) : undefined, @@ -397,8 +517,17 @@ const CompareDiffPerXBarChart = ({ tooltip: { callbacks: { label: (context: TooltipItem<'bar'>) => { - const value = context.parsed.y ?? 0; - return percentFormat ? `${value}%` : value.toLocaleString(); + const key = + context.datasetIndex === 0 ? yField : `${yField} (cmp)`; + const pt = plotData[context.dataIndex]; + return withMargin( + format(context.parsed.y ?? 0), + pt?.[moeKey(key)], + pt?.[reliabilityKey(key)], + format, + ' (low reliability)', + show, + ); }, }, }, @@ -412,13 +541,26 @@ const CompareDiffPerXBarChart = ({ }, }, }, - }), - [view, percentFormat, yLabel], - ); + }; + }, [view, percentFormat, yLabel, plotData, yField, show]); if (isPdfMode) return ; + if (!margins) return ; - return ; + return ( + + {view !== 'gallery' && ( + + )} + + + + + ); }; type AllowanceRow = { @@ -818,6 +960,9 @@ const RankedBarChart = ({ const colorField = chart.chartParams?.color ?? 'color'; const unit = chart.chartParams?.unit; const yLabel = (chart.chartParams?.yLabel as string | undefined) ?? yField; + const percent = unit === 'percent'; + const margins = hasMargins(rows, yField); + const show = useShowMargins(); const fmt = (v: unknown, short = false) => { if (v == null) return ''; const n = Number(v); @@ -826,18 +971,66 @@ const RankedBarChart = ({ ? `$${(n / 1000).toLocaleString(undefined, { maximumFractionDigits: 1 })}k` : `$${n.toLocaleString(undefined, { maximumFractionDigits: 0 })}`; const num = n.toLocaleString(undefined, { maximumFractionDigits: 1 }); - return unit === 'percent' ? `${num}%` : num; + return percent ? `${num}%` : num; }; const fmtShort = (v: unknown) => fmt(v, true); + const plotRows: Record[] = useMemo( + () => + margins && show + ? rows.map((r) => ({ ...r, error: errorOffsets(r, yField, percent) })) + : rows, + [rows, margins, show, yField, percent], + ); - // The table view shows place and value only, not the bar colors. + // The table view shows place and value only, not the bar colors, with the + // value's margin where it has one. useEffect(() => { - onPlotData?.(rows.map((r) => ({ Place: r[xField], [yLabel]: r[yField] }))); - }, [rows, xField, yField, yLabel, onPlotData]); + onPlotData?.( + rows.map((r) => ({ + Place: r[xField], + [yLabel]: r[yField], + ...(margins && { + [moeKey(yLabel)]: r[moeField(yField)] ?? null, + [reliabilityKey(yLabel)]: r[reliabilityField(yField)] ?? null, + }), + })), + ); + }, [rows, xField, yField, yLabel, margins, onPlotData]); + + // Each bar's value sits above its whisker, not under it. + const valueLabel = (props: { + x?: unknown; + y?: unknown; + width?: unknown; + height?: unknown; + value?: unknown; + index?: number; + }) => { + const x = Number(props.x); + const y = Number(props.y); + const width = Number(props.width); + const height = Number(props.height); + const error = plotRows[props.index ?? -1]?.error as number[] | null; + const value = Number(props.value); + const above = error && value > 0 ? (error[1] / value) * height : 0; + return ( + + {fmtShort(props.value)} + + ); + }; - return ( + const plot = ( - + - [fmt(v), yLabel]} /> + [ + withMargin( + fmt(v), + item.payload?.[moeField(yField)], + item.payload?.[reliabilityField(yField)], + (m) => fmt(m), + ' (low reliability)', + show, + ), + yLabel, + ]} + /> ( ))} - + {margins && show && ( + + )} + ); + if (!margins || view === 'gallery') return plot; + + return ( + + r[reliabilityField(yField)] === 'low')} + bars + show={show} + /> + {plot} + + ); }; export { diff --git a/frontend/src/components/Charts/DemographicsTable.tsx b/frontend/src/components/Charts/DemographicsTable.tsx index 39754ba3..3f16f647 100644 --- a/frontend/src/components/Charts/DemographicsTable.tsx +++ b/frontend/src/components/Charts/DemographicsTable.tsx @@ -5,6 +5,15 @@ import { ChartItem, DataRow } from '@/types/cachedCharts'; import { COLORS } from '@/app/theme'; import * as XLSX from 'xlsx'; import { saveWorksheet } from './TableView'; +import { + FieldOf, + MarginFootnote, + cellCompanion, + cellMarginFlags, + cellText, + moeField, + useShowMargins, +} from './margins'; // tidy ACS row shape consumed by the demographics tables interface AcsRow extends DataRow { @@ -20,18 +29,22 @@ const SPLIT_BORDER = '1px solid var(--mantine-color-gray-3)'; const DemographicsTableBase = ({ chart, - renderCell, - exportValue, + fieldOf, }: { chart: ChartItem; - renderCell: (row: AcsRow | undefined) => React.ReactNode; - /** The number behind a cell, for the Excel export. */ - exportValue: (row: AcsRow | undefined) => number | null; + fieldOf: FieldOf; }) => { const data = chart.data; const compareData = chart.compareData ?? []; const hasCompare = compareData.length > 0; const [showCompare, setShowCompare] = useState(false); + const show = useShowMargins(); + + // The export always writes the comparison columns, but the footnote + // explains only the cells on screen. + const allRows = [...data, ...compareData]; + const exported = cellMarginFlags(allRows, fieldOf); + const shown = cellMarginFlags(showCompare ? allRows : data, fieldOf); const labels = chart.chartParams?.legendLabels as [string, string] | undefined; @@ -48,21 +61,32 @@ const DemographicsTableBase = ({ ) => rows.find((r) => r.Variable === variable && r.year === year); // Mirrors the table: a "Variable" column, then one column per year that - // splits in two (main place | comparison) under a merged year header. + // splits in two (main place | comparison) under a merged year header. With + // margins, each variable's row is followed by a row of its margins. const exportTable = () => { const sub = hasCompare ? [homeLabel, compareLabel] : []; const width = hasCompare ? 2 : 1; - const num = (r: AcsRow | undefined) => exportValue(r) ?? ''; + const num = (r: AcsRow | undefined) => r?.[fieldOf(r)] ?? ''; + const moe = (r: AcsRow | undefined) => + cellCompanion(r, fieldOf, moeField) ?? ''; const header = [ 'Variable', ...years.flatMap((y) => (hasCompare ? [y, ''] : [y])), ]; - const body = variables.map((variable) => [ - variable, + const line = ( + label: string, + cell: (r: AcsRow | undefined) => unknown, + variable?: string, + ) => [ + label, ...years.flatMap((year) => [ - num(findRow(data, variable, year)), - ...(hasCompare ? [num(findRow(compareData, variable, year))] : []), + cell(findRow(data, variable, year)), + ...(hasCompare ? [cell(findRow(compareData, variable, year))] : []), ]), + ]; + const body = variables.flatMap((variable) => [ + line(String(variable), num, variable), + ...(exported.margins ? [line(`${variable} MOE`, moe, variable)] : []), ]); const rows = hasCompare ? [header, ['', ...years.flatMap(() => sub)], ...body] @@ -150,7 +174,9 @@ const DemographicsTableBase = ({ const mainRow = findRow(data, variable, year); if (!showCompare) { return ( - {renderCell(mainRow)} + + {cellText(mainRow, fieldOf, show)} + ); } const cmpRow = findRow(compareData, variable, year); @@ -165,7 +191,7 @@ const DemographicsTableBase = ({ textAlign: 'right', }} > - {renderCell(mainRow)} + {cellText(mainRow, fieldOf, show)}
- {renderCell(cmpRow)} + {cellText(cmpRow, fieldOf, show)}
@@ -187,43 +213,27 @@ const DemographicsTableBase = ({ + {shown.margins && ( + + )} ); }; export const renderTable = ({ chart }: { chart: ChartItem }) => ( - - row?.Percent != null ? `${row.Percent.toFixed(1)}%` : '—' - } - exportValue={(row) => row?.Percent ?? null} - /> + 'Percent'} /> ); export const renderTableEstimates = ({ chart, }: { chart: ChartItem; -}) => ( - - row?.Value != null ? row.Value.toLocaleString() : '—' - } - exportValue={(row) => row?.Value ?? null} - /> -); +}) => 'Value'} />; /** Shows Percent when available, falls back to Value — for mixed tables like Housing. */ export const renderTableMixed = ({ chart }: { chart: ChartItem }) => ( { - if (row?.Percent != null) return `${row.Percent.toFixed(1)}%`; - if (row?.Value != null) return row.Value.toLocaleString(); - return '—'; - }} - exportValue={(row) => row?.Percent ?? row?.Value ?? null} + fieldOf={(row) => (row?.Percent != null ? 'Percent' : 'Value')} /> ); diff --git a/frontend/src/components/Charts/MapCorrespondentScatter.tsx b/frontend/src/components/Charts/MapCorrespondentScatter.tsx index e5a95e46..d80714dc 100644 --- a/frontend/src/components/Charts/MapCorrespondentScatter.tsx +++ b/frontend/src/components/Charts/MapCorrespondentScatter.tsx @@ -19,6 +19,11 @@ import { ResponsiveContainer, } from 'recharts'; import { COLORS } from '@/app/theme'; +import { + isTooltipMeta, + tooltipLowReliability, + withTooltipMargin, +} from './margins'; type Legend = { measures: [string, string]; @@ -53,18 +58,21 @@ function makeScatterDot(activeId: string | null) { if (!Number.isFinite(cx) || !Number.isFinite(cy)) return null; const isActive = activeId != null && String(tooltip?.__title__ ?? '') === activeId; + // Hollow, in its map color, when either value has low reliability, as on + // the trend charts. + const low = !!tooltip && tooltipLowReliability(tooltip); return ( ); }; @@ -78,9 +86,9 @@ function PointTooltip({ content }: { content: Record }) { {Object.entries(content).map( ([k, v]) => - k !== '__title__' && ( + !isTooltipMeta(k) && (
- {k}: {String(v)} + {k}: {String(withTooltipMargin(content, k, v))}
), )} @@ -123,12 +131,19 @@ export default function VariableScatter({ .filter((p) => Number.isFinite(p.x) && Number.isFinite(p.y)); if (points.length === 0) return null; + const anyLow = points.some((p) => tooltipLowReliability(p.tooltip)); return ( - + {my} vs. {mx} + {anyLow && ( + + Hollow dots mark places where either estimate has low reliability: its + margin of error is about half the estimate or more. + + )} diff --git a/frontend/src/components/Charts/TableView.tsx b/frontend/src/components/Charts/TableView.tsx index 583df38c..4c3757c4 100644 --- a/frontend/src/components/Charts/TableView.tsx +++ b/frontend/src/components/Charts/TableView.tsx @@ -15,6 +15,15 @@ import { import { ChartItem, DataRow } from '@/types/cachedCharts'; import { COLORS } from '@/app/theme'; import { FileXlsIcon } from '@phosphor-icons/react'; +import { + MarginFootnote, + anyLowReliability, + isMarginKey, + moeKey, + reliabilityKey, + useShowMargins, + withMargin, +} from './margins'; interface TableViewProps { chart: ChartItem; @@ -26,13 +35,30 @@ const COMP_BG = 'var(--mantine-color-blue-0)'; const SPLIT_BORDER = '1px solid var(--mantine-color-gray-3)'; const CMP_SUFFIX = ' (cmp)'; +const formatNumber = (v: number) => + v.toLocaleString(undefined, { maximumFractionDigits: 1 }); + const formatCell = (v: unknown) => { if (v == null) return '—'; - if (typeof v === 'number') - return v.toLocaleString(undefined, { maximumFractionDigits: 1 }); + if (typeof v === 'number') return formatNumber(v); return String(v); }; +// A value's cell, with its margin of error (see margins.tsx) when the row +// has one and margins are shown, and an asterisk for a low-reliability +// estimate. +const valueCell = (row: DataRow | undefined, key: string, show: boolean) => + row?.[key] == null + ? formatCell(row?.[key]) + : withMargin( + formatCell(row[key]), + row[moeKey(key)], + row[reliabilityKey(key)], + formatNumber, + '*', + show, + ); + /** Write a worksheet to a one-sheet .xlsx named after the chart and download it. */ export function saveWorksheet( chart: ChartItem, @@ -104,6 +130,7 @@ export const TableView = ({ }: TableViewProps) => { const rows = (rowsOverride ?? chart.tableData ?? chart.data) as DataRow[]; const [showCompare, setShowCompare] = useState(false); + const show = useShowMargins(); const usePivot = !!chart.trendChart; if (!rows || rows.length === 0) return null; @@ -118,6 +145,21 @@ export const TableView = ({ const cmpKeys = allKeys.filter((k) => k.endsWith(CMP_SUFFIX)); const hasCompare = cmpKeys.length > 0 && rows.some((r) => cmpKeys.some((k) => r[k] != null)); + // The footnote explains only the cells on screen, and comparison cells stay + // hidden until "Show Comparison". + const withCmp = hasCompare && showCompare; + const margins = rows.some((r) => + Object.entries(r).some( + ([k, v]) => + v != null && isMarginKey(k) && (withCmp || !k.endsWith(CMP_SUFFIX)), + ), + ); + const footnote = margins && ( + + ); const handleExport = () => exportRowsToExcel(chart, rows); @@ -184,7 +226,7 @@ export const TableView = ({ ); const seriesKeys = allKeys.filter( - (k) => k !== xKey && !k.endsWith(CMP_SUFFIX), + (k) => k !== xKey && !k.endsWith(CMP_SUFFIX) && !isMarginKey(k), ); const findRow = (x: unknown) => rows.find((r) => r[xKey] === x); @@ -208,15 +250,13 @@ export const TableView = ({ {key} {xValues.map((x) => { const row = findRow(x); - const mainVal = row?.[key]; if (!showCompare || !hasCompare) { return ( - {formatCell(mainVal)} + {valueCell(row, key, show)} ); } - const cmpVal = row?.[`${key}${CMP_SUFFIX}`]; return (
@@ -228,7 +268,7 @@ export const TableView = ({ textAlign: 'right', }} > - {formatCell(mainVal)} + {valueCell(row, key, show)}
({ textAlign: 'right', }} > - {formatCell(cmpVal)} + {valueCell(row, `${key}${CMP_SUFFIX}`, show)}
@@ -250,13 +290,16 @@ export const TableView = ({ + {footnote} ); } // --- MODE B: DIRECT TABLE (FOR BAR CHARTS) --- // Determine standard columns vs comparison columns - const baseColumns = allKeys.filter((k) => !k.endsWith(CMP_SUFFIX)); + const baseColumns = allKeys.filter( + (k) => !k.endsWith(CMP_SUFFIX) && !isMarginKey(k), + ); return ( @@ -275,7 +318,6 @@ export const TableView = ({ {rows.map((row, i) => ( {baseColumns.map((column) => { - const mainVal = row[column]; const cmpVal = row[`${column}${CMP_SUFFIX}`]; const isComparableCell = hasCompare && showCompare && cmpVal != null; @@ -292,7 +334,7 @@ export const TableView = ({ textAlign: 'right', }} > - {formatCell(mainVal)} + {valueCell(row, column, show)}
({ textAlign: 'right', }} > - {formatCell(cmpVal)} + {valueCell(row, `${column}${CMP_SUFFIX}`, show)}
@@ -311,7 +353,9 @@ export const TableView = ({ } return ( - {formatCell(mainVal)} + + {valueCell(row, column, show)} + ); })} @@ -319,6 +363,7 @@ export const TableView = ({ + {footnote} ); }; diff --git a/frontend/src/components/Charts/TrendCharts.tsx b/frontend/src/components/Charts/TrendCharts.tsx index bcee6c37..cd5793b0 100644 --- a/frontend/src/components/Charts/TrendCharts.tsx +++ b/frontend/src/components/Charts/TrendCharts.tsx @@ -4,9 +4,9 @@ import { useState, useEffect, useMemo } from 'react'; import { CartesianGrid, + ComposedChart, Legend, Line, - LineChart, ResponsiveContainer, Tooltip, XAxis, @@ -16,6 +16,21 @@ import { ChartItem, DataRow } from '@/types/cachedCharts'; import { usePdfMode } from '@/contexts/PdfModeContext'; import MultiRowNotice from './MultiRowNotice'; import { lineName, splitIntoLines } from './seriesLines'; +import { + MarginNote, + anyLowReliability, + hasMargins, + marginBand, + moeField, + moeKey, + plotValue, + reliabilityDots, + reliabilityKey, + reliabilityOf, + sumMoe, + useShowMargins, + withMargin, +} from './margins'; // tidy ACS-style row consumed by the trend charts interface TrendRow extends DataRow { @@ -45,6 +60,9 @@ const FORMATTERS: Record< unit?: string; axisFormatter?: (v: number) => string; tooltip: (v: unknown, decimals?: number) => string; + /** A margin of error, shown after the value as "value ± margin" at the + * value's precision. */ + margin: (moe: number, decimals?: number) => string; } > = { currency: { @@ -53,6 +71,8 @@ const FORMATTERS: Record< v != null ? `$${Number(v).toLocaleString('en-US', { maximumFractionDigits: 0 })}` : '—', + margin: (m) => + `$${m.toLocaleString('en-US', { maximumFractionDigits: 0 })}`, }, percent: { unit: '%', @@ -60,17 +80,40 @@ const FORMATTERS: Record< v != null ? `${decimals != null ? Number(v).toFixed(decimals) : v}%` : '—', + margin: (m, decimals) => `${decimals != null ? m.toFixed(decimals) : m}%`, }, number: { axisFormatter: (v) => `${(v / 1000).toFixed(0)}k`, tooltip: (v) => (v != null ? Number(v).toLocaleString() : '—'), + margin: (m) => m.toLocaleString('en-US', { maximumFractionDigits: 0 }), }, years: { axisFormatter: (v) => Number(v).toFixed(0), tooltip: (v) => (v != null ? `${Number(v).toFixed(1)} years` : '—'), + margin: (m) => m.toFixed(1), }, }; +/** Tooltip text for a line's value: its margin follows it when the plot + * point has one, and a low-reliability estimate says so. */ +const tooltipText = ( + fmt: (typeof FORMATTERS)[FormatType], + value: unknown, + item: { dataKey?: unknown; payload?: DataRow }, + show: boolean, + decimals?: number, +) => { + const key = String(item.dataKey); + return withMargin( + fmt.tooltip(value, decimals), + item.payload?.[moeKey(key)], + item.payload?.[reliabilityKey(key)], + (m) => fmt.margin(m, decimals), + ' (low reliability)', + show, + ); +}; + const useToggle = () => { const [hidden, setHidden] = useState>(new Set()); const toggleSeries = (key: string) => @@ -180,23 +223,39 @@ export const SingleSeriesTrendChart = ({ [data], ); + const margins = + hasMargins(data, valueField) || hasMargins(compareData, valueField); + const show = useShowMargins(); + const plotData = useMemo(() => { if (!data || data.length === 0) return []; - const valueAt = (rows: DataRow[], year: unknown) => - rows.find((r) => r.year === year)?.[valueField] ?? null; + const rowAt = (rows: DataRow[], year: unknown) => + rows.find((r) => r.year === year); return years.map((year) => { const pt: DataRow = { year }; own.lines.forEach((line, i) => { - pt[lineKey(seriesName, i)] = valueAt(line.rows, year); + plotValue( + pt, + lineKey(seriesName, i), + rowAt(line.rows, year), + valueField, + margins, + ); }); if (compareData && compareData.length > 0) { other.lines.forEach((line, i) => { - pt[`${lineKey(seriesName, i)} (cmp)`] = valueAt(line.rows, year); + plotValue( + pt, + `${lineKey(seriesName, i)} (cmp)`, + rowAt(line.rows, year), + valueField, + margins, + ); }); } return pt; }); - }, [years, data, compareData, own, other, seriesName, valueField]); + }, [years, data, compareData, own, other, seriesName, valueField, margins]); useEffect(() => { onPlotData?.(plotData); @@ -205,6 +264,8 @@ export const SingleSeriesTrendChart = ({ if (!data || data.length === 0) return null; // early return now AFTER all hooks const fmt = FORMATTERS[format]; + const percent = format === 'percent'; + const hasCompare = compareData.length > 0; return ( <> @@ -218,8 +279,11 @@ export const SingleSeriesTrendChart = ({ Click legend items to show or hide locations. )} + {margins && !isGallery && ( + + )} - ({ tickFormatter={fmt.axisFormatter} /> {!isGallery && ( - fmt.tooltip(v, decimals)} /> + + tooltipText(fmt, v, item, show, decimals) + } + /> )} {!isGallery && ( ({ wrapperStyle={{ fontSize: isGallery ? 12 : 16 }} /> )} + {margins && + show && + own.lines.map((_, i) => + marginBand({ + dataKey: ownKey(i), + color, + percent, + hide: hidden.has(ownKey(i)), + type: 'monotone', + }), + )} + {margins && + show && + hasCompare && + other.lines.map((_, i) => + marginBand({ + dataKey: otherKey(i), + color: compareColor, + percent, + hide: hidden.has(otherKey(i)), + type: 'monotone', + }), + )} {own.lines.map((_, i) => ( ({ stroke={color} strokeWidth={lineWidth} strokeDasharray={SPLIT_DASHES[i % SPLIT_DASHES.length]} - dot={own.lines.length > 1} + dot={ + margins + ? reliabilityDots(ownKey(i), color, own.lines.length > 1) + : own.lines.length > 1 + } animationBegin={0} isAnimationActive={!isGallery && !isPdfMode} animationDuration={!isGallery ? 1500 : 0} hide={hidden.has(ownKey(i))} /> ))} - {compareData.length > 0 && + {hasCompare && other.lines.map((_, i) => ( ({ stroke={compareColor} strokeWidth={lineWidth} strokeDasharray={SPLIT_DASHES[i % SPLIT_DASHES.length]} - dot={other.lines.length > 1} + dot={ + margins + ? reliabilityDots( + otherKey(i), + compareColor, + other.lines.length > 1, + ) + : other.lines.length > 1 + } animationBegin={0} isAnimationActive={!isGallery && !isPdfMode} animationDuration={!isGallery ? 1500 : 0} hide={hidden.has(otherKey(i))} /> ))} - + ); @@ -328,27 +431,31 @@ export interface MultiSeriesConfig { nameSuffix?: boolean; } -// The series' value for a year; summed series add up their matches. -const getValue = ( +// Puts a summed series' value for a year on a plot point (Demographics 65+), +// with the sum's margin when the chart has margins. +const plotSum = ( + pt: DataRow, + key: string, rows: TrendRow[], year: number, s: SeriesDef, valueField: string, + margins: boolean, ) => { - if (s.aggregateFrom) { - const sum = s.aggregateFrom.reduce((acc, label) => { - const v = (rows.find((r) => r.year === year && r.Variable === label)?.[ - valueField - ] ?? 0) as number; - return acc + v; - }, 0); - return sum > 0 ? Math.round(sum * 10) / 10 : null; - } - const label = s.matchVariable ?? s.key; - return ( - rows.find((r) => r.year === year && r.Variable === label)?.[valueField] ?? - null + const parts = s.aggregateFrom!.map((label) => + rows.find((r) => r.year === year && r.Variable === label), ); + const sum = parts.reduce( + (acc, r) => acc + ((r?.[valueField] ?? 0) as number), + 0, + ); + const value = sum > 0 ? Math.round(sum * 10) / 10 : null; + pt[key] = value; + if (!margins) return; + const moe = + value == null ? null : sumMoe(parts.map((r) => r?.[moeField(valueField)])); + pt[moeKey(key)] = moe == null ? null : Math.round(moe * 10) / 10; + pt[reliabilityKey(key)] = reliabilityOf(value, moe); }; export const MultiSeriesTrendChart = ({ @@ -420,30 +527,44 @@ export const MultiSeriesTrendChart = ({ [data], ); + const margins = + hasMargins(data, valueField) || hasMargins(compareData, valueField); + const show = useShowMargins(); + const plotData = useMemo(() => { if (!data || data.length === 0) return []; return years.map((year) => { const pt: DataRow = { year }; series.forEach((s, si) => { - pt[s.key] = getValue(data, year, s, valueField); - if (compareData && compareData.length > 0) - pt[`${s.key} (cmp)`] = getValue(compareData, year, s, valueField); - if (s.aggregateFrom) return; + if (s.aggregateFrom) { + plotSum(pt, s.key, data, year, s, valueField, margins); + if (compareData && compareData.length > 0) + plotSum( + pt, + `${s.key} (cmp)`, + compareData, + year, + s, + valueField, + margins, + ); + return; + } const { own, other } = splitSeries[si]; const at = (line: { rows: DataRow[] }) => - line.rows.find((r) => r.year === year)?.[valueField] ?? null; + line.rows.find((r) => r.year === year); own.lines.forEach((line, i) => { - if (i === 0) pt[s.key] = at(line); - else pt[extraKey(s, i)] = at(line); + const key = i === 0 ? s.key : extraKey(s, i); + plotValue(pt, key, at(line), valueField, margins); }); other.lines.forEach((line, i) => { - if (i === 0) pt[`${s.key} (cmp)`] = at(line); - else pt[`${extraKey(s, i)} (cmp)`] = at(line); + const key = i === 0 ? s.key : extraKey(s, i); + plotValue(pt, `${key} (cmp)`, at(line), valueField, margins); }); }); return pt; }); - }, [years, data, compareData, series, valueField, splitSeries]); + }, [years, data, compareData, series, valueField, splitSeries, margins]); useEffect(() => { onPlotData?.(plotData); @@ -452,6 +573,8 @@ export const MultiSeriesTrendChart = ({ if (!data || data.length === 0) return null; const fmt = FORMATTERS[format]; + const percent = format === 'percent'; + const hasCompare = compareData.length > 0; return ( <> @@ -460,7 +583,7 @@ export const MultiSeriesTrendChart = ({ maxPerYear={maxPerYear} duplicates={duplicates} /> - {compareData.length > 0 && !isGallery && ( + {hasCompare && !isGallery && ( )} {!isGallery && ( @@ -468,8 +591,11 @@ export const MultiSeriesTrendChart = ({ Click legend items to show or hide categories. )} + {margins && !isGallery && ( + + )} - @@ -485,7 +611,11 @@ export const MultiSeriesTrendChart = ({ domain={['auto', 'auto']} tickFormatter={fmt.axisFormatter} /> - {!isGallery && fmt.tooltip(v)} />} + {!isGallery && ( + tooltipText(fmt, v, item, show)} + /> + )} {!isGallery && ( ({ wrapperStyle={{ fontSize: isGallery ? 12 : 16 }} /> )} + {margins && + show && + series.flatMap((s) => [ + marginBand({ + dataKey: s.key, + color: s.color, + percent, + hide: hidden.has(s.key), + }), + ...(hasCompare + ? [ + marginBand({ + dataKey: `${s.key} (cmp)`, + color: s.color, + percent, + hide: hidden.has(s.key), + }), + ] + : []), + ])} {series.map((s) => ( ({ } stroke={s.color} strokeWidth={2} - dot={false} + dot={margins && reliabilityDots(s.key, s.color, false)} isAnimationActive={!isGallery && !isPdfMode} animationDuration={!isGallery ? 1500 : 0} hide={hidden.has(s.key)} @@ -531,7 +681,7 @@ export const MultiSeriesTrendChart = ({ /> )), )} - {compareData.length > 0 && + {hasCompare && series.flatMap((s, si) => splitSeries[si].other.lines .slice(1) @@ -551,7 +701,7 @@ export const MultiSeriesTrendChart = ({ /> )), )} - {compareData.length > 0 && + {hasCompare && series.map((s) => ( ({ legendType="none" isAnimationActive={!isGallery && !isPdfMode} animationDuration={!isGallery ? 1500 : 0} - dot={false} + dot={ + margins && reliabilityDots(`${s.key} (cmp)`, s.color, false) + } hide={hidden.has(s.key)} /> ))} - + ); @@ -1003,13 +1155,18 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { new Set([...data, ...compareData].map((r) => r.year)), ).sort((a, b) => Number(a) - Number(b)); + const margins = hasMargins(data, 'Value') || hasMargins(compareData, 'Value'); + const show = useShowMargins(); + const plotData = allYears.map((year) => { const pt: DataRow = { year }; primary.lines.forEach((line, i) => { - pt[`primary${i}`] = line.rows.find((r) => r.year === year)?.Value ?? null; + const row = line.rows.find((r) => r.year === year); + plotValue(pt, `primary${i}`, row, 'Value', margins); }); compare.lines.forEach((line, i) => { - pt[`compare${i}`] = line.rows.find((r) => r.year === year)?.Value ?? null; + const row = line.rows.find((r) => r.year === year); + plotValue(pt, `compare${i}`, row, 'Value', margins); }); return pt; }); @@ -1023,8 +1180,11 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { maxPerYear={Math.max(primary.maxPerX, compare.maxPerX)} duplicates={primary.duplicates + compare.duplicates} /> + {margins && ( + + )} - @@ -1038,11 +1198,40 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { domain={['auto', 'auto']} /> { - return (Number(value) || 0).toLocaleString(); + formatter={(value, _name, item) => { + const key = String(item.dataKey); + return withMargin( + (Number(value) || 0).toLocaleString(), + item.payload?.[moeKey(key)], + item.payload?.[reliabilityKey(key)], + (m) => m.toLocaleString(), + ' (low reliability)', + show, + ); }} /> + {margins && + show && + primary.lines.map((_, i) => + marginBand({ + dataKey: `primary${i}`, + color: PRIMARY_SHADES[i % PRIMARY_SHADES.length], + percent: !!isPercent, + type: 'monotone', + }), + )} + {margins && + show && + compareData.length > 0 && + compare.lines.map((_, i) => + marginBand({ + dataKey: `compare${i}`, + color: COMPARE_SHADES[i % COMPARE_SHADES.length], + percent: !!isPercent, + type: 'monotone', + }), + )} {primary.lines.map((_, i) => ( }) => { name={lineName(primaryName, primary, i)} stroke={PRIMARY_SHADES[i % PRIMARY_SHADES.length]} strokeWidth={2} - dot={primary.lines.length > 1} + dot={ + margins + ? reliabilityDots( + `primary${i}`, + PRIMARY_SHADES[i % PRIMARY_SHADES.length], + primary.lines.length > 1, + ) + : primary.lines.length > 1 + } /> ))} {compareData.length > 0 && @@ -1066,10 +1263,18 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { stroke={COMPARE_SHADES[i % COMPARE_SHADES.length]} strokeWidth={1.5} strokeDasharray="6 4" - dot={compare.lines.length > 1} + dot={ + margins + ? reliabilityDots( + `compare${i}`, + COMPARE_SHADES[i % COMPARE_SHADES.length], + compare.lines.length > 1, + ) + : compare.lines.length > 1 + } /> ))} - + ); diff --git a/frontend/src/components/Charts/index.tsx b/frontend/src/components/Charts/index.tsx index 02e77be3..7265e1bb 100644 --- a/frontend/src/components/Charts/index.tsx +++ b/frontend/src/components/Charts/index.tsx @@ -64,6 +64,7 @@ import * as motion from 'motion/react-client'; import { AddChart, RemoveChart } from './saving'; import { useEffect, useRef, useState } from 'react'; import { TableView, ViewSwitch } from './TableView'; +import { MarginsSwitch } from './margins'; import { usePdfMode } from '@/contexts/PdfModeContext'; import { useItems } from '@/components/ItemsProvider'; import classes from './ChartCard.module.css'; @@ -97,6 +98,12 @@ interface ChartCardProps { listeners: DraggableSyntheticListeners; }; } +/** Whether any of a chart's rows has a margin of error to show or hide. */ +const carriesMargins = (chart: ChartItem) => + [...(chart.data ?? []), ...(chart.compareData ?? [])].some((row) => + Object.entries(row).some(([k, v]) => k.endsWith('_MOE') && v != null), + ); + export const ChartCard = ({ chart, ChartComponent, @@ -296,6 +303,9 @@ export const ChartCard = ({ )} + {!isPdfMode && !isGallery && carriesMargins(chart) && ( + + )} {!isPdfMode && showViewSwitch && ( )} diff --git a/frontend/src/components/Charts/margins.tsx b/frontend/src/components/Charts/margins.tsx new file mode 100644 index 00000000..63a74579 --- /dev/null +++ b/frontend/src/components/Charts/margins.tsx @@ -0,0 +1,390 @@ +// ACS estimates come from a survey sample, so each carries a 90% margin of +// error (MOE): the true value is likely within estimate ± MOE. The API sends a +// field's margin as `_MOE` and its reliability label (high, medium or +// low, from the estimate's coefficient of variation; see +// backend/data_cleaning/margins.py) as `_Reliability`. + +import { Area } from 'recharts'; +import { Switch, Text } from '@mantine/core'; +import { create } from 'zustand'; +import { persist } from 'zustand/middleware'; +import { DataRow } from '@/types/cachedCharts'; + +export type Reliability = 'high' | 'medium' | 'low'; + +/** Whether charts and tables show margins of error: one setting for every + * chart, saved in localStorage, on by default. Low-reliability marks (hollow + * dots, "*") show either way, since they matter most to someone who has + * hidden the detail. */ +const useMarginPreference = create<{ + show: boolean; + setShow: (show: boolean) => void; +}>()( + persist((set) => ({ show: true, setShow: (show) => set({ show }) }), { + name: 'show-margins', + }), +); + +export const useShowMargins = () => useMarginPreference((s) => s.show); + +/** The current setting, for code outside React (the PDF report). */ +export const showMarginsNow = () => useMarginPreference.getState().show; + +/** The show/hide switch, for a chart card that has margins to show. */ +export const MarginsSwitch = () => { + const { show, setShow } = useMarginPreference(); + return ( + setShow(e.currentTarget.checked)} + /> + ); +}; + +/** The API fields holding `field`'s margin of error and reliability label. */ +export const moeField = (field: string) => `${field}_MOE`; +export const reliabilityField = (field: string) => `${field}_Reliability`; + +/** Whether any row carries a margin for `field`. */ +export const hasMargins = (rows: DataRow[] | undefined, field: string) => + !!rows?.some((r) => r[moeField(field)] != null); + +// Plot rows keep a value's margin and reliability under keys derived from the +// value's key, ahead of any " (cmp)" suffix so the table view still pairs a +// comparison margin with its comparison value: "Under 18" -> "Under 18 MOE", +// "Under 18 (cmp)" -> "Under 18 MOE (cmp)". +const CMP_SUFFIX = ' (cmp)'; +const MOE_SUFFIX = ' MOE'; +const RELIABILITY_SUFFIX = ' Reliability'; + +const withoutCmp = (key: string) => + key.endsWith(CMP_SUFFIX) ? key.slice(0, -CMP_SUFFIX.length) : key; +const withSuffix = (key: string, suffix: string) => + key.endsWith(CMP_SUFFIX) + ? `${withoutCmp(key)}${suffix}${CMP_SUFFIX}` + : `${key}${suffix}`; + +export const moeKey = (key: string) => withSuffix(key, MOE_SUFFIX); +export const reliabilityKey = (key: string) => + withSuffix(key, RELIABILITY_SUFFIX); + +/** Whether a plot-row key holds a margin or reliability label, not a value. */ +export const isMarginKey = (key: string) => { + const base = withoutCmp(key); + return base.endsWith(MOE_SUFFIX) || base.endsWith(RELIABILITY_SUFFIX); +}; + +/** Puts a row's `field` on a plot point under `key`, and when the chart has + * margins, the row's margin and reliability alongside it. */ +export const plotValue = ( + pt: DataRow, + key: string, + row: DataRow | undefined, + field: string, + margins: boolean, +) => { + pt[key] = row?.[field] ?? null; + if (!margins) return; + pt[moeKey(key)] = row?.[moeField(field)] ?? null; + pt[reliabilityKey(key)] = row?.[reliabilityField(field)] ?? null; +}; + +/** The likely range, estimate ± margin, kept to values the measure can take. + * Every ACS measure with a margin is a count, dollar amount, age or share, so + * none goes below zero, and shares stay at or under 100% (a 100% share in a + * tiny tract can carry a ±257 margin). */ +export const interval = ( + value: number, + moe: number, + percent: boolean, +): [number, number] => [ + Math.max(0, value - moe), + percent ? Math.min(100, value + moe) : value + moe, +]; + +/** The interval around a plot point's value under `key`, or null when either + * the value or its margin is missing. */ +export const intervalAt = (pt: DataRow, key: string, percent: boolean) => { + const value = pt[key]; + const moe = pt[moeKey(key)]; + return typeof value === 'number' && typeof moe === 'number' + ? interval(value, moe, percent) + : null; +}; + +/** The margin of a sum: the root sum of squares of its parts' margins (Census + * ACS handbook). Summing shares of one total this way runs slightly wide, + * never narrow, since it leaves out the shared denominator. */ +export const sumMoe = (moes: unknown[]) => + moes.every((m) => typeof m === 'number') + ? Math.sqrt((moes as number[]).reduce((sum, m) => sum + m * m, 0)) + : null; + +// The cut points backend/data_cleaning/margins.py uses, for values the +// frontend derives itself: a coefficient of variation under 15% is high +// reliability, over 30% low. +const Z_90 = 1.645; + +export const reliabilityOf = ( + estimate: number | null, + moe: number | null, +): Reliability | null => { + if (estimate == null || moe == null) return null; + if (moe === 0) return 'high'; + if (estimate === 0) return 'low'; + const cv = moe / Z_90 / Math.abs(estimate); + return cv < 0.15 ? 'high' : cv <= 0.3 ? 'medium' : 'low'; +}; + +/** Whether any value in the plot rows has low reliability. Comparison values + * count only `withCmp`, since tables hide them until asked. */ +export const anyLowReliability = (rows: DataRow[], withCmp = true) => + rows.some((pt) => + Object.entries(pt).some( + ([k, v]) => + v === 'low' && + (withCmp || !k.endsWith(CMP_SUFFIX)) && + withoutCmp(k).endsWith(RELIABILITY_SUFFIX), + ), + ); + +/** "value ± margin" for a tooltip or table cell, plus `lowFlag` when the + * estimate has low reliability. Without a margin, or when margins are hidden + * (`show` false), the value and the flag. A no-break space keeps "± margin" + * together when a narrow cell wraps. */ +export const withMargin = ( + text: string, + moe: unknown, + reliability: unknown, + formatMoe: (moe: number) => string, + lowFlag: string, + show = true, +) => { + const flag = reliability === 'low' ? lowFlag : ''; + return show && typeof moe === 'number' + ? `${text} ±\u00a0${formatMoe(moe)}${flag}` + : `${text}${flag}`; +}; + +/** Which field a tidy-table row's cell shows: its share or its count. */ +export type FieldOf = (row: DataRow | undefined) => 'Percent' | 'Value'; + +const formatField = (field: 'Percent' | 'Value', v: number) => + field === 'Percent' ? `${v.toFixed(1)}%` : v.toLocaleString(); + +/** A tidy-table cell: the value `fieldOf` picks, with its margin of error and + * an asterisk for a low-reliability estimate. The on-screen tables and the + * PDF report's own table renderer both use it, so they read the same. */ +export const cellText = ( + row: DataRow | undefined, + fieldOf: FieldOf, + show = true, +) => { + const field = fieldOf(row); + const value = row?.[field]; + if (typeof value !== 'number') return '—'; + return withMargin( + formatField(field, value), + row?.[moeField(field)], + row?.[reliabilityField(field)], + (m) => formatField(field, m), + '*', + show, + ); +}; + +/** The margin (`moeField`) or reliability label (`reliabilityField`) of the + * value a tidy-table row's cell shows. */ +export const cellCompanion = ( + row: DataRow | undefined, + fieldOf: FieldOf, + name: (field: string) => string, +) => { + const field = fieldOf(row); + return row && row[field] != null ? (row[name(field)] ?? null) : null; +}; + +/** Whether the cells for `rows` show any margin, and any low-reliability + * estimate: what a table's footnote needs to explain. */ +export const cellMarginFlags = (rows: DataRow[], fieldOf: FieldOf) => ({ + margins: rows.some((r) => cellCompanion(r, fieldOf, moeField) != null), + lowReliability: rows.some( + (r) => cellCompanion(r, fieldOf, reliabilityField) === 'low', + ), +}); + +// A CV over 30% means a margin of at least 0.30 × 1.645 ≈ half the estimate. +const LOW_RELIABILITY = + 'Low reliability: the margin of error is about half the estimate or more.'; + +/** The footnote text for a table with margins; with margins hidden, only the + * low-reliability note, if any. */ +export const marginFootnote = (lowReliability: boolean, show = true) => { + const low = lowReliability ? `* ${LOW_RELIABILITY}` : ''; + if (!show) return low; + return ( + '± is the margin of error: the true value is likely (90%) within that ' + + `range.${low && ` ${low}`}` + ); +}; + +// Recharts draws a lone point's dot even with dot={false}; for a band, that +// would be a faint stray dot at the top of the range. +const noDot = () => ; + +/** The shaded range behind a trend line. Call it as a function inside the + * chart, not as a component, so Recharts sees the Area itself. */ +export const marginBand = ({ + dataKey, + color, + percent, + hide, + type = 'linear', +}: { + dataKey: string; + color: string; + percent: boolean; + hide?: boolean; + type?: 'linear' | 'monotone'; +}) => ( + intervalAt(pt, dataKey, percent)} + stroke="none" + fill={color} + fillOpacity={0.15} + legendType="none" + tooltipType="none" + dot={noDot} + activeDot={false} + isAnimationActive={false} + hide={hide} + /> +); + +/** A trend line's dots: hollow where the estimate has low reliability, and + * solid at every other point only when `allDots` (split series show them). */ +export const reliabilityDots = ( + dataKey: string, + color: string, + allDots: boolean, +) => + function ReliabilityDot({ + cx, + cy, + index, + payload, + points, + }: { + cx?: number; + cy?: number; + index?: number; + payload?: DataRow; + points?: readonly unknown[]; + }) { + const low = payload?.[reliabilityKey(dataKey)] === 'low'; + // Recharts draws no line through a single point, so its dot is the only + // mark the series gets. + const only = points?.length === 1; + if (cx == null || cy == null || (!low && !allDots && !only)) + return ; + return ( + + ); + }; + +/** The caption for a chart with margins: shaded bands behind trend lines, or + * whiskers on bars. */ +export const MarginNote = ({ + lowReliability, + bars = false, + show = true, +}: { + lowReliability: boolean; + bars?: boolean; + /** Whether the margins are drawn; hidden, only the low-reliability note + * remains. */ + show?: boolean; +}) => { + const low = + lowReliability && + (bars + ? 'Hover over a bar to see whether its estimate has low reliability.' + : 'Hollow dots mark low-reliability estimates, whose margin is about half the estimate or more.'); + if (!show && !low) return null; + return ( + + {show && + `${bars ? 'The black line on each bar shows' : 'Shading shows'} the margin of error: the true value is likely (90%) within it. `} + {low} + + ); +}; + +/** The footnote for a table with margins. */ +export const MarginFootnote = ({ + lowReliability, + show = true, +}: { + lowReliability: boolean; + show?: boolean; +}) => { + const text = marginFootnote(lowReliability, show); + return text ? ( + + {text} + + ) : null; +}; + +/** A comparison map tooltip's margins, keyed by measure: the backend + * (query/comparison.py) sends them under this key beside the values. */ +const TOOLTIP_MARGINS = '__margins__'; +type TooltipMargins = Record< + string, + { moe: number; reliability: Reliability | null } +>; + +/** Whether a map tooltip key holds data for the card (`__title__`, + * `__margins__`) rather than a field to list. */ +export const isTooltipMeta = (key: string) => + key.startsWith('__') && key.endsWith('__'); + +const tooltipMargin = (tooltip: Record, key: string) => + (tooltip[TOOLTIP_MARGINS] as TooltipMargins | undefined)?.[key]; + +/** A map tooltip field's value as "value ± margin" when the tooltip carries + * its margin; otherwise the value as is. */ +export const withTooltipMargin = ( + tooltip: Record, + key: string, + value: unknown, +) => { + const margin = tooltipMargin(tooltip, key); + return margin + ? withMargin( + String(value), + margin.moe, + margin.reliability, + (m) => m.toLocaleString(), + ' (low reliability)', + ) + : value; +}; + +/** Whether any value in a map tooltip has low reliability. */ +export const tooltipLowReliability = (tooltip: Record) => + Object.values( + (tooltip[TOOLTIP_MARGINS] as TooltipMargins | undefined) ?? {}, + ).some((m) => m.reliability === 'low'); diff --git a/frontend/src/components/mapping/index.tsx b/frontend/src/components/mapping/index.tsx index a256ddb1..af95d21d 100644 --- a/frontend/src/components/mapping/index.tsx +++ b/frontend/src/components/mapping/index.tsx @@ -21,6 +21,7 @@ import { } from '@mantine/core'; import { IconX } from '@tabler/icons-react'; import { COLORS, FONTS } from '@/app/theme'; +import { isTooltipMeta, withTooltipMargin } from '@/components/Charts/margins'; import 'maplibre-gl/dist/maplibre-gl.css'; import type { MapRef } from 'react-map-gl/maplibre'; @@ -454,7 +455,7 @@ export default function VTMap({ {Object.entries(tooltip.content).map( ([k, v]) => - k !== '__title__' && + !isTooltipMeta(k) && v !== null && v !== undefined && v !== '' && ( @@ -474,7 +475,13 @@ export default function VTMap({ ta="right" style={{ color: COLORS.ink }} > - {formatDetailValue(k, v)} + {String( + withTooltipMargin( + tooltip.content, + k, + formatDetailValue(k, v), + ), + )} ), @@ -537,9 +544,15 @@ export default function VTMap({ {(() => { // Named `fields` rather than `Map` to avoid shadowing the // react-map-gl `Map` component imported at the top of this file. - const fields = Object.entries(selected).filter( - ([k]) => k !== '__title__', - ); + const fields = Object.entries(selected) + .filter(([k]) => !isTooltipMeta(k)) + .map( + ([k, v]) => + [k, withTooltipMargin(selected, k, v)] as [ + string, + unknown, + ], + ); const claimed = new Set(); const sections = DETAIL_GROUPS.map((group) => { diff --git a/frontend/src/lib/pdfReport/ReportDocument.tsx b/frontend/src/lib/pdfReport/ReportDocument.tsx index 67fba174..9da519db 100644 --- a/frontend/src/lib/pdfReport/ReportDocument.tsx +++ b/frontend/src/lib/pdfReport/ReportDocument.tsx @@ -22,6 +22,13 @@ import { StyleSheet, } from '@react-pdf/renderer'; import { ChartItem } from '@/types/cachedCharts'; +import { + FieldOf, + cellMarginFlags, + cellText, + marginFootnote, + showMarginsNow, +} from '@/components/Charts/margins'; import { PdfChartEntry, PdfSection } from './types'; // --------------------------------------------------------------------------- @@ -202,6 +209,11 @@ const s = StyleSheet.create({ color: '#888', marginTop: 8, }, + tableFootnote: { + fontSize: 7, + color: '#666', + marginTop: 4, + }, noteCard: { backgroundColor: '#f9f9f9', padding: 8, @@ -221,24 +233,12 @@ const s = StyleSheet.create({ // Table renderer // --------------------------------------------------------------------------- -type RenderCellFn = (row: any) => string; - -const renderTablePercent: RenderCellFn = (row) => - row?.Percent != null ? `${row.Percent.toFixed(1)}%` : '—'; - -const renderTableValue: RenderCellFn = (row) => - row?.Value != null ? Number(row.Value).toLocaleString() : '—'; - -const renderTableMixed: RenderCellFn = (row) => { - if (row?.Percent != null) return `${row.Percent.toFixed(1)}%`; - if (row?.Value != null) return Number(row.Value).toLocaleString(); - return '—'; -}; - -function getCellRenderer(subtype: string): RenderCellFn { - if (subtype === 'renderTable') return renderTablePercent; - if (subtype === 'renderTableMixed') return renderTableMixed; - return renderTableValue; // renderTableEstimates and anything else +// The field each table subtype shows, as in Charts/DemographicsTable.tsx. +function getFieldOf(subtype: string): FieldOf { + if (subtype === 'renderTable') return () => 'Percent'; + if (subtype === 'renderTableMixed') + return (row) => (row?.Percent != null ? 'Percent' : 'Value'); + return () => 'Value'; // renderTableEstimates and anything else } const PdfTableSection = ({ chart }: { chart: ChartItem }) => { @@ -251,44 +251,51 @@ const PdfTableSection = ({ chart }: { chart: ChartItem }) => { const variables = Array.from( new Set(data.map((r) => r.Variable)), ) as string[]; - const renderCell = getCellRenderer(chart.subtype); + const fieldOf = getFieldOf(chart.subtype); + const { margins, lowReliability } = cellMarginFlags(data, fieldOf); + // Margins as the viewer set them on screen. + const show = showMarginsNow(); + const footnote = margins && marginFootnote(lowReliability, show); const findRow = (variable: string, year: number) => data.find((r) => r.Variable === variable && r.year === year); return ( - - {/* Header */} - - Variable - {years.map((y) => ( - - {y} - + <> + + {/* Header */} + + Variable + {years.map((y) => ( + + {y} + + ))} + + + {/* Rows */} + {variables.map((variable, idx) => ( + + {variable} + {years.map((year) => { + const row = findRow(variable, year); + return ( + + {cellText(row, fieldOf, show)} + + ); + })} + ))} - - {/* Rows */} - {variables.map((variable, idx) => ( - - {variable} - {years.map((year) => { - const row = findRow(variable, year); - return ( - - {renderCell(row)} - - ); - })} - - ))} - + {footnote && {footnote}} + ); };