From 0d29a3b2f1b88c2619d9c5f30c6db738394d57d7 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Thu, 29 Oct 2026 12:41:51 -0400 Subject: [PATCH 1/6] Serve ACS margins of error and reliability labels from the API The tidy endpoints add Value_MOE, Value_Reliability, Percent_MOE and Percent_Reliability; the timeseries endpoints add _MOE and _Reliability for the series that have margins. The frontend doesn't read them yet. --- backend/query/acs5.py | 7 +++++++ backend/query/sql/acs5/acs5_tidy.sql | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/query/acs5.py b/backend/query/acs5.py index f47e7aab..6266e6c6 100644 --- a/backend/query/acs5.py +++ b/backend/query/acs5.py @@ -126,6 +126,8 @@ # The cleaned timeseries tables store their measure columns in snake_case, but # the frontend charts read these Title_Case field names, so responses keep them. +# Where a series has a 90% margin of error, it follows as `_MOE`, with +# its reliability label (high, medium or low) as `_Reliability`. TIMESERIES_FIELD_NAMES = { "median_age": "Median_Age", "age_dependency_ratio": "Age_Dependency_Ratio", @@ -135,6 +137,11 @@ "median_home_value": "Median_Home_Value", "percent": "Percent", } +TIMESERIES_FIELD_NAMES |= { + f"{column}_{suffix.lower()}": f"{field}_{suffix}" + for column, field in TIMESERIES_FIELD_NAMES.items() + for suffix in ("MOE", "Reliability") +} def _acs5_source( diff --git a/backend/query/sql/acs5/acs5_tidy.sql b/backend/query/sql/acs5/acs5_tidy.sql index 4154c65a..4b7e84dc 100644 --- a/backend/query/sql/acs5/acs5_tidy.sql +++ b/backend/query/sql/acs5/acs5_tidy.sql @@ -5,7 +5,11 @@ SELECT section AS Section, variable AS Variable, value AS Value, - percent AS Percent + value_moe AS Value_MOE, + value_reliability AS Value_Reliability, + percent AS Percent, + percent_moe AS Percent_MOE, + percent_reliability AS Percent_Reliability FROM {{ table }} {{ where_string }} ORDER BY year, Section, Variable From a5a210d8461580b506d225218b65cfb906026730 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Fri, 30 Oct 2026 13:08:16 -0400 Subject: [PATCH 2/6] Show ACS margins of error in the charts and tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trend lines get a shaded band for the 90% margin of error, with hollow dots on low-reliability estimates; the age and education distribution bars get whiskers (error bars in the PDF version). Tooltips and table cells read "value ± margin", with low reliability flagged, and the Excel exports carry the margins. Intervals are clamped to 0 and, for shares, to 100%. --- frontend/src/components/Charts/Bar.tsx | 161 ++++++++++-- .../components/Charts/DemographicsTable.tsx | 99 +++++--- frontend/src/components/Charts/TableView.tsx | 54 +++- .../src/components/Charts/TrendCharts.tsx | 233 ++++++++++++++---- frontend/src/components/Charts/margins.tsx | 227 +++++++++++++++++ 5 files changed, 662 insertions(+), 112 deletions(-) create mode 100644 frontend/src/components/Charts/margins.tsx diff --git a/frontend/src/components/Charts/Bar.tsx b/frontend/src/components/Charts/Bar.tsx index 39e70f06..8d5c857a 100644 --- a/frontend/src/components/Charts/Bar.tsx +++ b/frontend/src/components/Charts/Bar.tsx @@ -4,6 +4,7 @@ import { useMemo, useEffect } from 'react'; import { BarChart, Bar, + ErrorBar, XAxis, YAxis, CartesianGrid, @@ -28,11 +29,24 @@ import { ChartEvent, LegendItem, LegendElement, + Plugin, } from 'chart.js'; ChartJS.register(CategoryScale, LinearScale, BarElement, TooltipJS, LegendJS); import { ChartItem, DataRow } from '@/types/cachedCharts'; import { usePdfMode } from '@/contexts/PdfModeContext'; +import { + MarginNote, + anyLowReliability, + hasMargins, + interval, + intervalAt, + moeField, + moeKey, + plotValue, + reliabilityKey, + withMargin, +} from './margins'; import { Box, Group, @@ -53,6 +67,56 @@ const d3Schemes = d3 as unknown as Record; const EMPTY_ROWS: DataRow[] = []; const stableRows = (rows?: DataRow[]) => (rows?.length ? rows : EMPTY_ROWS); +// A bar dataset may carry `intervals`, each bar's margin-of-error range (see +// margins.tsx); this plugin draws them as capped whiskers over the bars. +type MarginDataset = { intervals?: ([number, number] | null)[] }; + +const marginWhiskers: Plugin<'bar'> = { + id: 'marginWhiskers', + afterDatasetsDraw(chartJS) { + const { ctx } = chartJS; + const y = chartJS.scales.y; + chartJS.data.datasets.forEach((dataset, di) => { + const { intervals } = dataset as MarginDataset; + if (!intervals || !chartJS.isDatasetVisible(di)) return; + ctx.save(); + ctx.strokeStyle = '#333'; + ctx.lineWidth = 1.5; + chartJS.getDatasetMeta(di).data.forEach((bar, i) => { + const range = intervals[i]; + if (!range) return; + const top = y.getPixelForValue(range[1]); + const bottom = y.getPixelForValue(range[0]); + const { width } = bar.getProps(['width'], true) as { width: number }; + const cap = Math.min(6, width / 4); + ctx.beginPath(); + ctx.moveTo(bar.x, top); + ctx.lineTo(bar.x, bottom); + ctx.moveTo(bar.x - cap, top); + ctx.lineTo(bar.x + cap, top); + ctx.moveTo(bar.x - cap, bottom); + ctx.lineTo(bar.x + cap, bottom); + ctx.stroke(); + }); + ctx.restore(); + }); + }, +}; +const MARGIN_PLUGINS = [marginWhiskers]; + +/** A row's margin as Recharts ErrorBar offsets below and above its value. */ +const errorOffsets = ( + row: DataRow | undefined, + field: string, + percent: boolean, +) => { + const value = row?.[field]; + const moe = row?.[moeField(field)]; + if (typeof value !== 'number' || typeof moe !== 'number') return null; + const [low, high] = interval(value, moe, percent); + return [value - low, high - value]; +}; + const SamePerXBarChart = ({ chart, view, @@ -249,14 +313,19 @@ const CompareDiffPerXBarChartSVG = ({ colors = filteredData.map((_, i) => colorScale(i.toString())); } + const percent = chart.chartParams.percentFormat ?? false; + const margins = hasMargins(filteredData, chart.yField); + // Merge primary + compare into one array for grouped bars const merged = filteredData.map((entry, i) => ({ [chart.xField]: entry[chart.xField], primary: entry[chart.yField], compare: filteredCompareData[i]?.[chart.yField] ?? null, + primaryError: errorOffsets(entry, chart.yField, percent), + compareError: errorOffsets(filteredCompareData[i], chart.yField, percent), })); - return ( + const plot = ( ( ))} + {margins && ( + + )} + + + {margins && ( + + )} - ); + if (!margins) return plot; + + return ( + + + {plot} + + ); }; const CompareDiffPerXBarChart = ({ @@ -322,19 +406,21 @@ const CompareDiffPerXBarChart = ({ return cmp.length ? alignByX(filteredData, cmp, chart.xField) : cmp; }, [compareRows, filteredData, includeCategories, chart.xField]); + const margins = + hasMargins(filteredData, chart.yField) || + hasMargins(filteredCompareData, chart.yField); + // Derive exact plottable rows and report back to ChartCard for TableView const plotData = useMemo(() => { return filteredData.map((entry, i) => { const cmpEntry = filteredCompareData[i]; - return { - [chart.xField]: entry[chart.xField], - [chart.yField]: entry[chart.yField], - ...(cmpEntry - ? { [`${chart.yField} (cmp)`]: cmpEntry[chart.yField] } - : {}), - }; + const pt: DataRow = { [chart.xField]: entry[chart.xField] }; + plotValue(pt, chart.yField, entry, chart.yField, margins); + if (cmpEntry) + plotValue(pt, `${chart.yField} (cmp)`, cmpEntry, chart.yField, margins); + return pt; }); - }, [filteredData, filteredCompareData, chart.xField, chart.yField]); + }, [filteredData, filteredCompareData, chart.xField, chart.yField, margins]); useEffect(() => { onPlotData?.(plotData); @@ -346,6 +432,7 @@ const CompareDiffPerXBarChart = ({ const primaryLabel = chart.chartParams?.legendLabels?.[0] ?? yField; const compareLabel = chart.chartParams?.legendLabels?.[1] ?? `${yField} (compare)`; + const percentFormat = chart.chartParams?.percentFormat ?? false; const data = useMemo(() => { let colors: string[]; @@ -365,11 +452,19 @@ const CompareDiffPerXBarChart = ({ label: primaryLabel, data: filteredData.map((entry) => entry[yField] as number), backgroundColor: colors, + intervals: margins + ? plotData.map((pt) => intervalAt(pt, yField, percentFormat)) + : undefined, }, { label: compareLabel, data: filteredCompareData.map((entry) => entry[yField] as number), backgroundColor: filteredCompareData.map(() => '#D3D3D3'), + intervals: margins + ? plotData.map((pt) => + intervalAt(pt, `${yField} (cmp)`, percentFormat), + ) + : undefined, }, ], }; @@ -383,12 +478,20 @@ const CompareDiffPerXBarChart = ({ colorScheme, primaryLabel, compareLabel, + margins, + plotData, + percentFormat, ]); - const percentFormat = chart.chartParams?.percentFormat ?? false; const yLabel = chart.chartParams?.yLabel; - const options = useMemo( - () => ({ + const options = useMemo(() => { + const format = (v: number) => + percentFormat ? `${v}%` : v.toLocaleString(); + // Chart.js scales to the bars alone; leave room for the whiskers. + const whiskerTops = data.datasets.flatMap((ds) => + (ds.intervals ?? []).map((range) => range?.[1] ?? 0), + ); + return { responsive: true, maintainAspectRatio: false, animation: view === 'gallery' ? (false as const) : undefined, @@ -397,8 +500,16 @@ const CompareDiffPerXBarChart = ({ tooltip: { callbacks: { label: (context: TooltipItem<'bar'>) => { - const value = context.parsed.y ?? 0; - return percentFormat ? `${value}%` : value.toLocaleString(); + const key = + context.datasetIndex === 0 ? yField : `${yField} (cmp)`; + const pt = plotData[context.dataIndex]; + return withMargin( + format(context.parsed.y ?? 0), + pt?.[moeKey(key)], + pt?.[reliabilityKey(key)], + format, + ' (low reliability)', + ); }, }, }, @@ -406,19 +517,31 @@ const CompareDiffPerXBarChart = ({ scales: { y: { title: { display: !!yLabel, text: yLabel }, + suggestedMax: whiskerTops.length + ? Math.max(...whiskerTops) + : undefined, ticks: { callback: (value: string | number) => percentFormat ? `${value}%` : value.toLocaleString(), }, }, }, - }), - [view, percentFormat, yLabel], - ); + }; + }, [view, percentFormat, yLabel, data, plotData, yField]); if (isPdfMode) return ; + if (!margins) return ; - return ; + return ( + + {view !== 'gallery' && ( + + )} + + + + + ); }; type AllowanceRow = { diff --git a/frontend/src/components/Charts/DemographicsTable.tsx b/frontend/src/components/Charts/DemographicsTable.tsx index 39754ba3..8e92ea5d 100644 --- a/frontend/src/components/Charts/DemographicsTable.tsx +++ b/frontend/src/components/Charts/DemographicsTable.tsx @@ -5,6 +5,12 @@ import { ChartItem, DataRow } from '@/types/cachedCharts'; import { COLORS } from '@/app/theme'; import * as XLSX from 'xlsx'; import { saveWorksheet } from './TableView'; +import { + MarginFootnote, + moeField, + reliabilityField, + withMargin, +} from './margins'; // tidy ACS row shape consumed by the demographics tables interface AcsRow extends DataRow { @@ -18,21 +24,48 @@ const HOME_BG = 'var(--mantine-color-green-0)'; const COMP_BG = 'var(--mantine-color-blue-0)'; const SPLIT_BORDER = '1px solid var(--mantine-color-gray-3)'; +/** The field a row's cell shows. */ +type FieldOf = (row: AcsRow | undefined) => 'Percent' | 'Value'; + +const formatField = (field: 'Percent' | 'Value', v: number) => + field === 'Percent' ? `${v.toFixed(1)}%` : v.toLocaleString(); + +// A cell's value, with its margin of error (see margins.tsx) and an asterisk +// for a low-reliability estimate. +const cellText = (row: AcsRow | undefined, fieldOf: FieldOf) => { + const field = fieldOf(row); + const value = row?.[field]; + if (typeof value !== 'number') return '—'; + return withMargin( + formatField(field, value), + row?.[moeField(field)], + row?.[reliabilityField(field)], + (m) => formatField(field, m), + '*', + ); +}; + const DemographicsTableBase = ({ chart, - renderCell, - exportValue, + fieldOf, }: { chart: ChartItem; - renderCell: (row: AcsRow | undefined) => React.ReactNode; - /** The number behind a cell, for the Excel export. */ - exportValue: (row: AcsRow | undefined) => number | null; + fieldOf: FieldOf; }) => { const data = chart.data; const compareData = chart.compareData ?? []; const hasCompare = compareData.length > 0; const [showCompare, setShowCompare] = useState(false); + // The margin or reliability label of the value a row's cell shows. + const companion = (r: AcsRow | undefined, name: (field: string) => string) => + r?.[fieldOf(r)] != null ? r[name(fieldOf(r))] : null; + const allRows = [...data, ...compareData]; + const margins = allRows.some((r) => companion(r, moeField) != null); + const lowReliability = allRows.some( + (r) => companion(r, reliabilityField) === 'low', + ); + const labels = chart.chartParams?.legendLabels as [string, string] | undefined; const homeLabel = labels?.[0] ?? 'Primary'; @@ -48,21 +81,31 @@ const DemographicsTableBase = ({ ) => rows.find((r) => r.Variable === variable && r.year === year); // Mirrors the table: a "Variable" column, then one column per year that - // splits in two (main place | comparison) under a merged year header. + // splits in two (main place | comparison) under a merged year header. With + // margins, each variable's row is followed by a row of its margins. const exportTable = () => { const sub = hasCompare ? [homeLabel, compareLabel] : []; const width = hasCompare ? 2 : 1; - const num = (r: AcsRow | undefined) => exportValue(r) ?? ''; + const num = (r: AcsRow | undefined) => r?.[fieldOf(r)] ?? ''; + const moe = (r: AcsRow | undefined) => companion(r, moeField) ?? ''; const header = [ 'Variable', ...years.flatMap((y) => (hasCompare ? [y, ''] : [y])), ]; - const body = variables.map((variable) => [ - variable, + const line = ( + label: string, + cell: (r: AcsRow | undefined) => unknown, + variable?: string, + ) => [ + label, ...years.flatMap((year) => [ - num(findRow(data, variable, year)), - ...(hasCompare ? [num(findRow(compareData, variable, year))] : []), + cell(findRow(data, variable, year)), + ...(hasCompare ? [cell(findRow(compareData, variable, year))] : []), ]), + ]; + const body = variables.flatMap((variable) => [ + line(String(variable), num, variable), + ...(margins ? [line(`${variable} MOE`, moe, variable)] : []), ]); const rows = hasCompare ? [header, ['', ...years.flatMap(() => sub)], ...body] @@ -150,7 +193,9 @@ const DemographicsTableBase = ({ const mainRow = findRow(data, variable, year); if (!showCompare) { return ( - {renderCell(mainRow)} + + {cellText(mainRow, fieldOf)} + ); } const cmpRow = findRow(compareData, variable, year); @@ -165,7 +210,7 @@ const DemographicsTableBase = ({ textAlign: 'right', }} > - {renderCell(mainRow)} + {cellText(mainRow, fieldOf)}
- {renderCell(cmpRow)} + {cellText(cmpRow, fieldOf)}
@@ -187,43 +232,25 @@ const DemographicsTableBase = ({ + {margins && } ); }; export const renderTable = ({ chart }: { chart: ChartItem }) => ( - - row?.Percent != null ? `${row.Percent.toFixed(1)}%` : '—' - } - exportValue={(row) => row?.Percent ?? null} - /> + 'Percent'} /> ); export const renderTableEstimates = ({ chart, }: { chart: ChartItem; -}) => ( - - row?.Value != null ? row.Value.toLocaleString() : '—' - } - exportValue={(row) => row?.Value ?? null} - /> -); +}) => 'Value'} />; /** Shows Percent when available, falls back to Value — for mixed tables like Housing. */ export const renderTableMixed = ({ chart }: { chart: ChartItem }) => ( { - if (row?.Percent != null) return `${row.Percent.toFixed(1)}%`; - if (row?.Value != null) return row.Value.toLocaleString(); - return '—'; - }} - exportValue={(row) => row?.Percent ?? row?.Value ?? null} + fieldOf={(row) => (row?.Percent != null ? 'Percent' : 'Value')} /> ); diff --git a/frontend/src/components/Charts/TableView.tsx b/frontend/src/components/Charts/TableView.tsx index 583df38c..c9243178 100644 --- a/frontend/src/components/Charts/TableView.tsx +++ b/frontend/src/components/Charts/TableView.tsx @@ -15,6 +15,14 @@ import { import { ChartItem, DataRow } from '@/types/cachedCharts'; import { COLORS } from '@/app/theme'; import { FileXlsIcon } from '@phosphor-icons/react'; +import { + MarginFootnote, + anyLowReliability, + isMarginKey, + moeKey, + reliabilityKey, + withMargin, +} from './margins'; interface TableViewProps { chart: ChartItem; @@ -26,13 +34,28 @@ const COMP_BG = 'var(--mantine-color-blue-0)'; const SPLIT_BORDER = '1px solid var(--mantine-color-gray-3)'; const CMP_SUFFIX = ' (cmp)'; +const formatNumber = (v: number) => + v.toLocaleString(undefined, { maximumFractionDigits: 1 }); + const formatCell = (v: unknown) => { if (v == null) return '—'; - if (typeof v === 'number') - return v.toLocaleString(undefined, { maximumFractionDigits: 1 }); + if (typeof v === 'number') return formatNumber(v); return String(v); }; +// A value's cell, with its margin of error (see margins.tsx) when the row +// has one and an asterisk for a low-reliability estimate. +const valueCell = (row: DataRow | undefined, key: string) => + row?.[key] == null + ? formatCell(row?.[key]) + : withMargin( + formatCell(row[key]), + row[moeKey(key)], + row[reliabilityKey(key)], + formatNumber, + '*', + ); + /** Write a worksheet to a one-sheet .xlsx named after the chart and download it. */ export function saveWorksheet( chart: ChartItem, @@ -118,6 +141,10 @@ export const TableView = ({ const cmpKeys = allKeys.filter((k) => k.endsWith(CMP_SUFFIX)); const hasCompare = cmpKeys.length > 0 && rows.some((r) => cmpKeys.some((k) => r[k] != null)); + const margins = allKeys.some(isMarginKey); + const footnote = margins && ( + + ); const handleExport = () => exportRowsToExcel(chart, rows); @@ -184,7 +211,7 @@ export const TableView = ({ ); const seriesKeys = allKeys.filter( - (k) => k !== xKey && !k.endsWith(CMP_SUFFIX), + (k) => k !== xKey && !k.endsWith(CMP_SUFFIX) && !isMarginKey(k), ); const findRow = (x: unknown) => rows.find((r) => r[xKey] === x); @@ -208,15 +235,13 @@ export const TableView = ({ {key} {xValues.map((x) => { const row = findRow(x); - const mainVal = row?.[key]; if (!showCompare || !hasCompare) { return ( - {formatCell(mainVal)} + {valueCell(row, key)} ); } - const cmpVal = row?.[`${key}${CMP_SUFFIX}`]; return (
@@ -228,7 +253,7 @@ export const TableView = ({ textAlign: 'right', }} > - {formatCell(mainVal)} + {valueCell(row, key)}
({ textAlign: 'right', }} > - {formatCell(cmpVal)} + {valueCell(row, `${key}${CMP_SUFFIX}`)}
@@ -250,13 +275,16 @@ export const TableView = ({ + {footnote} ); } // --- MODE B: DIRECT TABLE (FOR BAR CHARTS) --- // Determine standard columns vs comparison columns - const baseColumns = allKeys.filter((k) => !k.endsWith(CMP_SUFFIX)); + const baseColumns = allKeys.filter( + (k) => !k.endsWith(CMP_SUFFIX) && !isMarginKey(k), + ); return ( @@ -275,7 +303,6 @@ export const TableView = ({ {rows.map((row, i) => ( {baseColumns.map((column) => { - const mainVal = row[column]; const cmpVal = row[`${column}${CMP_SUFFIX}`]; const isComparableCell = hasCompare && showCompare && cmpVal != null; @@ -292,7 +319,7 @@ export const TableView = ({ textAlign: 'right', }} > - {formatCell(mainVal)} + {valueCell(row, column)}
({ textAlign: 'right', }} > - {formatCell(cmpVal)} + {valueCell(row, `${column}${CMP_SUFFIX}`)}
@@ -311,7 +338,7 @@ export const TableView = ({ } return ( - {formatCell(mainVal)} + {valueCell(row, column)} ); })} @@ -319,6 +346,7 @@ export const TableView = ({ + {footnote} ); }; diff --git a/frontend/src/components/Charts/TrendCharts.tsx b/frontend/src/components/Charts/TrendCharts.tsx index bcee6c37..987a0cb4 100644 --- a/frontend/src/components/Charts/TrendCharts.tsx +++ b/frontend/src/components/Charts/TrendCharts.tsx @@ -4,6 +4,7 @@ import { useState, useEffect, useMemo } from 'react'; import { CartesianGrid, + ComposedChart, Legend, Line, LineChart, @@ -16,6 +17,20 @@ import { ChartItem, DataRow } from '@/types/cachedCharts'; import { usePdfMode } from '@/contexts/PdfModeContext'; import MultiRowNotice from './MultiRowNotice'; import { lineName, splitIntoLines } from './seriesLines'; +import { + MarginNote, + anyLowReliability, + hasMargins, + marginBand, + moeField, + moeKey, + plotValue, + reliabilityDots, + reliabilityKey, + reliabilityOf, + sumMoe, + withMargin, +} from './margins'; // tidy ACS-style row consumed by the trend charts interface TrendRow extends DataRow { @@ -45,6 +60,9 @@ const FORMATTERS: Record< unit?: string; axisFormatter?: (v: number) => string; tooltip: (v: unknown, decimals?: number) => string; + /** A margin of error, shown after the value as "value ± margin" at the + * value's precision. */ + margin: (moe: number, decimals?: number) => string; } > = { currency: { @@ -53,6 +71,8 @@ const FORMATTERS: Record< v != null ? `$${Number(v).toLocaleString('en-US', { maximumFractionDigits: 0 })}` : '—', + margin: (m) => + `$${m.toLocaleString('en-US', { maximumFractionDigits: 0 })}`, }, percent: { unit: '%', @@ -60,17 +80,38 @@ const FORMATTERS: Record< v != null ? `${decimals != null ? Number(v).toFixed(decimals) : v}%` : '—', + margin: (m, decimals) => `${decimals != null ? m.toFixed(decimals) : m}%`, }, number: { axisFormatter: (v) => `${(v / 1000).toFixed(0)}k`, tooltip: (v) => (v != null ? Number(v).toLocaleString() : '—'), + margin: (m) => m.toLocaleString('en-US', { maximumFractionDigits: 0 }), }, years: { axisFormatter: (v) => Number(v).toFixed(0), tooltip: (v) => (v != null ? `${Number(v).toFixed(1)} years` : '—'), + margin: (m) => m.toFixed(1), }, }; +/** Tooltip text for a line's value: its margin follows it when the plot + * point has one, and a low-reliability estimate says so. */ +const tooltipText = ( + fmt: (typeof FORMATTERS)[FormatType], + value: unknown, + item: { dataKey?: unknown; payload?: DataRow }, + decimals?: number, +) => { + const key = String(item.dataKey); + return withMargin( + fmt.tooltip(value, decimals), + item.payload?.[moeKey(key)], + item.payload?.[reliabilityKey(key)], + (m) => fmt.margin(m, decimals), + ' (low reliability)', + ); +}; + const useToggle = () => { const [hidden, setHidden] = useState>(new Set()); const toggleSeries = (key: string) => @@ -180,23 +221,38 @@ export const SingleSeriesTrendChart = ({ [data], ); + const margins = + hasMargins(data, valueField) || hasMargins(compareData, valueField); + const plotData = useMemo(() => { if (!data || data.length === 0) return []; - const valueAt = (rows: DataRow[], year: unknown) => - rows.find((r) => r.year === year)?.[valueField] ?? null; + const rowAt = (rows: DataRow[], year: unknown) => + rows.find((r) => r.year === year); return years.map((year) => { const pt: DataRow = { year }; own.lines.forEach((line, i) => { - pt[lineKey(seriesName, i)] = valueAt(line.rows, year); + plotValue( + pt, + lineKey(seriesName, i), + rowAt(line.rows, year), + valueField, + margins, + ); }); if (compareData && compareData.length > 0) { other.lines.forEach((line, i) => { - pt[`${lineKey(seriesName, i)} (cmp)`] = valueAt(line.rows, year); + plotValue( + pt, + `${lineKey(seriesName, i)} (cmp)`, + rowAt(line.rows, year), + valueField, + margins, + ); }); } return pt; }); - }, [years, data, compareData, own, other, seriesName, valueField]); + }, [years, data, compareData, own, other, seriesName, valueField, margins]); useEffect(() => { onPlotData?.(plotData); @@ -205,6 +261,8 @@ export const SingleSeriesTrendChart = ({ if (!data || data.length === 0) return null; // early return now AFTER all hooks const fmt = FORMATTERS[format]; + const percent = format === 'percent'; + const hasCompare = compareData.length > 0; return ( <> @@ -218,8 +276,11 @@ export const SingleSeriesTrendChart = ({ Click legend items to show or hide locations. )} + {margins && !isGallery && ( + + )} - ({ tickFormatter={fmt.axisFormatter} /> {!isGallery && ( - fmt.tooltip(v, decimals)} /> + + tooltipText(fmt, v, item, decimals) + } + /> )} {!isGallery && ( ({ wrapperStyle={{ fontSize: isGallery ? 12 : 16 }} /> )} + {margins && + own.lines.map((_, i) => + marginBand({ + dataKey: ownKey(i), + color, + percent, + hide: hidden.has(ownKey(i)), + type: 'monotone', + }), + )} + {margins && + hasCompare && + other.lines.map((_, i) => + marginBand({ + dataKey: otherKey(i), + color: compareColor, + percent, + hide: hidden.has(otherKey(i)), + type: 'monotone', + }), + )} {own.lines.map((_, i) => ( ({ stroke={color} strokeWidth={lineWidth} strokeDasharray={SPLIT_DASHES[i % SPLIT_DASHES.length]} - dot={own.lines.length > 1} + dot={ + margins + ? reliabilityDots(ownKey(i), color, own.lines.length > 1) + : own.lines.length > 1 + } animationBegin={0} isAnimationActive={!isGallery && !isPdfMode} animationDuration={!isGallery ? 1500 : 0} hide={hidden.has(ownKey(i))} /> ))} - {compareData.length > 0 && + {hasCompare && other.lines.map((_, i) => ( ({ stroke={compareColor} strokeWidth={lineWidth} strokeDasharray={SPLIT_DASHES[i % SPLIT_DASHES.length]} - dot={other.lines.length > 1} + dot={ + margins + ? reliabilityDots( + otherKey(i), + compareColor, + other.lines.length > 1, + ) + : other.lines.length > 1 + } animationBegin={0} isAnimationActive={!isGallery && !isPdfMode} animationDuration={!isGallery ? 1500 : 0} hide={hidden.has(otherKey(i))} /> ))} - + ); @@ -328,27 +426,31 @@ export interface MultiSeriesConfig { nameSuffix?: boolean; } -// The series' value for a year; summed series add up their matches. -const getValue = ( +// Puts a summed series' value for a year on a plot point (Demographics 65+), +// with the sum's margin when the chart has margins. +const plotSum = ( + pt: DataRow, + key: string, rows: TrendRow[], year: number, s: SeriesDef, valueField: string, + margins: boolean, ) => { - if (s.aggregateFrom) { - const sum = s.aggregateFrom.reduce((acc, label) => { - const v = (rows.find((r) => r.year === year && r.Variable === label)?.[ - valueField - ] ?? 0) as number; - return acc + v; - }, 0); - return sum > 0 ? Math.round(sum * 10) / 10 : null; - } - const label = s.matchVariable ?? s.key; - return ( - rows.find((r) => r.year === year && r.Variable === label)?.[valueField] ?? - null + const parts = s.aggregateFrom!.map((label) => + rows.find((r) => r.year === year && r.Variable === label), ); + const sum = parts.reduce( + (acc, r) => acc + ((r?.[valueField] ?? 0) as number), + 0, + ); + const value = sum > 0 ? Math.round(sum * 10) / 10 : null; + pt[key] = value; + if (!margins) return; + const moe = + value == null ? null : sumMoe(parts.map((r) => r?.[moeField(valueField)])); + pt[moeKey(key)] = moe == null ? null : Math.round(moe * 10) / 10; + pt[reliabilityKey(key)] = reliabilityOf(value, moe); }; export const MultiSeriesTrendChart = ({ @@ -420,30 +522,43 @@ export const MultiSeriesTrendChart = ({ [data], ); + const margins = + hasMargins(data, valueField) || hasMargins(compareData, valueField); + const plotData = useMemo(() => { if (!data || data.length === 0) return []; return years.map((year) => { const pt: DataRow = { year }; series.forEach((s, si) => { - pt[s.key] = getValue(data, year, s, valueField); - if (compareData && compareData.length > 0) - pt[`${s.key} (cmp)`] = getValue(compareData, year, s, valueField); - if (s.aggregateFrom) return; + if (s.aggregateFrom) { + plotSum(pt, s.key, data, year, s, valueField, margins); + if (compareData && compareData.length > 0) + plotSum( + pt, + `${s.key} (cmp)`, + compareData, + year, + s, + valueField, + margins, + ); + return; + } const { own, other } = splitSeries[si]; const at = (line: { rows: DataRow[] }) => - line.rows.find((r) => r.year === year)?.[valueField] ?? null; + line.rows.find((r) => r.year === year); own.lines.forEach((line, i) => { - if (i === 0) pt[s.key] = at(line); - else pt[extraKey(s, i)] = at(line); + const key = i === 0 ? s.key : extraKey(s, i); + plotValue(pt, key, at(line), valueField, margins); }); other.lines.forEach((line, i) => { - if (i === 0) pt[`${s.key} (cmp)`] = at(line); - else pt[`${extraKey(s, i)} (cmp)`] = at(line); + const key = i === 0 ? s.key : extraKey(s, i); + plotValue(pt, `${key} (cmp)`, at(line), valueField, margins); }); }); return pt; }); - }, [years, data, compareData, series, valueField, splitSeries]); + }, [years, data, compareData, series, valueField, splitSeries, margins]); useEffect(() => { onPlotData?.(plotData); @@ -452,6 +567,8 @@ export const MultiSeriesTrendChart = ({ if (!data || data.length === 0) return null; const fmt = FORMATTERS[format]; + const percent = format === 'percent'; + const hasCompare = compareData.length > 0; return ( <> @@ -460,7 +577,7 @@ export const MultiSeriesTrendChart = ({ maxPerYear={maxPerYear} duplicates={duplicates} /> - {compareData.length > 0 && !isGallery && ( + {hasCompare && !isGallery && ( )} {!isGallery && ( @@ -468,8 +585,11 @@ export const MultiSeriesTrendChart = ({ Click legend items to show or hide categories. )} + {margins && !isGallery && ( + + )} - @@ -485,7 +605,11 @@ export const MultiSeriesTrendChart = ({ domain={['auto', 'auto']} tickFormatter={fmt.axisFormatter} /> - {!isGallery && fmt.tooltip(v)} />} + {!isGallery && ( + tooltipText(fmt, v, item)} + /> + )} {!isGallery && ( ({ wrapperStyle={{ fontSize: isGallery ? 12 : 16 }} /> )} + {margins && + series.flatMap((s) => [ + marginBand({ + dataKey: s.key, + color: s.color, + percent, + hide: hidden.has(s.key), + }), + ...(hasCompare + ? [ + marginBand({ + dataKey: `${s.key} (cmp)`, + color: s.color, + percent, + hide: hidden.has(s.key), + }), + ] + : []), + ])} {series.map((s) => ( ({ } stroke={s.color} strokeWidth={2} - dot={false} + dot={margins && reliabilityDots(s.key, s.color, false)} isAnimationActive={!isGallery && !isPdfMode} animationDuration={!isGallery ? 1500 : 0} hide={hidden.has(s.key)} @@ -531,7 +674,7 @@ export const MultiSeriesTrendChart = ({ /> )), )} - {compareData.length > 0 && + {hasCompare && series.flatMap((s, si) => splitSeries[si].other.lines .slice(1) @@ -551,7 +694,7 @@ export const MultiSeriesTrendChart = ({ /> )), )} - {compareData.length > 0 && + {hasCompare && series.map((s) => ( ({ legendType="none" isAnimationActive={!isGallery && !isPdfMode} animationDuration={!isGallery ? 1500 : 0} - dot={false} + dot={ + margins && reliabilityDots(`${s.key} (cmp)`, s.color, false) + } hide={hidden.has(s.key)} /> ))} - + ); diff --git a/frontend/src/components/Charts/margins.tsx b/frontend/src/components/Charts/margins.tsx new file mode 100644 index 00000000..4b0118df --- /dev/null +++ b/frontend/src/components/Charts/margins.tsx @@ -0,0 +1,227 @@ +// ACS estimates come from a survey sample, so each carries a 90% margin of +// error (MOE): the true value is likely within estimate ± MOE. The API sends a +// field's margin as `_MOE` and its reliability label (high, medium or +// low, from the estimate's coefficient of variation; see +// backend/data_cleaning/margins.py) as `_Reliability`. + +import { Area } from 'recharts'; +import { Text } from '@mantine/core'; +import { DataRow } from '@/types/cachedCharts'; + +export type Reliability = 'high' | 'medium' | 'low'; + +/** The API fields holding `field`'s margin of error and reliability label. */ +export const moeField = (field: string) => `${field}_MOE`; +export const reliabilityField = (field: string) => `${field}_Reliability`; + +/** Whether any row carries a margin for `field`. */ +export const hasMargins = (rows: DataRow[] | undefined, field: string) => + !!rows?.some((r) => r[moeField(field)] != null); + +// Plot rows keep a value's margin and reliability under keys derived from the +// value's key, ahead of any " (cmp)" suffix so the table view still pairs a +// comparison margin with its comparison value: "Under 18" -> "Under 18 MOE", +// "Under 18 (cmp)" -> "Under 18 MOE (cmp)". +const CMP_SUFFIX = ' (cmp)'; +const MOE_SUFFIX = ' MOE'; +const RELIABILITY_SUFFIX = ' Reliability'; + +const withoutCmp = (key: string) => + key.endsWith(CMP_SUFFIX) ? key.slice(0, -CMP_SUFFIX.length) : key; +const withSuffix = (key: string, suffix: string) => + key.endsWith(CMP_SUFFIX) + ? `${withoutCmp(key)}${suffix}${CMP_SUFFIX}` + : `${key}${suffix}`; + +export const moeKey = (key: string) => withSuffix(key, MOE_SUFFIX); +export const reliabilityKey = (key: string) => + withSuffix(key, RELIABILITY_SUFFIX); + +/** Whether a plot-row key holds a margin or reliability label, not a value. */ +export const isMarginKey = (key: string) => { + const base = withoutCmp(key); + return base.endsWith(MOE_SUFFIX) || base.endsWith(RELIABILITY_SUFFIX); +}; + +/** Puts a row's `field` on a plot point under `key`, and when the chart has + * margins, the row's margin and reliability alongside it. */ +export const plotValue = ( + pt: DataRow, + key: string, + row: DataRow | undefined, + field: string, + margins: boolean, +) => { + pt[key] = row?.[field] ?? null; + if (!margins) return; + pt[moeKey(key)] = row?.[moeField(field)] ?? null; + pt[reliabilityKey(key)] = row?.[reliabilityField(field)] ?? null; +}; + +/** The likely range, estimate ± margin, kept to values the measure can take. + * Every ACS measure with a margin is a count, dollar amount, age or share, so + * none goes below zero, and shares stay at or under 100% (a 100% share in a + * tiny tract can carry a ±257 margin). */ +export const interval = ( + value: number, + moe: number, + percent: boolean, +): [number, number] => [ + Math.max(0, value - moe), + percent ? Math.min(100, value + moe) : value + moe, +]; + +/** The interval around a plot point's value under `key`, or null when either + * the value or its margin is missing. */ +export const intervalAt = (pt: DataRow, key: string, percent: boolean) => { + const value = pt[key]; + const moe = pt[moeKey(key)]; + return typeof value === 'number' && typeof moe === 'number' + ? interval(value, moe, percent) + : null; +}; + +/** The margin of a sum: the root sum of squares of its parts' margins (Census + * ACS handbook). Summing shares of one total this way runs slightly wide, + * never narrow, since it leaves out the shared denominator. */ +export const sumMoe = (moes: unknown[]) => + moes.every((m) => typeof m === 'number') + ? Math.sqrt((moes as number[]).reduce((sum, m) => sum + m * m, 0)) + : null; + +// The cut points backend/data_cleaning/margins.py uses, for values the +// frontend derives itself: a coefficient of variation under 15% is high +// reliability, over 30% low. +const Z_90 = 1.645; + +export const reliabilityOf = ( + estimate: number | null, + moe: number | null, +): Reliability | null => { + if (estimate == null || moe == null) return null; + if (moe === 0) return 'high'; + if (estimate === 0) return 'low'; + const cv = moe / Z_90 / Math.abs(estimate); + return cv < 0.15 ? 'high' : cv <= 0.3 ? 'medium' : 'low'; +}; + +/** Whether any value in the plot rows has low reliability. */ +export const anyLowReliability = (rows: DataRow[]) => + rows.some((pt) => + Object.entries(pt).some( + ([k, v]) => v === 'low' && withoutCmp(k).endsWith(RELIABILITY_SUFFIX), + ), + ); + +/** "value ± margin" for a tooltip or table cell, plus `lowFlag` when the + * estimate has low reliability. Without a margin, just the value. A + * no-break space keeps "± margin" together when a narrow cell wraps. */ +export const withMargin = ( + text: string, + moe: unknown, + reliability: unknown, + formatMoe: (moe: number) => string, + lowFlag: string, +) => + typeof moe === 'number' + ? `${text} ±\u00a0${formatMoe(moe)}${reliability === 'low' ? lowFlag : ''}` + : text; + +// A CV over 30% means a margin of at least 0.30 × 1.645 ≈ half the estimate. +const LOW_RELIABILITY = + 'Low reliability: the margin of error is about half the estimate or more.'; + +/** The shaded range behind a trend line. Call it as a function inside the + * chart, not as a component, so Recharts sees the Area itself. */ +export const marginBand = ({ + dataKey, + color, + percent, + hide, + type = 'linear', +}: { + dataKey: string; + color: string; + percent: boolean; + hide?: boolean; + type?: 'linear' | 'monotone'; +}) => ( + intervalAt(pt, dataKey, percent)} + stroke="none" + fill={color} + fillOpacity={0.15} + legendType="none" + tooltipType="none" + activeDot={false} + isAnimationActive={false} + hide={hide} + /> +); + +/** A trend line's dots: hollow where the estimate has low reliability, and + * solid at every other point only when `allDots` (split series show them). */ +export const reliabilityDots = ( + dataKey: string, + color: string, + allDots: boolean, +) => + function ReliabilityDot({ + cx, + cy, + index, + payload, + }: { + cx?: number; + cy?: number; + index?: number; + payload?: DataRow; + }) { + const low = payload?.[reliabilityKey(dataKey)] === 'low'; + if (cx == null || cy == null || (!low && !allDots)) + return ; + return ( + + ); + }; + +/** The caption for a chart with margins: shaded bands behind trend lines, or + * whiskers on bars. */ +export const MarginNote = ({ + lowReliability, + bars = false, +}: { + lowReliability: boolean; + bars?: boolean; +}) => ( + + {bars ? 'The black line on each bar shows' : 'Shading shows'} the margin of + error: the true value is likely (90%) within it. + {lowReliability && + (bars + ? ' Hover over a bar to see whether its estimate has low reliability.' + : ' Hollow dots mark low-reliability estimates, whose margin is about half the estimate or more.')} + +); + +/** The footnote for a table with margins. */ +export const MarginFootnote = ({ + lowReliability, +}: { + lowReliability: boolean; +}) => ( + + ± is the margin of error: the true value is likely (90%) within that range. + {lowReliability && ` * ${LOW_RELIABILITY}`} + +); From 70556b725ea3c8a19e7194930a30410619ce3a50 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Mon, 2 Nov 2026 14:42:46 -0500 Subject: [PATCH 3/6] Serve tidy rows without margins, and close margin display gaps The tidy query now selects every column and keeps the margin fields only where the warehouse has them, as the timeseries query already did, so the API can deploy before the ETL rerun. The PDF report's summary table shows margins, the low-reliability asterisk and the footnote, sharing the on-screen table's cell formatting. Table footnotes explain only the cells on screen, not hidden comparison values. Bar whisker headroom follows the visible series, the PDF bar chart draws comparison whiskers when only the comparison has margins, and a single-year trend keeps its dot. --- backend/query/acs5.py | 18 +++ backend/query/sql/acs5/acs5_tidy.sql | 17 +-- backend/tests/test_acs5_tidy_query.py | 69 ++++++++++++ frontend/src/components/Charts/Bar.tsx | 25 +++-- .../components/Charts/DemographicsTable.tsx | 47 +++----- frontend/src/components/Charts/TableView.tsx | 12 +- frontend/src/components/Charts/margins.tsx | 72 +++++++++++- frontend/src/lib/pdfReport/ReportDocument.tsx | 103 +++++++++--------- 8 files changed, 251 insertions(+), 112 deletions(-) create mode 100644 backend/tests/test_acs5_tidy_query.py diff --git a/backend/query/acs5.py b/backend/query/acs5.py index 6266e6c6..245bb811 100644 --- a/backend/query/acs5.py +++ b/backend/query/acs5.py @@ -143,6 +143,21 @@ for suffix in ("MOE", "Reliability") } +# The tidy tables' columns the tidy endpoints return, under the Title Case +# names the frontend reads. A warehouse built before margins were collected +# has no margin columns; its responses just leave them out. +TIDY_FIELD_NAMES = { + "year": "year", + "section": "Section", + "variable": "Variable", + "value": "Value", + "value_moe": "Value_MOE", + "value_reliability": "Value_Reliability", + "percent": "Percent", + "percent_moe": "Percent_MOE", + "percent_reliability": "Percent_Reliability", +} + def _acs5_source( table: str, @@ -179,6 +194,9 @@ def get_acs5_tidy(dataset: str, filters: dict | None = None) -> pd.DataFrame: sql, params = sql_filter_block(sql_path / "acs5_tidy.sql", [source]) result = DB.execute(sql, params).df() + result = result[[c for c in TIDY_FIELD_NAMES if c in result.columns]].rename( + columns=TIDY_FIELD_NAMES + ) if result.empty: logger.error( diff --git a/backend/query/sql/acs5/acs5_tidy.sql b/backend/query/sql/acs5/acs5_tidy.sql index 4b7e84dc..77a4fb34 100644 --- a/backend/query/sql/acs5/acs5_tidy.sql +++ b/backend/query/sql/acs5/acs5_tidy.sql @@ -1,15 +1,6 @@ --- The response keeps the Title Case field names the frontend reads; the tidy --- tables store them lowercase, and DuckDB would otherwise echo that casing. -SELECT - year, - section AS Section, - variable AS Variable, - value AS Value, - value_moe AS Value_MOE, - value_reliability AS Value_Reliability, - percent AS Percent, - percent_moe AS Percent_MOE, - percent_reliability AS Percent_Reliability +-- Every column, so the margin columns come along where the warehouse has +-- them; get_acs5_tidy keeps and renames the fields the frontend reads. +SELECT * FROM {{ table }} {{ where_string }} -ORDER BY year, Section, Variable +ORDER BY year, section, variable diff --git a/backend/tests/test_acs5_tidy_query.py b/backend/tests/test_acs5_tidy_query.py new file mode 100644 index 00000000..e3a35fdf --- /dev/null +++ b/backend/tests/test_acs5_tidy_query.py @@ -0,0 +1,69 @@ +""" +The tidy ACS endpoints' query, run against a small in-memory tidy table +instead of the warehouse: Title Case fields, with each estimate's margin of +error where the warehouse has one. +""" + +import importlib + +import duckdb +import pytest + +from query import production_db + +ESTIMATES = ( + "year BIGINT, geoid VARCHAR, name VARCHAR, section VARCHAR, " + "variable VARCHAR, value DOUBLE, percent DOUBLE" +) +MARGINS = ( + "value_moe DOUBLE, value_reliability VARCHAR, " + "percent_moe DOUBLE, percent_reliability VARCHAR" +) +ROW = (2023, "5000144350", "Middlebury town", "Race", "Asian", 150.0, 1.8) + + +@pytest.fixture +def warehouse(monkeypatch): + conn = duckdb.connect() + monkeypatch.setattr(production_db, "get_db", lambda: conn) + module = importlib.import_module("query.acs5") + monkeypatch.setattr(module, "DB", conn) + return module, conn + + +def test_margins_come_back_with_their_estimates(warehouse): + acs5, conn = warehouse + conn.execute(f"CREATE TABLE acs5_demographics_tidy ({ESTIMATES}, {MARGINS})") + conn.execute( + "INSERT INTO acs5_demographics_tidy VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [*ROW, 80.0, "low", 1.0, "low"], + ) + df = acs5.get_acs5_tidy("demographics", {"Location": "Middlebury town"}) + + assert df.to_dict("records") == [ + { + "year": 2023, + "Section": "Race", + "Variable": "Asian", + "Value": 150.0, + "Value_MOE": 80.0, + "Value_Reliability": "low", + "Percent": 1.8, + "Percent_MOE": 1.0, + "Percent_Reliability": "low", + } + ] + + +def test_warehouse_without_margins_still_serves_its_estimates(warehouse): + # Built before margins were collected: the endpoints must not fail on the + # missing columns while the ETL catches up. + acs5, conn = warehouse + conn.execute(f"CREATE TABLE acs5_demographics_tidy ({ESTIMATES})") + conn.execute( + "INSERT INTO acs5_demographics_tidy VALUES (?, ?, ?, ?, ?, ?, ?)", list(ROW) + ) + df = acs5.get_acs5_tidy("demographics", {"Location": "Middlebury town"}) + + assert list(df.columns) == ["year", "Section", "Variable", "Value", "Percent"] + assert df.iloc[0].Value == 150.0 diff --git a/frontend/src/components/Charts/Bar.tsx b/frontend/src/components/Charts/Bar.tsx index 8d5c857a..e229f6e3 100644 --- a/frontend/src/components/Charts/Bar.tsx +++ b/frontend/src/components/Charts/Bar.tsx @@ -73,6 +73,18 @@ type MarginDataset = { intervals?: ([number, number] | null)[] }; const marginWhiskers: Plugin<'bar'> = { id: 'marginWhiskers', + // Chart.js scales the y-axis to the bars alone. Raise it to the tallest + // visible whisker here, on every layout, so hiding a series in the legend + // drops its whiskers' headroom too. + afterDataLimits(chartJS, { scale }) { + if (scale.id !== 'y') return; + chartJS.data.datasets.forEach((dataset, di) => { + const { intervals } = dataset as MarginDataset; + if (!intervals || !chartJS.isDatasetVisible(di)) return; + for (const range of intervals) + if (range && range[1] > scale.max) scale.max = range[1]; + }); + }, afterDatasetsDraw(chartJS) { const { ctx } = chartJS; const y = chartJS.scales.y; @@ -314,7 +326,9 @@ const CompareDiffPerXBarChartSVG = ({ } const percent = chart.chartParams.percentFormat ?? false; - const margins = hasMargins(filteredData, chart.yField); + const margins = + hasMargins(filteredData, chart.yField) || + hasMargins(filteredCompareData, chart.yField); // Merge primary + compare into one array for grouped bars const merged = filteredData.map((entry, i) => ({ @@ -487,10 +501,6 @@ const CompareDiffPerXBarChart = ({ const options = useMemo(() => { const format = (v: number) => percentFormat ? `${v}%` : v.toLocaleString(); - // Chart.js scales to the bars alone; leave room for the whiskers. - const whiskerTops = data.datasets.flatMap((ds) => - (ds.intervals ?? []).map((range) => range?.[1] ?? 0), - ); return { responsive: true, maintainAspectRatio: false, @@ -517,9 +527,6 @@ const CompareDiffPerXBarChart = ({ scales: { y: { title: { display: !!yLabel, text: yLabel }, - suggestedMax: whiskerTops.length - ? Math.max(...whiskerTops) - : undefined, ticks: { callback: (value: string | number) => percentFormat ? `${value}%` : value.toLocaleString(), @@ -527,7 +534,7 @@ const CompareDiffPerXBarChart = ({ }, }, }; - }, [view, percentFormat, yLabel, data, plotData, yField]); + }, [view, percentFormat, yLabel, plotData, yField]); if (isPdfMode) return ; if (!margins) return ; diff --git a/frontend/src/components/Charts/DemographicsTable.tsx b/frontend/src/components/Charts/DemographicsTable.tsx index 8e92ea5d..c8ee58f7 100644 --- a/frontend/src/components/Charts/DemographicsTable.tsx +++ b/frontend/src/components/Charts/DemographicsTable.tsx @@ -6,10 +6,12 @@ import { COLORS } from '@/app/theme'; import * as XLSX from 'xlsx'; import { saveWorksheet } from './TableView'; import { + FieldOf, MarginFootnote, + cellCompanion, + cellMarginFlags, + cellText, moeField, - reliabilityField, - withMargin, } from './margins'; // tidy ACS row shape consumed by the demographics tables @@ -24,27 +26,6 @@ const HOME_BG = 'var(--mantine-color-green-0)'; const COMP_BG = 'var(--mantine-color-blue-0)'; const SPLIT_BORDER = '1px solid var(--mantine-color-gray-3)'; -/** The field a row's cell shows. */ -type FieldOf = (row: AcsRow | undefined) => 'Percent' | 'Value'; - -const formatField = (field: 'Percent' | 'Value', v: number) => - field === 'Percent' ? `${v.toFixed(1)}%` : v.toLocaleString(); - -// A cell's value, with its margin of error (see margins.tsx) and an asterisk -// for a low-reliability estimate. -const cellText = (row: AcsRow | undefined, fieldOf: FieldOf) => { - const field = fieldOf(row); - const value = row?.[field]; - if (typeof value !== 'number') return '—'; - return withMargin( - formatField(field, value), - row?.[moeField(field)], - row?.[reliabilityField(field)], - (m) => formatField(field, m), - '*', - ); -}; - const DemographicsTableBase = ({ chart, fieldOf, @@ -57,14 +38,11 @@ const DemographicsTableBase = ({ const hasCompare = compareData.length > 0; const [showCompare, setShowCompare] = useState(false); - // The margin or reliability label of the value a row's cell shows. - const companion = (r: AcsRow | undefined, name: (field: string) => string) => - r?.[fieldOf(r)] != null ? r[name(fieldOf(r))] : null; + // The export always writes the comparison columns, but the footnote + // explains only the cells on screen. const allRows = [...data, ...compareData]; - const margins = allRows.some((r) => companion(r, moeField) != null); - const lowReliability = allRows.some( - (r) => companion(r, reliabilityField) === 'low', - ); + const exported = cellMarginFlags(allRows, fieldOf); + const shown = cellMarginFlags(showCompare ? allRows : data, fieldOf); const labels = chart.chartParams?.legendLabels as [string, string] | undefined; @@ -87,7 +65,8 @@ const DemographicsTableBase = ({ const sub = hasCompare ? [homeLabel, compareLabel] : []; const width = hasCompare ? 2 : 1; const num = (r: AcsRow | undefined) => r?.[fieldOf(r)] ?? ''; - const moe = (r: AcsRow | undefined) => companion(r, moeField) ?? ''; + const moe = (r: AcsRow | undefined) => + cellCompanion(r, fieldOf, moeField) ?? ''; const header = [ 'Variable', ...years.flatMap((y) => (hasCompare ? [y, ''] : [y])), @@ -105,7 +84,7 @@ const DemographicsTableBase = ({ ]; const body = variables.flatMap((variable) => [ line(String(variable), num, variable), - ...(margins ? [line(`${variable} MOE`, moe, variable)] : []), + ...(exported.margins ? [line(`${variable} MOE`, moe, variable)] : []), ]); const rows = hasCompare ? [header, ['', ...years.flatMap(() => sub)], ...body] @@ -232,7 +211,9 @@ const DemographicsTableBase = ({ - {margins && } + {shown.margins && ( + + )} ); }; diff --git a/frontend/src/components/Charts/TableView.tsx b/frontend/src/components/Charts/TableView.tsx index c9243178..dff4bb79 100644 --- a/frontend/src/components/Charts/TableView.tsx +++ b/frontend/src/components/Charts/TableView.tsx @@ -141,9 +141,17 @@ export const TableView = ({ const cmpKeys = allKeys.filter((k) => k.endsWith(CMP_SUFFIX)); const hasCompare = cmpKeys.length > 0 && rows.some((r) => cmpKeys.some((k) => r[k] != null)); - const margins = allKeys.some(isMarginKey); + // The footnote explains only the cells on screen, and comparison cells stay + // hidden until "Show Comparison". + const withCmp = hasCompare && showCompare; + const margins = rows.some((r) => + Object.entries(r).some( + ([k, v]) => + v != null && isMarginKey(k) && (withCmp || !k.endsWith(CMP_SUFFIX)), + ), + ); const footnote = margins && ( - + ); const handleExport = () => exportRowsToExcel(chart, rows); diff --git a/frontend/src/components/Charts/margins.tsx b/frontend/src/components/Charts/margins.tsx index 4b0118df..aa4cc60d 100644 --- a/frontend/src/components/Charts/margins.tsx +++ b/frontend/src/components/Charts/margins.tsx @@ -105,11 +105,15 @@ export const reliabilityOf = ( return cv < 0.15 ? 'high' : cv <= 0.3 ? 'medium' : 'low'; }; -/** Whether any value in the plot rows has low reliability. */ -export const anyLowReliability = (rows: DataRow[]) => +/** Whether any value in the plot rows has low reliability. Comparison values + * count only `withCmp`, since tables hide them until asked. */ +export const anyLowReliability = (rows: DataRow[], withCmp = true) => rows.some((pt) => Object.entries(pt).some( - ([k, v]) => v === 'low' && withoutCmp(k).endsWith(RELIABILITY_SUFFIX), + ([k, v]) => + v === 'low' && + (withCmp || !k.endsWith(CMP_SUFFIX)) && + withoutCmp(k).endsWith(RELIABILITY_SUFFIX), ), ); @@ -127,10 +131,61 @@ export const withMargin = ( ? `${text} ±\u00a0${formatMoe(moe)}${reliability === 'low' ? lowFlag : ''}` : text; +/** Which field a tidy-table row's cell shows: its share or its count. */ +export type FieldOf = (row: DataRow | undefined) => 'Percent' | 'Value'; + +const formatField = (field: 'Percent' | 'Value', v: number) => + field === 'Percent' ? `${v.toFixed(1)}%` : v.toLocaleString(); + +/** A tidy-table cell: the value `fieldOf` picks, with its margin of error and + * an asterisk for a low-reliability estimate. The on-screen tables and the + * PDF report's own table renderer both use it, so they read the same. */ +export const cellText = (row: DataRow | undefined, fieldOf: FieldOf) => { + const field = fieldOf(row); + const value = row?.[field]; + if (typeof value !== 'number') return '—'; + return withMargin( + formatField(field, value), + row?.[moeField(field)], + row?.[reliabilityField(field)], + (m) => formatField(field, m), + '*', + ); +}; + +/** The margin (`moeField`) or reliability label (`reliabilityField`) of the + * value a tidy-table row's cell shows. */ +export const cellCompanion = ( + row: DataRow | undefined, + fieldOf: FieldOf, + name: (field: string) => string, +) => { + const field = fieldOf(row); + return row && row[field] != null ? (row[name(field)] ?? null) : null; +}; + +/** Whether the cells for `rows` show any margin, and any low-reliability + * estimate: what a table's footnote needs to explain. */ +export const cellMarginFlags = (rows: DataRow[], fieldOf: FieldOf) => ({ + margins: rows.some((r) => cellCompanion(r, fieldOf, moeField) != null), + lowReliability: rows.some( + (r) => cellCompanion(r, fieldOf, reliabilityField) === 'low', + ), +}); + // A CV over 30% means a margin of at least 0.30 × 1.645 ≈ half the estimate. const LOW_RELIABILITY = 'Low reliability: the margin of error is about half the estimate or more.'; +/** The footnote text for a table with margins. */ +export const marginFootnote = (lowReliability: boolean) => + '± is the margin of error: the true value is likely (90%) within that ' + + `range.${lowReliability ? ` * ${LOW_RELIABILITY}` : ''}`; + +// Recharts draws a lone point's dot even with dot={false}; for a band, that +// would be a faint stray dot at the top of the range. +const noDot = () => ; + /** The shaded range behind a trend line. Call it as a function inside the * chart, not as a component, so Recharts sees the Area itself. */ export const marginBand = ({ @@ -155,6 +210,7 @@ export const marginBand = ({ fillOpacity={0.15} legendType="none" tooltipType="none" + dot={noDot} activeDot={false} isAnimationActive={false} hide={hide} @@ -173,14 +229,19 @@ export const reliabilityDots = ( cy, index, payload, + points, }: { cx?: number; cy?: number; index?: number; payload?: DataRow; + points?: readonly unknown[]; }) { const low = payload?.[reliabilityKey(dataKey)] === 'low'; - if (cx == null || cy == null || (!low && !allDots)) + // Recharts draws no line through a single point, so its dot is the only + // mark the series gets. + const only = points?.length === 1; + if (cx == null || cy == null || (!low && !allDots && !only)) return ; return ( ( - ± is the margin of error: the true value is likely (90%) within that range. - {lowReliability && ` * ${LOW_RELIABILITY}`} + {marginFootnote(lowReliability)} ); diff --git a/frontend/src/lib/pdfReport/ReportDocument.tsx b/frontend/src/lib/pdfReport/ReportDocument.tsx index 67fba174..e350fba5 100644 --- a/frontend/src/lib/pdfReport/ReportDocument.tsx +++ b/frontend/src/lib/pdfReport/ReportDocument.tsx @@ -22,6 +22,12 @@ import { StyleSheet, } from '@react-pdf/renderer'; import { ChartItem } from '@/types/cachedCharts'; +import { + FieldOf, + cellMarginFlags, + cellText, + marginFootnote, +} from '@/components/Charts/margins'; import { PdfChartEntry, PdfSection } from './types'; // --------------------------------------------------------------------------- @@ -202,6 +208,11 @@ const s = StyleSheet.create({ color: '#888', marginTop: 8, }, + tableFootnote: { + fontSize: 7, + color: '#666', + marginTop: 4, + }, noteCard: { backgroundColor: '#f9f9f9', padding: 8, @@ -221,24 +232,12 @@ const s = StyleSheet.create({ // Table renderer // --------------------------------------------------------------------------- -type RenderCellFn = (row: any) => string; - -const renderTablePercent: RenderCellFn = (row) => - row?.Percent != null ? `${row.Percent.toFixed(1)}%` : '—'; - -const renderTableValue: RenderCellFn = (row) => - row?.Value != null ? Number(row.Value).toLocaleString() : '—'; - -const renderTableMixed: RenderCellFn = (row) => { - if (row?.Percent != null) return `${row.Percent.toFixed(1)}%`; - if (row?.Value != null) return Number(row.Value).toLocaleString(); - return '—'; -}; - -function getCellRenderer(subtype: string): RenderCellFn { - if (subtype === 'renderTable') return renderTablePercent; - if (subtype === 'renderTableMixed') return renderTableMixed; - return renderTableValue; // renderTableEstimates and anything else +// The field each table subtype shows, as in Charts/DemographicsTable.tsx. +function getFieldOf(subtype: string): FieldOf { + if (subtype === 'renderTable') return () => 'Percent'; + if (subtype === 'renderTableMixed') + return (row) => (row?.Percent != null ? 'Percent' : 'Value'); + return () => 'Value'; // renderTableEstimates and anything else } const PdfTableSection = ({ chart }: { chart: ChartItem }) => { @@ -251,44 +250,50 @@ const PdfTableSection = ({ chart }: { chart: ChartItem }) => { const variables = Array.from( new Set(data.map((r) => r.Variable)), ) as string[]; - const renderCell = getCellRenderer(chart.subtype); + const fieldOf = getFieldOf(chart.subtype); + const { margins, lowReliability } = cellMarginFlags(data, fieldOf); const findRow = (variable: string, year: number) => data.find((r) => r.Variable === variable && r.year === year); return ( - - {/* Header */} - - Variable - {years.map((y) => ( - - {y} - + <> + + {/* Header */} + + Variable + {years.map((y) => ( + + {y} + + ))} + + + {/* Rows */} + {variables.map((variable, idx) => ( + + {variable} + {years.map((year) => { + const row = findRow(variable, year); + return ( + + {cellText(row, fieldOf)} + + ); + })} + ))} - - {/* Rows */} - {variables.map((variable, idx) => ( - - {variable} - {years.map((year) => { - const row = findRow(variable, year); - return ( - - {renderCell(row)} - - ); - })} - - ))} - + {margins && ( + {marginFootnote(lowReliability)} + )} + ); }; From a8d0e290d0ee2a43988f9b10d231e77532b46bd0 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Fri, 6 Nov 2026 10:00:39 -0500 Subject: [PATCH 4/6] Show margins of error on the comparison pages The variable explorer's map tooltips and scatter, the DP explorer's trend chart and value cards, and the county comparison's ranked bars now carry margins and reliability labels. Each query adds the margin columns only where the warehouse has them, so an older warehouse serves these pages as before. --- backend/.sqlfluff | 1 + .../api/routes/post_routes/post_acs5_db.py | 23 ++-- backend/query/comparison.py | 37 +++++- backend/query/county_comparison.py | 10 +- .../query/sql/acs5/acs5_county_compare.sql | 11 ++ .../sql/acs5/acs5_dp_county_comparison.sql | 6 +- backend/query/sql/acs5/acs5_town_compare.sql | 11 ++ backend/query/sql/acs5/acs5_tract_compare.sql | 6 + backend/query/sql_render.py | 7 +- backend/tests/test_comparison_margins.py | 106 ++++++++++++++++++ backend/tests/test_county_comparison.py | 35 ++++++ backend/tests/test_dp_series_route.py | 25 ++++- backend/tests/test_sql_render.py | 7 +- .../county-comparison/page.tsx | 6 + .../app/data-comparison/dp-explorer/page.tsx | 21 ++++ frontend/src/components/Charts/Bar.tsx | 95 +++++++++++++--- .../Charts/MapCorrespondentScatter.tsx | 27 ++++- .../src/components/Charts/TrendCharts.tsx | 64 +++++++++-- frontend/src/components/Charts/margins.tsx | 41 +++++++ frontend/src/components/mapping/index.tsx | 23 +++- 20 files changed, 509 insertions(+), 53 deletions(-) create mode 100644 backend/tests/test_comparison_margins.py diff --git a/backend/.sqlfluff b/backend/.sqlfluff index f3740201..04036b76 100644 --- a/backend/.sqlfluff +++ b/backend/.sqlfluff @@ -38,3 +38,4 @@ rule_string = CAST("Residential_Min_Lot" AS VARCHAR) AS residential_min_lot indicators = 'ASTHMA', 'DIABETES' path = dummy.csv min_acres = 10 +margins = True diff --git a/backend/api/routes/post_routes/post_acs5_db.py b/backend/api/routes/post_routes/post_acs5_db.py index cf78f00a..378e3a37 100644 --- a/backend/api/routes/post_routes/post_acs5_db.py +++ b/backend/api/routes/post_routes/post_acs5_db.py @@ -448,12 +448,14 @@ def dp_combined_tree(): # (see query/dp_variable_names.py). DP_IDENTITY_COLUMNS = ("variable_code", "source_label") +# Each value's 90% margin of error and reliability label, under the names the +# frontend reads beside Value. A warehouse built before margins were collected +# has neither, and responses then leave them out. +DP_MARGIN_COLUMNS = {"moe": "Value_MOE", "reliability": "Value_Reliability"} -def _dp_identity_columns() -> list[str]: - present = { - row[0] for row in DB.execute("DESCRIBE acs5_dp_combined_tidy").fetchall() - } - return [c for c in DP_IDENTITY_COLUMNS if c in present] + +def _dp_columns() -> set[str]: + return {row[0] for row in DB.execute("DESCRIBE acs5_dp_combined_tidy").fetchall()} @router.post("/load/acs5-db/dp-combined/series") @@ -465,8 +467,12 @@ def dp_combined_series(request: DPSeriesRequest): in a stable order so each keeps its place across years: by variable code when the warehouse has it, otherwise by load order. """ - identity = _dp_identity_columns() + present = _dp_columns() + identity = [c for c in DP_IDENTITY_COLUMNS if c in present] extra = "".join(f", {c}" for c in identity) + extra += "".join( + f", {c} AS {name}" for c, name in DP_MARGIN_COLUMNS.items() if c in present + ) tiebreak = "variable_code" if "variable_code" in identity else "rowid" rows = DB.execute( f"""--sql @@ -512,11 +518,12 @@ def county_comparison_indicators(): @router.post("/load/acs5-db/county-comparison") def county_comparison(request: CountyComparisonRequest): """Return {place, geo_type, year, value} for each county, Vermont and the - United States, every year the indicator has data.""" + United States, every year the indicator has data, with value_MOE and + value_Reliability where the warehouse has margins.""" indicator = INDICATORS_BY_KEY.get(request.indicator) if indicator is None or not indicator.available: raise HTTPException(404, f"Unknown indicator: {request.indicator}") - sql, params = comparison_query(indicator) + sql, params = comparison_query(indicator, margins="moe" in _dp_columns()) rows = DB.execute(sql, params).df() return make_response( data=rows, diff --git a/backend/query/comparison.py b/backend/query/comparison.py index b7746f4c..2aba0e5c 100644 --- a/backend/query/comparison.py +++ b/backend/query/comparison.py @@ -30,6 +30,8 @@ logger = logging.getLogger(__name__) sql_dir = Path(__file__).resolve().parent / "sql" +# A value's 90% margin of error and reliability label, where the source has them +MARGIN_COLUMNS = ("moe", "reliability") MIN_COMPLETE_ROWS = 3 # fewest geographies a composite index can be fit from # Census ACS sentinel for a suppressed/not-applicable estimate (e.g. median age # in a town too small to estimate). Never a real value, so drop it up front. @@ -223,6 +225,14 @@ def dataset_for_table(table: str) -> str: raise ValueError(f"unregistered comparison table: {table}") from e +def _has_margins(table: str) -> bool: + """Whether `table` carries ACS margins of error. A warehouse built before + margins were collected has none, and the ACS templates then leave their + `moe`/`reliability` columns out (CDC PLACES never has them).""" + columns = {row[0] for row in DB.execute(f"DESCRIBE {table}").fetchall()} + return "percent_moe" in columns + + def _fetch(cfg: dict, extra_filters: dict | None = None) -> pd.DataFrame: filters = dict(extra_filters or {}) if "geo_type" in cfg: @@ -230,7 +240,9 @@ def _fetch(cfg: dict, extra_filters: dict | None = None) -> pd.DataFrame: filters["year"] = [_latest_year(cfg["table"])] source = FilterSource(filter_table=cfg["table"], filters=filters) - sql, params = sql_filter_block(cfg["sql"], [source]) + sql, params = sql_filter_block( + cfg["sql"], [source], margins=_has_margins(cfg["table"]) + ) df = DB.execute(sql, params).df() df = df.rename( @@ -241,6 +253,9 @@ def _fetch(cfg: dict, extra_filters: dict | None = None) -> pd.DataFrame: cfg["name_col"]: "name", } ) + for column in MARGIN_COLUMNS: + if column not in df.columns: + df[column] = None return df[df["data_value"] != ACS_SENTINEL] @@ -261,7 +276,8 @@ def _single_variable( filters = dict(extra_filters or {}) filters[cfg["var_col"]] = [var] df = _fetch(cfg, filters) - sub = df[df["measure"] == var][["geoid", "geometry", "data_value", "name"]].copy() + columns = ["geoid", "geometry", "data_value", "name", *MARGIN_COLUMNS] + sub = df[df["measure"] == var][columns].copy() # Rows whose Jurisdiction/geo-id failed to match a geometry (a handful of # unparsed or non-geographic entries in the source tidy table) carry no # geoid at all. Left in, two such rows from either side of a comparison @@ -299,7 +315,8 @@ def compare_variables( its Crude and Age-adjusted rows, doubling every geography on the map). """ d1 = _single_variable(dataset1, level, var1, filters1) - d2 = _single_variable(dataset2, level, var2, filters2)[["geoid", "data_value"]] + d2 = _single_variable(dataset2, level, var2, filters2) + d2 = d2[["geoid", "data_value", *MARGIN_COLUMNS]] d1["bin"], edges_x = _bin3(d1["data_value"]) d2["bin"], edges_y = _bin3(d2["data_value"]) @@ -321,6 +338,20 @@ def compare_variables( var1: round(r.data_value_1, 2), var2: round(r.data_value_2, 2), } + margins = { + var: { + "moe": round(moe, 2), + "reliability": reliability if pd.notna(reliability) else None, + } + for var, moe, reliability in [ + (var1, r.moe_1, r.reliability_1), + (var2, r.moe_2, r.reliability_2), + ] + if pd.notna(moe) + } + if margins: + # Read by the map tooltip, which shows each value with its margin. + tooltip["__margins__"] = margins features.append( { "type": "Feature", diff --git a/backend/query/county_comparison.py b/backend/query/county_comparison.py index 2886a6e9..94d958af 100644 --- a/backend/query/county_comparison.py +++ b/backend/query/county_comparison.py @@ -19,11 +19,12 @@ GEO_TYPES = ("county", "state", "national") -def comparison_query(indicator: Indicator) -> tuple[str, list]: +def comparison_query(indicator: Indicator, margins: bool = False) -> tuple[str, list]: """Render the comparison query for one indicator; returns (sql, params). Rows are {place, geo_type, year, value}, where `place` is the short display - name ("Essex", "Vermont", "United States"). Values travel as $N + name ("Essex", "Vermont", "United States"), plus value_MOE and + value_Reliability when `margins` (the table has them). Values travel as $N placeholders: execute with `DB.execute(sql, params)`. """ params: list = [] @@ -46,4 +47,7 @@ def comparison_query(indicator: Indicator) -> tuple[str, list]: # with nulls, as query/acs5.py does. clauses.append("value >= 0") where_string = "WHERE " + "\n AND ".join(clauses) - return render_sql(SQL_PATH, table=SOURCE_TABLE, where_string=where_string), params + sql = render_sql( + SQL_PATH, table=SOURCE_TABLE, where_string=where_string, margins=margins + ) + return sql, params diff --git a/backend/query/sql/acs5/acs5_county_compare.sql b/backend/query/sql/acs5/acs5_county_compare.sql index 7e75262d..65a5098e 100644 --- a/backend/query/sql/acs5/acs5_county_compare.sql +++ b/backend/query/sql/acs5/acs5_county_compare.sql @@ -7,6 +7,17 @@ SELECT -- present; count-only variables (Population, Median Age, ...) have no -- Percent and fall back to Value unchanged. COALESCE(a.percent, a.value) AS value, + {% if margins %} + -- The margin and reliability label of whichever figure `value` is. A + -- warehouse built before margins were collected has neither column. + CASE + WHEN a.percent IS NOT NULL THEN a.percent_moe ELSE a.value_moe + END AS moe, + CASE + WHEN a.percent IS NOT NULL THEN a.percent_reliability + ELSE a.value_reliability + END AS reliability, + {% endif %} c.geoid, c.county, ST_ASGEOJSON(ST_GeomFromWKB(c.geometry)) AS geometry diff --git a/backend/query/sql/acs5/acs5_dp_county_comparison.sql b/backend/query/sql/acs5/acs5_dp_county_comparison.sql index b0836937..0b28a85f 100644 --- a/backend/query/sql/acs5/acs5_dp_county_comparison.sql +++ b/backend/query/sql/acs5/acs5_dp_county_comparison.sql @@ -9,7 +9,11 @@ SELECT REGEXP_REPLACE(name, ' County, Vermont$', '') AS place, geo_type, CAST(year AS INTEGER) AS year, - CAST(value AS DOUBLE) AS value + CAST(value AS DOUBLE) AS value{% if margins %}, + -- Named for the frontend, which reads a field's margin as _MOE. + moe AS value_MOE, + reliability AS value_Reliability +{% endif %} FROM {{ table }} {{ where_string }} ORDER BY year, place diff --git a/backend/query/sql/acs5/acs5_town_compare.sql b/backend/query/sql/acs5/acs5_town_compare.sql index eb727fc0..5680633b 100644 --- a/backend/query/sql/acs5/acs5_town_compare.sql +++ b/backend/query/sql/acs5/acs5_town_compare.sql @@ -3,6 +3,17 @@ SELECT -- see acs5_county_compare.sql: prefer the share-of-total (Percent) over -- the raw count (Value) when the variable has one. COALESCE(a.percent, a.value) AS value, + {% if margins %} + -- see acs5_county_compare.sql: the margin of whichever figure + -- `value` is. + CASE + WHEN a.percent IS NOT NULL THEN a.percent_moe ELSE a.value_moe + END AS moe, + CASE + WHEN a.percent IS NOT NULL THEN a.percent_reliability + ELSE a.value_reliability + END AS reliability, + {% endif %} t.FIPS_ID, t.TOWN_NAME, ST_ASGEOJSON(ST_GeomFromWKB(t.geometry)) AS geometry diff --git a/backend/query/sql/acs5/acs5_tract_compare.sql b/backend/query/sql/acs5/acs5_tract_compare.sql index b34abcf9..881829bd 100644 --- a/backend/query/sql/acs5/acs5_tract_compare.sql +++ b/backend/query/sql/acs5/acs5_tract_compare.sql @@ -3,6 +3,12 @@ SELECT a.variable, COALESCE(a.percent, a.value) AS value, + {% if margins %} + -- The share's margin and reliability label, where the warehouse has + -- them. + a.percent_moe AS moe, + a.percent_reliability AS reliability, + {% endif %} a.geoid, a.name, ST_ASGEOJSON(ST_GEOMFROMWKB(t.geometry)) AS geometry diff --git a/backend/query/sql_render.py b/backend/query/sql_render.py index 970ff751..417c7952 100644 --- a/backend/query/sql_render.py +++ b/backend/query/sql_render.py @@ -129,18 +129,21 @@ def compile_filters(sources: list[FilterSource], params: list) -> tuple[str, str return cte_block, join_block -def sql_filter_block(sql_path: Path, sources: list[FilterSource]) -> tuple[str, list]: +def sql_filter_block( + sql_path: Path, sources: list[FilterSource], **extra +) -> tuple[str, list]: """Render a query template with compiled filters; returns (sql, params). Only the fragments the template actually references are compiled, so `params` contains exactly the values behind the $N placeholders present in the SQL. + `extra` is passed to the template as is (e.g. a flag for optional columns). Execute with `DB.execute(sql, params)`. """ source_text = sql_path.read_text() used = meta.find_undeclared_variables(JINJA_ENV.parse(source_text)) params: list = [] - context: dict = {} + context: dict = dict(extra) if used & {"cte_filter_block", "join_filter_block"}: cte_block, join_block = compile_filters(sources, params) context["cte_filter_block"] = cte_block diff --git a/backend/tests/test_comparison_margins.py b/backend/tests/test_comparison_margins.py new file mode 100644 index 00000000..1069dcd4 --- /dev/null +++ b/backend/tests/test_comparison_margins.py @@ -0,0 +1,106 @@ +""" +Margins of error on the variable explorer's comparison map +(query.comparison.compare_variables), run against small in-memory tidy tables +instead of the warehouse: each town's tooltip carries the margin and +reliability label of both values, and a warehouse built before margins were +collected still serves its map without them. +""" + +import duckdb + +from query import comparison + +TOWNS = [ + ("5000100001", "ADDISON"), + ("5000100002", "BRISTOL"), + ("5000100003", "LINCOLN"), +] + +# (geoid, value, percent, value_moe, percent_moe, value_reliability, +# percent_reliability) per variable. Unemployment Rate is a count with its +# rate in percent, so the map shows the rate's margin; Median Age has no +# percent, so it shows the count's. +UNEMPLOYMENT = [ + ("5000100001", 40.0, 3.1, 15.0, 1.2, "medium", "medium"), + ("5000100002", 25.0, 2.0, 20.0, 1.5, "low", "low"), + ("5000100003", 60.0, 4.4, 18.0, 1.0, "high", "high"), +] +MEDIAN_AGE = [ + ("5000100001", 41.2, None, 2.1, None, "high", None), + ("5000100002", 38.5, None, 3.4, None, "high", None), + ("5000100003", 45.0, None, 6.0, None, "high", None), +] + + +def _db(margins: bool) -> duckdb.DuckDBPyConnection: + con = duckdb.connect() + con.execute("LOAD spatial") + con.execute( + """ + CREATE TABLE vt_town_lines_geom AS + SELECT * FROM (VALUES + ('5000100001', 'ADDISON', ST_AsWKB(ST_Point(0, 0))), + ('5000100002', 'BRISTOL', ST_AsWKB(ST_Point(1, 0))), + ('5000100003', 'LINCOLN', ST_AsWKB(ST_Point(2, 0))) + ) AS t(FIPS_ID, TOWN_NAME, geometry) + """ + ) + margin_columns = ( + ", value_moe DOUBLE, percent_moe DOUBLE, value_reliability VARCHAR," + " percent_reliability VARCHAR" + if margins + else "" + ) + for table, variable, rows in [ + ("acs5_economics_tidy", "Unemployment Rate", UNEMPLOYMENT), + ("acs5_demographics_tidy", "Median Age", MEDIAN_AGE), + ]: + con.execute( + f""" + CREATE TABLE {table} ( + year VARCHAR, geo_type VARCHAR, geoid VARCHAR, variable VARCHAR, + value DOUBLE, percent DOUBLE{margin_columns} + ) + """ + ) + for row in rows: + values = ["2023", "town", row[0], variable, *row[1:3]] + if margins: + values += row[3:] + con.execute( + f"INSERT INTO {table} VALUES ({', '.join('?' * len(values))})", + values, + ) + return con + + +def _tooltips(monkeypatch, margins: bool) -> dict[str, dict]: + monkeypatch.setattr(comparison, "DB", _db(margins)) + geojson, _legend = comparison.compare_variables( + "economics", "town", "Unemployment Rate", "demographics", "Median Age" + ) + return { + f["properties"]["tooltip"]["__title__"]: f["properties"]["tooltip"] + for f in geojson["features"] + } + + +def test_tooltips_carry_each_values_margin(monkeypatch): + bristol = _tooltips(monkeypatch, margins=True)["Bristol"] + + assert bristol["Unemployment Rate"] == 2.0 + assert bristol["__margins__"] == { + "Unemployment Rate": {"moe": 1.5, "reliability": "low"}, + "Median Age": {"moe": 3.4, "reliability": "high"}, + } + + +def test_warehouse_without_margins_still_serves_its_map(monkeypatch): + tooltips = _tooltips(monkeypatch, margins=False) + + assert tooltips["Bristol"] == { + "__title__": "Bristol", + "Unemployment Rate": 2.0, + "Median Age": 38.5, + } + assert len(tooltips) == len(TOWNS) diff --git a/backend/tests/test_county_comparison.py b/backend/tests/test_county_comparison.py index a059567e..2c31e15a 100644 --- a/backend/tests/test_county_comparison.py +++ b/backend/tests/test_county_comparison.py @@ -349,3 +349,38 @@ def test_warehouse_has_16_places_every_year(monkeypatch, indicator): first = 2013 if INDICATORS_BY_KEY[indicator].topic == "Disability" else 2010 _expect(min(per_year), first, f"{indicator}: first year") _expect(max(per_year), 2024, f"{indicator}: last year") + + +def test_margins_come_with_their_values(monkeypatch): + conn = duckdb.connect() + conn.execute( + """ + CREATE TABLE acs5_dp_combined_tidy ( + year BIGINT, name VARCHAR, geo_type VARCHAR, "table" VARCHAR, + category VARCHAR, subcategory VARCHAR, variable VARCHAR, + measure VARCHAR, value DOUBLE, moe DOUBLE, reliability VARCHAR + ) + """ + ) + conn.execute( + "INSERT INTO acs5_dp_combined_tidy VALUES " + "(2020, ?, 'county', 'DP02', ?, ?, 'Bachelor''s degree or higher', " + "'Percent', 18.4, 2.6, 'high')", + [ESSEX, EDU, ADULTS], + ) + response = _client(monkeypatch, conn).post( + "/load/acs5-db/county-comparison", json={"indicator": "edu_bachelors_or_higher"} + ) + (row,) = response.json()["data"] + assert (row["value"], row["value_MOE"], row["value_Reliability"]) == ( + 18.4, + 2.6, + "high", + ) + + +def test_warehouse_without_margins_leaves_them_out(monkeypatch): + response = _client(monkeypatch).post( + "/load/acs5-db/county-comparison", json={"indicator": "edu_bachelors_or_higher"} + ) + assert all("value_MOE" not in row for row in response.json()["data"]) diff --git a/backend/tests/test_dp_series_route.py b/backend/tests/test_dp_series_route.py index 437d8064..4a6aed2e 100644 --- a/backend/tests/test_dp_series_route.py +++ b/backend/tests/test_dp_series_route.py @@ -29,6 +29,9 @@ (2012, "DP04_0100E", "Housing units with a mortgage", "1764"), ] +# Published margins, by variable code +MOES = {"DP04_0107E": 61.0, "DP04_0100E": 88.0} + REQUEST = { "name": BURLINGTON, "table": "DP04", @@ -41,9 +44,11 @@ } -def _client(monkeypatch, with_identity: bool) -> TestClient: +def _client(monkeypatch, with_identity: bool, with_margins=False) -> TestClient: conn = duckdb.connect() identity = ", variable_code VARCHAR, source_label VARCHAR" if with_identity else "" + if with_margins: + identity += ", moe DOUBLE, reliability VARCHAR" conn.execute( f""" CREATE TABLE acs5_dp_combined_tidy ( @@ -58,6 +63,8 @@ def _client(monkeypatch, with_identity: bool) -> TestClient: base += ["Estimate", value] if with_identity: base += [code, LABEL.format(part)] + if with_margins: + base += [MOES[code], "high"] conn.execute( f"INSERT INTO acs5_dp_combined_tidy VALUES ({', '.join('?' * len(base))})", base, @@ -129,3 +136,19 @@ def test_without_identity_columns_rows_keep_load_order(monkeypatch): assert "variable_code" not in data[0] # Same position each year, so the chart's nth line stays one observation. assert [r["Value"] for r in data] == [699, 1731, 737, 1764] + + +@pytest.mark.parametrize("with_margins", [True, False]) +def test_margins_come_with_their_values(monkeypatch, with_margins): + rows = _client(monkeypatch, True, with_margins).post( + "/load/acs5-db/dp-combined/series", json=REQUEST + ) + first = rows.json()["data"][0] + if with_margins: + assert (first["Value"], first["Value_MOE"], first["Value_Reliability"]) == ( + 1731, + 88.0, + "high", + ) + else: + assert "Value_MOE" not in first and "Value_Reliability" not in first diff --git a/backend/tests/test_sql_render.py b/backend/tests/test_sql_render.py index 34851b63..bffc812a 100644 --- a/backend/tests/test_sql_render.py +++ b/backend/tests/test_sql_render.py @@ -291,10 +291,13 @@ def test_templates_pass_sqlfluff(): assert violations == [], [str(v) for v in violations] +@pytest.mark.parametrize("margins", [True, False]) @pytest.mark.parametrize("template", sorted(QUERY_TEMPLATE_SOURCES)) -def test_rendered_query_sql_parses(template): +def test_rendered_query_sql_parses(template, margins): + # `margins` adds the ACS templates' optional margin columns; the rest + # ignore it. sql, _params = sql_filter_block( - BACKEND / template, QUERY_TEMPLATE_SOURCES[template] + BACKEND / template, QUERY_TEMPLATE_SOURCES[template], margins=margins ) _assert_parses(sql, template) diff --git a/frontend/src/app/data-comparison/county-comparison/page.tsx b/frontend/src/app/data-comparison/county-comparison/page.tsx index 11d13862..d7fc4cc4 100644 --- a/frontend/src/app/data-comparison/county-comparison/page.tsx +++ b/frontend/src/app/data-comparison/county-comparison/page.tsx @@ -45,6 +45,10 @@ interface ComparisonRow { geo_type: 'county' | 'state' | 'national'; year: number; value: number; + /** The value's 90% margin of error and reliability label, where the + * warehouse has them. */ + value_MOE?: number | null; + value_Reliability?: string | null; } interface ComparisonMeta { @@ -215,6 +219,8 @@ export default function CountyComparisonPage() { .map((r) => ({ place: r.place, value: r.value, + value_MOE: r.value_MOE ?? null, + value_Reliability: r.value_Reliability ?? null, color: barColor(r, highlighted), })), [rows, year, highlighted], diff --git a/frontend/src/app/data-comparison/dp-explorer/page.tsx b/frontend/src/app/data-comparison/dp-explorer/page.tsx index a6c1d2e1..3569d67e 100644 --- a/frontend/src/app/data-comparison/dp-explorer/page.tsx +++ b/frontend/src/app/data-comparison/dp-explorer/page.tsx @@ -224,12 +224,17 @@ function ValueCard({ location, valueLabel, value, + margin, + lowReliability, accent, }: { location: string; /** Which of several values this is (see ValueRow). */ valueLabel?: string | null; value: string; + /** The value's margin of error, formatted, when it has one. */ + margin?: string | null; + lowReliability?: boolean; accent: string; }) { return ( @@ -266,6 +271,12 @@ function ValueCard({ > {value} + {margin && ( + + ± {margin} + {lowReliability && ' · low reliability'} + + )} ); } @@ -635,6 +646,12 @@ export default function DPExplorerPage() { const fmtVal = (v: number | null) => v != null ? (isPercent ? `${v}%` : Number(v).toLocaleString()) : '—'; const valueRows = pairValues(pointsA, pointsB); + const fmtMoe = (row?: DataRow) => { + const m = row?.Value_MOE; + if (typeof m !== 'number') return null; + return isPercent ? `${m}%` : m.toLocaleString(); + }; + const isLow = (row?: DataRow) => row?.Value_Reliability === 'low'; // --------------------------------------------------------------------------- // Render @@ -954,6 +971,8 @@ export default function DPExplorerPage() { location={makeLabel(sideA)} valueLabel={row.label} value={fmtVal(a)} + margin={a != null ? fmtMoe(row.a) : null} + lowReliability={isLow(row.a)} accent={COLORS.spruce} /> @@ -970,6 +989,8 @@ export default function DPExplorerPage() { location={makeLabel(sideB)} valueLabel={row.label} value={fmtVal(b)} + margin={b != null ? fmtMoe(row.b) : null} + lowReliability={isLow(row.b)} accent={COLORS.amber} /> diff --git a/frontend/src/components/Charts/Bar.tsx b/frontend/src/components/Charts/Bar.tsx index e229f6e3..76a1a8b0 100644 --- a/frontend/src/components/Charts/Bar.tsx +++ b/frontend/src/components/Charts/Bar.tsx @@ -44,6 +44,7 @@ import { moeField, moeKey, plotValue, + reliabilityField, reliabilityKey, withMargin, } from './margins'; @@ -948,6 +949,8 @@ const RankedBarChart = ({ const colorField = chart.chartParams?.color ?? 'color'; const unit = chart.chartParams?.unit; const yLabel = (chart.chartParams?.yLabel as string | undefined) ?? yField; + const percent = unit === 'percent'; + const margins = hasMargins(rows, yField); const fmt = (v: unknown, short = false) => { if (v == null) return ''; const n = Number(v); @@ -956,18 +959,66 @@ const RankedBarChart = ({ ? `$${(n / 1000).toLocaleString(undefined, { maximumFractionDigits: 1 })}k` : `$${n.toLocaleString(undefined, { maximumFractionDigits: 0 })}`; const num = n.toLocaleString(undefined, { maximumFractionDigits: 1 }); - return unit === 'percent' ? `${num}%` : num; + return percent ? `${num}%` : num; }; const fmtShort = (v: unknown) => fmt(v, true); + const plotRows: Record[] = useMemo( + () => + margins + ? rows.map((r) => ({ ...r, error: errorOffsets(r, yField, percent) })) + : rows, + [rows, margins, yField, percent], + ); - // The table view shows place and value only, not the bar colors. + // The table view shows place and value only, not the bar colors, with the + // value's margin where it has one. useEffect(() => { - onPlotData?.(rows.map((r) => ({ Place: r[xField], [yLabel]: r[yField] }))); - }, [rows, xField, yField, yLabel, onPlotData]); + onPlotData?.( + rows.map((r) => ({ + Place: r[xField], + [yLabel]: r[yField], + ...(margins && { + [moeKey(yLabel)]: r[moeField(yField)] ?? null, + [reliabilityKey(yLabel)]: r[reliabilityField(yField)] ?? null, + }), + })), + ); + }, [rows, xField, yField, yLabel, margins, onPlotData]); + + // Each bar's value sits above its whisker, not under it. + const valueLabel = (props: { + x?: unknown; + y?: unknown; + width?: unknown; + height?: unknown; + value?: unknown; + index?: number; + }) => { + const x = Number(props.x); + const y = Number(props.y); + const width = Number(props.width); + const height = Number(props.height); + const error = plotRows[props.index ?? -1]?.error as number[] | null; + const value = Number(props.value); + const above = error && value > 0 ? (error[1] / value) * height : 0; + return ( + + {fmtShort(props.value)} + + ); + }; - return ( + const plot = ( - + - [fmt(v), yLabel]} /> + [ + withMargin( + fmt(v), + item.payload?.[moeField(yField)], + item.payload?.[reliabilityField(yField)], + (m) => fmt(m), + ' (low reliability)', + ), + yLabel, + ]} + /> ( ))} - + {margins && } + ); + if (!margins || view === 'gallery') return plot; + + return ( + + r[reliabilityField(yField)] === 'low')} + bars + /> + {plot} + + ); }; export { diff --git a/frontend/src/components/Charts/MapCorrespondentScatter.tsx b/frontend/src/components/Charts/MapCorrespondentScatter.tsx index e5a95e46..d80714dc 100644 --- a/frontend/src/components/Charts/MapCorrespondentScatter.tsx +++ b/frontend/src/components/Charts/MapCorrespondentScatter.tsx @@ -19,6 +19,11 @@ import { ResponsiveContainer, } from 'recharts'; import { COLORS } from '@/app/theme'; +import { + isTooltipMeta, + tooltipLowReliability, + withTooltipMargin, +} from './margins'; type Legend = { measures: [string, string]; @@ -53,18 +58,21 @@ function makeScatterDot(activeId: string | null) { if (!Number.isFinite(cx) || !Number.isFinite(cy)) return null; const isActive = activeId != null && String(tooltip?.__title__ ?? '') === activeId; + // Hollow, in its map color, when either value has low reliability, as on + // the trend charts. + const low = !!tooltip && tooltipLowReliability(tooltip); return ( ); }; @@ -78,9 +86,9 @@ function PointTooltip({ content }: { content: Record }) { {Object.entries(content).map( ([k, v]) => - k !== '__title__' && ( + !isTooltipMeta(k) && (
- {k}: {String(v)} + {k}: {String(withTooltipMargin(content, k, v))}
), )} @@ -123,12 +131,19 @@ export default function VariableScatter({ .filter((p) => Number.isFinite(p.x) && Number.isFinite(p.y)); if (points.length === 0) return null; + const anyLow = points.some((p) => tooltipLowReliability(p.tooltip)); return ( - + {my} vs. {mx} + {anyLow && ( + + Hollow dots mark places where either estimate has low reliability: its + margin of error is about half the estimate or more. + + )} diff --git a/frontend/src/components/Charts/TrendCharts.tsx b/frontend/src/components/Charts/TrendCharts.tsx index 987a0cb4..2cfdea47 100644 --- a/frontend/src/components/Charts/TrendCharts.tsx +++ b/frontend/src/components/Charts/TrendCharts.tsx @@ -7,7 +7,6 @@ import { ComposedChart, Legend, Line, - LineChart, ResponsiveContainer, Tooltip, XAxis, @@ -1148,13 +1147,17 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { new Set([...data, ...compareData].map((r) => r.year)), ).sort((a, b) => Number(a) - Number(b)); + const margins = hasMargins(data, 'Value') || hasMargins(compareData, 'Value'); + const plotData = allYears.map((year) => { const pt: DataRow = { year }; primary.lines.forEach((line, i) => { - pt[`primary${i}`] = line.rows.find((r) => r.year === year)?.Value ?? null; + const row = line.rows.find((r) => r.year === year); + plotValue(pt, `primary${i}`, row, 'Value', margins); }); compare.lines.forEach((line, i) => { - pt[`compare${i}`] = line.rows.find((r) => r.year === year)?.Value ?? null; + const row = line.rows.find((r) => r.year === year); + plotValue(pt, `compare${i}`, row, 'Value', margins); }); return pt; }); @@ -1168,8 +1171,9 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { maxPerYear={Math.max(primary.maxPerX, compare.maxPerX)} duplicates={primary.duplicates + compare.duplicates} /> + {margins && } - @@ -1183,11 +1187,37 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { domain={['auto', 'auto']} /> { - return (Number(value) || 0).toLocaleString(); + formatter={(value, _name, item) => { + const key = String(item.dataKey); + return withMargin( + (Number(value) || 0).toLocaleString(), + item.payload?.[moeKey(key)], + item.payload?.[reliabilityKey(key)], + (m) => m.toLocaleString(), + ' (low reliability)', + ); }} /> + {margins && + primary.lines.map((_, i) => + marginBand({ + dataKey: `primary${i}`, + color: PRIMARY_SHADES[i % PRIMARY_SHADES.length], + percent: !!isPercent, + type: 'monotone', + }), + )} + {margins && + compareData.length > 0 && + compare.lines.map((_, i) => + marginBand({ + dataKey: `compare${i}`, + color: COMPARE_SHADES[i % COMPARE_SHADES.length], + percent: !!isPercent, + type: 'monotone', + }), + )} {primary.lines.map((_, i) => ( }) => { name={lineName(primaryName, primary, i)} stroke={PRIMARY_SHADES[i % PRIMARY_SHADES.length]} strokeWidth={2} - dot={primary.lines.length > 1} + dot={ + margins + ? reliabilityDots( + `primary${i}`, + PRIMARY_SHADES[i % PRIMARY_SHADES.length], + primary.lines.length > 1, + ) + : primary.lines.length > 1 + } /> ))} {compareData.length > 0 && @@ -1211,10 +1249,18 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { stroke={COMPARE_SHADES[i % COMPARE_SHADES.length]} strokeWidth={1.5} strokeDasharray="6 4" - dot={compare.lines.length > 1} + dot={ + margins + ? reliabilityDots( + `compare${i}`, + COMPARE_SHADES[i % COMPARE_SHADES.length], + compare.lines.length > 1, + ) + : compare.lines.length > 1 + } /> ))} - + ); diff --git a/frontend/src/components/Charts/margins.tsx b/frontend/src/components/Charts/margins.tsx index aa4cc60d..027e3905 100644 --- a/frontend/src/components/Charts/margins.tsx +++ b/frontend/src/components/Charts/margins.tsx @@ -285,3 +285,44 @@ export const MarginFootnote = ({ {marginFootnote(lowReliability)} ); + +/** A comparison map tooltip's margins, keyed by measure: the backend + * (query/comparison.py) sends them under this key beside the values. */ +const TOOLTIP_MARGINS = '__margins__'; +type TooltipMargins = Record< + string, + { moe: number; reliability: Reliability | null } +>; + +/** Whether a map tooltip key holds data for the card (`__title__`, + * `__margins__`) rather than a field to list. */ +export const isTooltipMeta = (key: string) => + key.startsWith('__') && key.endsWith('__'); + +const tooltipMargin = (tooltip: Record, key: string) => + (tooltip[TOOLTIP_MARGINS] as TooltipMargins | undefined)?.[key]; + +/** A map tooltip field's value as "value ± margin" when the tooltip carries + * its margin; otherwise the value as is. */ +export const withTooltipMargin = ( + tooltip: Record, + key: string, + value: unknown, +) => { + const margin = tooltipMargin(tooltip, key); + return margin + ? withMargin( + String(value), + margin.moe, + margin.reliability, + (m) => m.toLocaleString(), + ' (low reliability)', + ) + : value; +}; + +/** Whether any value in a map tooltip has low reliability. */ +export const tooltipLowReliability = (tooltip: Record) => + Object.values( + (tooltip[TOOLTIP_MARGINS] as TooltipMargins | undefined) ?? {}, + ).some((m) => m.reliability === 'low'); diff --git a/frontend/src/components/mapping/index.tsx b/frontend/src/components/mapping/index.tsx index a256ddb1..af95d21d 100644 --- a/frontend/src/components/mapping/index.tsx +++ b/frontend/src/components/mapping/index.tsx @@ -21,6 +21,7 @@ import { } from '@mantine/core'; import { IconX } from '@tabler/icons-react'; import { COLORS, FONTS } from '@/app/theme'; +import { isTooltipMeta, withTooltipMargin } from '@/components/Charts/margins'; import 'maplibre-gl/dist/maplibre-gl.css'; import type { MapRef } from 'react-map-gl/maplibre'; @@ -454,7 +455,7 @@ export default function VTMap({ {Object.entries(tooltip.content).map( ([k, v]) => - k !== '__title__' && + !isTooltipMeta(k) && v !== null && v !== undefined && v !== '' && ( @@ -474,7 +475,13 @@ export default function VTMap({ ta="right" style={{ color: COLORS.ink }} > - {formatDetailValue(k, v)} + {String( + withTooltipMargin( + tooltip.content, + k, + formatDetailValue(k, v), + ), + )} ), @@ -537,9 +544,15 @@ export default function VTMap({ {(() => { // Named `fields` rather than `Map` to avoid shadowing the // react-map-gl `Map` component imported at the top of this file. - const fields = Object.entries(selected).filter( - ([k]) => k !== '__title__', - ); + const fields = Object.entries(selected) + .filter(([k]) => !isTooltipMeta(k)) + .map( + ([k, v]) => + [k, withTooltipMargin(selected, k, v)] as [ + string, + unknown, + ], + ); const claimed = new Set(); const sections = DETAIL_GROUPS.map((group) => { From f5114332a1c46a03d186ebe9e10313c947de0fe3 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Fri, 6 Nov 2026 10:32:17 -0500 Subject: [PATCH 5/6] Let people hide margins of error on charts and tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A "Margins of error" switch on each chart card with margins, saved as one setting for every chart and on by default. Hidden, charts drop their bands and whiskers and tables their "± margin", but low-reliability estimates keep their hollow dots and asterisks. --- .../app/data-comparison/dp-explorer/page.tsx | 14 ++- frontend/src/components/Charts/Bar.tsx | 48 +++++--- .../components/Charts/DemographicsTable.tsx | 10 +- frontend/src/components/Charts/TableView.tsx | 27 ++-- .../src/components/Charts/TrendCharts.tsx | 24 +++- frontend/src/components/Charts/index.tsx | 10 ++ frontend/src/components/Charts/margins.tsx | 116 ++++++++++++++---- frontend/src/lib/pdfReport/ReportDocument.tsx | 10 +- 8 files changed, 189 insertions(+), 70 deletions(-) diff --git a/frontend/src/app/data-comparison/dp-explorer/page.tsx b/frontend/src/app/data-comparison/dp-explorer/page.tsx index 3569d67e..2e6d9e12 100644 --- a/frontend/src/app/data-comparison/dp-explorer/page.tsx +++ b/frontend/src/app/data-comparison/dp-explorer/page.tsx @@ -38,6 +38,7 @@ import { BASE_API_URL } from '@/config'; import { ChartStack } from '@/components/Charts'; import { createChartItem } from '@/utils/itemFactory'; import { identityField, identityNames } from '@/components/Charts/seriesLines'; +import { useShowMargins } from '@/components/Charts/margins'; import county_town_names from '@/data/county_town_names.json'; import { DataRow } from '@/types/cachedCharts'; import { COLORS, FONTS } from '@/app/theme'; @@ -232,7 +233,8 @@ function ValueCard({ /** Which of several values this is (see ValueRow). */ valueLabel?: string | null; value: string; - /** The value's margin of error, formatted, when it has one. */ + /** The value's margin of error, formatted, when it has one and margins + * are shown. */ margin?: string | null; lowReliability?: boolean; accent: string; @@ -271,10 +273,11 @@ function ValueCard({ > {value} - {margin && ( + {(margin || lowReliability) && ( - ± {margin} - {lowReliability && ' · low reliability'} + {[margin && `± ${margin}`, lowReliability && 'low reliability'] + .filter(Boolean) + .join(' · ')} )} @@ -360,6 +363,7 @@ function StatusPanel({ export default function DPExplorerPage() { const { myLocation, comparison, yearMax } = useProfile(); + const showMargins = useShowMargins(); // --------------------------------------------------------------------------- // Side A defaults from profile myLocation @@ -648,7 +652,7 @@ export default function DPExplorerPage() { const valueRows = pairValues(pointsA, pointsB); const fmtMoe = (row?: DataRow) => { const m = row?.Value_MOE; - if (typeof m !== 'number') return null; + if (!showMargins || typeof m !== 'number') return null; return isPercent ? `${m}%` : m.toLocaleString(); }; const isLow = (row?: DataRow) => row?.Value_Reliability === 'low'; diff --git a/frontend/src/components/Charts/Bar.tsx b/frontend/src/components/Charts/Bar.tsx index 76a1a8b0..9a20cf5b 100644 --- a/frontend/src/components/Charts/Bar.tsx +++ b/frontend/src/components/Charts/Bar.tsx @@ -46,6 +46,7 @@ import { plotValue, reliabilityField, reliabilityKey, + useShowMargins, withMargin, } from './margins'; import { @@ -330,6 +331,7 @@ const CompareDiffPerXBarChartSVG = ({ const margins = hasMargins(filteredData, chart.yField) || hasMargins(filteredCompareData, chart.yField); + const show = useShowMargins(); // Merge primary + compare into one array for grouped bars const merged = filteredData.map((entry, i) => ({ @@ -365,19 +367,19 @@ const CompareDiffPerXBarChartSVG = ({ {merged.map((_, i) => ( ))} - {margins && ( + {margins && show && ( )} - {margins && ( + {margins && show && ( )} ); - if (!margins) return plot; + if (!margins || !show) return plot; return ( @@ -424,6 +426,7 @@ const CompareDiffPerXBarChart = ({ const margins = hasMargins(filteredData, chart.yField) || hasMargins(filteredCompareData, chart.yField); + const show = useShowMargins(); // Derive exact plottable rows and report back to ChartCard for TableView const plotData = useMemo(() => { @@ -467,19 +470,21 @@ const CompareDiffPerXBarChart = ({ label: primaryLabel, data: filteredData.map((entry) => entry[yField] as number), backgroundColor: colors, - intervals: margins - ? plotData.map((pt) => intervalAt(pt, yField, percentFormat)) - : undefined, + intervals: + margins && show + ? plotData.map((pt) => intervalAt(pt, yField, percentFormat)) + : undefined, }, { label: compareLabel, data: filteredCompareData.map((entry) => entry[yField] as number), backgroundColor: filteredCompareData.map(() => '#D3D3D3'), - intervals: margins - ? plotData.map((pt) => - intervalAt(pt, `${yField} (cmp)`, percentFormat), - ) - : undefined, + intervals: + margins && show + ? plotData.map((pt) => + intervalAt(pt, `${yField} (cmp)`, percentFormat), + ) + : undefined, }, ], }; @@ -494,6 +499,7 @@ const CompareDiffPerXBarChart = ({ primaryLabel, compareLabel, margins, + show, plotData, percentFormat, ]); @@ -520,6 +526,7 @@ const CompareDiffPerXBarChart = ({ pt?.[reliabilityKey(key)], format, ' (low reliability)', + show, ); }, }, @@ -535,7 +542,7 @@ const CompareDiffPerXBarChart = ({ }, }, }; - }, [view, percentFormat, yLabel, plotData, yField]); + }, [view, percentFormat, yLabel, plotData, yField, show]); if (isPdfMode) return ; if (!margins) return ; @@ -543,7 +550,11 @@ const CompareDiffPerXBarChart = ({ return ( {view !== 'gallery' && ( - + )} @@ -951,6 +962,7 @@ const RankedBarChart = ({ const yLabel = (chart.chartParams?.yLabel as string | undefined) ?? yField; const percent = unit === 'percent'; const margins = hasMargins(rows, yField); + const show = useShowMargins(); const fmt = (v: unknown, short = false) => { if (v == null) return ''; const n = Number(v); @@ -964,10 +976,10 @@ const RankedBarChart = ({ const fmtShort = (v: unknown) => fmt(v, true); const plotRows: Record[] = useMemo( () => - margins + margins && show ? rows.map((r) => ({ ...r, error: errorOffsets(r, yField, percent) })) : rows, - [rows, margins, yField, percent], + [rows, margins, show, yField, percent], ); // The table view shows place and value only, not the bar colors, with the @@ -1041,6 +1053,7 @@ const RankedBarChart = ({ item.payload?.[reliabilityField(yField)], (m) => fmt(m), ' (low reliability)', + show, ), yLabel, ]} @@ -1052,7 +1065,9 @@ const RankedBarChart = ({ {rows.map((r, i) => ( ))} - {margins && } + {margins && show && ( + + )} @@ -1065,6 +1080,7 @@ const RankedBarChart = ({ r[reliabilityField(yField)] === 'low')} bars + show={show} /> {plot} diff --git a/frontend/src/components/Charts/DemographicsTable.tsx b/frontend/src/components/Charts/DemographicsTable.tsx index c8ee58f7..3f16f647 100644 --- a/frontend/src/components/Charts/DemographicsTable.tsx +++ b/frontend/src/components/Charts/DemographicsTable.tsx @@ -12,6 +12,7 @@ import { cellMarginFlags, cellText, moeField, + useShowMargins, } from './margins'; // tidy ACS row shape consumed by the demographics tables @@ -37,6 +38,7 @@ const DemographicsTableBase = ({ const compareData = chart.compareData ?? []; const hasCompare = compareData.length > 0; const [showCompare, setShowCompare] = useState(false); + const show = useShowMargins(); // The export always writes the comparison columns, but the footnote // explains only the cells on screen. @@ -173,7 +175,7 @@ const DemographicsTableBase = ({ if (!showCompare) { return ( - {cellText(mainRow, fieldOf)} + {cellText(mainRow, fieldOf, show)} ); } @@ -189,7 +191,7 @@ const DemographicsTableBase = ({ textAlign: 'right', }} > - {cellText(mainRow, fieldOf)} + {cellText(mainRow, fieldOf, show)}
- {cellText(cmpRow, fieldOf)} + {cellText(cmpRow, fieldOf, show)}
@@ -212,7 +214,7 @@ const DemographicsTableBase = ({ {shown.margins && ( - + )} ); diff --git a/frontend/src/components/Charts/TableView.tsx b/frontend/src/components/Charts/TableView.tsx index dff4bb79..4c3757c4 100644 --- a/frontend/src/components/Charts/TableView.tsx +++ b/frontend/src/components/Charts/TableView.tsx @@ -21,6 +21,7 @@ import { isMarginKey, moeKey, reliabilityKey, + useShowMargins, withMargin, } from './margins'; @@ -44,8 +45,9 @@ const formatCell = (v: unknown) => { }; // A value's cell, with its margin of error (see margins.tsx) when the row -// has one and an asterisk for a low-reliability estimate. -const valueCell = (row: DataRow | undefined, key: string) => +// has one and margins are shown, and an asterisk for a low-reliability +// estimate. +const valueCell = (row: DataRow | undefined, key: string, show: boolean) => row?.[key] == null ? formatCell(row?.[key]) : withMargin( @@ -54,6 +56,7 @@ const valueCell = (row: DataRow | undefined, key: string) => row[reliabilityKey(key)], formatNumber, '*', + show, ); /** Write a worksheet to a one-sheet .xlsx named after the chart and download it. */ @@ -127,6 +130,7 @@ export const TableView = ({ }: TableViewProps) => { const rows = (rowsOverride ?? chart.tableData ?? chart.data) as DataRow[]; const [showCompare, setShowCompare] = useState(false); + const show = useShowMargins(); const usePivot = !!chart.trendChart; if (!rows || rows.length === 0) return null; @@ -151,7 +155,10 @@ export const TableView = ({ ), ); const footnote = margins && ( - + ); const handleExport = () => exportRowsToExcel(chart, rows); @@ -246,7 +253,7 @@ export const TableView = ({ if (!showCompare || !hasCompare) { return ( - {valueCell(row, key)} + {valueCell(row, key, show)} ); } @@ -261,7 +268,7 @@ export const TableView = ({ textAlign: 'right', }} > - {valueCell(row, key)} + {valueCell(row, key, show)}
({ textAlign: 'right', }} > - {valueCell(row, `${key}${CMP_SUFFIX}`)} + {valueCell(row, `${key}${CMP_SUFFIX}`, show)}
@@ -327,7 +334,7 @@ export const TableView = ({ textAlign: 'right', }} > - {valueCell(row, column)} + {valueCell(row, column, show)}
({ textAlign: 'right', }} > - {valueCell(row, `${column}${CMP_SUFFIX}`)} + {valueCell(row, `${column}${CMP_SUFFIX}`, show)}
@@ -346,7 +353,9 @@ export const TableView = ({ } return ( - {valueCell(row, column)} + + {valueCell(row, column, show)} + ); })} diff --git a/frontend/src/components/Charts/TrendCharts.tsx b/frontend/src/components/Charts/TrendCharts.tsx index 2cfdea47..cd5793b0 100644 --- a/frontend/src/components/Charts/TrendCharts.tsx +++ b/frontend/src/components/Charts/TrendCharts.tsx @@ -28,6 +28,7 @@ import { reliabilityKey, reliabilityOf, sumMoe, + useShowMargins, withMargin, } from './margins'; @@ -99,6 +100,7 @@ const tooltipText = ( fmt: (typeof FORMATTERS)[FormatType], value: unknown, item: { dataKey?: unknown; payload?: DataRow }, + show: boolean, decimals?: number, ) => { const key = String(item.dataKey); @@ -108,6 +110,7 @@ const tooltipText = ( item.payload?.[reliabilityKey(key)], (m) => fmt.margin(m, decimals), ' (low reliability)', + show, ); }; @@ -222,6 +225,7 @@ export const SingleSeriesTrendChart = ({ const margins = hasMargins(data, valueField) || hasMargins(compareData, valueField); + const show = useShowMargins(); const plotData = useMemo(() => { if (!data || data.length === 0) return []; @@ -276,7 +280,7 @@ export const SingleSeriesTrendChart = ({ )} {margins && !isGallery && ( - + )} ({ {!isGallery && ( - tooltipText(fmt, v, item, decimals) + tooltipText(fmt, v, item, show, decimals) } /> )} @@ -317,6 +321,7 @@ export const SingleSeriesTrendChart = ({ /> )} {margins && + show && own.lines.map((_, i) => marginBand({ dataKey: ownKey(i), @@ -327,6 +332,7 @@ export const SingleSeriesTrendChart = ({ }), )} {margins && + show && hasCompare && other.lines.map((_, i) => marginBand({ @@ -523,6 +529,7 @@ export const MultiSeriesTrendChart = ({ const margins = hasMargins(data, valueField) || hasMargins(compareData, valueField); + const show = useShowMargins(); const plotData = useMemo(() => { if (!data || data.length === 0) return []; @@ -585,7 +592,7 @@ export const MultiSeriesTrendChart = ({ )} {margins && !isGallery && ( - + )} ({ /> {!isGallery && ( tooltipText(fmt, v, item)} + formatter={(v, _name, item) => tooltipText(fmt, v, item, show)} /> )} {!isGallery && ( @@ -620,6 +627,7 @@ export const MultiSeriesTrendChart = ({ /> )} {margins && + show && series.flatMap((s) => [ marginBand({ dataKey: s.key, @@ -1148,6 +1156,7 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { ).sort((a, b) => Number(a) - Number(b)); const margins = hasMargins(data, 'Value') || hasMargins(compareData, 'Value'); + const show = useShowMargins(); const plotData = allYears.map((year) => { const pt: DataRow = { year }; @@ -1171,7 +1180,9 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { maxPerYear={Math.max(primary.maxPerX, compare.maxPerX)} duplicates={primary.duplicates + compare.duplicates} /> - {margins && } + {margins && ( + + )} }) => { item.payload?.[reliabilityKey(key)], (m) => m.toLocaleString(), ' (low reliability)', + show, ); }} /> {margins && + show && primary.lines.map((_, i) => marginBand({ dataKey: `primary${i}`, @@ -1209,6 +1222,7 @@ export const DPTrendChart = ({ chart }: { chart: ChartItem }) => { }), )} {margins && + show && compareData.length > 0 && compare.lines.map((_, i) => marginBand({ diff --git a/frontend/src/components/Charts/index.tsx b/frontend/src/components/Charts/index.tsx index 02e77be3..7265e1bb 100644 --- a/frontend/src/components/Charts/index.tsx +++ b/frontend/src/components/Charts/index.tsx @@ -64,6 +64,7 @@ import * as motion from 'motion/react-client'; import { AddChart, RemoveChart } from './saving'; import { useEffect, useRef, useState } from 'react'; import { TableView, ViewSwitch } from './TableView'; +import { MarginsSwitch } from './margins'; import { usePdfMode } from '@/contexts/PdfModeContext'; import { useItems } from '@/components/ItemsProvider'; import classes from './ChartCard.module.css'; @@ -97,6 +98,12 @@ interface ChartCardProps { listeners: DraggableSyntheticListeners; }; } +/** Whether any of a chart's rows has a margin of error to show or hide. */ +const carriesMargins = (chart: ChartItem) => + [...(chart.data ?? []), ...(chart.compareData ?? [])].some((row) => + Object.entries(row).some(([k, v]) => k.endsWith('_MOE') && v != null), + ); + export const ChartCard = ({ chart, ChartComponent, @@ -296,6 +303,9 @@ export const ChartCard = ({ )} + {!isPdfMode && !isGallery && carriesMargins(chart) && ( + + )} {!isPdfMode && showViewSwitch && ( )} diff --git a/frontend/src/components/Charts/margins.tsx b/frontend/src/components/Charts/margins.tsx index 027e3905..63a74579 100644 --- a/frontend/src/components/Charts/margins.tsx +++ b/frontend/src/components/Charts/margins.tsx @@ -5,11 +5,44 @@ // backend/data_cleaning/margins.py) as `_Reliability`. import { Area } from 'recharts'; -import { Text } from '@mantine/core'; +import { Switch, Text } from '@mantine/core'; +import { create } from 'zustand'; +import { persist } from 'zustand/middleware'; import { DataRow } from '@/types/cachedCharts'; export type Reliability = 'high' | 'medium' | 'low'; +/** Whether charts and tables show margins of error: one setting for every + * chart, saved in localStorage, on by default. Low-reliability marks (hollow + * dots, "*") show either way, since they matter most to someone who has + * hidden the detail. */ +const useMarginPreference = create<{ + show: boolean; + setShow: (show: boolean) => void; +}>()( + persist((set) => ({ show: true, setShow: (show) => set({ show }) }), { + name: 'show-margins', + }), +); + +export const useShowMargins = () => useMarginPreference((s) => s.show); + +/** The current setting, for code outside React (the PDF report). */ +export const showMarginsNow = () => useMarginPreference.getState().show; + +/** The show/hide switch, for a chart card that has margins to show. */ +export const MarginsSwitch = () => { + const { show, setShow } = useMarginPreference(); + return ( + setShow(e.currentTarget.checked)} + /> + ); +}; + /** The API fields holding `field`'s margin of error and reliability label. */ export const moeField = (field: string) => `${field}_MOE`; export const reliabilityField = (field: string) => `${field}_Reliability`; @@ -118,18 +151,22 @@ export const anyLowReliability = (rows: DataRow[], withCmp = true) => ); /** "value ± margin" for a tooltip or table cell, plus `lowFlag` when the - * estimate has low reliability. Without a margin, just the value. A - * no-break space keeps "± margin" together when a narrow cell wraps. */ + * estimate has low reliability. Without a margin, or when margins are hidden + * (`show` false), the value and the flag. A no-break space keeps "± margin" + * together when a narrow cell wraps. */ export const withMargin = ( text: string, moe: unknown, reliability: unknown, formatMoe: (moe: number) => string, lowFlag: string, -) => - typeof moe === 'number' - ? `${text} ±\u00a0${formatMoe(moe)}${reliability === 'low' ? lowFlag : ''}` - : text; + show = true, +) => { + const flag = reliability === 'low' ? lowFlag : ''; + return show && typeof moe === 'number' + ? `${text} ±\u00a0${formatMoe(moe)}${flag}` + : `${text}${flag}`; +}; /** Which field a tidy-table row's cell shows: its share or its count. */ export type FieldOf = (row: DataRow | undefined) => 'Percent' | 'Value'; @@ -140,7 +177,11 @@ const formatField = (field: 'Percent' | 'Value', v: number) => /** A tidy-table cell: the value `fieldOf` picks, with its margin of error and * an asterisk for a low-reliability estimate. The on-screen tables and the * PDF report's own table renderer both use it, so they read the same. */ -export const cellText = (row: DataRow | undefined, fieldOf: FieldOf) => { +export const cellText = ( + row: DataRow | undefined, + fieldOf: FieldOf, + show = true, +) => { const field = fieldOf(row); const value = row?.[field]; if (typeof value !== 'number') return '—'; @@ -150,6 +191,7 @@ export const cellText = (row: DataRow | undefined, fieldOf: FieldOf) => { row?.[reliabilityField(field)], (m) => formatField(field, m), '*', + show, ); }; @@ -177,10 +219,16 @@ export const cellMarginFlags = (rows: DataRow[], fieldOf: FieldOf) => ({ const LOW_RELIABILITY = 'Low reliability: the margin of error is about half the estimate or more.'; -/** The footnote text for a table with margins. */ -export const marginFootnote = (lowReliability: boolean) => - '± is the margin of error: the true value is likely (90%) within that ' + - `range.${lowReliability ? ` * ${LOW_RELIABILITY}` : ''}`; +/** The footnote text for a table with margins; with margins hidden, only the + * low-reliability note, if any. */ +export const marginFootnote = (lowReliability: boolean, show = true) => { + const low = lowReliability ? `* ${LOW_RELIABILITY}` : ''; + if (!show) return low; + return ( + '± is the margin of error: the true value is likely (90%) within that ' + + `range.${low && ` ${low}`}` + ); +}; // Recharts draws a lone point's dot even with dot={false}; for a band, that // would be a faint stray dot at the top of the range. @@ -261,30 +309,44 @@ export const reliabilityDots = ( export const MarginNote = ({ lowReliability, bars = false, + show = true, }: { lowReliability: boolean; bars?: boolean; -}) => ( - - {bars ? 'The black line on each bar shows' : 'Shading shows'} the margin of - error: the true value is likely (90%) within it. - {lowReliability && - (bars - ? ' Hover over a bar to see whether its estimate has low reliability.' - : ' Hollow dots mark low-reliability estimates, whose margin is about half the estimate or more.')} - -); + /** Whether the margins are drawn; hidden, only the low-reliability note + * remains. */ + show?: boolean; +}) => { + const low = + lowReliability && + (bars + ? 'Hover over a bar to see whether its estimate has low reliability.' + : 'Hollow dots mark low-reliability estimates, whose margin is about half the estimate or more.'); + if (!show && !low) return null; + return ( + + {show && + `${bars ? 'The black line on each bar shows' : 'Shading shows'} the margin of error: the true value is likely (90%) within it. `} + {low} + + ); +}; /** The footnote for a table with margins. */ export const MarginFootnote = ({ lowReliability, + show = true, }: { lowReliability: boolean; -}) => ( - - {marginFootnote(lowReliability)} - -); + show?: boolean; +}) => { + const text = marginFootnote(lowReliability, show); + return text ? ( + + {text} + + ) : null; +}; /** A comparison map tooltip's margins, keyed by measure: the backend * (query/comparison.py) sends them under this key beside the values. */ diff --git a/frontend/src/lib/pdfReport/ReportDocument.tsx b/frontend/src/lib/pdfReport/ReportDocument.tsx index e350fba5..9da519db 100644 --- a/frontend/src/lib/pdfReport/ReportDocument.tsx +++ b/frontend/src/lib/pdfReport/ReportDocument.tsx @@ -27,6 +27,7 @@ import { cellMarginFlags, cellText, marginFootnote, + showMarginsNow, } from '@/components/Charts/margins'; import { PdfChartEntry, PdfSection } from './types'; @@ -252,6 +253,9 @@ const PdfTableSection = ({ chart }: { chart: ChartItem }) => { ) as string[]; const fieldOf = getFieldOf(chart.subtype); const { margins, lowReliability } = cellMarginFlags(data, fieldOf); + // Margins as the viewer set them on screen. + const show = showMarginsNow(); + const footnote = margins && marginFootnote(lowReliability, show); const findRow = (variable: string, year: number) => data.find((r) => r.Variable === variable && r.year === year); @@ -283,16 +287,14 @@ const PdfTableSection = ({ chart }: { chart: ChartItem }) => { const row = findRow(variable, year); return ( - {cellText(row, fieldOf)} + {cellText(row, fieldOf, show)} ); })} ))} - {margins && ( - {marginFootnote(lowReliability)} - )} + {footnote && {footnote}} ); }; From 0bdcd8681b1429075464b80a32c8bd7d44229854 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Fri, 6 Nov 2026 10:32:45 -0500 Subject: [PATCH 6/6] Install DuckDB's spatial extension in the comparison map test CI's fresh environment doesn't have it, so the test failed to load it. --- backend/tests/test_comparison_margins.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/tests/test_comparison_margins.py b/backend/tests/test_comparison_margins.py index 1069dcd4..f2ed1b45 100644 --- a/backend/tests/test_comparison_margins.py +++ b/backend/tests/test_comparison_margins.py @@ -7,6 +7,7 @@ """ import duckdb +import pytest from query import comparison @@ -34,7 +35,14 @@ def _db(margins: bool) -> duckdb.DuckDBPyConnection: con = duckdb.connect() - con.execute("LOAD spatial") + # The comparison queries build GeoJSON with the spatial extension, which a + # fresh environment (CI) doesn't have yet; installing is a no-op once it's + # there. + try: + con.install_extension("spatial") + con.load_extension("spatial") + except duckdb.Error as e: + pytest.skip(f"DuckDB spatial extension unavailable: {e}") con.execute( """ CREATE TABLE vt_town_lines_geom AS