From ac98b48621cc6dbb4ae148d21b21c8e98347793b Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Thu, 29 Oct 2026 12:21:40 -0400 Subject: [PATCH 1/3] Get the backend test suite green and run it in CI - Retire the dead /load/census route, query/timeseries_db.py, its tests and golden CSVs. The frontend uses /load/acs5-db/timeseries/*; the only caller (Filter_Call.tsx, also removed) was never imported and hit a path the route did not match, and the acs5_* tables it read are gone from the warehouse. - Point test_sql_render at the current templates (build/sql moved to data_cleaning/sql) and fix the stale compiled-CTE expectation. Fix the SQL lint debt that test now catches; rendered output is unchanged. - Update test_clean_acs5 for the DOUBLE value column and add a fixture test for the "(X)" and sentinel-code nulling. - conftest: give import-time get_db() an empty in-memory DuckDB when no warehouse is built, and skip `requires_warehouse` tests instead. - Rename the DuckLake diagnostic test_lake.py to inspect_lake.py so pytest no longer opens the lake on collection. - Add a Backend tests workflow running the whole suite. - Fix stale build/sql paths in the justfile, CLAUDE.md and .sqlfluff. --- .github/workflows/backend-tests.yml | 38 ++++ CLAUDE.md | 10 +- backend/.sqlfluff | 4 +- backend/api/routes/post_routes/__init__.py | 2 - backend/api/routes/post_routes/post_census.py | 67 ------- .../data_cleaning/sql/zoning_empty_geom.sql | 4 +- backend/data_cleaning/sql/zoning_rules.sql | 2 +- .../query/sql/acs5/acs5_county_compare.sql | 4 +- backend/query/sql/acs5/acs5_timeseries.sql | 2 +- .../sql/ambulance/ambulance_geo_query.sql | 4 +- backend/query/sql/cdc/county_places.sql | 12 +- .../wastewater/soil_suitability_geo_query.sql | 8 +- backend/query/sql/zoning/footprints.sql | 6 +- backend/query/sql/zoning/unzoned.sql | 4 +- backend/query/sql_render.py | 7 +- backend/query/timeseries_db.py | 135 -------------- backend/tests/conftest.py | 34 ++++ .../golden/commute_habits_addison_town.csv | 40 ---- .../tests/golden/commute_habits_vergennes.csv | 1 - .../golden/commute_time_addison_town.csv | 14 -- .../tests/golden/commute_time_vergennes.csv | 1 - .../historic_population_addison_town.csv | 25 --- .../golden/historic_population_vergennes.csv | 1 - .../golden/med_home_value_addison_town.csv | 16 -- .../tests/golden/med_home_value_vergennes.csv | 1 - .../tests/golden/med_smoc_addison_town.csv | 31 ---- backend/tests/golden/med_smoc_vergennes.csv | 1 - .../golden/median_earnings_addison_town.csv | 40 ---- .../golden/median_earnings_vergennes.csv | 1 - .../golden/unemployment_rate_addison_town.csv | 14 -- .../golden/unemployment_rate_vergennes.csv | 1 - .../tests/{test_lake.py => inspect_lake.py} | 2 +- backend/tests/test_census_timeseries_db.py | 171 ------------------ backend/tests/test_clean_acs5.py | 59 +++++- backend/tests/test_sql_render.py | 80 +++++--- docs/guides/mcp.md | 13 +- .../src/components/FilterUI/Filter_Call.tsx | 13 -- justfile | 4 +- 38 files changed, 228 insertions(+), 644 deletions(-) create mode 100644 .github/workflows/backend-tests.yml delete mode 100644 backend/api/routes/post_routes/post_census.py delete mode 100644 backend/query/timeseries_db.py delete mode 100644 backend/tests/golden/commute_habits_addison_town.csv delete mode 100644 backend/tests/golden/commute_habits_vergennes.csv delete mode 100644 backend/tests/golden/commute_time_addison_town.csv delete mode 100644 backend/tests/golden/commute_time_vergennes.csv delete mode 100644 backend/tests/golden/historic_population_addison_town.csv delete mode 100644 backend/tests/golden/historic_population_vergennes.csv delete mode 100644 backend/tests/golden/med_home_value_addison_town.csv delete mode 100644 backend/tests/golden/med_home_value_vergennes.csv delete mode 100644 backend/tests/golden/med_smoc_addison_town.csv delete mode 100644 backend/tests/golden/med_smoc_vergennes.csv delete mode 100644 backend/tests/golden/median_earnings_addison_town.csv delete mode 100644 backend/tests/golden/median_earnings_vergennes.csv delete mode 100644 backend/tests/golden/unemployment_rate_addison_town.csv delete mode 100644 backend/tests/golden/unemployment_rate_vergennes.csv rename backend/tests/{test_lake.py => inspect_lake.py} (99%) delete mode 100644 backend/tests/test_census_timeseries_db.py delete mode 100644 frontend/src/components/FilterUI/Filter_Call.tsx diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml new file mode 100644 index 00000000..49426f5e --- /dev/null +++ b/.github/workflows/backend-tests.yml @@ -0,0 +1,38 @@ +name: Backend tests + +on: + pull_request: + paths: + - "backend/**" + - ".github/workflows/backend-tests.yml" + push: + branches: [main] + paths: + - "backend/**" + - ".github/workflows/backend-tests.yml" + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: false + - uses: astral-sh/setup-uv@v6 + with: + version: "0.11.21" + enable-cache: true + cache-dependency-glob: backend/uv.lock + - name: Install locked Python dependencies + working-directory: backend + run: uv sync --frozen --python 3.13 + # No warehouse.duckdb or DuckLake exists here: tests that read real data + # are marked `requires_warehouse` and skip (see tests/conftest.py). + # test_base.py still imports the deleted data_collection_OLD package; drop + # the --ignore once it is ported to data_collection.base. + - name: Run the backend test suite + working-directory: backend + run: uv run --frozen pytest -q --ignore=tests/test_base.py diff --git a/CLAUDE.md b/CLAUDE.md index 21857d15..caea5ac0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,11 +28,11 @@ npm run format:check # Check formatting without modifying uvicorn api.main:app --reload --port 6767 # API server on localhost:6767 pip install -r requirements.txt # Install dependencies streamlit run Home.py # Legacy Streamlit interface (not primary) -uv run sqlfluff lint query/sql build/sql # Lint the Jinja SQL templates -uv run sqlfluff fix query/sql build/sql # Auto-fix SQL lint issues +uv run sqlfluff lint query/sql data_cleaning/sql # Lint the Jinja SQL templates +uv run sqlfluff fix query/sql data_cleaning/sql # Auto-fix SQL lint issues ``` -SQL templates under `backend/query/sql/` and `backend/build/sql/` are Jinja +SQL templates under `backend/query/sql/` and `backend/data_cleaning/sql/` are Jinja templates rendered by `query/sql_render.py`, which also holds the general-purpose filter compiler (it emits `$N` prepared-statement placeholders plus a params list — execute with `DB.execute(sql, params)`). Lint-time @@ -72,7 +72,7 @@ template context lives in `backend/.sqlfluff`. - `api/routes/get_routes/get_filters.py` — `GET /load/mapping/{dataset}/filters` — returns hierarchical filter tree - `api/routes/get_routes/get_wholedata.py` — full dataset fetch with optional filters -- `api/routes/post_routes/post_census.py` — Census/ACS-5 data +- `api/routes/post_routes/post_acs5_db.py` — Census ACS-5 tidy and time-series data (`/load/acs5-db/...`) - `api/routes/post_routes/post_zoning.py` — Zoning data **Response model:** `api/models/response_models.py` — `APIResponse` with `data`, `tableData`, and `metadata` fields. @@ -80,7 +80,7 @@ template context lives in `backend/.sqlfluff`. **Query layer (`query/`):** - `sql_render.py` — Jinja SQL rendering and the filter compiler (`sql_filter_block`, `compile_where`) -- `timeseries_db.py` — DuckDB access to census time-series tables +- `acs5.py` — ACS-5 tidy and time-series queries against `warehouse.duckdb` - `production_db.py` — lazy DuckDB connection **ETL census helpers:** `data_collection/census.py` (`tidy_census`, `split_name_col`). diff --git a/backend/.sqlfluff b/backend/.sqlfluff index 83c36c4d..f3740201 100644 --- a/backend/.sqlfluff +++ b/backend/.sqlfluff @@ -1,5 +1,5 @@ -# SQL linting for the Jinja SQL templates in query/sql/ and build/sql/. -# Run from backend/: uv run sqlfluff lint query/sql build/sql +# SQL linting for the Jinja SQL templates in query/sql/ and data_cleaning/sql/. +# Run from backend/: uv run sqlfluff lint query/sql data_cleaning/sql # # The [templater:jinja:context] section supplies representative values for the # fragments that Python (sql_render.py) injects at runtime, so the linter diff --git a/backend/api/routes/post_routes/__init__.py b/backend/api/routes/post_routes/__init__.py index 5ce904ed..db50bf72 100644 --- a/backend/api/routes/post_routes/__init__.py +++ b/backend/api/routes/post_routes/__init__.py @@ -1,7 +1,6 @@ from .post_acs5_db import router as post_acs5_router from .post_ambulance import router as post_ambulance_router from .post_cdc import router as post_cdc_router -from .post_census import router as post_census_router from .post_comparison import router as post_comparison_router from .post_export import router as post_export_router from .post_flood import router as post_flood_router @@ -12,7 +11,6 @@ all_post_routers = [ post_zoning_router, - post_census_router, post_acs5_router, post_qcew_router, post_wastewater_router, diff --git a/backend/api/routes/post_routes/post_census.py b/backend/api/routes/post_routes/post_census.py deleted file mode 100644 index ff9e4d21..00000000 --- a/backend/api/routes/post_routes/post_census.py +++ /dev/null @@ -1,67 +0,0 @@ -from pathlib import Path - -from fastapi import APIRouter, HTTPException - -from api.models import FilterRequest, make_response -from query import timeseries_db -from query.production_db import get_db - -DB = get_db() - -router = APIRouter() - -# Fixed this path to four levels up to reach the Data directory -DATADIR = Path(__file__).parent.parent.parent.parent / "Data" -CENSUS_DATADIR = DATADIR / "Census" - - -CENSUS_DATASETS = ["demographics", "economics", "housing", "social"] - -# Maps (category, subcategory) to the timeseries_db view name. -# These subcategories are served via DuckDB instead of pandas/CSV. -_TIMESERIES_VIEWS: dict[tuple[str, str], str] = { - ("demographics", "historic_population"): "VCGI_historicPopulation_timeseries", - ( - "demographics", - "age_dependency_ratio", - ): "acs5Demographics_ageDependencyRatio_timeseries", - ("demographics", "median_age"): "acs5Demographics_medianAge_timeseries", - ("economics", "health_insurance"): "acs5Economics_healthInsurance_timeseries", - ("economics", "household_income"): "acs5Economics_medianHouseholdIncome_timeseries", - ("economics", "per_capita_income"): "acs5Economics_perCapitaIncome_timeseries", - ("economics", "unemployment_rate"): "acs5Economics_unemploymentRate_timeseries", - ("housing", "housing_units"): "acs5Housing_housingUnits_timeseries", - ("housing", "median_home_value"): "acs5Housing_medianHomeValue_timeseries", - ("housing", "vacancy_rates"): "acs5Demographics_vacancyRates_timeseries", -} - - -# Load the Census Dataset by `category`(housing, economic, etc.) -# and `subcategory`(special time series tables) -@router.post("/load/census/{category}/{subcategory}") -def read_census_data_subcat( - category: str, request: FilterRequest, subcategory: str = "main" -): - if category not in CENSUS_DATASETS: - raise HTTPException( - status_code=404, detail=f"Census category '{category}' was not found" - ) - - # Time-series subcategories are served via DuckDB - ts_key = (category, subcategory) - if ts_key in _TIMESERIES_VIEWS: - view_name = _TIMESERIES_VIEWS[ts_key] - filters = request.filters if request else None - - data = timeseries_db.query_timeseries(view_name, filters) - if data.empty: - raise HTTPException( - status_code=404, - detail=f"No data found for the given filters: {filters}", - ) - return make_response(data, {}) - - raise HTTPException( - status_code=404, - detail=f"Census subcategory '{subcategory}' was not found in category '{category}'", - ) diff --git a/backend/data_cleaning/sql/zoning_empty_geom.sql b/backend/data_cleaning/sql/zoning_empty_geom.sql index 6a23ae55..d85f200a 100644 --- a/backend/data_cleaning/sql/zoning_empty_geom.sql +++ b/backend/data_cleaning/sql/zoning_empty_geom.sql @@ -28,7 +28,7 @@ gaps AS ( t.id, t.NAME, ST_DIFFERENCE( - ST_GeomFromWKB(geometry), + ST_GeomFromWKB(t.geometry), COALESCE(c.geom, ST_GEOMFROMTEXT('POLYGON EMPTY')) ) AS geom FROM town_boundaries AS t @@ -41,7 +41,7 @@ exploded AS ( SELECT g.id, g.NAME, - UNNEST(ST_DUMP(geom), recursive := true) -- noqa: AL03 + UNNEST(ST_DUMP(g.geom), recursive := true) -- noqa: AL03 FROM gaps AS g WHERE NOT ST_ISEMPTY(g.geom) ), diff --git a/backend/data_cleaning/sql/zoning_rules.sql b/backend/data_cleaning/sql/zoning_rules.sql index 1c219b26..e986eb7b 100644 --- a/backend/data_cleaning/sql/zoning_rules.sql +++ b/backend/data_cleaning/sql/zoning_rules.sql @@ -9,4 +9,4 @@ FROM ( INTO NAME col_name VALUE val -) \ No newline at end of file +) diff --git a/backend/query/sql/acs5/acs5_county_compare.sql b/backend/query/sql/acs5/acs5_county_compare.sql index 3cbacb09..7e75262d 100644 --- a/backend/query/sql/acs5/acs5_county_compare.sql +++ b/backend/query/sql/acs5/acs5_county_compare.sql @@ -11,9 +11,9 @@ SELECT c.county, ST_ASGEOJSON(ST_GeomFromWKB(c.geometry)) AS geometry FROM {{ table }} AS a +-- County rows in the tidy tables carry no geoid (it's only populated for +-- towns); the county FIPS is what identifies them. LEFT JOIN vt_county_lines_geom AS c - -- County rows in the tidy tables carry no geoid (it's only populated for - -- towns); the county FIPS is what identifies them. ON a.county_fips = c.geoid {{ where_string }} diff --git a/backend/query/sql/acs5/acs5_timeseries.sql b/backend/query/sql/acs5/acs5_timeseries.sql index 1b0fda1d..fad0ef19 100644 --- a/backend/query/sql/acs5/acs5_timeseries.sql +++ b/backend/query/sql/acs5/acs5_timeseries.sql @@ -1,4 +1,4 @@ SELECT * FROM {{ table }} {{ where_string }} -ORDER BY year \ No newline at end of file +ORDER BY year diff --git a/backend/query/sql/ambulance/ambulance_geo_query.sql b/backend/query/sql/ambulance/ambulance_geo_query.sql index 0ebdcfee..faf4d23d 100644 --- a/backend/query/sql/ambulance/ambulance_geo_query.sql +++ b/backend/query/sql/ambulance/ambulance_geo_query.sql @@ -10,7 +10,7 @@ FROM ( 'type', 'Feature', 'geometry', ST_AsGeoJSON( ST_Simplify(ST_GeomFromWKB(g.geometry), 0.0001) - )::JSON, + )::JSON, 'properties', json_object( 'Certification Level', i.cert_level, 'Acres', ROUND(g.shape__area, 2), @@ -29,4 +29,4 @@ FROM ( LEFT JOIN VCGI_ambulanceService_colors AS c ON i.cert_level = c.certification_level {{ join_filter_block }} -) AS features \ No newline at end of file +) AS features diff --git a/backend/query/sql/cdc/county_places.sql b/backend/query/sql/cdc/county_places.sql index 61c82d23..62013807 100644 --- a/backend/query/sql/cdc/county_places.sql +++ b/backend/query/sql/cdc/county_places.sql @@ -1,3 +1,10 @@ +-- County lines carry their own county column; drop it so an unqualified +-- "county" filter in the WHERE clause resolves to the PLACES row. +WITH county_lines AS ( + SELECT * EXCLUDE (county) + FROM vt_county_lines_geom +) + SELECT p.geoid, p.measure, @@ -7,9 +14,6 @@ SELECT p.county, ST_ASGEOJSON(ST_GeomFromWKB(c.geometry)) AS geometry FROM cdc_places_county AS p -LEFT JOIN ( - SELECT * EXCLUDE(county) - FROM vt_county_lines_geom - ) AS c +LEFT JOIN county_lines AS c ON p.geoid = c.geoid {{ where_string }} diff --git a/backend/query/sql/wastewater/soil_suitability_geo_query.sql b/backend/query/sql/wastewater/soil_suitability_geo_query.sql index fa34634d..52608e01 100644 --- a/backend/query/sql/wastewater/soil_suitability_geo_query.sql +++ b/backend/query/sql/wastewater/soil_suitability_geo_query.sql @@ -8,9 +8,9 @@ FROM ( json_object( 'type', 'Feature', 'geometry', - ST_AsGeoJSON( - ST_Simplify(g.geometry, 0.0001) - )::JSON, + ST_AsGeoJSON( + ST_Simplify(g.geometry, 0.0001) + )::JSON, 'properties', json_object( 'Suitability', i.suitability, 'Acres', ROUND(i.acres, 2), @@ -29,4 +29,4 @@ FROM ( LEFT JOIN VersoWastewater_soilSuitability_colors AS c ON i.suitability = c.soil_suitability {{ where_string }} -) AS features; \ No newline at end of file +) AS features; diff --git a/backend/query/sql/zoning/footprints.sql b/backend/query/sql/zoning/footprints.sql index d06217f3..b67f620f 100644 --- a/backend/query/sql/zoning/footprints.sql +++ b/backend/query/sql/zoning/footprints.sql @@ -1,12 +1,12 @@ {{ cte_filter_block }} -SELECT +SELECT b.object_id, b.town, b.county, - b.geoid, + b.geoid, b.name, b.height_ft, b.building_type, - ST_AsGeoJSON(b.geometry) AS geometry, + ST_AsGeoJSON(b.geometry) AS geometry FROM VCGI_buildingFootprints_geom AS b {{ join_filter_block }} diff --git a/backend/query/sql/zoning/unzoned.sql b/backend/query/sql/zoning/unzoned.sql index ec917b5e..6cb00a3b 100644 --- a/backend/query/sql/zoning/unzoned.sql +++ b/backend/query/sql/zoning/unzoned.sql @@ -1,6 +1,6 @@ -- Towns' unzoned areas as a GeoJSON FeatureCollection, drawn as a grey layer --- underneath the zoning districts. Built by build/sql/zoning_empty_geom.sql, --- which already drops sliver-sized gaps. +-- underneath the zoning districts. Built by +-- data_cleaning/sql/zoning_empty_geom.sql, which already drops sliver-sized gaps. -- -- This layer does not depend on the sidebar filters: it is the same "we have no -- data here" backdrop regardless of which districts are being shown. diff --git a/backend/query/sql_render.py b/backend/query/sql_render.py index bb4babf3..970ff751 100644 --- a/backend/query/sql_render.py +++ b/backend/query/sql_render.py @@ -4,9 +4,10 @@ **Description**: Jinja SQL rendering plus the general-purpose filter compiler. - SQL files under `query/sql/` and `build/sql/` are Jinja templates, so they can - be linted with sqlfluff (`uv run sqlfluff lint query/sql build/sql` from - `backend/`; lint-time template context lives in `backend/.sqlfluff`). + SQL files under `query/sql/` and `data_cleaning/sql/` are Jinja templates, so + they can be linted with sqlfluff (`uv run sqlfluff lint query/sql + data_cleaning/sql` from `backend/`; lint-time template context lives in + `backend/.sqlfluff`). Python compiles FilterSource objects into SQL fragments (filter CTEs, joins, and WHERE clauses) and injects them through the Jinja context. This keeps the diff --git a/backend/query/timeseries_db.py b/backend/query/timeseries_db.py deleted file mode 100644 index 99f16629..00000000 --- a/backend/query/timeseries_db.py +++ /dev/null @@ -1,135 +0,0 @@ -""" -DuckDB-backed access to the census time-series tables in Data/_Processed/acs5/. - -Use query_timeseries(table_name, filters) to filter any registered table. -""" - -import logging - -from query.production_db import get_db - -DB = get_db() - -logger = logging.getLogger(__name__) - -# Per-table registry. "select"/"where" default to the whole table; the two -# entries that override them reproduce the column names and town-level row set -# these endpoints have always returned (build/acs5.py renames the measure to -# Value and appends county/state rollups for the /load/acs5-db routes). -# "cols" whitelists what a caller may filter on, which guards against injection. -_TABLES: dict[str, dict] = { - "unemployment_rate": { - "table": "acs5_unemployment_rate", - "select": 'year, GEOID, NAME, Jurisdiction, County, "Value" AS Unemployment_Rate', - "where": "geotype = 'county_subdivision'", - "cols": { - "year", - "GEOID", - "NAME", - "Jurisdiction", - "County", - "Unemployment_Rate", - }, - }, - "median_earnings": { - "table": "acs5_median_earnings", - "select": "year, GEOID, NAME, Jurisdiction, County, " - '"Variable" AS variable, "Value" AS estimate', - "where": "geotype = 'county_subdivision'", - "cols": { - "year", - "GEOID", - "NAME", - "Jurisdiction", - "County", - "variable", - "estimate", - }, - }, - "median_home_value": { - "table": "acs5_median_home_value", - "cols": { - "year", - "GEOID", - "NAME", - "Jurisdiction", - "County", - "estimate", - }, - }, - "median_smoc": { - "table": "acs5_median_smoc", - "cols": { - "year", - "GEOID", - "NAME", - "Jurisdiction", - "County", - "variable", - "estimate", - }, - }, - "commute_time": { - "table": "acs5_commute_time", - "cols": {"year", "GEOID", "NAME", "Jurisdiction", "County", "estimate"}, - }, - "commute_habits": { - "table": "acs5_commute_habits", - "cols": { - "year", - "GEOID", - "NAME", - "Jurisdiction", - "County", - "variable", - "estimate", - }, - }, - "historic_population": { - "table": "acs5_historic_population", - "cols": { - "geoid", - "NAME", - "geo_type", - "Jurisdiction", - "County", - # "Year", - "Population", - }, - }, -} - -# Frontend filter labels whose name differs from the column they filter on. -# Anything else is taken as the column name itself and checked against "cols". -_LABEL_ALIASES = {"Location": "NAME"} - - -def query_timeseries(table_name: str, filters: dict | None = None): - if table_name not in _TABLES: - raise KeyError(f"No timeseries table registered under '{table_name}'") - - spec = _TABLES[table_name] - valid_cols = spec["cols"] - where_clauses = [spec["where"]] if spec.get("where") else [] - params = [] - - for label, values in (filters or {}).items(): - col = _LABEL_ALIASES.get(label, label) - if col not in valid_cols: - logger.warning( - "timeseries_db: skipping unknown filter column '%s' on table '%s'", - col, - table_name, - ) - continue - - if isinstance(values, str): - values = [values] - - placeholders = ", ".join("?" for _ in values) - where_clauses.append(f'"{col}" IN ({placeholders})') - params.extend(values) - - where_sql = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else "" - sql = f'SELECT {spec.get("select", "*")} FROM "{spec["table"]}" {where_sql}' - return DB.execute(sql, params).df() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 373a7b34..49ac29ef 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,12 +1,46 @@ """ Ensure backend/ is on sys.path and the working directory is set to backend/ so relative paths (e.g., logger/logsapi.log) resolve correctly. + +Route and query modules open the production warehouse when imported +(`DB = get_db()`). Without a built warehouse (e.g. in CI) they get an empty +in-memory DuckDB instead, so tests that swap in fixture tables can still import +them. Tests that read real warehouse data are marked `requires_warehouse` and +skip when it is absent. """ import os import sys from pathlib import Path +import duckdb +import pytest + backend_dir = Path(__file__).resolve().parent.parent sys.path.insert(0, str(backend_dir)) os.chdir(backend_dir) + +from query import production_db + +WAREHOUSE_AVAILABLE = production_db.WAREHOUSE_PATH.exists() + +if not WAREHOUSE_AVAILABLE: + production_db.get_db = lambda: production_db._ThreadSafeDB(duckdb.connect()) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "requires_warehouse: reads the built warehouse.duckdb; skipped without it", + ) + + +def pytest_collection_modifyitems(config, items): + if WAREHOUSE_AVAILABLE: + return + skip = pytest.mark.skip( + reason=f"no warehouse at {production_db.WAREHOUSE_PATH} (build it with `just etl`)" + ) + for item in items: + if "requires_warehouse" in item.keywords: + item.add_marker(skip) diff --git a/backend/tests/golden/commute_habits_addison_town.csv b/backend/tests/golden/commute_habits_addison_town.csv deleted file mode 100644 index 372bbb22..00000000 --- a/backend/tests/golden/commute_habits_addison_town.csv +++ /dev/null @@ -1,40 +0,0 @@ -year,GEOID,NAME,variable,estimate,Jurisdiction,County -2011,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,72.1,Addison town,Addison -2011,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,0.0,Addison town,Addison -2011,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,12.4,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,75.2,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,1.3,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,12.1,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,72.5,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,1.3,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,10.7,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,73.7,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,1.1,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,10.7,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,76.9,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,1.0,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,7.6,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,78.6,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,1.0,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,6.8,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,80.4,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,0.0,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,6.7,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,85.5,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,0.0,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,6.5,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,87.0,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,0.0,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,6.0,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,87.4,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,0.0,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,6.5,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,83.0,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,0.0,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,6.8,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,76.7,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,0.0,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,10.0,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",DP03_0019P,69.7,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",DP03_0021P,0.0,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",DP03_0024P,12.0,Addison town,Addison diff --git a/backend/tests/golden/commute_habits_vergennes.csv b/backend/tests/golden/commute_habits_vergennes.csv deleted file mode 100644 index 90ca1cc0..00000000 --- a/backend/tests/golden/commute_habits_vergennes.csv +++ /dev/null @@ -1 +0,0 @@ -year,GEOID,NAME,variable,estimate,Jurisdiction,County diff --git a/backend/tests/golden/commute_time_addison_town.csv b/backend/tests/golden/commute_time_addison_town.csv deleted file mode 100644 index 6392c456..00000000 --- a/backend/tests/golden/commute_time_addison_town.csv +++ /dev/null @@ -1,14 +0,0 @@ -year,GEOID,NAME,estimate,Jurisdiction,County -2011,5000100325,"Addison town, Addison County, Vermont",23.4,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",25.4,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",25.3,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",25.5,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",26.5,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",27.1,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",25.4,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",26.8,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",26.3,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",26.4,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",28.9,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",31.0,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",30.5,Addison town,Addison diff --git a/backend/tests/golden/commute_time_vergennes.csv b/backend/tests/golden/commute_time_vergennes.csv deleted file mode 100644 index 45164242..00000000 --- a/backend/tests/golden/commute_time_vergennes.csv +++ /dev/null @@ -1 +0,0 @@ -year,GEOID,NAME,estimate,Jurisdiction,County diff --git a/backend/tests/golden/historic_population_addison_town.csv b/backend/tests/golden/historic_population_addison_town.csv deleted file mode 100644 index ae8d0be5..00000000 --- a/backend/tests/golden/historic_population_addison_town.csv +++ /dev/null @@ -1,25 +0,0 @@ -Unnamed: 0,X_geoid,NAME,Year,Population,Jurisdiction,County -1,5000100325,"Addison town, Addison County, Vermont",1791,402,Addison town,Addison -2,5000100325,"Addison town, Addison County, Vermont",1800,734,Addison town,Addison -3,5000100325,"Addison town, Addison County, Vermont",1810,1100,Addison town,Addison -4,5000100325,"Addison town, Addison County, Vermont",1820,1210,Addison town,Addison -5,5000100325,"Addison town, Addison County, Vermont",1830,1306,Addison town,Addison -6,5000100325,"Addison town, Addison County, Vermont",1840,1232,Addison town,Addison -7,5000100325,"Addison town, Addison County, Vermont",1850,1279,Addison town,Addison -8,5000100325,"Addison town, Addison County, Vermont",1860,1000,Addison town,Addison -9,5000100325,"Addison town, Addison County, Vermont",1870,911,Addison town,Addison -10,5000100325,"Addison town, Addison County, Vermont",1880,847,Addison town,Addison -11,5000100325,"Addison town, Addison County, Vermont",1890,900,Addison town,Addison -12,5000100325,"Addison town, Addison County, Vermont",1900,851,Addison town,Addison -13,5000100325,"Addison town, Addison County, Vermont",1910,796,Addison town,Addison -14,5000100325,"Addison town, Addison County, Vermont",1920,743,Addison town,Addison -15,5000100325,"Addison town, Addison County, Vermont",1930,684,Addison town,Addison -16,5000100325,"Addison town, Addison County, Vermont",1940,576,Addison town,Addison -17,5000100325,"Addison town, Addison County, Vermont",1950,628,Addison town,Addison -18,5000100325,"Addison town, Addison County, Vermont",1960,645,Addison town,Addison -19,5000100325,"Addison town, Addison County, Vermont",1970,717,Addison town,Addison -20,5000100325,"Addison town, Addison County, Vermont",1980,889,Addison town,Addison -21,5000100325,"Addison town, Addison County, Vermont",1990,1023,Addison town,Addison -22,5000100325,"Addison town, Addison County, Vermont",2000,1393,Addison town,Addison -23,5000100325,"Addison town, Addison County, Vermont",2010,1371,Addison town,Addison -24,5000100325,"Addison town, Addison County, Vermont",2020,1365,Addison town,Addison diff --git a/backend/tests/golden/historic_population_vergennes.csv b/backend/tests/golden/historic_population_vergennes.csv deleted file mode 100644 index c3973981..00000000 --- a/backend/tests/golden/historic_population_vergennes.csv +++ /dev/null @@ -1 +0,0 @@ -Unnamed: 0,X_geoid,NAME,Year,Population,Jurisdiction,County diff --git a/backend/tests/golden/med_home_value_addison_town.csv b/backend/tests/golden/med_home_value_addison_town.csv deleted file mode 100644 index f5f91a58..00000000 --- a/backend/tests/golden/med_home_value_addison_town.csv +++ /dev/null @@ -1,16 +0,0 @@ -year,GEOID,NAME,estimate,Jurisdiction,County -2009,5000100325,"Addison town, Addison County, Vermont",222500.0,Addison town,Addison -2010,5000100325,"Addison town, Addison County, Vermont",229600.0,Addison town,Addison -2011,5000100325,"Addison town, Addison County, Vermont",234000.0,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",252400.0,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",247300.0,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",238100.0,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",243900.0,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",238100.0,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",238200.0,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",243500.0,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",270000.0,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",282200.0,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",298300.0,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",330900.0,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",353000.0,Addison town,Addison diff --git a/backend/tests/golden/med_home_value_vergennes.csv b/backend/tests/golden/med_home_value_vergennes.csv deleted file mode 100644 index 45164242..00000000 --- a/backend/tests/golden/med_home_value_vergennes.csv +++ /dev/null @@ -1 +0,0 @@ -year,GEOID,NAME,estimate,Jurisdiction,County diff --git a/backend/tests/golden/med_smoc_addison_town.csv b/backend/tests/golden/med_smoc_addison_town.csv deleted file mode 100644 index 23fc6bd8..00000000 --- a/backend/tests/golden/med_smoc_addison_town.csv +++ /dev/null @@ -1,31 +0,0 @@ -year,GEOID,NAME,variable,estimate,Jurisdiction,County -2009,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1481.0,Addison town,Addison -2009,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,605.0,Addison town,Addison -2010,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1566.0,Addison town,Addison -2010,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,643.0,Addison town,Addison -2011,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1691.0,Addison town,Addison -2011,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,682.0,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1801.0,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,722.0,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1780.0,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,731.0,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1745.0,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,729.0,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1773.0,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,722.0,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1705.0,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,750.0,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1650.0,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,740.0,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1676.0,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,748.0,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1693.0,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,756.0,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1755.0,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,750.0,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1782.0,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,782.0,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1842.0,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,854.0,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",Mortgaged SMOC,1871.0,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",Non-Mortgaged SMOC,868.0,Addison town,Addison diff --git a/backend/tests/golden/med_smoc_vergennes.csv b/backend/tests/golden/med_smoc_vergennes.csv deleted file mode 100644 index 90ca1cc0..00000000 --- a/backend/tests/golden/med_smoc_vergennes.csv +++ /dev/null @@ -1 +0,0 @@ -year,GEOID,NAME,variable,estimate,Jurisdiction,County diff --git a/backend/tests/golden/median_earnings_addison_town.csv b/backend/tests/golden/median_earnings_addison_town.csv deleted file mode 100644 index 9d5c4a14..00000000 --- a/backend/tests/golden/median_earnings_addison_town.csv +++ /dev/null @@ -1,40 +0,0 @@ -year,GEOID,NAME,variable,estimate,Jurisdiction,County -2011,5000100325,"Addison town, Addison County, Vermont",DP03_0092,32250.0,Addison town,Addison -2011,5000100325,"Addison town, Addison County, Vermont",DP03_0093,42500.0,Addison town,Addison -2011,5000100325,"Addison town, Addison County, Vermont",DP03_0094,32200.0,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",DP03_0092,33125.0,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",DP03_0093,46250.0,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",DP03_0094,33958.0,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",DP03_0092,33289.0,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",DP03_0093,43750.0,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",DP03_0094,34500.0,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",DP03_0092,34046.0,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",DP03_0093,46944.0,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",DP03_0094,35833.0,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",DP03_0092,36853.0,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",DP03_0093,53438.0,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",DP03_0094,39583.0,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",DP03_0092,36714.0,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",DP03_0093,52500.0,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",DP03_0094,45000.0,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",DP03_0092,36591.0,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",DP03_0093,46111.0,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",DP03_0094,45417.0,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",DP03_0092,37063.0,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",DP03_0093,49750.0,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",DP03_0094,51000.0,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",DP03_0092,40000.0,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",DP03_0093,48300.0,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",DP03_0094,53646.0,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",DP03_0092,40039.0,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",DP03_0093,50391.0,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",DP03_0094,54722.0,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",DP03_0092,44833.0,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",DP03_0093,56923.0,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",DP03_0094,62778.0,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",DP03_0092,50972.0,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",DP03_0093,71250.0,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",DP03_0094,69265.0,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",DP03_0092,58438.0,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",DP03_0093,72679.0,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",DP03_0094,67917.0,Addison town,Addison diff --git a/backend/tests/golden/median_earnings_vergennes.csv b/backend/tests/golden/median_earnings_vergennes.csv deleted file mode 100644 index 90ca1cc0..00000000 --- a/backend/tests/golden/median_earnings_vergennes.csv +++ /dev/null @@ -1 +0,0 @@ -year,GEOID,NAME,variable,estimate,Jurisdiction,County diff --git a/backend/tests/golden/unemployment_rate_addison_town.csv b/backend/tests/golden/unemployment_rate_addison_town.csv deleted file mode 100644 index b7731495..00000000 --- a/backend/tests/golden/unemployment_rate_addison_town.csv +++ /dev/null @@ -1,14 +0,0 @@ -year,GEOID,NAME,Unemployment_Rate,Jurisdiction,County -2011,5000100325,"Addison town, Addison County, Vermont",4.2,Addison town,Addison -2012,5000100325,"Addison town, Addison County, Vermont",4.6,Addison town,Addison -2013,5000100325,"Addison town, Addison County, Vermont",4.5,Addison town,Addison -2014,5000100325,"Addison town, Addison County, Vermont",3.6,Addison town,Addison -2015,5000100325,"Addison town, Addison County, Vermont",3.8,Addison town,Addison -2016,5000100325,"Addison town, Addison County, Vermont",2.9,Addison town,Addison -2017,5000100325,"Addison town, Addison County, Vermont",2.9,Addison town,Addison -2018,5000100325,"Addison town, Addison County, Vermont",3.0,Addison town,Addison -2019,5000100325,"Addison town, Addison County, Vermont",2.2,Addison town,Addison -2020,5000100325,"Addison town, Addison County, Vermont",1.7,Addison town,Addison -2021,5000100325,"Addison town, Addison County, Vermont",2.2,Addison town,Addison -2022,5000100325,"Addison town, Addison County, Vermont",0.7,Addison town,Addison -2023,5000100325,"Addison town, Addison County, Vermont",1.1,Addison town,Addison diff --git a/backend/tests/golden/unemployment_rate_vergennes.csv b/backend/tests/golden/unemployment_rate_vergennes.csv deleted file mode 100644 index c7fe953b..00000000 --- a/backend/tests/golden/unemployment_rate_vergennes.csv +++ /dev/null @@ -1 +0,0 @@ -year,GEOID,NAME,Unemployment_Rate,Jurisdiction,County diff --git a/backend/tests/test_lake.py b/backend/tests/inspect_lake.py similarity index 99% rename from backend/tests/test_lake.py rename to backend/tests/inspect_lake.py index 9c619110..dcd0e3a3 100644 --- a/backend/tests/test_lake.py +++ b/backend/tests/inspect_lake.py @@ -2,7 +2,7 @@ Diagnostic script for inspecting DuckLake table structure. Usage (from backend): -python -m tests.test_lake +python -m tests.inspect_lake """ from lake_build import get_connection diff --git a/backend/tests/test_census_timeseries_db.py b/backend/tests/test_census_timeseries_db.py deleted file mode 100644 index 71bf0a79..00000000 --- a/backend/tests/test_census_timeseries_db.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Golden-output and correctness tests for the DuckDB-backed time-series census endpoints. - -Golden CSVs in tests/golden/ were generated from the original pandas/CSV pipeline -before the DuckDB migration. Tests assert that the new DuckDB path returns -numerically identical results. - -Run: - cd backend && conda run -n leahy_data pytest tests/test_census_timeseries_db.py -v -""" - -from pathlib import Path - -import pandas as pd -import pytest - -from query import timeseries_db - -GOLDEN_DIR = Path(__file__).parent / "golden" -FILTER_TOWN = "Addison town" - -# --------------------------------------------------------------------------- -# Helper -# --------------------------------------------------------------------------- - - -def _load_golden(filename: str) -> pd.DataFrame: - return pd.read_csv(GOLDEN_DIR / filename) - - -def _normalize(df: pd.DataFrame) -> pd.DataFrame: - """Sort + reset index so frame comparison is order-independent. - - Coerces all columns that look numeric to float so that int/str/float - dtype differences between the DuckDB result and the golden CSV don't - produce false failures. - """ - df = df.copy() - for col in df.columns: - coerced = pd.to_numeric(df[col], errors="coerce") - # Only coerce if *all* non-null values converted successfully - if coerced.notna().sum() == df[col].notna().sum(): - df[col] = coerced - return df.sort_values(list(df.columns)).reset_index(drop=True) - - -# --------------------------------------------------------------------------- -# Golden-output (value) tests -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "view_name,golden_file", - [ - ("unemployment_rate", "unemployment_rate_addison_town.csv"), - ("median_earnings", "median_earnings_addison_town.csv"), - ("median_home_value", "med_home_value_addison_town.csv"), - ("median_smoc", "med_smoc_addison_town.csv"), - ("commute_time", "commute_time_addison_town.csv"), - ("commute_habits", "commute_habits_addison_town.csv"), - ("historic_population", "historic_population_addison_town.csv"), - ], -) -def test_golden_output_matches(view_name: str, golden_file: str): - """DuckDB result must match pre-migration pandas result exactly.""" - actual = timeseries_db.query_timeseries(view_name, {"Jurisdiction": [FILTER_TOWN]}) - expected = _load_golden(golden_file) - - # Align column sets (golden may have extra cols like 'Unnamed: 0' from to_csv) - extra_in_golden = set(expected.columns) - set(actual.columns) - expected = expected.drop(columns=list(extra_in_golden)) - - assert set(actual.columns) == set(expected.columns), ( - f"{view_name}: column mismatch. " - f"actual={sorted(actual.columns)}, expected={sorted(expected.columns)}" - ) - - pd.testing.assert_frame_equal( - _normalize(actual[sorted(actual.columns)]), - _normalize(expected[sorted(actual.columns)]), - check_like=True, - check_dtype=False, - rtol=1e-4, - ) - - -# --------------------------------------------------------------------------- -# Column-presence tests (Jurisdiction and County must always be present) -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "view_name", - [ - "unemployment_rate", - "median_earnings", - "median_home_value", - "median_smoc", - "commute_time", - "commute_habits", - "historic_population", - ], -) -def test_required_columns_present(view_name: str): - df = timeseries_db.query_timeseries(view_name) - assert "Jurisdiction" in df.columns, f"{view_name}: missing Jurisdiction" - assert "County" in df.columns, f"{view_name}: missing County" - assert "NAME" in df.columns, f"{view_name}: missing NAME" - assert len(df) > 0, f"{view_name}: returned empty (no filter)" - - -# --------------------------------------------------------------------------- -# Filter-correctness tests -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "view_name", - [ - "unemployment_rate", - "median_earnings", - "median_home_value", - "median_smoc", - "commute_time", - "commute_habits", - ], -) -def test_jurisdiction_filter_restricts_rows(view_name: str): - df = timeseries_db.query_timeseries(view_name, {"Jurisdiction": [FILTER_TOWN]}) - assert len(df) > 0, f"{view_name}: empty result for {FILTER_TOWN!r}" - assert (df["Jurisdiction"] == FILTER_TOWN).all(), ( - f"{view_name}: rows outside filter returned" - ) - - -def test_empty_filter_returns_all(): - df_all = timeseries_db.query_timeseries("unemployment_rate") - df_empty = timeseries_db.query_timeseries("unemployment_rate", {}) - assert len(df_all) == len(df_empty) - - -def test_nonsense_filter_returns_empty(): - df = timeseries_db.query_timeseries( - "unemployment_rate", {"Jurisdiction": ["__nonexistent__"]} - ) - assert len(df) == 0 - - -def test_county_filter_works(): - df = timeseries_db.query_timeseries("unemployment_rate", {"County": ["Addison"]}) - assert len(df) > 0 - assert (df["County"] == "Addison").all() - - -def test_unknown_column_in_filter_is_ignored(): - """Unknown filter keys must be silently skipped, not raise an error.""" - df = timeseries_db.query_timeseries( - "unemployment_rate", - {"Jurisdiction": [FILTER_TOWN], "__bad_col__": ["x"]}, - ) - assert len(df) > 0 - - -# --------------------------------------------------------------------------- -# View-registry test -# --------------------------------------------------------------------------- - - -def test_unknown_view_raises_key_error(): - with pytest.raises(KeyError): - timeseries_db.query_timeseries("nonexistent_view") diff --git a/backend/tests/test_clean_acs5.py b/backend/tests/test_clean_acs5.py index e824f0e5..4ebf0160 100644 --- a/backend/tests/test_clean_acs5.py +++ b/backend/tests/test_clean_acs5.py @@ -89,10 +89,49 @@ def test_colliding_smoc_observations_survive_distinctly(): ).fetchall() assert rows == [ - (2011, "DP04_0100E", "1731"), - (2011, "DP04_0107E", "699"), - (2012, "DP04_0100E", "1764"), - (2012, "DP04_0107E", "737"), + (2011, "DP04_0100E", 1731.0), + (2011, "DP04_0107E", 699.0), + (2012, "DP04_0100E", 1764.0), + (2012, "DP04_0107E", 737.0), + ] + + +def test_dp_values_are_numeric_with_unavailable_codes_nulled(): + con = _base_con() + # Census marks values it cannot publish with "(X)" or a sentinel code. + con.execute( + _housing_rows( + [ + (2011, "DP04_0100E", "1731"), + (2011, "DP04_0101E", "(X)"), + (2011, "DP04_0102E", "666666666"), + (2011, "DP04_0103E", "-666666666"), + (2011, "DP04_0104E", "-888888888"), + (2011, "DP04_0105E", "-999999999"), + ] + ) + ) + build_dp_combined(con) + + assert con.execute( + """ + SELECT data_type FROM duckdb_columns() + WHERE table_name = 'acs5_dp_combined_tidy' AND column_name = 'value' + """ + ).fetchone() == ("DOUBLE",) + rows = con.execute( + """ + SELECT variable_code, value FROM lake.CLEANED.acs5_dp_combined_tidy + ORDER BY variable_code + """ + ).fetchall() + assert rows == [ + ("DP04_0100E", 1731.0), + ("DP04_0101E", None), + ("DP04_0102E", None), + ("DP04_0103E", None), + ("DP04_0104E", None), + ("DP04_0105E", None), ] @@ -129,8 +168,20 @@ def test_duplicate_town_lines_row_does_not_fan_out_observations(): _assert_unique_observations(con) # should not raise despite the dup join row +@pytest.mark.requires_warehouse def test_dp_value_col(): con = get_db() + (data_type,) = con.execute( + """ + SELECT data_type FROM information_schema.columns + WHERE table_name = 'acs5_dp_combined_tidy' AND column_name = 'value' + """ + ).fetchone() + if data_type != "DOUBLE": + pytest.skip( + f"warehouse.duckdb predates the DOUBLE value cast ({data_type}); " + "rebuild it with `just etl`" + ) df = con.execute("SELECT value FROM main.acs5_dp_combined_tidy").df() print(df["value"].dtype, df["value"].min(), df["value"].max()) diff --git a/backend/tests/test_sql_render.py b/backend/tests/test_sql_render.py index 84c56cb4..bfcba7a8 100644 --- a/backend/tests/test_sql_render.py +++ b/backend/tests/test_sql_render.py @@ -22,7 +22,7 @@ BACKEND = Path(__file__).resolve().parent.parent QUERY_SQL = BACKEND / "query" / "sql" -BUILD_SQL = BACKEND / "build" / "sql" +CLEANING_SQL = BACKEND / "data_cleaning" / "sql" # --------------------------------------------------------------------------- # Representative render arguments per template @@ -41,31 +41,69 @@ join_key="OBJECT_ID", join_type="inner", ) +PARCELS_SOURCE = FilterSource( + filter_table="VCGIParcels_info", + filters={"county": ["Addison"]}, + join_key="object_id", +) +CDC_SOURCE = FilterSource( + filter_table="cdc_places_county", + filters={"measure": ["Depression among adults"]}, +) # template path (relative to backend/) -> FilterSource list for sql_filter_block QUERY_TEMPLATE_SOURCES = { + "query/sql/acs5/acs5_county_compare.sql": [WHERE_SOURCE], "query/sql/acs5/acs5_tidy.sql": [WHERE_SOURCE], - "query/sql/acs5/median_earnings.sql": [WHERE_SOURCE], - "query/sql/acs5/snapshot.sql": [WHERE_SOURCE], - "query/sql/acs5/unemployment_rate.sql": [WHERE_SOURCE], + "query/sql/acs5/acs5_timeseries.sql": [WHERE_SOURCE], + "query/sql/acs5/acs5_town_compare.sql": [WHERE_SOURCE], + "query/sql/acs5/acs5_tract_compare.sql": [WHERE_SOURCE], + "query/sql/ambulance/ambulance_geo_query.sql": [ + FilterSource( + filter_table="VCGI_ambulanceService_info", + filters={"cert_level": ["Advanced EMT"]}, + join_key="object_id", + ) + ], "query/sql/cdc/county_places.sql": [ FilterSource( filter_table="cdc_places_county", filters={"Measure": ["Depression among adults"]}, ) ], + "query/sql/cdc/places_by_county.sql": [CDC_SOURCE], + "query/sql/cdc/places_tidy.sql": [CDC_SOURCE], "query/sql/cdc/tract_places.sql": [ FilterSource( filter_table="cdc_tract_places", filters={"Measure": ["Depression among adults"]}, ) ], + "query/sql/flood/flood_geo_query.sql": [ + FilterSource( + filter_table="FEMA_floodHazard_geom", + filters={"flood_zone_type": ["AE"]}, + ) + ], + "query/sql/parcels/agg_category_table.sql": [PARCELS_SOURCE], + "query/sql/parcels/agg_flood_table.sql": [PARCELS_SOURCE], + "query/sql/parcels/agg_owner_table.sql": [PARCELS_SOURCE], + "query/sql/parcels/agg_value_table.sql": [PARCELS_SOURCE], + "query/sql/parcels/category_table.sql": [PARCELS_SOURCE], + "query/sql/parcels/flood_table.sql": [PARCELS_SOURCE], + "query/sql/parcels/geo_query.sql": [PARCELS_SOURCE], + "query/sql/parcels/owner_table.sql": [PARCELS_SOURCE], + "query/sql/parcels/value_table.sql": [PARCELS_SOURCE], + "query/sql/zoning/agg_footprints.sql": [CTE_SOURCE], "query/sql/zoning/agg_info_table.sql": [CTE_SOURCE], + "query/sql/zoning/agg_lot_size.sql": [CTE_SOURCE], "query/sql/zoning/info_table.sql": [CTE_SOURCE], "query/sql/zoning/geo_query.sql": [CTE_SOURCE], "query/sql/zoning/unzoned.sql": [], "query/sql/zoning/agg_rules_table.sql": [CTE_SOURCE], + "query/sql/zoning/footprints.sql": [CTE_SOURCE], "query/sql/zoning/geo_rule_table.sql": [CTE_SOURCE], + "query/sql/zoning/lot_size_table.sql": [CTE_SOURCE], "query/sql/zoning/rules_table.sql": [CTE_SOURCE], "query/sql/zoning/rules.sql": [CTE_SOURCE], "query/sql/wastewater/service_area_geo_query.sql": [ @@ -93,16 +131,12 @@ "query/sql/wastewater/waste_treatment_permit_table.sql": [], } -# build templates take plain string context instead of FilterSources -BUILD_TEMPLATE_CONTEXT = { - "build/sql/cdc_places.sql": { - "indicators": "'ASTHMA', 'DIABETES'", - "path": "dummy.csv", - }, - "build/sql/zoning_colors.sql": {}, - "build/sql/zoning_empty_geom.sql": {"min_acres": 10}, - "build/sql/zoning_info.sql": {"info_string": "OBJECT_ID, County"}, - "build/sql/zoning_rules.sql": { +# data-cleaning templates take plain string context instead of FilterSources +CLEANING_TEMPLATE_CONTEXT = { + "data_cleaning/sql/zoning_colors.sql": {}, + "data_cleaning/sql/zoning_empty_geom.sql": {"min_acres": 10}, + "data_cleaning/sql/zoning_info.sql": {"info_string": "OBJECT_ID, County"}, + "data_cleaning/sql/zoning_rules.sql": { "rule_string": 'CAST("Residential_Min_Lot" AS VARCHAR) AS residential_min_lot' }, } @@ -164,7 +198,7 @@ def test_inner_join(self): cte, join = compile_filters([CTE_SOURCE], params) assert cte == ( "WITH f0 AS (SELECT DISTINCT OBJECT_ID " - 'FROM zoning_info WHERE "County" IN ($1))' + 'FROM VersoZoning_info WHERE "County" IN ($1))' ) assert join == "JOIN f0 USING (OBJECT_ID)" assert params == ["Addison"] @@ -230,7 +264,7 @@ def test_undefined_variable_raises(self): from jinja2 import UndefinedError with pytest.raises(UndefinedError): - render_sql(BACKEND / "build/sql/zoning_info.sql") + render_sql(BACKEND / "data_cleaning/sql/zoning_info.sql") # --------------------------------------------------------------------------- @@ -242,16 +276,16 @@ def test_all_templates_accounted_for(): """Every .sql file has a representative context in this test module.""" templates = { str(p.relative_to(BACKEND)) - for d in (QUERY_SQL, BUILD_SQL) + for d in (QUERY_SQL, CLEANING_SQL) for p in d.rglob("*.sql") } - assert templates == set(QUERY_TEMPLATE_SOURCES) | set(BUILD_TEMPLATE_CONTEXT) + assert templates == set(QUERY_TEMPLATE_SOURCES) | set(CLEANING_TEMPLATE_CONTEXT) def test_templates_pass_sqlfluff(): - """Same check as `sqlfluff lint query/sql build/sql` (config in .sqlfluff).""" + """Same check as `sqlfluff lint query/sql data_cleaning/sql` (config in .sqlfluff).""" linter = Linter(config=FluffConfig.from_path(str(BACKEND))) - result = linter.lint_paths((str(QUERY_SQL), str(BUILD_SQL))) + result = linter.lint_paths((str(QUERY_SQL), str(CLEANING_SQL))) violations = result.get_violations() assert violations == [], [str(v) for v in violations] @@ -264,9 +298,9 @@ def test_rendered_query_sql_parses(template): _assert_parses(sql, template) -@pytest.mark.parametrize("template", sorted(BUILD_TEMPLATE_CONTEXT)) -def test_rendered_build_sql_parses(template): - sql = render_sql(BACKEND / template, **BUILD_TEMPLATE_CONTEXT[template]) +@pytest.mark.parametrize("template", sorted(CLEANING_TEMPLATE_CONTEXT)) +def test_rendered_cleaning_sql_parses(template): + sql = render_sql(BACKEND / template, **CLEANING_TEMPLATE_CONTEXT[template]) _assert_parses(sql, template) diff --git a/docs/guides/mcp.md b/docs/guides/mcp.md index 0baa27f5..092162a8 100644 --- a/docs/guides/mcp.md +++ b/docs/guides/mcp.md @@ -569,14 +569,11 @@ plus a known historic population value. It is skipped in the fixture-only run unless `MCP_LIVE_WAREHOUSE` is set. Live-warehouse and deployed-proxy checks complement fixture tests; fixtures cannot establish the freshness of deployed data. -The existing whole-backend suite is not green on the pre-MCP baseline commit -`3a50bf0`: against the same warehouse, an isolated baseline run excluding the -DuckLake diagnostic reported 105 passed and 30 failed. Those failures involve -legacy Census table names and SQL template/expectation mismatches. In addition, -`tests/test_lake.py` initializes DuckLake at import time and requires its extension -and data; it is not an isolated unit test. The MCP workflow runs the new fixture -suite explicitly. A passing MCP check does not claim that these existing tests -have been repaired. +The MCP workflow runs the MCP fixture suite explicitly. The whole backend suite +runs separately in `.github/workflows/backend-tests.yml`, without a warehouse: +tests that read real warehouse data are marked `requires_warehouse` and skip +there. The DuckLake diagnostic, which opens the lake at import time, lives in +`tests/inspect_lake.py` so pytest does not collect it. This release supplies data tools for report generation. It does not implement an agent orchestration loop, report templates, scheduled reports, a PDF renderer, diff --git a/frontend/src/components/FilterUI/Filter_Call.tsx b/frontend/src/components/FilterUI/Filter_Call.tsx deleted file mode 100644 index 68c8dd3c..00000000 --- a/frontend/src/components/FilterUI/Filter_Call.tsx +++ /dev/null @@ -1,13 +0,0 @@ -import axios from 'axios'; - -export async function fetchFilteredData( - selectedFilters: Record, -) { - const filters: Record = {}; - Object.values(selectedFilters).forEach((val, idx) => { - if (val && val !== 'All') filters[`level${idx}`] = val; - }); - - const response = await axios.post('/load/census/data', { filters }); - return response.data; -} diff --git a/justfile b/justfile index ab23f086..ca3d1fea 100644 --- a/justfile +++ b/justfile @@ -273,13 +273,13 @@ lint-backend: [group("Lint & Format")] [working-directory("backend")] lint-sql: - uv run sqlfluff lint query/sql build/sql + uv run sqlfluff lint query/sql data_cleaning/sql [doc("auto-fix the Jinja SQL templates with sqlfluff")] [group("Lint & Format")] [working-directory("backend")] fix-sql: - uv run sqlfluff fix query/sql build/sql + uv run sqlfluff fix query/sql data_cleaning/sql [doc("run the backend test suite")] [group("Test")] From 547d4f298092d811c7d7f4eaba5f49f9cfdd1a6c Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Mon, 2 Nov 2026 10:17:11 -0500 Subject: [PATCH 2/3] Filter CDC PLACES before joining the county lines The compiled WHERE clause names columns unqualified, and the county lines share county, county_fips and geoid with PLACES. Excluding county from the join only covered that one column; a county_fips filter still failed with an ambiguous reference. Filtering PLACES in a CTE first leaves nothing to clash. --- backend/query/sql/cdc/county_places.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/query/sql/cdc/county_places.sql b/backend/query/sql/cdc/county_places.sql index 62013807..d908d2b3 100644 --- a/backend/query/sql/cdc/county_places.sql +++ b/backend/query/sql/cdc/county_places.sql @@ -1,8 +1,9 @@ --- County lines carry their own county column; drop it so an unqualified --- "county" filter in the WHERE clause resolves to the PLACES row. -WITH county_lines AS ( - SELECT * EXCLUDE (county) - FROM vt_county_lines_geom +-- Filter PLACES before joining the county lines: the compiled WHERE clause +-- names columns unqualified, and both tables have county, county_fips and geoid. +WITH places AS ( + SELECT * + FROM cdc_places_county + {{ where_string }} ) SELECT @@ -13,7 +14,6 @@ SELECT ROUND(p.natl_pct * 100, 2) AS natl_pct, p.county, ST_ASGEOJSON(ST_GeomFromWKB(c.geometry)) AS geometry -FROM cdc_places_county AS p -LEFT JOIN county_lines AS c +FROM places AS p +LEFT JOIN vt_county_lines_geom AS c ON p.geoid = c.geoid -{{ where_string }} From 2f4bfed3e74315a02799ae0191eec94a6ace71ed Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Mon, 2 Nov 2026 10:25:38 -0500 Subject: [PATCH 3/3] Register the county comparison template in the SQL render test --- backend/tests/test_sql_render.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/tests/test_sql_render.py b/backend/tests/test_sql_render.py index bfcba7a8..34851b63 100644 --- a/backend/tests/test_sql_render.py +++ b/backend/tests/test_sql_render.py @@ -54,6 +54,7 @@ # template path (relative to backend/) -> FilterSource list for sql_filter_block QUERY_TEMPLATE_SOURCES = { "query/sql/acs5/acs5_county_compare.sql": [WHERE_SOURCE], + "query/sql/acs5/acs5_dp_county_comparison.sql": [WHERE_SOURCE], "query/sql/acs5/acs5_tidy.sql": [WHERE_SOURCE], "query/sql/acs5/acs5_timeseries.sql": [WHERE_SOURCE], "query/sql/acs5/acs5_town_compare.sql": [WHERE_SOURCE],