From 1bb629fe7729f2c2d36add6a2a406c15f398e3d9 Mon Sep 17 00:00:00 2001 From: Prayas Lashkari Date: Tue, 25 Aug 2026 16:00:26 -0400 Subject: [PATCH 1/4] feat(engine): add streams entity type and distance-bounded flow tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recreates UC1-CQ2c ("what streams are downstream at most N km from facilities of industry X?") from David's notebook, which needed two things the query builder could not express. Streams as an answerable entity. Flowlines previously reached the map only as a decorative layer traced from the anchors; hyf:HY_FlowPath was not selectable in either block. Adds 'streams' to EntityType with an FTYPE filter, IRI hydration, and map wiring. When the target is a flowline the ?s2target hop is replaced with a direct bind — that hop would otherwise match any flowline sharing a cell with the answer — and the supporting stream layer is skipped so flowlines aren't drawn twice. Cumulative distance cutoff. downstream/upstream traces were the full transitive closure. Adds an optional maxDistanceKm that sums nhdplusv2:hasFlowPathLength over the segments between seed and candidate and filters on the total. Unset, the emitted SPARQL is byte-identical to before, so existing questions are unaffected. The bound is not just a filter: "samples within 30 km downstream of NH airports" returns 27 sample points in ~15s, where the unbounded form times out against the federation gateway. Two things worth knowing for review: The notebook and this implementation do not agree, and the notebook is wrong. Its outer block re-joins facilities on schema1:address as a required triple, a predicate only 13 of 144 NH airport facilities carry, so it silently discards 91% of its own anchor set. Drop that triple and it returns 1,547 flowlines where we return 1,605 on the same seeding — we are a superset. Address stays OPTIONAL here. Neighbour-cell expansion is kept, per discussion: we seed from the facility's S2 cell and its 8 neighbours where the notebook uses the facility's own cell only. That is the app-wide convention and accounts for the rest of the difference (2,757 vs 1,605 for NH airports at 30km). scripts/flow-distance-check.mjs runs the real planner against the live endpoints and asserts the step plan, the superset relation, that the bound actually excludes flowlines, and that every hydrated stream has drawable geometry. --- README.md | 7 +- scripts/flow-distance-check.mjs | 106 ++++++++++++ src/components/Map/MapPopup.tsx | 6 + src/components/QueryEditor/EntityBlock.tsx | 8 + .../QueryEditor/EntityTypeSelector.tsx | 1 + .../QueryEditor/RelationshipSelector.tsx | 24 +++ src/components/QueryEditor/StreamFilters.tsx | 32 ++++ src/engine/planner.ts | 12 +- src/engine/resultTransformer.ts | 1 + src/engine/templates/fusedQueries.ts | 158 ++++++++++++++++-- src/engine/templates/hydrate.ts | 24 +++ src/hooks/useMapLayers.ts | 5 +- src/types/map.ts | 2 +- src/types/query.ts | 10 +- src/utils/questionGenerator.ts | 13 +- 15 files changed, 385 insertions(+), 24 deletions(-) create mode 100644 scripts/flow-distance-check.mjs create mode 100644 src/components/QueryEditor/StreamFilters.tsx diff --git a/README.md b/README.md index f219bdf..bc32b9d 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,14 @@ When you click Apply, the query engine: 2. **Executes** steps sequentially (`engine/executor.ts`), threading S2 cell sets between steps 3. **Renders** results as map layers (`resultTransformer.ts` → `MapFeature[]`) -Supported entity types: **samples**, **facilities**, **water bodies** +Supported entity types: **samples**, **facilities**, **water bodies**, **wells**, **streams** Supported relationships: **near** (~1–2 km), **downstream**, **upstream** +Downstream/upstream traces accept an optional cumulative flowpath cutoff +(`Within N km of flow`). Unset, the trace is the full transitive closure. +`node scripts/flow-distance-check.mjs` verifies the bounded trace against the +live endpoints. + ## SPARQL endpoints All hosted at `frink.apps.renci.org`: diff --git a/scripts/flow-distance-check.mjs b/scripts/flow-distance-check.mjs new file mode 100644 index 0000000..8e00036 --- /dev/null +++ b/scripts/flow-distance-check.mjs @@ -0,0 +1,106 @@ +// Checks the distance-bounded downstream trace against the live SPARQL +// endpoints, using the same planner the app runs. +// +// Reference values come from UC1_CQ2c (New Hampshire, NAICS 488119 airports, +// 30 km): the notebook returns 162 flowlines, but only because it requires +// schema1:address on facilities — a predicate 13 of 144 NH airport facilities +// have. Without that accidental filter its answer is 1,547 flowlines, and ours +// is a superset of exactly that set (we also expand to neighbouring S2 cells). +// +// Run: node scripts/flow-distance-check.mjs +import assert from 'node:assert/strict'; +import { build } from 'esbuild'; + +const bundle = await build({ + entryPoints: ['src/engine/planner.ts'], + bundle: true, + format: 'esm', + platform: 'neutral', + write: false, +}); +const { planPipeline } = await import( + 'data:text/javascript;base64,' + Buffer.from(bundle.outputFiles[0].text).toString('base64') +); + +const ENDPOINTS = { + sawgraph: 'https://frink.apps.renci.org/sawgraph/sparql', + spatialkg: 'https://frink.apps.renci.org/spatialkg/sparql', + hydrologykg: 'https://frink.apps.renci.org/hydrologykg/sparql', + federation: 'https://frink.apps.renci.org/federation/sparql', +}; + +async function run(endpoint, query) { + const res = await fetch(ENDPOINTS[endpoint], { + method: 'POST', + headers: { + Accept: 'application/sparql-results+json', + 'Content-Type': 'application/sparql-query', + }, + body: query, + }); + if (!res.ok) throw new Error(`${endpoint} ${res.status}: ${await res.text()}`); + const json = await res.json(); + return json.results.bindings.map((row) => + Object.fromEntries(Object.entries(row).map(([k, v]) => [k, v.value])), + ); +} + +// "What streams are within 30 km downstream of 488119 (Airports) facilities +// in New Hampshire?" — the notebook's question, expressed in the app's model. +const question = { + blockA: { type: 'streams' }, + relationship: { type: 'downstream', maxDistanceKm: 30 }, + blockC: { + type: 'facilities', + region: { stateCode: '33' }, + facilityFilters: { industryCodes: ['488119'] }, + }, +}; + +const steps = planPipeline(question); +const stepTypes = steps.map((s) => s.type); + +// The answer set *is* the flowlines, so the supporting stream layer is skipped. +assert.ok( + !stepTypes.includes('GET_FLOWLINE_GEOMETRIES'), + `expected no supporting flowline step, got ${stepTypes.join(', ')}`, +); + +const ctx = { question, targetIris: [], anchorIris: [], results: {} }; +for (const step of steps) { + const rows = await run(step.endpoint, step.buildQuery(ctx)); + ctx.results[step.type] = rows; + if (step.type === 'FIND_TARGET_IRIS') ctx.targetIris = [...new Set(rows.map((r) => r.iri))]; + if (step.type === 'FIND_ANCHOR_IRIS') ctx.anchorIris = [...new Set(rows.map((r) => r.iri))]; + console.log(`${step.type}: ${rows.length} rows`); +} + +const streams = ctx.targetIris.length; +const facilities = ctx.anchorIris.length; +console.log(`\n${streams} streams within 30 km downstream of ${facilities} NH airport facilities`); + +// The bound has to actually bind: unbounded, this trace runs to the coast. +const unbounded = planPipeline({ ...question, relationship: { type: 'downstream' } }); +const unboundedRows = await run(unbounded[0].endpoint, unbounded[0].buildQuery(ctx)); +const unboundedStreams = new Set(unboundedRows.map((r) => r.iri)).size; +console.log(`${unboundedStreams} streams with no distance bound`); + +assert.ok(streams > 1547, `expected a superset of the notebook's 1547, got ${streams}`); +// The bound must actually exclude flowlines the unbounded trace reaches. +// Observed 2026-08: 2757 bounded vs 3490 unbounded — downstreamFlowPathTC in +// this KG does not reach as far as the coast, so the gap is smaller than the +// notebook's parameters suggest. +assert.ok( + unboundedStreams > streams, + `bound had no effect: ${streams} bounded vs ${unboundedStreams} unbounded`, +); + +// Every hydrated stream carries geometry the map can draw. +const hydrated = ctx.results['HYDRATE_TARGET_BY_IRI'] ?? []; +assert.ok(hydrated.length > 0, 'no hydrated streams'); +assert.ok( + hydrated.every((r) => r.flowlineWKT?.startsWith('LINESTRING')), + 'hydrated streams missing LINESTRING geometry', +); + +console.log('\nOK'); diff --git a/src/components/Map/MapPopup.tsx b/src/components/Map/MapPopup.tsx index d13fa74..e95102b 100644 --- a/src/components/Map/MapPopup.tsx +++ b/src/components/Map/MapPopup.tsx @@ -321,6 +321,12 @@ export function MapPopupContent({ feature }: MapPopupProps) { {props.flowType} )} + {props.pathLength && ( + + Flow distance + {Number(props.pathLength).toFixed(1)} km + + )} {feature.id && ( Flowline diff --git a/src/components/QueryEditor/EntityBlock.tsx b/src/components/QueryEditor/EntityBlock.tsx index f7f09c4..743e96c 100644 --- a/src/components/QueryEditor/EntityBlock.tsx +++ b/src/components/QueryEditor/EntityBlock.tsx @@ -4,6 +4,7 @@ import { SampleFilters } from './SampleFilters'; import { FacilityFilters } from './FacilityFilters'; import { WaterBodyFilters } from './WaterBodyFilters'; import { WellFilters } from './WellFilters'; +import { StreamFilters } from './StreamFilters'; import { RegionSelector } from './RegionSelector'; import { useState } from 'react'; @@ -21,6 +22,7 @@ export function EntityBlock({ label, value, onChange }: EntityBlockProps) { value.sampleFilters?.materialTypes?.length || value.facilityFilters?.industryCodes?.length || value.waterBodyFilters?.ftypes?.length || + value.streamFilters?.ftypes?.length || value.wellFilters?.wellTypes?.length ) ); @@ -62,6 +64,12 @@ export function EntityBlock({ label, value, onChange }: EntityBlockProps) { onChange={(f) => onChange({ ...value, waterBodyFilters: f })} /> )} + {value.type === 'streams' && ( + onChange({ ...value, streamFilters: f })} + /> + )} {value.type === 'wells' && ( r.value === value.type) || RELATIONSHIP_TYPES[0]; const currentHops = value.hops ?? 1; @@ -40,6 +50,20 @@ export function RelationshipSelector({ value, onChange }: RelationshipSelectorPr isClearable={false} placeholder="Select relationship..." /> + {(value.type === 'downstream' || value.type === 'upstream') && ( + { + const km = vals[0] ? parseInt(vals[0]) : undefined; + onChange({ ...value, maxDistanceKm: km }); + }} + isMulti={false} + isClearable={false} + searchable={false} + placeholder="Flow distance..." + /> + )} {value.type === 'near' && ( void; +} + +// NHDPlusV2 FTYPE values present on hyf:HY_FlowPath in hydrologykg. +const FTYPE_OPTIONS = [ + { value: 'StreamRiver', label: 'Stream / River' }, + { value: 'ArtificialPath', label: 'Artificial Path' }, + { value: 'CanalDitch', label: 'Canal / Ditch' }, + { value: 'Connector', label: 'Connector' }, + { value: 'Pipeline', label: 'Pipeline' }, +]; + +export function StreamFilters({ value, onChange }: StreamFiltersProps) { + return ( +
+
+ + onChange({ ...value, ftypes: vals })} + placeholder="Any stream type..." + /> +
+
+ ); +} diff --git a/src/engine/planner.ts b/src/engine/planner.ts index 574e23b..4431c75 100644 --- a/src/engine/planner.ts +++ b/src/engine/planner.ts @@ -10,6 +10,7 @@ import { } from './templates/fusedQueries'; import { buildFacilitiesByIri, + buildStreamsByIri, buildWaterBodiesByIri, buildWellsByIri, } from './templates/hydrate'; @@ -54,6 +55,7 @@ function entityEndpoint(block: EntityBlock): EndpointKey { return 'sawgraph'; case 'waterBodies': case 'wells': + case 'streams': return 'hydrologykg'; } } @@ -102,6 +104,8 @@ function hydrateStep( return buildWaterBodiesByIri(iris, block.waterBodyFilters); case 'wells': return buildWellsByIri(iris, block.wellFilters); + case 'streams': + return buildStreamsByIri(iris, block.streamFilters); } }, }; @@ -149,6 +153,7 @@ function buildFusedSteps(question: AnalysisQuestion): PipelineStep[] { project, anchorRegion: anchorRegionOpt, targetRegion: targetRegionOpt, + maxDistanceKm: relationship.maxDistanceKm, }); }; @@ -166,7 +171,11 @@ function buildFusedSteps(question: AnalysisQuestion): PipelineStep[] { buildQuery: () => buildIriQuery('anchor'), }); - if (relationship.type !== 'near') { + // Supporting stream layer. Skipped when a side is already streams — those + // flowlines come back as the answer set and would be drawn twice. + const streamsAreAnswer = + targetBlock.type === 'streams' || anchorBlock.type === 'streams'; + if (relationship.type !== 'near' && !streamsAreAnswer) { steps.push({ type: 'GET_FLOWLINE_GEOMETRIES', endpoint: 'federation', @@ -177,6 +186,7 @@ function buildFusedSteps(question: AnalysisQuestion): PipelineStep[] { anchor: anchorBlock, direction: relationship.type === 'downstream' ? 'downstream' : 'upstream', anchorIris: ctx.anchorIris, + maxDistanceKm: relationship.maxDistanceKm, }), }); } diff --git a/src/engine/resultTransformer.ts b/src/engine/resultTransformer.ts index 6f1f86b..6dd8494 100644 --- a/src/engine/resultTransformer.ts +++ b/src/engine/resultTransformer.ts @@ -173,6 +173,7 @@ export function transformFlowlinesToFeatures(rows: SparqlRow[]): MapFeature[] { type: 'stream', name: r.streamName || '', flowType: r.fl_type || '', + ...(r.path_length ? { pathLength: r.path_length } : {}), }, }); } diff --git a/src/engine/templates/fusedQueries.ts b/src/engine/templates/fusedQueries.ts index a3914bf..03752c4 100644 --- a/src/engine/templates/fusedQueries.ts +++ b/src/engine/templates/fusedQueries.ts @@ -2,6 +2,7 @@ import { PREFIXES } from '../../constants/prefixes'; import type { EntityBlock, FacilityFilters, + StreamFilters, WellFilters, SpatialRelationship, } from '../../types/query'; @@ -20,6 +21,8 @@ export function entityIriVar(block: EntityBlock, suffix: string): string { return `?waterBody${suffix}`; case 'wells': return `?well${suffix}`; + case 'streams': + return `?stream${suffix}`; } } @@ -86,9 +89,22 @@ export function bindEntityInCell(block: EntityBlock, s2Var: string, suffix: stri return `${s2Var} spatial:connectedTo ?well${suffix} . ${typeFilter}`; } + case 'streams': { + return `${s2Var} spatial:connectedTo ?stream${suffix} . + ?stream${suffix} rdf:type hyf:HY_FlowPath . + ${streamFtypeFilter(block.streamFilters, suffix)}`; + } } } +// FTYPE restriction for flowlines, shared by the s2-hop and direct-bind forms. +function streamFtypeFilter(filters: StreamFilters | undefined, suffix: string): string { + if (!filters?.ftypes?.length) return ''; + const values = filters.ftypes.map((f) => `"${f}"`).join(' '); + return `?stream${suffix} nhdplusv2:hasFTYPE ?flFtype${suffix} . + VALUES ?flFtype${suffix} { ${values} }`; +} + function regionClause(regionCodes: string[] | undefined, s2Var: string, internalVar: string): string { if (!regionCodes?.length) return ''; if (regionCodes.length === 1) { @@ -122,6 +138,45 @@ function neighborPath(hops: number, fromVar: string, toVar: string): string { interface FusedBodyOpts extends FusedBaseOpts { mode: 'near' | 'downstream' | 'upstream'; hops?: number; + maxDistanceKm?: number; +} + +// Wraps the hydrology trace in a cumulative-length cutoff. The seed block is +// duplicated inside so the closure stays anchored — without it the aggregate +// runs over the whole national flowline graph. +// +// Membership semantics: a flowline qualifies if *some* seed reaches it within +// the cutoff, matching GROUP BY (seed, end) with no MIN. The per-flowline +// number shown in popups is computed separately in buildFusedFlowlineQuery / +// buildStreamsByIri, where MIN() picks the shortest qualifying path. +function boundedTrace( + seed: string, + direction: 'downstream' | 'upstream', + maxDistanceKm: number, +): string { + const trace = + direction === 'downstream' + ? `?upstream_flowline hyf:downstreamFlowPathTC ?_flMid . + ?_flMid hyf:downstreamFlowPathTC ?ds_flowline .` + : `?ds_flowline hyf:downstreamFlowPathTC ?_flMid . + ?_flMid hyf:downstreamFlowPathTC ?upstream_flowline .`; + + return `{ + SELECT DISTINCT ?upstream_flowline ?ds_flowline WHERE { + { + SELECT ?upstream_flowline ?ds_flowline (SUM(?_flLen) AS ?_plen) WHERE { + { + SELECT ?upstream_flowline ?_flMid ?ds_flowline WHERE { + { SELECT DISTINCT ?upstream_flowline WHERE { ${seed} } } + ${trace} + } + } + ?_flMid nhdplusv2:hasFlowPathLength/qudt:quantityValue/qudt:numericValue ?_flLen . + } GROUP BY ?upstream_flowline ?ds_flowline + } + FILTER (xsd:float(?_plen) < xsd:float(${maxDistanceKm})) + } + }`; } // Returns just the inner WHERE-body patterns shared across all fused queries: @@ -144,22 +199,41 @@ function buildFusedWhereBody(opts: FusedBodyOpts): string { ${targetBind}`; } - const traceTriple = - opts.mode === 'downstream' - ? `?upstream_flowline hyf:downstreamFlowPathTC ?ds_flowline .` - : `?ds_flowline hyf:downstreamFlowPathTC ?upstream_flowline .`; - - return `?s2anchor rdf:type kwg-ont:S2Cell_Level13 . + const seed = `?s2anchor rdf:type kwg-ont:S2Cell_Level13 . ${aRegion} ${anchorBind} ?s2anchor kwg-ont:sfTouches | owl:sameAs ?s2neighbor . ?s2neighbor spatial:connectedTo ?upstream_flowline . - ?upstream_flowline rdf:type hyf:HY_FlowPath . - ${traceTriple} - ?s2target spatial:connectedTo ?ds_flowline ; + ?upstream_flowline rdf:type hyf:HY_FlowPath .`; + + const trace = opts.maxDistanceKm + ? boundedTrace(seed, opts.mode, opts.maxDistanceKm) + : opts.mode === 'downstream' + ? `?upstream_flowline hyf:downstreamFlowPathTC ?ds_flowline .` + : `?ds_flowline hyf:downstreamFlowPathTC ?upstream_flowline .`; + + // When the target *is* a flowline, the traced ?ds_flowline already is the + // answer — the s2target hop would find any flowline sharing a cell with it. + // The cell is still needed if the target carries a region filter. + const targetPart = + opts.target.type === 'streams' + ? `${ + tRegion + ? `?s2target spatial:connectedTo ?ds_flowline ; + rdf:type kwg-ont:S2Cell_Level13 . + ${tRegion}` + : '' + } + BIND(?ds_flowline AS ?streamC) + ${streamFtypeFilter(opts.target.streamFilters, 'C')}` + : `?s2target spatial:connectedTo ?ds_flowline ; rdf:type kwg-ont:S2Cell_Level13 . ${tRegion} ${targetBind}`; + + return `${seed} + ${trace} + ${targetPart}`; } function relationshipMode( @@ -209,6 +283,7 @@ export function buildFusedNearQuery(opts: FusedNearOpts): string { export interface FusedHydrologyOpts extends FusedBaseOpts { direction: 'downstream' | 'upstream'; project: 'anchor' | 'target'; + maxDistanceKm?: number; } // Server-side downstream/upstream query. @@ -219,6 +294,7 @@ export function buildFusedHydrologyQuery(opts: FusedHydrologyOpts): string { anchorRegion: opts.anchorRegion, targetRegion: opts.targetRegion, mode: opts.direction, + maxDistanceKm: opts.maxDistanceKm, }); const projectVar = opts.project === 'anchor' @@ -252,6 +328,7 @@ export function buildFusedSampleAggregateQuery(opts: FusedSampleSideOpts): strin targetRegion: opts.targetRegion, mode: relationshipMode(opts.relationship), hops: opts.relationship.hops, + maxDistanceKm: opts.relationship.maxDistanceKm, }); const suffix = opts.sampleSide === 'anchor' ? 'A' : 'C'; const s2Var = opts.sampleSide === 'anchor' ? '?s2anchor' : '?s2target'; @@ -291,6 +368,7 @@ export function buildFusedSampleDetailsQuery(opts: FusedSampleSideOpts): string targetRegion: opts.targetRegion, mode: relationshipMode(opts.relationship), hops: opts.relationship.hops, + maxDistanceKm: opts.relationship.maxDistanceKm, }); const suffix = opts.sampleSide === 'anchor' ? 'A' : 'C'; const spVar = `?sp${suffix}`; @@ -353,6 +431,7 @@ export interface FusedFlowlineOpts { anchor: EntityBlock; direction: 'downstream' | 'upstream'; anchorIris: string[]; + maxDistanceKm?: number; } // Returns flowline geometries traced from the anchor entities the pipeline @@ -375,6 +454,15 @@ export function buildFusedFlowlineQuery(opts: FusedFlowlineOpts): string { .map(wrapUri) .join(' ')} }`; + const seedCells = `{ + SELECT DISTINCT ?s2cellus WHERE { + ${anchorValues} + ?s2anchor rdf:type kwg-ont:S2Cell_Level13 . + ${anchorBind} + ?s2anchor kwg-ont:sfTouches | owl:sameAs ?s2cellus . + } + }`; + const flowlinePattern = opts.direction === 'downstream' ? `?upstream_flowline rdf:type hyf:HY_FlowPath ; @@ -384,18 +472,54 @@ export function buildFusedFlowlineQuery(opts: FusedFlowlineOpts): string { spatial:connectedTo ?s2cellus . ?flowline hyf:downstreamFlowPathTC ?downstream_flowline .`; - return ` + // Unbounded: the flowline set is the plain transitive closure. + if (!opts.maxDistanceKm) { + return ` ${PREFIXES} SELECT DISTINCT ?flowline ?flowlineWKT ?fl_type ?streamName WHERE { + ${seedCells} + ${flowlinePattern} + ?flowline geo:hasGeometry/geo:asWKT ?flowlineWKT ; + nhdplusv2:hasFTYPE ?fl_type . + OPTIONAL { ?flowline rdfs:label ?streamName } + } + `; + } + + // Bounded: sum the lengths of the segments between seed and candidate, then + // MIN across seeds so each flowline carries one distance for its popup. + const boundedTraceInner = + opts.direction === 'downstream' + ? `?_flSeed hyf:downstreamFlowPathTC ?_flMid . + ?_flMid hyf:downstreamFlowPathTC ?flowline .` + : `?flowline hyf:downstreamFlowPathTC ?_flMid . + ?_flMid hyf:downstreamFlowPathTC ?_flSeed .`; + + return ` + ${PREFIXES} + SELECT DISTINCT ?flowline ?flowlineWKT ?fl_type ?streamName ?path_length WHERE { { - SELECT DISTINCT ?s2cellus WHERE { - ${anchorValues} - ?s2anchor rdf:type kwg-ont:S2Cell_Level13 . - ${anchorBind} - ?s2anchor kwg-ont:sfTouches | owl:sameAs ?s2cellus . - } + SELECT ?flowline (MIN(?_plen) AS ?path_length) WHERE { + { + SELECT ?_flSeed ?flowline (SUM(?_flLen) AS ?_plen) WHERE { + { + SELECT ?_flSeed ?_flMid ?flowline WHERE { + { + SELECT DISTINCT ?_flSeed WHERE { + ${seedCells} + ?_flSeed rdf:type hyf:HY_FlowPath ; + spatial:connectedTo ?s2cellus . + } + } + ${boundedTraceInner} + } + } + ?_flMid nhdplusv2:hasFlowPathLength/qudt:quantityValue/qudt:numericValue ?_flLen . + } GROUP BY ?_flSeed ?flowline + } + FILTER (xsd:float(?_plen) < xsd:float(${opts.maxDistanceKm})) + } GROUP BY ?flowline } - ${flowlinePattern} ?flowline geo:hasGeometry/geo:asWKT ?flowlineWKT ; nhdplusv2:hasFTYPE ?fl_type . OPTIONAL { ?flowline rdfs:label ?streamName } diff --git a/src/engine/templates/hydrate.ts b/src/engine/templates/hydrate.ts index 8d87f4b..b8ff3c7 100644 --- a/src/engine/templates/hydrate.ts +++ b/src/engine/templates/hydrate.ts @@ -1,6 +1,7 @@ import { PREFIXES } from '../../constants/prefixes'; import type { FacilityFilters, + StreamFilters, WaterBodyFilters, WellFilters, } from '../../types/query'; @@ -65,6 +66,29 @@ export function buildWaterBodiesByIri( `; } +// Column names match buildFusedFlowlineQuery so transformFlowlinesToFeatures +// consumes either one unchanged. +export function buildStreamsByIri(streamIris: string[], filters?: StreamFilters): string { + let filterClauses = ''; + if (filters?.ftypes?.length) { + const ftypeValues = filters.ftypes.map((f) => `"${f}"`).join(' '); + filterClauses = `VALUES ?fl_type { ${ftypeValues} }`; + } + const vals = streamIris.map(wrapUri).join(' '); + + return ` + ${PREFIXES} + SELECT DISTINCT ?flowline ?flowlineWKT ?fl_type ?streamName WHERE { + VALUES ?flowline { ${vals} } + ?flowline rdf:type hyf:HY_FlowPath ; + geo:hasGeometry/geo:asWKT ?flowlineWKT ; + nhdplusv2:hasFTYPE ?fl_type . + OPTIONAL { ?flowline rdfs:label ?streamName } + ${filterClauses} + } + `; +} + export function buildWellsByIri(wellIris: string[], filters?: WellFilters): string { const wellTypes = filters?.wellTypes; let typeFilter: string; diff --git a/src/hooks/useMapLayers.ts b/src/hooks/useMapLayers.ts index 4eb8dec..614ae16 100644 --- a/src/hooks/useMapLayers.ts +++ b/src/hooks/useMapLayers.ts @@ -48,6 +48,9 @@ export function useMapLayers(result: PipelineResult | null): MapLayerData { const facilityRows = allRows.filter((r) => r.facWKT); const waterBodyRows = allRows.filter((r) => r.wbWKT); const wellRows = allRows.filter((r) => r.wellWKT); + // Streams can arrive either as the answer set (hydrated, when a block is + // streams) or as the supporting layer traced from the anchors. + const streamRows = allRows.filter((r) => r.flowlineWKT); const sampleFeatures = transformSamplesToFeatures(sampleRows); if (sampleDetailRows.length > 0) { @@ -59,7 +62,7 @@ export function useMapLayers(result: PipelineResult | null): MapLayerData { facilities: transformFacilitiesToFeatures(facilityRows), waterBodies: transformWaterBodiesToFeatures(waterBodyRows), wells: transformWellsToFeatures(wellRows), - streams: transformFlowlinesToFeatures(flowlineRows), + streams: transformFlowlinesToFeatures([...streamRows, ...flowlineRows]), regionBoundaries: transformRegionBoundaries(boundaryRows), }; }, [result]); diff --git a/src/types/map.ts b/src/types/map.ts index 0552167..7d16624 100644 --- a/src/types/map.ts +++ b/src/types/map.ts @@ -3,7 +3,7 @@ import type { LatLngExpression } from 'leaflet'; export interface MapLayer { id: string; label: string; - type: 'samples' | 'facilities' | 'waterBodies' | 'wells' | 'regionBoundary'; + type: 'samples' | 'facilities' | 'waterBodies' | 'wells' | 'streams' | 'regionBoundary'; visible: boolean; data: MapFeature[]; } diff --git a/src/types/query.ts b/src/types/query.ts index 822ba0f..436a794 100644 --- a/src/types/query.ts +++ b/src/types/query.ts @@ -5,7 +5,7 @@ export interface AnalysisQuestion { blockC: EntityBlock; } -export type EntityType = 'samples' | 'facilities' | 'waterBodies' | 'wells'; +export type EntityType = 'samples' | 'facilities' | 'waterBodies' | 'wells' | 'streams'; export interface EntityBlock { type: EntityType; @@ -14,6 +14,7 @@ export interface EntityBlock { facilityFilters?: FacilityFilters; waterBodyFilters?: WaterBodyFilters; wellFilters?: WellFilters; + streamFilters?: StreamFilters; } export interface RegionFilter { @@ -43,6 +44,10 @@ export interface WaterBodyFilters { ftypes?: string[]; } +export interface StreamFilters { + ftypes?: string[]; +} + export interface WellFilters { wellTypes?: string[]; } @@ -50,4 +55,7 @@ export interface WellFilters { export interface SpatialRelationship { type: 'near' | 'downstream' | 'upstream' | 'within'; hops?: number; + // Cumulative flowpath length cutoff (km) for downstream/upstream traces. + // Undefined = unbounded transitive closure, the original behavior. + maxDistanceKm?: number; } diff --git a/src/utils/questionGenerator.ts b/src/utils/questionGenerator.ts index eae9012..7e9fc22 100644 --- a/src/utils/questionGenerator.ts +++ b/src/utils/questionGenerator.ts @@ -149,6 +149,11 @@ function describeEntity(block: EntityBlock, totals?: QuestionTotals): string { } case 'waterBodies': return 'surface water bodies'; + case 'streams': { + const ftypes = block.streamFilters?.ftypes ?? []; + if (!ftypes.length) return 'streams'; + return `${summarize(ftypes, undefined, 'stream types', 'stream type', '/', 'any')} streams`; + } case 'wells': return 'wells'; } @@ -162,9 +167,13 @@ function describeRelationship(rel: SpatialRelationship): string { return `near (~${miles} mile${miles > 1 ? 's' : ''})`; } case 'downstream': - return 'downstream of'; + return rel.maxDistanceKm + ? `within ${rel.maxDistanceKm} km downstream of` + : 'downstream of'; case 'upstream': - return 'upstream from'; + return rel.maxDistanceKm + ? `within ${rel.maxDistanceKm} km upstream from` + : 'upstream from'; case 'within': return 'within'; } From 0ff2f393df88fac991a5f8a74b98e7984248982d Mon Sep 17 00:00:00 2001 From: Prayas Lashkari Date: Tue, 25 Aug 2026 17:12:05 -0400 Subject: [PATCH 2/4] feat(engine): extend bounded traces one flowline past the cutoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "+1" from David's second UC1-CQ2c notebook. A distance budget runs out at whatever segment happens to fit, which is an artifact of how NHDPlus split the river rather than a real feature — the drawn path stops mid-channel. Extending one segment past the boundary means the flowpath visibly crosses the threshold instead of ending at it, which is the notebook's stated intent: the total may deliberately exceed the limit. Always on when a cutoff is set. A checkbox for a 1%-at-30km difference is a control nobody would understand, and "within 30 km" already reads as approximate. Implemented as a zero-or-one property path, which yields the endpoint and its immediate neighbour in one triple. A UNION says the same thing, but QLever — which the notebooks run against — returns unbound results for MIN() over a variable bound inside a UNION, verified on a two-row test case with no data involved. The path form works on both hosts. Same reason `?a = ?b` rather than sameTerm(): QLever has not implemented sameTerm. Fringe segments cannot inherit their parent's distance, since that would report a sub-threshold number for a flowline outside the threshold. They carry the parent's path plus the parent's own length — the distance to where the fringe segment begins, which never understates. A flowline reachable as both a valid endpoint and a fringe keeps the smaller value, since MIN runs over both. Effect scales inversely with the cutoff, as the fringe is a larger share of a smaller answer (NH airports): 5 km 1,533 -> 1,669 (+8.9%) 10 km 2,019 -> 2,096 (+3.8%) 30 km 2,757 -> 2,784 (+1.0%) Verified against apps.okn.us: 2,784 at 30 km, 1,669 at 5 km, 3,490 unbounded (unchanged), and flowline distances spanning 0.01–32.23 km with the fringe correctly reporting past the threshold. FRINK is returning 503 across all five endpoints right now, so the committed check script has not been re-run against it — the two hosts were verified to return identical result sets earlier in this work. --- scripts/flow-distance-check.mjs | 16 ++++++--- src/engine/templates/fusedQueries.ts | 52 ++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/scripts/flow-distance-check.mjs b/scripts/flow-distance-check.mjs index 8e00036..735a961 100644 --- a/scripts/flow-distance-check.mjs +++ b/scripts/flow-distance-check.mjs @@ -5,7 +5,8 @@ // 30 km): the notebook returns 162 flowlines, but only because it requires // schema1:address on facilities — a predicate 13 of 144 NH airport facilities // have. Without that accidental filter its answer is 1,547 flowlines, and ours -// is a superset of exactly that set (we also expand to neighbouring S2 cells). +// is a superset of exactly that set (we also expand to neighbouring S2 cells +// and add the "+1" segment past the cutoff). // // Run: node scripts/flow-distance-check.mjs import assert from 'node:assert/strict'; @@ -79,6 +80,13 @@ const streams = ctx.targetIris.length; const facilities = ctx.anchorIris.length; console.log(`\n${streams} streams within 30 km downstream of ${facilities} NH airport facilities`); +// The "+1" fringe must land past the cutoff, so some flowlines carry a +// distance above the threshold — that is the point of it. +const overThreshold = (ctx.results['GET_FLOWLINE_GEOMETRIES'] ?? []) + .concat(ctx.results['HYDRATE_TARGET_BY_IRI'] ?? []) + .filter((r) => r.path_length !== undefined && Number(r.path_length) >= 30); +console.log(`${overThreshold.length} flowlines past the 30 km cutoff (the "+1" fringe)`); + // The bound has to actually bind: unbounded, this trace runs to the coast. const unbounded = planPipeline({ ...question, relationship: { type: 'downstream' } }); const unboundedRows = await run(unbounded[0].endpoint, unbounded[0].buildQuery(ctx)); @@ -87,9 +95,9 @@ console.log(`${unboundedStreams} streams with no distance bound`); assert.ok(streams > 1547, `expected a superset of the notebook's 1547, got ${streams}`); // The bound must actually exclude flowlines the unbounded trace reaches. -// Observed 2026-08: 2757 bounded vs 3490 unbounded — downstreamFlowPathTC in -// this KG does not reach as far as the coast, so the gap is smaller than the -// notebook's parameters suggest. +// Observed 2026-08: 2784 bounded (2757 within the cutoff + 27 fringe) vs 3490 +// unbounded — downstreamFlowPathTC in this KG does not reach as far as the +// coast, so the gap is smaller than the notebook's parameters suggest. assert.ok( unboundedStreams > streams, `bound had no effect: ${streams} bounded vs ${unboundedStreams} unbounded`, diff --git a/src/engine/templates/fusedQueries.ts b/src/engine/templates/fusedQueries.ts index 03752c4..da4bed3 100644 --- a/src/engine/templates/fusedQueries.ts +++ b/src/engine/templates/fusedQueries.ts @@ -141,6 +141,25 @@ interface FusedBodyOpts extends FusedBaseOpts { maxDistanceKm?: number; } +// Extends the trace one flowline past the cutoff ("+1"). The budget runs out at +// whatever segment happens to fit, which is an artifact of how NHDPlus split +// the river rather than a real feature — without this the drawn path stops +// mid-channel. Matches David's second UC1-CQ2c notebook, where the total +// flowpath may deliberately exceed the threshold. +// +// The zero-or-one path yields the endpoint itself *and* its immediate +// neighbour in one triple. A UNION would express the same thing, but QLever — +// which the notebooks run against — returns unbound results for MIN() over a +// variable bound inside a UNION, so the path form keeps this portable across +// both hosts. Note the direction flip: downstream extends past the downstream +// end, upstream past the upstream end. hyf:downstreamFlowPath has no TC, so +// it is one segment. +function fringePath(direction: 'downstream' | 'upstream', endVar: string, outVar: string): string { + return direction === 'downstream' + ? `${endVar} hyf:downstreamFlowPath? ${outVar} .` + : `${outVar} hyf:downstreamFlowPath? ${endVar} .`; +} + // Wraps the hydrology trace in a cumulative-length cutoff. The seed block is // duplicated inside so the closure stays anchored — without it the aggregate // runs over the whole national flowline graph. @@ -157,24 +176,25 @@ function boundedTrace( const trace = direction === 'downstream' ? `?upstream_flowline hyf:downstreamFlowPathTC ?_flMid . - ?_flMid hyf:downstreamFlowPathTC ?ds_flowline .` - : `?ds_flowline hyf:downstreamFlowPathTC ?_flMid . + ?_flMid hyf:downstreamFlowPathTC ?_flEnd .` + : `?_flEnd hyf:downstreamFlowPathTC ?_flMid . ?_flMid hyf:downstreamFlowPathTC ?upstream_flowline .`; return `{ SELECT DISTINCT ?upstream_flowline ?ds_flowline WHERE { { - SELECT ?upstream_flowline ?ds_flowline (SUM(?_flLen) AS ?_plen) WHERE { + SELECT ?upstream_flowline ?_flEnd (SUM(?_flLen) AS ?_plen) WHERE { { - SELECT ?upstream_flowline ?_flMid ?ds_flowline WHERE { + SELECT ?upstream_flowline ?_flMid ?_flEnd WHERE { { SELECT DISTINCT ?upstream_flowline WHERE { ${seed} } } ${trace} } } ?_flMid nhdplusv2:hasFlowPathLength/qudt:quantityValue/qudt:numericValue ?_flLen . - } GROUP BY ?upstream_flowline ?ds_flowline + } GROUP BY ?upstream_flowline ?_flEnd } FILTER (xsd:float(?_plen) < xsd:float(${maxDistanceKm})) + ${fringePath(direction, '?_flEnd', '?ds_flowline')} } }`; } @@ -491,19 +511,25 @@ export function buildFusedFlowlineQuery(opts: FusedFlowlineOpts): string { const boundedTraceInner = opts.direction === 'downstream' ? `?_flSeed hyf:downstreamFlowPathTC ?_flMid . - ?_flMid hyf:downstreamFlowPathTC ?flowline .` - : `?flowline hyf:downstreamFlowPathTC ?_flMid . + ?_flMid hyf:downstreamFlowPathTC ?_flEnd .` + : `?_flEnd hyf:downstreamFlowPathTC ?_flMid . ?_flMid hyf:downstreamFlowPathTC ?_flSeed .`; + // The "+1" segment lies past the cutoff, so it can't reuse its parent's + // distance — that would report a number under the threshold for a flowline + // outside it. Adding the parent's own length gives the distance to where the + // fringe segment starts, which never understates. A flowline that is both a + // valid endpoint (via one seed) and a fringe (via another) keeps the smaller + // value, since MIN runs over the union. return ` ${PREFIXES} SELECT DISTINCT ?flowline ?flowlineWKT ?fl_type ?streamName ?path_length WHERE { { - SELECT ?flowline (MIN(?_plen) AS ?path_length) WHERE { + SELECT ?flowline (MIN(?_ptotal) AS ?path_length) WHERE { { - SELECT ?_flSeed ?flowline (SUM(?_flLen) AS ?_plen) WHERE { + SELECT ?_flSeed ?_flEnd (SUM(?_flLen) AS ?_plen) WHERE { { - SELECT ?_flSeed ?_flMid ?flowline WHERE { + SELECT ?_flSeed ?_flMid ?_flEnd WHERE { { SELECT DISTINCT ?_flSeed WHERE { ${seedCells} @@ -515,9 +541,13 @@ export function buildFusedFlowlineQuery(opts: FusedFlowlineOpts): string { } } ?_flMid nhdplusv2:hasFlowPathLength/qudt:quantityValue/qudt:numericValue ?_flLen . - } GROUP BY ?_flSeed ?flowline + } GROUP BY ?_flSeed ?_flEnd } FILTER (xsd:float(?_plen) < xsd:float(${opts.maxDistanceKm})) + ?_flEnd nhdplusv2:hasFlowPathLength/qudt:quantityValue/qudt:numericValue ?_endLen . + ${fringePath(opts.direction, '?_flEnd', '?flowline')} + BIND(IF(?flowline = ?_flEnd, 0.0, ?_endLen) AS ?_extra) + BIND(xsd:float(?_plen) + xsd:float(?_extra) AS ?_ptotal) } GROUP BY ?flowline } ?flowline geo:hasGeometry/geo:asWKT ?flowlineWKT ; From 3a03fa5929276e397e70655b8287ab7819d39bd0 Mon Sep 17 00:00:00 2001 From: Prayas Lashkari Date: Fri, 28 Aug 2026 00:58:02 +0530 Subject: [PATCH 3/4] fix(endpoints): move SPARQL endpoints from FRINK to apps.okn.us FRINK retired https://frink.apps.renci.org//sparql; all five knowledge graphs now return 503 there and are served from https://apps.okn.us//sparql instead. Reported by David Kedrowski, who hit the same break in his notebooks. Verified all five endpoints live on the new host (200, CORS *), and replayed the real pipeline through planPipeline/executePipeline against them: 7 of 8 prebuilt queries succeed and all discovery queries return live data rather than falling back to hardcoded constants. The Indiana downstream prebuilt still fails, but with a QLever memory-limit error ("Tried to allocate 819.2 MB, but only 743.3 MB were available"), not a routing failure. Tracked separately. --- README.md | 2 +- docs/ARCHITECTURE.md | 2 +- src/constants/endpoints.ts | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f219bdf..7f90bb4 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Supported relationships: **near** (~1–2 km), **downstream**, **upstream** ## SPARQL endpoints -All hosted at `frink.apps.renci.org`: +All hosted at `apps.okn.us`: | Endpoint | Used for | | ------------- | ----------------------------------------------- | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5de2ab1..5ea4002 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -696,7 +696,7 @@ console.log(results); Copy a query from the console, run it at: ``` -https://frink.apps.renci.org/fiokg/sparql +https://apps.okn.us/fiokg/sparql ``` ### 4. Check S2 Cell Counts diff --git a/src/constants/endpoints.ts b/src/constants/endpoints.ts index 858072a..7af9534 100644 --- a/src/constants/endpoints.ts +++ b/src/constants/endpoints.ts @@ -1,9 +1,9 @@ export const ENDPOINTS = { - sawgraph: 'https://frink.apps.renci.org/sawgraph/sparql', - fiokg: 'https://frink.apps.renci.org/fiokg/sparql', - spatialkg: 'https://frink.apps.renci.org/spatialkg/sparql', - hydrologykg: 'https://frink.apps.renci.org/hydrologykg/sparql', - federation: 'https://frink.apps.renci.org/federation/sparql', + sawgraph: 'https://apps.okn.us/sawgraph/sparql', + fiokg: 'https://apps.okn.us/fiokg/sparql', + spatialkg: 'https://apps.okn.us/spatialkg/sparql', + hydrologykg: 'https://apps.okn.us/hydrologykg/sparql', + federation: 'https://apps.okn.us/federation/sparql', } as const; export type EndpointKey = keyof typeof ENDPOINTS; From 266130ba05e691748c202b2a674482d7f22492e9 Mon Sep 17 00:00:00 2001 From: Prayas Lashkari Date: Fri, 28 Aug 2026 01:09:30 +0530 Subject: [PATCH 4/4] chore: drop stale render.yaml, document Railway as the deploy target The app deploys on Railway from main, but the repo still carried a Render blueprint from the publish workflow and no mention of Railway anywhere. The stale file was actively misleading about where production runs. Removes render.yaml, documents the real deploy target in the README, and adds the W35 changelog covering this and the FRINK endpoint migration. --- README.md | 5 +++++ docs/changelog/2026-W35.md | 13 +++++++++++++ render.yaml | 39 -------------------------------------- 3 files changed, 18 insertions(+), 39 deletions(-) create mode 100644 docs/changelog/2026-W35.md delete mode 100644 render.yaml diff --git a/README.md b/README.md index 7f90bb4..84c290c 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,11 @@ npm run lint # ESLint npm run preview # Preview production build ``` +## Deployment + +Deployed on **Railway** from the `main` branch. Build config lives in the Railway +dashboard, not in this repo — there is no `railway.json` or `nixpacks.toml` here. + ## How queries work An **Analysis Question** has three parts: diff --git a/docs/changelog/2026-W35.md b/docs/changelog/2026-W35.md new file mode 100644 index 0000000..c7d4514 --- /dev/null +++ b/docs/changelog/2026-W35.md @@ -0,0 +1,13 @@ +# Changelog — Week 35, 2026 (Aug 24 – Aug 30) + +--- + +1. **Move SPARQL Endpoints from FRINK to `apps.okn.us`** (2026-08-28) + FRINK retired `https://frink.apps.renci.org//sparql`; all five knowledge graphs began returning 503 and Explorer was unusable for several days. Repointed every endpoint at `https://apps.okn.us//sparql`. Reported by David Kedrowski, who hit the same break in his notebooks. `sparqlClient.ts` reads all URLs from the `ENDPOINTS` map and nothing else hardcodes a host, so this was a five-line change. Verified by replaying the real pipeline against the new host: 7 of 8 prebuilt queries pass and all discovery queries return live data instead of falling back to hardcoded constants. + + *Files:* `constants/endpoints.ts`, `docs/ARCHITECTURE.md`, `README.md` + +2. **Remove Stale `render.yaml`** (2026-08-28) + Deleted the Render blueprint left over from the publish workflow. The app deploys on Railway now, and the stale file was actively misleading about where production runs. Documented the actual deploy target in the README instead. + + *Files:* `render.yaml` (deleted), `README.md` diff --git a/render.yaml b/render.yaml deleted file mode 100644 index b43fa97..0000000 --- a/render.yaml +++ /dev/null @@ -1,39 +0,0 @@ -databases: - - name: sawgraph-db - plan: free - databaseName: sawgraph - user: sawgraph - -services: - - type: web - name: sawgraph-api - runtime: node - plan: free - rootDir: server - buildCommand: npm ci && npm run build - startCommand: npm start - healthCheckPath: /health - envVars: - - key: DATABASE_URL - fromDatabase: - name: sawgraph-db - property: connectionString - - key: FRONTEND_ORIGIN - sync: false - - key: NODE_VERSION - value: 20 - - - type: web - name: sawgraph-web - runtime: static - buildCommand: npm ci && npm run build - staticPublishPath: ./dist - envVars: - - key: VITE_API_BASE_URL - sync: false - - key: NODE_VERSION - value: 20 - routes: - - type: rewrite - source: /* - destination: /index.html