From d5db8cf5bcc14fd972be12ac5dbf48b1748a3645 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 13:47:55 +0000 Subject: [PATCH 1/8] fix(dsm): find Major depressive disorder when searching for depression DSM search has no stemming, so 'depression' never matched MDD's text ('depressed', 'depressive') and the diagnosis was missing from the results while records that only mention depression in passing filled the list. Add a small curated alias list: depression, major depression and clinical depression for MDD, chronic depression for persistent depressive disorder. Local catalogue search only, no retrieval change. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- src/lib/dsm.ts | 9 +++++++++ tests/dsm.test.ts | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/src/lib/dsm.ts b/src/lib/dsm.ts index 7b0ae2e26c..302c1af307 100644 --- a/src/lib/dsm.ts +++ b/src/lib/dsm.ts @@ -160,8 +160,17 @@ for (const diagnosis of dsmDiagnoses) { } } +// Plain clinical names a clinician types that never appear in the catalogue text. Search has +// no stemming, so "depression" cannot reach "depressive" and Major depressive disorder was +// missing from a search for "depression" while incidental mentions filled the list. +const DSM_CLINICAL_SEARCH_ALIASES: Readonly> = { + "major-depressive-disorder": ["depression", "major depression", "clinical depression"], + "persistent-depressive-disorder-dysthymia": ["chronic depression"], +}; + function dsmDiagnosisAliases(diagnosis: DsmDiagnosis) { const aliases = new Set(); + for (const alias of DSM_CLINICAL_SEARCH_ALIASES[diagnosis.slug] ?? []) aliases.add(normalizeSearchText(alias)); const parenthetical = PARENTHETICAL_ABBREV_RE.exec(diagnosis.title)?.[1]; if (parenthetical) aliases.add(normalizeSearchText(parenthetical)); const initialism = dsmTitleInitialism(diagnosis.title); diff --git a/tests/dsm.test.ts b/tests/dsm.test.ts index 1dc184d269..cfb313d745 100644 --- a/tests/dsm.test.ts +++ b/tests/dsm.test.ts @@ -80,6 +80,14 @@ describe("DSM clinical catalogue", () => { expect(rankDsmDiagnoses("PDD", 1)[0]?.diagnosis.slug).toBe("persistent-depressive-disorder-dysthymia"); }); + it("ranks Major depressive disorder first for a plain search for depression", () => { + expect(rankDsmDiagnoses("depression", 4, [], true)[0]?.diagnosis.slug).toBe("major-depressive-disorder"); + expect(rankDsmDiagnoses("major depression", 1, [], true)[0]?.diagnosis.slug).toBe("major-depressive-disorder"); + expect(rankDsmDiagnoses("chronic depression", 1, [], true)[0]?.diagnosis.slug).toBe( + "persistent-depressive-disorder-dysthymia", + ); + }); + it("ranks US DSM spellings to the matching catalogue diagnoses", () => { expect(rankDsmDiagnoses("generalized anxiety disorder", 1)[0]?.diagnosis.slug).toBe("generalised-anxiety-disorder"); }); From b249be6df055d7c5a4abdebf7b301715d7639d16 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 14:07:41 +0000 Subject: [PATCH 2/8] perf(db): filter keyword-only retrieval to matching chunks before building rows match_governed_candidate_chunks_v3 kept the text match as a select-list column only, so a keyword-only call (match_document_chunks_text_v3) built labels, summary, images and metadata for every eligible public chunk before discarding the non-matching ones. On the live corpus that took 30-48 s per call. Add one WHERE predicate that drops non-matching chunks first when there is no embedding. Hybrid calls are unchanged, and keyword-only output is unchanged because those calls can only rank text-matched rows. Approved by the owner on 15 Oct 2026 (open as draft, live only when he marks it ready), with a live canary pair around it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- supabase/drift-manifest.json | 8 +- ...prefilter_text_only_governed_retrieval.sql | 429 ++++++++++++++++++ supabase/schema.sql | 10 + 3 files changed, 443 insertions(+), 4 deletions(-) create mode 100644 supabase/migrations/20261015140000_prefilter_text_only_governed_retrieval.sql diff --git a/supabase/drift-manifest.json b/supabase/drift-manifest.json index 90122fd320..41ddac1117 100644 --- a/supabase/drift-manifest.json +++ b/supabase/drift-manifest.json @@ -1,9 +1,9 @@ { - "generated_at": "2026-10-14T16:34:21.177Z", + "generated_at": "2026-10-15T14:07:10.413Z", "generator": "scripts/generate-drift-manifest.ts", "postgres_image": "supabase/postgres:17.6.1.127@sha256:be60aee15997daca475b710b734bc6bfe52cd544dcd7e9fd2ff58210b6747d83", - "schema_sha256": "8b544451a3604bc191db8a76c0176ef1bcfc1c236d84a8f2a5da701c340b5db2", - "replay_seconds": 38, + "schema_sha256": "dad0927eec040c6280b6b1c54ccd0d8927ce2818c8645f7fa97d3eec96286360", + "replay_seconds": 23, "snapshot": { "views": [ { @@ -19377,7 +19377,7 @@ "acl": [ "postgres=X/postgres" ], - "def_hash": "92bf14fee49b501305bd81ac583a43cd", + "def_hash": "090e9afd33a764ce6ec81c632abd83b4", "signature": "public.match_governed_candidate_chunks_v3(extensions.vector,text,integer,double precision,uuid[],uuid,boolean,text[],uuid,text,bigint,text[])" }, { diff --git a/supabase/migrations/20261015140000_prefilter_text_only_governed_retrieval.sql b/supabase/migrations/20261015140000_prefilter_text_only_governed_retrieval.sql new file mode 100644 index 0000000000..f7dfb1fa0d --- /dev/null +++ b/supabase/migrations/20261015140000_prefilter_text_only_governed_retrieval.sql @@ -0,0 +1,429 @@ +-- Keyword-only retrieval took 30-48 s on the live corpus because the text match was only a +-- select-list column, so every eligible public chunk was fully built before filtering. +-- Same function body as 20260901130000 plus one WHERE predicate; output is unchanged because +-- a call without an embedding can only rank text-matched rows (vector_ranked is empty). + +begin; + +create or replace function public.match_governed_candidate_chunks_v3( + query_embedding extensions.vector(1536), + query_text text, + match_count integer, + min_similarity double precision, + document_filters uuid[], + owner_filter uuid, + include_public boolean, + corpus_scopes text[], + expected_site_release_id uuid, + expected_site_release_digest text, + expected_site_change_epoch bigint, + site_content_domains text[] +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + document_labels jsonb, + document_summary text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + rrf_score double precision, + lexical_score double precision, + images jsonb, + corpus_scope text, + site_content_domain text, + site_release_id uuid, + site_change_epoch bigint, + pending_exclusion_exact boolean +) +language sql +stable +security definer +set search_path = '' +set work_mem = '64MB' +as $$ + with + admitted as ( + select + least(greatest(coalesce(match_count, 12), 1), 96) as bounded_count, + case + when query_text is null or pg_catalog.btrim(query_text) = '' then null + else pg_catalog.websearch_to_tsquery('english', query_text) + end as tsq + where include_public is true + and owner_filter = '00000000-0000-0000-0000-000000000000'::uuid + and corpus_scopes is not null + and pg_catalog.cardinality(corpus_scopes) between 1 and 4 + and corpus_scopes <@ array[ + 'uploaded_local', + 'clinical_kb_site', + 'australian_public', + 'international_supplementary' + ]::text[] + ), + site_authority as ( + select + state.active_release_id as release_id, + state.change_epoch, + state.served_change_epoch, + release.release_digest + from public.site_content_sync_state state + join public.site_content_releases release + on release.id = state.active_release_id + and release.state = 'active' + and release.target_change_epoch = state.served_change_epoch + and release.release_digest = state.active_release_digest + cross join admitted + where expected_site_release_id is not null + and expected_site_release_digest ~ '^[0-9a-f]{64}$' + and expected_site_change_epoch is not null + and state.initialized + and state.active_release_id = expected_site_release_id + and state.active_release_digest = expected_site_release_digest + and state.change_epoch = expected_site_change_epoch + and state.served_change_epoch <= state.change_epoch + ), + pending_site_set as ( + select not exists ( + select 1 + from public.site_content_public_records record + left join public.site_content_sync_events event + on event.event_sequence = record.pending_event_sequence + where record.pending_event_sequence is not null + and ( + event.event_sequence is null + or event.logical_id is distinct from record.logical_id + or event.target_publication_id is distinct from record.current_publication_id + or event.target_change_epoch is distinct from record.head_change_epoch + or event.target_change_epoch > site_authority.change_epoch + ) + ) and not exists ( + select 1 + from public.site_content_public_records record + where record.head_change_epoch > site_authority.served_change_epoch + and ( + record.pending_event_sequence is null + or record.head_change_epoch > site_authority.change_epoch + ) + ) as exact + from site_authority + ), + pending_site_logical_ids as ( + select record.logical_id + from public.site_content_public_records record + cross join site_authority + where record.pending_event_sequence is not null + ), + document_candidates as ( + select + chunk.id, + document.id as document_id, + document.title, + document.file_name, + chunk.page_number, + chunk.chunk_index, + chunk.section_heading, + chunk.content, + chunk.retrieval_synopsis, + chunk.image_ids, + pg_catalog.jsonb_strip_nulls(pg_catalog.jsonb_build_object( + 'source_kind', 'document', + 'source_title', document.metadata->>'source_title', + 'publisher', document.metadata->>'publisher', + 'publisher_code', document.metadata->>'publisher_code', + 'jurisdiction', document.metadata->>'jurisdiction', + 'version', document.metadata->>'version', + 'publication_date', document.metadata->>'publication_date', + 'review_date', document.metadata->>'review_date', + 'uploaded_at', document.metadata->>'uploaded_at', + 'indexed_at', document.metadata->>'indexed_at', + 'corpus_scope', document.metadata->>'corpus_scope', + 'source_role', document.metadata->>'source_role', + 'content_mode', document.metadata->>'content_mode', + 'source_catalogue_key', document.metadata->>'source_catalogue_key', + 'source_policy_version', document.metadata->>'source_policy_version', + 'canonical_url', document.metadata->>'canonical_url', + 'effective_date', document.metadata->>'effective_date', + 'expiry_date', document.metadata->>'expiry_date', + 'supersedes_document_id', document.metadata->>'supersedes_document_id', + 'superseded_by_document_id', document.metadata->>'superseded_by_document_id', + 'retrieved_at', document.metadata->>'retrieved_at', + 'content_hash', document.metadata->>'content_hash', + 'change_state', document.metadata->>'change_state', + 'licence_policy', document.metadata->>'licence_policy', + 'document_status', document.metadata->>'document_status', + 'clinical_validation_status', document.metadata->>'clinical_validation_status', + 'extraction_quality', document.metadata->>'extraction_quality' + )) as source_metadata, + coalesce( + ( + select pg_catalog.jsonb_agg( + pg_catalog.jsonb_build_object('id', label.id, 'label', label.label, 'label_type', label.label_type) + order by label.label_type, label.label, label.id + ) + from public.document_labels label + where label.document_id = document.id + ), + '[]'::jsonb + ) as document_labels, + ( + select summary.summary + from public.document_summaries summary + where summary.document_id = document.id + order by summary.created_at desc, summary.id + limit 1 + ) as document_summary, + case + when query_embedding is null then 0::double precision + else (1 - (chunk.embedding OPERATOR(extensions.<=>) query_embedding))::double precision + end as similarity, + case + when admitted.tsq is null then 0::double precision + else pg_catalog.ts_rank_cd(chunk.search_tsv, admitted.tsq)::double precision + end as text_rank, + coalesce( + ( + select pg_catalog.jsonb_agg( + pg_catalog.jsonb_build_object( + 'id', image.id, + 'page_number', image.page_number, + 'storage_path', image.storage_path, + 'caption', image.caption + ) + order by image.id + ) + from public.document_images image + where image.id = any(chunk.image_ids) + ), + '[]'::jsonb + ) as images, + document.metadata->>'corpus_scope' as corpus_scope, + ( + admitted.tsq is not null + and (chunk.search_tsv @@ admitted.tsq or document.title_search_tsv @@ admitted.tsq) + ) as text_matched + from public.document_chunks chunk + join public.documents document on document.id = chunk.document_id + cross join admitted + where document.owner_id is null + -- Text-only calls (match_document_chunks_text_v3) can only return text-matched rows, so + -- filter here, before the per-row labels, summary, images and jsonb work, instead of + -- building every eligible public chunk first. Hybrid calls with an embedding are unchanged. + and ( + query_embedding is not null + or ( + admitted.tsq is not null + and (chunk.search_tsv @@ admitted.tsq or document.title_search_tsv @@ admitted.tsq) + ) + ) + and document.status = 'indexed' + and public.is_committed_document_generation(chunk.index_generation_id, document.index_generation_id) + and (document_filters is null or document.id = any(document_filters)) + and document.metadata->>'corpus_scope' = any(corpus_scopes) + and document.metadata->>'corpus_scope' = 'australian_public' + and document.metadata->>'source_kind' = 'document' + and document.metadata->'public_corpus' = 'true'::jsonb + and document.metadata->>'content_mode' = 'indexed_content' + and document.metadata->>'document_status' = 'current' + and coalesce(document.metadata->>'change_state', '') in ('changed', 'unchanged') + and document.metadata->>'licence_policy' = 'public_index_permitted' + and nullif(document.metadata->>'source_catalogue_key', '') is not null + and nullif(document.metadata->>'source_policy_version', '') is not null + and nullif(document.metadata->>'publication_approval_id', '') is not null + and nullif(document.metadata->>'publication_manifest_digest', '') is not null + ), + site_candidates as ( + select + record.logical_chunk_id as id, + record.logical_document_id as document_id, + record.record->>'title' as title, + record.record->>'route' as file_name, + null::integer as page_number, + 0::integer as chunk_index, + record.record->>'domain' as section_heading, + record.normalized_text as content, + null::text as retrieval_synopsis, + '{}'::uuid[] as image_ids, + pg_catalog.jsonb_build_object( + 'corpus_scope', 'clinical_kb_site', + 'source_kind', 'registry_record', + 'source_role', record.record->>'sourceRole', + 'content_mode', 'indexed_content', + 'source_title', record.record->>'title', + 'version', record.record->>'publicationVersion', + 'document_status', record.record->>'sourceStatus', + 'clinical_validation_status', record.record->>'validationStatus', + 'extraction_quality', 'good', + 'content_hash', record.record->>'contentHash', + 'site_content_lineage', record.record->'sourceLineage' + ) as source_metadata, + '[]'::jsonb as document_labels, + null::text as document_summary, + case + when query_embedding is null or record.embedding is null then 0::double precision + else (1 - (record.embedding OPERATOR(extensions.<=>) query_embedding))::double precision + end as similarity, + case + when admitted.tsq is null then 0::double precision + else pg_catalog.ts_rank_cd( + pg_catalog.to_tsvector( + 'english', + coalesce(record.record->>'title', '') || ' ' || record.normalized_text + ), + admitted.tsq + )::double precision + end as text_rank, + '[]'::jsonb as images, + 'clinical_kb_site'::text as corpus_scope, + ( + admitted.tsq is not null + and pg_catalog.to_tsvector( + 'english', + coalesce(record.record->>'title', '') || ' ' || record.normalized_text + ) @@ admitted.tsq + ) as text_matched, + record.record->>'domain' as site_content_domain, + site_authority.release_id as site_release_id, + site_authority.change_epoch as site_change_epoch + from public.site_content_release_records record + cross join admitted + cross join site_authority + cross join pending_site_set + where 'clinical_kb_site' = any(corpus_scopes) + and record.release_id = site_authority.release_id + and pending_site_set.exact + and (document_filters is null or record.logical_document_id = any(document_filters)) + and record.public_visible + and not record.tombstone + and pg_catalog.jsonb_typeof(record.record) = 'object' + and record.record->>'version' = 'site-content-record-v1' + and record.record->>'access' = 'public' + and record.record->>'logicalId' = record.logical_id + and record.record->>'validationStatus' in ('locally_reviewed', 'approved') + and record.record->>'sourceStatus' in ('current', 'review_due') + and record.record->>'domain' in ( + 'services', 'forms', 'medications', 'differentials', 'specifiers', 'dsm', + 'formulation', 'therapies', 'dictionary', 'factsheets', 'calculators', 'tools' + ) + and ( + site_content_domains is null + or ( + pg_catalog.cardinality(site_content_domains) between 1 and 12 + and site_content_domains <@ array[ + 'services', 'forms', 'medications', 'differentials', 'specifiers', 'dsm', + 'formulation', 'therapies', 'dictionary', 'factsheets', 'calculators', 'tools' + ]::text[] + and record.record->>'domain' = any(site_content_domains) + ) + ) + and not exists ( + select 1 + from pending_site_logical_ids pending + where pending.logical_id = record.logical_id + ) + ), + combined as ( + select + document.*, + null::text as site_content_domain, + null::uuid as site_release_id, + null::bigint as site_change_epoch + from document_candidates document + union all + select * from site_candidates + ), + vector_ranked as ( + select + combined.id, + combined.corpus_scope, + row_number() over (order by combined.similarity desc, combined.id) as vector_rank, + null::bigint as text_match_rank + from combined + where query_embedding is not null + and combined.similarity >= greatest(coalesce(min_similarity, 0.12), 0) + order by combined.similarity desc, combined.id + limit (select greatest(bounded_count * 6, 48) from admitted) + ), + text_ranked as ( + select + combined.id, + combined.corpus_scope, + null::bigint as vector_rank, + row_number() over (order by combined.text_rank desc, combined.id) as text_match_rank + from combined + where combined.text_matched + order by combined.text_rank desc, combined.id + limit (select greatest(bounded_count * 6, 48) from admitted) + ), + rank_positions as ( + select + positions.id, + positions.corpus_scope, + min(positions.vector_rank) as vector_rank, + min(positions.text_match_rank) as text_match_rank + from ( + select * from vector_ranked + union all + select * from text_ranked + ) positions + group by positions.id, positions.corpus_scope + ), + ranked as ( + select + combined.*, + ((combined.similarity * 0.65) + (least(combined.text_rank, 1) * 0.35))::double precision as hybrid_score, + ( + coalesce(1.0 / (60 + rank_positions.vector_rank), 0) + + coalesce(1.0 / (60 + rank_positions.text_match_rank), 0) + )::double precision as rrf_score + from combined + join rank_positions using (id, corpus_scope) + ) + select + ranked.id, + ranked.document_id, + ranked.title, + ranked.file_name, + ranked.page_number, + ranked.chunk_index, + ranked.section_heading, + ranked.content, + ranked.retrieval_synopsis, + ranked.image_ids, + ranked.source_metadata, + ranked.document_labels, + ranked.document_summary, + ranked.similarity, + ranked.text_rank, + ranked.hybrid_score, + ranked.rrf_score, + least(ranked.text_rank, 1)::double precision as lexical_score, + ranked.images, + ranked.corpus_scope, + ranked.site_content_domain, + ranked.site_release_id, + ranked.site_change_epoch, + true as pending_exclusion_exact + from ranked + order by ranked.hybrid_score desc, ranked.rrf_score desc, ranked.id + limit (select bounded_count from admitted); +$$; + +revoke all on function public.match_governed_candidate_chunks_v3( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean, text[], uuid, text, bigint, text[] +) from public, anon, authenticated, service_role; + +alter function public.match_governed_candidate_chunks_v3( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean, text[], uuid, text, bigint, text[] +) owner to postgres; + +commit; diff --git a/supabase/schema.sql b/supabase/schema.sql index 41c7cdd4cf..28cc12b1d8 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -17755,6 +17755,16 @@ as $$ join public.documents document on document.id = chunk.document_id cross join admitted where document.owner_id is null + -- Text-only calls (match_document_chunks_text_v3) can only return text-matched rows, so + -- filter here, before the per-row labels, summary, images and jsonb work, instead of + -- building every eligible public chunk first. Hybrid calls with an embedding are unchanged. + and ( + query_embedding is not null + or ( + admitted.tsq is not null + and (chunk.search_tsv @@ admitted.tsq or document.title_search_tsv @@ admitted.tsq) + ) + ) and document.status = 'indexed' and public.is_committed_document_generation(chunk.index_generation_id, document.index_generation_id) and (document_filters is null or document.id = any(document_filters)) From 7383d79a6344f450fbc8e277177bd263e80691d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 16:17:29 +0000 Subject: [PATCH 3/8] fix(search): bound search database calls at 90 seconds instead of no limit PR #3554 removed the 30 s deadline so slow keyword searches could finish. A finite 90 s limit keeps that while bounding a stalled call, including the query-correction RPC that does not carry the caller's abort signal. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- src/lib/rag/rag.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 1a3f85458b..a2d579e778 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -355,6 +355,9 @@ function asReviewOnlyCitations(citations: readonly Citation[]): Citation[] { return citations.map((citation) => ({ ...citation, provenance: "review_only" as const })); } +// Search deadline for the admin client. See searchChunksWithTiming for why it exceeds the shared 30 s default. +const SEARCH_SUPABASE_TIMEOUT_MS = 90_000; + const confidenceOrder = { unsupported: 0, low: 1, @@ -1148,11 +1151,12 @@ async function searchChunksWithTiming( recordSearchScoreTelemetry(telemetry, []); return finishSearch(searchTiming, { results: [] as SearchResult[], telemetry }); } - // No client-side deadline here. PR #3454 (11 Oct 2026) gave the shared admin client a - // 30 s default, and the keyword RPC can take 30-48 s on the live corpus, so it was - // aborted and returned nothing for 9 of the 36 golden cases (eval canaries from 11 Oct). - // Search keeps the caller's own abort signal, as it did before that change. - const supabase = createAdminClient({ timeoutMs: null }); + // Longer deadline than the shared 30 s default. PR #3454 (11 Oct 2026) gave the shared + // admin client that default, and the keyword RPC can take 30-48 s on the live corpus, so it + // was aborted and returned nothing for 9 of the 36 golden cases (eval canaries from 11 Oct). + // 90 s clears the slowest observed keyword call while still bounding a stalled request, + // including the query-correction RPC that does not carry the caller's abort signal. + const supabase = createAdminClient({ timeoutMs: SEARCH_SUPABASE_TIMEOUT_MS }); // Source-only retrieval never calls embeddings. const sourceOnlyRetrieval = isSourceOnlyMode(); if (args.forceEmbedding && sourceOnlyRetrieval) { From a77cdb077036bb61030c73635f0bdc52284ad9de Mon Sep 17 00:00:00 2001 From: "qodo-code-review[bot]" <151058649+qodo-code-review[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:37:54 +0000 Subject: [PATCH 4/8] fix: Probe text indexes separately --- ...prefilter_text_only_governed_retrieval.sql | 30 +++++++++---- supabase/schema.sql | 44 ++++++++++++++++--- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/supabase/migrations/20261015140000_prefilter_text_only_governed_retrieval.sql b/supabase/migrations/20261015140000_prefilter_text_only_governed_retrieval.sql index f7dfb1fa0d..fbee675211 100644 --- a/supabase/migrations/20261015140000_prefilter_text_only_governed_retrieval.sql +++ b/supabase/migrations/20261015140000_prefilter_text_only_governed_retrieval.sql @@ -1,7 +1,7 @@ -- Keyword-only retrieval took 30-48 s on the live corpus because the text match was only a -- select-list column, so every eligible public chunk was fully built before filtering. --- Same function body as 20260901130000 plus one WHERE predicate; output is unchanged because --- a call without an embedding can only rank text-matched rows (vector_ranked is empty). +-- Same function body as 20260901130000 with indexed text candidate probes; output is unchanged +-- because a call without an embedding can only rank text-matched rows (vector_ranked is empty). begin; @@ -123,6 +123,23 @@ as $$ cross join site_authority where record.pending_event_sequence is not null ), + text_match_ids as ( + -- Keep the two GIN probes separate so either index can select candidates. + select chunk.id + from public.document_chunks chunk + cross join admitted + where query_embedding is null + and admitted.tsq is not null + and chunk.search_tsv @@ admitted.tsq + union + select chunk.id + from public.documents document + join public.document_chunks chunk on chunk.document_id = document.id + cross join admitted + where query_embedding is null + and admitted.tsq is not null + and document.title_search_tsv @@ admitted.tsq + ), document_candidates as ( select chunk.id, @@ -214,17 +231,12 @@ as $$ from public.document_chunks chunk join public.documents document on document.id = chunk.document_id cross join admitted + left join text_match_ids matched on matched.id = chunk.id where document.owner_id is null -- Text-only calls (match_document_chunks_text_v3) can only return text-matched rows, so -- filter here, before the per-row labels, summary, images and jsonb work, instead of -- building every eligible public chunk first. Hybrid calls with an embedding are unchanged. - and ( - query_embedding is not null - or ( - admitted.tsq is not null - and (chunk.search_tsv @@ admitted.tsq or document.title_search_tsv @@ admitted.tsq) - ) - ) + and (query_embedding is not null or matched.id is not null) and document.status = 'indexed' and public.is_committed_document_generation(chunk.index_generation_id, document.index_generation_id) and (document_filters is null or document.id = any(document_filters)) diff --git a/supabase/schema.sql b/supabase/schema.sql index 28cc12b1d8..2355eb38d6 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -15947,6 +15947,23 @@ as $$ cross join site_authority where record.pending_event_sequence is not null ), + text_match_ids as ( + -- Keep the two GIN probes separate so either index can select candidates. + select chunk.id + from public.document_chunks chunk + cross join admitted + where query_embedding is null + and admitted.tsq is not null + and chunk.search_tsv @@ admitted.tsq + union + select chunk.id + from public.documents document + join public.document_chunks chunk on chunk.document_id = document.id + cross join admitted + where query_embedding is null + and admitted.tsq is not null + and document.title_search_tsv @@ admitted.tsq + ), document_candidates as ( select chunk.id, @@ -16038,6 +16055,7 @@ as $$ from public.document_chunks chunk join public.documents document on document.id = chunk.document_id cross join admitted + left join text_match_ids matched on matched.id = chunk.id where document.owner_id is null and document.status = 'indexed' and public.is_committed_document_generation(chunk.index_generation_id, document.index_generation_id) @@ -17663,6 +17681,23 @@ as $$ cross join site_authority where record.pending_event_sequence is not null ), + text_match_ids as ( + -- Keep the two GIN probes separate so either index can select candidates. + select chunk.id + from public.document_chunks chunk + cross join admitted + where query_embedding is null + and admitted.tsq is not null + and chunk.search_tsv @@ admitted.tsq + union + select chunk.id + from public.documents document + join public.document_chunks chunk on chunk.document_id = document.id + cross join admitted + where query_embedding is null + and admitted.tsq is not null + and document.title_search_tsv @@ admitted.tsq + ), document_candidates as ( select chunk.id, @@ -17754,17 +17789,12 @@ as $$ from public.document_chunks chunk join public.documents document on document.id = chunk.document_id cross join admitted + left join text_match_ids matched on matched.id = chunk.id where document.owner_id is null -- Text-only calls (match_document_chunks_text_v3) can only return text-matched rows, so -- filter here, before the per-row labels, summary, images and jsonb work, instead of -- building every eligible public chunk first. Hybrid calls with an embedding are unchanged. - and ( - query_embedding is not null - or ( - admitted.tsq is not null - and (chunk.search_tsv @@ admitted.tsq or document.title_search_tsv @@ admitted.tsq) - ) - ) + and (query_embedding is not null or matched.id is not null) and document.status = 'indexed' and public.is_committed_document_generation(chunk.index_generation_id, document.index_generation_id) and (document_filters is null or document.id = any(document_filters)) From 98f3b652efeb8d3b9f82b663b9e8c49dd321e646 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 17:48:48 +0000 Subject: [PATCH 5/8] Pass the search abort signal to the query-correction call A cancelled search no longer leaves correct_clinical_query_terms running until the 90 s client deadline (CodeAnt nitpick on #3560). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- src/lib/rag/rag.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index a2d579e778..eeedf8c266 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -1155,7 +1155,7 @@ async function searchChunksWithTiming( // admin client that default, and the keyword RPC can take 30-48 s on the live corpus, so it // was aborted and returned nothing for 9 of the 36 golden cases (eval canaries from 11 Oct). // 90 s clears the slowest observed keyword call while still bounding a stalled request, - // including the query-correction RPC that does not carry the caller's abort signal. + // including the query-correction RPC. const supabase = createAdminClient({ timeoutMs: SEARCH_SUPABASE_TIMEOUT_MS }); // Source-only retrieval never calls embeddings. const sourceOnlyRetrieval = isSourceOnlyMode(); @@ -1341,10 +1341,14 @@ async function searchChunksWithTiming( ) { // RC6: correct a would-be unsupported typo once before retrieval short-circuits. if (!args.typoCorrected && !sourceOnlyRetrieval) { - const { data: corrected } = await supabase.rpc("correct_clinical_query_terms", { + // Carry the caller's abort signal so a cancelled search does not leave this call running + // until the 90 s client deadline. + let correctionQuery = supabase.rpc("correct_clinical_query_terms", { input_query: retrievalQuery, min_sim: 0.45, }); + if (args.signal) correctionQuery = correctionQuery.abortSignal(args.signal); + const { data: corrected } = await correctionQuery; if (typeof corrected === "string" && corrected && corrected.toLowerCase() !== retrievalQuery.toLowerCase()) { const correctedSearch = await searchChunksWithTelemetry({ ...args, From 7f86952ec821295fca5bdf772b6d6e666283f0ed Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 17:52:40 +0000 Subject: [PATCH 6/8] Stop a cancelled search before it caches an empty result The aborted query-correction call resolves with an error instead of throwing, so check the signal straight after it (Gitar finding on #3560). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- src/lib/rag/rag.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index eeedf8c266..fa8d85884f 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -1349,6 +1349,9 @@ async function searchChunksWithTiming( }); if (args.signal) correctionQuery = correctionQuery.abortSignal(args.signal); const { data: corrected } = await correctionQuery; + // A cancelled call resolves with an error rather than throwing; stop before the empty + // short-circuit result below is written to the search cache. + throwIfAborted(args.signal); if (typeof corrected === "string" && corrected && corrected.toLowerCase() !== retrievalQuery.toLowerCase()) { const correctedSearch = await searchChunksWithTelemetry({ ...args, From 6c326b28758ee0c13b8eb241df6f65ab2b7fd0ea Mon Sep 17 00:00:00 2001 From: "qodo-code-review[bot]" <151058649+qodo-code-review[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:18:18 +0000 Subject: [PATCH 7/8] fix: Handle correction RPC errors before caching --- src/lib/rag/rag.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index fa8d85884f..96885e4833 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -1348,9 +1348,10 @@ async function searchChunksWithTiming( min_sim: 0.45, }); if (args.signal) correctionQuery = correctionQuery.abortSignal(args.signal); - const { data: corrected } = await correctionQuery; - // A cancelled call resolves with an error rather than throwing; stop before the empty - // short-circuit result below is written to the search cache. + const { data: corrected, error: correctionError } = await correctionQuery; + // A cancelled or failed call resolves with an error rather than throwing; do not treat it + // as an uncorrected query or write the empty short-circuit result to the search cache. + if (correctionError) throw correctionError; throwIfAborted(args.signal); if (typeof corrected === "string" && corrected && corrected.toLowerCase() !== retrievalQuery.toLowerCase()) { const correctedSearch = await searchChunksWithTelemetry({ From 9b8de62ab6fa8e0ce7c3e4c2f5cf8495b545739c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:21:22 +0000 Subject: [PATCH 8/8] Do not cache an empty search when the query-correction call fails A timed-out or failed correction resolves with an error and null data, which is not evidence the query has no match (Qodo finding on #3560). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- src/lib/rag/rag.ts | 7 ++- ...ag-unsupported-short-circuit-cache.test.ts | 51 +++++++++++++++++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index fa8d85884f..701a867e17 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -1340,6 +1340,7 @@ async function searchChunksWithTiming( shouldApplyUnsupportedSearchShortCircuit(retrievalQuery, queryAnalysis, ragAliasExpansions) ) { // RC6: correct a would-be unsupported typo once before retrieval short-circuits. + let correctionFailed = false; if (!args.typoCorrected && !sourceOnlyRetrieval) { // Carry the caller's abort signal so a cancelled search does not leave this call running // until the 90 s client deadline. @@ -1348,10 +1349,13 @@ async function searchChunksWithTiming( min_sim: 0.45, }); if (args.signal) correctionQuery = correctionQuery.abortSignal(args.signal); - const { data: corrected } = await correctionQuery; + const { data: corrected, error: correctionError } = await correctionQuery; // A cancelled call resolves with an error rather than throwing; stop before the empty // short-circuit result below is written to the search cache. throwIfAborted(args.signal); + // A failed or timed-out correction is not proof the query has no match, so the empty + // result below must not be cached. + if (correctionError) correctionFailed = true; if (typeof corrected === "string" && corrected && corrected.toLowerCase() !== retrievalQuery.toLowerCase()) { const correctedSearch = await searchChunksWithTelemetry({ ...args, @@ -1371,6 +1375,7 @@ async function searchChunksWithTiming( recordSearchScoreTelemetry(telemetry, []); // Skip only when a reachable classifier could have produced a nondeterministic soft-tail zero. if ( + !correctionFailed && !shouldSkipUnsupportedSoftTailCacheWrite(retrievalQuery, queryAnalysis, { openAiApiKeyPresent: Boolean(env.OPENAI_API_KEY), }) diff --git a/tests/rag-unsupported-short-circuit-cache.test.ts b/tests/rag-unsupported-short-circuit-cache.test.ts index 01e7d10405..28bb2a2151 100644 --- a/tests/rag-unsupported-short-circuit-cache.test.ts +++ b/tests/rag-unsupported-short-circuit-cache.test.ts @@ -45,7 +45,7 @@ class EmptyQuery implements PromiseLike<{ data: unknown[]; error: null }> { async function loadSearch( corpusGroundingVerdict: "inconclusive" | "out_of_corpus" | "in_corpus_topic", - options: { openAiApiKey?: string } = {}, + options: { openAiApiKey?: string; sourceOnly?: boolean; correctionError?: boolean } = {}, ) { const setCachedSearch = vi.fn(async () => undefined); @@ -84,7 +84,7 @@ async function loadSearch( }; }); vi.doMock("@/lib/rag/rag-provider", () => ({ - isSourceOnlyMode: () => true, + isSourceOnlyMode: () => options.sourceOnly ?? true, allowsAutoDegrade: () => true, sourceOnlyReason: () => "source_only", classifyProviderFailure: () => "provider_failure", @@ -92,7 +92,19 @@ async function loadSearch( })); vi.doMock("@/lib/supabase/admin", () => ({ createAdminClient: () => ({ - rpc: vi.fn(async () => ({ data: [], error: null })), + rpc: vi.fn((name: string) => { + // The query-correction call is awaited after an optional abortSignal(), so it returns a + // thenable builder. A timeout surfaces as an error with null data. + const response = + name === "correct_clinical_query_terms" + ? options.correctionError + ? { data: null, error: { message: "AbortError: signal timed out" } } + : { data: null, error: null } + : { data: [], error: null }; + const builder = Promise.resolve(response) as Promise & { abortSignal: () => unknown }; + builder.abortSignal = () => builder; + return builder; + }), from: vi.fn(() => new EmptyQuery()), }), })); @@ -178,6 +190,39 @@ describe("unsupported short-circuit cache write", () => { expect(setCachedSearch).toHaveBeenCalledTimes(1); }, 60_000); + it("does not cache the zero when the query-correction call failed or timed out", async () => { + // Qodo review on PR #3560: a timed-out correction resolves with an error and null data, which + // is not evidence that the query has no match, so the empty result must not become sticky. + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive", { + sourceOnly: false, + correctionError: true, + }); + + const result = await searchChunksWithTelemetry({ + query: "Show me the airport travel policy", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).not.toHaveBeenCalled(); + }, 60_000); + + it("still caches the deterministic zero when the query-correction call succeeded", async () => { + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive", { sourceOnly: false }); + + const result = await searchChunksWithTelemetry({ + query: "Show me the airport travel policy", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).toHaveBeenCalledTimes(1); + }, 60_000); + it("rescues an in-corpus bare topic before ever reaching the short circuit", async () => { const { searchChunksWithTelemetry } = await loadSearch("in_corpus_topic");