diff --git a/src/lib/dsm.ts b/src/lib/dsm.ts index 7b0ae2e26c..302c1af307 100644 --- a/src/lib/dsm.ts +++ b/src/lib/dsm.ts @@ -160,8 +160,17 @@ for (const diagnosis of dsmDiagnoses) { } } +// Plain clinical names a clinician types that never appear in the catalogue text. Search has +// no stemming, so "depression" cannot reach "depressive" and Major depressive disorder was +// missing from a search for "depression" while incidental mentions filled the list. +const DSM_CLINICAL_SEARCH_ALIASES: Readonly> = { + "major-depressive-disorder": ["depression", "major depression", "clinical depression"], + "persistent-depressive-disorder-dysthymia": ["chronic depression"], +}; + function dsmDiagnosisAliases(diagnosis: DsmDiagnosis) { const aliases = new Set(); + for (const alias of DSM_CLINICAL_SEARCH_ALIASES[diagnosis.slug] ?? []) aliases.add(normalizeSearchText(alias)); const parenthetical = PARENTHETICAL_ABBREV_RE.exec(diagnosis.title)?.[1]; if (parenthetical) aliases.add(normalizeSearchText(parenthetical)); const initialism = dsmTitleInitialism(diagnosis.title); diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 1a3f85458b..701a867e17 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -355,6 +355,9 @@ function asReviewOnlyCitations(citations: readonly Citation[]): Citation[] { return citations.map((citation) => ({ ...citation, provenance: "review_only" as const })); } +// Search deadline for the admin client. See searchChunksWithTiming for why it exceeds the shared 30 s default. +const SEARCH_SUPABASE_TIMEOUT_MS = 90_000; + const confidenceOrder = { unsupported: 0, low: 1, @@ -1148,11 +1151,12 @@ async function searchChunksWithTiming( recordSearchScoreTelemetry(telemetry, []); return finishSearch(searchTiming, { results: [] as SearchResult[], telemetry }); } - // No client-side deadline here. PR #3454 (11 Oct 2026) gave the shared admin client a - // 30 s default, and the keyword RPC can take 30-48 s on the live corpus, so it was - // aborted and returned nothing for 9 of the 36 golden cases (eval canaries from 11 Oct). - // Search keeps the caller's own abort signal, as it did before that change. - const supabase = createAdminClient({ timeoutMs: null }); + // Longer deadline than the shared 30 s default. PR #3454 (11 Oct 2026) gave the shared + // admin client that default, and the keyword RPC can take 30-48 s on the live corpus, so it + // was aborted and returned nothing for 9 of the 36 golden cases (eval canaries from 11 Oct). + // 90 s clears the slowest observed keyword call while still bounding a stalled request, + // including the query-correction RPC. + const supabase = createAdminClient({ timeoutMs: SEARCH_SUPABASE_TIMEOUT_MS }); // Source-only retrieval never calls embeddings. const sourceOnlyRetrieval = isSourceOnlyMode(); if (args.forceEmbedding && sourceOnlyRetrieval) { @@ -1336,11 +1340,22 @@ async function searchChunksWithTiming( shouldApplyUnsupportedSearchShortCircuit(retrievalQuery, queryAnalysis, ragAliasExpansions) ) { // RC6: correct a would-be unsupported typo once before retrieval short-circuits. + let correctionFailed = false; if (!args.typoCorrected && !sourceOnlyRetrieval) { - const { data: corrected } = await supabase.rpc("correct_clinical_query_terms", { + // Carry the caller's abort signal so a cancelled search does not leave this call running + // until the 90 s client deadline. + let correctionQuery = supabase.rpc("correct_clinical_query_terms", { input_query: retrievalQuery, min_sim: 0.45, }); + if (args.signal) correctionQuery = correctionQuery.abortSignal(args.signal); + const { data: corrected, error: correctionError } = await correctionQuery; + // A cancelled call resolves with an error rather than throwing; stop before the empty + // short-circuit result below is written to the search cache. + throwIfAborted(args.signal); + // A failed or timed-out correction is not proof the query has no match, so the empty + // result below must not be cached. + if (correctionError) correctionFailed = true; if (typeof corrected === "string" && corrected && corrected.toLowerCase() !== retrievalQuery.toLowerCase()) { const correctedSearch = await searchChunksWithTelemetry({ ...args, @@ -1360,6 +1375,7 @@ async function searchChunksWithTiming( recordSearchScoreTelemetry(telemetry, []); // Skip only when a reachable classifier could have produced a nondeterministic soft-tail zero. if ( + !correctionFailed && !shouldSkipUnsupportedSoftTailCacheWrite(retrievalQuery, queryAnalysis, { openAiApiKeyPresent: Boolean(env.OPENAI_API_KEY), }) diff --git a/supabase/applied-migration-hashes.json b/supabase/applied-migration-hashes.json index 0a76a4211b..c371d7fbfb 100644 --- a/supabase/applied-migration-hashes.json +++ b/supabase/applied-migration-hashes.json @@ -1,7 +1,7 @@ { "note": "sha256 of every migration this repository has shipped, with newlines normalised. An entry whose hash changes means an applied migration was edited, which cannot reach the live database. See scripts/check-migration-immutability.mjs.", "generator": "scripts/check-migration-immutability.mjs", - "sealed_at": "2026-10-15T15:57:11.538Z", + "sealed_at": "2026-10-15T17:36:28.021Z", "versions": { "20260527000000_bulk_ingestion.sql": "813e8e314064a6dcbaaa55e2765a8ce901d5783b304b4a7af7bb7417a07daf7c", "20260527001000_documents_policy_cleanup.sql": "559c26f3172bf75e545a41adef75538b2bf648ea7d53c8eb3d3ea7c244c65a48", @@ -313,6 +313,7 @@ "20261013200007_managed_native_pdf_terminal_append.sql": "94ac297bad4a1f9d28fcb13d018237df081aa5b3da328073be8c9e17774bd292", "20261014164220_validate_managed_ingestion_historical_reconciliation.sql": "8ffd75292620e82065300eb1e1d008c229545dde37826862820bfd6ea1314587", "20261015160000_document_child_policies_check_parent_owner.sql": "71210bbc5aecdca74bee2ae32181ba2861e65017ca1a5e87995aa8e557946094", - "20261015174500_roster_open_post_once.sql": "a4edfe07497e3b833c4bedecf1b41d0d7b9fb0b73a798a71b3c17bccb9f851ff" + "20261015174500_roster_open_post_once.sql": "a4edfe07497e3b833c4bedecf1b41d0d7b9fb0b73a798a71b3c17bccb9f851ff", + "20261015180000_prefilter_text_only_governed_retrieval.sql": "8ff1f445269ad6413d5f00b304fc169dd2521f0d01e215415bbc4f573d2a926a" } } diff --git a/supabase/drift-manifest.json b/supabase/drift-manifest.json index 2ef9197500..c291d98311 100644 --- a/supabase/drift-manifest.json +++ b/supabase/drift-manifest.json @@ -1,9 +1,9 @@ { - "generated_at": "2026-10-15T15:56:45.785Z", + "generated_at": "2026-10-15T23:31:52.942Z", "generator": "scripts/generate-drift-manifest.ts", "postgres_image": "supabase/postgres:17.6.1.127@sha256:be60aee15997daca475b710b734bc6bfe52cd544dcd7e9fd2ff58210b6747d83", - "schema_sha256": "e561ea277c34ddec08d6b676ebcb8603c0e98b04e0ca311c9fbf4a4f49246895", - "replay_seconds": 38, + "schema_sha256": "067f617ef440b12532d63d7a4e89054919da9258e403c1beaace99a7578033f2", + "replay_seconds": 34, "snapshot": { "views": [ { @@ -19441,7 +19441,7 @@ "acl": [ "postgres=X/postgres" ], - "def_hash": "92bf14fee49b501305bd81ac583a43cd", + "def_hash": "090e9afd33a764ce6ec81c632abd83b4", "signature": "public.match_governed_candidate_chunks_v3(extensions.vector,text,integer,double precision,uuid[],uuid,boolean,text[],uuid,text,bigint,text[])" }, { diff --git a/supabase/migrations/20261015180000_prefilter_text_only_governed_retrieval.sql b/supabase/migrations/20261015180000_prefilter_text_only_governed_retrieval.sql new file mode 100644 index 0000000000..f7dfb1fa0d --- /dev/null +++ b/supabase/migrations/20261015180000_prefilter_text_only_governed_retrieval.sql @@ -0,0 +1,429 @@ +-- Keyword-only retrieval took 30-48 s on the live corpus because the text match was only a +-- select-list column, so every eligible public chunk was fully built before filtering. +-- Same function body as 20260901130000 plus one WHERE predicate; output is unchanged because +-- a call without an embedding can only rank text-matched rows (vector_ranked is empty). + +begin; + +create or replace function public.match_governed_candidate_chunks_v3( + query_embedding extensions.vector(1536), + query_text text, + match_count integer, + min_similarity double precision, + document_filters uuid[], + owner_filter uuid, + include_public boolean, + corpus_scopes text[], + expected_site_release_id uuid, + expected_site_release_digest text, + expected_site_change_epoch bigint, + site_content_domains text[] +) +returns table ( + id uuid, + document_id uuid, + title text, + file_name text, + page_number integer, + chunk_index integer, + section_heading text, + content text, + retrieval_synopsis text, + image_ids uuid[], + source_metadata jsonb, + document_labels jsonb, + document_summary text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + rrf_score double precision, + lexical_score double precision, + images jsonb, + corpus_scope text, + site_content_domain text, + site_release_id uuid, + site_change_epoch bigint, + pending_exclusion_exact boolean +) +language sql +stable +security definer +set search_path = '' +set work_mem = '64MB' +as $$ + with + admitted as ( + select + least(greatest(coalesce(match_count, 12), 1), 96) as bounded_count, + case + when query_text is null or pg_catalog.btrim(query_text) = '' then null + else pg_catalog.websearch_to_tsquery('english', query_text) + end as tsq + where include_public is true + and owner_filter = '00000000-0000-0000-0000-000000000000'::uuid + and corpus_scopes is not null + and pg_catalog.cardinality(corpus_scopes) between 1 and 4 + and corpus_scopes <@ array[ + 'uploaded_local', + 'clinical_kb_site', + 'australian_public', + 'international_supplementary' + ]::text[] + ), + site_authority as ( + select + state.active_release_id as release_id, + state.change_epoch, + state.served_change_epoch, + release.release_digest + from public.site_content_sync_state state + join public.site_content_releases release + on release.id = state.active_release_id + and release.state = 'active' + and release.target_change_epoch = state.served_change_epoch + and release.release_digest = state.active_release_digest + cross join admitted + where expected_site_release_id is not null + and expected_site_release_digest ~ '^[0-9a-f]{64}$' + and expected_site_change_epoch is not null + and state.initialized + and state.active_release_id = expected_site_release_id + and state.active_release_digest = expected_site_release_digest + and state.change_epoch = expected_site_change_epoch + and state.served_change_epoch <= state.change_epoch + ), + pending_site_set as ( + select not exists ( + select 1 + from public.site_content_public_records record + left join public.site_content_sync_events event + on event.event_sequence = record.pending_event_sequence + where record.pending_event_sequence is not null + and ( + event.event_sequence is null + or event.logical_id is distinct from record.logical_id + or event.target_publication_id is distinct from record.current_publication_id + or event.target_change_epoch is distinct from record.head_change_epoch + or event.target_change_epoch > site_authority.change_epoch + ) + ) and not exists ( + select 1 + from public.site_content_public_records record + where record.head_change_epoch > site_authority.served_change_epoch + and ( + record.pending_event_sequence is null + or record.head_change_epoch > site_authority.change_epoch + ) + ) as exact + from site_authority + ), + pending_site_logical_ids as ( + select record.logical_id + from public.site_content_public_records record + cross join site_authority + where record.pending_event_sequence is not null + ), + document_candidates as ( + select + chunk.id, + document.id as document_id, + document.title, + document.file_name, + chunk.page_number, + chunk.chunk_index, + chunk.section_heading, + chunk.content, + chunk.retrieval_synopsis, + chunk.image_ids, + pg_catalog.jsonb_strip_nulls(pg_catalog.jsonb_build_object( + 'source_kind', 'document', + 'source_title', document.metadata->>'source_title', + 'publisher', document.metadata->>'publisher', + 'publisher_code', document.metadata->>'publisher_code', + 'jurisdiction', document.metadata->>'jurisdiction', + 'version', document.metadata->>'version', + 'publication_date', document.metadata->>'publication_date', + 'review_date', document.metadata->>'review_date', + 'uploaded_at', document.metadata->>'uploaded_at', + 'indexed_at', document.metadata->>'indexed_at', + 'corpus_scope', document.metadata->>'corpus_scope', + 'source_role', document.metadata->>'source_role', + 'content_mode', document.metadata->>'content_mode', + 'source_catalogue_key', document.metadata->>'source_catalogue_key', + 'source_policy_version', document.metadata->>'source_policy_version', + 'canonical_url', document.metadata->>'canonical_url', + 'effective_date', document.metadata->>'effective_date', + 'expiry_date', document.metadata->>'expiry_date', + 'supersedes_document_id', document.metadata->>'supersedes_document_id', + 'superseded_by_document_id', document.metadata->>'superseded_by_document_id', + 'retrieved_at', document.metadata->>'retrieved_at', + 'content_hash', document.metadata->>'content_hash', + 'change_state', document.metadata->>'change_state', + 'licence_policy', document.metadata->>'licence_policy', + 'document_status', document.metadata->>'document_status', + 'clinical_validation_status', document.metadata->>'clinical_validation_status', + 'extraction_quality', document.metadata->>'extraction_quality' + )) as source_metadata, + coalesce( + ( + select pg_catalog.jsonb_agg( + pg_catalog.jsonb_build_object('id', label.id, 'label', label.label, 'label_type', label.label_type) + order by label.label_type, label.label, label.id + ) + from public.document_labels label + where label.document_id = document.id + ), + '[]'::jsonb + ) as document_labels, + ( + select summary.summary + from public.document_summaries summary + where summary.document_id = document.id + order by summary.created_at desc, summary.id + limit 1 + ) as document_summary, + case + when query_embedding is null then 0::double precision + else (1 - (chunk.embedding OPERATOR(extensions.<=>) query_embedding))::double precision + end as similarity, + case + when admitted.tsq is null then 0::double precision + else pg_catalog.ts_rank_cd(chunk.search_tsv, admitted.tsq)::double precision + end as text_rank, + coalesce( + ( + select pg_catalog.jsonb_agg( + pg_catalog.jsonb_build_object( + 'id', image.id, + 'page_number', image.page_number, + 'storage_path', image.storage_path, + 'caption', image.caption + ) + order by image.id + ) + from public.document_images image + where image.id = any(chunk.image_ids) + ), + '[]'::jsonb + ) as images, + document.metadata->>'corpus_scope' as corpus_scope, + ( + admitted.tsq is not null + and (chunk.search_tsv @@ admitted.tsq or document.title_search_tsv @@ admitted.tsq) + ) as text_matched + from public.document_chunks chunk + join public.documents document on document.id = chunk.document_id + cross join admitted + where document.owner_id is null + -- Text-only calls (match_document_chunks_text_v3) can only return text-matched rows, so + -- filter here, before the per-row labels, summary, images and jsonb work, instead of + -- building every eligible public chunk first. Hybrid calls with an embedding are unchanged. + and ( + query_embedding is not null + or ( + admitted.tsq is not null + and (chunk.search_tsv @@ admitted.tsq or document.title_search_tsv @@ admitted.tsq) + ) + ) + and document.status = 'indexed' + and public.is_committed_document_generation(chunk.index_generation_id, document.index_generation_id) + and (document_filters is null or document.id = any(document_filters)) + and document.metadata->>'corpus_scope' = any(corpus_scopes) + and document.metadata->>'corpus_scope' = 'australian_public' + and document.metadata->>'source_kind' = 'document' + and document.metadata->'public_corpus' = 'true'::jsonb + and document.metadata->>'content_mode' = 'indexed_content' + and document.metadata->>'document_status' = 'current' + and coalesce(document.metadata->>'change_state', '') in ('changed', 'unchanged') + and document.metadata->>'licence_policy' = 'public_index_permitted' + and nullif(document.metadata->>'source_catalogue_key', '') is not null + and nullif(document.metadata->>'source_policy_version', '') is not null + and nullif(document.metadata->>'publication_approval_id', '') is not null + and nullif(document.metadata->>'publication_manifest_digest', '') is not null + ), + site_candidates as ( + select + record.logical_chunk_id as id, + record.logical_document_id as document_id, + record.record->>'title' as title, + record.record->>'route' as file_name, + null::integer as page_number, + 0::integer as chunk_index, + record.record->>'domain' as section_heading, + record.normalized_text as content, + null::text as retrieval_synopsis, + '{}'::uuid[] as image_ids, + pg_catalog.jsonb_build_object( + 'corpus_scope', 'clinical_kb_site', + 'source_kind', 'registry_record', + 'source_role', record.record->>'sourceRole', + 'content_mode', 'indexed_content', + 'source_title', record.record->>'title', + 'version', record.record->>'publicationVersion', + 'document_status', record.record->>'sourceStatus', + 'clinical_validation_status', record.record->>'validationStatus', + 'extraction_quality', 'good', + 'content_hash', record.record->>'contentHash', + 'site_content_lineage', record.record->'sourceLineage' + ) as source_metadata, + '[]'::jsonb as document_labels, + null::text as document_summary, + case + when query_embedding is null or record.embedding is null then 0::double precision + else (1 - (record.embedding OPERATOR(extensions.<=>) query_embedding))::double precision + end as similarity, + case + when admitted.tsq is null then 0::double precision + else pg_catalog.ts_rank_cd( + pg_catalog.to_tsvector( + 'english', + coalesce(record.record->>'title', '') || ' ' || record.normalized_text + ), + admitted.tsq + )::double precision + end as text_rank, + '[]'::jsonb as images, + 'clinical_kb_site'::text as corpus_scope, + ( + admitted.tsq is not null + and pg_catalog.to_tsvector( + 'english', + coalesce(record.record->>'title', '') || ' ' || record.normalized_text + ) @@ admitted.tsq + ) as text_matched, + record.record->>'domain' as site_content_domain, + site_authority.release_id as site_release_id, + site_authority.change_epoch as site_change_epoch + from public.site_content_release_records record + cross join admitted + cross join site_authority + cross join pending_site_set + where 'clinical_kb_site' = any(corpus_scopes) + and record.release_id = site_authority.release_id + and pending_site_set.exact + and (document_filters is null or record.logical_document_id = any(document_filters)) + and record.public_visible + and not record.tombstone + and pg_catalog.jsonb_typeof(record.record) = 'object' + and record.record->>'version' = 'site-content-record-v1' + and record.record->>'access' = 'public' + and record.record->>'logicalId' = record.logical_id + and record.record->>'validationStatus' in ('locally_reviewed', 'approved') + and record.record->>'sourceStatus' in ('current', 'review_due') + and record.record->>'domain' in ( + 'services', 'forms', 'medications', 'differentials', 'specifiers', 'dsm', + 'formulation', 'therapies', 'dictionary', 'factsheets', 'calculators', 'tools' + ) + and ( + site_content_domains is null + or ( + pg_catalog.cardinality(site_content_domains) between 1 and 12 + and site_content_domains <@ array[ + 'services', 'forms', 'medications', 'differentials', 'specifiers', 'dsm', + 'formulation', 'therapies', 'dictionary', 'factsheets', 'calculators', 'tools' + ]::text[] + and record.record->>'domain' = any(site_content_domains) + ) + ) + and not exists ( + select 1 + from pending_site_logical_ids pending + where pending.logical_id = record.logical_id + ) + ), + combined as ( + select + document.*, + null::text as site_content_domain, + null::uuid as site_release_id, + null::bigint as site_change_epoch + from document_candidates document + union all + select * from site_candidates + ), + vector_ranked as ( + select + combined.id, + combined.corpus_scope, + row_number() over (order by combined.similarity desc, combined.id) as vector_rank, + null::bigint as text_match_rank + from combined + where query_embedding is not null + and combined.similarity >= greatest(coalesce(min_similarity, 0.12), 0) + order by combined.similarity desc, combined.id + limit (select greatest(bounded_count * 6, 48) from admitted) + ), + text_ranked as ( + select + combined.id, + combined.corpus_scope, + null::bigint as vector_rank, + row_number() over (order by combined.text_rank desc, combined.id) as text_match_rank + from combined + where combined.text_matched + order by combined.text_rank desc, combined.id + limit (select greatest(bounded_count * 6, 48) from admitted) + ), + rank_positions as ( + select + positions.id, + positions.corpus_scope, + min(positions.vector_rank) as vector_rank, + min(positions.text_match_rank) as text_match_rank + from ( + select * from vector_ranked + union all + select * from text_ranked + ) positions + group by positions.id, positions.corpus_scope + ), + ranked as ( + select + combined.*, + ((combined.similarity * 0.65) + (least(combined.text_rank, 1) * 0.35))::double precision as hybrid_score, + ( + coalesce(1.0 / (60 + rank_positions.vector_rank), 0) + + coalesce(1.0 / (60 + rank_positions.text_match_rank), 0) + )::double precision as rrf_score + from combined + join rank_positions using (id, corpus_scope) + ) + select + ranked.id, + ranked.document_id, + ranked.title, + ranked.file_name, + ranked.page_number, + ranked.chunk_index, + ranked.section_heading, + ranked.content, + ranked.retrieval_synopsis, + ranked.image_ids, + ranked.source_metadata, + ranked.document_labels, + ranked.document_summary, + ranked.similarity, + ranked.text_rank, + ranked.hybrid_score, + ranked.rrf_score, + least(ranked.text_rank, 1)::double precision as lexical_score, + ranked.images, + ranked.corpus_scope, + ranked.site_content_domain, + ranked.site_release_id, + ranked.site_change_epoch, + true as pending_exclusion_exact + from ranked + order by ranked.hybrid_score desc, ranked.rrf_score desc, ranked.id + limit (select bounded_count from admitted); +$$; + +revoke all on function public.match_governed_candidate_chunks_v3( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean, text[], uuid, text, bigint, text[] +) from public, anon, authenticated, service_role; + +alter function public.match_governed_candidate_chunks_v3( + extensions.vector, text, integer, double precision, uuid[], uuid, boolean, text[], uuid, text, bigint, text[] +) owner to postgres; + +commit; diff --git a/supabase/schema.sql b/supabase/schema.sql index 5c41424857..fc13616da7 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -17767,6 +17767,16 @@ as $$ join public.documents document on document.id = chunk.document_id cross join admitted where document.owner_id is null + -- Text-only calls (match_document_chunks_text_v3) can only return text-matched rows, so + -- filter here, before the per-row labels, summary, images and jsonb work, instead of + -- building every eligible public chunk first. Hybrid calls with an embedding are unchanged. + and ( + query_embedding is not null + or ( + admitted.tsq is not null + and (chunk.search_tsv @@ admitted.tsq or document.title_search_tsv @@ admitted.tsq) + ) + ) and document.status = 'indexed' and public.is_committed_document_generation(chunk.index_generation_id, document.index_generation_id) and (document_filters is null or document.id = any(document_filters)) diff --git a/tests/dsm.test.ts b/tests/dsm.test.ts index 1dc184d269..cfb313d745 100644 --- a/tests/dsm.test.ts +++ b/tests/dsm.test.ts @@ -80,6 +80,14 @@ describe("DSM clinical catalogue", () => { expect(rankDsmDiagnoses("PDD", 1)[0]?.diagnosis.slug).toBe("persistent-depressive-disorder-dysthymia"); }); + it("ranks Major depressive disorder first for a plain search for depression", () => { + expect(rankDsmDiagnoses("depression", 4, [], true)[0]?.diagnosis.slug).toBe("major-depressive-disorder"); + expect(rankDsmDiagnoses("major depression", 1, [], true)[0]?.diagnosis.slug).toBe("major-depressive-disorder"); + expect(rankDsmDiagnoses("chronic depression", 1, [], true)[0]?.diagnosis.slug).toBe( + "persistent-depressive-disorder-dysthymia", + ); + }); + it("ranks US DSM spellings to the matching catalogue diagnoses", () => { expect(rankDsmDiagnoses("generalized anxiety disorder", 1)[0]?.diagnosis.slug).toBe("generalised-anxiety-disorder"); }); diff --git a/tests/rag-unsupported-short-circuit-cache.test.ts b/tests/rag-unsupported-short-circuit-cache.test.ts index 01e7d10405..28bb2a2151 100644 --- a/tests/rag-unsupported-short-circuit-cache.test.ts +++ b/tests/rag-unsupported-short-circuit-cache.test.ts @@ -45,7 +45,7 @@ class EmptyQuery implements PromiseLike<{ data: unknown[]; error: null }> { async function loadSearch( corpusGroundingVerdict: "inconclusive" | "out_of_corpus" | "in_corpus_topic", - options: { openAiApiKey?: string } = {}, + options: { openAiApiKey?: string; sourceOnly?: boolean; correctionError?: boolean } = {}, ) { const setCachedSearch = vi.fn(async () => undefined); @@ -84,7 +84,7 @@ async function loadSearch( }; }); vi.doMock("@/lib/rag/rag-provider", () => ({ - isSourceOnlyMode: () => true, + isSourceOnlyMode: () => options.sourceOnly ?? true, allowsAutoDegrade: () => true, sourceOnlyReason: () => "source_only", classifyProviderFailure: () => "provider_failure", @@ -92,7 +92,19 @@ async function loadSearch( })); vi.doMock("@/lib/supabase/admin", () => ({ createAdminClient: () => ({ - rpc: vi.fn(async () => ({ data: [], error: null })), + rpc: vi.fn((name: string) => { + // The query-correction call is awaited after an optional abortSignal(), so it returns a + // thenable builder. A timeout surfaces as an error with null data. + const response = + name === "correct_clinical_query_terms" + ? options.correctionError + ? { data: null, error: { message: "AbortError: signal timed out" } } + : { data: null, error: null } + : { data: [], error: null }; + const builder = Promise.resolve(response) as Promise & { abortSignal: () => unknown }; + builder.abortSignal = () => builder; + return builder; + }), from: vi.fn(() => new EmptyQuery()), }), })); @@ -178,6 +190,39 @@ describe("unsupported short-circuit cache write", () => { expect(setCachedSearch).toHaveBeenCalledTimes(1); }, 60_000); + it("does not cache the zero when the query-correction call failed or timed out", async () => { + // Qodo review on PR #3560: a timed-out correction resolves with an error and null data, which + // is not evidence that the query has no match, so the empty result must not become sticky. + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive", { + sourceOnly: false, + correctionError: true, + }); + + const result = await searchChunksWithTelemetry({ + query: "Show me the airport travel policy", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).not.toHaveBeenCalled(); + }, 60_000); + + it("still caches the deterministic zero when the query-correction call succeeded", async () => { + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive", { sourceOnly: false }); + + const result = await searchChunksWithTelemetry({ + query: "Show me the airport travel policy", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).toHaveBeenCalledTimes(1); + }, 60_000); + it("rescues an in-corpus bare topic before ever reaching the short circuit", async () => { const { searchChunksWithTelemetry } = await loadSearch("in_corpus_topic"); diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts index 75b8c41241..a1fc51c249 100644 --- a/tests/supabase-schema.test.ts +++ b/tests/supabase-schema.test.ts @@ -25,6 +25,19 @@ const failClosedInternationalGovernedRetrievalV3 = readFileSync( new URL("../supabase/migrations/20260901130000_fail_closed_unactivated_international_retrieval.sql", import.meta.url), "utf8", ).replace(/\s+/g, " "); +// The effective match_governed_candidate_chunks_v3 body: 20260901130000 plus the text-only +// prefilter (it may only add that one WHERE predicate, so the 0901 assertions still describe it). +const textOnlyPrefilterGovernedRetrievalV3 = readFileSync( + new URL("../supabase/migrations/20261015180000_prefilter_text_only_governed_retrieval.sql", import.meta.url), + "utf8", +).replace(/\s+/g, " "); +const textOnlyPrefilterPredicate = + " and ( query_embedding is not null or ( admitted.tsq is not null and (chunk.search_tsv @@ admitted.tsq or document.title_search_tsv @@ admitted.tsq) ) )"; +function governedCandidateV3Body(sql: string) { + const start = sql.indexOf("create or replace function public.match_governed_candidate_chunks_v3("); + const end = sql.indexOf("revoke all on function public.match_governed_candidate_chunks_v3(", start); + return sql.slice(start, end); +} const siteContentHealthMigration = readFileSync( new URL("../supabase/migrations/20260824123000_add_site_content_health_probe.sql", import.meta.url), "utf8", @@ -196,7 +209,7 @@ describe("governed corpus retrieval v3 schema", () => { "document.metadata->>'corpus_scope' in ('australian_public', 'international_supplementary')", ); expect(candidateFunction).not.toContain("from public.document_publication_approvals approval"); - expect(schema.slice(schemaStart, schemaEnd)).toBe(candidateFunction); + expect(schema.slice(schemaStart, schemaEnd)).toBe(governedCandidateV3Body(textOnlyPrefilterGovernedRetrievalV3)); }); it("emits canonical registry metadata and mirrors the effective function in the final schema", () => { @@ -214,7 +227,23 @@ describe("governed corpus retrieval v3 schema", () => { const schemaFunction = schema.slice(schemaStart, schemaEnd); expect(migrationFunction).toContain("'source_kind', 'registry_record'"); expect(migrationFunction).not.toContain("'source_kind', 'site_content_release_record'"); - expect(schemaFunction).toBe(migrationFunction); + expect(schemaFunction).toBe(governedCandidateV3Body(textOnlyPrefilterGovernedRetrievalV3)); + }); + + it("changes only the text-only prefilter predicate on top of 20260901130000", () => { + // Compare with SQL line comments removed, so the explanatory comment beside the predicate is ignored. + const withoutComments = (file: string) => + readFileSync(new URL(`../supabase/migrations/${file}`, import.meta.url), "utf8") + .replace(/--[^\n]*/g, "") + .replace(/\s+/g, " "); + const base = governedCandidateV3Body( + withoutComments("20260901130000_fail_closed_unactivated_international_retrieval.sql"), + ); + const prefiltered = governedCandidateV3Body( + withoutComments("20261015180000_prefilter_text_only_governed_retrieval.sql"), + ); + expect(prefiltered).toContain(textOnlyPrefilterPredicate); + expect(prefiltered.replace(textOnlyPrefilterPredicate, "")).toBe(base); }); });