diff --git a/docs/outstanding-issues-inbox/4f15ebaf-d311-4971-a52f-46e26a2a963d.json b/docs/outstanding-issues-inbox/4f15ebaf-d311-4971-a52f-46e26a2a963d.json new file mode 100644 index 0000000000..913a516b75 --- /dev/null +++ b/docs/outstanding-issues-inbox/4f15ebaf-d311-4971-a52f-46e26a2a963d.json @@ -0,0 +1,12 @@ +{ + "version": 2, + "id": "4f15ebaf-d311-4971-a52f-46e26a2a963d", + "createdOn": "2026-10-21", + "action": "done", + "payload": { + "id": "#VXB8XA", + "outcome": "Fixed on main (verified offline 21 Oct 2026, answer safety hardening thread). cite-mismatched-attribution no longer has a KNOWN_DIVERGENCES entry in tests/rag-adversarial-harness.test.ts, and npm run eval:rag:adversarial:offline passes 25 of 25 on main 81ba5aefa, via PR #3628 (62558e5c7, 18 Oct, document attribution fix), so the case now meets its normative fixture expectation (cites only the claim-bearing chunk). The pin was self-expiring by design, so its removal records the behaviour change.", + "baseRowFingerprint": "1f48f81cb2e3a3db5cd3994a709d63e2ff2e3d2967790a017a6c6d99e4a3ceee", + "targetUlid": "01M08C19FYVXB8XATJE3TYZJMS" + } +} diff --git a/docs/outstanding-issues-inbox/62c5cf9f-95c6-4a40-9112-6f63ac631d9b.json b/docs/outstanding-issues-inbox/62c5cf9f-95c6-4a40-9112-6f63ac631d9b.json new file mode 100644 index 0000000000..ff21e74c24 --- /dev/null +++ b/docs/outstanding-issues-inbox/62c5cf9f-95c6-4a40-9112-6f63ac631d9b.json @@ -0,0 +1,12 @@ +{ + "version": 2, + "id": "62c5cf9f-95c6-4a40-9112-6f63ac631d9b", + "createdOn": "2026-10-21", + "action": "done", + "payload": { + "id": "#F3B2XC", + "outcome": "Done on main (verified 21 Oct 2026, answer safety hardening thread). src/lib/medications/tall-man.ts was rebuilt from the ACSQHC National Mixed-Case Lettering List (April 2024) in PR #3562 (5319dab78, 16 Oct), pinned by tests/tall-man-lettering.test.ts. All 26 lettered names were re-checked against the Commission PDF (safetyandquality.gov.au national_mixed-case_lettering_list.pdf, Tables 1 and 3) on 21 Oct and every one matches. The doubtful forms in this row (citaloPRAM, fluPENtixol, VALproate, midazolAM, temazePAM) are gone. One local exception remains, cloZAPine (not on the ACSQHC list), which the owner confirmed keeping on 21 Oct 2026 ('Go ahead with all recommendations', project chat). Brand names on the list (Largactil, Lamictal, Pristiq and others) are not lettered because only generic record names are formatted.", + "baseRowFingerprint": "ceae0146e0b33521b704f0c9830ae582d28aa18ecf652e60b4f337f204126c7f", + "targetUlid": "01M4ZM794RF3B2XCAJ2YDRJ8FG" + } +} diff --git a/docs/outstanding-issues-inbox/d7678153-b6d7-4571-8f46-48f4a59ca47a.json b/docs/outstanding-issues-inbox/d7678153-b6d7-4571-8f46-48f4a59ca47a.json new file mode 100644 index 0000000000..c2ac7abfbe --- /dev/null +++ b/docs/outstanding-issues-inbox/d7678153-b6d7-4571-8f46-48f4a59ca47a.json @@ -0,0 +1,12 @@ +{ + "version": 2, + "id": "d7678153-b6d7-4571-8f46-48f4a59ca47a", + "createdOn": "2026-10-21", + "action": "update", + "payload": { + "id": "#ZZ4RAP", + "detail": "UPDATE 2026-10-21 (answer safety hardening thread, verified offline on main 81ba5aefa): 18 of the 20 cases are wired into answerQualityEvalCases as high-risk- on the owner go-ahead of 2026-10-10, with verbatim evidence pinned by tests/high-risk-eval-cases-draft.test.ts. Two stay out as source_needed: breastfeeding-lithium and hepatic-valproate, because no WA passage for either is held in the repository. Draft expected facts for both were sent to the owner for correction on 2026-10-21 (not approved). Next: owner corrects the drafts and approves a free read-only fetch of the KEMH lithium breastfeeding and CG.MHSP.Valproate passages into a fixture, then the two cases are wired in. PRIOR DETAIL: OWNER DECISION 2026-09-25: approved. Engineering drafts the 20 cases (question, key facts a correct answer must contain, facts it must not state, expected source); the owner reviews and corrects them before any are added to the harness. A first draft of all 20 was delivered to the owner on 2026-09-25 as a Word document (not committed: clinical content awaiting review). Verified offline by topic count over src/lib/rag/rag-eval-cases.ts (101 cases): hepatic 0, older adult/elderly 0, drug interaction 0, unsafe/false premise 0, Mental Health Act 1, QTc 2, pregnancy 3, paediatric 2. Owner recommendation (2026-09-25): start with ~20 clinician-reviewed cases (3 pregnancy/breastfeeding, 3 renal/hepatic, 2 older adults, 2 QTc, 2 lithium/clozapine monitoring, 2 overdose, 2 interactions, 2 MHA, 2 false-premise where the right answer corrects the premise); engineering drafts expected key facts + source, owner reviews. Adding golden cases touches the eval harness (RAG-protected surface): flag before editing, and live canary dispatches need approval.", + "baseRowFingerprint": "33b1e815aec207b8f894527ab4e39d00df4a2c81e69bcfb8a22cfcdf0842e01f", + "targetUlid": "01M3CJ78GDZZ4RAPXAS64MSVYH" + } +} diff --git a/docs/outstanding-issues-inbox/e9b59bb8-6de5-4041-a086-c6d01bae8593.json b/docs/outstanding-issues-inbox/e9b59bb8-6de5-4041-a086-c6d01bae8593.json new file mode 100644 index 0000000000..15604ebd87 --- /dev/null +++ b/docs/outstanding-issues-inbox/e9b59bb8-6de5-4041-a086-c6d01bae8593.json @@ -0,0 +1,12 @@ +{ + "version": 2, + "id": "e9b59bb8-6de5-4041-a086-c6d01bae8593", + "createdOn": "2026-10-21", + "action": "done", + "payload": { + "id": "#HD1C6W", + "outcome": "Done on main (verified 21 Oct 2026, answer safety hardening thread). The word-form fix for all three sealed hazard-control files landed in PR #3571 (e526f8a4d, 16 Oct) with tests in answer-verification, rag-claim-support and source-text-sanitizer; docs/clinical-hazard-controls.json was then resealed with owner approval on 21 Oct (5714539c5, comment 6313455205), and npm run check:clinical-hazard-controls passes on main 81ba5aefa. A fresh grep finds no bare escalat/contraindicat/prescrib/initiat stems left inside word boundaries in those files.", + "baseRowFingerprint": "b35fbe964bb903a420fef3a4ea99e440f64bc026c7e60818d7e569314cc0cc65", + "targetUlid": "01M3H9CJH5HD1C6WEV9E9KKASJ" + } +} diff --git a/docs/outstanding-issues-inbox/eed86710-2875-44cf-80bf-8a79dc8fe069.json b/docs/outstanding-issues-inbox/eed86710-2875-44cf-80bf-8a79dc8fe069.json new file mode 100644 index 0000000000..6f8f66532b --- /dev/null +++ b/docs/outstanding-issues-inbox/eed86710-2875-44cf-80bf-8a79dc8fe069.json @@ -0,0 +1,12 @@ +{ + "version": 2, + "id": "eed86710-2875-44cf-80bf-8a79dc8fe069", + "createdOn": "2026-10-21", + "action": "update", + "payload": { + "id": "#BFCTST", + "detail": "UPDATE 2026-10-21 (answer safety hardening thread, offline): tier-rule tests added to tests/australian-source-priority.test.ts (WA ahead of national ahead of other states ahead of supplementary at equal relevance, relevance always wins across bands, supplementary padding dropped once two Australian documents suffice). Both owner additions remain unbuilt: (a) outdated sources are only penalised in rag-second-stage.ts and warned about in the sealed src/lib/source-governance.ts, so they can still carry a claim; (b) buildSourceConflictSection (src/lib/rag/source-conflict-section.ts) only compares uploaded_local with australian_public, and uploaded_local retrieval is fail-closed (#9V1TJ5), so a WA versus national public disagreement is never disclosed. Both are answer-behaviour changes: RAG notice, canary pair and hazard reseal needed; plan sent to the owner 2026-10-21. PRIOR DETAIL: OWNER DECISION 2026-09-25: confirmed the existing tier order (WA validated > Australian national > other Australian state > supplementary, within a relevance band) and asked for two additions: a superseded source is never used as evidence, and a clinically material WA-vs-national disagreement is disclosed with both sources shown. Found 2026-09-25: outdated sources are currently KEPT and shown with a danger-severity OUTDATED warning (src/lib/source-governance.ts), and authority conflict in source-authority-registry.ts means publisher-metadata mismatch, not clinical disagreement. Both additions change retrieval/answer behaviour: flag before editing and run a live canary pair (owner approval, about USD 1-2) around the change. A rule already exists: src/lib/australian-source-priority.ts ranks wa_validated < australian_national < australian_state < supplementary WITHIN a relevance band, and tracks authorityConflictCount. Only 3 eval cases touch conflict/superseded sources. Owner recommendation (2026-09-25): confirm that order, add (a) superseded versions never used as evidence and (b) disclose both sources when WA and national disagree on something clinically material. Next action: owner confirms the rule, then add conflict/currentness/attribution cases. Ranking is RAG-protected: flag before editing and require a canary pair for any ordering change.", + "baseRowFingerprint": "2d864f3aad6ae3255c85943918a427e80f2718cdf482e1b332e2fd1c2a1e4be4", + "targetUlid": "01M3CJ78QHBFCTST7YFDZZHKE3" + } +} diff --git a/docs/outstanding-issues-inbox/f2e5b2bc-8d0d-4b8c-86a6-a93dfa6cdc7e.json b/docs/outstanding-issues-inbox/f2e5b2bc-8d0d-4b8c-86a6-a93dfa6cdc7e.json new file mode 100644 index 0000000000..67d361e67c --- /dev/null +++ b/docs/outstanding-issues-inbox/f2e5b2bc-8d0d-4b8c-86a6-a93dfa6cdc7e.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "id": "f2e5b2bc-8d0d-4b8c-86a6-a93dfa6cdc7e", + "createdOn": "2026-10-21", + "action": "add", + "payload": { + "pri": "P2", + "type": "issue", + "summary": "The WA valproate guideline (CG.MHSP.Valproate.pdf) is not in the live corpus, although an answer-quality eval case expects it", + "detail": "Read-only live query 2026-10-21 (owner-approved free read, answer safety hardening thread): no document in public.documents has a title or file name containing 'valpro' other than the registry JSON records. quality-valproate-pregnancy-contraindication in src/lib/rag/rag-eval-cases.ts lists CG.MHSP.Valproate.pdf as an expected file, and the high-risk case hepatic-valproate (#ZZ4RAP) cannot be sourced without it. The only hepatic valproate passage found was BMJ Best Practice 'Bipolar disorder in adults' p130, whose licence is unconfirmed. Next: owner supplies the WA guideline PDF for ingestion (same path as the CAMHS ADHD guideline, #018); no re-index of existing documents is needed.", + "source": "Answer safety hardening thread, 2026-10-21, read-only Supabase query", + "issueUlid": "01M5F5FS6D7B6MMMAM9NCTSB3M" + } +} diff --git a/docs/outstanding-issues-inbox/f4f9f0cd-7579-4f7e-8beb-6c30ee1fc5cb.json b/docs/outstanding-issues-inbox/f4f9f0cd-7579-4f7e-8beb-6c30ee1fc5cb.json new file mode 100644 index 0000000000..ae235be851 --- /dev/null +++ b/docs/outstanding-issues-inbox/f4f9f0cd-7579-4f7e-8beb-6c30ee1fc5cb.json @@ -0,0 +1,12 @@ +{ + "version": 2, + "id": "f4f9f0cd-7579-4f7e-8beb-6c30ee1fc5cb", + "createdOn": "2026-10-21", + "action": "done", + "payload": { + "id": "#E6142T", + "outcome": "Closed as won't fix by owner decision 2026-10-21 ('Go ahead with all recommendations', project chat, on the answer safety hardening plan). Part 1 was already closed by owner ruling 2026-09-25. Part 2 (services mode nouns in modeSearchWords) has no observable effect on main because the services ranker and universal search never read modeSearchWords; reopen only if services search starts consuming smartSearchContentTerms.", + "baseRowFingerprint": "2f70b014d807047cbc1309c4c28591101c2de8f050989661fda6168df6fffa67", + "targetUlid": "01M2R5W5F8E6142TR092JVPGF2" + } +} diff --git a/src/components/clinical-dashboard/answer-content.tsx b/src/components/clinical-dashboard/answer-content.tsx index 06034060a4..0a610b8491 100644 --- a/src/components/clinical-dashboard/answer-content.tsx +++ b/src/components/clinical-dashboard/answer-content.tsx @@ -3,6 +3,7 @@ import { primaryAnswerDisplayText as canonicalPrimaryAnswerDisplayText } from "@/lib/answer-display-text"; import { Fragment, memo, type ReactNode } from "react"; +import Link from "next/link"; import { Copy } from "lucide-react"; import { SafeBoldText } from "@/components/SafeBoldText"; @@ -20,8 +21,14 @@ import { SignedImage } from "@/components/clinical-dashboard/signed-image"; import { clinicalProseUsefulness } from "@/lib/source-text-sanitizer"; import { type ClaimMarkCluster, type SupportedClaim, resolveClaimMarks } from "@/lib/answer-claim-marks"; import { type SourceLink } from "@/lib/answer-render-policy"; +import { documentCitationHref } from "@/lib/citations"; +import { guidanceDifferenceLabel } from "@/lib/rag/guidance-difference-note"; import type { AnswerSection, AnswerSectionKind, RagAnswer, VisualEvidenceCard } from "@/lib/types"; -import type { ClientBestSourceRecommendation, ClientSearchResult } from "@/lib/answer-client-payload"; +import type { + ClientBestSourceRecommendation, + ClientRagAnswerPayload, + ClientSearchResult, +} from "@/lib/answer-client-payload"; export const SourceImage = memo(function SourceImage({ endpoint, @@ -260,6 +267,7 @@ export function NaturalLanguageAnswer({ showCopyAction = true, isRefusal = false, hideSourceRail = false, + guidanceDifference, }: { // Raw answer text (server bold intact); this component owns display // sanitization so can render the high-yield emphasis. @@ -315,6 +323,11 @@ export function NaturalLanguageAnswer({ * so the list of them is left off rather than shown as if it backed something. */ hideSourceRail?: boolean; + /** + * Owner-signed WA versus national difference (`answer.guidanceDifference`). The answer still + * follows WA; one quiet line under the prose names the national source. Approved 2026-10-21. + */ + guidanceDifference?: ClientRagAnswerPayload["guidanceDifference"]; }) { const { preferences } = useAppPreferences(); const fragments = primaryAnswerDisplayFragments(text, { preformatted, preserveBold: true }); @@ -375,6 +388,7 @@ export function NaturalLanguageAnswer({ ))}

+ {guidanceDifference && !isRefusal ? : null} {/* The Source-only pill is NOT here any more (owner decision, 2026-09-03). It sat below the prose, apart from the three status chips above it and built from a third disclosure mechanism, so the @@ -419,6 +433,33 @@ export function NaturalLanguageAnswer({ ); } +function GuidanceDifferenceLine({ + difference, +}: { + difference: NonNullable; +}) { + const label = guidanceDifferenceLabel(difference); + return ( +

+ National guidance differs on this point. See{" "} + + {label} + + . +

+ ); +} + export function UserQuestionBubble({ query }: { query: string }) { const cleaned = query.trim(); if (!cleaned) return null; diff --git a/src/components/clinical-dashboard/answer-copy-payload.ts b/src/components/clinical-dashboard/answer-copy-payload.ts index 7be0f84ce5..29f3668c63 100644 --- a/src/components/clinical-dashboard/answer-copy-payload.ts +++ b/src/components/clinical-dashboard/answer-copy-payload.ts @@ -25,6 +25,7 @@ import { isPreformattedGroundedAnswer, primaryAnswerDisplayText } from "@/compon import { projectAnswerForMainSurface } from "@/components/clinical-dashboard/answer-section-projector"; import { composeAnswerClipboardText } from "@/lib/answer-clipboard"; import { publicFallbackReason } from "@/lib/rag/rag-fallback-reason"; +import { guidanceDifferenceSentence } from "@/lib/rag/guidance-difference-note"; import type { AnswerPayload } from "@/components/clinical-dashboard/search-utils"; import type { ClientRagAnswerPayload, ClientSearchResult } from "@/lib/answer-client-payload"; import { ragAdaptiveAnswerPromptVersion } from "@/lib/rag/rag-versioning"; @@ -178,6 +179,20 @@ export function singleDocumentClipboardMetadata( return withMetadata?.source_metadata ?? undefined; } +/** + * The national guidance line travels with a copied answer, directly after the answer text, so a + * pasted WA recommendation never loses its "national guidance differs" caveat. + */ +function withGuidanceDifferenceLine(text: string, answer: ClientRagAnswerPayload): string { + if (!answer.guidanceDifference) return text; + const line = guidanceDifferenceSentence(answer.guidanceDifference); + const lead = answerTextForClipboard(answer); + const at = lead ? text.indexOf(lead) : -1; + if (at < 0) return `${text}\n\n${line}`; + const end = at + lead.length; + return `${text.slice(0, end)}\n\n${line}${text.slice(end)}`; +} + /** * The clipboard payload for an answer. `renderCopyText` remains the primary * product string, except its finalized answer lead is replaced with the same @@ -208,7 +223,7 @@ export function buildAnswerClipboardText({ routingMode: answer.routingMode, }); const copied = composeAnswerClipboardText({ - renderCopyText: renderCopyTextWithCanonicalAnswer(renderCopyText, answer), + renderCopyText: withGuidanceDifferenceLine(renderCopyTextWithCanonicalAnswer(renderCopyText, answer), answer), sourceOnly, state, degradedReason: degraded diff --git a/src/components/clinical-dashboard/answer-result-surface.tsx b/src/components/clinical-dashboard/answer-result-surface.tsx index 7c41f7fa8b..dcfdf9602f 100644 --- a/src/components/clinical-dashboard/answer-result-surface.tsx +++ b/src/components/clinical-dashboard/answer-result-surface.tsx @@ -599,6 +599,7 @@ function StagedAnswerResultSurfaceImpl({ openSourceIndex={openSourceIndex} showCopyAction={false} isRefusal={refusalAnswer} + guidanceDifference={answer.guidanceDifference} copied={copiedAnswer} onCopy={onCopyAnswer} /> diff --git a/src/components/clinical-dashboard/answer-thread-turn.tsx b/src/components/clinical-dashboard/answer-thread-turn.tsx index 3ab0c8648a..3a98f732b3 100644 --- a/src/components/clinical-dashboard/answer-thread-turn.tsx +++ b/src/components/clinical-dashboard/answer-thread-turn.tsx @@ -163,6 +163,7 @@ export const PriorAnswerTurnSurface = memo(function PriorAnswerTurnSurface({ sourceLinks={leadSourceLinks} railRows={railSources} isRefusal={refusalAnswer} + guidanceDifference={turn.answer.guidanceDifference} copied={copied} onCopy={() => onCopy( diff --git a/src/data/reviewed-guidance-differences.json b/src/data/reviewed-guidance-differences.json new file mode 100644 index 0000000000..e3b31d84ad --- /dev/null +++ b/src/data/reviewed-guidance-differences.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "note": "Owner-signed WA versus national guidance differences. An entry shows the 'National guidance differs on this point' line under any answer whose direct support cites one of its WA chunks. Only the owner adds or signs off an entry, in his own words. Expired entries are ignored.", + "differences": [] +} diff --git a/src/lib/answer-client-fields.ts b/src/lib/answer-client-fields.ts index 9ab5e28aaa..b75ccd01e1 100644 --- a/src/lib/answer-client-fields.ts +++ b/src/lib/answer-client-fields.ts @@ -159,6 +159,17 @@ export const clientAnswerFieldsSchema = z.object({ .check(z.maxLength(100)), ), sourceCurrencyWarning: z.optional(z.enum(["supporting", "retrieved"])), + // Owner-signed WA versus national difference (src/lib/rag/guidance-difference-note.ts). + guidanceDifference: z.optional( + z.object({ + differenceId: id, + title: z.string().check(z.minLength(1), z.maxLength(300)), + year: z.int().check(z.minimum(1900), z.maximum(2100)), + page: z.nullable(z.int().check(z.minimum(1), z.maximum(10_000))), + documentId: id, + chunkId: id, + }), + ), answerContractVersion: z.optional(z.literal(ragAdaptiveAnswerPromptVersion)), renderAdaptiveAnswer: z.optional(z.boolean()), // Disclosure persists with the answer; feedback credentials deliberately do not. diff --git a/src/lib/answer-response.ts b/src/lib/answer-response.ts index 9d860c85d5..0730174e33 100644 --- a/src/lib/answer-response.ts +++ b/src/lib/answer-response.ts @@ -6,6 +6,7 @@ import { sourceGovernanceWarnings, } from "@/lib/source-governance"; import type { RagAnswer } from "@/lib/types"; +import { guidanceDifferenceNoteForAnswer } from "@/lib/rag/guidance-difference-note"; import { carryRagProgrammeTelemetry } from "@/lib/rag/rag-programme-telemetry"; import { classifyRagFallbackReason, @@ -14,6 +15,12 @@ import { publicFallbackReason, } from "@/lib/rag/rag-fallback-reason"; +/** Owner-signed WA versus national note; absent unless a signed-off WA chunk directly supports the answer. */ +function withGuidanceDifference(answer: RagAnswer) { + const guidanceDifference = guidanceDifferenceNoteForAnswer(answer); + return guidanceDifference ? { guidanceDifference } : {}; +} + function answerFallbackReasonCode( answer?: Pick< RagAnswer, @@ -113,6 +120,7 @@ export function buildGovernedAnswerClientResponse(answer: RagAnswer) { telemetryAnswer: answer, payload: { ...toClientAnswerPayload({ ...answer, fallbackReasonCode }), + ...withGuidanceDifference(answer), fallbackReasonCode, degradedMode: answerDegradedModeSignal({ ...answer, fallbackReasonCode }), sourceGovernanceWarnings: warnings, diff --git a/src/lib/rag/guidance-difference-note.ts b/src/lib/rag/guidance-difference-note.ts new file mode 100644 index 0000000000..e9c60e85ce --- /dev/null +++ b/src/lib/rag/guidance-difference-note.ts @@ -0,0 +1,128 @@ +import registry from "@/data/reviewed-guidance-differences.json"; +import { clientAnswerFieldsSchema } from "@/lib/answer-client-fields"; +import type { RagAnswer } from "@/lib/types"; + +/** + * Owner decision (Josh, 2026-10-21): when a WA guideline and a national guideline differ on a + * point, the answer still follows WA and one plain line under it names the national source. + * The line only ever comes from an owner-signed entry in `src/data/reviewed-guidance-differences.json`, + * never from model or heuristic comparison, because a wrong "guidance differs" claim is itself a + * clinical hazard. Mockup approved 2026-10-21 (Live site visual checks, item 8). + */ +export type ReviewedGuidanceDifference = { + id: string; + topic: string; + waDocumentId: string; + waChunkIds: string[]; + national: { + documentId: string; + chunkId: string; + title: string; + publisher: string; + year: number; + page: number | null; + }; + signedOffBy: "owner"; + signedOffAt: string; + reviewBy: string; +}; + +export type GuidanceDifferenceNote = { + differenceId: string; + title: string; + year: number; + page: number | null; + documentId: string; + chunkId: string; +}; + +const datePattern = /^\d{4}-\d{2}-\d{2}$/; + +function isReviewedDifference(value: unknown): value is ReviewedGuidanceDifference { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const entry = value as ReviewedGuidanceDifference; + const national = entry.national; + return ( + typeof entry.id === "string" && + entry.id.length > 0 && + typeof entry.waDocumentId === "string" && + Array.isArray(entry.waChunkIds) && + entry.waChunkIds.length > 0 && + entry.waChunkIds.every((chunkId) => typeof chunkId === "string" && chunkId.length > 0) && + entry.signedOffBy === "owner" && + datePattern.test(entry.signedOffAt) && + datePattern.test(entry.reviewBy) && + Boolean(national) && + typeof national.documentId === "string" && + national.documentId.length > 0 && + national.documentId !== entry.waDocumentId && + typeof national.chunkId === "string" && + national.chunkId.length > 0 && + typeof national.title === "string" && + national.title.trim().length > 0 && + Number.isInteger(national.year) && + (national.page === null || (Number.isInteger(national.page) && national.page > 0)) + ); +} + +function noteFor(entry: ReviewedGuidanceDifference): GuidanceDifferenceNote { + return { + differenceId: entry.id, + title: entry.national.title.trim(), + year: entry.national.year, + page: entry.national.page, + documentId: entry.national.documentId, + chunkId: entry.national.chunkId, + }; +} + +/** + * The client boundary rejects a whole answer whose `guidanceDifference` fails its schema, so an + * entry is only active when the note it would build already passes that same schema. + */ +function passesClientBoundary(entry: ReviewedGuidanceDifference) { + return clientAnswerFieldsSchema.safeParse({ guidanceDifference: noteFor(entry) }).success; +} + +/** Entries that are well formed and not past their review-by date; malformed entries are dropped, not repaired. */ +export function activeReviewedGuidanceDifferences( + entries: readonly unknown[] = registry.differences, + now: Date = new Date(), +): ReviewedGuidanceDifference[] { + const today = now.toISOString().slice(0, 10); + return entries + .filter(isReviewedDifference) + .filter(passesClientBoundary) + .filter((entry) => entry.reviewBy >= today) + .sort((left, right) => left.id.localeCompare(right.id)); +} + +/** + * The note for an answer, or `undefined`. It needs a grounded answer whose directly supported + * claims cite a signed-off WA chunk; partial support, refusals and ungrounded answers never carry it. + */ +export function guidanceDifferenceNoteForAnswer( + answer: Pick, + differences: readonly ReviewedGuidanceDifference[] = activeReviewedGuidanceDifferences(), +): GuidanceDifferenceNote | undefined { + if (!answer.grounded || differences.length === 0) return undefined; + const servedChunkIds = new Set(answer.sources.map((source) => source.id)); + const directChunkIds = new Set( + (answer.supportedClaims ?? []) + .filter((claim) => claim.supportStatus === "direct") + .flatMap((claim) => claim.supportingChunkIds) + .filter((chunkId) => servedChunkIds.has(chunkId)), + ); + const match = differences.find((entry) => entry.waChunkIds.some((chunkId) => directChunkIds.has(chunkId))); + return match ? noteFor(match) : undefined; +} + +/** "Example National Guideline, 2023, p 11": the linked part of the line, shared by screen and clipboard. */ +export function guidanceDifferenceLabel(note: Pick) { + return [note.title, String(note.year), note.page ? `p ${note.page}` : null].filter(Boolean).join(", "); +} + +/** The whole line as plain text, for copied answers. */ +export function guidanceDifferenceSentence(note: Pick) { + return `National guidance differs on this point. See ${guidanceDifferenceLabel(note)}.`; +} diff --git a/src/lib/rag/rag-document-summary-context.ts b/src/lib/rag/rag-document-summary-context.ts index bda3bc58c6..24368eeed3 100644 --- a/src/lib/rag/rag-document-summary-context.ts +++ b/src/lib/rag/rag-document-summary-context.ts @@ -1,5 +1,7 @@ +import { excludeSupersededEvidence } from "@/lib/rag/superseded-evidence"; import { committedIndexGeneration } from "@/lib/reindex-pipeline"; import { createAdminClient } from "@/lib/supabase/admin"; +import type { RagAnswer, SearchResult } from "@/lib/types"; function throwIfAborted(signal?: AbortSignal) { if (signal?.aborted) { @@ -7,6 +9,19 @@ function throwIfAborted(signal?: AbortSignal) { } } +/** + * The refusal for a document that cannot be summarised, or null when it can. A superseded document + * is never used as evidence, including when it is summarised directly (#BFCTST). + */ +export function documentSummaryUnavailableAnswer(results: SearchResult[]): RagAnswer | null { + const answer = !results.length + ? "This document has not been indexed yet, so no summary can be generated." + : excludeSupersededEvidence(results).length === 0 + ? "This document is marked as superseded, so it is not summarised as clinical guidance. Use its current replacement instead." + : null; + return answer ? { answer, grounded: false, confidence: "unsupported", citations: [], sources: [] } : null; +} + export async function loadDocumentSummaryContext(documentId: string, ownerId?: string, signal?: AbortSignal) { throwIfAborted(signal); const supabase = createAdminClient(); diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 003d53a6c2..96482da6cd 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -24,7 +24,11 @@ import { withRagProgrammeRollout, answerContractForRollout, adaptiveAnswerRender import { loadReviewedPolicyRequest } from "@/lib/rag/rag-reviewed-policy-input"; import { createGenerationDegradationRecorder } from "@/lib/rag/rag-generation-degradation"; import { createAdminClient } from "@/lib/supabase/admin"; -import { loadDocumentSummaryContext, sanitizeUnsupportedSummaryAnswer } from "@/lib/rag/rag-document-summary-context"; +import { + documentSummaryUnavailableAnswer, + loadDocumentSummaryContext, + sanitizeUnsupportedSummaryAnswer, +} from "@/lib/rag/rag-document-summary-context"; import { generationFailureDetailToken } from "@/lib/rag/rag-generation-failure-diagnostics"; import { answerLatencyMetadata, answerScopedEvidenceMetadata } from "@/lib/rag/rag-answer-telemetry-metadata"; import { assertRetrievalRows, buildDocumentSummaryResults } from "@/lib/rag/rag-row-contracts"; @@ -99,6 +103,7 @@ import { } from "@/lib/answer-preview"; export { applyNumericVerification, unboldUnverifiedNumbers } from "@/lib/answer-verification"; import { selectModelContextEvidencePair, summarizeAustralianSourceSelection } from "@/lib/rag/rag-context-selection"; +import { excludeSupersededEvidence } from "@/lib/rag/superseded-evidence"; import { relatedInformationMenuLine } from "@/lib/rag/answer-composition"; export { capPerDocumentCrowding, @@ -1546,13 +1551,17 @@ async function searchChunksWithTiming( if (textData.length) { const rerankElapsedMs = startRerankClock(searchTiming); - const textCandidates = await measureSearchPhase(searchTiming, "metadata_hydration", () => - attachDocumentRankingMetadata( - supabase, - textData as SearchResult[], - args.ownerId, - documentRankingMetadataCache, - args.signal, + // Superseded rows leave before any fast-path decision, so an outdated lexical match cannot + // short-circuit the lanes that would find the current guidance (#BFCTST). + const textCandidates = excludeSupersededEvidence( + await measureSearchPhase(searchTiming, "metadata_hydration", () => + attachDocumentRankingMetadata( + supabase, + textData as SearchResult[], + args.ownerId, + documentRankingMetadataCache, + args.signal, + ), ), ); expandedQuery = expandClinicalQueryWithCandidateMetadata(args.query, expandedQuery, textCandidates); @@ -1611,7 +1620,7 @@ async function searchChunksWithTiming( textFastResults = selectRankedRetrievalResults({ query: retrievalQuery, queryClass: queryClassification.queryClass, - candidates: memoryBoost.results, + candidates: excludeSupersededEvidence(memoryBoost.results), topK: args.topK ?? 8, maxResultsPerDocument, telemetry, @@ -1710,7 +1719,7 @@ async function searchChunksWithTiming( selectRankedRetrievalResults({ query: retrievalQuery, queryClass: queryClassification.queryClass, - candidates: memoryBoost.results, + candidates: excludeSupersededEvidence(memoryBoost.results), topK: args.topK ?? 8, maxResultsPerDocument, telemetry, @@ -1977,7 +1986,7 @@ async function searchChunksWithTiming( selectRankedRetrievalResults({ query: retrievalQuery, queryClass: queryClassification.queryClass, - candidates: memoryBoost.results, + candidates: excludeSupersededEvidence(memoryBoost.results), topK: args.topK ?? 8, maxResultsPerDocument, telemetry, @@ -2071,7 +2080,7 @@ async function searchChunksWithTiming( selectRankedRetrievalResults({ query: retrievalQuery, queryClass: queryClassification.queryClass, - candidates: memoryBoost.results, + candidates: excludeSupersededEvidence(memoryBoost.results), topK: args.topK ?? 8, maxResultsPerDocument, telemetry, @@ -2352,7 +2361,7 @@ async function answerQuestionWithScopeUncoalesced( const queryAnalysis = analyzeClinicalQuery(answerFocusQuery); if (queryClassForClinicalMode(args.queryMode ?? "auto")) queryAnalysis.queryClass = queryClass; const answerRanking = rankAnswerEvidence(answerFocusQuery, normalizeSearchResults(search.results), queryClass); - const results = annotateSearchResults(answerFocusQuery, answerRanking.rankedResults); + const results = excludeSupersededEvidence(annotateSearchResults(answerFocusQuery, answerRanking.rankedResults)); const { crossDocumentPlan, eligibleResults: eligibleAnswerResults, @@ -4279,20 +4288,9 @@ export async function summarizeDocument( options?: { signal?: AbortSignal; observationContext?: import("@/lib/rag/rag-contracts").RagObservationContext }, ) { const { document, chunks } = await loadDocumentSummaryContext(documentId, ownerId, options?.signal); - if (!chunks.length) { - return observeRagAnswer( - { - answer: "This document has not been indexed yet, so no summary can be generated.", - grounded: false, - confidence: "unsupported", - citations: [], - sources: [], - } satisfies RagAnswer, - options?.observationContext, - ); - } - const results = buildDocumentSummaryResults(chunks, document); + const unavailableAnswer = documentSummaryUnavailableAnswer(results); + if (unavailableAnswer) return observeRagAnswer(unavailableAnswer, options?.observationContext); const summaryInstructions = `Summarize a clinical document for practical psychiatric use in Perth, Australia. Use only the excerpts provided. Use a layered response: make the answer field a plain high-yield clinical paragraph, diff --git a/src/lib/rag/superseded-evidence.ts b/src/lib/rag/superseded-evidence.ts new file mode 100644 index 0000000000..bf23d928d4 --- /dev/null +++ b/src/lib/rag/superseded-evidence.ts @@ -0,0 +1,12 @@ +import type { SearchResult } from "@/lib/types"; + +/** + * Owner rule (Josh, 2026-09-25, ledger #BFCTST): a superseded source is never used as evidence. + * A document marked "outdated" has been replaced, so it is removed before any answer route + * (extractive or generated) sees it. "review_due" and "unknown" sources stay: they are still the + * current version and keep their own governance warnings. When only outdated sources matched, the + * answer path receives no evidence and degrades to its existing evidence-gap behaviour. + */ +export function excludeSupersededEvidence>(results: T[]): T[] { + return results.filter((result) => result.source_metadata?.document_status !== "outdated"); +} diff --git a/tests/australian-source-priority.test.ts b/tests/australian-source-priority.test.ts index c2d2bfe58d..66fd189904 100644 --- a/tests/australian-source-priority.test.ts +++ b/tests/australian-source-priority.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { selectAustralianClinicalContext } from "../src/lib/australian-source-priority"; +import { + australianSourceTier, + selectAustralianClinicalContext, + summarizeAustralianSourceSelection, +} from "../src/lib/australian-source-priority"; import type { ClinicalSourceMetadata, SearchResult } from "../src/lib/types"; function relevance(verdict: "direct" | "partial") { @@ -181,3 +185,92 @@ describe("selectAustralianClinicalContext role eligibility", () => { ).toEqual(["wa-direct-first", "wa-direct-second", "state-direct", "wa-partial"]); }); }); + +// Owner tier rule (Josh, 2026-09-25, ledger #BFCTST): WA validated, then Australian national, then +// other Australian states, then supplementary, applied only inside a relevance band. These tests pin +// the rule itself so a change to the registry or the comparator cannot reorder tiers unnoticed. +describe("owner source tier rule (#BFCTST)", () => { + const wa = (id: string, verdict: "direct" | "partial" = "direct") => + australianResult({ id, verdict, publisher: "WA Health", publisherCode: "WAHEALTH", jurisdiction: "Australia/WA" }); + const national = (id: string, verdict: "direct" | "partial" = "direct") => + australianResult({ + id, + verdict, + publisher: "Australian Commission on Safety and Quality in Health Care", + publisherCode: "ACSQHC", + jurisdiction: "Australia", + }); + const state = (id: string, verdict: "direct" | "partial" = "direct") => + australianResult({ + id, + verdict, + publisher: "NSW Health", + publisherCode: "NSWHEALTH", + jurisdiction: "Australia/NSW", + }); + const supplementary = (id: string, verdict: "direct" | "partial" = "direct") => + australianResult({ + id, + verdict, + publisher: "Substance Abuse and Mental Health Services Administration", + publisherCode: "SAMHSA", + jurisdiction: "United States", + }); + const order = (results: SearchResult[]) => + selectAustralianClinicalContext(results, { omitSupplementaryPadding: false, limit: 10 }).map((item) => item.id); + + it("classifies each sample publisher into the tier the rule names", () => { + expect(australianSourceTier(wa("wa"))).toBe("wa_validated"); + expect(australianSourceTier(national("national"))).toBe("australian_national"); + expect(australianSourceTier(state("state"))).toBe("australian_state"); + expect(australianSourceTier(supplementary("supplementary"))).toBe("supplementary"); + }); + + it("puts WA ahead of national, national ahead of other states, and supplementary last at equal relevance", () => { + expect(order([supplementary("supplementary"), state("state"), national("national"), wa("wa")])).toEqual([ + "wa", + "national", + "state", + "supplementary", + ]); + }); + + it("never lets a tier outrank a more relevant source", () => { + expect(order([wa("wa-partial", "partial"), national("national-direct")])).toEqual([ + "national-direct", + "wa-partial", + ]); + expect(order([national("national-partial", "partial"), state("state-direct")])).toEqual([ + "state-direct", + "national-partial", + ]); + }); + + it("drops supplementary padding once two Australian documents carry enough equally relevant evidence", () => { + const selected = selectAustralianClinicalContext( + [supplementary("supplementary"), wa("wa-1"), wa("wa-2"), national("national-1"), national("national-2")], + { limit: 10 }, + ).map((item) => item.id); + expect(selected).toEqual(["wa-1", "wa-2", "national-1", "national-2"]); + }); + + it("keeps supplementary evidence when Australian evidence is not sufficient on its own", () => { + const selected = selectAustralianClinicalContext([supplementary("supplementary"), wa("wa-1")], { limit: 10 }).map( + (item) => item.id, + ); + expect(selected).toEqual(["wa-1", "supplementary"]); + }); + + it("counts WA and supplementary documents in the selection summary", () => { + const candidates = [wa("wa-1"), national("national-1"), supplementary("supplementary")]; + const summary = summarizeAustralianSourceSelection(candidates, candidates.slice(0, 2)); + expect(summary).toMatchObject({ + candidateCount: 3, + selectedCount: 2, + australianCandidateCount: 2, + waCandidateCount: 1, + waSelectedCount: 1, + usedSupplementaryFallback: false, + }); + }); +}); diff --git a/tests/fixtures/kemh-lithium-breastfeeding-live-excerpt.json b/tests/fixtures/kemh-lithium-breastfeeding-live-excerpt.json new file mode 100644 index 0000000000..ddda3a226d --- /dev/null +++ b/tests/fixtures/kemh-lithium-breastfeeding-live-excerpt.json @@ -0,0 +1,21 @@ +{ + "description": "Public indexed passage captured read-only from the live index on 2026-10-21 (owner-approved free read) for the high-risk case breastfeeding-lithium (#ZZ4RAP). Original PDF line wrapping retained. Capture is evidence text, not a new clinical approval or currency attestation.", + "sources": [ + { + "id": "56760e1e-774c-43d6-a6e2-4d239c7e9f82", + "document_id": "64d94538-2e3f-417b-aeef-2915b1958853", + "title": "Women S And Perinatal Mental Health Referral And Management Guideline(KEMH)", + "file_name": "Women's and Perinatal Mental Health Referral and Management Guideline (KEMH).pdf", + "page_number": 28, + "content": "Drug class Risks in pregnancy* Risks while breastfeeding* Recommendations\n\nexposure and an increased risk of risk of infant toxicity as well lithium during pregnancy\ncongenital malformations. Importantly, the as additional risks from Breastfeeding while taking lithium is not\nrisk estimates from these studies are lower lithium exposure for thyroid recommended\nthan previously reported. Risk of Ebstein’s function, renal function and\nAnomaly 0.05-0.1% neurological side effects When lithium treatment must be continued, focus\non ways to support mothers to feel confident and\nLittle is known about the association with positive about parenting their baby, and to\npreterm birth and birthweight, and experience the closeness of the mother-infant\ndevelopmental consequences of bond without breastfeeding\nintrauterine exposure to lithium\nThere is an association between lithium\nuse and a range of pregnancy or delivery-\nrelated outcomes, but more research is\nneeded" + }, + { + "id": "3846321f-ae8a-4c88-983f-eea69ae3aa49", + "document_id": "64d94538-2e3f-417b-aeef-2915b1958853", + "title": "Women S And Perinatal Mental Health Referral And Management Guideline(KEMH)", + "file_name": "Women's and Perinatal Mental Health Referral and Management Guideline (KEMH).pdf", + "page_number": 27, + "content": "Lithium Recent large studies have confirmed the Potential for high passage of See Monitoring During Pregnancy\nassociation between first trimester lithium lithium into breastmilk and See Specific delivery planning for women using" + } + ] +} diff --git a/tests/guidance-difference-note.dom.test.tsx b/tests/guidance-difference-note.dom.test.tsx new file mode 100644 index 0000000000..758766bce9 --- /dev/null +++ b/tests/guidance-difference-note.dom.test.tsx @@ -0,0 +1,171 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/components/clinical-dashboard/signed-image", () => ({ + SignedImage: ({ caption }: { caption?: string }) =>

{caption}

, +})); + +import { NaturalLanguageAnswer } from "@/components/clinical-dashboard/answer-content"; +import { buildAnswerClipboardText } from "@/components/clinical-dashboard/answer-copy-payload"; +import { clientAnswerFieldsSchema } from "@/lib/answer-client-fields"; +import { + activeReviewedGuidanceDifferences, + guidanceDifferenceNoteForAnswer, + type ReviewedGuidanceDifference, +} from "@/lib/rag/guidance-difference-note"; +import registry from "@/data/reviewed-guidance-differences.json"; +import type { RagAnswer, SearchResult } from "@/lib/types"; + +/** + * Owner decision 2026-10-21: when WA and national guidance differ, the answer follows WA and one + * plain line names the national source. The line only comes from an owner-signed entry. + * All identifiers and titles below are synthetic. + */ + +const ENTRY: ReviewedGuidanceDifference = { + id: "synthetic-renal-interval", + topic: "renal monitoring interval", + waDocumentId: "doc-wa", + waChunkIds: ["chunk-wa-1"], + national: { + documentId: "doc-national", + chunkId: "chunk-national-4", + title: "Example National Guideline", + publisher: "Example national body", + year: 2023, + page: 11, + }, + signedOffBy: "owner", + signedOffAt: "2026-10-21", + reviewBy: "2099-12-31", +}; + +function answer(overrides: Partial> = {}) { + return { + grounded: true, + sources: [{ id: "chunk-wa-1" }, { id: "chunk-other" }] as SearchResult[], + supportedClaims: [ + { + claimId: "c1", + text: "Check renal function.", + riskClass: "routine", + supportingChunkIds: ["chunk-wa-1"], + supportStatus: "direct", + }, + ], + ...overrides, + } as Pick; +} + +describe("guidance difference note", () => { + it("ships with no signed-off differences, so no answer changes until the owner adds one", () => { + expect(registry.differences).toEqual([]); + expect(guidanceDifferenceNoteForAnswer(answer())).toBeUndefined(); + }); + + it("attaches the national source when a signed-off WA chunk directly supports the answer", () => { + expect(guidanceDifferenceNoteForAnswer(answer(), [ENTRY])).toEqual({ + differenceId: "synthetic-renal-interval", + title: "Example National Guideline", + year: 2023, + page: 11, + documentId: "doc-national", + chunkId: "chunk-national-4", + }); + }); + + it("stays silent for partial support, ungrounded answers and chunks the answer did not serve", () => { + const partial = answer({ + supportedClaims: [ + { + claimId: "c1", + text: "x", + riskClass: "routine", + supportingChunkIds: ["chunk-wa-1"], + supportStatus: "partial", + }, + ], + }); + expect(guidanceDifferenceNoteForAnswer(partial, [ENTRY])).toBeUndefined(); + expect(guidanceDifferenceNoteForAnswer(answer({ grounded: false }), [ENTRY])).toBeUndefined(); + expect(guidanceDifferenceNoteForAnswer(answer({ sources: [] }), [ENTRY])).toBeUndefined(); + }); + + it("drops expired, unsigned and malformed entries", () => { + const now = new Date("2027-01-01T00:00:00Z"); + expect( + activeReviewedGuidanceDifferences( + [ + { ...ENTRY, id: "expired", reviewBy: "2026-12-31" }, + { ...ENTRY, id: "unsigned", signedOffBy: "agent" }, + { ...ENTRY, id: "same-document", national: { ...ENTRY.national, documentId: "doc-wa" } }, + { ...ENTRY, id: "no-chunks", waChunkIds: [] }, + { ...ENTRY, id: "year-typo", national: { ...ENTRY.national, year: 20233 } }, + { ...ENTRY, id: "long-title", national: { ...ENTRY.national, title: "x".repeat(301) } }, + ENTRY, + ], + now, + ).map((entry) => entry.id), + ).toEqual(["synthetic-renal-interval"]); + }); + + it("passes the client field boundary", () => { + const note = guidanceDifferenceNoteForAnswer(answer(), [ENTRY]); + expect(clientAnswerFieldsSchema.safeParse({ guidanceDifference: note }).success).toBe(true); + expect(clientAnswerFieldsSchema.safeParse({ guidanceDifference: { ...note, title: "" } }).success).toBe(false); + }); + + function renderAnswer(props: { isRefusal?: boolean; withNote?: boolean }) { + return render( + , + ); + } + + it("renders one plain line under the answer, linking to the national source page", () => { + renderAnswer({}); + const line = screen.getByTestId("answer-guidance-difference"); + expect(line.textContent).toBe( + "National guidance differs on this point. See Example National Guideline, 2023, p 11.", + ); + const link = screen.getByRole("link", { name: "Example National Guideline, 2023, p 11" }); + expect(link.getAttribute("href")).toBe("/documents/doc-national?page=11&chunk=chunk-national-4"); + expect(screen.getByTestId("plain-answer-prose").closest("p")?.nextElementSibling).toBe(line); + }); + + it("shows nothing without a note or on a refusal", () => { + const { unmount } = renderAnswer({ withNote: false }); + expect(screen.queryByTestId("answer-guidance-difference")).toBeNull(); + unmount(); + renderAnswer({ isRefusal: true }); + expect(screen.queryByTestId("answer-guidance-difference")).toBeNull(); + }); + + it("keeps the line in a copied answer, directly after the answer text", () => { + const note = guidanceDifferenceNoteForAnswer(answer(), [ENTRY]); + const copied = buildAnswerClipboardText({ + answer: { + answer: "Check renal function at baseline and six weeks after starting.", + grounded: true, + confidence: "high", + citations: [], + sources: [], + guidanceDifference: note, + }, + renderCopyText: + "Clinical answer draft\n\nAnswer\nCheck renal function at baseline and six weeks after starting.\n\nSources\n1. WA guideline", + }); + expect(copied).toContain( + "Check renal function at baseline and six weeks after starting.\n\nNational guidance differs on this point. See Example National Guideline, 2023, p 11.\n\nSources", + ); + }); +}); diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index b0705320fa..7aa76e72d3 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -2866,9 +2866,9 @@ describe("RAG structured-output fallback", () => { uploaded_at: null, indexed_at: null, uploaded_by: null, - document_status: "outdated", + document_status: "current", clinical_validation_status: "approved", - extraction_quality: "good", + extraction_quality: "poor", }, table_facts: [ { @@ -5591,9 +5591,9 @@ describe("RAG structured-output fallback", () => { uploaded_at: null, indexed_at: null, uploaded_by: null, - document_status: "outdated", + document_status: "current", clinical_validation_status: "approved", - extraction_quality: "good", + extraction_quality: "poor", }, }); const packedButNotServed = source({ @@ -5691,8 +5691,8 @@ describe("RAG structured-output fallback", () => { cross_document_fusion_source_chunk_ids: [], memory_card_count: 0, memory_top_score: 0, - indexing_extraction_quality: "good", - indexing_stale: true, + indexing_extraction_quality: "poor", + indexing_stale: false, }); expect(JSON.stringify(routeWideLogs[0])).not.toMatch(/POST_GENERATION_(?:MEMORY|INDEX)_SENTINEL/); }); diff --git a/tests/rag-document-summary.test.ts b/tests/rag-document-summary.test.ts index 8324ba8967..9d0541f5eb 100644 --- a/tests/rag-document-summary.test.ts +++ b/tests/rag-document-summary.test.ts @@ -225,6 +225,75 @@ describe("document summary context", () => { expect(answer.evidenceAssessments).toBeUndefined(); }); + it("never summarises a superseded document as clinical guidance (#BFCTST)", async () => { + vi.stubEnv("OPENAI_API_KEY", "test-key"); + const documentId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const documentQuery = { + select: vi.fn().mockReturnThis(), + or: vi.fn().mockReturnThis(), + eq: vi.fn().mockReturnThis(), + abortSignal: vi.fn().mockReturnThis(), + maybeSingle: vi.fn(async () => ({ + data: { + id: documentId, + title: "Superseded guideline", + file_name: "superseded.pdf", + metadata: { + index_generation_id: "generation-current", + document_status: "outdated", + clinical_validation_status: "approved", + extraction_quality: "good", + }, + }, + error: null, + })), + }; + const chunkQuery = { + or: vi.fn().mockReturnThis(), + select: vi.fn().mockReturnThis(), + eq: vi.fn().mockReturnThis(), + order: vi.fn().mockReturnThis(), + limit: vi.fn().mockReturnThis(), + abortSignal: vi.fn(async () => ({ + data: [ + { + id: "superseded-chunk-1", + document_id: documentId, + page_number: 1, + chunk_index: 0, + section_heading: "Dosing", + content: "Superseded dosing text.", + retrieval_synopsis: null, + image_ids: [], + index_generation_id: "generation-current", + }, + ], + error: null, + })), + }; + const from = vi.fn((table: string) => (table === "documents" ? documentQuery : chunkQuery)); + const generateStructuredTextResult = vi.fn(); + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ from, rpc: vi.fn() }), + })); + vi.doMock("@/lib/openai", () => ({ + embedTextWithTelemetry: vi.fn(), + generateStructuredTextResult, + })); + + const { summarizeDocument } = await import("../src/lib/rag/rag"); + const answer = await summarizeDocument(documentId, "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", { + signal: new AbortController().signal, + }); + + expect(chunkQuery.abortSignal).toHaveBeenCalled(); + expect(generateStructuredTextResult).not.toHaveBeenCalled(); + expect(answer.grounded).toBe(false); + expect(answer.confidence).toBe("unsupported"); + expect(answer.sources).toEqual([]); + expect(answer.answer).toMatch(/superseded/i); + }); + it("refuses to summarize non-indexed documents without querying chunks or OpenAI", async () => { vi.stubEnv("OPENAI_API_KEY", "test-key"); const documentId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; diff --git a/tests/rag-superseded-evidence.test.ts b/tests/rag-superseded-evidence.test.ts new file mode 100644 index 0000000000..9fac09f886 --- /dev/null +++ b/tests/rag-superseded-evidence.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; +import { excludeSupersededEvidence } from "../src/lib/rag/superseded-evidence"; +import type { SearchResult } from "../src/lib/types"; + +type Status = NonNullable["document_status"]; +const row = (id: string, status?: Status) => + ({ id, source_metadata: status ? { document_status: status } : undefined }) as unknown as SearchResult; + +describe("superseded sources are never used as evidence (#BFCTST)", () => { + it("removes outdated sources and keeps current, review-due, unknown and unlabelled ones in order", () => { + const kept = excludeSupersededEvidence([ + row("current", "current"), + row("old", "outdated"), + row("review-due", "review_due"), + row("unknown", "unknown"), + row("unlabelled"), + ]).map((result) => result.id); + expect(kept).toEqual(["current", "review-due", "unknown", "unlabelled"]); + }); + + it("leaves no evidence when every match is superseded, so the answer degrades to an evidence gap", () => { + expect(excludeSupersededEvidence([row("old-1", "outdated"), row("old-2", "outdated")])).toEqual([]); + }); +}); diff --git a/tests/registered-public-answer-options.test.ts b/tests/registered-public-answer-options.test.ts new file mode 100644 index 0000000000..3bf5d8c555 --- /dev/null +++ b/tests/registered-public-answer-options.test.ts @@ -0,0 +1,196 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { SearchResult } from "../src/lib/types"; + +// Synthetic coverage for the registered public answer path (audit DOC-18). The planned tests for +// this file depended on a private capture and never landed, so these pin its own gating rules: +// who may get the loader at all, which selected rows may ask for a service-role read, and that +// every failure is the named binding error rather than a silent fallback. + +const mocks = vi.hoisted(() => ({ + createAdminClient: vi.fn(), + resolvePlan: vi.fn(), + readLayout: vi.fn(), +})); + +vi.mock("@/lib/supabase/admin", () => ({ createAdminClient: mocks.createAdminClient })); +vi.mock("@/lib/registered-published-admin-source-plan", () => ({ + createRegisteredPublishedAdminSourcePlanResolver: () => mocks.resolvePlan, +})); +vi.mock("../worker/published-admin-source-layout", () => ({ + createPublishedAdminTextSectionLayoutLoader: () => mocks.readLayout, +})); +vi.mock("@/lib/rag/rag-source-block", () => ({ + publishedOriginalSectionMetadataForResult: (row: SearchResult) => ({ + index_generation_id: (row.source_metadata as Record).index_generation_id, + publication_approval_id: (row.source_metadata as Record).publication_approval_id, + }), + validateCheckedPublishedOriginalSectionLayout: (_request: unknown, projection: unknown) => projection, +})); + +const { registeredPublicAnswerOptions } = await import("../src/lib/registered-public-answer-options"); + +const requesterId = "11111111-1111-4111-8111-111111111111"; +const documentId = "22222222-2222-4222-8222-222222222222"; +const generationId = "33333333-3333-4333-8333-333333333333"; +const approvalId = "44444444-4444-4444-8444-444444444444"; +const publicScope = { ownerId: requesterId, includePublic: true } as never; + +function row(id: string, overrides: Partial = {}, metadata: Record = {}) { + return { + id, + document_id: documentId, + corpus_scope: "australian_public", + source_metadata: { + publication_manifest_version: 2, + index_generation_id: generationId, + publication_approval_id: approvalId, + ...metadata, + }, + ...overrides, + } as unknown as SearchResult; +} + +const rowIds = Array.from({ length: 7 }, (_, index) => `5555555${index}-5555-4555-8555-555555555555`); + +function loader(input: Partial[0]> = {}) { + const options = registeredPublicAnswerOptions({ requesterId, accessScope: publicScope, ...input }); + if (!options) throw new Error("expected registered public answer options"); + return options.loadRegisteredPublishedOriginalSectionLayout; +} + +beforeEach(() => { + mocks.createAdminClient.mockReset().mockReturnValue({ rpc: vi.fn() }); + mocks.resolvePlan.mockReset().mockResolvedValue({ + documentId, + generationId, + publicationApprovalId: approvalId, + sourceExpectation: {}, + sourceSectionLayout: {}, + }); + mocks.readLayout.mockReset().mockResolvedValue({ checked: true }); +}); + +describe("registeredPublicAnswerOptions gating", () => { + it("offers no loader without a requester, without public scope, or in summary mode", () => { + expect(registeredPublicAnswerOptions({ accessScope: publicScope })).toBeUndefined(); + expect( + registeredPublicAnswerOptions({ + requesterId, + accessScope: { ownerId: requesterId, includePublic: false } as never, + }), + ).toBeUndefined(); + expect(registeredPublicAnswerOptions({ requesterId, accessScope: publicScope, summaryMode: true })).toBeUndefined(); + }); + + it("refuses a requester that is not the scope owner or not a UUID", () => { + const otherOwner = { ownerId: "99999999-9999-4999-8999-999999999999", includePublic: true } as never; + expect(() => registeredPublicAnswerOptions({ requesterId, accessScope: otherOwner })).toThrow( + expect.objectContaining({ name: "OriginalSectionLayoutBindingError" }), + ); + expect(() => + registeredPublicAnswerOptions({ + requesterId: "not-a-uuid", + accessScope: { ownerId: "not-a-uuid", includePublic: true } as never, + }), + ).toThrow(expect.objectContaining({ name: "OriginalSectionLayoutBindingError" })); + }); +}); + +describe("registered public section layout loader", () => { + it("keeps ordinary non-V2 rows on baseline behaviour with no service-role read", async () => { + const result = await loader()({ + selectedResults: [row(rowIds[0], {}, { publication_manifest_version: 1 })], + } as never); + expect(result).toBeNull(); + expect(mocks.createAdminClient).not.toHaveBeenCalled(); + }); + + it("returns the checked layout when registration and every selected row agree", async () => { + const result = await loader({ explicitDocumentIds: [documentId] })({ + selectedResults: [row(rowIds[0]), row(rowIds[1])], + } as never); + expect(result).toEqual({ checked: true }); + expect(mocks.resolvePlan).toHaveBeenCalledWith(expect.objectContaining({ requesterId, documentId })); + expect(mocks.readLayout).toHaveBeenCalledWith( + expect.objectContaining({ + requesterId, + documentId, + expectedIndexGenerationId: generationId, + publicationApprovalId: approvalId, + }), + ); + }); + + const bindingFailures: Array< + [string, () => Parameters>[0], Parameters[0]?] + > = [ + ["more than six selected rows", () => ({ selectedResults: rowIds.map((id) => row(id)) }) as never], + [ + "a document outside the explicit document filter", + () => ({ selectedResults: [row(rowIds[0])] }) as never, + { explicitDocumentIds: ["66666666-6666-4666-8666-666666666666"] }, + ], + [ + "a document outside the resolved document set", + () => ({ selectedResults: [row(rowIds[0])] }) as never, + { resolvedDocumentIds: [] }, + ], + [ + "rows from two documents", + () => + ({ + selectedResults: [row(rowIds[0]), row(rowIds[1], { document_id: "77777777-7777-4777-8777-777777777777" })], + }) as never, + ], + ["a repeated row", () => ({ selectedResults: [row(rowIds[0]), row(rowIds[0])] }) as never], + [ + "a row from a non-public corpus", + () => ({ selectedResults: [row(rowIds[0], { corpus_scope: "uploaded_local" } as never)] }) as never, + ], + [ + "rows from two index generations", + () => + ({ + selectedResults: [ + row(rowIds[0]), + row(rowIds[1], {}, { index_generation_id: "88888888-8888-4888-8888-888888888888" }), + ], + }) as never, + ], + ]; + + it.each(bindingFailures)("fails with the named binding error for %s", async (_label, request, input) => { + await expect(loader(input)(request())).rejects.toMatchObject({ name: "OriginalSectionLayoutBindingError" }); + expect(mocks.readLayout).not.toHaveBeenCalled(); + }); + + it("fails closed when the registration plan disagrees with the selected rows", async () => { + mocks.resolvePlan.mockResolvedValue({ + documentId, + generationId: "88888888-8888-4888-8888-888888888888", + publicationApprovalId: approvalId, + sourceExpectation: {}, + sourceSectionLayout: {}, + }); + await expect(loader()({ selectedResults: [row(rowIds[0])] } as never)).rejects.toMatchObject({ + name: "OriginalSectionLayoutBindingError", + }); + expect(mocks.readLayout).not.toHaveBeenCalled(); + }); + + it("turns a supplier error into the named binding error", async () => { + mocks.readLayout.mockRejectedValue(new Error("supplier unavailable")); + await expect(loader()({ selectedResults: [row(rowIds[0])] } as never)).rejects.toMatchObject({ + name: "OriginalSectionLayoutBindingError", + }); + }); + + it("reports caller cancellation as cancellation, not as a binding failure", async () => { + const controller = new AbortController(); + controller.abort(new DOMException("caller left", "AbortError")); + await expect( + loader({ signal: controller.signal })({ selectedResults: [row(rowIds[0])] } as never), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(mocks.createAdminClient).not.toHaveBeenCalled(); + }); +});