diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index e2cd5827d7..17092fdd86 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -31,6 +31,15 @@ import type { AnswerRouteMode } from "@/lib/rag/rag-routing"; import type { RagAnswer } from "@/lib/types"; import { buildRagDiagnosticDumpRecord, resolveLocalDiagnosticOutputPath } from "./eval-answer-quality"; import { isDirectEntrypoint } from "./lib/is-entrypoint.mjs"; +import { + blockingGateRetryCaseIds, + finalBlockingFailures, + latencySamplesForRetry, + mergeRetriedRagResults, + renderGateRetryMarkdown, + summarizeGateRetry, + type GateRetrySummary, +} from "./lib/eval-quality-retry"; loadEnvConfig(process.cwd()); @@ -49,6 +58,8 @@ type EvalQualityArgs = { sourceMetadataDebt?: string; json: boolean; failOnThreshold: boolean; + /** Re-run cases implicated in a blocking RAG gate once before blocking (default on). */ + gateRetry: boolean; retrievalOnly: boolean; ragOnly: boolean; skipPreflight: boolean; @@ -320,6 +331,7 @@ export function parseEvalQualityArgs(argv: string[]): EvalQualityArgs { outputDir: join(process.cwd(), "output", "evals"), json: false, failOnThreshold: false, + gateRetry: true, retrievalOnly: false, ragOnly: false, skipPreflight: false, @@ -339,6 +351,10 @@ export function parseEvalQualityArgs(argv: string[]): EvalQualityArgs { args.failOnThreshold = true; continue; } + if (token === "--no-gate-retry") { + args.gateRetry = false; + continue; + } if (token === "--retrieval-only") { args.retrievalOnly = true; continue; @@ -630,10 +646,27 @@ function topResultGovernanceCounts(results: GoldenRetrievalResult[]) { }; } +/** A first attempt replaced by its retry: still counted in latency populations and run cost. */ +export type SupersededAttemptSample = { + route: string; + latencyMs: number; + estimatedCostUsd: number | null; + /** A first attempt over its route ceiling still counts against the ceiling gate. */ + routeCeilingExceeded?: boolean; +}; + +export function isGroundedSupported(result: RagQualityResult, providerMode: EvalQualityProviderMode) { + if (providerMode === "offline") { + return result.model === null && result.expectedHit && result.citations > 0; + } + return result.grounded || (result.acceptSourceOnly && result.expectedHit && result.citations > 0); +} + function summarizeRagQualityResults( results: RagQualityResult[], providerMode: EvalQualityProviderMode, allowanceOverride?: ReadonlyArray<{ id: string; reason: string }>, + additionalLatencySamples: ReadonlyArray = [], ) { const fallbackAllowance = allowanceOverride ?? qualityThresholds.ragSourceBackedReviewFallbackAllowance; const supported = results.filter((result) => result.supported); @@ -644,12 +677,7 @@ function summarizeRagQualityResults( // Requiring expectedHit keeps the guard honest: a real retrieval regression that // stops surfacing the expected docs is NOT accepted and still drags the rate below // threshold, hard-failing the canary. - const groundedSupported = supported.filter((result) => { - if (providerMode === "offline") { - return result.model === null && result.expectedHit && result.citations > 0; - } - return result.grounded || (result.acceptSourceOnly && result.expectedHit && result.citations > 0); - }).length; + const groundedSupported = supported.filter((result) => isGroundedSupported(result, providerMode)).length; const unsupportedCorrect = unsupported.filter((result) => !result.grounded).length; const citationFailures = results.filter((result) => result.failures.some((failure) => qualityFailureCategory(failure) === "citation"), @@ -667,7 +695,9 @@ function summarizeRagQualityResults( const expectedDangerWarningMissing = results.filter((result) => result.failures.includes("expected danger source governance warning missing"), ).length; - const routeCeilingFailures = results.filter((result) => result.routeCeilingExceeded).length; + const routeCeilingFailures = + results.filter((result) => result.routeCeilingExceeded).length + + additionalLatencySamples.filter((sample) => sample.routeCeilingExceeded).length; const generationFallbacks = results.filter((result) => /(?:^|;\s*)generation_fallback:/i.test(result.routingReason ?? ""), ).length; @@ -693,20 +723,32 @@ function summarizeRagQualityResults( const comparisonSourceExtractiveFallbacks = results.filter((result) => /(?:^|;\s*)comparison_source_extractive_fallback(?:;|$)/i.test(result.routingReason ?? ""), ).length; - const latencies = results.map((result) => result.latencyMs); + const latencies = [ + ...results.map((result) => result.latencyMs), + ...additionalLatencySamples.map((sample) => sample.latencyMs), + ]; const routeLatencyP95 = Object.fromEntries( Array.from( - results.reduce((accumulator, result) => { - const current = accumulator.get(result.latencyRoute) ?? []; - current.push(result.latencyMs); - accumulator.set(result.latencyRoute, current); + [ + ...results.map((result) => ({ route: result.latencyRoute, latencyMs: result.latencyMs })), + ...additionalLatencySamples, + ].reduce((accumulator, sample) => { + // Buckets come from final results AND retained samples, so a route whose only + // case moved route on retry still has its slow first attempt gated. + const current = accumulator.get(sample.route) ?? []; + current.push(sample.latencyMs); + accumulator.set(sample.route, current); return accumulator; }, new Map()), ).map(([route, routeLatencies]) => [route, percentile(routeLatencies, 95)]), ); - const estimatedCostUsd = results.some((result) => result.estimatedCostUsd === null) + const costSamples = [ + ...results.map((result) => result.estimatedCostUsd), + ...additionalLatencySamples.map((sample) => sample.estimatedCostUsd), + ]; + const estimatedCostUsd = costSamples.some((cost) => cost === null) ? null - : results.reduce((sum, result) => sum + (result.estimatedCostUsd ?? 0), 0); + : costSamples.reduce((sum, cost) => sum + (cost ?? 0), 0); return { case_count: results.length, @@ -760,6 +802,10 @@ export function buildEvalQualityReport(args: { question_filter: string | null; ranking_config_overridden: boolean; }; + /** Blocking-gate retry receipt (scripts/lib/eval-quality-retry.ts); null when no retry ran. */ + gateRetry?: GateRetrySummary | null; + /** First-attempt latency samples retained when latency-triggered cases are retried. */ + additionalLatencySamples?: ReadonlyArray; }) { const providerMode = args.providerMode ?? "openai"; const retrievalSummary = summarizeGoldenRetrievalResults(args.retrievalResults); @@ -767,6 +813,7 @@ export function buildEvalQualityReport(args: { args.ragResults, providerMode, args.sourceBackedReviewFallbackAllowance, + args.additionalLatencySamples, ); // `--rag-only` intentionally leaves retrieval metrics and gates empty. The // canary can still supply the preceding golden-retrieval artifact so this @@ -922,6 +969,7 @@ export function buildEvalQualityReport(args: { accepted_threshold_failures: sourceMetadataDebtAcceptance.accepted_failures, blocking_threshold_failures: blockingThresholdFailures, source_metadata_debt_acceptance: sourceMetadataDebtAcceptance, + gate_retry: args.gateRetry ?? null, }; } @@ -1108,7 +1156,7 @@ ${acceptedFailures} All threshold failures: ${failures} - +${renderGateRetryMarkdown(report.gate_retry)} ## Source Metadata Debt Acceptance ${debtAcceptanceRows} @@ -1270,6 +1318,10 @@ async function runRagQualityCases(args: { limit?: number; question?: string; captureDiagnostics?: boolean; + /** Run exactly these case ids (blocking-gate retry); ignores `limit`. */ + caseIds?: readonly string[]; + /** Receives each result as it completes, so a later throw does not lose earlier ones. */ + sink?: { results: RagQualityResult[]; diagnosticCases: Array> }; providerMode: EvalQualityProviderMode; supabase: Awaited>; }) { @@ -1287,10 +1339,12 @@ async function runRagQualityCases(args: { // cases in rag-eval-cases.ts, so a limited run can push registry cases off the // end entirely. `case_count` alone cannot distinguish that from a full run: it // reads 44 either way. Record which IDs ran and whether anything was dropped. - const cases = allCases.slice(0, args.limit ?? allCases.length); - const droppedCaseIds = allCases.slice(cases.length).map((testCase) => testCase.id); - const results: RagQualityResult[] = []; - const diagnosticCases: Array> = []; + const cases = args.caseIds + ? allCases.filter((testCase) => args.caseIds!.includes(testCase.id)) + : allCases.slice(0, args.limit ?? allCases.length); + const droppedCaseIds = args.caseIds ? [] : allCases.slice(cases.length).map((testCase) => testCase.id); + const results: RagQualityResult[] = args.sink?.results ?? []; + const diagnosticCases: Array> = args.sink?.diagnosticCases ?? []; for (const testCase of cases) { const answer = (await withProviderBackoff(`quality-rag:${testCase.id}`, () => @@ -1540,18 +1594,88 @@ async function main() { captureDiagnostics: Boolean(args.dumpRagCases), }); const ragResults = ragRun.results; - const report = buildEvalQualityReport({ - retrievalResults, - sourceGovernanceResults, - ragResults, - sourceMetadataDebtAcceptance, - providerMode: args.providerMode, - // A gate that did not run is not a gate that passed. `--rag-only` and - // `--retrieval-only` leave the other half's results empty, and an empty - // section renders identically to a passing one, so name what was skipped. - skippedComponents: [...(args.ragOnly ? ["retrieval"] : []), ...(args.retrievalOnly ? ["rag-answer-quality"] : [])], - scope: "scope" in ragRun ? ragRun.scope : undefined, - }); + let retryDiagnosticCases: Array> = []; + const reportFor = ( + results: RagQualityResult[], + gateRetry?: GateRetrySummary | null, + additionalLatencySamples?: ReadonlyArray, + ) => + buildEvalQualityReport({ + retrievalResults, + sourceGovernanceResults, + ragResults: results, + sourceMetadataDebtAcceptance, + providerMode: args.providerMode, + // A gate that did not run is not a gate that passed. `--rag-only` and + // `--retrieval-only` leave the other half's results empty, and an empty + // section renders identically to a passing one, so name what was skipped. + skippedComponents: [ + ...(args.ragOnly ? ["retrieval"] : []), + ...(args.retrievalOnly ? ["rag-answer-quality"] : []), + ], + scope: "scope" in ragRun ? ragRun.scope : undefined, + gateRetry, + additionalLatencySamples, + }); + let report = reportFor(ragResults); + if (args.failOnThreshold && args.gateRetry && report.blocking_threshold_failures.length > 0) { + const { caseIds, nonRetryable } = blockingGateRetryCaseIds(report, qualityFailureCategory, args.providerMode); + const initial = report; + const retryRun: { + results: RagQualityResult[]; + diagnosticCases: Array>; + } = { results: [], diagnosticCases: [] }; + let retryError: string | null = null; + try { + if (caseIds.length) { + await runRagQualityCases({ + ...args, + ownerId, + supabase, + caseIds, + captureDiagnostics: Boolean(args.dumpRagCases), + sink: retryRun, + }); + } + } catch (error) { + // A failed rerun must not discard the completed first evaluation or the retries that + // finished before the error: keep them and record the failure in the receipt. + retryError = error instanceof Error ? error.message : String(error); + console.error(`[eval:quality] blocking-gate retry failed; keeping completed attempts: ${retryError}`); + } + const retried = retryRun.results; + retryDiagnosticCases = retryRun.diagnosticCases.map((record) => ({ ...record, attempt: "retry" })); + const additionalLatencySamples = latencySamplesForRetry( + initial, + retried.map((result) => result.id), + ); + const mergedResults = mergeRetriedRagResults(ragResults, retried); + const merged = reportFor(mergedResults, null, additionalLatencySamples); + const retryReceipt = { + ...summarizeGateRetry({ initial, final: merged, retried, nonRetryable }), + attempted: caseIds.length > 0, + retry_error: retryError, + }; + report = reportFor(mergedResults, retryReceipt, additionalLatencySamples); + // Non-retryable first-attempt failures (offline provider use, latency, retrieval) stand: a + // substituted result must never erase them from the final verdict. + const finalFailures = finalBlockingFailures({ + initial: initial.blocking_threshold_failures, + merged: report.blocking_threshold_failures, + nonRetryable, + retryError, + }); + if (finalFailures.length !== report.blocking_threshold_failures.length) { + report = { + ...report, + blocking_threshold_failures: finalFailures, + gate_retry: { ...retryReceipt, final_blocking_threshold_failures: finalFailures }, + }; + } + console.error( + `[eval:quality] blocking-gate retry: re-ran ${retried.length} case(s) [${caseIds.join(", ")}]; blocking before=${initial.blocking_threshold_failures.length} after=${report.blocking_threshold_failures.length}`, + ); + } const paths = await writeReports(report, args.outputDir); if (args.dumpRagCases) { const dumpPath = resolveLocalDiagnosticOutputPath(args.dumpRagCases); @@ -1561,8 +1685,11 @@ async function main() { `${JSON.stringify( { generated_at: new Date().toISOString(), - case_count: ragRun.diagnosticCases.length, - cases: ragRun.diagnosticCases, + case_count: ragRun.diagnosticCases.length + retryDiagnosticCases.length, + cases: [ + ...ragRun.diagnosticCases.map((record) => ({ ...record, attempt: "initial" })), + ...retryDiagnosticCases, + ], }, null, 2, diff --git a/scripts/lib/eval-quality-retry.ts b/scripts/lib/eval-quality-retry.ts new file mode 100644 index 0000000000..bafe6aaa62 --- /dev/null +++ b/scripts/lib/eval-quality-retry.ts @@ -0,0 +1,214 @@ +/** + * Blocking-gate retry for `eval:quality --fail-on-threshold`. + * + * Every Eval Canary red between 2026-09-13 and 2026-10-13 came from a different single-case or + * single-sample gate (citation_failure_rate 0.0227 = one case in 44; one unaccounted review + * fallback; one 6.4 s unsupported retrieval tipping the route p95). Those gates are zero-tolerance + * by design and their values must not move, so instead of loosening them the harness re-runs ONLY + * the cases implicated in a blocking RAG failure, once, and rebuilds the report with the retry + * outcome substituted for each retried case. A gate therefore blocks only when the same case fails + * it again. Latency gates are not retried at all: every first-attempt sample stays in the p95 + * population, so a rerun could never clear one and would only add provider cost. If the retry + * itself errors, the whole first-attempt verdict stands. Thresholds are read, never written. Retrieval and offline-invariant failures are not + * retryable and keep blocking exactly as before. + * + * Pure apart from types, so every branch is testable without a provider call. + */ +import { + isGroundedSupported, + type EvalQualityProviderMode, + type EvalQualityReport, + type RagQualityResult, +} from "../eval-quality"; + +export type GateRetryCase = { + id: string; + initial_failures: string[]; + retry_failures: string[]; + initial_latency_ms: number; + retry_latency_ms: number; + initial_routing_reason: string | null; + retry_routing_reason: string | null; +}; + +export type GateRetrySummary = { + attempted: boolean; + initial_blocking_threshold_failures: string[]; + non_retryable_blocking_failures: string[]; + retried_case_ids: string[]; + cases: GateRetryCase[]; + final_blocking_threshold_failures: string[]; + /** Set when the rerun itself threw; the first-attempt verdict then stands. */ + retry_error?: string | null; +}; + +const p95Pattern = /^RAG p95_latency_ms \S+ above (\d+(?:\.\d+)?)$/; +const routeP95Pattern = /^RAG route (\S+) p95_latency_ms \S+ above (\d+(?:\.\d+)?)$/; + +/** + * The RAG case ids each blocking failure points at. A failure with no case-level owner (retrieval + * gates, offline invariants, anything unrecognised) is returned as non-retryable. + */ +export function blockingGateRetryCaseIds( + report: EvalQualityReport, + failureCategory: (message: string) => string, + providerMode: EvalQualityProviderMode = "openai", +) { + const results = report.rag.results as RagQualityResult[]; + const unaccounted = new Set( + report.rag.summary.source_backed_review_fallback_unaccounted.map((entry: string) => entry.split(" (")[0]), + ); + const ids = new Set(); + const nonRetryable: string[] = []; + const add = (matches: RagQualityResult[]) => matches.forEach((result) => ids.add(result.id)); + + for (const failure of report.blocking_threshold_failures) { + const routeP95 = routeP95Pattern.exec(failure); + const p95 = p95Pattern.exec(failure); + if (routeP95 || p95) { + // Latency gates keep every first-attempt sample (latencySamplesForRetry), so a rerun can + // never clear them; re-running would only add provider cost. They stand as-is. + nonRetryable.push(failure); + } else if (failure.startsWith("RAG citation_failure_rate")) { + add(results.filter((result) => result.failures.some((item) => failureCategory(item) === "citation"))); + } else if (failure.startsWith("RAG source_backed_review_fallback unaccounted")) { + add(results.filter((result) => unaccounted.has(result.id))); + } else if (failure.startsWith("RAG route_ceiling_failure_count")) { + // Every retried case keeps its first-attempt routeCeilingExceeded flag in the count, so a + // rerun can never clear this gate either; retrying would only add provider cost. + nonRetryable.push(failure); + } else if (failure.startsWith("RAG grounded_supported_rate")) { + add(results.filter((result) => result.supported && !isGroundedSupported(result, providerMode))); + } else if (failure.startsWith("RAG unsupported_correct_rate")) { + add(results.filter((result) => !result.supported && result.grounded)); + } else if (failure.startsWith("RAG numeric_grounding_failure_rate")) { + add( + results.filter( + (result) => + result.unverifiedNumericTokenCount > 0 || + result.hasFaithfulnessWarning || + result.failures.some((item) => failureCategory(item) === "numeric_grounding"), + ), + ); + } else if (failure.startsWith("RAG source_governance_danger_failure_rate")) { + add(results.filter((result) => result.grounded && result.sourceDangerWarningCount > 0)); + } else if (failure.startsWith("RAG expected_danger_warning_missing_count")) { + add(results.filter((result) => result.failures.includes("expected danger source governance warning missing"))); + } else { + nonRetryable.push(failure); + } + } + return { caseIds: [...ids].sort(), nonRetryable }; +} + +/** + * First attempts superseded by a retry. Every retried case keeps its first-attempt latency in the + * latency-gate population (whichever gate selected it) and its cost in the run total, so a retry + * can never hide a slow first attempt or understate spend. + */ +export function latencySamplesForRetry(report: EvalQualityReport, caseIds: readonly string[]) { + const results = report.rag.results as RagQualityResult[]; + const ids = new Set(caseIds); + return results + .filter((result) => ids.has(result.id)) + .map((result) => ({ + route: result.latencyRoute, + latencyMs: result.latencyMs, + estimatedCostUsd: result.estimatedCostUsd, + routeCeilingExceeded: result.routeCeilingExceeded, + })); +} + +/** Substitute each retried case's single retry result for its first attempt, preserving case order. */ +export function mergeRetriedRagResults(original: RagQualityResult[], retried: RagQualityResult[]) { + const byId = new Map(retried.map((result) => [result.id, result])); + return original.map((result) => byId.get(result.id) ?? result); +} + +export function summarizeGateRetry(args: { + initial: EvalQualityReport; + final: EvalQualityReport; + retried: RagQualityResult[]; + nonRetryable: string[]; +}): GateRetrySummary { + const initialById = new Map((args.initial.rag.results as RagQualityResult[]).map((result) => [result.id, result])); + return { + attempted: args.retried.length > 0, + initial_blocking_threshold_failures: args.initial.blocking_threshold_failures, + non_retryable_blocking_failures: args.nonRetryable, + retried_case_ids: args.retried.map((result) => result.id), + cases: args.retried.map((retry) => { + const first = initialById.get(retry.id); + return { + id: retry.id, + initial_failures: first?.failures ?? [], + retry_failures: retry.failures, + initial_latency_ms: first?.latencyMs ?? 0, + retry_latency_ms: retry.latencyMs, + initial_routing_reason: first?.routingReason ?? null, + retry_routing_reason: retry.routingReason ?? null, + }; + }), + final_blocking_threshold_failures: args.final.blocking_threshold_failures, + }; +} + +export function renderGateRetryMarkdown(retry: GateRetrySummary | null | undefined) { + if (!retry) return ""; + if (!retry.attempted) { + return `\n## Blocking-Gate Retry\n\n${retry.retry_error ? `Retry failed (first-attempt verdict kept): ${retry.retry_error}\n\n` : ""}No retry: ${ + retry.initial_blocking_threshold_failures.length ? "no retryable RAG case was implicated." : "nothing blocked." + }\n`; + } + const list = (items: string[]) => (items.length ? items.map((item) => `- ${item}`).join("\n") : "- None"); + const rows = retry.cases + .map( + (item) => + `| ${item.id} | ${item.initial_failures.join("; ") || "none"} | ${item.retry_failures.join("; ") || "none"} | ${item.initial_latency_ms} | ${item.retry_latency_ms} | ${item.initial_routing_reason ?? "n/a"} | ${item.retry_routing_reason ?? "n/a"} |`, + ) + .join("\n"); + const errorNote = retry.retry_error ? `\nRetry failed (first-attempt verdict kept): ${retry.retry_error}\n` : ""; + return ` +## Blocking-Gate Retry +${errorNote} +Each case implicated in a blocking RAG gate was re-run once. Answer-quality gates were re-evaluated with the retry result in place of the first attempt. Latency gates keep BOTH samples (first attempt and retry), so a slow first attempt still counts and a retry cannot clear a latency gate on its own; estimated cost includes both attempts. Threshold values are unchanged. + +Initial blocking failures (first attempt): + +${list(retry.initial_blocking_threshold_failures)} + +Not retryable (kept as-is): + +${list(retry.non_retryable_blocking_failures)} + +| Case | First-attempt failures | Retry failures | First ms | Retry ms | First routing reason | Retry routing reason | +| --- | --- | --- | ---: | ---: | --- | --- | +${rows} + +Blocking failures after retry: + +${list(retry.final_blocking_threshold_failures)} +`; +} + +/** + * The verdict after a retry. Non-retryable first-attempt failures always stand. A crashed retry is + * incomplete evidence, so then the whole first-attempt verdict stands: a partial set of passing + * retries can never lift a rate gate over its threshold and let the run pass. + */ +export function finalBlockingFailures(args: { + initial: readonly string[]; + merged: readonly string[]; + nonRetryable: readonly string[]; + retryError: string | null; +}): string[] { + // One entry per gate: messages carry the measured value, so match on the gate with numbers masked. + const gate = (failure: string) => failure.replace(/\d+(?:\.\d+)?/g, "#"); + if (args.retryError) { + // The first-attempt value is the verdict; a partial-retry value of the same gate is dropped. + const initialGates = new Set(args.initial.map(gate)); + return [...args.initial, ...args.merged.filter((failure) => !initialGates.has(gate(failure)))]; + } + const mergedGates = new Set(args.merged.map(gate)); + return [...args.merged, ...args.nonRetryable.filter((failure) => !mergedGates.has(gate(failure)))]; +} diff --git a/src/lib/rag/rag-best-practice-prescription-evidence.ts b/src/lib/rag/rag-best-practice-prescription-evidence.ts new file mode 100644 index 0000000000..b16c7a4906 --- /dev/null +++ b/src/lib/rag/rag-best-practice-prescription-evidence.ts @@ -0,0 +1,41 @@ +import type { RagQueryClass, SearchResult } from "@/lib/types"; +import { + bestPracticePrescriptionProfileEvidence, + bestPracticePrescriptionProgramUseEvidence, + isSourceBoundBestPracticePrescriptionRequirementsQuery, +} from "@/lib/rag/rag-extractive-answer"; + +/** + * Give the measured Best Practice Prescription extractive-first builder the two chunks it pairs. + * Answer-evidence selection can fill the fast four-chunk budget with the procedure's legislative and + * monitoring chunks (live canaries 38383361787 and 38419027058 cited only p4 chunks 05a8819b and 212f78ad), + * so the builder found no program-use + medication-profile pair and the query fell through to model + * generation, which the final gate then rejected as ungrounded. When the selected evidence cannot + * form that pair, append at most one governed eligible chunk per role from the same AKG document, + * using the builder's own evidence predicates. Nothing is appended for any other query, and the + * short-circuit still requires the candidate answer to pass every final quality gate. + */ +export function withBestPracticePrescriptionEvidence(args: { + query: string; + queryClass: RagQueryClass; + selected: SearchResult[]; + candidates: readonly SearchResult[]; +}) { + if (!isSourceBoundBestPracticePrescriptionRequirementsQuery(args.query, args.queryClass)) return args.selected; + const selectedIds = new Set(args.selected.map((result) => result.id)); + const pool = [...args.selected, ...args.candidates.filter((result) => !selectedIds.has(result.id))]; + const useSources = pool.filter(bestPracticePrescriptionProgramUseEvidence); + const profileSources = pool.filter(bestPracticePrescriptionProfileEvidence); + const cost = (result: SearchResult) => (selectedIds.has(result.id) ? 0 : 1); + let best: { use: SearchResult; profile: SearchResult; cost: number } | undefined; + for (const use of useSources) { + for (const profile of profileSources) { + if (use.id === profile.id || !use.document_id || use.document_id !== profile.document_id) continue; + const pairCost = cost(use) + cost(profile); + if (!best || pairCost < best.cost) best = { use, profile, cost: pairCost }; + } + } + if (!best || best.cost === 0) return args.selected; + const additions = [best.use, best.profile].filter((result) => !selectedIds.has(result.id)); + return [...args.selected, ...additions]; +} diff --git a/src/lib/rag/rag-extractive-answer.ts b/src/lib/rag/rag-extractive-answer.ts index ce58c28a9c..42796de96d 100644 --- a/src/lib/rag/rag-extractive-answer.ts +++ b/src/lib/rag/rag-extractive-answer.ts @@ -2573,7 +2573,7 @@ function isAkgBestPracticePrescriptionSource(result: SearchResult) { return /\bBest Practice Prescription(?:\s+Program)?\s*\(?AKG\)?\b/i.test(comparisonSourceLabel(result)); } -function bestPracticePrescriptionProgramUseEvidence(result: SearchResult) { +export function bestPracticePrescriptionProgramUseEvidence(result: SearchResult) { if (!isAkgBestPracticePrescriptionSource(result)) return false; const text = boundedSourceText(result); return ( @@ -2586,7 +2586,7 @@ function bestPracticePrescriptionProgramUseEvidence(result: SearchResult) { ); } -function bestPracticePrescriptionProfileEvidence(result: SearchResult) { +export function bestPracticePrescriptionProfileEvidence(result: SearchResult) { if (!isAkgBestPracticePrescriptionSource(result)) return false; const text = boundedSourceText(result); return ( diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 2467b424eb..3ca9a41301 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -113,6 +113,7 @@ import { } from "@/lib/rag/rag-extractive-answer"; import { contextPackAdmissionMatches } from "./rag-context-admission"; import { withAdmissionDischargeSideEvidence } from "./rag-admission-discharge-side-evidence"; +import { withBestPracticePrescriptionEvidence } from "./rag-best-practice-prescription-evidence"; import { chooseValidatedExtractiveShortCircuit, hasValidatedExtractiveCandidate } from "@/lib/rag/rag-extractive-first"; import { buildComparisonMatrix, comparisonEvidenceGuide, selectSafeComparisonFallback } from "@/lib/rag/rag-comparison"; export { @@ -2389,10 +2390,16 @@ async function answerQuestionWithScopeUncoalesced( }); // Same top-up as the generation fallback below, so the source-bound // admission/discharge extractive-first answer can fire before generation. - const extractiveFirstInputResults = withAdmissionDischargeSideEvidence({ + // The measured Best Practice Prescription pair gets the same governed top-up (#3527 pattern). + const extractiveFirstInputResults = withBestPracticePrescriptionEvidence({ query: args.query, queryClass, - selected: answerInputResults, + selected: withAdmissionDischargeSideEvidence({ + query: args.query, + queryClass, + selected: answerInputResults, + candidates: sideEvidenceCandidates, + }), candidates: sideEvidenceCandidates, }); const validatedExtractiveShortCircuit = chooseValidatedExtractiveShortCircuit({ diff --git a/tests/eval-quality-retry.test.ts b/tests/eval-quality-retry.test.ts new file mode 100644 index 0000000000..6b04b5dea5 --- /dev/null +++ b/tests/eval-quality-retry.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from "vitest"; + +import { + buildEvalQualityReport, + qualityFailureCategory, + qualityThresholds, + renderEvalQualityMarkdown, + type RagQualityResult, +} from "../scripts/eval-quality"; +import { + blockingGateRetryCaseIds, + finalBlockingFailures, + latencySamplesForRetry, + mergeRetriedRagResults, + summarizeGateRetry, +} from "../scripts/lib/eval-quality-retry"; + +function ragResult(overrides: Partial = {}): RagQualityResult { + return { + id: "rag-1", + question: "What safety monitoring is required for clozapine?", + category: "routine", + supported: true, + expectedFiles: ["clozapine.pdf"], + matchedFiles: ["clozapine.pdf"], + missingFiles: [], + topFiles: ["clozapine.pdf"], + expectedHit: true, + grounded: true, + latencyMs: 900, + searchLatencyMs: 200, + generationLatencyMs: 650, + rpcLatencyMs: 150, + embeddingLatencyMs: 25, + route: "fast", + executionType: "api", + latencyRoute: "generation", + model: "test-model", + citations: 2, + visualEvidence: 0, + failures: [], + sourceWarningCount: 0, + sourceDangerWarningCount: 0, + unverifiedNumericTokenCount: 0, + hasFaithfulnessWarning: false, + routingReason: "test_route", + timings: { + retrievalMs: 500, + routingMs: 10, + generationMs: 350, + verificationMs: 40, + totalMs: 900, + routeBudgetMs: 25_000, + routeDeadlineExceeded: false, + }, + routeCeilingExceeded: false, + estimatedCostUsd: 0.001, + ...overrides, + }; +} + +const unsupportedLimit = qualityThresholds.ragRouteP95LatencyMs.unsupported; + +function report(results: RagQualityResult[]) { + return buildEvalQualityReport({ retrievalResults: [], ragResults: results, skippedComponents: ["retrieval"] }); +} + +// Shapes of the three reds before this change: 38363682098 (one citation failure in 44), +// 38383361787 (one unaccounted review fallback), 38419027058 (one slow refusal, route p95 6360 > 4000). +const healthy = Array.from({ length: 10 }, (_, index) => ragResult({ id: `ok-${index}` })); +const refusals = Array.from({ length: 4 }, (_, index) => + ragResult({ + id: `refusal-${index}`, + supported: false, + grounded: false, + route: "unsupported", + latencyRoute: "unsupported", + latencyMs: 300, + model: null, + }), +); +const slowRefusal = ragResult({ + id: "unsupported-coffee-machine", + supported: false, + grounded: false, + route: "unsupported", + latencyRoute: "unsupported", + latencyMs: 6360, + model: null, +}); +const citationMiss = ragResult({ id: "citation-miss", failures: ["citation chunk ids missing from sources"] }); +const reviewFallback = ragResult({ + id: "discharge-documentation", + routingReason: + "strong_routine_retrieval; generation_fallback:x; source_backed_review_fallback; extractive_quality_gate:guidance_wrapper_fragment", +}); + +describe("blocking-gate retry", () => { + it("keeps every threshold value untouched", () => { + expect(qualityThresholds.ragCitationFailureRate).toBe(0); + expect(unsupportedLimit).toBe(4_000); + }); + + it("names only the cases behind each blocking RAG failure", () => { + const initial = report([...healthy, ...refusals, slowRefusal, citationMiss, reviewFallback]); + expect(initial.blocking_threshold_failures).toEqual( + expect.arrayContaining([ + expect.stringMatching(/^RAG citation_failure_rate/), + expect.stringMatching(/^RAG source_backed_review_fallback unaccounted/), + expect.stringMatching(/^RAG route unsupported p95_latency_ms 6360 above 4000$/), + ]), + ); + // Latency gates keep the first-attempt sample, so they are not re-run (a retry cannot clear them). + expect(blockingGateRetryCaseIds(initial, qualityFailureCategory)).toEqual({ + caseIds: ["citation-miss", "discharge-documentation"], + nonRetryable: ["RAG route unsupported p95_latency_ms 6360 above 4000"], + }); + }); + + it("keeps the original slow sample when a latency retry passes", () => { + const first = [...healthy, ...refusals, slowRefusal]; + const initial = report(first); + const retried = [{ ...slowRefusal, latencyMs: 280 }]; + const merged = mergeRetriedRagResults(first, retried); + const additionalLatencySamples = latencySamplesForRetry(initial, ["unsupported-coffee-machine"]); + const final = buildEvalQualityReport({ + retrievalResults: [], + ragResults: merged, + skippedComponents: ["retrieval"], + additionalLatencySamples, + }); + expect(final.blocking_threshold_failures).toEqual(["RAG route unsupported p95_latency_ms 6360 above 4000"]); + expect(merged.map((result) => result.id)).toEqual(first.map((result) => result.id)); + + const withRetry = buildEvalQualityReport({ + retrievalResults: [], + ragResults: merged, + skippedComponents: ["retrieval"], + additionalLatencySamples, + gateRetry: summarizeGateRetry({ initial, final, retried, nonRetryable: [] }), + }); + const markdown = renderEvalQualityMarkdown(withRetry); + expect(markdown).toContain("## Blocking-Gate Retry"); + expect(markdown).toContain("RAG route unsupported p95_latency_ms 6360 above 4000"); + expect(markdown).toMatch(/\| unsupported-coffee-machine \| none \| none \| 6360 \| 280 \|/); + expect(withRetry.gate_retry?.final_blocking_threshold_failures).toEqual([ + "RAG route unsupported p95_latency_ms 6360 above 4000", + ]); + }); + + it("gates a route whose only slow first attempt retried onto another route", () => { + const first = [...healthy, ...refusals, slowRefusal]; + const initial = report(first); + const retried = [{ ...slowRefusal, latencyMs: 280, latencyRoute: "moved-route" }]; + const final = buildEvalQualityReport({ + retrievalResults: [], + ragResults: mergeRetriedRagResults(first, retried), + skippedComponents: ["retrieval"], + additionalLatencySamples: latencySamplesForRetry(initial, ["unsupported-coffee-machine"]), + }); + expect(final.blocking_threshold_failures).toContain("RAG route unsupported p95_latency_ms 6360 above 4000"); + }); + + it("keeps a first attempt's route-ceiling breach in the final gate", () => { + const over = ragResult({ id: "ceiling-case", routeCeilingExceeded: true }); + const first = [...healthy, over]; + const initial = report(first); + const final = buildEvalQualityReport({ + retrievalResults: [], + ragResults: mergeRetriedRagResults(first, [{ ...over, routeCeilingExceeded: false }]), + skippedComponents: ["retrieval"], + additionalLatencySamples: latencySamplesForRetry(initial, ["ceiling-case"]), + }); + expect(final.rag.summary.route_ceiling_failure_count).toBe(1); + }); + + it("shows a crashed retry in the markdown instead of 'nothing retryable'", () => { + const initial = report([...healthy, citationMiss]); + const receipt = { + ...summarizeGateRetry({ initial, final: initial, retried: [], nonRetryable: [] }), + attempted: true, + retry_error: "provider exploded", + }; + const markdown = renderEvalQualityMarkdown( + buildEvalQualityReport({ + retrievalResults: [], + ragResults: [...healthy, citationMiss], + skippedComponents: ["retrieval"], + gateRetry: receipt, + }), + ); + expect(markdown).toContain("Retry failed (first-attempt verdict kept): provider exploded"); + }); + + it("still blocks when the same case fails again", () => { + const first = [...healthy, citationMiss]; + const final = report(mergeRetriedRagResults(first, [{ ...citationMiss, latencyMs: 950 }])); + expect(final.blocking_threshold_failures).toEqual([expect.stringMatching(/^RAG citation_failure_rate/)]); + }); + + it("treats failures without a case owner as non-retryable", () => { + const initial = report(healthy); + const forged = { ...initial, blocking_threshold_failures: ["retrieval document_recall_at_5 0.9 below 1"] }; + expect(blockingGateRetryCaseIds(forged, qualityFailureCategory)).toEqual({ + caseIds: [], + nonRetryable: ["retrieval document_recall_at_5 0.9 below 1"], + }); + }); + + it("renders nothing extra when no retry ran", () => { + expect(renderEvalQualityMarkdown(report(healthy))).not.toContain("Blocking-Gate Retry"); + }); +}); + +describe("finalBlockingFailures", () => { + const rate = "RAG grounded_supported_rate 0.9 below 0.95"; + const latency = "RAG p95_latency_ms 6360 above 6000"; + + it("keeps non-retryable failures when the retry completed", () => { + expect( + finalBlockingFailures({ initial: [rate, latency], merged: [], nonRetryable: [latency], retryError: null }), + ).toEqual([latency]); + }); + + it("keeps the whole first-attempt verdict when the retry crashed part-way", () => { + // One retry passed and lifted the rate above its threshold before the next one threw. + expect( + finalBlockingFailures({ initial: [rate], merged: [], nonRetryable: [], retryError: "provider exploded" }), + ).toEqual([rate]); + }); + + it("lists a gate once when a crashed retry left it failing at a different value", () => { + expect( + finalBlockingFailures({ + initial: [rate], + merged: ["RAG grounded_supported_rate 0.92 below 0.95"], + nonRetryable: [], + retryError: "provider exploded", + }), + ).toEqual([rate]); + }); +}); diff --git a/tests/rag-best-practice-prescription-evidence.test.ts b/tests/rag-best-practice-prescription-evidence.test.ts new file mode 100644 index 0000000000..16be528783 --- /dev/null +++ b/tests/rag-best-practice-prescription-evidence.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from "vitest"; + +import { withBestPracticePrescriptionEvidence } from "../src/lib/rag/rag-best-practice-prescription-evidence"; +import { chooseValidatedExtractiveShortCircuit } from "../src/lib/rag/rag-extractive-first"; +import type { SearchResult } from "../src/lib/types"; + +// Live canaries 38383361787 and 38419027058: the fast four-chunk selection carried only the AKG +// procedure's p4 legislative/monitoring chunks, so the measured extractive-first builder never saw +// its program-use + medication-profile pair and the query fell through to model generation. + +const QUERY = "What does the best practice prescription document require?"; +const DOC = "best-practice-prescription-akg"; + +function chunk(overrides: Partial): SearchResult { + return { + id: "chunk", + document_id: DOC, + title: "Best Practice Prescription(AKG)", + file_name: "Best Practice Prescription (AKG).pdf", + page_number: 4, + chunk_index: 0, + section_heading: null, + content: "", + image_ids: [], + similarity: 0.6, + hybrid_score: 0.6, + images: [], + ...overrides, + }; +} + +const legislative = chunk({ + id: "p4-legislative", + content: + "7. Legislative context The following documents are required to give effect to this procedure: Medicines and Poisons Act 2014; Mental Health Act 2014.", +}); +const monitoring = chunk({ + id: "p4-monitoring", + content: + "Monitoring activities include: Quarterly auditing of medical records and Best Practice Prescription Program profiles to identify if staff have complied with history taking and record keeping requirements.", +}); +const otherDocument = chunk({ + id: "co-payment", + document_id: "co-payment-fsh", + title: "Exemption from Patient Prescription Co-Payment(FSH)", + file_name: "Exemption from Patient Prescription Co-Payment (FSH).pdf", + content: "Patients may be exempt from the prescription co-payment when eligibility criteria are documented.", +}); +const programUse = chunk({ + id: "p1-program-use", + page_number: 1, + content: `Best Practice Prescription Program is an electronic program implemented in the AHS outpatient mental health clinics that is used for the following activities: +• Prescription generation (replacing handwritten prescriptions*) +• Maintenance of electronic medication profiles (replacing paper-based profiles) +• Internal and external correspondence generation +• Recording other relevant clinical information (e.g.: allergies/ adverse drug reactions).`, +}); +const profile = chunk({ + id: "p2-profile", + page_number: 2, + content: `The prescriber is responsible for the following: • At the first clinic appointment, medical officers are to inquire with the consumer (or carer) about their current medications and the medical officer or care co-ordinator may need to verify this information with the consumers own medications or a suitable alternative source (e.g.: General Practitioner). +• The details (i.e.: medication name, formulation, route, dose and directions for use) of each psychiatric and non-psychiatric medication must be entered into the Best Practice Prescription Program medication profile.`, +}); + +const selected = [legislative, monitoring, otherDocument]; +const candidates = [legislative, monitoring, otherDocument, programUse, profile]; + +function shortCircuit(results: SearchResult[], query = QUERY) { + return chooseValidatedExtractiveShortCircuit({ + query, + queryClass: "document_lookup", + results, + route: { mode: "fast", reason: "strong_routine_retrieval" }, + sourceBacked: true, + gateStatus: "passed", + }); +} + +describe("withBestPracticePrescriptionEvidence", () => { + it("reproduces the miss: the live p4-only selection cannot short-circuit", () => { + expect(shortCircuit(selected)?.reasonMarker).not.toBe( + "validated_best_practice_prescription_requirements_extractive_first", + ); + }); + + it("appends the same-document program-use and profile chunks so the deterministic answer fires", () => { + const topped = withBestPracticePrescriptionEvidence({ + query: QUERY, + queryClass: "document_lookup", + selected, + candidates, + }); + expect(topped.map((result) => result.id)).toEqual([ + ...selected.map((result) => result.id), + "p1-program-use", + "p2-profile", + ]); + expect(shortCircuit(topped)).toEqual({ + reasonMarker: "validated_best_practice_prescription_requirements_extractive_first", + }); + }); + + it("leaves every other query untouched", () => { + const other = "What does the clozapine prescribing document require?"; + expect( + withBestPracticePrescriptionEvidence({ query: other, queryClass: "document_lookup", selected, candidates }), + ).toBe(selected); + expect( + withBestPracticePrescriptionEvidence({ + query: QUERY, + queryClass: "unsupported_or_general", + selected, + candidates, + }), + ).toBe(selected); + }); + + it("adds nothing when the pair is already selected or cannot be formed from one document", () => { + const complete = [programUse, profile]; + expect( + withBestPracticePrescriptionEvidence({ + query: QUERY, + queryClass: "document_lookup", + selected: complete, + candidates, + }), + ).toBe(complete); + const split = [legislative, programUse, { ...profile, document_id: "different-procedure" }]; + expect( + withBestPracticePrescriptionEvidence({ + query: QUERY, + queryClass: "document_lookup", + selected: [legislative], + candidates: split, + }), + ).toEqual([legislative]); + }); + + it("only adds the missing half when one role is already selected", () => { + const topped = withBestPracticePrescriptionEvidence({ + query: QUERY, + queryClass: "document_lookup", + selected: [legislative, programUse], + candidates, + }); + expect(topped.map((result) => result.id)).toEqual(["p4-legislative", "p1-program-use", "p2-profile"]); + }); +});