diff --git a/.github/workflows/eval-canary.yml b/.github/workflows/eval-canary.yml index 2e71f14789..0928edd27b 100644 --- a/.github/workflows/eval-canary.yml +++ b/.github/workflows/eval-canary.yml @@ -14,7 +14,7 @@ name: Eval Canary on: # repository_dispatch always loads this secret-bearing workflow and its code # from the trusted default branch. Optional client_payload keys are - # answer_case_limit, rag_ranking_config, and answer_quality_eval. + # answer_case_limit, rag_ranking_config, answer_quality_eval, and dump_rag_cases. repository_dispatch: types: [eval-canary] schedule: @@ -147,6 +147,10 @@ jobs: id: answer_quality env: ANSWER_CASE_LIMIT: ${{ github.event.client_payload.answer_case_limit || '44' }} + # Dispatch-only: when 'true', also write per-case answers and each source's ranking + # scores and pipeline reasons, so a live misorder is diagnosable offline. Off for the + # weekly schedule; costs nothing extra because the same answers are already generated. + DUMP_RAG_CASES: ${{ github.event.client_payload.dump_rag_cases || '' }} SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} # Canary-only latency allowance: cross-region runner -> Sydney Supabase @@ -159,7 +163,11 @@ jobs: fi mkdir -p .local/eval-canary set -o pipefail - npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --source-governance-results .local/eval-canary/golden-retrieval.json --output-dir .local/eval-canary/quality-reports --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log + dump_args=() + if [[ "$DUMP_RAG_CASES" == "true" ]]; then + dump_args=(--dump-rag-cases .local/eval-canary/rag-cases.json) + fi + npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --source-governance-results .local/eval-canary/golden-retrieval.json --output-dir .local/eval-canary/quality-reports --fail-on-threshold "${dump_args[@]}" 2>&1 | tee .local/eval-canary/answer-quality.log # Phase E baseline/paired-run instrument: the five answer-quality metrics (relevance, # readability, artifact leaks, intent coverage, fail-closed) + per-intent structural diff --git a/scripts/eval-answer-quality.ts b/scripts/eval-answer-quality.ts index 7c706fdee8..20cb928d39 100644 --- a/scripts/eval-answer-quality.ts +++ b/scripts/eval-answer-quality.ts @@ -17,6 +17,7 @@ import { type AnswerQualityEvalCase, type AnswerQualityMetric, } from "@/lib/rag/rag-eval-cases"; +import { rankAnswerEvidence } from "@/lib/answer-ranking"; import type { RagAnswer } from "@/lib/types"; import { estimateCostUsd, loadAdminClient, resolveEvalOwnerId, withProviderBackoff } from "./eval-utils"; import { isDirectEntrypoint } from "./lib/is-entrypoint.mjs"; @@ -148,6 +149,7 @@ export function selectAnswerQualityCases(args: { caseIds?: string[]; intent?: st } const MAX_DIAGNOSTIC_SOURCES = 10; +const MAX_DIAGNOSTIC_MATCH_REASONS = 48; const MAX_SOURCE_PREVIEW_CHARS = 800; const MAX_IMAGE_PREVIEW_CHARS = 800; const MAX_DIAGNOSTIC_QUESTION_CHARS = 1_000; @@ -192,7 +194,14 @@ export function resolveLocalDiagnosticOutputPath(requestedPath: string, cwd = pr return resolvedPath; } -export function buildBoundedRagSourceDiagnostics(sources: RagAnswer["sources"]) { +function diagnosticScore(value: number | null | undefined) { + return typeof value === "number" && Number.isFinite(value) ? Number(value.toFixed(4)) : null; +} + +export function buildBoundedRagSourceDiagnostics( + sources: RagAnswer["sources"], + answerEvidenceScores?: ReadonlyMap, +) { return sources.slice(0, MAX_DIAGNOSTIC_SOURCES).map((source, index) => { const tableFacts = source.table_facts ?? []; const images = source.images ?? []; @@ -243,6 +252,22 @@ export function buildBoundedRagSourceDiagnostics(sources: RagAnswer["sources"]) image_table_preview: imageTablePreview ? displaySafeDiagnosticText(imageTablePreview, MAX_IMAGE_PREVIEW_CHARS) : null, + // Ordering evidence: which scores and pipeline steps put this source at this rank, so a + // live misorder (e.g. a later rerank undoing a selection boost) is diagnosable offline. + ranking: { + // rankAnswerEvidence score, the first answer-stage reorder after retrieval. Context + // selection can still move sources afterwards, so where this score's order disagrees + // with `rank`, that later stage moved the source. Retrieval scores follow. + answer_evidence_score: diagnosticScore(answerEvidenceScores?.get(source.id)), + hybrid_score: diagnosticScore(source.hybrid_score), + rank_score: diagnosticScore(source.score_explanation?.rankScore), + release_rank_score: diagnosticScore(source.score_explanation?.releaseRankScore), + final_score: diagnosticScore(source.score_explanation?.finalScore), + final_rank: source.score_explanation?.finalRank ?? null, + match_reasons: (source.match_explanation?.reasons ?? []) + .slice(0, MAX_DIAGNOSTIC_MATCH_REASONS) + .map((reason) => displaySafeDiagnosticText(reason, 120)), + }, content_preview: displaySafeDiagnosticText(source.content, MAX_SOURCE_PREVIEW_CHARS), }; }); @@ -257,6 +282,9 @@ export function buildRagDiagnosticDumpRecord(id: string, question: string, answe const hasOpenAIUsage = openAIUsage.input_tokens + openAIUsage.cached_input_tokens + openAIUsage.output_tokens > 0; const observedProviderRequestIdCount = answer.openAIRequestIds?.filter(Boolean).length ?? 0; const generationLatencyMs = answer.latencyTimings?.generation_latency_ms ?? 0; + // Re-score the emitted sources with the answer-stage ranking, as answerScopedEvidenceMetadata + // does for telemetry. It explains rankAnswerEvidence's reorder, not later context selection. + const answerEvidenceScores = rankAnswerEvidence(question, answer.sources, answer.queryClass).scoresByChunkId; const providerAttempted = Boolean( hasOpenAIUsage || observedProviderRequestIdCount > 0 || @@ -306,6 +334,7 @@ export function buildRagDiagnosticDumpRecord(id: string, question: string, answe answer.sources.filter((source) => answer.citations.some((citation) => citation?.chunk_id && citation.chunk_id === source.id), ), + answerEvidenceScores, ), answer_length: answer.answer?.length ?? 0, answer: displaySafeDiagnosticText(answer.answer ?? "", MAX_DIAGNOSTIC_ANSWER_CHARS), @@ -313,7 +342,7 @@ export function buildRagDiagnosticDumpRecord(id: string, question: string, answe heading: displaySafeDiagnosticText(section.heading, 240), body: displaySafeDiagnosticText(section.body, MAX_DIAGNOSTIC_SECTION_CHARS), })), - sources: buildBoundedRagSourceDiagnostics(answer.sources), + sources: buildBoundedRagSourceDiagnostics(answer.sources, answerEvidenceScores), }; } diff --git a/tests/eval-answer-quality.test.ts b/tests/eval-answer-quality.test.ts index 481cdd1711..e3f460f074 100644 --- a/tests/eval-answer-quality.test.ts +++ b/tests/eval-answer-quality.test.ts @@ -225,6 +225,103 @@ describe("answer dump records", () => { expect(Object.keys(diagnostics[0])).not.toContain("document_id"); }); + it("records the scores and pipeline reasons behind each source's rank", () => { + const [ranked, bare] = buildBoundedRagSourceDiagnostics([ + { + id: "chunk-ranked", + document_id: "doc-ranked", + title: "Assessment Documentation (AKG)", + file_name: "Assessment Documentation (AKG).pdf", + page_number: 3, + chunk_index: 0, + section_heading: null, + content: "Assessment documentation must be completed.", + image_ids: [], + similarity: 0.48, + hybrid_score: 0.600049, + images: [], + score_explanation: { rankScore: 1.34751, releaseRankScore: 0.6, finalScore: 1, finalRank: 4 }, + match_explanation: { reasons: ["retrieval_signal:document_title_phrase", "second_stage_rerank"] }, + }, + { + id: "chunk-bare", + document_id: "doc-bare", + title: "Bare", + file_name: "bare.pdf", + page_number: 1, + chunk_index: 0, + section_heading: null, + content: "Text.", + image_ids: [], + similarity: 0.5, + images: [], + }, + ] as unknown as RagAnswer["sources"]); + expect(ranked.ranking).toEqual({ + answer_evidence_score: null, + hybrid_score: 0.6, + rank_score: 1.3475, + release_rank_score: 0.6, + final_score: 1, + final_rank: 4, + match_reasons: ["retrieval_signal:document_title_phrase", "second_stage_rerank"], + }); + expect(bare.ranking).toEqual({ + answer_evidence_score: null, + hybrid_score: null, + rank_score: null, + release_rank_score: null, + final_score: null, + final_rank: null, + match_reasons: [], + }); + }); + + it("scores each dumped source with the answer-stage ranking that set the emitted order", () => { + const sources = [ + { + id: "chunk-on-topic", + document_id: "doc-on-topic", + title: "Lithium Monitoring", + file_name: "lithium-monitoring.pdf", + page_number: 2, + chunk_index: 0, + section_heading: "Monitoring", + content: "Lithium monitoring requires serum levels, renal function and thyroid function.", + image_ids: [], + similarity: 0.4, + hybrid_score: 0.4, + images: [], + score_explanation: { rankScore: 0.4, finalScore: 0.4, finalRank: 2 }, + }, + { + id: "chunk-off-topic", + document_id: "doc-off-topic", + title: "Pressure Injury Prevention", + file_name: "pressure-injury.pdf", + page_number: 7, + chunk_index: 0, + section_heading: null, + content: "Complete the skin integrity form.", + image_ids: [], + similarity: 0.6, + hybrid_score: 0.6, + images: [], + score_explanation: { rankScore: 0.6, finalScore: 0.6, finalRank: 1 }, + }, + ] as unknown as RagAnswer["sources"]; + const record = buildRagDiagnosticDumpRecord("answer-order", "What lithium monitoring is required?", { + ...answer, + queryClass: "document_lookup", + sources, + } as RagAnswer); + const [onTopic, offTopic] = record.sources; + expect(onTopic.ranking.final_rank).toBe(2); + expect(onTopic.ranking.answer_evidence_score).toEqual(expect.any(Number)); + expect(offTopic.ranking.answer_evidence_score).toEqual(expect.any(Number)); + expect(onTopic.ranking.answer_evidence_score!).toBeGreaterThan(offTopic.ranking.answer_evidence_score!); + }); + it("redacts the question, answer, routing and section fields while preserving clinical prose", () => { const hostileAnswer = { ...answer,