From e48badf512d1c86d46618b70457cedc0869bc76f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 12:08:37 +0000 Subject: [PATCH 1/3] eval: record each answer source's ranking scores and reasons, opt-in on canary dispatch The answer-quality canary only reports which files an answer used, not why they landed in that order, so a live misorder (assessment-documentation still lists the named guideline 4th after PR #3572) cannot be traced offline. Add hybrid/rank/release/final scores, final rank and match reasons to the bounded per-source diagnostics, and a dump_rag_cases dispatch payload that writes the existing --dump-rag-cases file into the canary artifact. Off for the weekly schedule. No retrieval or answer behaviour change. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- .github/workflows/eval-canary.yml | 12 ++++++-- scripts/eval-answer-quality.ts | 17 +++++++++++ tests/eval-answer-quality.test.ts | 50 +++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval-canary.yml b/.github/workflows/eval-canary.yml index 2e71f14789..0928edd27b 100644 --- a/.github/workflows/eval-canary.yml +++ b/.github/workflows/eval-canary.yml @@ -14,7 +14,7 @@ name: Eval Canary on: # repository_dispatch always loads this secret-bearing workflow and its code # from the trusted default branch. Optional client_payload keys are - # answer_case_limit, rag_ranking_config, and answer_quality_eval. + # answer_case_limit, rag_ranking_config, answer_quality_eval, and dump_rag_cases. repository_dispatch: types: [eval-canary] schedule: @@ -147,6 +147,10 @@ jobs: id: answer_quality env: ANSWER_CASE_LIMIT: ${{ github.event.client_payload.answer_case_limit || '44' }} + # Dispatch-only: when 'true', also write per-case answers and each source's ranking + # scores and pipeline reasons, so a live misorder is diagnosable offline. Off for the + # weekly schedule; costs nothing extra because the same answers are already generated. + DUMP_RAG_CASES: ${{ github.event.client_payload.dump_rag_cases || '' }} SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} # Canary-only latency allowance: cross-region runner -> Sydney Supabase @@ -159,7 +163,11 @@ jobs: fi mkdir -p .local/eval-canary set -o pipefail - npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --source-governance-results .local/eval-canary/golden-retrieval.json --output-dir .local/eval-canary/quality-reports --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log + dump_args=() + if [[ "$DUMP_RAG_CASES" == "true" ]]; then + dump_args=(--dump-rag-cases .local/eval-canary/rag-cases.json) + fi + npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --source-governance-results .local/eval-canary/golden-retrieval.json --output-dir .local/eval-canary/quality-reports --fail-on-threshold "${dump_args[@]}" 2>&1 | tee .local/eval-canary/answer-quality.log # Phase E baseline/paired-run instrument: the five answer-quality metrics (relevance, # readability, artifact leaks, intent coverage, fail-closed) + per-intent structural diff --git a/scripts/eval-answer-quality.ts b/scripts/eval-answer-quality.ts index 7c706fdee8..deef1a2662 100644 --- a/scripts/eval-answer-quality.ts +++ b/scripts/eval-answer-quality.ts @@ -148,6 +148,7 @@ export function selectAnswerQualityCases(args: { caseIds?: string[]; intent?: st } const MAX_DIAGNOSTIC_SOURCES = 10; +const MAX_DIAGNOSTIC_MATCH_REASONS = 48; const MAX_SOURCE_PREVIEW_CHARS = 800; const MAX_IMAGE_PREVIEW_CHARS = 800; const MAX_DIAGNOSTIC_QUESTION_CHARS = 1_000; @@ -192,6 +193,10 @@ export function resolveLocalDiagnosticOutputPath(requestedPath: string, cwd = pr return resolvedPath; } +function diagnosticScore(value: number | null | undefined) { + return typeof value === "number" && Number.isFinite(value) ? Number(value.toFixed(4)) : null; +} + export function buildBoundedRagSourceDiagnostics(sources: RagAnswer["sources"]) { return sources.slice(0, MAX_DIAGNOSTIC_SOURCES).map((source, index) => { const tableFacts = source.table_facts ?? []; @@ -243,6 +248,18 @@ export function buildBoundedRagSourceDiagnostics(sources: RagAnswer["sources"]) image_table_preview: imageTablePreview ? displaySafeDiagnosticText(imageTablePreview, MAX_IMAGE_PREVIEW_CHARS) : null, + // Ordering evidence: which scores and pipeline steps put this source at this rank, so a + // live misorder (e.g. a later rerank undoing a selection boost) is diagnosable offline. + ranking: { + hybrid_score: diagnosticScore(source.hybrid_score), + rank_score: diagnosticScore(source.score_explanation?.rankScore), + release_rank_score: diagnosticScore(source.score_explanation?.releaseRankScore), + final_score: diagnosticScore(source.score_explanation?.finalScore), + final_rank: source.score_explanation?.finalRank ?? null, + match_reasons: (source.match_explanation?.reasons ?? []) + .slice(0, MAX_DIAGNOSTIC_MATCH_REASONS) + .map((reason) => displaySafeDiagnosticText(reason, 120)), + }, content_preview: displaySafeDiagnosticText(source.content, MAX_SOURCE_PREVIEW_CHARS), }; }); diff --git a/tests/eval-answer-quality.test.ts b/tests/eval-answer-quality.test.ts index 481cdd1711..ca03eca7fe 100644 --- a/tests/eval-answer-quality.test.ts +++ b/tests/eval-answer-quality.test.ts @@ -225,6 +225,56 @@ describe("answer dump records", () => { expect(Object.keys(diagnostics[0])).not.toContain("document_id"); }); + it("records the scores and pipeline reasons behind each source's rank", () => { + const [ranked, bare] = buildBoundedRagSourceDiagnostics([ + { + id: "chunk-ranked", + document_id: "doc-ranked", + title: "Assessment Documentation (AKG)", + file_name: "Assessment Documentation (AKG).pdf", + page_number: 3, + chunk_index: 0, + section_heading: null, + content: "Assessment documentation must be completed.", + image_ids: [], + similarity: 0.48, + hybrid_score: 0.600049, + images: [], + score_explanation: { rankScore: 1.34751, releaseRankScore: 0.6, finalScore: 1, finalRank: 4 }, + match_explanation: { reasons: ["retrieval_signal:document_title_phrase", "second_stage_rerank"] }, + }, + { + id: "chunk-bare", + document_id: "doc-bare", + title: "Bare", + file_name: "bare.pdf", + page_number: 1, + chunk_index: 0, + section_heading: null, + content: "Text.", + image_ids: [], + similarity: 0.5, + images: [], + }, + ] as unknown as RagAnswer["sources"]); + expect(ranked.ranking).toEqual({ + hybrid_score: 0.6, + rank_score: 1.3475, + release_rank_score: 0.6, + final_score: 1, + final_rank: 4, + match_reasons: ["retrieval_signal:document_title_phrase", "second_stage_rerank"], + }); + expect(bare.ranking).toEqual({ + hybrid_score: null, + rank_score: null, + release_rank_score: null, + final_score: null, + final_rank: null, + match_reasons: [], + }); + }); + it("redacts the question, answer, routing and section fields while preserving clinical prose", () => { const hostileAnswer = { ...answer, From 2c289f4efbb3c19717f08dced2140cce7d664736 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 13:18:49 +0000 Subject: [PATCH 2/3] eval: record the answer-stage score that sets each dumped source's order rankAnswerEvidence reorders sources after retrieval selection, so the retrieval scores alone cannot explain the emitted order (Codex review). Re-score the emitted sources with it, as answerScopedEvidenceMetadata does, and record answer_evidence_score beside the retrieval scores. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- scripts/eval-answer-quality.ts | 15 ++++++++-- tests/eval-answer-quality.test.ts | 47 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/scripts/eval-answer-quality.ts b/scripts/eval-answer-quality.ts index deef1a2662..83de71c92b 100644 --- a/scripts/eval-answer-quality.ts +++ b/scripts/eval-answer-quality.ts @@ -17,6 +17,7 @@ import { type AnswerQualityEvalCase, type AnswerQualityMetric, } from "@/lib/rag/rag-eval-cases"; +import { rankAnswerEvidence } from "@/lib/answer-ranking"; import type { RagAnswer } from "@/lib/types"; import { estimateCostUsd, loadAdminClient, resolveEvalOwnerId, withProviderBackoff } from "./eval-utils"; import { isDirectEntrypoint } from "./lib/is-entrypoint.mjs"; @@ -197,7 +198,10 @@ function diagnosticScore(value: number | null | undefined) { return typeof value === "number" && Number.isFinite(value) ? Number(value.toFixed(4)) : null; } -export function buildBoundedRagSourceDiagnostics(sources: RagAnswer["sources"]) { +export function buildBoundedRagSourceDiagnostics( + sources: RagAnswer["sources"], + answerEvidenceScores?: ReadonlyMap, +) { return sources.slice(0, MAX_DIAGNOSTIC_SOURCES).map((source, index) => { const tableFacts = source.table_facts ?? []; const images = source.images ?? []; @@ -251,6 +255,9 @@ export function buildBoundedRagSourceDiagnostics(sources: RagAnswer["sources"]) // Ordering evidence: which scores and pipeline steps put this source at this rank, so a // live misorder (e.g. a later rerank undoing a selection boost) is diagnosable offline. ranking: { + // The answer-stage score (rankAnswerEvidence) sets the emitted order; the retrieval + // scores below are from the earlier selection and can disagree with `rank`. + answer_evidence_score: diagnosticScore(answerEvidenceScores?.get(source.id)), hybrid_score: diagnosticScore(source.hybrid_score), rank_score: diagnosticScore(source.score_explanation?.rankScore), release_rank_score: diagnosticScore(source.score_explanation?.releaseRankScore), @@ -274,6 +281,9 @@ export function buildRagDiagnosticDumpRecord(id: string, question: string, answe const hasOpenAIUsage = openAIUsage.input_tokens + openAIUsage.cached_input_tokens + openAIUsage.output_tokens > 0; const observedProviderRequestIdCount = answer.openAIRequestIds?.filter(Boolean).length ?? 0; const generationLatencyMs = answer.latencyTimings?.generation_latency_ms ?? 0; + // Re-score the emitted sources with the answer-stage ranking that ordered them, as + // answerScopedEvidenceMetadata does for telemetry, so the dump shows why each sits where it does. + const answerEvidenceScores = rankAnswerEvidence(question, answer.sources, answer.queryClass).scoresByChunkId; const providerAttempted = Boolean( hasOpenAIUsage || observedProviderRequestIdCount > 0 || @@ -323,6 +333,7 @@ export function buildRagDiagnosticDumpRecord(id: string, question: string, answe answer.sources.filter((source) => answer.citations.some((citation) => citation?.chunk_id && citation.chunk_id === source.id), ), + answerEvidenceScores, ), answer_length: answer.answer?.length ?? 0, answer: displaySafeDiagnosticText(answer.answer ?? "", MAX_DIAGNOSTIC_ANSWER_CHARS), @@ -330,7 +341,7 @@ export function buildRagDiagnosticDumpRecord(id: string, question: string, answe heading: displaySafeDiagnosticText(section.heading, 240), body: displaySafeDiagnosticText(section.body, MAX_DIAGNOSTIC_SECTION_CHARS), })), - sources: buildBoundedRagSourceDiagnostics(answer.sources), + sources: buildBoundedRagSourceDiagnostics(answer.sources, answerEvidenceScores), }; } diff --git a/tests/eval-answer-quality.test.ts b/tests/eval-answer-quality.test.ts index ca03eca7fe..e3f460f074 100644 --- a/tests/eval-answer-quality.test.ts +++ b/tests/eval-answer-quality.test.ts @@ -258,6 +258,7 @@ describe("answer dump records", () => { }, ] as unknown as RagAnswer["sources"]); expect(ranked.ranking).toEqual({ + answer_evidence_score: null, hybrid_score: 0.6, rank_score: 1.3475, release_rank_score: 0.6, @@ -266,6 +267,7 @@ describe("answer dump records", () => { match_reasons: ["retrieval_signal:document_title_phrase", "second_stage_rerank"], }); expect(bare.ranking).toEqual({ + answer_evidence_score: null, hybrid_score: null, rank_score: null, release_rank_score: null, @@ -275,6 +277,51 @@ describe("answer dump records", () => { }); }); + it("scores each dumped source with the answer-stage ranking that set the emitted order", () => { + const sources = [ + { + id: "chunk-on-topic", + document_id: "doc-on-topic", + title: "Lithium Monitoring", + file_name: "lithium-monitoring.pdf", + page_number: 2, + chunk_index: 0, + section_heading: "Monitoring", + content: "Lithium monitoring requires serum levels, renal function and thyroid function.", + image_ids: [], + similarity: 0.4, + hybrid_score: 0.4, + images: [], + score_explanation: { rankScore: 0.4, finalScore: 0.4, finalRank: 2 }, + }, + { + id: "chunk-off-topic", + document_id: "doc-off-topic", + title: "Pressure Injury Prevention", + file_name: "pressure-injury.pdf", + page_number: 7, + chunk_index: 0, + section_heading: null, + content: "Complete the skin integrity form.", + image_ids: [], + similarity: 0.6, + hybrid_score: 0.6, + images: [], + score_explanation: { rankScore: 0.6, finalScore: 0.6, finalRank: 1 }, + }, + ] as unknown as RagAnswer["sources"]; + const record = buildRagDiagnosticDumpRecord("answer-order", "What lithium monitoring is required?", { + ...answer, + queryClass: "document_lookup", + sources, + } as RagAnswer); + const [onTopic, offTopic] = record.sources; + expect(onTopic.ranking.final_rank).toBe(2); + expect(onTopic.ranking.answer_evidence_score).toEqual(expect.any(Number)); + expect(offTopic.ranking.answer_evidence_score).toEqual(expect.any(Number)); + expect(onTopic.ranking.answer_evidence_score!).toBeGreaterThan(offTopic.ranking.answer_evidence_score!); + }); + it("redacts the question, answer, routing and section fields while preserving clinical prose", () => { const hostileAnswer = { ...answer, From df28429d9ae096b33682d37951599cec42c02a3f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 13:32:07 +0000 Subject: [PATCH 3/3] eval: say what the dumped answer-evidence score does and does not explain Context selection can still move sources after rankAnswerEvidence, so the score explains that reorder, not every later move (Qodo review). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- scripts/eval-answer-quality.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/eval-answer-quality.ts b/scripts/eval-answer-quality.ts index 83de71c92b..20cb928d39 100644 --- a/scripts/eval-answer-quality.ts +++ b/scripts/eval-answer-quality.ts @@ -255,8 +255,9 @@ export function buildBoundedRagSourceDiagnostics( // Ordering evidence: which scores and pipeline steps put this source at this rank, so a // live misorder (e.g. a later rerank undoing a selection boost) is diagnosable offline. ranking: { - // The answer-stage score (rankAnswerEvidence) sets the emitted order; the retrieval - // scores below are from the earlier selection and can disagree with `rank`. + // rankAnswerEvidence score, the first answer-stage reorder after retrieval. Context + // selection can still move sources afterwards, so where this score's order disagrees + // with `rank`, that later stage moved the source. Retrieval scores follow. answer_evidence_score: diagnosticScore(answerEvidenceScores?.get(source.id)), hybrid_score: diagnosticScore(source.hybrid_score), rank_score: diagnosticScore(source.score_explanation?.rankScore), @@ -281,8 +282,8 @@ export function buildRagDiagnosticDumpRecord(id: string, question: string, answe const hasOpenAIUsage = openAIUsage.input_tokens + openAIUsage.cached_input_tokens + openAIUsage.output_tokens > 0; const observedProviderRequestIdCount = answer.openAIRequestIds?.filter(Boolean).length ?? 0; const generationLatencyMs = answer.latencyTimings?.generation_latency_ms ?? 0; - // Re-score the emitted sources with the answer-stage ranking that ordered them, as - // answerScopedEvidenceMetadata does for telemetry, so the dump shows why each sits where it does. + // Re-score the emitted sources with the answer-stage ranking, as answerScopedEvidenceMetadata + // does for telemetry. It explains rankAnswerEvidence's reorder, not later context selection. const answerEvidenceScores = rankAnswerEvidence(question, answer.sources, answer.queryClass).scoresByChunkId; const providerAttempted = Boolean( hasOpenAIUsage ||