eval: record why each answer source ranks where it does, opt-in on canary dispatch

.github/workflows/eval-canary.yml

@@ -14,7 +14,7 @@ name: Eval Canary
 on:
   # repository_dispatch always loads this secret-bearing workflow and its code
   # from the trusted default branch. Optional client_payload keys are
-  # answer_case_limit, rag_ranking_config, and answer_quality_eval.
+  # answer_case_limit, rag_ranking_config, answer_quality_eval, and dump_rag_cases.
   repository_dispatch:
     types: [eval-canary]
   schedule:
@@ -147,6 +147,10 @@ jobs:
         id: answer_quality
         env:
           ANSWER_CASE_LIMIT: ${{ github.event.client_payload.answer_case_limit || '44' }}
+          # Dispatch-only: when 'true', also write per-case answers and each source's ranking
+          # scores and pipeline reasons, so a live misorder is diagnosable offline. Off for the
+          # weekly schedule; costs nothing extra because the same answers are already generated.
+          DUMP_RAG_CASES: ${{ github.event.client_payload.dump_rag_cases || '' }}
           SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           # Canary-only latency allowance: cross-region runner -> Sydney Supabase
@@ -159,7 +163,11 @@ jobs:
           fi
           mkdir -p .local/eval-canary
           set -o pipefail
-          npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --source-governance-results .local/eval-canary/golden-retrieval.json --output-dir .local/eval-canary/quality-reports --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log
+          dump_args=()
+          if [[ "$DUMP_RAG_CASES" == "true" ]]; then
+            dump_args=(--dump-rag-cases .local/eval-canary/rag-cases.json)
+          fi
+          npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --source-governance-results .local/eval-canary/golden-retrieval.json --output-dir .local/eval-canary/quality-reports --fail-on-threshold "${dump_args[@]}" 2>&1 | tee .local/eval-canary/answer-quality.log
 
       # Phase E baseline/paired-run instrument: the five answer-quality metrics (relevance,
       # readability, artifact leaks, intent coverage, fail-closed) + per-intent structural

scripts/eval-answer-quality.ts

@@ -17,6 +17,7 @@ import {
   type AnswerQualityEvalCase,
   type AnswerQualityMetric,
 } from "@/lib/rag/rag-eval-cases";
+import { rankAnswerEvidence } from "@/lib/answer-ranking";
 import type { RagAnswer } from "@/lib/types";
 import { estimateCostUsd, loadAdminClient, resolveEvalOwnerId, withProviderBackoff } from "./eval-utils";
 import { isDirectEntrypoint } from "./lib/is-entrypoint.mjs";
@@ -148,6 +149,7 @@ export function selectAnswerQualityCases(args: { caseIds?: string[]; intent?: st
 }
 
 const MAX_DIAGNOSTIC_SOURCES = 10;
+const MAX_DIAGNOSTIC_MATCH_REASONS = 48;
 const MAX_SOURCE_PREVIEW_CHARS = 800;
 const MAX_IMAGE_PREVIEW_CHARS = 800;
 const MAX_DIAGNOSTIC_QUESTION_CHARS = 1_000;
@@ -192,7 +194,14 @@ export function resolveLocalDiagnosticOutputPath(requestedPath: string, cwd = pr
   return resolvedPath;
 }
 
-export function buildBoundedRagSourceDiagnostics(sources: RagAnswer["sources"]) {
+function diagnosticScore(value: number | null | undefined) {
+  return typeof value === "number" && Number.isFinite(value) ? Number(value.toFixed(4)) : null;
+}
+
+export function buildBoundedRagSourceDiagnostics(
+  sources: RagAnswer["sources"],
+  answerEvidenceScores?: ReadonlyMap<string, number>,
+) {
   return sources.slice(0, MAX_DIAGNOSTIC_SOURCES).map((source, index) => {
     const tableFacts = source.table_facts ?? [];
     const images = source.images ?? [];
@@ -243,6 +252,22 @@ export function buildBoundedRagSourceDiagnostics(sources: RagAnswer["sources"])
       image_table_preview: imageTablePreview
         ? displaySafeDiagnosticText(imageTablePreview, MAX_IMAGE_PREVIEW_CHARS)
         : null,
+      // Ordering evidence: which scores and pipeline steps put this source at this rank, so a
+      // live misorder (e.g. a later rerank undoing a selection boost) is diagnosable offline.
+      ranking: {
+        // rankAnswerEvidence score, the first answer-stage reorder after retrieval. Context
+        // selection can still move sources afterwards, so where this score's order disagrees
+        // with `rank`, that later stage moved the source. Retrieval scores follow.
+        answer_evidence_score: diagnosticScore(answerEvidenceScores?.get(source.id)),
+        hybrid_score: diagnosticScore(source.hybrid_score),
+        rank_score: diagnosticScore(source.score_explanation?.rankScore),
+        release_rank_score: diagnosticScore(source.score_explanation?.releaseRankScore),
+        final_score: diagnosticScore(source.score_explanation?.finalScore),
+        final_rank: source.score_explanation?.finalRank ?? null,
+        match_reasons: (source.match_explanation?.reasons ?? [])
+          .slice(0, MAX_DIAGNOSTIC_MATCH_REASONS)
+          .map((reason) => displaySafeDiagnosticText(reason, 120)),
+      },
       content_preview: displaySafeDiagnosticText(source.content, MAX_SOURCE_PREVIEW_CHARS),
     };
   });
@@ -257,6 +282,9 @@ export function buildRagDiagnosticDumpRecord(id: string, question: string, answe
   const hasOpenAIUsage = openAIUsage.input_tokens + openAIUsage.cached_input_tokens + openAIUsage.output_tokens > 0;
   const observedProviderRequestIdCount = answer.openAIRequestIds?.filter(Boolean).length ?? 0;
   const generationLatencyMs = answer.latencyTimings?.generation_latency_ms ?? 0;
+  // Re-score the emitted sources with the answer-stage ranking, as answerScopedEvidenceMetadata
+  // does for telemetry. It explains rankAnswerEvidence's reorder, not later context selection.
+  const answerEvidenceScores = rankAnswerEvidence(question, answer.sources, answer.queryClass).scoresByChunkId;
   const providerAttempted = Boolean(
     hasOpenAIUsage ||
     observedProviderRequestIdCount > 0 ||
@@ -306,14 +334,15 @@ export function buildRagDiagnosticDumpRecord(id: string, question: string, answe
       answer.sources.filter((source) =>
         answer.citations.some((citation) => citation?.chunk_id && citation.chunk_id === source.id),
       ),
+      answerEvidenceScores,
     ),
     answer_length: answer.answer?.length ?? 0,
     answer: displaySafeDiagnosticText(answer.answer ?? "", MAX_DIAGNOSTIC_ANSWER_CHARS),
     answer_sections: (answer.answerSections ?? []).map((section) => ({
       heading: displaySafeDiagnosticText(section.heading, 240),
       body: displaySafeDiagnosticText(section.body, MAX_DIAGNOSTIC_SECTION_CHARS),
     })),
-    sources: buildBoundedRagSourceDiagnostics(answer.sources),
+    sources: buildBoundedRagSourceDiagnostics(answer.sources, answerEvidenceScores),
   };
 }
 

tests/eval-answer-quality.test.ts

@@ -225,6 +225,103 @@ describe("answer dump records", () => {
     expect(Object.keys(diagnostics[0])).not.toContain("document_id");
   });
 
+  it("records the scores and pipeline reasons behind each source's rank", () => {
+    const [ranked, bare] = buildBoundedRagSourceDiagnostics([
+      {
+        id: "chunk-ranked",
+        document_id: "doc-ranked",
+        title: "Assessment Documentation (AKG)",
+        file_name: "Assessment Documentation (AKG).pdf",
+        page_number: 3,
+        chunk_index: 0,
+        section_heading: null,
+        content: "Assessment documentation must be completed.",
+        image_ids: [],
+        similarity: 0.48,
+        hybrid_score: 0.600049,
+        images: [],
+        score_explanation: { rankScore: 1.34751, releaseRankScore: 0.6, finalScore: 1, finalRank: 4 },
+        match_explanation: { reasons: ["retrieval_signal:document_title_phrase", "second_stage_rerank"] },
+      },
+      {
+        id: "chunk-bare",
+        document_id: "doc-bare",
+        title: "Bare",
+        file_name: "bare.pdf",
+        page_number: 1,
+        chunk_index: 0,
+        section_heading: null,
+        content: "Text.",
+        image_ids: [],
+        similarity: 0.5,
+        images: [],
+      },
+    ] as unknown as RagAnswer["sources"]);
+    expect(ranked.ranking).toEqual({
+      answer_evidence_score: null,
+      hybrid_score: 0.6,
+      rank_score: 1.3475,
+      release_rank_score: 0.6,
+      final_score: 1,
+      final_rank: 4,
+      match_reasons: ["retrieval_signal:document_title_phrase", "second_stage_rerank"],
+    });
+    expect(bare.ranking).toEqual({
+      answer_evidence_score: null,
+      hybrid_score: null,
+      rank_score: null,
+      release_rank_score: null,
+      final_score: null,
+      final_rank: null,
+      match_reasons: [],
+    });
+  });
+
+  it("scores each dumped source with the answer-stage ranking that set the emitted order", () => {
+    const sources = [
+      {
+        id: "chunk-on-topic",
+        document_id: "doc-on-topic",
+        title: "Lithium Monitoring",
+        file_name: "lithium-monitoring.pdf",
+        page_number: 2,
+        chunk_index: 0,
+        section_heading: "Monitoring",
+        content: "Lithium monitoring requires serum levels, renal function and thyroid function.",
+        image_ids: [],
+        similarity: 0.4,
+        hybrid_score: 0.4,
+        images: [],
+        score_explanation: { rankScore: 0.4, finalScore: 0.4, finalRank: 2 },
+      },
+      {
+        id: "chunk-off-topic",
+        document_id: "doc-off-topic",
+        title: "Pressure Injury Prevention",
+        file_name: "pressure-injury.pdf",
+        page_number: 7,
+        chunk_index: 0,
+        section_heading: null,
+        content: "Complete the skin integrity form.",
+        image_ids: [],
+        similarity: 0.6,
+        hybrid_score: 0.6,
+        images: [],
+        score_explanation: { rankScore: 0.6, finalScore: 0.6, finalRank: 1 },
+      },
+    ] as unknown as RagAnswer["sources"];
+    const record = buildRagDiagnosticDumpRecord("answer-order", "What lithium monitoring is required?", {
+      ...answer,
+      queryClass: "document_lookup",
+      sources,
+    } as RagAnswer);
+    const [onTopic, offTopic] = record.sources;
+    expect(onTopic.ranking.final_rank).toBe(2);
+    expect(onTopic.ranking.answer_evidence_score).toEqual(expect.any(Number));
+    expect(offTopic.ranking.answer_evidence_score).toEqual(expect.any(Number));
+    expect(onTopic.ranking.answer_evidence_score!).toBeGreaterThan(offTopic.ranking.answer_evidence_score!);
+  });
+
   it("redacts the question, answer, routing and section fields while preserving clinical prose", () => {
     const hostileAnswer = {
       ...answer,