diff --git a/scripts/classify-documents.ts b/scripts/classify-documents.ts index 1166c3166f..4ed6b2ad48 100644 --- a/scripts/classify-documents.ts +++ b/scripts/classify-documents.ts @@ -197,6 +197,8 @@ function preservedReviewMetadata(metadata: unknown) { const preserved: Record = {}; for (const key of [ "review_status", + "label_review_status", + "hidden", "reviewed_at", "reviewed_by", "review_note", @@ -409,21 +411,31 @@ async function writeClassifications(supabase: SupabaseAdmin, plans: Classificati const documentIds = batch.map((plan) => plan.document.id); const generatedLabels = dedupeGeneratedLabels(batch.flatMap((plan) => generatedLabelsForPlan(plan, stampedAt))); const desiredLabelKeys = new Set(generatedLabels.map(labelIdentity)); - const { data: existingGenerated, error: existingGeneratedError } = (await supabase - .from("document_labels") - .select("id,document_id,label_type,label,metadata") - .in("document_id", documentIds) - .eq("source", "generated") - .in("label_type", [...generatedLabelTypes])) as { - data: ExistingGeneratedLabelRow[] | null; - error: DatabaseError | null; - }; - if (existingGeneratedError) throw new Error(existingGeneratedError.message); + // Paged, because 100 documents can hold more generated labels than one response returns, + // and a label missing here would lose its review state on upsert and never be cleaned up. + const existingGenerated: ExistingGeneratedLabelRow[] = []; + const existingPageSize = 1000; + for (let start = 0; ; start += existingPageSize) { + const { data, error: existingGeneratedError } = (await supabase + .from("document_labels") + .select("id,document_id,label_type,label,metadata") + .in("document_id", documentIds) + .eq("source", "generated") + .in("label_type", [...generatedLabelTypes]) + .order("id", { ascending: true }) + .range(start, start + existingPageSize - 1)) as { + data: ExistingGeneratedLabelRow[] | null; + error: DatabaseError | null; + }; + if (existingGeneratedError) throw new Error(existingGeneratedError.message); + existingGenerated.push(...(data ?? [])); + if ((data ?? []).length < existingPageSize) break; + } - const labelsToDelete = (existingGenerated ?? []) + const labelsToDelete = existingGenerated .filter((label) => !desiredLabelKeys.has(labelIdentity(label))) .map((label) => label.id); - const existingGeneratedByKey = new Map((existingGenerated ?? []).map((label) => [labelIdentity(label), label])); + const existingGeneratedByKey = new Map(existingGenerated.map((label) => [labelIdentity(label), label])); const labelsForUpsert = generatedLabels.map((label) => { const preserved = preservedReviewMetadata(existingGeneratedByKey.get(labelIdentity(label))?.metadata); return { diff --git a/scripts/compare-retrieval-eval.ts b/scripts/compare-retrieval-eval.ts index 370fd4645a..63b59299f7 100644 --- a/scripts/compare-retrieval-eval.ts +++ b/scripts/compare-retrieval-eval.ts @@ -190,6 +190,26 @@ export function compareRetrievalEval(baseline: EvalSummary, candidate: EvalSumma return { rows, missingRequired }; } +const PINNED_RECALL_METRICS = new Set(["document_recall_at_5", "content_recall_at_5"]); + +/** + * Recall rows that fail the canary-pair protocol: doc and content recall are pinned at 1.0, so a + * candidate below 1.0, or below its baseline, is a regression even when every per-case rank holds. + */ +export function recallFailures(rows: readonly ComparisonRow[]): string[] { + const failures: string[] = []; + for (const row of rows) { + if (!PINNED_RECALL_METRICS.has(row.name) || !row.candidate.present) continue; + const candidate = row.candidate.value; + if (candidate < 1) failures.push(`${row.name} is ${candidate.toFixed(row.digits)}, below the pinned 1.0`); + else if (row.baseline.present && candidate < row.baseline.value) + failures.push( + `${row.name} fell from ${row.baseline.value.toFixed(row.digits)} to ${candidate.toFixed(row.digits)}`, + ); + } + return failures; +} + function formatCell(row: ComparisonRow): string { if (!row.baseline.present || !row.candidate.present) { const candidate = row.candidate.present ? row.candidate.value.toFixed(row.digits) : "n/a"; @@ -224,6 +244,12 @@ function main() { console.log(` ${row.name}: ${formatCell(row)}`); } + const recallDrops = recallFailures(rows); + if (failOnRegression && recallDrops.length > 0) { + console.error(`\nRecall gate failed: ${recallDrops.join("; ")}.`); + process.exitCode = 1; + } + if (missingRequired.length > 0) { console.error(`\nMissing required metric(s), refusing a clean comparison: ${missingRequired.join(", ")}`); console.error("A missing decision metric is not the same as 0 — regenerate the eval JSON with the full summary."); diff --git a/scripts/embed-registry-records.ts b/scripts/embed-registry-records.ts index 122738f187..e0ed4616be 100644 --- a/scripts/embed-registry-records.ts +++ b/scripts/embed-registry-records.ts @@ -74,25 +74,45 @@ function registryKinds(kind: EmbedKind): RegistryRecordKind[] { return []; } +/** + * Every row a query matches. One PostgREST response stops at the project's row cap (1000 by + * default), so an unpaged read silently skips records once a table outgrows it. + */ +async function readAllPages( + page: (from: number, to: number) => PromiseLike<{ data: unknown[] | null; error: { message: string } | null }>, + what: string, +): Promise { + const pageSize = 1000; + const rows: Row[] = []; + for (let from = 0; ; from += pageSize) { + const { data, error } = await page(from, from + pageSize - 1); + if (error) throw new Error(`Could not load ${what}: ${error.message}`); + rows.push(...((data ?? []) as Row[])); + if ((data ?? []).length < pageSize) return rows; + } +} + /** Load registry rows. */ async function loadRegistryRows(supabase: Awaited>, args: Args, ownerId: string) { const kinds = registryKinds(args.kind); if (kinds.length === 0) return [] as RegistryRecordRow[]; - let query = supabase.from("clinical_registry_records").select("*").eq("owner_id", ownerId).in("kind", kinds); - if (args.slug) query = query.eq("slug", args.slug); - const { data, error } = await query.order("kind").order("title"); - if (error) throw new Error(`Could not load registry records: ${error.message}`); - return (data ?? []) as RegistryRecordRow[]; + const page = (from: number, to: number) => { + let query = supabase.from("clinical_registry_records").select("*").eq("owner_id", ownerId).in("kind", kinds); + if (args.slug) query = query.eq("slug", args.slug); + return query.order("kind").order("title").order("id").range(from, to); + }; + return readAllPages(page, "registry records"); } /** Load medication rows. */ async function loadMedicationRows(supabase: Awaited>, args: Args, ownerId: string) { if (args.kind !== "all" && args.kind !== "medication") return [] as MedicationRecordRow[]; - let query = supabase.from("medication_records").select("*").eq("owner_id", ownerId); - if (args.slug) query = query.eq("slug", args.slug); - const { data, error } = await query.order("name"); - if (error) throw new Error(`Could not load medication records: ${error.message}`); - return (data ?? []) as MedicationRecordRow[]; + const page = (from: number, to: number) => { + let query = supabase.from("medication_records").select("*").eq("owner_id", ownerId); + if (args.slug) query = query.eq("slug", args.slug); + return query.order("name").order("id").range(from, to); + }; + return readAllPages(page, "medication records"); } /** Load differential rows. */ @@ -102,11 +122,12 @@ async function loadDifferentialRows( ownerId: string, ) { if (args.kind !== "all" && args.kind !== "differential") return [] as DifferentialRecordRow[]; - let query = supabase.from("differential_records").select("*").eq("owner_id", ownerId); - if (args.slug) query = query.eq("slug", args.slug); - const { data, error } = await query.order("kind").order("title"); - if (error) throw new Error(`Could not load differential records: ${error.message}`); - return (data ?? []) as DifferentialRecordRow[]; + const page = (from: number, to: number) => { + let query = supabase.from("differential_records").select("*").eq("owner_id", ownerId); + if (args.slug) query = query.eq("slug", args.slug); + return query.order("kind").order("title").order("id").range(from, to); + }; + return readAllPages(page, "differential records"); } type OwnerCounts = { diff --git a/scripts/eval-rag.ts b/scripts/eval-rag.ts index c795585187..0dd9351423 100644 --- a/scripts/eval-rag.ts +++ b/scripts/eval-rag.ts @@ -98,7 +98,9 @@ function parseArgs(argv: string[]): EvalArgs { for (let index = 0; index < argv.length; index += 1) { const token = argv[index]; - if (!token.startsWith("--")) continue; + // A mistyped flag used to be skipped, or to swallow the next token, so a run could + // quietly drop --fail-on-threshold and report green. Refuse anything not recognised. + if (!token.startsWith("--")) throw new Error(`Unexpected argument: ${token}`); if (token === "--json") { args.json = true; @@ -113,6 +115,9 @@ function parseArgs(argv: string[]): EvalArgs { continue; } + if (!["--owner-email", "--owner-id", "--limit", "--question"].includes(token)) { + throw new Error(`Unknown flag: ${token}`); + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); index += 1; @@ -350,6 +355,10 @@ async function main() { } } + if (args.failOnThreshold && results.length === 0) { + console.error("--fail-on-threshold: no eval cases were selected, so nothing was checked."); + process.exit(1); + } if (args.failOnThreshold && thresholdFailures.length > 0) process.exit(1); } diff --git a/scripts/ledger-inbox.mjs b/scripts/ledger-inbox.mjs index 23cb6c7c14..918f17e9b6 100644 --- a/scripts/ledger-inbox.mjs +++ b/scripts/ledger-inbox.mjs @@ -549,9 +549,22 @@ function assertReopenIdentity(markdown, request) { } } +/** + * Known encoding damage in the hand-written preamble, which no request can reach. An arrow saved + * through a Windows code page became "ΓåÆ" (#WWTRM5). Repaired only inside a real reconcile, so + * a batch with no requests still reproduces its base byte for byte for check:ledger-write-discipline. + */ +const LEDGER_MOJIBAKE_REPAIRS = [["\u0393\u00e5\u00c6", "\u2192"]]; + +export function repairLedgerMojibake(markdown) { + let next = markdown; + for (const [broken, intended] of LEDGER_MOJIBAKE_REPAIRS) next = next.split(broken).join(intended); + return next; +} + export function applyRequestBatch(markdown, requests, options = {}) { const plan = planRequestBatch(requests, options); - let next = markdown; + let next = requests.length > 0 ? repairLedgerMojibake(markdown) : markdown; const closedInBatch = new Set(); const ulidToDisplayId = new Map(); diff --git a/scripts/sql/operator-export-site-content-population.sql b/scripts/sql/operator-export-site-content-population.sql index ec8b179947..0f00a7ece5 100644 --- a/scripts/sql/operator-export-site-content-population.sql +++ b/scripts/sql/operator-export-site-content-population.sql @@ -73,7 +73,7 @@ select jsonb_pretty(jsonb_build_object( 'embedding', null ) order by r.logical_id - ), + ) filter (where r.release_id is not null), '[]'::jsonb ) )) as population_export diff --git a/src/components/dsm/dsm-search-page.tsx b/src/components/dsm/dsm-search-page.tsx index 0b8e15a763..0bb5b8bbda 100644 --- a/src/components/dsm/dsm-search-page.tsx +++ b/src/components/dsm/dsm-search-page.tsx @@ -510,6 +510,8 @@ export function DsmSearchPage({ ) : (
+ {/* Keeps the cards' h3 titles from skipping a level (a11y A11Y-12). */} +

Matching diagnoses

{results.map((result) => (
-
+ {/* A real heading, so the ranked rows' h3 titles do not skip a level (a11y A11Y-12). */} +

Ranked clinical matches -

+

{matchLabel}

diff --git a/src/components/therapy-compass/screens/search-screen.tsx b/src/components/therapy-compass/screens/search-screen.tsx index 5102ca017f..639637929f 100644 --- a/src/components/therapy-compass/screens/search-screen.tsx +++ b/src/components/therapy-compass/screens/search-screen.tsx @@ -270,15 +270,19 @@ export function SearchScreen() { onClearSearch={b.clearSearch} /> ) : results.length > 0 ? ( - // One hairline card with divided compact rows (no cards inside cards). -
    - {shown.map((t, index) => ( - - ))} -
+ // One hairline card with divided compact rows (no cards inside cards). The hidden + // heading keeps the rows' h3 titles from skipping a level (a11y A11Y-12). + <> +

Matching therapies

+
    + {shown.map((t, index) => ( + + ))} +
+ ) : null} {outside.length > 0 ? (
diff --git a/src/components/work-screens/admin/requests-page.tsx b/src/components/work-screens/admin/requests-page.tsx index f75112e608..b2d554663d 100644 --- a/src/components/work-screens/admin/requests-page.tsx +++ b/src/components/work-screens/admin/requests-page.tsx @@ -559,7 +559,8 @@ function RequestCard({ Copy and mark sent - {isExampleRecord(request) ? ( + {/* A blocked message never goes into an href, where a middle click or copied link would skip onClick. */} + {isExampleRecord(request) || requestProblem(request) ? ( void onEmail()} testId="admin-requests-email"> Email draft diff --git a/src/lib/legacy-home-redirect.ts b/src/lib/legacy-home-redirect.ts index 1447a0360e..a53eedf7ef 100644 --- a/src/lib/legacy-home-redirect.ts +++ b/src/lib/legacy-home-redirect.ts @@ -28,6 +28,16 @@ const aliasOnlyModePaths = { tools: "/tools", } as const; +/** + * Mode names people type or bookmark that are not the mode's id. `/?mode=medications` + * used to fail the mode check and land on Answer with no sign anything was wrong + * (design audit D26-52), so these keep the visitor on `/` under the mode they meant. + */ +const renamedModeIds = { + medication: "prescribing", + medications: "prescribing", +} as const; + /** * Dropped when a mode forwards to its own page: `mode` is consumed by the * pathname, and `run=1` would submit the draft on a page with nothing to @@ -49,6 +59,14 @@ export function legacyHomeRedirectUrl(requestUrl: LegacyHomeRequestUrl, method: const mode = requestUrl.searchParams.get("mode"); + const renamedMode = mode ? renamedModeIds[mode as keyof typeof renamedModeIds] : undefined; + if (renamedMode) { + const renamedDestination = new URL(requestUrl.toString()); + renamedDestination.searchParams.set("mode", renamedMode); + renamedDestination.hash = ""; + return renamedDestination; + } + const aliasOnlyPath = mode ? aliasOnlyModePaths[mode as keyof typeof aliasOnlyModePaths] : undefined; if (aliasOnlyPath) { const aliasDestination = new URL(requestUrl.toString()); diff --git a/src/lib/teaching/assessments/model.ts b/src/lib/teaching/assessments/model.ts index 0b3ba98119..69b07d82f2 100644 --- a/src/lib/teaching/assessments/model.ts +++ b/src/lib/teaching/assessments/model.ts @@ -926,6 +926,7 @@ export function assessmentsReducer(s: AssessmentsState, a: AssessmentsAction): A if (![1, 2, 3, 4].includes(a.epa) || !SUPERVISION_LEVELS.some((l) => l.id === a.level)) return s; if (a.complexity !== undefined && !validComplexity(a.complexity)) return s; const note = a.note?.trim(); + if (!validText(note)) return s; const done: EpaRequest = { epa: a.epa, who: "sup", diff --git a/tests/compare-retrieval-eval.test.ts b/tests/compare-retrieval-eval.test.ts index 012cf858f8..3fd437c862 100644 --- a/tests/compare-retrieval-eval.test.ts +++ b/tests/compare-retrieval-eval.test.ts @@ -3,7 +3,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, describe, expect, it } from "vitest"; -import { comparePerCaseRanks, compareRetrievalEval } from "../scripts/compare-retrieval-eval"; +import { comparePerCaseRanks, compareRetrievalEval, recallFailures } from "../scripts/compare-retrieval-eval"; // A complete retrieval eval summary as emitted by summarizeGoldenRetrievalResults, trimmed to the // fields the comparison reads. @@ -199,3 +199,23 @@ describe("compare-retrieval-eval CLI --fail-on-regression", () => { expect(out.status).toBe(1); }); }); + +describe("recallFailures", () => { + it("passes a pair that holds recall at 1.0", () => { + expect(recallFailures(compareRetrievalEval(summary(), summary()).rows)).toEqual([]); + }); + + it("fails a candidate below the pinned 1.0 even when the baseline was lower", () => { + const rows = compareRetrievalEval( + summary({ document_recall_at_5: 0.9 }), + summary({ document_recall_at_5: 0.95 }), + ).rows; + expect(recallFailures(rows)).toEqual(["document_recall_at_5 is 0.9500, below the pinned 1.0"]); + }); + + it("fails a content recall drop", () => { + const rows = compareRetrievalEval(summary(), summary({ content_recall_at_5: 0.9583 })).rows; + expect(recallFailures(rows)).toHaveLength(1); + expect(recallFailures(rows)[0]).toContain("content_recall_at_5"); + }); +}); diff --git a/tests/ledger-inbox.test.ts b/tests/ledger-inbox.test.ts index f7d89e3250..af7b88ed88 100644 --- a/tests/ledger-inbox.test.ts +++ b/tests/ledger-inbox.test.ts @@ -8,6 +8,7 @@ import { applyRequest, applyRequestBatch, planRequestBatch, + repairLedgerMojibake, validateRequest, } from "../scripts/ledger-inbox.mjs"; import { @@ -1121,3 +1122,15 @@ describe("ledger-inbox idempotent close and duplicate done handling", () => { }); }); }); + +describe("ledger preamble mojibake (#WWTRM5)", () => { + const broken = "- Say `/issues` in Claude Code \u0393\u00e5\u00c6 the skill reads this file\n"; + + it("repairs the damaged arrow", () => { + expect(repairLedgerMojibake(broken)).toBe("- Say `/issues` in Claude Code \u2192 the skill reads this file\n"); + }); + + it("leaves a batch with no requests byte for byte, so other PRs still match their base", () => { + expect(applyRequestBatch(broken + BASE_LEDGER, []).markdown).toBe(broken + BASE_LEDGER); + }); +}); diff --git a/tests/own-page-mode-home.test.ts b/tests/own-page-mode-home.test.ts index dcbdd04f8d..6efc7265f0 100644 --- a/tests/own-page-mode-home.test.ts +++ b/tests/own-page-mode-home.test.ts @@ -53,3 +53,14 @@ describe("modes with nothing to search open their own page", () => { expect(legacyHomeRedirectUrl(new URL("https://clinical-kb.test/?mode=documents"), "GET")).toBeNull(); }); }); + +describe("renamed mode links", () => { + it("sends /?mode=medications to the Medication mode instead of Answer, keeping the draft", () => { + expect(legacyHomeRedirectUrl(new URL("https://clinical-kb.test/?mode=medications"), "GET")?.toString()).toBe( + "https://clinical-kb.test/?mode=prescribing", + ); + expect( + legacyHomeRedirectUrl(new URL("https://clinical-kb.test/?mode=medication&q=lithium"), "GET")?.toString(), + ).toBe("https://clinical-kb.test/?mode=prescribing&q=lithium"); + }); +}); diff --git a/tests/teaching-assessments-model.test.ts b/tests/teaching-assessments-model.test.ts index 6df0dc16cc..26f22c8aca 100644 --- a/tests/teaching-assessments-model.test.ts +++ b/tests/teaching-assessments-model.test.ts @@ -91,6 +91,18 @@ describe("Teaching assessments: home on Mon 5 Oct (week 6)", () => { expect(twice.epaRequests).toHaveLength(1); }); + it("refuses a direct EPA record whose note looks like a patient detail or is too long", () => { + const patient = assessmentsReducer(s, { + type: "record-epa-direct", + epa: 1, + level: "minimal", + note: "Mrs Smith in bed 4", + }); + expect(patient).toBe(s); + const long = assessmentsReducer(s, { type: "record-epa-direct", epa: 1, level: "minimal", note: "x".repeat(5000) }); + expect(long).toBe(s); + }); + it("lets the supervisor record an EPA without a request, and Undo removes it", () => { const saved = assessmentsReducer(s, { type: "record-epa-direct", epa: 1, level: "minimal", note: " Clear plans " }); expect(epaRecords(saved)).toHaveLength(9);