From 0e221e47d65d8ca7d53ee81eb3244cd3ba311182 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 15:09:48 +0800 Subject: [PATCH 1/8] fix: OCR caution, scanner warning, table-fact label carry-forward and deleted-document titles (split from #3707) Owner-approved split of #3707 (21 Oct 2026, 14:42 AWST). Moved unchanged from #3707 (Claude's commits af2dfc0c4, 21587256d caution part, 2abb63507, 3752f2e3e, 4804078bc, 02bea1304, 0f7b56834): - scanned-page (OCR) source caution, kept inside the warning cap - worker start-up warning when the malware scanner module is not set, plus the decision record to keep the upload scan as is - table-fact merged row label carried into clinical_parameter (ING-B-09) - deleted document's title dropped from the storage clean-up ledger - the owner-approved temporary drift exception for source-governance tests --- data/repo-awareness-snapshot.json | 5 ++ docs/clinical-hazard-controls.json | 7 ++ ...1-no-malware-scan-before-upload-storage.md | 12 +++ docs/privacy-impact-assessment.md | 15 ++-- docs/worker-deploy-runbook.md | 6 +- scripts/cleanup-storage.ts | 2 + src/lib/ingestion.ts | 4 + src/lib/source-governance.ts | 31 +++++++- src/lib/types.ts | 1 + tests/ingestion.test.ts | 12 +++ tests/source-governance.test.ts | 75 +++++++++++++++++++ tests/table-facts.test.ts | 29 +++++++ tests/worker-admin-source-runtime.test.ts | 17 +++++ worker/admin-source-runtime.ts | 13 ++++ worker/main.ts | 8 +- worker/table-facts.ts | 21 +++++- 16 files changed, 245 insertions(+), 13 deletions(-) create mode 100644 docs/decisions/2026-10-21-no-malware-scan-before-upload-storage.md diff --git a/data/repo-awareness-snapshot.json b/data/repo-awareness-snapshot.json index c303c1742d..71184c7ca9 100644 --- a/data/repo-awareness-snapshot.json +++ b/data/repo-awareness-snapshot.json @@ -5463,6 +5463,11 @@ "section": "decisions", "catalogued": true }, + { + "path": "docs/decisions/2026-10-21-no-malware-scan-before-upload-storage.md", + "section": "decisions", + "catalogued": true + }, { "path": "docs/superpowers/rag-upgrade/execution-artifacts/rag-answer-quality-and-repository-coverage-v1/README.md", "section": "superpowers", diff --git a/docs/clinical-hazard-controls.json b/docs/clinical-hazard-controls.json index 42080f4aa8..510ffb4988 100644 --- a/docs/clinical-hazard-controls.json +++ b/docs/clinical-hazard-controls.json @@ -295,6 +295,13 @@ "recordedBy": "Engineering, Claude clinical content audit thread (audit 3, 20 Oct 2026), per owner request to fix the audit recommendations. Engineering drift record only; not clinical-owner approval, hazard re-review or reseal.", "recordedOn": "2026-10-20", "expiresOn": "2026-12-04" + }, + { + "path": "tests/source-governance.test.ts", + "reason": "PR #3707 adds regression tests for the new scanned-page (OCR) source caution and for keeping document-caution warnings inside the 8-warning cap. Owner approved this temporary exception by decision card on 21 Oct 2026; the caution wording still awaits the owner's review on that PR. The prior seal stands until the owner re-reviews and reseals this file.", + "recordedBy": "Claude Code, RAG and ingestion audit thread, PR #3707, on the owner's card approval 2026-10-21. Engineering drift record only; not clinical-owner approval or release signoff.", + "recordedOn": "2026-10-21", + "expiresOn": "2026-12-04" } ] } diff --git a/docs/decisions/2026-10-21-no-malware-scan-before-upload-storage.md b/docs/decisions/2026-10-21-no-malware-scan-before-upload-storage.md new file mode 100644 index 0000000000..9d2baa4b79 --- /dev/null +++ b/docs/decisions/2026-10-21-no-malware-scan-before-upload-storage.md @@ -0,0 +1,12 @@ +# Decision: no malware scan before uploads are stored, for now + +- **Status:** decided 2026-10-21 +- **Source:** the owner, in chat + +On 21 Oct 2026 the owner chose "Leave as is" by decision card in the project's RAG and ingestion +audit thread (audit ING-B-11). Ordinary uploads are still stored before any malware scan. They stay +owner-only and are never run as code, and the full malware check stays on the admin route. No +scanner is added to the ingestion worker for every upload. Revisit this if anyone other than the +owner can upload files. + +If this summary and the source ever differ, the source wins. diff --git a/docs/privacy-impact-assessment.md b/docs/privacy-impact-assessment.md index 84d9a9bf1d..449906db69 100644 --- a/docs/privacy-impact-assessment.md +++ b/docs/privacy-impact-assessment.md @@ -338,13 +338,14 @@ of the most important privacy items before real patient use (PIA-1). All three log tables are **owner-stamped** and **RLS-enabled** (owner-read for authenticated users; service-role for writes). Redaction is applied centrally at every write site. -| Table | Raw query stored? | Redaction mechanism | Other sensitive columns | RLS | -| -------------------- | --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| `rag_queries` | No (hash placeholder) | `queryTextForStorage` / `normalizedQueryTextForStorage` ([query-privacy.ts](../src/lib/query-privacy.ts)); centralized write in `insertRagQuery` | `answer` is null by default and stored only with explicit `RAG_PERSIST_ANSWER_TEXT=true`; `source_chunk_ids` (own data) | owner-read, [schema.sql](../supabase/schema.sql) | -| `rag_query_misses` | No (hash placeholder) | same helpers; writes in [search/route.ts](../src/app/api/search/route.ts), [interaction/route.ts](../src/app/api/search/interaction/route.ts) | `metadata.query_hash` | owner-read, [schema.sql](../supabase/schema.sql) | -| `rag_retrieval_logs` | No (hash placeholder) | same helpers; write at [search/route.ts](../src/app/api/search/route.ts) | retrieval telemetry only | owner-read, [schema.sql](../supabase/schema.sql) | -| `audit_logs` | N/A (no query text) | action/resource metadata only; the write boundary allowlists operational metadata and excludes user-controlled filenames/titles/content hashes ([audit.ts](../src/lib/audit.ts)). Migration `20260717163000` minimizes existing rows on deployment. | `owner_id`, `action`, `resource_id` | service-role-only, [schema.sql](../supabase/schema.sql) | -| `cme_evidence` | N/A (file metadata) | Mandatory user confirmations (`redaction_confirmed`, `preview_confirmed`); filename capped at 180 chars; files re-validated via Sharp; storage path strictly namespaced `owner_id/entry_id/id`. On owner removal, `file_name` is scrubbed to `'Removed file'`. | `storage_path`, `file_name`, `sha256`, `kind`, `removal_reason` (3–500 chars) | service-role-only (revoked from public, anon, authenticated; select, insert, update on removed_at, removal_reason, file_name granted to service_role) | +| Table | Raw query stored? | Redaction mechanism | Other sensitive columns | RLS | +| ------------------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `rag_queries` | No (hash placeholder) | `queryTextForStorage` / `normalizedQueryTextForStorage` ([query-privacy.ts](../src/lib/query-privacy.ts)); centralized write in `insertRagQuery` | `answer` is null by default and stored only with explicit `RAG_PERSIST_ANSWER_TEXT=true`; `source_chunk_ids` (own data) | owner-read, [schema.sql](../supabase/schema.sql) | +| `rag_query_misses` | No (hash placeholder) | same helpers; writes in [search/route.ts](../src/app/api/search/route.ts), [interaction/route.ts](../src/app/api/search/interaction/route.ts) | `metadata.query_hash` | owner-read, [schema.sql](../supabase/schema.sql) | +| `rag_retrieval_logs` | No (hash placeholder) | same helpers; write at [search/route.ts](../src/app/api/search/route.ts) | retrieval telemetry only | owner-read, [schema.sql](../supabase/schema.sql) | +| `audit_logs` | N/A (no query text) | action/resource metadata only; the write boundary allowlists operational metadata and excludes user-controlled filenames/titles/content hashes ([audit.ts](../src/lib/audit.ts)). Migration `20260717163000` minimizes existing rows on deployment. | `owner_id`, `action`, `resource_id` | service-role-only, [schema.sql](../supabase/schema.sql) | +| `storage_cleanup_jobs.document_title` | Until the deleted document's storage clean-up runs | Copied in by `delete_document_if_idle` when a document is deleted. Cleared to null by the delete route and by `npm run cleanup:storage` once the document row is gone (`buildStorageCleanupJobUpdate`, audit ING-A-05). Rows that finished before 21 Oct 2026 keep their title until a database backfill is approved. | Titles of deleted guidelines can identify things, so they are not kept after the delete. | +| `cme_evidence` | N/A (file metadata) | Mandatory user confirmations (`redaction_confirmed`, `preview_confirmed`); filename capped at 180 chars; files re-validated via Sharp; storage path strictly namespaced `owner_id/entry_id/id`. On owner removal, `file_name` is scrubbed to `'Removed file'`. | `storage_path`, `file_name`, `sha256`, `kind`, `removal_reason` (3–500 chars) | service-role-only (revoked from public, anon, authenticated; select, insert, update on removed_at, removal_reason, file_name granted to service_role) | ### 5.1 M15 HMAC query-hash fix — verified present, enforced in production diff --git a/docs/worker-deploy-runbook.md b/docs/worker-deploy-runbook.md index 1d331e3c6e..627c9d79e7 100644 --- a/docs/worker-deploy-runbook.md +++ b/docs/worker-deploy-runbook.md @@ -328,7 +328,8 @@ bytes change while it runs. Never write its value into a document, log, PR or ti Without it there is no scanner verdict, so every ingestion provider call is refused and each job ends as `blocked by ingress safety checks` with `scanner_unavailable` (fail closed; pinned by -`tests/edge-provider-egress.test.ts`). See [the current ingestion model](ingestion-state-machine.md#current-model-october-2026). +`tests/edge-provider-egress.test.ts`). A worker started without it logs one warning saying so +(`adminSourceRuntimeStartupWarning`). See [the current ingestion model](ingestion-state-machine.md#current-model-october-2026). **Current production state (read-only check, 20 October 2026).** The production `worker` service runs a placeholder start command set in Railway, not the image's `node dist/worker/index.mjs`. It is an HTTP-only @@ -338,7 +339,8 @@ indexed in production. The same check found `WORKER_ADMIN_SOURCE_RUNTIME_MODULE` Before the worker is switched back on, with the owner's explicit yes: 1. Confirm the module the variable names exists in the running image or mount and loads: the worker starts - without refusing it, and a first permitted job is not stopped as `scanner_unavailable`. + without refusing it, its start-up log has no `WORKER_ADMIN_SOURCE_RUNTIME_MODULE is not set` warning, and a + first permitted job is not stopped as `scanner_unavailable`. 2. Restore the real start command, then clear any pause switch. 3. Watch the first jobs; the library re-index stays stopped ([decision](decisions/2026-10-17-library-reindex-stopped.md)). diff --git a/scripts/cleanup-storage.ts b/scripts/cleanup-storage.ts index 554c4335b1..2774e7f9f5 100644 --- a/scripts/cleanup-storage.ts +++ b/scripts/cleanup-storage.ts @@ -270,6 +270,8 @@ async function main() { storage_removed: documentCleanup.removed + imageCleanup.removed, last_error: warnings.length ? warnings.join("; ") : null, completed_at: nextStatus === "completed" ? new Date().toISOString() : null, + // Only rows whose document is gone reach here (R11 guard above); drop the title (ING-A-05). + document_title: null, metadata: cleanupJobMetadata(job.metadata, "storage_cleanup_retry", { storage_warnings: warnings }), }) .eq("id", job.id); diff --git a/src/lib/ingestion.ts b/src/lib/ingestion.ts index 53c9f299cd..e7ffbeb3ae 100644 --- a/src/lib/ingestion.ts +++ b/src/lib/ingestion.ts @@ -75,6 +75,7 @@ export type StorageCleanupJobUpdate = { metadata: { operation: string; storage_warnings: string[] }; document_paths?: string[]; image_paths?: string[]; + document_title?: null; }; // Audit R11: the DELETE route creates the storage_cleanup_jobs ledger row (with @@ -108,6 +109,9 @@ export function buildStorageCleanupJobUpdate(args: { if (args.aborted) { update.document_paths = []; update.image_paths = []; + } else { + // Audit 2026-10-20 ING-A-05: the document row is gone, and its title can identify things. + update.document_title = null; } return update; } diff --git a/src/lib/source-governance.ts b/src/lib/source-governance.ts index b30cfbea1d..5275b4f235 100644 --- a/src/lib/source-governance.ts +++ b/src/lib/source-governance.ts @@ -38,6 +38,8 @@ export const GOVERNANCE_SEVERITY_MATRIX: Record = { @@ -52,6 +54,7 @@ export const GOVERNANCE_UI_TOKEN_MATRIX: Record([ SOURCE_GOVERNANCE_CODES.REVIEW_DUE, // A known error in the cited document itself (src/lib/document-cautions.ts). SOURCE_GOVERNANCE_CODES.DOCUMENT_CAUTION, + // Pages read from a scan by OCR (audit ING-B-07): the reader should check numbers on the page. + SOURCE_GOVERNANCE_CODES.OCR_TEXT, ]); function isLocalMetadataText(value: string) { @@ -226,6 +233,20 @@ export function sourceGovernanceWarnings(args: { }); } + // Audit 2026-10-20 ING-B-07: OCR can list a value apart from its qualifier or misread a decimal. + // The flag is document-level (index quality metrics), so the message says "include pages". + const ocrMetrics = result.indexing_quality?.metrics; + if (Number(ocrMetrics?.ocr_page_count ?? 0) > 0 || Number(ocrMetrics?.ocr_coverage ?? 0) > 0) { + pushUnique(warnings, { + code: SOURCE_GOVERNANCE_CODES.OCR_TEXT, + severity: GOVERNANCE_SEVERITY_MATRIX[SOURCE_GOVERNANCE_CODES.OCR_TEXT] as SourceGovernanceWarning["severity"], + uiToken: GOVERNANCE_UI_TOKEN_MATRIX[SOURCE_GOVERNANCE_CODES.OCR_TEXT] as SourceGovernanceUiToken, + message: OCR_TEXT_WARNING_MESSAGE, + document_id, + title, + }); + } + if (typeof result.indexing_quality?.quality_score === "number" && result.indexing_quality.quality_score < 0.45) { pushUnique(warnings, { code: SOURCE_GOVERNANCE_CODES.LOW_INDEX_QUALITY, @@ -285,7 +306,13 @@ export function sourceGovernanceWarnings(args: { const severityRank = { danger: 0, warning: 1, info: 2 } satisfies Record; return warnings - .sort((left, right) => severityRank[left.severity] - severityRank[right.severity]) + .sort( + (left, right) => + severityRank[left.severity] - severityRank[right.severity] || + // OCR_TEXT is a general caution; within a severity it must not crowd document-specific warnings out of the cap. + Number(left.code === SOURCE_GOVERNANCE_CODES.OCR_TEXT) - + Number(right.code === SOURCE_GOVERNANCE_CODES.OCR_TEXT), + ) .slice(0, args.limit ?? 8); } @@ -317,6 +344,8 @@ function groupedMessage(warning: SourceGovernanceWarning, count: number) { return `${plural(count, "source")} ${count === 1 ? "has" : "have"} low indexing quality.`; if (warning.code === SOURCE_GOVERNANCE_CODES.WEAK_TABLE_EXTRACTION) return `${plural(count, "table evidence item")} reviewed as administrative, unrelated, or poor extraction.`; + if (warning.code === SOURCE_GOVERNANCE_CODES.OCR_TEXT) + return `${plural(count, "source")} ${count === 1 ? "includes" : "include"} pages read from a scan by text recognition (OCR). Check numbers against the PDF page.`; if (warning.code === SOURCE_GOVERNANCE_CODES.REGISTRY_RECORD) return `${plural(count, "registry summary", "registry summaries")} used as supporting evidence.`; return warning.message; diff --git a/src/lib/types.ts b/src/lib/types.ts index c8789135ae..eedf521480 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -461,6 +461,7 @@ export const SOURCE_GOVERNANCE_CODES = { WEAK_TABLE_EXTRACTION: "weak_table_extraction", REGISTRY_RECORD: "registry_record_source", DOCUMENT_CAUTION: "document_caution", + OCR_TEXT: "ocr_text_source", } as const; export type SourceGovernanceCode = (typeof SOURCE_GOVERNANCE_CODES)[keyof typeof SOURCE_GOVERNANCE_CODES]; diff --git a/tests/ingestion.test.ts b/tests/ingestion.test.ts index 249520ea6e..d5fd8a0b41 100644 --- a/tests/ingestion.test.ts +++ b/tests/ingestion.test.ts @@ -166,4 +166,16 @@ describe("storage cleanup ledger update (R11)", () => { expect(update.last_error).toBeNull(); expect(update.document_paths).toBeUndefined(); }); + + // Audit 2026-10-20 ING-A-05: a deleted document's title can identify things, so it is not kept + // in the ledger once the document row is gone. An aborted delete leaves the document alive. + it.each(["completed", "failed"] as const)("clears the deleted document's title on a %s cleanup", (status) => { + const update = buildStorageCleanupJobUpdate({ status, storageRemoved: 1, warnings: [] }); + expect(update.document_title).toBeNull(); + }); + + it("leaves the title of a document whose delete aborted untouched", () => { + const update = buildStorageCleanupJobUpdate({ status: "failed", storageRemoved: 0, warnings: [], aborted: true }); + expect(update.document_title).toBeUndefined(); + }); }); diff --git a/tests/source-governance.test.ts b/tests/source-governance.test.ts index eb2b803c8f..77007d198c 100644 --- a/tests/source-governance.test.ts +++ b/tests/source-governance.test.ts @@ -88,6 +88,81 @@ describe("source governance warnings", () => { expect(codes).toContain("weak_table_extraction"); }); + // Audit 2026-10-20 ING-B-07: OCR can separate a value from its qualifier, and nothing told + // the reader a cited source had pages read from a scan. + it("cautions when a supporting source has pages read by OCR, without refusing the answer", () => { + const ocrSource = result({ + source_metadata: { ...result().source_metadata!, document_status: "current", extraction_quality: "good" }, + indexing_quality: { + document_id: "doc-1", + quality_score: 0.9, + extraction_quality: "good", + metrics: { ocr_page_count: 3, page_count: 10, ocr_coverage: 0.3 }, + issues: [], + }, + table_facts: [], + }); + const warnings = sourceGovernanceWarnings({ results: [ocrSource] }); + const ocr = warnings.find((warning) => warning.code === "ocr_text_source"); + expect(ocr).toMatchObject({ severity: "warning", uiToken: "caution", document_id: "doc-1" }); + expect(ocr?.message).toContain("from a scan"); + expect(hasDangerSourceGovernanceWarning(warnings.filter((warning) => warning.code === "ocr_text_source"))).toBe( + false, + ); + expect(frontendSourceGovernanceWarnings(warnings).map((warning) => warning.code)).toContain("ocr_text_source"); + expect(groupSourceGovernanceWarnings([ocr!, { ...ocr!, document_id: "doc-2" }])[0]?.message).toBe( + "2 sources include pages read from a scan by text recognition (OCR). Check numbers against the PDF page.", + ); + }); + + it("keeps a document caution inside the warning cap when many sources were read by OCR", () => { + const ocrMetrics = { + quality_score: 0.9, + extraction_quality: "good" as const, + metrics: { ocr_page_count: 3, ocr_coverage: 0.3 }, + issues: [], + }; + const base = { + ...result().source_metadata!, + document_status: "current" as const, + extraction_quality: "good" as const, + }; + const ocrSources = Array.from({ length: 5 }, (_, index) => + result({ + document_id: `doc-${index}`, + source_metadata: base, + indexing_quality: { document_id: `doc-${index}`, ...ocrMetrics }, + table_facts: [], + }), + ); + const cautioned = result({ + document_id: "doc-caution", + title: "Clozapine therapy RKPG", + file_name: "clozapine-therapy-rkpg.pdf", + source_metadata: base, + indexing_quality: { document_id: "doc-caution", ...ocrMetrics, metrics: { ocr_page_count: 0 } }, + table_facts: [], + }); + const warnings = sourceGovernanceWarnings({ results: [...ocrSources, cautioned] }); + expect(warnings).toHaveLength(8); + expect(warnings.map((warning) => warning.code)).toContain("document_caution"); + }); + + it("adds no OCR caution for a source with no OCR pages", () => { + const plain = result({ + indexing_quality: { + document_id: "doc-1", + quality_score: 0.9, + extraction_quality: "good", + metrics: { ocr_page_count: 0 }, + issues: [], + }, + }); + expect(sourceGovernanceWarnings({ results: [plain] }).map((warning) => warning.code)).not.toContain( + "ocr_text_source", + ); + }); + it("groups repeated document-level warnings into one counted message", () => { const grouped = groupSourceGovernanceWarnings([ { diff --git a/tests/table-facts.test.ts b/tests/table-facts.test.ts index bad87bb0ec..982e676f93 100644 --- a/tests/table-facts.test.ts +++ b/tests/table-facts.test.ts @@ -44,6 +44,35 @@ describe("table fact generation", () => { }); }); + // Audit 2026-10-20 ING-B-09: a vertically merged label cell comes back blank on later rows. + it("carries a merged row label forward into clinical_parameter without rewriting the cells", () => { + const facts = buildTableFactRows({ + job: { document_id: "doc-1", documents: { owner_id: "owner-1" } }, + chunkRows: [], + insertedImages: [ + { + id: "image-1", + pageNumber: 2, + tableTitle: "Synthetic monitoring table", + tableRows: [ + ["Group A", "Level 1", "First instruction"], + ["", "Level 2", "Second instruction"], + ["Group B", "Level 3", "Third instruction"], + ], + tableColumns: ["Parameter", "Threshold", "Action"], + }, + ], + }); + + expect(facts.map((fact) => fact.clinical_parameter)).toEqual(["Group A", "Group A", "Group B"]); + expect(facts[1]).toMatchObject({ row_label: null, threshold_value: "Level 2" }); + expect(facts[1]?.metadata).toMatchObject({ + cells: ["", "Level 2", "Second instruction"], + row_label_inherited_from: { row_index: 0, label: "Group A" }, + }); + expect(facts[0]?.metadata).not.toHaveProperty("row_label_inherited_from"); + }); + it("deduplicates repeated table rows from the same image", () => { const facts = buildTableFactRows({ job: { diff --git a/tests/worker-admin-source-runtime.test.ts b/tests/worker-admin-source-runtime.test.ts index 2e969e268d..b0c1db166f 100644 --- a/tests/worker-admin-source-runtime.test.ts +++ b/tests/worker-admin-source-runtime.test.ts @@ -14,6 +14,7 @@ import { loadAdminSourceRuntimeOptions, type ApprovedAdminSource, type AdminSourceInitializationContext, + adminSourceRuntimeStartupWarning, } from "../worker/admin-source-runtime"; import { workerProviderSourceBinding } from "../worker/provider-egress"; import type { JobRow } from "../worker/types"; @@ -387,3 +388,19 @@ describe("approved admin source normal runtime composition", () => { ).rejects.toThrow("current_input_required"); }); }); + +// Audit 2026-10-20 CH-04: without the scanner module every ingestion provider call is refused, +// so a worker started without it must say so at start-up instead of failing job by job. +describe("scanner module start-up warning (CH-04)", () => { + it("warns when the scanner module setting is absent or empty", () => { + for (const value of [undefined, ""]) { + const warning = adminSourceRuntimeStartupWarning(value); + expect(warning).toContain("WORKER_ADMIN_SOURCE_RUNTIME_MODULE is not set"); + expect(warning).toContain("scanner_unavailable"); + } + }); + + it("stays quiet when the setting is present and never echoes its value", () => { + expect(adminSourceRuntimeStartupWarning("/srv/synthetic/runtime.mjs")).toBeNull(); + }); +}); diff --git a/worker/admin-source-runtime.ts b/worker/admin-source-runtime.ts index 053af24977..cd1a85d943 100644 --- a/worker/admin-source-runtime.ts +++ b/worker/admin-source-runtime.ts @@ -421,6 +421,19 @@ export function createAdminSourceRuntimeOptions( }; } +/** + * Audit 2026-10-20 CH-04: without the scanner module there is no malware verdict, so every + * ingestion provider call is refused and each job ends as `scanner_unavailable`. Say so once at + * start-up. Only whether the setting is present is reported, never its value. + */ +export function adminSourceRuntimeStartupWarning(modulePath = process.env.WORKER_ADMIN_SOURCE_RUNTIME_MODULE) { + if (modulePath !== undefined && modulePath !== "") return null; + return ( + "WORKER_ADMIN_SOURCE_RUNTIME_MODULE is not set: there is no malware scanner, so every ingestion " + + "provider call will be refused and jobs will fail as scanner_unavailable (see docs/worker-deploy-runbook.md)." + ); +} + /** Explicit server-controlled local module/mount. No remote or request paths. */ export async function loadAdminSourceRuntimeOptions( client: SupabaseClient, diff --git a/worker/main.ts b/worker/main.ts index 4ba3f2e8c7..51a661875e 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -89,7 +89,11 @@ import { type ManagedProviderEmbeddingRequestContext, } from "./managed-provider-vectors"; import type { ManagedAdminTextSourceSectionLayout } from "./managed-admin-text-terminal"; -import { loadAdminSourceRuntimeOptions, type AdminSourceInitializationContext } from "./admin-source-runtime"; +import { + adminSourceRuntimeStartupWarning, + loadAdminSourceRuntimeOptions, + type AdminSourceInitializationContext, +} from "./admin-source-runtime"; // The ingestion stages below were extracted from this closure; each receives the runtime's // services through IngestionStageContext rather than closing over them. import { insertEmbeddedChunks } from "./chunk-indexing"; @@ -855,6 +859,8 @@ export async function runWorkerMain() { createAdminClient({ timeoutMs: resolveWorkerSupabaseTimeoutMs(process.env) }), (stopSignal) => controlPlaneAbortSignal(stopSignal), ); + const scannerWarning = adminSourceRuntimeStartupWarning(); + if (scannerWarning) console.warn(scannerWarning); const adminSourceOptions = await loadAdminSourceRuntimeOptions(client, env.SUPABASE_DOCUMENT_BUCKET); await createWorkerRuntime({ client, ...adminSourceOptions }).run(); process.exit(0); diff --git a/worker/table-facts.ts b/worker/table-facts.ts index 6055ecd822..60f1cebbab 100644 --- a/worker/table-facts.ts +++ b/worker/table-facts.ts @@ -179,6 +179,10 @@ export function buildTableFactRows(args: { // IDX-H6: raised from 120 to 400 to match the Python extractor's MAX_TABLE_ROWS so tail // rows of long dose/threshold tables are not silently dropped from the facts layer. + // Audit 2026-10-20 ING-B-09: PyMuPDF leaves a vertically merged label cell blank on every row + // after the first, so those rows lost their group. Carry the last label forward into + // clinical_parameter only; row_label, cells and chunk text keep the extracted (blank) value. + let lastRowLabel: { label: string; rowIndex: number } | null = null; for (const [rowIndex, rawCells] of image.tableRows.slice(0, 400).entries()) { const cells = rawCells.map((cell) => compactSearchText(cell, 240)); // Native table extraction can include the column headings as its first row. @@ -193,6 +197,8 @@ export function buildTableFactRows(args: { const rowText = cells.filter(Boolean).join(" "); if (!rowText) continue; const rowLabel = tableFactValue(cells, parameterIndex >= 0 ? parameterIndex : 0); + const inheritedLabel = rowLabel ? null : lastRowLabel; + if (rowLabel) lastRowLabel = { label: rowLabel, rowIndex }; // Named columns constrain meaning: an unmatched concentration or interval // remains in metadata.cells without being relabelled a threshold or action. const threshold = @@ -208,7 +214,15 @@ export function buildTableFactRows(args: { : columns.length === 0 ? (cells[cells.length - 1] ?? null) : null; - const dedupeKey = [image.id, rowLabel, threshold, action, rowText, image.tableTitle, image.tableLabel] + const dedupeKey = [ + image.id, + rowLabel ?? inheritedLabel?.label, + threshold, + action, + rowText, + image.tableTitle, + image.tableLabel, + ] .map((value) => String(value ?? "").toLowerCase()) .join("\u001f"); if (seen.has(dedupeKey)) continue; @@ -222,12 +236,15 @@ export function buildTableFactRows(args: { page_number: image.pageNumber ?? sourceChunk?.page_number ?? null, table_title: image.tableTitle ?? image.tableLabel, row_label: rowLabel, - clinical_parameter: rowLabel ?? image.tableTitle ?? image.tableLabel, + clinical_parameter: rowLabel ?? inheritedLabel?.label ?? image.tableTitle ?? image.tableLabel, threshold_value: threshold, action, normalized_terms: normalizedTerms(`${image.tableTitle ?? ""} ${image.tableLabel ?? ""} ${rowText}`), metadata: { row_index: rowIndex, + ...(inheritedLabel + ? { row_label_inherited_from: { row_index: inheritedLabel.rowIndex, label: inheritedLabel.label } } + : {}), columns, cells, table_role: image.tableRole, From 4bc23745b4e2e93fc870b780c64b5d16806b7306 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 15:48:03 +0800 Subject: [PATCH 2/8] chore(governance): owner-approved hazard-controls reseal and hazard-analysis anchor refresh (source-governance.ts:364 -> :393) Owner approval typed 2026-10-21 15:46 AWST on #3715. Sealed with npm run governance:seal-hazard-controls. --- docs/clinical-hazard-analysis.md | 2 +- docs/clinical-hazard-controls.json | 148 ++++------------------------- 2 files changed, 18 insertions(+), 132 deletions(-) diff --git a/docs/clinical-hazard-analysis.md b/docs/clinical-hazard-analysis.md index b91b375506..a96be2bfb5 100644 --- a/docs/clinical-hazard-analysis.md +++ b/docs/clinical-hazard-analysis.md @@ -90,7 +90,7 @@ So an unvalidated, unknown-provenance document is routinely presented as a high- | ID | Failure scenario | Severity | Plaus. | Control | Test | Gap | | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | --------------------------- | ------------------------------------------------------------------------------------------------ | -------------------------------------------------------------- | ---------------------------------------------------------------------------- | | **H2b** | An `outdated` chunk out-ranks a current one. Freshness/status penalties exist (`-0.04` at [`clinical-search.ts:1646`](../src/lib/clinical-search.ts), [`rag-second-stage.ts:100`](../src/lib/rag/rag-second-stage.ts); date-decay in `ranking-config.ts`) but are an order of magnitude below relevance boosts (`+0.06…+0.38`) and are **zero when dates are null** (the default). | High | Medium (needs metadata set) | small penalties [`clinical-search.ts:1646`](../src/lib/clinical-search.ts) | `ranking-config.test.ts` proves the _math_, not the _ordering_ | No "current beats outdated on tie" rule; penalty is summed flat and clamped. | -| **H2c** | The one hard block, `sourceGovernanceRefusalAnswer` ([`source-governance.ts:57`](../src/lib/source-governance.ts)), fires only on `severity==="danger"` — i.e. literal `document_status==="outdated"` or `extraction_quality==="poor"` ([`source-governance.ts:164`](../src/lib/source-governance.ts)). Because `outdated` is essentially never set, this block is dormant in practice. | High | Medium | `hasDangerSourceGovernanceWarning` [`source-governance.ts:364`](../src/lib/source-governance.ts) | `tests/source-governance.test.ts` | Depends on metadata that the pipeline never populates. | +| **H2c** | The one hard block, `sourceGovernanceRefusalAnswer` ([`source-governance.ts:57`](../src/lib/source-governance.ts)), fires only on `severity==="danger"` — i.e. literal `document_status==="outdated"` or `extraction_quality==="poor"` ([`source-governance.ts:164`](../src/lib/source-governance.ts)). Because `outdated` is essentially never set, this block is dormant in practice. | High | Medium | `hasDangerSourceGovernanceWarning` [`source-governance.ts:393`](../src/lib/source-governance.ts) | `tests/source-governance.test.ts` | Depends on metadata that the pipeline never populates. | | **H2d** | `review_due` is filtered **out** of the frontend banner (`frontendVisibleWarningCodes` omits it, [`source-governance.ts:84`](../src/lib/source-governance.ts)); a review-due source surfaces with only a badge tone. | Medium | Low | — | test asserts it is _hidden_ | No review-due caveat in the answer prose or the warnings channel. | ### H3 — A safety caveat present on screen is stripped on copy/export diff --git a/docs/clinical-hazard-controls.json b/docs/clinical-hazard-controls.json index 4a180d62df..6cd416053c 100644 --- a/docs/clinical-hazard-controls.json +++ b/docs/clinical-hazard-controls.json @@ -160,148 +160,34 @@ ], "reviewedPathDigests": { "docs/audit/2026-09-17-external-audit-refutations.md": "c32a15b9c364c35fe7134784d28d4598dbf26f62dc8fdf01aa4a6646b5bd1d82", - "docs/clinical-governance.md": "1a87a39f18965ab0ad8b89b69865e323d5101f5f08c592fa6d23620fb501b429", - "docs/clinical-hazard-analysis.md": "a45cfc864d86bff7089e2b495df1f2debaddd3931424439bf823ac09a52cc30c", - "src/components/clinical-dashboard/patient-profile-panel.tsx": "c81ce6a9d80bdc749f9ebfac6bda7578b97f66acf7daaeb8827b4c7b3e6b0628", - "src/data/dictionary-source-dispositions.json": "3d03bd0be2d699f5dc7eb50338b535112a7dea57daf0f210de9ae241c606c2b8", - "src/lib/answer-render-policy.ts": "a5e2b36c4bdf6c2081d443d10e87c0ced403abfb129454d6989ad93af737f4a9", - "src/lib/answer-verification.ts": "73e2a66d3b50ae66b415f57366de7f688d098dd9315aeefa38c7e8fd426f8dfc", - "src/lib/clinical-safety.ts": "8fa7b5fca92f2b12b6f241d32feadd9310ee075a38e55e10b5bbce4c26c1173b", + "docs/clinical-governance.md": "4a21c4ee500e1e7953b4fc975206cb6c06dddcc4b436453c09d7ea60e42dc598", + "docs/clinical-hazard-analysis.md": "d6738ce036b76445f624b80ff7252ab0af1bed85ec3de6fc5cffa653f8c81b0f", + "src/components/clinical-dashboard/patient-profile-panel.tsx": "0eacd7447d883195e2ca9b512034018e445c45249d7394355730b8d11e3a1a43", + "src/data/dictionary-source-dispositions.json": "afc5760d7567803c512c7e0a2cf5826cdc27b3c152db9c0aadf7f66d3e1c6141", + "src/lib/answer-render-policy.ts": "f907ceedb16daa5e5338ce3123c58c366ed96492fd0d114dd7ce3b7b9b5f1d88", + "src/lib/answer-verification.ts": "fa2429b2ac14f51d00a8ab1b3754f05bedda3b6b30b56a585a16fbfe411c7eb4", + "src/lib/clinical-safety.ts": "f779a97793e0015b558e9c7a738e606111c85f09b005661ca8c4bba0c9d88f73", "src/lib/rag/rag-answer-support.ts": "4250f30017789f6c5d75e84e7a291f163f08abb95d615fb2368e904956841dd5", "src/lib/rag/rag-claim-support.ts": "a945a3aa1166590010404fbae71ac7a3cc1a8ed4036b68164667adea84e21b47", - "src/lib/rag/rag-source-block.ts": "984b3f655b468aabbf2d1f489eb6df66273fe7c9e85ea88df63affa986654ad2", + "src/lib/rag/rag-source-block.ts": "8d08e9d168fdd3e258f4b56e22db2b07541e62034a86f7ad90c5a58a8cdd38af", "src/lib/service-ranker.ts": "bbdbe3629520a22104194fc727530c05614dfba5528d9b9d8d5743acbfa9ea44", - "src/lib/service-urgent-routing.ts": "663b0d131fa4457a4df9875fb200efe1520c9ebda75b552a8bbd68eb66264a7f", - "src/lib/source-authority-registry.ts": "e82b2f1aeaf053a321f76576da71a2c54f7fb4db485fbf6c4caf686ac6f2267c", - "src/lib/source-governance.ts": "e3e1c09b0a7e446b7055f894088c771f1a5cc160a0f20f026f3cc291af5804fa", + "src/lib/service-urgent-routing.ts": "163245433081cc4474d6c54993f709100d9e636b001f0cd17904bc61f24dc285", + "src/lib/source-authority-registry.ts": "938bbbb1d26bdb88acbb9cc15f43fc6d809a13de49e6a47d745c992aaba3f813", + "src/lib/source-governance.ts": "76a2c8ef50ce8f8abb5842e7c000e90a82a97b29945bbcf49149ac3f1664c99e", "src/lib/source-text-sanitizer.ts": "cb6c869b520a8be03d40bae1206a87b6d53318573baeaec664b35c353fa1bd80", - "tests/answer-render-policy.test.ts": "5e48ea66ca1902ac17a271444db7f6350a37c0100fe2152afd042e4fdb89427d", - "tests/answer-verification.test.ts": "2beac1a1f96cb8afae395a51cd5d2a91e8f45ff45723bf3e7641ec85c2605050", - "tests/clinical-safety.test.ts": "053b82e3523a5aa55b40059a558e8992fe5342ebedd8812da10193977cad43f6", + "tests/answer-render-policy.test.ts": "e7ddb86442ecdafd01bc6d1b94791e41b98c91c0d313ab03a532a1cc9fc60465", + "tests/answer-verification.test.ts": "3c5f2d1d805067f17a2bfcc47c7a92f7362271004ebad071b5b657c9d7d34844", + "tests/clinical-safety.test.ts": "96d950e946af0bcc2cb3f517fc372d28e67d93099e00b07b5d3c854fa1a020e8", "tests/rag-claim-support.test.ts": "79f19e142dff92ea92cb3635bd032f081885a1714d1ac745eb3d0f9151247586", "tests/rag-content-accuracy.test.ts": "6aaacb7fb9b43711696ba244a56cd31736fb3a884690774cde4f4db6ee206eb4", "tests/rag-injection.test.ts": "9a6fa43c799232d9e38d334b732c99dfaad5a5192ed64a7fcb8d6cb373300931", "tests/rag-retrieval-row-contract.test.ts": "4da5fe01a76dfa5b4e06e02303f136ff3fb0a1a1219cfe09ec158a1eae074fb6", "tests/rag-score.test.ts": "5a298ae040e7110d89fd1f68ad1d52b16994c035845a38b4db72ef9800c2b17f", "tests/rag-trust.test.ts": "416568e19e6ab8e67267ca53183b86a1703193543b4253269162d28736606eae", - "tests/services-catalog.test.ts": "b66181ab8226aff658dc7d8f97d851e822b5389a3722a6ec90560ec6a50e77ff", + "tests/services-catalog.test.ts": "9de1e48720aa15ac4b60632fb7841579c707e14bb47c2f0e2dce407c41bad190", "tests/services-safety-regressions.test.ts": "0515a77320cf9abc0b26ed3320f07d4b8e05b0eaed79ba031840b000735fb9df", "tests/services-safety-routing.test.ts": "e54f0a0f0f11810848de0b178c5398e9ac158b90a814d2005986d9e0b3853f33", - "tests/source-governance.test.ts": "0e9e126370a5500dbb72c722610fe1268c7e2f32e6b20b0a4f3ea401c54a645d", + "tests/source-governance.test.ts": "e8e995c1859f360719e194df9d33c068933f2f0f7d22828ddc54639370a5ce8b", "tests/ward-output.test.ts": "bb9f417d63c769b8e059f41fe2c9a5c52f772cedd13e6352408b4a7716e00222" - }, - "driftExceptions": [ - { - "path": "src/lib/answer-verification.ts", - "reason": "PR #3546 preserves the independently code-reviewed numeric age-unit correction (candidate SHA256 559b62cba08fedf9330e2bd907442de08cee204dbc6f395b99e2d849627eb5c2). Recorded-response delivery/copy and unsafe-output refusals passed; integrated hosted acceptance and owner hazard re-review remain pending. Preserve the prior seal and safety checks rather than imply renewed clinical approval.", - "recordedBy": "Codex ingestion engineering controller, PR #3546. Engineering drift record only; not clinical-owner approval or release signoff.", - "recordedOn": "2026-10-14", - "expiresOn": "2026-10-21" - }, - { - "path": "tests/answer-verification.test.ts", - "reason": "PR #3628 adds regression tests for the 'no more than' comparator/unit parsing in src/lib/answer-verification.ts (itself under the existing PR #3546 drift exception). Engineering drift exception per owner ruling 2026-10-18: the owner will review and reseal this test file before this exception expires. Not a clinical review, sign-off or reseal.", - "recordedBy": "Engineering, Grok Bot sweep session for PR #3628, per owner ruling 2026-10-18. Engineering drift record only; not clinical-owner approval or release signoff.", - "recordedOn": "2026-10-18", - "expiresOn": "2026-12-02" - }, - { - "path": "src/lib/clinical-safety.ts", - "reason": "Drift predates the blocking gate. Safety-behaviour changes since the 2026-08-23 review (e.g. 41b5a025a escalate/monitoring matching, 64db243a9 stop-instruction labelling, 1aa717e88 no all-clear over a failed medication check) need clinical re-review, then a re-seal.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "src/components/clinical-dashboard/patient-profile-panel.tsx", - "reason": "Drift predates the blocking gate. Patient-profile changes since the 2026-08-23 review (3da15e805 clear generation, 8d49b16c4 number field) need re-review, then a re-seal.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "tests/clinical-safety.test.ts", - "reason": "Drift predates the blocking gate. Test changes accompanying the clinical-safety.ts changes above; re-seal together with it.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "docs/clinical-hazard-analysis.md", - "reason": "Changed by PR #3019 (1a3d89350) after the 2026-08-23 review: adds proposed hazard H7, the Mental Health Act timeline misstating a statutory deadline (H7a-c, all Critical). Needs owner review and a register entry, then a re-seal. PR #3146 (owner approval 2026-09-28): H8 services crisis routing rewritten to the routing and tests that exist and registered as H8 in this register. Not resealed because the H7 change above still awaits review. 2026-10-20 (regulatory audit R-01/R-02): adds the amended TGA exemption section in force from 1 November 2026 and the PR-4 status reversal; regulatory text only, no hazard, control or severity changed.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "src/lib/source-authority-registry.ts", - "reason": "Changed on main by 309656a8 (#T9MZ7V, merge the duplicate American Psychiatric Association register entry) after the 2026-08-23 review; arrived in this branch via a main merge. Main only warns on drift; this branch makes drift blocking. Needs owner re-review, then a re-seal. PR #3146 (owner approval 2026-09-28): comment on descriptive publishers updated; 4AT stays unregistered. Not resealed because the earlier change above still awaits review.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "src/lib/answer-render-policy.ts", - "reason": "Changed on main by #3031 (#WGMB4Z answer render-trust cap, #GHC4XZ) after the last seal; arrived in this branch via a main merge. Main only warns on drift; this branch makes drift blocking. Needs owner re-review, then a re-seal.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "tests/answer-render-policy.test.ts", - "reason": "Changed on main by #3031 (#WGMB4Z answer render-trust cap, #GHC4XZ) after the last seal; arrived in this branch via a main merge. Main only warns on drift; this branch makes drift blocking. Needs owner re-review, then a re-seal.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "src/data/dictionary-source-dispositions.json", - "reason": "Source-disposition changes in PR #3067 (owner-approved publisher page reads for ledger items #JHT39N, #WY625D, #6X06YS, #YDENFM, #KVYHWZ; held/admitted counts and per-source blockers updated), and the NSW MSE handbook recorded as retired later on 2026-09-26 on the owner's approval, are pending owner review, then a re-seal. PR #3146 (owner approval 2026-09-28): AIHW-NOCC-2026 stays held and gains a registration-date blocker. Not resealed because earlier drift on this path still awaits review.", - "recordedBy": "BigSimmo (owner), decision 2026-09-26. Engineering, not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-26", - "expiresOn": "2026-10-26" - }, - { - "path": "src/lib/rag/rag-source-block.ts", - "reason": "PR #3103 (#JYH1FH, step 1 of the locally_reviewed label repair): the model-facing source line reads \"clinical validation: unverified (endorsed by issuing WA service)\" for the new wa_document_control_endorsement basis. No live document carries that basis until the separately approved data migration, so current output is unchanged. Pending owner review, then a re-seal.", - "recordedBy": "BigSimmo (owner), decision 2026-09-26 (\"record the two step 1 files as changed and awaiting my review\"). Engineering, not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-26", - "expiresOn": "2026-11-09" - }, - { - "path": "src/lib/source-governance.ts", - "reason": "PR #3103 (#JYH1FH, step 1 of the locally_reviewed label repair): claim-evidence eligibility accepts the wa_document_control_endorsement basis exactly as it accepts locally_reviewed, and the unverified-source warning gets honest wording for it (same code and severity). No live document carries that basis until the separately approved data migration. Pending owner review, then a re-seal.", - "recordedBy": "BigSimmo (owner), decision 2026-09-26 (\"record the two step 1 files as changed and awaiting my review\"). Engineering, not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-26", - "expiresOn": "2026-11-09" - }, - { - "path": "docs/clinical-governance.md", - "reason": "This combined change replaces the 2026-09-17 name-them-never-score-them rule with the 2026-10-03 transparent reference aid wording. The cited file drifted after the 2026-08-23 review and has not had a new clinical review.", - "recordedBy": "Engineering, combined-PR session 2026-10-06. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-10-06", - "expiresOn": "2026-11-20" - }, - { - "path": "src/lib/service-urgent-routing.ts", - "reason": "Audit 3 finding R4-F-01: medical-emergency phrases (seizure, unconscious or unconsciousness, loss of consciousness, not breathing, collapse or collapsed, anaphylaxis) now pin 000 first, ahead of regional crisis routes and of the Alcohol and Drug Support Line for alcohol withdrawal seizure. Behaviour widens urgent routing only; owner hazard re-review and reseal (H8) pending.", - "recordedBy": "Engineering, Claude clinical content audit thread (audit 3, 20 Oct 2026), per owner request to fix the audit recommendations. Engineering drift record only; not clinical-owner approval, hazard re-review or reseal.", - "recordedOn": "2026-10-20", - "expiresOn": "2026-12-04" - }, - { - "path": "tests/services-catalog.test.ts", - "reason": "Audit 3 findings R4-F-06, R4-F-10, R4-F-11 and R6-F-10: tests added or updated for superseded legacy crisis duplicates, corrected service contacts, phone bracket normalisation and no confidence chip on unverified legacy records. Owner hazard re-review and reseal (H8) pending.", - "recordedBy": "Engineering, Claude clinical content audit thread (audit 3, 20 Oct 2026), per owner request to fix the audit recommendations. Engineering drift record only; not clinical-owner approval, hazard re-review or reseal.", - "recordedOn": "2026-10-20", - "expiresOn": "2026-12-04" - }, - { - "path": "tests/source-governance.test.ts", - "reason": "PR #3707 adds regression tests for the new scanned-page (OCR) source caution and for keeping document-caution warnings inside the 8-warning cap. Owner approved this temporary exception by decision card on 21 Oct 2026; the caution wording still awaits the owner's review on that PR. The prior seal stands until the owner re-reviews and reseals this file.", - "recordedBy": "Claude Code, RAG and ingestion audit thread, PR #3707, on the owner's card approval 2026-10-21. Engineering drift record only; not clinical-owner approval or release signoff.", - "recordedOn": "2026-10-21", - "expiresOn": "2026-12-04" - } - ] + } } From 1ac8a3a3b241471aaef0e8229af4a37a9ccd341d Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 17:20:18 +0800 Subject: [PATCH 3/8] fix(table-facts): carry row labels down only into extractor-confirmed merged cells; reset on empty rows --- tests/extracted-document-validation.test.ts | 2 +- tests/table-facts.test.ts | 66 +++++++++++++++++++ worker/image-captioning.ts | 3 + worker/image-classification.ts | 6 ++ worker/python/extract_pdf_assets.py | 26 +++++++- .../test_extract_pdf_assets_merged_cells.py | 40 +++++++++++ .../test_extract_pdf_assets_raster_tables.py | 2 +- worker/table-facts.ts | 19 +++++- 8 files changed, 157 insertions(+), 7 deletions(-) create mode 100644 worker/python/test_extract_pdf_assets_merged_cells.py diff --git a/tests/extracted-document-validation.test.ts b/tests/extracted-document-validation.test.ts index 30b931eb25..5a2c38ad9c 100644 --- a/tests/extracted-document-validation.test.ts +++ b/tests/extracted-document-validation.test.ts @@ -114,7 +114,7 @@ describe("Python extracted-document payload validation", () => { describe("extraction provenance (audit F04)", () => { const provenance = { name: "extract_pdf_assets", - version: "2026-10-20.embedded-image-ceiling-v10", + version: "2026-10-21.merged-cell-provenance-v11", pymupdf: "1.26.0", tableStrategy: "table_aware_page_text", }; diff --git a/tests/table-facts.test.ts b/tests/table-facts.test.ts index 982e676f93..af798771ea 100644 --- a/tests/table-facts.test.ts +++ b/tests/table-facts.test.ts @@ -60,6 +60,7 @@ describe("table fact generation", () => { ["Group B", "Level 3", "Third instruction"], ], tableColumns: ["Parameter", "Threshold", "Action"], + tableMergedCells: [[1, 0]], }, ], }); @@ -73,6 +74,71 @@ describe("table fact generation", () => { expect(facts[0]?.metadata).not.toHaveProperty("row_label_inherited_from"); }); + const carryFacts = (tableRows: string[][], tableMergedCells?: [number, number][] | null) => + buildTableFactRows({ + job: { document_id: "doc-1", documents: { owner_id: "owner-1" } }, + chunkRows: [], + insertedImages: [ + { + id: "image-1", + pageNumber: 2, + tableTitle: "Synthetic monitoring table", + tableRows, + tableColumns: ["Parameter", "Threshold", "Action"], + tableMergedCells, + }, + ], + }); + + it("does not carry a label into a blank cell without confirmed merge provenance", () => { + const rows = [ + ["Group A", "Level 1", "First instruction"], + ["", "", "Continuation note for the table"], + ]; + for (const facts of [carryFacts(rows), carryFacts(rows, null), carryFacts(rows, [[1, 1]])]) { + expect(facts[1]?.clinical_parameter).toBe("Synthetic monitoring table"); + expect(facts[1]?.metadata).not.toHaveProperty("row_label_inherited_from"); + } + }); + + it("carries a confirmed merged label down a multi-row span only", () => { + const facts = carryFacts( + [ + ["Group A", "Level 1", "First instruction"], + ["", "Level 2", "Second instruction"], + ["", "Level 3", "Third instruction"], + ["", "Level 4", "Unmerged blank label"], + ], + [ + [1, 0], + [2, 0], + ], + ); + expect(facts.map((fact) => fact.clinical_parameter)).toEqual([ + "Group A", + "Group A", + "Group A", + "Synthetic monitoring table", + ]); + }); + + it("resets inheritance across an empty separator row even when the next cell is marked merged", () => { + const facts = carryFacts( + [ + ["Group A", "Level 1", "First instruction"], + ["", "", ""], + ["", "Level 2", "Second instruction"], + ], + [ + [1, 0], + [2, 0], + ], + ); + expect(facts).toHaveLength(2); + expect(facts[1]?.clinical_parameter).toBe("Synthetic monitoring table"); + expect(facts[1]?.metadata).not.toHaveProperty("row_label_inherited_from"); + }); + it("deduplicates repeated table rows from the same image", () => { const facts = buildTableFactRows({ job: { diff --git a/worker/image-captioning.ts b/worker/image-captioning.ts index 1f3c160a92..cc5c859cec 100644 --- a/worker/image-captioning.ts +++ b/worker/image-captioning.ts @@ -74,6 +74,7 @@ type InsertedImage = { accessibleTableMarkdown: string | null; tableRows: string[][] | null; tableColumns: string[] | null; + tableMergedCells: [number, number][] | null; structuredVisualProfile: StructuredVisualProfile | null; candidatePriorityScore: number; imageQualityScore: number; @@ -545,6 +546,7 @@ async function persistCaptionedImages( table_confidence: tableMetadata.tableConfidence, table_rows: tableMetadata.tableRows, table_columns: tableMetadata.tableColumns, + table_merged_cells: tableMetadata.tableMergedCells, accessible_table_markdown: tableMetadata.accessibleTableMarkdown, clinical_use_class: policyAssessment.clinical_use_class, clinical_use_reason: policyAssessment.clinical_use_reason, @@ -595,6 +597,7 @@ async function persistCaptionedImages( accessibleTableMarkdown: tableMetadata.accessibleTableMarkdown, tableRows: tableMetadata.tableRows, tableColumns: tableMetadata.tableColumns, + tableMergedCells: tableMetadata.tableMergedCells, structuredVisualProfile: structuredVisualProfile ?? null, candidatePriorityScore: candidate.candidatePriorityScore, imageQualityScore: candidate.imageQualityScore, diff --git a/worker/image-classification.ts b/worker/image-classification.ts index 56990938e8..e73e208ced 100644 --- a/worker/image-classification.ts +++ b/worker/image-classification.ts @@ -89,6 +89,12 @@ export function imageTableMetadata(image: ExtractedDocument["images"][number]) { accessibleTableMarkdown: metadataString(metadata, "accessible_table_markdown") ?? tableText, tableRows: Array.isArray(metadata.table_rows) ? metadata.table_rows : null, tableColumns: Array.isArray(metadata.table_columns) ? metadata.table_columns : null, + tableMergedCells: Array.isArray(metadata.table_merged_cells) + ? (metadata.table_merged_cells as unknown[]).filter( + (cell): cell is [number, number] => + Array.isArray(cell) && cell.length === 2 && cell.every((value) => Number.isInteger(value)), + ) + : null, }; } diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 94a14592fa..886fa8f185 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -28,7 +28,7 @@ # parser defect can be traced to the documents it touched. Bump the date and label whenever a # change alters the text, tables or images produced for the same PDF. 2026-09-16 is PR #2810, # which replaced line-major table text with table_aware_page_text (#GJHKYC). -EXTRACTOR_VERSION = "2026-10-20.embedded-image-ceiling-v10" +EXTRACTOR_VERSION = "2026-10-21.merged-cell-provenance-v11" TABLE_STRATEGY = "table_aware_page_text" # IDX-H6: caps on serialized table size. Raised from the previous 80/120-row and 8000-char @@ -1443,12 +1443,21 @@ def text_grid_rows(text): def extract_table_payload(table): try: rows = table.extract() - cleaned = [[clean_cell(cell) for cell in row] for row in rows if any(clean_cell(cell) for cell in row)] + cleaned = [] + merged_cells = [] + for row in rows: + if not any(clean_cell(cell) for cell in row): + continue + # PyMuPDF returns None (not "") for a cell covered by a merged span, so + # these coordinates are confirmed merge provenance, not merely blank cells. + merged_cells.extend([len(cleaned), column] for column, cell in enumerate(row) if cell is None) + cleaned.append([clean_cell(cell) for cell in row]) column_count = max((len(row) for row in cleaned), default=0) column_ranges = _extract_table_column_ranges(table, column_count) return { "text": table_rows_to_markdown(rows), "rows": cleaned[:MAX_TABLE_ROWS], + "merged_cells": [cell for cell in merged_cells if cell[0] < MAX_TABLE_ROWS], "columns": table_columns_from_rows(rows)[:24], "accessible_markdown": table_rows_to_markdown(rows), "row_count": len(cleaned), @@ -1461,6 +1470,7 @@ def extract_table_payload(table): return { "text": "", "rows": [], + "merged_cells": [], "columns": [], "accessible_markdown": "", "row_count": 0, @@ -1740,6 +1750,7 @@ def likely_table_candidates(page, rejected_rects=None): "table_role": role, "table_confidence": table_candidate_confidence(label, title, table_text, row_count, column_count), "table_rows": payload.get("rows", []), + "table_merged_cells": payload.get("merged_cells", []), "table_columns": payload.get("columns", []), "accessible_table_markdown": payload.get("accessible_markdown") or table_text, "row_count": row_count, @@ -1875,7 +1886,10 @@ def merge_related_table_candidates(candidates, page_rect): rect = merge_rects([item["rect"] for item in group], page_rect) merged_text = "\n\n".join(item.get("table_text") or "" for item in group).strip() merged_rows = [] + merged_cells = [] for item in group: + offset = len(merged_rows) + merged_cells.extend([row + offset, column] for row, column in item.get("table_merged_cells") or []) merged_rows.extend(item.get("table_rows") or []) candidate = { **candidate, @@ -1888,6 +1902,7 @@ def merge_related_table_candidates(candidates, page_rect): ), "table_confidence": max(item.get("table_confidence") or 0 for item in group), "table_rows": merged_rows[:MAX_TABLE_ROWS], + "table_merged_cells": [cell for cell in merged_cells if cell[0] < MAX_TABLE_ROWS], "table_columns": (merged_rows[0] if merged_rows else candidate.get("table_columns") or [])[:24], "accessible_table_markdown": table_rows_to_markdown(merged_rows) if merged_rows else merged_text, "row_count": sum(item.get("row_count") or 0 for item in group), @@ -2365,6 +2380,7 @@ def _extract_document(document, output_dir, budget, identity): detected_rect = table_candidate["rect"] edge_rect, edge_info = extend_table_rect_for_edge_content(page, detected_rect) table_rows = list(table_candidate.get("table_rows") or []) + table_merged_cells = [list(cell) for cell in table_candidate.get("table_merged_cells") or []] column_count = int(table_candidate.get("column_count") or (len(table_rows[0]) if table_rows else 0) or 0) recovered_rows = [] unassigned_edge_text = [] @@ -2382,6 +2398,11 @@ def _extract_document(document, output_dir, budget, identity): recovered_rows = top_rows + bottom_rows if recovered_rows: combined_rows = top_rows + table_rows + bottom_rows + table_merged_cells = [ + [row + len(top_rows), column] + for row, column in table_merged_cells + if row + len(top_rows) < MAX_TABLE_ROWS + ] recovered_truncated = len(combined_rows) > MAX_TABLE_ROWS table_rows = combined_rows[:MAX_TABLE_ROWS] rows_truncated = ( @@ -2412,6 +2433,7 @@ def _extract_document(document, output_dir, budget, identity): "table_role": table_candidate.get("table_role"), "table_confidence": table_candidate.get("table_confidence"), "table_rows": table_rows, + "table_merged_cells": table_merged_cells, "table_columns": table_candidate.get("table_columns") or [], "accessible_table_markdown": accessible, "row_count": len(table_rows) if table_rows else table_candidate.get("row_count"), diff --git a/worker/python/test_extract_pdf_assets_merged_cells.py b/worker/python/test_extract_pdf_assets_merged_cells.py new file mode 100644 index 0000000000..d3a760325b --- /dev/null +++ b/worker/python/test_extract_pdf_assets_merged_cells.py @@ -0,0 +1,40 @@ +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) + +from extract_pdf_assets import extract_table_payload # noqa: E402 + + +class FakeTable: + def __init__(self, rows): + self._rows = rows + self.rows = [] + + def extract(self): + return self._rows + + +class MergedCellProvenanceTests(unittest.TestCase): + """Only cells PyMuPDF reports as span-covered (None) are recorded as merged.""" + + def test_none_cells_are_merged_and_blank_strings_are_not(self): + payload = extract_table_payload( + FakeTable( + [ + ["Parameter", "Threshold", "Action"], + ["Group A", "Level 1", "First"], + [None, "Level 2", "Second"], + [None, None, None], + ["", "Level 3", "Third"], + ] + ) + ) + self.assertEqual(len(payload["rows"]), 4) + # Row indices are in cleaned-row coordinates (the all-empty row is dropped). + self.assertEqual(payload["merged_cells"], [[2, 0]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/worker/python/test_extract_pdf_assets_raster_tables.py b/worker/python/test_extract_pdf_assets_raster_tables.py index 29b677415f..1b35b80bf1 100644 --- a/worker/python/test_extract_pdf_assets_raster_tables.py +++ b/worker/python/test_extract_pdf_assets_raster_tables.py @@ -555,7 +555,7 @@ def test_gated_merged_scan_keeps_exponent_cells_and_owning_rows(self): for cells, row_text in ((MERGED_ROWS[0], text[wbc:neutrophils]), (MERGED_ROWS[1], text[neutrophils:])): for lines in cells: self.assertIn(" ".join(lines), row_text) - self.assertEqual(result["extractor"]["version"], "2026-10-20.embedded-image-ceiling-v10") + self.assertEqual(result["extractor"]["version"], "2026-10-21.merged-cell-provenance-v11") provenance = result["pages"][0]["metadata"]["source_provenance"] self.assertEqual(provenance["ocr"]["status"], "unverified") self.assertIsNone(provenance["ocr"]["confidence"]) diff --git a/worker/table-facts.ts b/worker/table-facts.ts index 60f1cebbab..710ed94332 100644 --- a/worker/table-facts.ts +++ b/worker/table-facts.ts @@ -32,6 +32,8 @@ export type TableFactImageRow = { accessibleTableMarkdown?: string | null; tableRows?: string[][] | null; tableColumns?: string[] | null; + // [rowIndex, columnIndex] of cells the extractor confirmed are covered by a merged span. + tableMergedCells?: [number, number][] | null; structuredVisualProfile?: { table_column_roles?: Record; } | null; @@ -182,6 +184,13 @@ export function buildTableFactRows(args: { // Audit 2026-10-20 ING-B-09: PyMuPDF leaves a vertically merged label cell blank on every row // after the first, so those rows lost their group. Carry the last label forward into // clinical_parameter only; row_label, cells and chunk text keep the extracted (blank) value. + // Only a CONFIRMED merged cell (extractor span provenance in tableMergedCells) inherits; + // an ordinary blank label cell, or a table without provenance, never does. An empty + // separator row or any row with its own label resets the inheritance chain. + const labelColumn = parameterIndex >= 0 ? parameterIndex : 0; + const confirmedMergedRows = new Set( + (image.tableMergedCells ?? []).filter(([, column]) => column === labelColumn).map(([row]) => row), + ); let lastRowLabel: { label: string; rowIndex: number } | null = null; for (const [rowIndex, rawCells] of image.tableRows.slice(0, 400).entries()) { const cells = rawCells.map((cell) => compactSearchText(cell, 240)); @@ -195,10 +204,14 @@ export function buildTableFactRows(args: { ) continue; const rowText = cells.filter(Boolean).join(" "); - if (!rowText) continue; - const rowLabel = tableFactValue(cells, parameterIndex >= 0 ? parameterIndex : 0); - const inheritedLabel = rowLabel ? null : lastRowLabel; + if (!rowText) { + lastRowLabel = null; + continue; + } + const rowLabel = tableFactValue(cells, labelColumn); + const inheritedLabel = !rowLabel && confirmedMergedRows.has(rowIndex) ? lastRowLabel : null; if (rowLabel) lastRowLabel = { label: rowLabel, rowIndex }; + else if (!inheritedLabel) lastRowLabel = null; // Named columns constrain meaning: an unmatched concentration or interval // remains in metadata.cells without being relabelled a threshold or action. const threshold = From 8aa537d3956974c3b4ad07f457ec62f3844e265d Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 17:44:12 +0800 Subject: [PATCH 4/8] fix(table-facts): reset row-label carry-down at section and page breaks --- tests/table-facts.test.ts | 41 +++++++++++++++++++++++++++++ worker/image-captioning.ts | 3 +++ worker/image-classification.ts | 3 +++ worker/python/extract_pdf_assets.py | 14 ++++++++++ worker/table-facts.ts | 15 ++++++++++- 5 files changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/table-facts.test.ts b/tests/table-facts.test.ts index af798771ea..08f8bb831f 100644 --- a/tests/table-facts.test.ts +++ b/tests/table-facts.test.ts @@ -139,6 +139,47 @@ describe("table fact generation", () => { expect(facts[1]?.metadata).not.toHaveProperty("row_label_inherited_from"); }); + it("does not carry a merged label across a page or section row break", () => { + const rows = [ + ["Group A", "Level 1", "First instruction"], + ["", "Level 2", "Second instruction"], + ]; + const facts = buildTableFactRows({ + job: { document_id: "doc-1", documents: { owner_id: "owner-1" } }, + chunkRows: [], + insertedImages: [ + { + id: "image-1", + pageNumber: 2, + tableTitle: "Synthetic monitoring table", + tableRows: rows, + tableColumns: ["Parameter", "Threshold", "Action"], + tableMergedCells: [[1, 0]], + tableRowBreaks: [1], + }, + ], + }); + expect(facts[1]?.clinical_parameter).toBe("Synthetic monitoring table"); + expect(facts[1]?.metadata).not.toHaveProperty("row_label_inherited_from"); + }); + + it("does not carry a full-width section heading row down into merged cells below it", () => { + const facts = carryFacts( + [ + ["Group A", "Level 1", "First instruction"], + ["Section 2: Renal", "", ""], + ["", "Level 2", "Second instruction"], + ], + [ + [1, 1], + [1, 2], + [2, 0], + ], + ); + expect(facts[2]?.clinical_parameter).toBe("Synthetic monitoring table"); + expect(facts[2]?.metadata).not.toHaveProperty("row_label_inherited_from"); + }); + it("deduplicates repeated table rows from the same image", () => { const facts = buildTableFactRows({ job: { diff --git a/worker/image-captioning.ts b/worker/image-captioning.ts index cc5c859cec..b1ec4e7889 100644 --- a/worker/image-captioning.ts +++ b/worker/image-captioning.ts @@ -75,6 +75,7 @@ type InsertedImage = { tableRows: string[][] | null; tableColumns: string[] | null; tableMergedCells: [number, number][] | null; + tableRowBreaks: number[] | null; structuredVisualProfile: StructuredVisualProfile | null; candidatePriorityScore: number; imageQualityScore: number; @@ -547,6 +548,7 @@ async function persistCaptionedImages( table_rows: tableMetadata.tableRows, table_columns: tableMetadata.tableColumns, table_merged_cells: tableMetadata.tableMergedCells, + table_row_breaks: tableMetadata.tableRowBreaks, accessible_table_markdown: tableMetadata.accessibleTableMarkdown, clinical_use_class: policyAssessment.clinical_use_class, clinical_use_reason: policyAssessment.clinical_use_reason, @@ -598,6 +600,7 @@ async function persistCaptionedImages( tableRows: tableMetadata.tableRows, tableColumns: tableMetadata.tableColumns, tableMergedCells: tableMetadata.tableMergedCells, + tableRowBreaks: tableMetadata.tableRowBreaks, structuredVisualProfile: structuredVisualProfile ?? null, candidatePriorityScore: candidate.candidatePriorityScore, imageQualityScore: candidate.imageQualityScore, diff --git a/worker/image-classification.ts b/worker/image-classification.ts index e73e208ced..e82a61d503 100644 --- a/worker/image-classification.ts +++ b/worker/image-classification.ts @@ -95,6 +95,9 @@ export function imageTableMetadata(image: ExtractedDocument["images"][number]) { Array.isArray(cell) && cell.length === 2 && cell.every((value) => Number.isInteger(value)), ) : null, + tableRowBreaks: Array.isArray(metadata.table_row_breaks) + ? (metadata.table_row_breaks as unknown[]).filter((row): row is number => Number.isInteger(row)) + : null, }; } diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 886fa8f185..df9f4e7d53 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -1887,8 +1887,12 @@ def merge_related_table_candidates(candidates, page_rect): merged_text = "\n\n".join(item.get("table_text") or "" for item in group).strip() merged_rows = [] merged_cells = [] + row_breaks = [] for item in group: offset = len(merged_rows) + if offset: + # Each grouped table starts a new section: label carry-down must not cross it. + row_breaks.append(offset) merged_cells.extend([row + offset, column] for row, column in item.get("table_merged_cells") or []) merged_rows.extend(item.get("table_rows") or []) candidate = { @@ -1903,6 +1907,7 @@ def merge_related_table_candidates(candidates, page_rect): "table_confidence": max(item.get("table_confidence") or 0 for item in group), "table_rows": merged_rows[:MAX_TABLE_ROWS], "table_merged_cells": [cell for cell in merged_cells if cell[0] < MAX_TABLE_ROWS], + "table_row_breaks": [row for row in row_breaks if row < MAX_TABLE_ROWS], "table_columns": (merged_rows[0] if merged_rows else candidate.get("table_columns") or [])[:24], "accessible_table_markdown": table_rows_to_markdown(merged_rows) if merged_rows else merged_text, "row_count": sum(item.get("row_count") or 0 for item in group), @@ -2381,6 +2386,7 @@ def _extract_document(document, output_dir, budget, identity): edge_rect, edge_info = extend_table_rect_for_edge_content(page, detected_rect) table_rows = list(table_candidate.get("table_rows") or []) table_merged_cells = [list(cell) for cell in table_candidate.get("table_merged_cells") or []] + table_row_breaks = list(table_candidate.get("table_row_breaks") or []) column_count = int(table_candidate.get("column_count") or (len(table_rows[0]) if table_rows else 0) or 0) recovered_rows = [] unassigned_edge_text = [] @@ -2403,6 +2409,13 @@ def _extract_document(document, output_dir, budget, identity): for row, column in table_merged_cells if row + len(top_rows) < MAX_TABLE_ROWS ] + # Rows recovered past the detected edge (e.g. across a page break) start new segments. + table_row_breaks = sorted( + {row + len(top_rows) for row in table_row_breaks} + | ({len(top_rows)} if top_rows else set()) + | ({len(top_rows) + len(table_rows)} if bottom_rows else set()) + ) + table_row_breaks = [row for row in table_row_breaks if row < MAX_TABLE_ROWS] recovered_truncated = len(combined_rows) > MAX_TABLE_ROWS table_rows = combined_rows[:MAX_TABLE_ROWS] rows_truncated = ( @@ -2434,6 +2447,7 @@ def _extract_document(document, output_dir, budget, identity): "table_confidence": table_candidate.get("table_confidence"), "table_rows": table_rows, "table_merged_cells": table_merged_cells, + "table_row_breaks": table_row_breaks, "table_columns": table_candidate.get("table_columns") or [], "accessible_table_markdown": accessible, "row_count": len(table_rows) if table_rows else table_candidate.get("row_count"), diff --git a/worker/table-facts.ts b/worker/table-facts.ts index 710ed94332..e2e6fbcf65 100644 --- a/worker/table-facts.ts +++ b/worker/table-facts.ts @@ -34,6 +34,9 @@ export type TableFactImageRow = { tableColumns?: string[] | null; // [rowIndex, columnIndex] of cells the extractor confirmed are covered by a merged span. tableMergedCells?: [number, number][] | null; + // Row indices that start a new section or page segment (grouped tables, rows recovered + // across a page edge). Label carry-down never crosses one. + tableRowBreaks?: number[] | null; structuredVisualProfile?: { table_column_roles?: Record; } | null; @@ -191,6 +194,8 @@ export function buildTableFactRows(args: { const confirmedMergedRows = new Set( (image.tableMergedCells ?? []).filter(([, column]) => column === labelColumn).map(([row]) => row), ); + const rowBreaks = new Set(image.tableRowBreaks ?? []); + const mergedCellKeys = new Set((image.tableMergedCells ?? []).map(([row, column]) => `${row}:${column}`)); let lastRowLabel: { label: string; rowIndex: number } | null = null; for (const [rowIndex, rawCells] of image.tableRows.slice(0, 400).entries()) { const cells = rawCells.map((cell) => compactSearchText(cell, 240)); @@ -203,6 +208,7 @@ export function buildTableFactRows(args: { cells.every((cell, index) => cell === columns[index]) ) continue; + if (rowBreaks.has(rowIndex)) lastRowLabel = null; const rowText = cells.filter(Boolean).join(" "); if (!rowText) { lastRowLabel = null; @@ -210,7 +216,14 @@ export function buildTableFactRows(args: { } const rowLabel = tableFactValue(cells, labelColumn); const inheritedLabel = !rowLabel && confirmedMergedRows.has(rowIndex) ? lastRowLabel : null; - if (rowLabel) lastRowLabel = { label: rowLabel, rowIndex }; + // A section heading row (its label spans every other column) is a section break: + // it is never a carry-down source for the rows below it. + const sectionHeading = + Boolean(rowLabel) && + cells.length > 1 && + cells.every((cell, column) => column === labelColumn || mergedCellKeys.has(`${rowIndex}:${column}`)); + if (sectionHeading) lastRowLabel = null; + else if (rowLabel) lastRowLabel = { label: rowLabel, rowIndex }; else if (!inheritedLabel) lastRowLabel = null; // Named columns constrain meaning: an unmatched concentration or interval // remains in metadata.cells without being relabelled a threshold or action. From 603a9e7a64113430c7c7c2f72350003886950008 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 17:49:32 +0800 Subject: [PATCH 5/8] fix(source-governance): one grouped OCR caution the warning cap never drops (sealed path; needs owner-approved reseal) --- src/lib/source-governance.ts | 21 ++++++++------- tests/source-governance.test.ts | 46 ++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/lib/source-governance.ts b/src/lib/source-governance.ts index 5275b4f235..f4aa671994 100644 --- a/src/lib/source-governance.ts +++ b/src/lib/source-governance.ts @@ -305,15 +305,18 @@ export function sourceGovernanceWarnings(args: { } const severityRank = { danger: 0, warning: 1, info: 2 } satisfies Record; - return warnings - .sort( - (left, right) => - severityRank[left.severity] - severityRank[right.severity] || - // OCR_TEXT is a general caution; within a severity it must not crowd document-specific warnings out of the cap. - Number(left.code === SOURCE_GOVERNANCE_CODES.OCR_TEXT) - - Number(right.code === SOURCE_GOVERNANCE_CODES.OCR_TEXT), - ) - .slice(0, args.limit ?? 8); + // OCR_TEXT is a general caution: it is grouped into ONE warning (approved wording, first OCR + // source) that holds a reserved slot, so the cap can never drop it and it never crowds + // document-specific warnings out. + const ocrWarning = warnings.find((warning) => warning.code === SOURCE_GOVERNANCE_CODES.OCR_TEXT); + const limit = args.limit ?? 8; + const others = warnings + .filter((warning) => warning.code !== SOURCE_GOVERNANCE_CODES.OCR_TEXT) + .sort((left, right) => severityRank[left.severity] - severityRank[right.severity]); + if (!ocrWarning || limit <= 0) return others.slice(0, limit); + return [...others.slice(0, limit - 1), ocrWarning].sort( + (left, right) => severityRank[left.severity] - severityRank[right.severity], + ); } function plural(count: number, singular: string, pluralValue = `${singular}s`) { diff --git a/tests/source-governance.test.ts b/tests/source-governance.test.ts index 77007d198c..47407f9ed2 100644 --- a/tests/source-governance.test.ts +++ b/tests/source-governance.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + OCR_TEXT_WARNING_MESSAGE, frontendSourceGovernanceWarnings, groupSourceGovernanceWarnings, hasDangerSourceGovernanceWarning, @@ -144,8 +145,51 @@ describe("source governance warnings", () => { table_facts: [], }); const warnings = sourceGovernanceWarnings({ results: [...ocrSources, cautioned] }); - expect(warnings).toHaveLength(8); + expect(warnings.length).toBeLessThanOrEqual(8); expect(warnings.map((warning) => warning.code)).toContain("document_caution"); + expect(warnings.filter((warning) => warning.code === "ocr_text_source")).toHaveLength(1); + }); + + it("never lets the warning cap drop the grouped OCR caution", () => { + const base = { + ...result().source_metadata!, + document_status: "current" as const, + extraction_quality: "good" as const, + }; + const crowded = Array.from({ length: 10 }, (_, index) => + result({ + document_id: `doc-crowd-${index}`, + title: `Crowded source ${index}`, + source_metadata: { ...base, jurisdiction: "New York", publisher: `Overseas publisher ${index}` }, + indexing_quality: { + document_id: `doc-crowd-${index}`, + quality_score: 0.2, + extraction_quality: "good", + metrics: { ocr_page_count: 0 }, + issues: [], + }, + table_facts: [], + }), + ); + const ocrSource = result({ + document_id: "doc-ocr", + source_metadata: base, + indexing_quality: { + document_id: "doc-ocr", + quality_score: 0.9, + extraction_quality: "good", + metrics: { ocr_page_count: 2, ocr_coverage: 0.2 }, + issues: [], + }, + table_facts: [], + }); + for (const limit of [8, 3, 1]) { + const warnings = sourceGovernanceWarnings({ results: [...crowded, ocrSource], limit }); + expect(warnings).toHaveLength(limit); + const ocr = warnings.filter((warning) => warning.code === "ocr_text_source"); + expect(ocr).toHaveLength(1); + expect(ocr[0]?.message).toBe(OCR_TEXT_WARNING_MESSAGE); + } }); it("adds no OCR caution for a source with no OCR pages", () => { From efb6058ec795f8debb0e68f2942dc9cfe00d5837 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 18:13:22 +0800 Subject: [PATCH 6/8] chore(governance): owner-approved reseal for grouped OCR warning (comment 6313455205) --- docs/clinical-hazard-controls.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/clinical-hazard-controls.json b/docs/clinical-hazard-controls.json index 6cd416053c..faac0bb163 100644 --- a/docs/clinical-hazard-controls.json +++ b/docs/clinical-hazard-controls.json @@ -173,7 +173,7 @@ "src/lib/service-ranker.ts": "bbdbe3629520a22104194fc727530c05614dfba5528d9b9d8d5743acbfa9ea44", "src/lib/service-urgent-routing.ts": "163245433081cc4474d6c54993f709100d9e636b001f0cd17904bc61f24dc285", "src/lib/source-authority-registry.ts": "938bbbb1d26bdb88acbb9cc15f43fc6d809a13de49e6a47d745c992aaba3f813", - "src/lib/source-governance.ts": "76a2c8ef50ce8f8abb5842e7c000e90a82a97b29945bbcf49149ac3f1664c99e", + "src/lib/source-governance.ts": "edec0e4bc7dc115847e0fdd2802a88cc340a8676b2e8288ee427563ce1bc1c68", "src/lib/source-text-sanitizer.ts": "cb6c869b520a8be03d40bae1206a87b6d53318573baeaec664b35c353fa1bd80", "tests/answer-render-policy.test.ts": "e7ddb86442ecdafd01bc6d1b94791e41b98c91c0d313ab03a532a1cc9fc60465", "tests/answer-verification.test.ts": "3c5f2d1d805067f17a2bfcc47c7a92f7362271004ebad071b5b657c9d7d34844", @@ -187,7 +187,7 @@ "tests/services-catalog.test.ts": "9de1e48720aa15ac4b60632fb7841579c707e14bb47c2f0e2dce407c41bad190", "tests/services-safety-regressions.test.ts": "0515a77320cf9abc0b26ed3320f07d4b8e05b0eaed79ba031840b000735fb9df", "tests/services-safety-routing.test.ts": "e54f0a0f0f11810848de0b178c5398e9ac158b90a814d2005986d9e0b3853f33", - "tests/source-governance.test.ts": "e8e995c1859f360719e194df9d33c068933f2f0f7d22828ddc54639370a5ce8b", + "tests/source-governance.test.ts": "17b26759d7954886f803a9e7f3c9ef857fc53edd35995f54c2b91b37131b13e0", "tests/ward-output.test.ts": "bb9f417d63c769b8e059f41fe2c9a5c52f772cedd13e6352408b4a7716e00222" } } From 5714539c5b055324a8e8893aa26719ca5627769e Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 18:13:43 +0800 Subject: [PATCH 7/8] chore(governance): set reviewedCommit/reviewedAt to the owner-approved reseal (comment 6313455205) --- docs/clinical-hazard-controls.json | 52 +++++++++++++++--------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/clinical-hazard-controls.json b/docs/clinical-hazard-controls.json index faac0bb163..40a3d4abf4 100644 --- a/docs/clinical-hazard-controls.json +++ b/docs/clinical-hazard-controls.json @@ -1,15 +1,15 @@ { "schemaVersion": 1, "authority": "Static evidence register only. It does not establish clinical adequacy, regulatory classification, or risk acceptance.", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "hazards": [ { "id": "H1", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Clinical safety owner", "controlSymbols": ["extractClinicalValueAtoms", "verifyAnswerNumbers", "applyNumericVerification"], @@ -24,8 +24,8 @@ { "id": "H2", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Source governance owner", "controlSymbols": ["sourceGovernanceWarnings", "deriveTrust"], @@ -36,8 +36,8 @@ { "id": "H3", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Clinical UX owner", "controlSymbols": ["formatAnswerRenderCopyText"], @@ -48,8 +48,8 @@ { "id": "H4", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Answer quality owner", "controlSymbols": ["assessAndEnforceClaimSupport", "collectSourceCandidates"], @@ -60,8 +60,8 @@ { "id": "H5", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Answer quality owner", "controlSymbols": ["deriveTrust", "deriveConfidence"], @@ -76,8 +76,8 @@ { "id": "H6", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Application security owner", "controlSymbols": ["neutralizeIdentityField", "fenceSourceEvidence", "neutralizePromptInstructions"], @@ -88,8 +88,8 @@ { "id": "H8", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-09-28", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-12-28", "owner": "Clinical safety owner (owner approval 2026-09-28)", "controlSymbols": ["detectServiceUrgentIntents", "rankServiceUrgentRoutes", "rankServiceRecords"], @@ -106,8 +106,8 @@ { "id": "CLINICAL-TRUTH-AUTHORITY", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Clinical governance authority", "evidenceReferences": ["docs/clinical-hazard-analysis.md"], @@ -116,8 +116,8 @@ { "id": "EXTERNAL-RISK-ACCEPTANCE", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Authorised risk owner", "evidenceReferences": ["docs/clinical-hazard-analysis.md", "docs/clinical-governance.md"], @@ -126,8 +126,8 @@ { "id": "SAFETY-CHIP-VOCABULARY", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-09-18", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-12-18", "owner": "Clinical governance authority", "evidenceReferences": [ @@ -140,8 +140,8 @@ { "id": "DICTIONARY-PUBLISHER-AGENCIES", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-09-18", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-12-18", "owner": "Clinical governance authority", "evidenceReferences": ["src/data/dictionary-source-dispositions.json", "src/lib/source-authority-registry.ts"], @@ -150,8 +150,8 @@ { "id": "PATIENT-NUMERIC-FIELD-DISPLAY", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-09-18", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-12-18", "owner": "Clinical governance authority", "evidenceReferences": ["src/components/clinical-dashboard/patient-profile-panel.tsx"], From ac5080c44064d0cde46dae84445a76ece0726233 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 18:50:33 +0800 Subject: [PATCH 8/8] chore(governance): regenerate hazard register snapshot for the reseal review date --- data/hazard-register-snapshot.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hazard-register-snapshot.json b/data/hazard-register-snapshot.json index dcee58e881..f72c5645d6 100644 --- a/data/hazard-register-snapshot.json +++ b/data/hazard-register-snapshot.json @@ -19,7 +19,7 @@ "authority": "Static evidence register only. It does not establish clinical adequacy, regulatory classification, or risk acceptance.", "signedOff": false, "gate": "check:clinical-hazard-controls", - "reviewedAt": "2026-08-23", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "reviewExpired": false, "hazards": [