diff --git a/data/hazard-register-snapshot.json b/data/hazard-register-snapshot.json index dcee58e881..f72c5645d6 100644 --- a/data/hazard-register-snapshot.json +++ b/data/hazard-register-snapshot.json @@ -19,7 +19,7 @@ "authority": "Static evidence register only. It does not establish clinical adequacy, regulatory classification, or risk acceptance.", "signedOff": false, "gate": "check:clinical-hazard-controls", - "reviewedAt": "2026-08-23", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "reviewExpired": false, "hazards": [ diff --git a/data/repo-awareness-snapshot.json b/data/repo-awareness-snapshot.json index a5aa3c3909..105d167828 100644 --- a/data/repo-awareness-snapshot.json +++ b/data/repo-awareness-snapshot.json @@ -5483,6 +5483,11 @@ "section": "decisions", "catalogued": true }, + { + "path": "docs/decisions/2026-10-21-no-malware-scan-before-upload-storage.md", + "section": "decisions", + "catalogued": true + }, { "path": "docs/superpowers/rag-upgrade/execution-artifacts/rag-answer-quality-and-repository-coverage-v1/README.md", "section": "superpowers", diff --git a/docs/clinical-hazard-analysis.md b/docs/clinical-hazard-analysis.md index b91b375506..a96be2bfb5 100644 --- a/docs/clinical-hazard-analysis.md +++ b/docs/clinical-hazard-analysis.md @@ -90,7 +90,7 @@ So an unvalidated, unknown-provenance document is routinely presented as a high- | ID | Failure scenario | Severity | Plaus. | Control | Test | Gap | | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | --------------------------- | ------------------------------------------------------------------------------------------------ | -------------------------------------------------------------- | ---------------------------------------------------------------------------- | | **H2b** | An `outdated` chunk out-ranks a current one. Freshness/status penalties exist (`-0.04` at [`clinical-search.ts:1646`](../src/lib/clinical-search.ts), [`rag-second-stage.ts:100`](../src/lib/rag/rag-second-stage.ts); date-decay in `ranking-config.ts`) but are an order of magnitude below relevance boosts (`+0.06…+0.38`) and are **zero when dates are null** (the default). | High | Medium (needs metadata set) | small penalties [`clinical-search.ts:1646`](../src/lib/clinical-search.ts) | `ranking-config.test.ts` proves the _math_, not the _ordering_ | No "current beats outdated on tie" rule; penalty is summed flat and clamped. | -| **H2c** | The one hard block, `sourceGovernanceRefusalAnswer` ([`source-governance.ts:57`](../src/lib/source-governance.ts)), fires only on `severity==="danger"` — i.e. literal `document_status==="outdated"` or `extraction_quality==="poor"` ([`source-governance.ts:164`](../src/lib/source-governance.ts)). Because `outdated` is essentially never set, this block is dormant in practice. | High | Medium | `hasDangerSourceGovernanceWarning` [`source-governance.ts:364`](../src/lib/source-governance.ts) | `tests/source-governance.test.ts` | Depends on metadata that the pipeline never populates. | +| **H2c** | The one hard block, `sourceGovernanceRefusalAnswer` ([`source-governance.ts:57`](../src/lib/source-governance.ts)), fires only on `severity==="danger"` — i.e. literal `document_status==="outdated"` or `extraction_quality==="poor"` ([`source-governance.ts:164`](../src/lib/source-governance.ts)). Because `outdated` is essentially never set, this block is dormant in practice. | High | Medium | `hasDangerSourceGovernanceWarning` [`source-governance.ts:393`](../src/lib/source-governance.ts) | `tests/source-governance.test.ts` | Depends on metadata that the pipeline never populates. | | **H2d** | `review_due` is filtered **out** of the frontend banner (`frontendVisibleWarningCodes` omits it, [`source-governance.ts:84`](../src/lib/source-governance.ts)); a review-due source surfaces with only a badge tone. | Medium | Low | — | test asserts it is _hidden_ | No review-due caveat in the answer prose or the warnings channel. | ### H3 — A safety caveat present on screen is stripped on copy/export diff --git a/docs/clinical-hazard-controls.json b/docs/clinical-hazard-controls.json index fdaadea6c4..40a3d4abf4 100644 --- a/docs/clinical-hazard-controls.json +++ b/docs/clinical-hazard-controls.json @@ -1,15 +1,15 @@ { "schemaVersion": 1, "authority": "Static evidence register only. It does not establish clinical adequacy, regulatory classification, or risk acceptance.", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "hazards": [ { "id": "H1", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Clinical safety owner", "controlSymbols": ["extractClinicalValueAtoms", "verifyAnswerNumbers", "applyNumericVerification"], @@ -24,8 +24,8 @@ { "id": "H2", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Source governance owner", "controlSymbols": ["sourceGovernanceWarnings", "deriveTrust"], @@ -36,8 +36,8 @@ { "id": "H3", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Clinical UX owner", "controlSymbols": ["formatAnswerRenderCopyText"], @@ -48,8 +48,8 @@ { "id": "H4", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Answer quality owner", "controlSymbols": ["assessAndEnforceClaimSupport", "collectSourceCandidates"], @@ -60,8 +60,8 @@ { "id": "H5", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Answer quality owner", "controlSymbols": ["deriveTrust", "deriveConfidence"], @@ -76,8 +76,8 @@ { "id": "H6", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Application security owner", "controlSymbols": ["neutralizeIdentityField", "fenceSourceEvidence", "neutralizePromptInstructions"], @@ -88,8 +88,8 @@ { "id": "H8", "state": "partial", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-09-28", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-12-28", "owner": "Clinical safety owner (owner approval 2026-09-28)", "controlSymbols": ["detectServiceUrgentIntents", "rankServiceUrgentRoutes", "rankServiceRecords"], @@ -106,8 +106,8 @@ { "id": "CLINICAL-TRUTH-AUTHORITY", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Clinical governance authority", "evidenceReferences": ["docs/clinical-hazard-analysis.md"], @@ -116,8 +116,8 @@ { "id": "EXTERNAL-RISK-ACCEPTANCE", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-08-23", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-11-23", "owner": "Authorised risk owner", "evidenceReferences": ["docs/clinical-hazard-analysis.md", "docs/clinical-governance.md"], @@ -126,8 +126,8 @@ { "id": "SAFETY-CHIP-VOCABULARY", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-09-18", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-12-18", "owner": "Clinical governance authority", "evidenceReferences": [ @@ -140,8 +140,8 @@ { "id": "DICTIONARY-PUBLISHER-AGENCIES", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-09-18", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-12-18", "owner": "Clinical governance authority", "evidenceReferences": ["src/data/dictionary-source-dispositions.json", "src/lib/source-authority-registry.ts"], @@ -150,8 +150,8 @@ { "id": "PATIENT-NUMERIC-FIELD-DISPLAY", "state": "open", - "reviewedCommit": "ff57ec6d0903ead5647db968436be911a744567d", - "reviewedAt": "2026-09-18", + "reviewedCommit": "efb6058ec795f8debb0e68f2942dc9cfe00d5837", + "reviewedAt": "2026-10-21", "reviewExpiresAt": "2026-12-18", "owner": "Clinical governance authority", "evidenceReferences": ["src/components/clinical-dashboard/patient-profile-panel.tsx"], @@ -160,141 +160,34 @@ ], "reviewedPathDigests": { "docs/audit/2026-09-17-external-audit-refutations.md": "c32a15b9c364c35fe7134784d28d4598dbf26f62dc8fdf01aa4a6646b5bd1d82", - "docs/clinical-governance.md": "1a87a39f18965ab0ad8b89b69865e323d5101f5f08c592fa6d23620fb501b429", - "docs/clinical-hazard-analysis.md": "a45cfc864d86bff7089e2b495df1f2debaddd3931424439bf823ac09a52cc30c", - "src/components/clinical-dashboard/patient-profile-panel.tsx": "c81ce6a9d80bdc749f9ebfac6bda7578b97f66acf7daaeb8827b4c7b3e6b0628", - "src/data/dictionary-source-dispositions.json": "3d03bd0be2d699f5dc7eb50338b535112a7dea57daf0f210de9ae241c606c2b8", - "src/lib/answer-render-policy.ts": "a5e2b36c4bdf6c2081d443d10e87c0ced403abfb129454d6989ad93af737f4a9", - "src/lib/answer-verification.ts": "73e2a66d3b50ae66b415f57366de7f688d098dd9315aeefa38c7e8fd426f8dfc", - "src/lib/clinical-safety.ts": "8fa7b5fca92f2b12b6f241d32feadd9310ee075a38e55e10b5bbce4c26c1173b", + "docs/clinical-governance.md": "4a21c4ee500e1e7953b4fc975206cb6c06dddcc4b436453c09d7ea60e42dc598", + "docs/clinical-hazard-analysis.md": "d6738ce036b76445f624b80ff7252ab0af1bed85ec3de6fc5cffa653f8c81b0f", + "src/components/clinical-dashboard/patient-profile-panel.tsx": "0eacd7447d883195e2ca9b512034018e445c45249d7394355730b8d11e3a1a43", + "src/data/dictionary-source-dispositions.json": "afc5760d7567803c512c7e0a2cf5826cdc27b3c152db9c0aadf7f66d3e1c6141", + "src/lib/answer-render-policy.ts": "f907ceedb16daa5e5338ce3123c58c366ed96492fd0d114dd7ce3b7b9b5f1d88", + "src/lib/answer-verification.ts": "fa2429b2ac14f51d00a8ab1b3754f05bedda3b6b30b56a585a16fbfe411c7eb4", + "src/lib/clinical-safety.ts": "f779a97793e0015b558e9c7a738e606111c85f09b005661ca8c4bba0c9d88f73", "src/lib/rag/rag-answer-support.ts": "4250f30017789f6c5d75e84e7a291f163f08abb95d615fb2368e904956841dd5", "src/lib/rag/rag-claim-support.ts": "a945a3aa1166590010404fbae71ac7a3cc1a8ed4036b68164667adea84e21b47", - "src/lib/rag/rag-source-block.ts": "984b3f655b468aabbf2d1f489eb6df66273fe7c9e85ea88df63affa986654ad2", + "src/lib/rag/rag-source-block.ts": "8d08e9d168fdd3e258f4b56e22db2b07541e62034a86f7ad90c5a58a8cdd38af", "src/lib/service-ranker.ts": "bbdbe3629520a22104194fc727530c05614dfba5528d9b9d8d5743acbfa9ea44", - "src/lib/service-urgent-routing.ts": "663b0d131fa4457a4df9875fb200efe1520c9ebda75b552a8bbd68eb66264a7f", - "src/lib/source-authority-registry.ts": "e82b2f1aeaf053a321f76576da71a2c54f7fb4db485fbf6c4caf686ac6f2267c", - "src/lib/source-governance.ts": "e3e1c09b0a7e446b7055f894088c771f1a5cc160a0f20f026f3cc291af5804fa", + "src/lib/service-urgent-routing.ts": "163245433081cc4474d6c54993f709100d9e636b001f0cd17904bc61f24dc285", + "src/lib/source-authority-registry.ts": "938bbbb1d26bdb88acbb9cc15f43fc6d809a13de49e6a47d745c992aaba3f813", + "src/lib/source-governance.ts": "edec0e4bc7dc115847e0fdd2802a88cc340a8676b2e8288ee427563ce1bc1c68", "src/lib/source-text-sanitizer.ts": "cb6c869b520a8be03d40bae1206a87b6d53318573baeaec664b35c353fa1bd80", - "tests/answer-render-policy.test.ts": "5e48ea66ca1902ac17a271444db7f6350a37c0100fe2152afd042e4fdb89427d", - "tests/answer-verification.test.ts": "2beac1a1f96cb8afae395a51cd5d2a91e8f45ff45723bf3e7641ec85c2605050", - "tests/clinical-safety.test.ts": "053b82e3523a5aa55b40059a558e8992fe5342ebedd8812da10193977cad43f6", + "tests/answer-render-policy.test.ts": "e7ddb86442ecdafd01bc6d1b94791e41b98c91c0d313ab03a532a1cc9fc60465", + "tests/answer-verification.test.ts": "3c5f2d1d805067f17a2bfcc47c7a92f7362271004ebad071b5b657c9d7d34844", + "tests/clinical-safety.test.ts": "96d950e946af0bcc2cb3f517fc372d28e67d93099e00b07b5d3c854fa1a020e8", "tests/rag-claim-support.test.ts": "79f19e142dff92ea92cb3635bd032f081885a1714d1ac745eb3d0f9151247586", "tests/rag-content-accuracy.test.ts": "6aaacb7fb9b43711696ba244a56cd31736fb3a884690774cde4f4db6ee206eb4", "tests/rag-injection.test.ts": "9a6fa43c799232d9e38d334b732c99dfaad5a5192ed64a7fcb8d6cb373300931", "tests/rag-retrieval-row-contract.test.ts": "4da5fe01a76dfa5b4e06e02303f136ff3fb0a1a1219cfe09ec158a1eae074fb6", "tests/rag-score.test.ts": "5a298ae040e7110d89fd1f68ad1d52b16994c035845a38b4db72ef9800c2b17f", "tests/rag-trust.test.ts": "416568e19e6ab8e67267ca53183b86a1703193543b4253269162d28736606eae", - "tests/services-catalog.test.ts": "b66181ab8226aff658dc7d8f97d851e822b5389a3722a6ec90560ec6a50e77ff", + "tests/services-catalog.test.ts": "9de1e48720aa15ac4b60632fb7841579c707e14bb47c2f0e2dce407c41bad190", "tests/services-safety-regressions.test.ts": "0515a77320cf9abc0b26ed3320f07d4b8e05b0eaed79ba031840b000735fb9df", "tests/services-safety-routing.test.ts": "e54f0a0f0f11810848de0b178c5398e9ac158b90a814d2005986d9e0b3853f33", - "tests/source-governance.test.ts": "0e9e126370a5500dbb72c722610fe1268c7e2f32e6b20b0a4f3ea401c54a645d", + "tests/source-governance.test.ts": "17b26759d7954886f803a9e7f3c9ef857fc53edd35995f54c2b91b37131b13e0", "tests/ward-output.test.ts": "bb9f417d63c769b8e059f41fe2c9a5c52f772cedd13e6352408b4a7716e00222" - }, - "driftExceptions": [ - { - "path": "src/lib/answer-verification.ts", - "reason": "PR #3546 preserves the independently code-reviewed numeric age-unit correction (candidate SHA256 559b62cba08fedf9330e2bd907442de08cee204dbc6f395b99e2d849627eb5c2). Recorded-response delivery/copy and unsafe-output refusals passed; integrated hosted acceptance and owner hazard re-review remain pending. Preserve the prior seal and safety checks rather than imply renewed clinical approval.", - "recordedBy": "Codex ingestion engineering controller, PR #3546. Engineering drift record only; not clinical-owner approval or release signoff.", - "recordedOn": "2026-10-14", - "expiresOn": "2026-10-21" - }, - { - "path": "tests/answer-verification.test.ts", - "reason": "PR #3628 adds regression tests for the 'no more than' comparator/unit parsing in src/lib/answer-verification.ts (itself under the existing PR #3546 drift exception). Engineering drift exception per owner ruling 2026-10-18: the owner will review and reseal this test file before this exception expires. Not a clinical review, sign-off or reseal.", - "recordedBy": "Engineering, Grok Bot sweep session for PR #3628, per owner ruling 2026-10-18. Engineering drift record only; not clinical-owner approval or release signoff.", - "recordedOn": "2026-10-18", - "expiresOn": "2026-12-02" - }, - { - "path": "src/lib/clinical-safety.ts", - "reason": "Drift predates the blocking gate. Safety-behaviour changes since the 2026-08-23 review (e.g. 41b5a025a escalate/monitoring matching, 64db243a9 stop-instruction labelling, 1aa717e88 no all-clear over a failed medication check) need clinical re-review, then a re-seal.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "src/components/clinical-dashboard/patient-profile-panel.tsx", - "reason": "Drift predates the blocking gate. Patient-profile changes since the 2026-08-23 review (3da15e805 clear generation, 8d49b16c4 number field) need re-review, then a re-seal.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "tests/clinical-safety.test.ts", - "reason": "Drift predates the blocking gate. Test changes accompanying the clinical-safety.ts changes above; re-seal together with it.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "docs/clinical-hazard-analysis.md", - "reason": "Changed by PR #3019 (1a3d89350) after the 2026-08-23 review: adds proposed hazard H7, the Mental Health Act timeline misstating a statutory deadline (H7a-c, all Critical). Needs owner review and a register entry, then a re-seal. PR #3146 (owner approval 2026-09-28): H8 services crisis routing rewritten to the routing and tests that exist and registered as H8 in this register. Not resealed because the H7 change above still awaits review. 2026-10-20 (regulatory audit R-01/R-02): adds the amended TGA exemption section in force from 1 November 2026 and the PR-4 status reversal; regulatory text only, no hazard, control or severity changed.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "src/lib/source-authority-registry.ts", - "reason": "Changed on main by 309656a8 (#T9MZ7V, merge the duplicate American Psychiatric Association register entry) after the 2026-08-23 review; arrived in this branch via a main merge. Main only warns on drift; this branch makes drift blocking. Needs owner re-review, then a re-seal. PR #3146 (owner approval 2026-09-28): comment on descriptive publishers updated; 4AT stays unregistered. Not resealed because the earlier change above still awaits review.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "src/lib/answer-render-policy.ts", - "reason": "Changed on main by #3031 (#WGMB4Z answer render-trust cap, #GHC4XZ) after the last seal; arrived in this branch via a main merge. Main only warns on drift; this branch makes drift blocking. Needs owner re-review, then a re-seal.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "tests/answer-render-policy.test.ts", - "reason": "Changed on main by #3031 (#WGMB4Z answer render-trust cap, #GHC4XZ) after the last seal; arrived in this branch via a main merge. Main only warns on drift; this branch makes drift blocking. Needs owner re-review, then a re-seal.", - "recordedBy": "Engineering, Claude session 2026-09-25. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-25", - "expiresOn": "2026-10-25" - }, - { - "path": "src/data/dictionary-source-dispositions.json", - "reason": "Source-disposition changes in PR #3067 (owner-approved publisher page reads for ledger items #JHT39N, #WY625D, #6X06YS, #YDENFM, #KVYHWZ; held/admitted counts and per-source blockers updated), and the NSW MSE handbook recorded as retired later on 2026-09-26 on the owner's approval, are pending owner review, then a re-seal. PR #3146 (owner approval 2026-09-28): AIHW-NOCC-2026 stays held and gains a registration-date blocker. Not resealed because earlier drift on this path still awaits review.", - "recordedBy": "BigSimmo (owner), decision 2026-09-26. Engineering, not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-26", - "expiresOn": "2026-10-26" - }, - { - "path": "src/lib/rag/rag-source-block.ts", - "reason": "PR #3103 (#JYH1FH, step 1 of the locally_reviewed label repair): the model-facing source line reads \"clinical validation: unverified (endorsed by issuing WA service)\" for the new wa_document_control_endorsement basis. No live document carries that basis until the separately approved data migration, so current output is unchanged. Pending owner review, then a re-seal.", - "recordedBy": "BigSimmo (owner), decision 2026-09-26 (\"record the two step 1 files as changed and awaiting my review\"). Engineering, not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-26", - "expiresOn": "2026-11-09" - }, - { - "path": "src/lib/source-governance.ts", - "reason": "PR #3103 (#JYH1FH, step 1 of the locally_reviewed label repair): claim-evidence eligibility accepts the wa_document_control_endorsement basis exactly as it accepts locally_reviewed, and the unverified-source warning gets honest wording for it (same code and severity). No live document carries that basis until the separately approved data migration. Pending owner review, then a re-seal.", - "recordedBy": "BigSimmo (owner), decision 2026-09-26 (\"record the two step 1 files as changed and awaiting my review\"). Engineering, not a clinical review; the owner must re-review.", - "recordedOn": "2026-09-26", - "expiresOn": "2026-11-09" - }, - { - "path": "docs/clinical-governance.md", - "reason": "This combined change replaces the 2026-09-17 name-them-never-score-them rule with the 2026-10-03 transparent reference aid wording. The cited file drifted after the 2026-08-23 review and has not had a new clinical review.", - "recordedBy": "Engineering, combined-PR session 2026-10-06. Not a clinical review; the owner must re-review.", - "recordedOn": "2026-10-06", - "expiresOn": "2026-11-20" - }, - { - "path": "src/lib/service-urgent-routing.ts", - "reason": "Audit 3 finding R4-F-01: medical-emergency phrases (seizure, unconscious or unconsciousness, loss of consciousness, not breathing, collapse or collapsed, anaphylaxis) now pin 000 first, ahead of regional crisis routes and of the Alcohol and Drug Support Line for alcohol withdrawal seizure. Behaviour widens urgent routing only; owner hazard re-review and reseal (H8) pending.", - "recordedBy": "Engineering, Claude clinical content audit thread (audit 3, 20 Oct 2026), per owner request to fix the audit recommendations. Engineering drift record only; not clinical-owner approval, hazard re-review or reseal.", - "recordedOn": "2026-10-20", - "expiresOn": "2026-12-04" - }, - { - "path": "tests/services-catalog.test.ts", - "reason": "Audit 3 findings R4-F-06, R4-F-10, R4-F-11 and R6-F-10: tests added or updated for superseded legacy crisis duplicates, corrected service contacts, phone bracket normalisation and no confidence chip on unverified legacy records. Owner hazard re-review and reseal (H8) pending.", - "recordedBy": "Engineering, Claude clinical content audit thread (audit 3, 20 Oct 2026), per owner request to fix the audit recommendations. Engineering drift record only; not clinical-owner approval, hazard re-review or reseal.", - "recordedOn": "2026-10-20", - "expiresOn": "2026-12-04" - } - ] + } } diff --git a/docs/decisions/2026-10-21-no-malware-scan-before-upload-storage.md b/docs/decisions/2026-10-21-no-malware-scan-before-upload-storage.md new file mode 100644 index 0000000000..9d2baa4b79 --- /dev/null +++ b/docs/decisions/2026-10-21-no-malware-scan-before-upload-storage.md @@ -0,0 +1,12 @@ +# Decision: no malware scan before uploads are stored, for now + +- **Status:** decided 2026-10-21 +- **Source:** the owner, in chat + +On 21 Oct 2026 the owner chose "Leave as is" by decision card in the project's RAG and ingestion +audit thread (audit ING-B-11). Ordinary uploads are still stored before any malware scan. They stay +owner-only and are never run as code, and the full malware check stays on the admin route. No +scanner is added to the ingestion worker for every upload. Revisit this if anyone other than the +owner can upload files. + +If this summary and the source ever differ, the source wins. diff --git a/docs/privacy-impact-assessment.md b/docs/privacy-impact-assessment.md index 84d9a9bf1d..449906db69 100644 --- a/docs/privacy-impact-assessment.md +++ b/docs/privacy-impact-assessment.md @@ -338,13 +338,14 @@ of the most important privacy items before real patient use (PIA-1). All three log tables are **owner-stamped** and **RLS-enabled** (owner-read for authenticated users; service-role for writes). Redaction is applied centrally at every write site. -| Table | Raw query stored? | Redaction mechanism | Other sensitive columns | RLS | -| -------------------- | --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| `rag_queries` | No (hash placeholder) | `queryTextForStorage` / `normalizedQueryTextForStorage` ([query-privacy.ts](../src/lib/query-privacy.ts)); centralized write in `insertRagQuery` | `answer` is null by default and stored only with explicit `RAG_PERSIST_ANSWER_TEXT=true`; `source_chunk_ids` (own data) | owner-read, [schema.sql](../supabase/schema.sql) | -| `rag_query_misses` | No (hash placeholder) | same helpers; writes in [search/route.ts](../src/app/api/search/route.ts), [interaction/route.ts](../src/app/api/search/interaction/route.ts) | `metadata.query_hash` | owner-read, [schema.sql](../supabase/schema.sql) | -| `rag_retrieval_logs` | No (hash placeholder) | same helpers; write at [search/route.ts](../src/app/api/search/route.ts) | retrieval telemetry only | owner-read, [schema.sql](../supabase/schema.sql) | -| `audit_logs` | N/A (no query text) | action/resource metadata only; the write boundary allowlists operational metadata and excludes user-controlled filenames/titles/content hashes ([audit.ts](../src/lib/audit.ts)). Migration `20260717163000` minimizes existing rows on deployment. | `owner_id`, `action`, `resource_id` | service-role-only, [schema.sql](../supabase/schema.sql) | -| `cme_evidence` | N/A (file metadata) | Mandatory user confirmations (`redaction_confirmed`, `preview_confirmed`); filename capped at 180 chars; files re-validated via Sharp; storage path strictly namespaced `owner_id/entry_id/id`. On owner removal, `file_name` is scrubbed to `'Removed file'`. | `storage_path`, `file_name`, `sha256`, `kind`, `removal_reason` (3–500 chars) | service-role-only (revoked from public, anon, authenticated; select, insert, update on removed_at, removal_reason, file_name granted to service_role) | +| Table | Raw query stored? | Redaction mechanism | Other sensitive columns | RLS | +| ------------------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `rag_queries` | No (hash placeholder) | `queryTextForStorage` / `normalizedQueryTextForStorage` ([query-privacy.ts](../src/lib/query-privacy.ts)); centralized write in `insertRagQuery` | `answer` is null by default and stored only with explicit `RAG_PERSIST_ANSWER_TEXT=true`; `source_chunk_ids` (own data) | owner-read, [schema.sql](../supabase/schema.sql) | +| `rag_query_misses` | No (hash placeholder) | same helpers; writes in [search/route.ts](../src/app/api/search/route.ts), [interaction/route.ts](../src/app/api/search/interaction/route.ts) | `metadata.query_hash` | owner-read, [schema.sql](../supabase/schema.sql) | +| `rag_retrieval_logs` | No (hash placeholder) | same helpers; write at [search/route.ts](../src/app/api/search/route.ts) | retrieval telemetry only | owner-read, [schema.sql](../supabase/schema.sql) | +| `audit_logs` | N/A (no query text) | action/resource metadata only; the write boundary allowlists operational metadata and excludes user-controlled filenames/titles/content hashes ([audit.ts](../src/lib/audit.ts)). Migration `20260717163000` minimizes existing rows on deployment. | `owner_id`, `action`, `resource_id` | service-role-only, [schema.sql](../supabase/schema.sql) | +| `storage_cleanup_jobs.document_title` | Until the deleted document's storage clean-up runs | Copied in by `delete_document_if_idle` when a document is deleted. Cleared to null by the delete route and by `npm run cleanup:storage` once the document row is gone (`buildStorageCleanupJobUpdate`, audit ING-A-05). Rows that finished before 21 Oct 2026 keep their title until a database backfill is approved. | Titles of deleted guidelines can identify things, so they are not kept after the delete. | +| `cme_evidence` | N/A (file metadata) | Mandatory user confirmations (`redaction_confirmed`, `preview_confirmed`); filename capped at 180 chars; files re-validated via Sharp; storage path strictly namespaced `owner_id/entry_id/id`. On owner removal, `file_name` is scrubbed to `'Removed file'`. | `storage_path`, `file_name`, `sha256`, `kind`, `removal_reason` (3–500 chars) | service-role-only (revoked from public, anon, authenticated; select, insert, update on removed_at, removal_reason, file_name granted to service_role) | ### 5.1 M15 HMAC query-hash fix — verified present, enforced in production diff --git a/docs/worker-deploy-runbook.md b/docs/worker-deploy-runbook.md index 1d331e3c6e..627c9d79e7 100644 --- a/docs/worker-deploy-runbook.md +++ b/docs/worker-deploy-runbook.md @@ -328,7 +328,8 @@ bytes change while it runs. Never write its value into a document, log, PR or ti Without it there is no scanner verdict, so every ingestion provider call is refused and each job ends as `blocked by ingress safety checks` with `scanner_unavailable` (fail closed; pinned by -`tests/edge-provider-egress.test.ts`). See [the current ingestion model](ingestion-state-machine.md#current-model-october-2026). +`tests/edge-provider-egress.test.ts`). A worker started without it logs one warning saying so +(`adminSourceRuntimeStartupWarning`). See [the current ingestion model](ingestion-state-machine.md#current-model-october-2026). **Current production state (read-only check, 20 October 2026).** The production `worker` service runs a placeholder start command set in Railway, not the image's `node dist/worker/index.mjs`. It is an HTTP-only @@ -338,7 +339,8 @@ indexed in production. The same check found `WORKER_ADMIN_SOURCE_RUNTIME_MODULE` Before the worker is switched back on, with the owner's explicit yes: 1. Confirm the module the variable names exists in the running image or mount and loads: the worker starts - without refusing it, and a first permitted job is not stopped as `scanner_unavailable`. + without refusing it, its start-up log has no `WORKER_ADMIN_SOURCE_RUNTIME_MODULE is not set` warning, and a + first permitted job is not stopped as `scanner_unavailable`. 2. Restore the real start command, then clear any pause switch. 3. Watch the first jobs; the library re-index stays stopped ([decision](decisions/2026-10-17-library-reindex-stopped.md)). diff --git a/scripts/cleanup-storage.ts b/scripts/cleanup-storage.ts index 554c4335b1..2774e7f9f5 100644 --- a/scripts/cleanup-storage.ts +++ b/scripts/cleanup-storage.ts @@ -270,6 +270,8 @@ async function main() { storage_removed: documentCleanup.removed + imageCleanup.removed, last_error: warnings.length ? warnings.join("; ") : null, completed_at: nextStatus === "completed" ? new Date().toISOString() : null, + // Only rows whose document is gone reach here (R11 guard above); drop the title (ING-A-05). + document_title: null, metadata: cleanupJobMetadata(job.metadata, "storage_cleanup_retry", { storage_warnings: warnings }), }) .eq("id", job.id); diff --git a/src/lib/ingestion.ts b/src/lib/ingestion.ts index 53c9f299cd..e7ffbeb3ae 100644 --- a/src/lib/ingestion.ts +++ b/src/lib/ingestion.ts @@ -75,6 +75,7 @@ export type StorageCleanupJobUpdate = { metadata: { operation: string; storage_warnings: string[] }; document_paths?: string[]; image_paths?: string[]; + document_title?: null; }; // Audit R11: the DELETE route creates the storage_cleanup_jobs ledger row (with @@ -108,6 +109,9 @@ export function buildStorageCleanupJobUpdate(args: { if (args.aborted) { update.document_paths = []; update.image_paths = []; + } else { + // Audit 2026-10-20 ING-A-05: the document row is gone, and its title can identify things. + update.document_title = null; } return update; } diff --git a/src/lib/source-governance.ts b/src/lib/source-governance.ts index b30cfbea1d..f4aa671994 100644 --- a/src/lib/source-governance.ts +++ b/src/lib/source-governance.ts @@ -38,6 +38,8 @@ export const GOVERNANCE_SEVERITY_MATRIX: Record = { @@ -52,6 +54,7 @@ export const GOVERNANCE_UI_TOKEN_MATRIX: Record([ SOURCE_GOVERNANCE_CODES.REVIEW_DUE, // A known error in the cited document itself (src/lib/document-cautions.ts). SOURCE_GOVERNANCE_CODES.DOCUMENT_CAUTION, + // Pages read from a scan by OCR (audit ING-B-07): the reader should check numbers on the page. + SOURCE_GOVERNANCE_CODES.OCR_TEXT, ]); function isLocalMetadataText(value: string) { @@ -226,6 +233,20 @@ export function sourceGovernanceWarnings(args: { }); } + // Audit 2026-10-20 ING-B-07: OCR can list a value apart from its qualifier or misread a decimal. + // The flag is document-level (index quality metrics), so the message says "include pages". + const ocrMetrics = result.indexing_quality?.metrics; + if (Number(ocrMetrics?.ocr_page_count ?? 0) > 0 || Number(ocrMetrics?.ocr_coverage ?? 0) > 0) { + pushUnique(warnings, { + code: SOURCE_GOVERNANCE_CODES.OCR_TEXT, + severity: GOVERNANCE_SEVERITY_MATRIX[SOURCE_GOVERNANCE_CODES.OCR_TEXT] as SourceGovernanceWarning["severity"], + uiToken: GOVERNANCE_UI_TOKEN_MATRIX[SOURCE_GOVERNANCE_CODES.OCR_TEXT] as SourceGovernanceUiToken, + message: OCR_TEXT_WARNING_MESSAGE, + document_id, + title, + }); + } + if (typeof result.indexing_quality?.quality_score === "number" && result.indexing_quality.quality_score < 0.45) { pushUnique(warnings, { code: SOURCE_GOVERNANCE_CODES.LOW_INDEX_QUALITY, @@ -284,9 +305,18 @@ export function sourceGovernanceWarnings(args: { } const severityRank = { danger: 0, warning: 1, info: 2 } satisfies Record; - return warnings - .sort((left, right) => severityRank[left.severity] - severityRank[right.severity]) - .slice(0, args.limit ?? 8); + // OCR_TEXT is a general caution: it is grouped into ONE warning (approved wording, first OCR + // source) that holds a reserved slot, so the cap can never drop it and it never crowds + // document-specific warnings out. + const ocrWarning = warnings.find((warning) => warning.code === SOURCE_GOVERNANCE_CODES.OCR_TEXT); + const limit = args.limit ?? 8; + const others = warnings + .filter((warning) => warning.code !== SOURCE_GOVERNANCE_CODES.OCR_TEXT) + .sort((left, right) => severityRank[left.severity] - severityRank[right.severity]); + if (!ocrWarning || limit <= 0) return others.slice(0, limit); + return [...others.slice(0, limit - 1), ocrWarning].sort( + (left, right) => severityRank[left.severity] - severityRank[right.severity], + ); } function plural(count: number, singular: string, pluralValue = `${singular}s`) { @@ -317,6 +347,8 @@ function groupedMessage(warning: SourceGovernanceWarning, count: number) { return `${plural(count, "source")} ${count === 1 ? "has" : "have"} low indexing quality.`; if (warning.code === SOURCE_GOVERNANCE_CODES.WEAK_TABLE_EXTRACTION) return `${plural(count, "table evidence item")} reviewed as administrative, unrelated, or poor extraction.`; + if (warning.code === SOURCE_GOVERNANCE_CODES.OCR_TEXT) + return `${plural(count, "source")} ${count === 1 ? "includes" : "include"} pages read from a scan by text recognition (OCR). Check numbers against the PDF page.`; if (warning.code === SOURCE_GOVERNANCE_CODES.REGISTRY_RECORD) return `${plural(count, "registry summary", "registry summaries")} used as supporting evidence.`; return warning.message; diff --git a/src/lib/types.ts b/src/lib/types.ts index c8789135ae..eedf521480 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -461,6 +461,7 @@ export const SOURCE_GOVERNANCE_CODES = { WEAK_TABLE_EXTRACTION: "weak_table_extraction", REGISTRY_RECORD: "registry_record_source", DOCUMENT_CAUTION: "document_caution", + OCR_TEXT: "ocr_text_source", } as const; export type SourceGovernanceCode = (typeof SOURCE_GOVERNANCE_CODES)[keyof typeof SOURCE_GOVERNANCE_CODES]; diff --git a/tests/extracted-document-validation.test.ts b/tests/extracted-document-validation.test.ts index 30b931eb25..5a2c38ad9c 100644 --- a/tests/extracted-document-validation.test.ts +++ b/tests/extracted-document-validation.test.ts @@ -114,7 +114,7 @@ describe("Python extracted-document payload validation", () => { describe("extraction provenance (audit F04)", () => { const provenance = { name: "extract_pdf_assets", - version: "2026-10-20.embedded-image-ceiling-v10", + version: "2026-10-21.merged-cell-provenance-v11", pymupdf: "1.26.0", tableStrategy: "table_aware_page_text", }; diff --git a/tests/ingestion.test.ts b/tests/ingestion.test.ts index 249520ea6e..d5fd8a0b41 100644 --- a/tests/ingestion.test.ts +++ b/tests/ingestion.test.ts @@ -166,4 +166,16 @@ describe("storage cleanup ledger update (R11)", () => { expect(update.last_error).toBeNull(); expect(update.document_paths).toBeUndefined(); }); + + // Audit 2026-10-20 ING-A-05: a deleted document's title can identify things, so it is not kept + // in the ledger once the document row is gone. An aborted delete leaves the document alive. + it.each(["completed", "failed"] as const)("clears the deleted document's title on a %s cleanup", (status) => { + const update = buildStorageCleanupJobUpdate({ status, storageRemoved: 1, warnings: [] }); + expect(update.document_title).toBeNull(); + }); + + it("leaves the title of a document whose delete aborted untouched", () => { + const update = buildStorageCleanupJobUpdate({ status: "failed", storageRemoved: 0, warnings: [], aborted: true }); + expect(update.document_title).toBeUndefined(); + }); }); diff --git a/tests/source-governance.test.ts b/tests/source-governance.test.ts index eb2b803c8f..47407f9ed2 100644 --- a/tests/source-governance.test.ts +++ b/tests/source-governance.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + OCR_TEXT_WARNING_MESSAGE, frontendSourceGovernanceWarnings, groupSourceGovernanceWarnings, hasDangerSourceGovernanceWarning, @@ -88,6 +89,124 @@ describe("source governance warnings", () => { expect(codes).toContain("weak_table_extraction"); }); + // Audit 2026-10-20 ING-B-07: OCR can separate a value from its qualifier, and nothing told + // the reader a cited source had pages read from a scan. + it("cautions when a supporting source has pages read by OCR, without refusing the answer", () => { + const ocrSource = result({ + source_metadata: { ...result().source_metadata!, document_status: "current", extraction_quality: "good" }, + indexing_quality: { + document_id: "doc-1", + quality_score: 0.9, + extraction_quality: "good", + metrics: { ocr_page_count: 3, page_count: 10, ocr_coverage: 0.3 }, + issues: [], + }, + table_facts: [], + }); + const warnings = sourceGovernanceWarnings({ results: [ocrSource] }); + const ocr = warnings.find((warning) => warning.code === "ocr_text_source"); + expect(ocr).toMatchObject({ severity: "warning", uiToken: "caution", document_id: "doc-1" }); + expect(ocr?.message).toContain("from a scan"); + expect(hasDangerSourceGovernanceWarning(warnings.filter((warning) => warning.code === "ocr_text_source"))).toBe( + false, + ); + expect(frontendSourceGovernanceWarnings(warnings).map((warning) => warning.code)).toContain("ocr_text_source"); + expect(groupSourceGovernanceWarnings([ocr!, { ...ocr!, document_id: "doc-2" }])[0]?.message).toBe( + "2 sources include pages read from a scan by text recognition (OCR). Check numbers against the PDF page.", + ); + }); + + it("keeps a document caution inside the warning cap when many sources were read by OCR", () => { + const ocrMetrics = { + quality_score: 0.9, + extraction_quality: "good" as const, + metrics: { ocr_page_count: 3, ocr_coverage: 0.3 }, + issues: [], + }; + const base = { + ...result().source_metadata!, + document_status: "current" as const, + extraction_quality: "good" as const, + }; + const ocrSources = Array.from({ length: 5 }, (_, index) => + result({ + document_id: `doc-${index}`, + source_metadata: base, + indexing_quality: { document_id: `doc-${index}`, ...ocrMetrics }, + table_facts: [], + }), + ); + const cautioned = result({ + document_id: "doc-caution", + title: "Clozapine therapy RKPG", + file_name: "clozapine-therapy-rkpg.pdf", + source_metadata: base, + indexing_quality: { document_id: "doc-caution", ...ocrMetrics, metrics: { ocr_page_count: 0 } }, + table_facts: [], + }); + const warnings = sourceGovernanceWarnings({ results: [...ocrSources, cautioned] }); + expect(warnings.length).toBeLessThanOrEqual(8); + expect(warnings.map((warning) => warning.code)).toContain("document_caution"); + expect(warnings.filter((warning) => warning.code === "ocr_text_source")).toHaveLength(1); + }); + + it("never lets the warning cap drop the grouped OCR caution", () => { + const base = { + ...result().source_metadata!, + document_status: "current" as const, + extraction_quality: "good" as const, + }; + const crowded = Array.from({ length: 10 }, (_, index) => + result({ + document_id: `doc-crowd-${index}`, + title: `Crowded source ${index}`, + source_metadata: { ...base, jurisdiction: "New York", publisher: `Overseas publisher ${index}` }, + indexing_quality: { + document_id: `doc-crowd-${index}`, + quality_score: 0.2, + extraction_quality: "good", + metrics: { ocr_page_count: 0 }, + issues: [], + }, + table_facts: [], + }), + ); + const ocrSource = result({ + document_id: "doc-ocr", + source_metadata: base, + indexing_quality: { + document_id: "doc-ocr", + quality_score: 0.9, + extraction_quality: "good", + metrics: { ocr_page_count: 2, ocr_coverage: 0.2 }, + issues: [], + }, + table_facts: [], + }); + for (const limit of [8, 3, 1]) { + const warnings = sourceGovernanceWarnings({ results: [...crowded, ocrSource], limit }); + expect(warnings).toHaveLength(limit); + const ocr = warnings.filter((warning) => warning.code === "ocr_text_source"); + expect(ocr).toHaveLength(1); + expect(ocr[0]?.message).toBe(OCR_TEXT_WARNING_MESSAGE); + } + }); + + it("adds no OCR caution for a source with no OCR pages", () => { + const plain = result({ + indexing_quality: { + document_id: "doc-1", + quality_score: 0.9, + extraction_quality: "good", + metrics: { ocr_page_count: 0 }, + issues: [], + }, + }); + expect(sourceGovernanceWarnings({ results: [plain] }).map((warning) => warning.code)).not.toContain( + "ocr_text_source", + ); + }); + it("groups repeated document-level warnings into one counted message", () => { const grouped = groupSourceGovernanceWarnings([ { diff --git a/tests/table-facts.test.ts b/tests/table-facts.test.ts index bad87bb0ec..08f8bb831f 100644 --- a/tests/table-facts.test.ts +++ b/tests/table-facts.test.ts @@ -44,6 +44,142 @@ describe("table fact generation", () => { }); }); + // Audit 2026-10-20 ING-B-09: a vertically merged label cell comes back blank on later rows. + it("carries a merged row label forward into clinical_parameter without rewriting the cells", () => { + const facts = buildTableFactRows({ + job: { document_id: "doc-1", documents: { owner_id: "owner-1" } }, + chunkRows: [], + insertedImages: [ + { + id: "image-1", + pageNumber: 2, + tableTitle: "Synthetic monitoring table", + tableRows: [ + ["Group A", "Level 1", "First instruction"], + ["", "Level 2", "Second instruction"], + ["Group B", "Level 3", "Third instruction"], + ], + tableColumns: ["Parameter", "Threshold", "Action"], + tableMergedCells: [[1, 0]], + }, + ], + }); + + expect(facts.map((fact) => fact.clinical_parameter)).toEqual(["Group A", "Group A", "Group B"]); + expect(facts[1]).toMatchObject({ row_label: null, threshold_value: "Level 2" }); + expect(facts[1]?.metadata).toMatchObject({ + cells: ["", "Level 2", "Second instruction"], + row_label_inherited_from: { row_index: 0, label: "Group A" }, + }); + expect(facts[0]?.metadata).not.toHaveProperty("row_label_inherited_from"); + }); + + const carryFacts = (tableRows: string[][], tableMergedCells?: [number, number][] | null) => + buildTableFactRows({ + job: { document_id: "doc-1", documents: { owner_id: "owner-1" } }, + chunkRows: [], + insertedImages: [ + { + id: "image-1", + pageNumber: 2, + tableTitle: "Synthetic monitoring table", + tableRows, + tableColumns: ["Parameter", "Threshold", "Action"], + tableMergedCells, + }, + ], + }); + + it("does not carry a label into a blank cell without confirmed merge provenance", () => { + const rows = [ + ["Group A", "Level 1", "First instruction"], + ["", "", "Continuation note for the table"], + ]; + for (const facts of [carryFacts(rows), carryFacts(rows, null), carryFacts(rows, [[1, 1]])]) { + expect(facts[1]?.clinical_parameter).toBe("Synthetic monitoring table"); + expect(facts[1]?.metadata).not.toHaveProperty("row_label_inherited_from"); + } + }); + + it("carries a confirmed merged label down a multi-row span only", () => { + const facts = carryFacts( + [ + ["Group A", "Level 1", "First instruction"], + ["", "Level 2", "Second instruction"], + ["", "Level 3", "Third instruction"], + ["", "Level 4", "Unmerged blank label"], + ], + [ + [1, 0], + [2, 0], + ], + ); + expect(facts.map((fact) => fact.clinical_parameter)).toEqual([ + "Group A", + "Group A", + "Group A", + "Synthetic monitoring table", + ]); + }); + + it("resets inheritance across an empty separator row even when the next cell is marked merged", () => { + const facts = carryFacts( + [ + ["Group A", "Level 1", "First instruction"], + ["", "", ""], + ["", "Level 2", "Second instruction"], + ], + [ + [1, 0], + [2, 0], + ], + ); + expect(facts).toHaveLength(2); + expect(facts[1]?.clinical_parameter).toBe("Synthetic monitoring table"); + expect(facts[1]?.metadata).not.toHaveProperty("row_label_inherited_from"); + }); + + it("does not carry a merged label across a page or section row break", () => { + const rows = [ + ["Group A", "Level 1", "First instruction"], + ["", "Level 2", "Second instruction"], + ]; + const facts = buildTableFactRows({ + job: { document_id: "doc-1", documents: { owner_id: "owner-1" } }, + chunkRows: [], + insertedImages: [ + { + id: "image-1", + pageNumber: 2, + tableTitle: "Synthetic monitoring table", + tableRows: rows, + tableColumns: ["Parameter", "Threshold", "Action"], + tableMergedCells: [[1, 0]], + tableRowBreaks: [1], + }, + ], + }); + expect(facts[1]?.clinical_parameter).toBe("Synthetic monitoring table"); + expect(facts[1]?.metadata).not.toHaveProperty("row_label_inherited_from"); + }); + + it("does not carry a full-width section heading row down into merged cells below it", () => { + const facts = carryFacts( + [ + ["Group A", "Level 1", "First instruction"], + ["Section 2: Renal", "", ""], + ["", "Level 2", "Second instruction"], + ], + [ + [1, 1], + [1, 2], + [2, 0], + ], + ); + expect(facts[2]?.clinical_parameter).toBe("Synthetic monitoring table"); + expect(facts[2]?.metadata).not.toHaveProperty("row_label_inherited_from"); + }); + it("deduplicates repeated table rows from the same image", () => { const facts = buildTableFactRows({ job: { diff --git a/tests/worker-admin-source-runtime.test.ts b/tests/worker-admin-source-runtime.test.ts index 2e969e268d..b0c1db166f 100644 --- a/tests/worker-admin-source-runtime.test.ts +++ b/tests/worker-admin-source-runtime.test.ts @@ -14,6 +14,7 @@ import { loadAdminSourceRuntimeOptions, type ApprovedAdminSource, type AdminSourceInitializationContext, + adminSourceRuntimeStartupWarning, } from "../worker/admin-source-runtime"; import { workerProviderSourceBinding } from "../worker/provider-egress"; import type { JobRow } from "../worker/types"; @@ -387,3 +388,19 @@ describe("approved admin source normal runtime composition", () => { ).rejects.toThrow("current_input_required"); }); }); + +// Audit 2026-10-20 CH-04: without the scanner module every ingestion provider call is refused, +// so a worker started without it must say so at start-up instead of failing job by job. +describe("scanner module start-up warning (CH-04)", () => { + it("warns when the scanner module setting is absent or empty", () => { + for (const value of [undefined, ""]) { + const warning = adminSourceRuntimeStartupWarning(value); + expect(warning).toContain("WORKER_ADMIN_SOURCE_RUNTIME_MODULE is not set"); + expect(warning).toContain("scanner_unavailable"); + } + }); + + it("stays quiet when the setting is present and never echoes its value", () => { + expect(adminSourceRuntimeStartupWarning("/srv/synthetic/runtime.mjs")).toBeNull(); + }); +}); diff --git a/worker/admin-source-runtime.ts b/worker/admin-source-runtime.ts index 053af24977..cd1a85d943 100644 --- a/worker/admin-source-runtime.ts +++ b/worker/admin-source-runtime.ts @@ -421,6 +421,19 @@ export function createAdminSourceRuntimeOptions( }; } +/** + * Audit 2026-10-20 CH-04: without the scanner module there is no malware verdict, so every + * ingestion provider call is refused and each job ends as `scanner_unavailable`. Say so once at + * start-up. Only whether the setting is present is reported, never its value. + */ +export function adminSourceRuntimeStartupWarning(modulePath = process.env.WORKER_ADMIN_SOURCE_RUNTIME_MODULE) { + if (modulePath !== undefined && modulePath !== "") return null; + return ( + "WORKER_ADMIN_SOURCE_RUNTIME_MODULE is not set: there is no malware scanner, so every ingestion " + + "provider call will be refused and jobs will fail as scanner_unavailable (see docs/worker-deploy-runbook.md)." + ); +} + /** Explicit server-controlled local module/mount. No remote or request paths. */ export async function loadAdminSourceRuntimeOptions( client: SupabaseClient, diff --git a/worker/image-captioning.ts b/worker/image-captioning.ts index 1f3c160a92..b1ec4e7889 100644 --- a/worker/image-captioning.ts +++ b/worker/image-captioning.ts @@ -74,6 +74,8 @@ type InsertedImage = { accessibleTableMarkdown: string | null; tableRows: string[][] | null; tableColumns: string[] | null; + tableMergedCells: [number, number][] | null; + tableRowBreaks: number[] | null; structuredVisualProfile: StructuredVisualProfile | null; candidatePriorityScore: number; imageQualityScore: number; @@ -545,6 +547,8 @@ async function persistCaptionedImages( table_confidence: tableMetadata.tableConfidence, table_rows: tableMetadata.tableRows, table_columns: tableMetadata.tableColumns, + table_merged_cells: tableMetadata.tableMergedCells, + table_row_breaks: tableMetadata.tableRowBreaks, accessible_table_markdown: tableMetadata.accessibleTableMarkdown, clinical_use_class: policyAssessment.clinical_use_class, clinical_use_reason: policyAssessment.clinical_use_reason, @@ -595,6 +599,8 @@ async function persistCaptionedImages( accessibleTableMarkdown: tableMetadata.accessibleTableMarkdown, tableRows: tableMetadata.tableRows, tableColumns: tableMetadata.tableColumns, + tableMergedCells: tableMetadata.tableMergedCells, + tableRowBreaks: tableMetadata.tableRowBreaks, structuredVisualProfile: structuredVisualProfile ?? null, candidatePriorityScore: candidate.candidatePriorityScore, imageQualityScore: candidate.imageQualityScore, diff --git a/worker/image-classification.ts b/worker/image-classification.ts index 56990938e8..e82a61d503 100644 --- a/worker/image-classification.ts +++ b/worker/image-classification.ts @@ -89,6 +89,15 @@ export function imageTableMetadata(image: ExtractedDocument["images"][number]) { accessibleTableMarkdown: metadataString(metadata, "accessible_table_markdown") ?? tableText, tableRows: Array.isArray(metadata.table_rows) ? metadata.table_rows : null, tableColumns: Array.isArray(metadata.table_columns) ? metadata.table_columns : null, + tableMergedCells: Array.isArray(metadata.table_merged_cells) + ? (metadata.table_merged_cells as unknown[]).filter( + (cell): cell is [number, number] => + Array.isArray(cell) && cell.length === 2 && cell.every((value) => Number.isInteger(value)), + ) + : null, + tableRowBreaks: Array.isArray(metadata.table_row_breaks) + ? (metadata.table_row_breaks as unknown[]).filter((row): row is number => Number.isInteger(row)) + : null, }; } diff --git a/worker/main.ts b/worker/main.ts index 319dba8b50..09ac8e40e7 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -90,7 +90,11 @@ import { type ManagedProviderEmbeddingRequestContext, } from "./managed-provider-vectors"; import type { ManagedAdminTextSourceSectionLayout } from "./managed-admin-text-terminal"; -import { loadAdminSourceRuntimeOptions, type AdminSourceInitializationContext } from "./admin-source-runtime"; +import { + adminSourceRuntimeStartupWarning, + loadAdminSourceRuntimeOptions, + type AdminSourceInitializationContext, +} from "./admin-source-runtime"; // The ingestion stages below were extracted from this closure; each receives the runtime's // services through IngestionStageContext rather than closing over them. import { insertEmbeddedChunks } from "./chunk-indexing"; @@ -865,6 +869,8 @@ export async function runWorkerMain() { createAdminClient({ timeoutMs: resolveWorkerSupabaseTimeoutMs(process.env) }), (stopSignal) => controlPlaneAbortSignal(stopSignal), ); + const scannerWarning = adminSourceRuntimeStartupWarning(); + if (scannerWarning) console.warn(scannerWarning); const adminSourceOptions = await loadAdminSourceRuntimeOptions(client, env.SUPABASE_DOCUMENT_BUCKET); await createWorkerRuntime({ client, ...adminSourceOptions }).run(); process.exit(0); diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 94a14592fa..df9f4e7d53 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -28,7 +28,7 @@ # parser defect can be traced to the documents it touched. Bump the date and label whenever a # change alters the text, tables or images produced for the same PDF. 2026-09-16 is PR #2810, # which replaced line-major table text with table_aware_page_text (#GJHKYC). -EXTRACTOR_VERSION = "2026-10-20.embedded-image-ceiling-v10" +EXTRACTOR_VERSION = "2026-10-21.merged-cell-provenance-v11" TABLE_STRATEGY = "table_aware_page_text" # IDX-H6: caps on serialized table size. Raised from the previous 80/120-row and 8000-char @@ -1443,12 +1443,21 @@ def text_grid_rows(text): def extract_table_payload(table): try: rows = table.extract() - cleaned = [[clean_cell(cell) for cell in row] for row in rows if any(clean_cell(cell) for cell in row)] + cleaned = [] + merged_cells = [] + for row in rows: + if not any(clean_cell(cell) for cell in row): + continue + # PyMuPDF returns None (not "") for a cell covered by a merged span, so + # these coordinates are confirmed merge provenance, not merely blank cells. + merged_cells.extend([len(cleaned), column] for column, cell in enumerate(row) if cell is None) + cleaned.append([clean_cell(cell) for cell in row]) column_count = max((len(row) for row in cleaned), default=0) column_ranges = _extract_table_column_ranges(table, column_count) return { "text": table_rows_to_markdown(rows), "rows": cleaned[:MAX_TABLE_ROWS], + "merged_cells": [cell for cell in merged_cells if cell[0] < MAX_TABLE_ROWS], "columns": table_columns_from_rows(rows)[:24], "accessible_markdown": table_rows_to_markdown(rows), "row_count": len(cleaned), @@ -1461,6 +1470,7 @@ def extract_table_payload(table): return { "text": "", "rows": [], + "merged_cells": [], "columns": [], "accessible_markdown": "", "row_count": 0, @@ -1740,6 +1750,7 @@ def likely_table_candidates(page, rejected_rects=None): "table_role": role, "table_confidence": table_candidate_confidence(label, title, table_text, row_count, column_count), "table_rows": payload.get("rows", []), + "table_merged_cells": payload.get("merged_cells", []), "table_columns": payload.get("columns", []), "accessible_table_markdown": payload.get("accessible_markdown") or table_text, "row_count": row_count, @@ -1875,7 +1886,14 @@ def merge_related_table_candidates(candidates, page_rect): rect = merge_rects([item["rect"] for item in group], page_rect) merged_text = "\n\n".join(item.get("table_text") or "" for item in group).strip() merged_rows = [] + merged_cells = [] + row_breaks = [] for item in group: + offset = len(merged_rows) + if offset: + # Each grouped table starts a new section: label carry-down must not cross it. + row_breaks.append(offset) + merged_cells.extend([row + offset, column] for row, column in item.get("table_merged_cells") or []) merged_rows.extend(item.get("table_rows") or []) candidate = { **candidate, @@ -1888,6 +1906,8 @@ def merge_related_table_candidates(candidates, page_rect): ), "table_confidence": max(item.get("table_confidence") or 0 for item in group), "table_rows": merged_rows[:MAX_TABLE_ROWS], + "table_merged_cells": [cell for cell in merged_cells if cell[0] < MAX_TABLE_ROWS], + "table_row_breaks": [row for row in row_breaks if row < MAX_TABLE_ROWS], "table_columns": (merged_rows[0] if merged_rows else candidate.get("table_columns") or [])[:24], "accessible_table_markdown": table_rows_to_markdown(merged_rows) if merged_rows else merged_text, "row_count": sum(item.get("row_count") or 0 for item in group), @@ -2365,6 +2385,8 @@ def _extract_document(document, output_dir, budget, identity): detected_rect = table_candidate["rect"] edge_rect, edge_info = extend_table_rect_for_edge_content(page, detected_rect) table_rows = list(table_candidate.get("table_rows") or []) + table_merged_cells = [list(cell) for cell in table_candidate.get("table_merged_cells") or []] + table_row_breaks = list(table_candidate.get("table_row_breaks") or []) column_count = int(table_candidate.get("column_count") or (len(table_rows[0]) if table_rows else 0) or 0) recovered_rows = [] unassigned_edge_text = [] @@ -2382,6 +2404,18 @@ def _extract_document(document, output_dir, budget, identity): recovered_rows = top_rows + bottom_rows if recovered_rows: combined_rows = top_rows + table_rows + bottom_rows + table_merged_cells = [ + [row + len(top_rows), column] + for row, column in table_merged_cells + if row + len(top_rows) < MAX_TABLE_ROWS + ] + # Rows recovered past the detected edge (e.g. across a page break) start new segments. + table_row_breaks = sorted( + {row + len(top_rows) for row in table_row_breaks} + | ({len(top_rows)} if top_rows else set()) + | ({len(top_rows) + len(table_rows)} if bottom_rows else set()) + ) + table_row_breaks = [row for row in table_row_breaks if row < MAX_TABLE_ROWS] recovered_truncated = len(combined_rows) > MAX_TABLE_ROWS table_rows = combined_rows[:MAX_TABLE_ROWS] rows_truncated = ( @@ -2412,6 +2446,8 @@ def _extract_document(document, output_dir, budget, identity): "table_role": table_candidate.get("table_role"), "table_confidence": table_candidate.get("table_confidence"), "table_rows": table_rows, + "table_merged_cells": table_merged_cells, + "table_row_breaks": table_row_breaks, "table_columns": table_candidate.get("table_columns") or [], "accessible_table_markdown": accessible, "row_count": len(table_rows) if table_rows else table_candidate.get("row_count"), diff --git a/worker/python/test_extract_pdf_assets_merged_cells.py b/worker/python/test_extract_pdf_assets_merged_cells.py new file mode 100644 index 0000000000..d3a760325b --- /dev/null +++ b/worker/python/test_extract_pdf_assets_merged_cells.py @@ -0,0 +1,40 @@ +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) + +from extract_pdf_assets import extract_table_payload # noqa: E402 + + +class FakeTable: + def __init__(self, rows): + self._rows = rows + self.rows = [] + + def extract(self): + return self._rows + + +class MergedCellProvenanceTests(unittest.TestCase): + """Only cells PyMuPDF reports as span-covered (None) are recorded as merged.""" + + def test_none_cells_are_merged_and_blank_strings_are_not(self): + payload = extract_table_payload( + FakeTable( + [ + ["Parameter", "Threshold", "Action"], + ["Group A", "Level 1", "First"], + [None, "Level 2", "Second"], + [None, None, None], + ["", "Level 3", "Third"], + ] + ) + ) + self.assertEqual(len(payload["rows"]), 4) + # Row indices are in cleaned-row coordinates (the all-empty row is dropped). + self.assertEqual(payload["merged_cells"], [[2, 0]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/worker/python/test_extract_pdf_assets_raster_tables.py b/worker/python/test_extract_pdf_assets_raster_tables.py index 29b677415f..1b35b80bf1 100644 --- a/worker/python/test_extract_pdf_assets_raster_tables.py +++ b/worker/python/test_extract_pdf_assets_raster_tables.py @@ -555,7 +555,7 @@ def test_gated_merged_scan_keeps_exponent_cells_and_owning_rows(self): for cells, row_text in ((MERGED_ROWS[0], text[wbc:neutrophils]), (MERGED_ROWS[1], text[neutrophils:])): for lines in cells: self.assertIn(" ".join(lines), row_text) - self.assertEqual(result["extractor"]["version"], "2026-10-20.embedded-image-ceiling-v10") + self.assertEqual(result["extractor"]["version"], "2026-10-21.merged-cell-provenance-v11") provenance = result["pages"][0]["metadata"]["source_provenance"] self.assertEqual(provenance["ocr"]["status"], "unverified") self.assertIsNone(provenance["ocr"]["confidence"]) diff --git a/worker/table-facts.ts b/worker/table-facts.ts index 6055ecd822..e2e6fbcf65 100644 --- a/worker/table-facts.ts +++ b/worker/table-facts.ts @@ -32,6 +32,11 @@ export type TableFactImageRow = { accessibleTableMarkdown?: string | null; tableRows?: string[][] | null; tableColumns?: string[] | null; + // [rowIndex, columnIndex] of cells the extractor confirmed are covered by a merged span. + tableMergedCells?: [number, number][] | null; + // Row indices that start a new section or page segment (grouped tables, rows recovered + // across a page edge). Label carry-down never crosses one. + tableRowBreaks?: number[] | null; structuredVisualProfile?: { table_column_roles?: Record; } | null; @@ -179,6 +184,19 @@ export function buildTableFactRows(args: { // IDX-H6: raised from 120 to 400 to match the Python extractor's MAX_TABLE_ROWS so tail // rows of long dose/threshold tables are not silently dropped from the facts layer. + // Audit 2026-10-20 ING-B-09: PyMuPDF leaves a vertically merged label cell blank on every row + // after the first, so those rows lost their group. Carry the last label forward into + // clinical_parameter only; row_label, cells and chunk text keep the extracted (blank) value. + // Only a CONFIRMED merged cell (extractor span provenance in tableMergedCells) inherits; + // an ordinary blank label cell, or a table without provenance, never does. An empty + // separator row or any row with its own label resets the inheritance chain. + const labelColumn = parameterIndex >= 0 ? parameterIndex : 0; + const confirmedMergedRows = new Set( + (image.tableMergedCells ?? []).filter(([, column]) => column === labelColumn).map(([row]) => row), + ); + const rowBreaks = new Set(image.tableRowBreaks ?? []); + const mergedCellKeys = new Set((image.tableMergedCells ?? []).map(([row, column]) => `${row}:${column}`)); + let lastRowLabel: { label: string; rowIndex: number } | null = null; for (const [rowIndex, rawCells] of image.tableRows.slice(0, 400).entries()) { const cells = rawCells.map((cell) => compactSearchText(cell, 240)); // Native table extraction can include the column headings as its first row. @@ -190,9 +208,23 @@ export function buildTableFactRows(args: { cells.every((cell, index) => cell === columns[index]) ) continue; + if (rowBreaks.has(rowIndex)) lastRowLabel = null; const rowText = cells.filter(Boolean).join(" "); - if (!rowText) continue; - const rowLabel = tableFactValue(cells, parameterIndex >= 0 ? parameterIndex : 0); + if (!rowText) { + lastRowLabel = null; + continue; + } + const rowLabel = tableFactValue(cells, labelColumn); + const inheritedLabel = !rowLabel && confirmedMergedRows.has(rowIndex) ? lastRowLabel : null; + // A section heading row (its label spans every other column) is a section break: + // it is never a carry-down source for the rows below it. + const sectionHeading = + Boolean(rowLabel) && + cells.length > 1 && + cells.every((cell, column) => column === labelColumn || mergedCellKeys.has(`${rowIndex}:${column}`)); + if (sectionHeading) lastRowLabel = null; + else if (rowLabel) lastRowLabel = { label: rowLabel, rowIndex }; + else if (!inheritedLabel) lastRowLabel = null; // Named columns constrain meaning: an unmatched concentration or interval // remains in metadata.cells without being relabelled a threshold or action. const threshold = @@ -208,7 +240,15 @@ export function buildTableFactRows(args: { : columns.length === 0 ? (cells[cells.length - 1] ?? null) : null; - const dedupeKey = [image.id, rowLabel, threshold, action, rowText, image.tableTitle, image.tableLabel] + const dedupeKey = [ + image.id, + rowLabel ?? inheritedLabel?.label, + threshold, + action, + rowText, + image.tableTitle, + image.tableLabel, + ] .map((value) => String(value ?? "").toLowerCase()) .join("\u001f"); if (seen.has(dedupeKey)) continue; @@ -222,12 +262,15 @@ export function buildTableFactRows(args: { page_number: image.pageNumber ?? sourceChunk?.page_number ?? null, table_title: image.tableTitle ?? image.tableLabel, row_label: rowLabel, - clinical_parameter: rowLabel ?? image.tableTitle ?? image.tableLabel, + clinical_parameter: rowLabel ?? inheritedLabel?.label ?? image.tableTitle ?? image.tableLabel, threshold_value: threshold, action, normalized_terms: normalizedTerms(`${image.tableTitle ?? ""} ${image.tableLabel ?? ""} ${rowText}`), metadata: { row_index: rowIndex, + ...(inheritedLabel + ? { row_label_inherited_from: { row_index: inheritedLabel.rowIndex, label: inheritedLabel.label } } + : {}), columns, cells, table_role: image.tableRole,