From 11a431632a470396dd246578a2963803e5ddb1f0 Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 07:16:31 +0000 Subject: [PATCH 01/16] fix(ingestion): bound table evidence and separate integrity alerts --- docs/reindex-runbook.md | 13 +++ docs/worker-deploy-runbook.md | 7 ++ scripts/ingestion-autopilot.ts | 61 +++++++++-- src/lib/chunking.ts | 68 ++++++++++-- src/lib/extractors/document.ts | 10 ++ tests/chunker-version-ratchet.test.ts | 1 + tests/chunking-native-table-link.test.ts | 6 +- tests/chunking.test.ts | 128 +++++++++++++++++++++++ tests/ingestion-autopilot.test.ts | 121 ++++++++++++++++++++- tests/pdf-extraction-budget.test.ts | 35 +++++++ tests/worker-behavior.test.ts | 17 +++ 11 files changed, 447 insertions(+), 20 deletions(-) diff --git a/docs/reindex-runbook.md b/docs/reindex-runbook.md index 9e45460332..1ddf66aee2 100644 --- a/docs/reindex-runbook.md +++ b/docs/reindex-runbook.md @@ -71,6 +71,19 @@ If the response includes `safety.safeToRun: false`, do not retry repeatedly. Use The preflight response also includes active job counts and job metadata for operator review. +When the ingestion-autopilot probe supplies `jobs_stranded_final_attempt` or +`documents_indexed_without_chunks`, any positive count requires manual review and an alert, +even if the probe's overall `ok` is true. A supplied invalid count is also unassessed and alerts; +older probes that omit these fields remain compatible. These signals alone never trigger queue +recovery, including with `--apply`. If separate recoverable queue work also exists, bounded +recovery may run, but the integrity alert remains until a subsequent probe confirms resolution. + +`TABLE_ROW_EXCEEDS_CHUNK_BUDGET` is a permanent ingestion refusal: the title, column +header, one intact row and attached note cannot fit the configured character target. +The chunker returns no partial chunk array and never splits cells or truncates the +evidence. Review the extracted table and source structure before retrying; the refusal +does not establish OCR accuracy. An atomic reindex preserves the existing index on failure. + ## Do not do this - Do not run workers while Supabase SQL probes are timing out. diff --git a/docs/worker-deploy-runbook.md b/docs/worker-deploy-runbook.md index 1d331e3c6e..70f70dcedd 100644 --- a/docs/worker-deploy-runbook.md +++ b/docs/worker-deploy-runbook.md @@ -26,6 +26,13 @@ database or Python failures still return 503 and block promotion. The Supabase probe reports failures as a structured result; `/health` honours that result as well as thrown errors and omits provider failure details. +Production PDF extraction requires the Python prerequisites. If Python extraction +fails, `NODE_ENV=production` refuses the JavaScript fallback with a permanent +`PDF_EXTRACTION_BUDGET_EXCEEDED` error and removes partial artifacts: pdf.js has +no isolated memory limit inside the worker. Restore the Python runtime before +retrying the document. The developer fallback retains its deadline and pixel +limits; those limits do not establish a process memory boundary. + --- ## 0. Pre-deploy gate — do not deploy before this is live diff --git a/scripts/ingestion-autopilot.ts b/scripts/ingestion-autopilot.ts index f596263052..a01525f3cb 100644 --- a/scripts/ingestion-autopilot.ts +++ b/scripts/ingestion-autopilot.ts @@ -8,8 +8,9 @@ import { appendFileSync } from "node:fs"; // (reindex-health) to a decision, and — only when a stuck-queue signal is present // and --apply is passed — runs the existing recovery path (recover-ingestion-queue // --apply). It ALERTS (non-zero exit) only when Supabase is unreachable or recovery -// fails, or (scheduled, --alert-on-stuck) when recoverable work or a pending upload that -// nothing has claimed is detected; a healthy, idle or deliberately paused queue exits 0. +// fails, when integrity faults require manual review, or (scheduled, --alert-on-stuck) +// when recoverable work or a pending upload that nothing has claimed is detected; +// a healthy, idle or deliberately paused queue exits 0. // // Decision is based purely on reindex-health's JSON (a clean read-only probe). // check-indexing is intentionally NOT chained here: it additionally requires the @@ -48,11 +49,15 @@ export type WorkerState = "running" | "paused" | "unknown"; export type IngestionAssessment = { available: boolean; stuck: boolean; + recoveryNeeded: boolean; + integrityProblems: string[]; reasons: string[]; failedJobs: number; staleProcessingJobs: number; agedPendingJobs: number; strandedQueuedDocuments: number; + strandedFinalAttemptJobs: number; + indexedDocumentsWithoutChunks: number; workerState: WorkerState; }; @@ -72,7 +77,9 @@ export function describeWorkerState(assessment: IngestionAssessment, staleAfterM `until it is resumed; ${ assessment.agedPendingJobs > 0 ? `${assessment.agedPendingJobs} pending job(s) have waited > ${staleAfterMinutes}m, so this alerts` - : `no pending job has waited > ${staleAfterMinutes}m, so no alert` + : assessment.integrityProblems.length > 0 + ? "ingestion integrity requires attention" + : `no pending job has waited > ${staleAfterMinutes}m, so no alert` }` ); } @@ -88,8 +95,9 @@ export function describeWorkerState(assessment: IngestionAssessment, staleAfterM * claimed (the worker is stopped, paused or wedged; audit EVAL-03). A pending job is * due at `next_run_at` (a retry backs off into the future), or at `created_at` when * it has none. A deliberate pause is reported as `workerState: "paused"`, never as - * stuck by itself; only a pending job left waiting behind it alerts. Env-free and - * unit-tested. + * stuck by itself; a pending job left waiting behind it still alerts. Env-free and + * unit-tested. Optional integrity counts also require attention, but do not make + * queue recovery safe or sufficient; `recoveryNeeded` covers only the queue signals. */ export function assessIngestionHealth( health: IngestionHealth, @@ -104,11 +112,15 @@ export function assessIngestionHealth( return { available: false, stuck: false, + recoveryNeeded: false, + integrityProblems: [], reasons: ["supabase unavailable"], failedJobs: 0, staleProcessingJobs: 0, agedPendingJobs: 0, strandedQueuedDocuments: 0, + strandedFinalAttemptJobs: 0, + indexedDocumentsWithoutChunks: 0, workerState, }; } @@ -151,15 +163,39 @@ export function assessIngestionHealth( if (strandedQueuedDocuments > 0) { reasons.push(`${strandedQueuedDocuments} stranded queued document(s)`); } + const recoveryNeeded = reasons.length > 0; + const integrityProblems: string[] = []; + function integrityCount(name: string, label: string): number { + // Older probes omit these fields. A supplied but invalid value is unknown, + // never evidence of a healthy corpus; do not copy provider error text here. + if (!health.counts || !Object.hasOwn(health.counts, name)) return 0; + const value = health.counts[name]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + integrityProblems.push(`${label} count unavailable`); + return 0; + } + if (value > 0) integrityProblems.push(`${value} ${label}`); + return value; + } + const strandedFinalAttemptJobs = integrityCount("jobs_stranded_final_attempt", "stranded final-attempt job(s)"); + const indexedDocumentsWithoutChunks = integrityCount( + "documents_indexed_without_chunks", + "indexed document(s) without chunks", + ); + reasons.push(...integrityProblems); return { available: true, stuck: reasons.length > 0, + recoveryNeeded, + integrityProblems, reasons, failedJobs, staleProcessingJobs, agedPendingJobs, strandedQueuedDocuments, + strandedFinalAttemptJobs, + indexedDocumentsWithoutChunks, workerState, }; } @@ -221,7 +257,15 @@ function main() { return; } - console.log(`[autopilot] stuck-queue signal: ${assessment.reasons.join("; ")}`); + console.log(`[autopilot] ingestion attention signal: ${assessment.reasons.join("; ")}`); + const integrityAttentionRequired = assessment.integrityProblems.length > 0; + if (integrityAttentionRequired) { + console.error("[autopilot] integrity requires manual review; automatic queue recovery does not repair it."); + } + if (!assessment.recoveryNeeded) { + process.exitCode = 2; + return; + } if (!apply) { console.log( @@ -229,8 +273,8 @@ function main() { `--include-stranded-queued --stale-after-minutes ${staleAfterMinutes} --limit ${limit}\n` + `[autopilot] pass --apply to recover.`, ); - if (alertOnStuck) { - console.error("[autopilot] scheduled probe detected recoverable ingestion work — alerting."); + if (alertOnStuck || integrityAttentionRequired) { + console.error("[autopilot] ingestion work requires attention — alerting."); process.exitCode = 2; } return; @@ -254,6 +298,7 @@ function main() { return; } console.log("[autopilot] recovery applied. Re-probe with npm run reindex:health to confirm."); + if (integrityAttentionRequired) process.exitCode = 2; } // Only run as a CLI when invoked directly — importing (tests) must not execute. diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index 77fc666035..ac274ea8c4 100644 --- a/src/lib/chunking.ts +++ b/src/lib/chunking.ts @@ -8,8 +8,8 @@ import type { ChunkInput, DocumentChunk } from "@/lib/types"; // re-index can tell which chunker produced a row, and so bumping the strategy deliberately // invalidates stale chunks. Bump this when chunk boundaries change (e.g. when cross-page // document-mode chunking is enabled by default). -export const CHUNKER_VERSION = "1.3.0-page"; -export const DOCUMENT_CHUNKER_VERSION = "1.3.0-document"; +export const CHUNKER_VERSION = "1.3.1-page"; +export const DOCUMENT_CHUNKER_VERSION = "1.3.1-document"; const sentenceBoundary = /(?<=[.!?])\s+/; const sentenceBoundaryGlobal = new RegExp(sentenceBoundary.source, "g"); @@ -471,7 +471,12 @@ function chunkOversizedParagraph(paragraph: string, chunkSize: number, overlap: if (!segment.table || segment.lines.length < 2) { const next = segments[index + 1]; const captionStart = next?.table && next.lines.length >= 2 ? tableCaptionStart(segment.lines) : -1; - pushProse(captionStart >= 0 ? segment.lines.slice(0, captionStart) : segment.lines); + const previous = segments[index - 1]; + const footnoteCount = + previous?.table && previous.lines.length >= 2 + ? tableFootnoteLineCount(segment.lines, Boolean(next?.table && next.lines.length >= 2)) + : 0; + pushProse(segment.lines.slice(footnoteCount, captionStart >= 0 ? captionStart : undefined)); return; } const previous = segments[index - 1]; @@ -483,7 +488,18 @@ function chunkOversizedParagraph(paragraph: string, chunkSize: number, overlap: .map((line) => line.trim()) .join("\n") : ""; - chunks.push(...chunkTableBlock(segment.lines.join("\n"), chunkSize, caption)); + const next = segments[index + 1]; + const followingTable = segments[index + 2]; + const footnote = + next && !next.table + ? next.lines + .slice( + 0, + tableFootnoteLineCount(next.lines, Boolean(followingTable?.table && followingTable.lines.length >= 2)), + ) + .join("\n") + : ""; + chunks.push(...chunkTableBlock(segment.lines.join("\n"), chunkSize, caption, footnote)); }); return chunks; } @@ -516,35 +532,67 @@ function tableCaptionStart(lines: string[]) { return -1; } +// Only an immediately adjacent, explicit note or clinical prohibition is carried +// with a table. Keep its entire prose run: a line wrap can separate a prohibition +// from its condition. A paragraph boundary or the next table's caption provides +// a structural boundary; guessing from sentence/line length can change meaning. +const tableFootnoteLinePattern = /^(?:notes?\b\s*[:.]|footnotes?\b\s*[:.]|[*†‡]\s*\S)/i; +function tableFootnoteLineCount(lines: string[], followedByTable = false) { + const first = lines[0]?.trim() ?? ""; + if (!first || !(tableFootnoteLinePattern.test(first) || clinicalProhibitionPattern.test(first))) return 0; + const captionStart = followedByTable ? tableCaptionStart(lines) : -1; + return captionStart >= 0 ? captionStart : lines.length; +} + function isTableBlock(block: string) { const lines = block.split(lineBreakPattern).filter((line) => line.trim().length > 0); if (lines.length < 2) return false; return lines.every((line) => tableRowPattern.test(line)); } -function chunkTableBlock(block: string, chunkSize: number, caption = "") { +// Refuse unrepresentable table evidence rather than truncate or separate cells +// and clinical qualifiers. Keep diagnostics content-free, including the message. +export class OversizedTableRowError extends Error { + readonly code = "TABLE_ROW_EXCEEDS_CHUNK_BUDGET"; + + constructor() { + super("TABLE_ROW_EXCEEDS_CHUNK_BUDGET"); + this.name = "OversizedTableRowError"; + } +} + +function chunkTableBlock(block: string, chunkSize: number, caption = "", footnote = "") { const lines = block.split(lineBreakPattern).filter((line) => line.trim().length > 0); - const captioned = caption ? `${caption}\n${block.trim()}` : block.trim(); + const captioned = [caption, block.trim(), footnote].filter(Boolean).join("\n"); if (captioned.length <= chunkSize) return [captioned]; + // Reserve the same source qualification on every row group so retrieval cannot + // separate a dose/threshold from its immediately following warning. + const rowBudget = chunkSize - (footnote ? footnote.length + 1 : 0); + const withFootnote = (text: string) => (footnote ? `${text}\n${footnote}` : text); const separatorIndex = lines.findIndex((line) => tableSeparatorRowPattern.test(line) && line.includes("-")); const headerLines = separatorIndex > 0 ? lines.slice(0, separatorIndex + 1) : lines.slice(0, 1); const bodyLines = lines.slice(headerLines.length); const header = [caption, ...headerLines].filter(Boolean).join("\n"); + // Cells and their qualifiers are indivisible evidence. Refuse the whole build + // before emitting any row group when that evidence cannot fit the character target. + if (header.length > rowBudget || bodyLines.some((row) => header.length + 1 + row.length > rowBudget)) { + throw new OversizedTableRowError(); + } const chunks: string[] = []; let current = header; for (const row of bodyLines) { const candidate = `${current}\n${row}`; - if (candidate.length > chunkSize && current !== header) { - chunks.push(current.trim()); + if (candidate.length > rowBudget && current !== header) { + chunks.push(withFootnote(current.trim())); current = `${header}\n${row}`; } else { current = candidate; } } - if (current.trim()) chunks.push(current.trim()); - return chunks.length > 0 ? chunks : [block.trim()]; + if (current.trim()) chunks.push(withFootnote(current.trim())); + return chunks.length > 0 ? chunks : [captioned]; } const clinicalProhibitionPattern = diff --git a/src/lib/extractors/document.ts b/src/lib/extractors/document.ts index 1cc1015039..a00f197c79 100644 --- a/src/lib/extractors/document.ts +++ b/src/lib/extractors/document.ts @@ -536,6 +536,16 @@ export async function extractPdf( await rm(tempRoot, { recursive: true, force: true }).catch(() => undefined); throw error; } + // The fallback shares the worker's heap and native image allocations. Its + // deadline/pixel caps are not an isolated memory limit, so production must + // refuse this path rather than retry a failed PDF parser in-process. + if (process.env.NODE_ENV === "production") { + await rm(tempRoot, { recursive: true, force: true }).catch(() => undefined); + throw new PdfExtractionResourceError( + "PDF_EXTRACTION_BUDGET_EXCEEDED", + "JavaScript PDF fallback has no isolated memory limit and is disabled in production; restore Python PDF extraction prerequisites", + ); + } // Fallback for developer machines without PyMuPDF/pytesseract. It still // indexes text PDFs, but scanned PDFs and image extraction need the Python // helper dependencies listed in worker/python/requirements.txt. diff --git a/tests/chunker-version-ratchet.test.ts b/tests/chunker-version-ratchet.test.ts index 83d39cc339..6df0723206 100644 --- a/tests/chunker-version-ratchet.test.ts +++ b/tests/chunker-version-ratchet.test.ts @@ -32,6 +32,7 @@ const recorded: Record = { "1.1.1-page|1.1.1-document": "4af6b997fc7ead3f8678b027a396b8ee3352bec8d155a4cbc7e751da5a6f43d5", "1.2.0-page|1.2.0-document": "e0eb0b337d010f3c4f078ef2df970b3b91d0d420dd2916b0f0f98a83600957e8", "1.3.0-page|1.3.0-document": "e0eb0b337d010f3c4f078ef2df970b3b91d0d420dd2916b0f0f98a83600957e8", + "1.3.1-page|1.3.1-document": "e0eb0b337d010f3c4f078ef2df970b3b91d0d420dd2916b0f0f98a83600957e8", }; function fingerprint() { diff --git a/tests/chunking-native-table-link.test.ts b/tests/chunking-native-table-link.test.ts index 32309378c0..353d23258e 100644 --- a/tests/chunking-native-table-link.test.ts +++ b/tests/chunking-native-table-link.test.ts @@ -248,7 +248,11 @@ describe("page-local native table linkage through buildChunks", () => { }); it("does not associate a complete native descriptor with chunks containing only table fragments", async () => { - const chunks = await build([page(tableText)], "page", "120"); + // Fit one intact header-bound row, while still requiring separate row groups. + const lines = tableText.split("\n"); + const rowBudget = lines.slice(0, 2).join("\n").length + 1 + Math.max(...lines.slice(2).map((row) => row.length)); + expect(rowBudget).toBeLessThan(tableText.length); + const chunks = await build([page(tableText)], "page", String(rowBudget)); expect(chunks.length).toBeGreaterThan(1); for (const chunk of chunks) { const sourceText = chunk.content.replace(/\[\[IMAGE_DATA_START\]\][\s\S]*?\[\[IMAGE_DATA_END\]\]/g, "").trim(); diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index cc36f4221d..200c660342 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { CHUNKER_VERSION, DOCUMENT_CHUNKER_VERSION, + OversizedTableRowError, buildChunks, buildImageTag, chunkContentKey, @@ -726,6 +727,69 @@ describe("table-aware chunking with a title and footnote (ING-B-03)", () => { expect(combined).toContain("Do not give lithium if the patient is dehydrated."); }); + it("keeps the immediately following clinical footnote with every row group", () => { + const footnote = "Do not give lithium if the patient is dehydrated."; + const chunks = chunkTextWithOverlap(pageText, 1200, 200); + expect(chunks.length).toBeGreaterThan(3); + for (const chunk of chunks) { + expect(chunk).toContain("| Level | Concentration | Action |"); + expect(chunk).toContain(footnote); + expect(chunk.length).toBeLessThanOrEqual(1200); + } + const emittedRows = chunks.flatMap((chunk) => chunk.split("\n").filter((line) => /^\| Level \d/.test(line))); + expect(emittedRows).toEqual(rows); + }); + + it("attaches a marked table note without absorbing subsequent prose", () => { + const note = "* Repeat the measurement before changing treatment."; + const after = "Next section discusses discharge planning."; + const chunks = chunkTextWithOverlap( + [ + "Table 3. Monitoring", + "| Level | Concentration | Action |", + "| --- | --- | --- |", + ...rows, + note, + "", + after, + ].join("\n"), + 1200, + 200, + ); + const tableChunks = chunks.filter((chunk) => chunk.includes("| Level |")); + for (const chunk of tableChunks) { + expect(chunk).toContain(note); + expect(chunk).not.toContain(after); + expect(chunk.length).toBeLessThanOrEqual(1200); + } + expect(chunks).toContain(after); + expect(chunks).not.toContain(note); + }); + + it("retains a complete wrapped conditional note on every table row group", () => { + const note = "Note: Do not give lithium\nunless renal function has been assessed."; + const chunks = chunkTextWithOverlap( + ["Table 3. Monitoring", "| Level | Concentration | Action |", "| --- | --- | --- |", ...rows, note].join("\n"), + 1200, + 200, + ); + expect(chunks.length).toBeGreaterThan(3); + for (const chunk of chunks) { + expect(chunk).toContain(note); + expect(chunk).toContain("| Level | Concentration | Action |"); + expect(chunk.length).toBeLessThanOrEqual(1200); + } + }); + + it("keeps an adjacent note block intact while reserving the next table's title", () => { + const first = ["Table 1. First", "| Group | Action |", "| --- | --- |", "| A | Repeat measurement. |"]; + const note = "Note: Use the source condition\nunless a later review applies."; + const second = ["Table 2. Second", "| Group | Review |", "| --- | --- |", "| B | Review the result. |"]; + const budget = [...first, note].join("\n").length; + const chunks = chunkTextWithOverlap([...first, note, ...second].join("\n"), budget, 0); + expect(chunks).toEqual([[...first, note].join("\n"), second.join("\n")]); + }); + it("keeps prose before an untitled table and splits the table with its column header", () => { const prose = "Lithium monitoring applies to every patient on maintenance therapy."; const chunks = chunkTextWithOverlap( @@ -765,3 +829,67 @@ describe("table-aware chunking with a title and footnote (ING-B-03)", () => { for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(1200); }); }); + +// Table cells stay atomic: a character target cannot justify splitting a +// clinical condition from the assertion or another cell it qualifies. +describe("oversized table row refusal", () => { + const header = "| Group | Action |\n| --- | --- |"; + const row = `| GROUP-A | Do not change treatment unless ${"the original condition remains applicable ".repeat(20).trim()}. |`; + + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + + it("refuses an indivisible long row with a content-free typed error", () => { + const text = `${header}\n${row}`; + expect(() => chunkTextWithOverlap(text, 400, 40)).toThrow(OversizedTableRowError); + try { + chunkTextWithOverlap(text, 400, 40); + } catch (error) { + expect(error).toMatchObject({ + name: "OversizedTableRowError", + code: "TABLE_ROW_EXCEEDS_CHUNK_BUDGET", + message: "TABLE_ROW_EXCEEDS_CHUNK_BUDGET", + }); + expect(JSON.stringify(error)).not.toContain("GROUP-A"); + expect(String(error)).not.toContain("original condition"); + } + }); + + it("accepts a row exactly at the character limit including title and footnote", () => { + const title = "Table 4. Synthetic monitoring"; + const note = "Note: Preserve the original clinical qualification."; + const rows = ["| GROUP-A | Repeat the measurement. |", "| GROUP-B | Review the result. |"]; + const atomic = [title, header, rows[0], note].join("\n"); + const text = [title, header, ...rows, note].join("\n"); + const chunks = chunkTextWithOverlap(text, atomic.length, 0); + expect(chunks).toEqual(rows.map((line) => [title, header, line, note].join("\n"))); + for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(atomic.length); + expect(() => chunkTextWithOverlap(text, atomic.length - 1, 0)).toThrow(OversizedTableRowError); + }); + + it("refuses when a header alone cannot fit the character budget", () => { + const text = `| Group | ${"Original column qualification ".repeat(20).trim()} |\n| --- | --- |`; + expect(() => chunkTextWithOverlap(text, 400, 0)).toThrow(OversizedTableRowError); + }); + + it("refuses when the attached qualification alone exhausts the budget", () => { + const note = `Note: ${"Preserve the original qualification. ".repeat(20).trim()}`; + const text = [header, "| GROUP-A | Repeat the measurement. |", note].join("\n"); + expect(() => chunkTextWithOverlap(text, 400, 0)).toThrow(OversizedTableRowError); + }); + + it.each(["page", "document"])("propagates refusal without returning earlier chunks in %s mode", async (strategy) => { + vi.resetModules(); + vi.stubEnv("CHUNK_STRATEGY", strategy); + vi.stubEnv("CHUNK_SIZE", "400"); + vi.stubEnv("CHUNK_OVERLAP", "40"); + const mod = await import("../src/lib/chunking"); + const inputs = [ + { documentId: "synthetic-document", pageNumber: 1, pageText: "Earlier Section\n\nEarlier complete guidance." }, + { documentId: "synthetic-document", pageNumber: 2, pageText: `Table 4. Monitoring\n${header}\n${row}` }, + ]; + expect(() => mod.buildChunks(inputs)).toThrow(mod.OversizedTableRowError); + }); +}); diff --git a/tests/ingestion-autopilot.test.ts b/tests/ingestion-autopilot.test.ts index 6aceeea219..3d529303bc 100644 --- a/tests/ingestion-autopilot.test.ts +++ b/tests/ingestion-autopilot.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { assessIngestionHealth, describeWorkerState, workerPausedFromEnv } from "../scripts/ingestion-autopilot"; @@ -9,6 +10,8 @@ const workflow = readFileSync(new URL("../.github/workflows/ingestion-autopilot. const NOW = Date.parse("2026-07-13T12:00:00Z"); const minutesAgo = (m: number) => new Date(NOW - m * 60_000).toISOString(); +vi.mock("node:child_process", () => ({ spawnSync: vi.fn() })); + describe("assessIngestionHealth", () => { it("reports unavailable (and not stuck) when Supabase is down", () => { const a = assessIngestionHealth({ ok: false, status: "supabase_unavailable" }, { now: NOW }); @@ -31,9 +34,54 @@ describe("assessIngestionHealth", () => { const a = assessIngestionHealth({ ok: true, counts: { jobs_failed: 3 }, openJobs: [] }, { now: NOW }); expect(a.stuck).toBe(true); expect(a.failedJobs).toBe(3); + expect(a.recoveryNeeded).toBe(true); expect(a.reasons[0]).toContain("3 failed job"); }); + it.each([ + ["jobs_stranded_final_attempt", "strandedFinalAttemptJobs", "stranded final-attempt job(s)"], + ["documents_indexed_without_chunks", "indexedDocumentsWithoutChunks", "indexed document(s) without chunks"], + ])("alerts on %s without authorizing queue recovery", (countName, assessmentName, label) => { + const a = assessIngestionHealth({ ok: true, counts: { [countName]: 2 } }, { now: NOW }); + expect(a).toMatchObject({ available: true, stuck: true, recoveryNeeded: false, [assessmentName]: 2 }); + expect(a.integrityProblems).toEqual([`2 ${label}`]); + expect(a.reasons).toEqual(a.integrityProblems); + }); + + it("keeps missing old-build and valid zero integrity counts compatible", () => { + for (const counts of [undefined, {}, { jobs_stranded_final_attempt: 0, documents_indexed_without_chunks: 0 }]) { + expect(assessIngestionHealth({ ok: true, counts }, { now: NOW })).toMatchObject({ + stuck: false, + recoveryNeeded: false, + integrityProblems: [], + }); + } + }); + + it.each([null, -1, 0.5, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1])( + "reports present invalid integrity telemetry %s without queue recovery", + (value) => { + const a = assessIngestionHealth( + { ok: true, counts: { jobs_stranded_final_attempt: value, documents_indexed_without_chunks: value } }, + { now: NOW }, + ); + expect(a).toMatchObject({ available: true, stuck: true, recoveryNeeded: false }); + expect(a.integrityProblems).toEqual([ + "stranded final-attempt job(s) count unavailable", + "indexed document(s) without chunks count unavailable", + ]); + }, + ); + + it("preserves queue recovery eligibility alongside an independent integrity fault", () => { + const a = assessIngestionHealth( + { ok: true, counts: { jobs_failed: 1, documents_indexed_without_chunks: 1 } }, + { now: NOW }, + ); + expect(a).toMatchObject({ stuck: true, recoveryNeeded: true, failedJobs: 1 }); + expect(a.reasons).toEqual(["1 failed job(s)", "1 indexed document(s) without chunks"]); + }); + it("flags a processing job whose lock is older than the stale threshold", () => { const a = assessIngestionHealth( { @@ -176,3 +224,74 @@ describe("assessIngestionHealth", () => { expect(workflow).toContain("--include-stranded-queued"); }); }); + +describe("ingestion-autopilot CLI integrity safeguards", () => { + async function runCli(counts: Record, args: string[]) { + const previousArgv = process.argv; + const previousExitCode = process.exitCode; + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + vi.resetModules(); + const run = vi.mocked(spawnSync); + run.mockReset(); + run.mockReturnValue({ + pid: 0, + output: [], + stdout: JSON.stringify({ ok: true, counts }), + stderr: "", + status: 0, + signal: null, + }); + vi.spyOn(process.stdout, "write").mockReturnValue(true); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + process.argv = [process.execPath, "/synthetic/ingestion-autopilot.ts", ...args]; + process.exitCode = 0; + delete process.env.GITHUB_STEP_SUMMARY; + try { + await import("../scripts/ingestion-autopilot"); + return { exitCode: process.exitCode, calls: run.mock.calls.map((call) => call[1]) }; + } finally { + process.argv = previousArgv; + process.exitCode = previousExitCode; + if (summaryPath === undefined) delete process.env.GITHUB_STEP_SUMMARY; + else process.env.GITHUB_STEP_SUMMARY = summaryPath; + vi.restoreAllMocks(); + } + } + + it.each([{ args: [] }, { args: ["--alert-on-stuck"] }, { args: ["--apply"] }])( + "alerts on integrity-only faults without running recovery with flags $args", + async ({ args }) => { + const result = await runCli({ jobs_stranded_final_attempt: 1, documents_indexed_without_chunks: 1 }, args); + expect(result.exitCode).toBe(2); + expect(result.calls).toHaveLength(1); + expect(result.calls[0]).toContain("scripts/reindex-health.ts"); + }, + ); + + it.each([null, "1", { unsafe: "synthetic provider error text" }])( + "does not run recovery for invalid integrity telemetry %j under apply", + async (value) => { + const result = await runCli({ jobs_stranded_final_attempt: value }, ["--apply"]); + expect(result.exitCode).toBe(2); + expect(result.calls).toHaveLength(1); + }, + ); + + it("retains the integrity alert after bounded recovery of separate queue faults", async () => { + const result = await runCli({ jobs_failed: 1, documents_indexed_without_chunks: 1 }, ["--apply"]); + expect(result.exitCode).toBe(2); + expect(result.calls).toHaveLength(2); + expect(result.calls[1]).toEqual([ + "scripts/run-tsx.mjs", + "scripts/recover-ingestion-queue.ts", + "--apply", + "--yes", + "--include-stranded-queued", + "--stale-after-minutes", + "30", + "--limit", + "20", + ]); + }); +}); diff --git a/tests/pdf-extraction-budget.test.ts b/tests/pdf-extraction-budget.test.ts index 6af87783f7..a36a9b7c39 100644 --- a/tests/pdf-extraction-budget.test.ts +++ b/tests/pdf-extraction-budget.test.ts @@ -51,6 +51,7 @@ async function createRecoverableMalformedTextPdf() { afterEach(async () => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); @@ -203,6 +204,40 @@ describe("PDF extraction budgets", () => { expect(existsSync(temporaryRoot)).toBe(false); }); + it("refuses unisolated JavaScript fallback in production and cleans up without retrying", async () => { + const root = await mkdtemp(path.join(tmpdir(), "clinical-kb-pdf-production-fallback-test-")); + roots.push(root); + const fakeExtractor = path.join(root, "failing-extractor.py"); + const rootRecord = path.join(root, "temporary-root.txt"); + await writeFile( + fakeExtractor, + [ + "import pathlib, sys", + "record = pathlib.Path(sys.argv[1]).read_text(encoding='utf-8')", + "temporary_root = pathlib.Path(sys.argv[2]).parent", + "pathlib.Path(record).write_text(str(temporary_root), encoding='utf-8')", + "(temporary_root / 'partial-artifact.bin').write_bytes(b'partial')", + "raise SystemExit(2)", + ].join("\n"), + "utf8", + ); + const getText = vi.spyOn(PdfjsFallbackParser.prototype, "getText"); + const getImage = vi.spyOn(PdfjsFallbackParser.prototype, "getImage"); + vi.stubEnv("NODE_ENV", "production"); + + const error = await extractPdf(Buffer.from(rootRecord, "utf8"), { + scriptPathOverride: fakeExtractor, + }).catch((failure: unknown) => failure); + + expect(error).toMatchObject({ code: "PDF_EXTRACTION_BUDGET_EXCEEDED" }); + expect(error).toBeInstanceOf(PdfExtractionResourceError); + expect((error as Error).message).toContain("isolated memory limit"); + expect(isRetryableIngestionError((error as Error).message)).toBe(false); + expect(getText).not.toHaveBeenCalled(); + expect(getImage).not.toHaveBeenCalled(); + expect(existsSync(await readFile(rootRecord, "utf8"))).toBe(false); + }); + it("stops a stalled JavaScript fallback at the document deadline and cleans up (ING-B-05)", async () => { const root = await mkdtemp(path.join(tmpdir(), "clinical-kb-pdf-fallback-deadline-test-")); roots.push(root); diff --git a/tests/worker-behavior.test.ts b/tests/worker-behavior.test.ts index 41a9efb25d..aec0c5c489 100644 --- a/tests/worker-behavior.test.ts +++ b/tests/worker-behavior.test.ts @@ -1,8 +1,25 @@ import { describe, expect, it } from "vitest"; import { sourceSegment } from "./helpers/source-contract"; import { enrichmentRepairDecision, ingestionFailureDecision, visionImageRejectionSkipReason } from "../worker/behavior"; +import { OversizedTableRowError } from "../src/lib/chunking"; describe("ingestion worker behavior", () => { + it.each([false, true])("does not retry an oversized atomic table row (atomic reindex: %s)", (atomicReindex) => { + expect( + ingestionFailureDecision({ + error: new OversizedTableRowError(), + attemptCount: 1, + maxAttempts: 4, + atomicReindex, + }), + ).toEqual({ + retry: false, + documentStatus: atomicReindex ? "indexed" : "failed", + stage: "failed", + errorMessage: "TABLE_ROW_EXCEEDS_CHUNK_BUDGET", + }); + }); + it("retries transient failures while preserving a previously indexed document", () => { const decision = ingestionFailureDecision({ error: new Error("network connection reset"), From 17b7745f50febe7ecb2e702bae20b7eec19ce640 Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 07:17:12 +0000 Subject: [PATCH 02/16] fix(extraction): preflight displayed image and mask decode budgets --- worker/python/extract_pdf_assets.py | 105 ++++++++--- .../test_extract_pdf_assets_raster_tables.py | 2 +- ...test_extract_pdf_assets_resource_limits.py | 171 ++++++++++++++++++ 3 files changed, 251 insertions(+), 27 deletions(-) diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 94a14592fa..72ba487518 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -28,7 +28,7 @@ # parser defect can be traced to the documents it touched. Bump the date and label whenever a # change alters the text, tables or images produced for the same PDF. 2026-09-16 is PR #2810, # which replaced line-major table text with table_aware_page_text (#GJHKYC). -EXTRACTOR_VERSION = "2026-10-20.embedded-image-ceiling-v10" +EXTRACTOR_VERSION = "2026-10-21.image-decode-preflight-v11" TABLE_STRATEGY = "table_aware_page_text" # IDX-H6: caps on serialized table size. Raised from the previous 80/120-row and 8000-char @@ -182,6 +182,56 @@ def oversized_embedded_image_warning(image_info, max_pixels=MAX_EMBEDDED_IMAGE_P ) +def page_image_decode_preflight(page, image_infos): + """Inspect image headers, including inline images and separate soft masks. + + Image hashes, dict image bytes and page renders can decode every source + image even when their output pixels are bounded. Metadata without hashes + reads the compressed image headers and displayed placements only. + """ + skipped = {} + for image_info in image_infos: + xref, smask = image_info[:2] + warning = oversized_embedded_image_warning(image_info) + if not warning and smask: + width = int(page.parent.xref_get_key(smask, "Width")[1]) + height = int(page.parent.xref_get_key(smask, "Height")[1]) + if width * height > MAX_EMBEDDED_IMAGE_PIXELS: + warning = (f"embedded_image_skipped: xref {xref} soft mask {smask} declares " + f"{width}x{height}, above the {MAX_EMBEDDED_IMAGE_PIXELS}-pixel " + "embedded-image ceiling; not decoded") + if warning: + skipped[xref] = warning + displayed = page.get_image_info(hashes=False, xrefs=False) + unsafe = any( + int(image["width"]) * int(image["height"]) > MAX_EMBEDDED_IMAGE_PIXELS + for image in displayed + ) + # get_images also lists dead resources: an unused oversized XObject must + # still be skipped as an artifact, but cannot make a safe page render unsafe. + # Displayed image headers above cover the base pixels, including inline and + # Form images. Separate masks need their owning image's placement as well. + for image_info in image_infos: + if image_info[0] not in skipped or oversized_embedded_image_warning(image_info): + continue + if image_info[-1] != 0: + # get_image_bbox delegates Form referencers to hash-based placement. + # Unknown mask placement stays conservative without decoding it. + skipped[image_info[0]] += "; Form mask placement unverified without decoding" + unsafe = True + continue + try: + # Only top-level resources with transform=False use the metadata-only + # path; an unused resource returns an empty/infinite rectangle. + bbox = page.get_image_bbox(image_info) + unsafe = unsafe or not (bbox.is_empty or bbox.is_infinite) + except MemoryError: + raise + except Exception: + unsafe = True + return skipped, unsafe + + def is_memory_exhaustion(exc): """True for Python or MuPDF allocation failures under the address-space limit.""" return isinstance(exc, MemoryError) or bool(MEMORY_EXHAUSTION_PATTERN.search(str(exc))) @@ -801,26 +851,11 @@ def page_image_coverage_ratio(page): if page_area <= 0: return 0.0 covered = 0.0 - seen = set() - for image_info in page.get_images(full=True): - xref = image_info[0] - if xref in seen: - continue - seen.add(xref) - try: - if oversized_embedded_image_warning(image_info): - # ING-B-04: get_image_rects decodes the image. Use the declared - # placement instead, which reads the page content only. - bbox = page.get_image_bbox(image_info) - rects = [] if bbox.is_infinite or bbox.is_empty else [bbox] - else: - rects = page.get_image_rects(xref) or [] - except MemoryError: - raise - except Exception: - # One unreadable image must not abort the document; it adds no coverage. - continue - for rect in rects: + # get_image_rects hashes and decodes ALL images on the page, including inline + # images absent from get_images(). Header-only placements need neither. + for image in page.get_image_info(hashes=False, xrefs=False): + rect = fitz.Rect(image["bbox"]) + if not rect.is_infinite and not rect.is_empty: covered += abs(float(rect.width) * float(rect.height)) return min(1.0, covered / page_area) @@ -1153,7 +1188,7 @@ def _extract_table_column_ranges(table, column_count=0): def _extract_page_line_segments(page): segments = [] try: - text_dict = page.get_text("dict") or {} + text_dict = page.get_text("dict", flags=fitz.TEXTFLAGS_DICT & ~fitz.TEXT_PRESERVE_IMAGES) or {} for block in text_dict.get("blocks") or []: if block.get("type", 0) != 0: continue @@ -2213,10 +2248,17 @@ def _extract_document(document, output_dir, budget, identity): pages = [] images = [] warnings = [] + unsafe_render_pages = set() budget.set_page_count(document.page_count) for page_index, page in enumerate(document): page_number = page_index + 1 + image_infos = page.get_images(full=True) + skipped_images, unsafe_image_decode = page_image_decode_preflight(page, image_infos) + if unsafe_image_decode: + unsafe_render_pages.add(page_index) + warnings.append(f"page_render_skipped: page {page_number} contains an image or soft mask " + f"above the {MAX_EMBEDDED_IMAGE_PIXELS}-pixel ceiling; not decoded") # Some reader helpers temporarily alter the page crop; retain the source # observations before table detection or crop rendering can touch them. source_geometry = { @@ -2252,7 +2294,10 @@ def _extract_document(document, output_dir, budget, identity): # rendered as an image with a short caption) are OCR'd and merged with the existing # text layer so caption + table contents are both retained. if should_ocr_page(raw_page_text, page): - ocr_text, ocr_warning = maybe_ocr_page(page, budget, raster_observations, ocr_runtime) + if unsafe_image_decode: + ocr_text, ocr_warning = "", f"OCR skipped on page {page_number}: unsafe source image dimensions" + else: + ocr_text, ocr_warning = maybe_ocr_page(page, budget, raster_observations, ocr_runtime) if ocr_warning: warnings.append(ocr_warning) if ocr_text.strip(): @@ -2279,10 +2324,10 @@ def _extract_document(document, output_dir, budget, identity): ) page_image_count = 0 - for image_index, image_info in enumerate(page.get_images(full=True)): + for image_index, image_info in enumerate(image_infos): xref = image_info[0] budget.ensure_artifact_slot() - oversized_warning = oversized_embedded_image_warning(image_info) + oversized_warning = skipped_images.get(xref) if oversized_warning: warnings.append(oversized_warning) continue @@ -2306,6 +2351,9 @@ def _extract_document(document, output_dir, budget, identity): width = normalized["width"] height = normalized["height"] try: + if unsafe_image_decode: + # Hash-based placement would decode unsafe sibling images. + raise ValueError("unsafe sibling image dimensions") rects = page.get_image_rects(xref) or [] except MemoryError: raise @@ -2354,6 +2402,11 @@ def _extract_document(document, output_dir, budget, identity): ) page_image_count += 1 + if unsafe_image_decode: + # A bounded thumbnail/crop still decodes its original source images. + # Native text above is retained; no page render is safe here. + continue + crop_index = 1 table_rects = [] table_candidates = list(grid_table_candidates) @@ -2532,7 +2585,7 @@ def _extract_document(document, output_dir, budget, identity): # Optional cover after searchable artifacts so a near-limit budget cannot # fail indexing because the decorative thumbnail consumed the last slot. - if document.page_count > 0: + if document.page_count > 0 and 0 not in unsafe_render_pages: try: cover = save_cover_page(document[0], output_dir, budget, warnings) if cover: diff --git a/worker/python/test_extract_pdf_assets_raster_tables.py b/worker/python/test_extract_pdf_assets_raster_tables.py index 29b677415f..57a8e05f6e 100644 --- a/worker/python/test_extract_pdf_assets_raster_tables.py +++ b/worker/python/test_extract_pdf_assets_raster_tables.py @@ -555,7 +555,7 @@ def test_gated_merged_scan_keeps_exponent_cells_and_owning_rows(self): for cells, row_text in ((MERGED_ROWS[0], text[wbc:neutrophils]), (MERGED_ROWS[1], text[neutrophils:])): for lines in cells: self.assertIn(" ".join(lines), row_text) - self.assertEqual(result["extractor"]["version"], "2026-10-20.embedded-image-ceiling-v10") + self.assertEqual(result["extractor"]["version"], "2026-10-21.image-decode-preflight-v11") provenance = result["pages"][0]["metadata"]["source_provenance"] self.assertEqual(provenance["ocr"]["status"], "unverified") self.assertIsNone(provenance["ocr"]["confidence"]) diff --git a/worker/python/test_extract_pdf_assets_resource_limits.py b/worker/python/test_extract_pdf_assets_resource_limits.py index 6a26ff309c..86dd66df84 100644 --- a/worker/python/test_extract_pdf_assets_resource_limits.py +++ b/worker/python/test_extract_pdf_assets_resource_limits.py @@ -17,8 +17,10 @@ from __future__ import annotations import gc +import importlib.util import io import os +import shutil import subprocess import sys import tempfile @@ -95,6 +97,33 @@ def write_small_image_pdf(path, text=PAGE_TEXT): document.save(path) +def write_indirect_decode_pdf(path, kind, side=9_000, text=PAGE_TEXT): + """Tiny synthetic streams declaring unsafe inline or soft-mask dimensions.""" + with fitz.open() as document: + page = document.new_page(width=400, height=400) + if text: + page.insert_text((40, 300), text, fontsize=9) + if kind == "inline": + compressed = zlib.compress(b"\x00" * (120 * 90 if side == 120 else 1)) + inline = (b"q 100 0 0 100 10 10 cm BI /W %d /H %d /CS /G /BPC 8 /F /Fl ID " + % (side, 90 if side == 120 else side)) + compressed + b" EI Q" + contents = b"\n".join(document.xref_stream(xref) for xref in page.get_contents()) + xref = document.get_new_xref() + document.update_object(xref, "<<>>") + document.update_stream(xref, contents + b"\n" + inline) + page.set_contents(xref) + elif kind == "soft_mask": + pixmap = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 120, 90), True) + pixmap.clear_with(200) + xref = page.insert_image(fitz.Rect(40, 40, 160, 130), pixmap=pixmap) + mask = int(document.xref_get_key(xref, "SMask")[1].split()[0]) + document.xref_set_key(mask, "Width", str(side)) + document.xref_set_key(mask, "Height", str(side)) + else: + raise ValueError(kind) + document.save(path) + + class TemporaryDirectoryTestCase(unittest.TestCase): def setUp(self): self._directory = tempfile.TemporaryDirectory() @@ -121,6 +150,8 @@ def refuse_decode(*_args, **_kwargs): with patch.object(fitz.Document, "extract_image", side_effect=refuse_decode), patch.object( fitz.Page, "get_image_rects", side_effect=refuse_decode + ), patch.object( + fitz.Page, "get_pixmap", side_effect=refuse_decode ): result = extractor.extract(path, self.out) @@ -133,6 +164,146 @@ def refuse_decode(*_args, **_kwargs): # Coverage still counts the oversized image from its declared placement. self.assertGreater(result["pages"][0]["metadata"]["image_coverage_ratio"], 0) + def test_oversized_inline_image_is_refused_before_hashing_or_rendering(self): + path = self.path("inline.pdf") + write_indirect_decode_pdf(path, "inline", text="") + original = fitz.Page.get_image_info + + def headers_only(page, hashes=False, xrefs=False): + self.assertFalse(hashes) + self.assertFalse(xrefs) + return original(page, hashes=hashes, xrefs=xrefs) + + with patch.object(fitz.Page, "get_image_info", headers_only), patch.object( + fitz.Page, "get_image_rects", side_effect=AssertionError("must not hash images") + ), patch.object(fitz.Page, "get_pixmap", side_effect=AssertionError("must not render images")): + result = extractor.extract(path, self.out) + self.assertEqual(result["images"], []) + self.assertTrue(result["pages"][0]["needsOcr"]) + self.assertFalse(result["pages"][0]["ocrUsed"]) + self.assertGreater(result["pages"][0]["metadata"]["image_coverage_ratio"], 0) + self.assertTrue(any(warning.startswith("page_render_skipped") for warning in result["warnings"])) + + def test_small_image_with_oversized_soft_mask_is_skipped_before_any_decode(self): + path = self.path("soft-mask.pdf") + write_indirect_decode_pdf(path, "soft_mask") + with patch.object(fitz.Document, "extract_image", side_effect=AssertionError("must not extract image")), patch.object( + fitz.Page, "get_image_rects", side_effect=AssertionError("must not hash images") + ), patch.object(fitz.Page, "get_pixmap", side_effect=AssertionError("must not render images")): + result = extractor.extract(path, self.out) + self.assertIn("lithium level 0.6 to 0.8 mmol/L", result["pages"][0]["text"]) + self.assertEqual(result["images"], []) + self.assertTrue(any("soft mask" in warning and "9000x9000" in warning for warning in result["warnings"])) + + def test_form_soft_mask_is_refused_without_hash_based_placement(self): + source = self.path("form-source.pdf") + path = self.path("form-mask.pdf") + write_indirect_decode_pdf(source, "soft_mask") + with fitz.open(source) as original, fitz.open() as document: + page = document.new_page(width=400, height=400) + page.show_pdf_page(page.rect, original, 0) + self.assertTrue(all(info[-1] != 0 for info in page.get_images(full=True))) + document.save(path) + with patch.object(fitz.Page, "get_image_bbox") as bbox, patch.object( + fitz.Page, "get_image_rects" + ) as rects, patch.object(fitz.Page, "get_pixmap") as render: + result = extractor.extract(path, self.out) + bbox.assert_not_called() + rects.assert_not_called() + render.assert_not_called() + self.assertEqual(result["images"], []) + self.assertIn("lithium level 0.6 to 0.8 mmol/L", result["pages"][0]["text"]) + self.assertTrue(any("Form mask placement unverified" in warning for warning in result["warnings"])) + + def test_safe_embedded_sibling_survives_without_hashing_unsafe_inline_image(self): + source = self.path("inline-source.pdf") + path = self.path("inline-sibling.pdf") + write_indirect_decode_pdf(source, "inline") + with fitz.open(source) as document: + pixels = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 120, 90), False) + pixels.clear_with(200) + document[0].insert_image(fitz.Rect(150, 40, 270, 130), pixmap=pixels) + document.save(path) + with patch.object(fitz.Page, "get_image_rects", side_effect=AssertionError("must not hash sibling")), patch.object( + fitz.Page, "get_pixmap", side_effect=AssertionError("must not render images") + ): + result = extractor.extract(path, self.out) + embedded = [image for image in result["images"] if image["sourceKind"] == "embedded"] + self.assertEqual(len(embedded), 1) + self.assertEqual((embedded[0]["width"], embedded[0]["height"]), (120, 90)) + self.assertIsNone(embedded[0]["bbox"]) + self.assertTrue(any(warning.startswith("embedded_image_placement_unavailable") for warning in result["warnings"])) + self.assertIn("lithium level 0.6 to 0.8 mmol/L", result["pages"][0]["text"]) + + @unittest.skipUnless(shutil.which("tesseract") and importlib.util.find_spec("pytesseract"), + "Actual Tesseract and pytesseract are required") + def test_unused_oversized_resources_keep_safe_scanned_page_ocr_and_renders(self): + for kind in ("embedded", "soft_mask"): + with self.subTest(kind=kind): + source = self.path(f"dead-{kind}.pdf") + path = self.path(f"safe-scan-{kind}.pdf") + if kind == "embedded": + write_declared_image_pdf(source, 9_000, text="") + else: + write_indirect_decode_pdf(source, "soft_mask", text="") + with fitz.open() as native: + page = native.new_page(width=400, height=400) + page.insert_text((40, 100), "SYNTHETIC OFFLINE SCAN", fontsize=16) + page.insert_text((40, 140), "Keep source text visible.", fontsize=16) + scan = page.get_pixmap(dpi=150).tobytes("png") + with fitz.open(source) as document: + page = document[0] + dead_xrefs = {image[0] for image in page.get_images(full=True)} + for xref in page.get_contents(): + document.update_stream(xref, b"") + page.insert_image(page.rect, stream=scan) + document.save(path) + original_extract = fitz.Document.extract_image + original_rects = fitz.Page.get_image_rects + + def safe_extract(document, xref): + self.assertNotIn(xref, dead_xrefs) + return original_extract(document, xref) + + def safe_rects(page, xref, *args, **kwargs): + self.assertNotIn(xref, dead_xrefs) + return original_rects(page, xref, *args, **kwargs) + + with patch.object(fitz.Document, "extract_image", safe_extract), patch.object( + fitz.Page, "get_image_rects", safe_rects + ): + result = extractor.extract(path, self.path(f"out-{kind}")) + self.assertTrue(result["pages"][0]["ocrUsed"]) + self.assertFalse(result["pages"][0]["needsOcr"]) + self.assertIn("Keep source text visible.", result["pages"][0]["text"]) + self.assertTrue(any(image["sourceKind"] == "cover_page" for image in result["images"])) + self.assertTrue(any(image["sourceKind"] == "page_region" for image in result["images"])) + self.assertFalse(any(warning.startswith("page_render_skipped") for warning in result["warnings"])) + self.assertTrue(any(warning.startswith("embedded_image_skipped") for warning in result["warnings"])) + + def test_normal_inline_image_coverage_is_read_without_decoding_image_bytes(self): + path = self.path("normal-inline.pdf") + write_indirect_decode_pdf(path, "inline", side=120) + with fitz.open(path) as document: + page = document[0] + self.assertEqual(page.get_images(full=True), []) + with patch.object(fitz.Page, "get_image_rects", side_effect=AssertionError("must not hash images")): + self.assertAlmostEqual(extractor.page_image_coverage_ratio(page), 100 * 100 / (400 * 400)) + + def test_native_line_segments_request_no_image_bytes_from_dict(self): + path = self.path("dict.pdf") + write_small_image_pdf(path) + original = fitz.Page.get_text + + def text_only(page, option="text", *args, **kwargs): + if option == "dict": + self.assertFalse(kwargs.get("flags", fitz.TEXTFLAGS_DICT) & fitz.TEXT_PRESERVE_IMAGES) + return original(page, option, *args, **kwargs) + + with fitz.open(path) as document, patch.object(fitz.Page, "get_text", text_only): + segments = extractor._extract_page_line_segments(document[0]) + self.assertTrue(any("lithium level" in segment[4] for segment in segments)) + def test_an_image_mupdf_refuses_to_load_does_not_abort_the_document(self): path = self.path("refused.pdf") write_small_image_pdf(path) From 550cf46c0ee5a424e56c310f4eef77dc681778a0 Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 07:19:40 +0000 Subject: [PATCH 03/16] test(ingestion): type optional integrity telemetry cases --- tests/ingestion-autopilot.test.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/ingestion-autopilot.test.ts b/tests/ingestion-autopilot.test.ts index 3d529303bc..1d3a36d764 100644 --- a/tests/ingestion-autopilot.test.ts +++ b/tests/ingestion-autopilot.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it, vi } from "vitest"; import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; -import { assessIngestionHealth, describeWorkerState, workerPausedFromEnv } from "../scripts/ingestion-autopilot"; +import { + assessIngestionHealth, + describeWorkerState, + workerPausedFromEnv, + type IngestionHealth, +} from "../scripts/ingestion-autopilot"; const autopilotSource = readFileSync(new URL("../scripts/ingestion-autopilot.ts", import.meta.url), "utf8"); const healthSource = readFileSync(new URL("../scripts/reindex-health.ts", import.meta.url), "utf8"); @@ -49,7 +54,12 @@ describe("assessIngestionHealth", () => { }); it("keeps missing old-build and valid zero integrity counts compatible", () => { - for (const counts of [undefined, {}, { jobs_stranded_final_attempt: 0, documents_indexed_without_chunks: 0 }]) { + const cases: IngestionHealth["counts"][] = [ + undefined, + {}, + { jobs_stranded_final_attempt: 0, documents_indexed_without_chunks: 0 }, + ]; + for (const counts of cases) { expect(assessIngestionHealth({ ok: true, counts }, { now: NOW })).toMatchObject({ stuck: false, recoveryNeeded: false, From add5da61fd7483fa0f365dab4ec350aac62ac0b0 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 15:34:30 +0800 Subject: [PATCH 04/16] fix(extraction): guard malformed soft-mask Width/Height in decode preflight A non-integer or missing soft-mask dimension now skips that image conservatively (not decoded) instead of aborting the whole PDF. Addresses gitar-bot review on #3716. --- worker/python/extract_pdf_assets.py | 13 ++++-- ...test_extract_pdf_assets_resource_limits.py | 40 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 72ba487518..bdf6ad57f7 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -194,9 +194,16 @@ def page_image_decode_preflight(page, image_infos): xref, smask = image_info[:2] warning = oversized_embedded_image_warning(image_info) if not warning and smask: - width = int(page.parent.xref_get_key(smask, "Width")[1]) - height = int(page.parent.xref_get_key(smask, "Height")[1]) - if width * height > MAX_EMBEDDED_IMAGE_PIXELS: + try: + width = int(page.parent.xref_get_key(smask, "Width")[1]) + height = int(page.parent.xref_get_key(smask, "Height")[1]) + except (IndexError, TypeError, ValueError): + # A malformed mask header must not abort the whole PDF: skip + # this image conservatively without decoding it. + width = height = None + warning = (f"embedded_image_skipped: xref {xref} soft mask {smask} has an " + "unreadable Width/Height; not decoded") + if width is not None and width * height > MAX_EMBEDDED_IMAGE_PIXELS: warning = (f"embedded_image_skipped: xref {xref} soft mask {smask} declares " f"{width}x{height}, above the {MAX_EMBEDDED_IMAGE_PIXELS}-pixel " "embedded-image ceiling; not decoded") diff --git a/worker/python/test_extract_pdf_assets_resource_limits.py b/worker/python/test_extract_pdf_assets_resource_limits.py index 86dd66df84..16d51493ae 100644 --- a/worker/python/test_extract_pdf_assets_resource_limits.py +++ b/worker/python/test_extract_pdf_assets_resource_limits.py @@ -337,6 +337,46 @@ def test_a_normal_embedded_image_is_still_extracted_with_its_placement(self): self.assertFalse(any(warning.startswith("embedded_image_skipped") for warning in result["warnings"])) +class MalformedSoftMaskPreflightTests(unittest.TestCase): + """A bad soft-mask Width/Height skips that image instead of aborting the PDF.""" + + def _page(self, width_value): + doc = types.SimpleNamespace( + xref_get_key=lambda xref, key: ("int", width_value) if key == "Width" else ("int", "10")) + bbox = types.SimpleNamespace(is_empty=True, is_infinite=False) + return types.SimpleNamespace( + parent=doc, + get_image_info=lambda hashes=False, xrefs=False: [], + get_image_bbox=lambda info: bbox, + ) + + def test_unparsable_mask_dimension_is_skipped_not_raised(self): + for bad in ("null", "", "1.5e3x"): + with self.subTest(bad=bad): + info = (7, 9, 10, 10, 8, "DeviceRGB", "", "Im1", "FlateDecode", 0) + skipped, unsafe = extractor.page_image_decode_preflight(self._page(bad), [info]) + self.assertIn(7, skipped) + self.assertIn("unreadable Width/Height", skipped[7]) + self.assertFalse(unsafe) + + def test_missing_mask_key_tuple_is_skipped_not_raised(self): + doc = types.SimpleNamespace(xref_get_key=lambda xref, key: ()) + page = types.SimpleNamespace( + parent=doc, + get_image_info=lambda hashes=False, xrefs=False: [], + get_image_bbox=lambda info: types.SimpleNamespace(is_empty=True, is_infinite=False), + ) + info = (7, 9, 10, 10, 8, "DeviceRGB", "", "Im1", "FlateDecode", 0) + skipped, _ = extractor.page_image_decode_preflight(page, [info]) + self.assertIn("unreadable Width/Height", skipped[7]) + + def test_valid_small_mask_is_not_skipped(self): + info = (7, 9, 10, 10, 8, "DeviceRGB", "", "Im1", "FlateDecode", 0) + skipped, unsafe = extractor.page_image_decode_preflight(self._page("10"), [info]) + self.assertEqual(skipped, {}) + self.assertFalse(unsafe) + + class ExtractorMemoryLimitTests(unittest.TestCase): def fake_resource(self, soft, hard): calls = [] From 049b1516b7719c808713318e96424d45c47bd2f6 Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 07:50:28 +0000 Subject: [PATCH 05/16] fix(ingestion): retain bounded atomic table rows and budget embedding batches --- docs/organisation/systems/source-intake.json | 1 + docs/reindex-runbook.md | 15 +- src/lib/chunking.ts | 128 +++++++++++++---- src/lib/embedding-input-budget.ts | 49 +++++++ src/lib/openai.ts | 6 +- tests/chunker-version-ratchet.test.ts | 38 +++++ tests/chunking.test.ts | 138 +++++++++++++++++-- tests/embed-texts-integrity.test.ts | 47 +++++++ tests/embedding-input-budget.test.ts | 45 ++++++ 9 files changed, 420 insertions(+), 47 deletions(-) create mode 100644 src/lib/embedding-input-budget.ts create mode 100644 tests/embedding-input-budget.test.ts diff --git a/docs/organisation/systems/source-intake.json b/docs/organisation/systems/source-intake.json index 9dee1b47f9..5a1659341a 100644 --- a/docs/organisation/systems/source-intake.json +++ b/docs/organisation/systems/source-intake.json @@ -92,6 +92,7 @@ "src/lib/document-organization.ts", "src/lib/document-tags.ts", "src/lib/embedding-dimensions.ts", + "src/lib/embedding-input-budget.ts", "src/lib/enrichment-repair.ts", "src/lib/extractors/**", "src/lib/image-filtering.ts", diff --git a/docs/reindex-runbook.md b/docs/reindex-runbook.md index 1ddf66aee2..a3edea1fed 100644 --- a/docs/reindex-runbook.md +++ b/docs/reindex-runbook.md @@ -78,11 +78,16 @@ older probes that omit these fields remain compatible. These signals alone never recovery, including with `--apply`. If separate recoverable queue work also exists, bounded recovery may run, but the integrity alert remains until a subsequent probe confirms resolution. -`TABLE_ROW_EXCEEDS_CHUNK_BUDGET` is a permanent ingestion refusal: the title, column -header, one intact row and attached note cannot fit the configured character target. -The chunker returns no partial chunk array and never splits cells or truncates the -evidence. Review the extracted table and source structure before retrying; the refusal -does not establish OCR accuracy. An atomic reindex preserves the existing index on failure. +The configured character target controls ordinary table groups. A larger indivisible +row is supported as a standalone chunk with its full title, header and adjacent note, +up to 8,191 UTF-8 bytes for known OpenAI embedding models. This is a conservative +byte-BPE token upper bound, not an exact token count. The chunk records this overflow +in metadata, and embedding requests are batched by both input count and total bytes. +`TABLE_ROW_EXCEEDS_CHUNK_BUDGET` remains a permanent refusal above that hard ceiling +or for an unknown model's target overflow. The chunker returns no partial chunk array +and never splits cells or truncates evidence. Review the source structure before +retrying; the refusal does not establish OCR accuracy. Atomic reindex failure preserves +the existing index. ## Do not do this diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index ac274ea8c4..9ef85bdb8a 100644 --- a/src/lib/chunking.ts +++ b/src/lib/chunking.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import { env } from "@/lib/env"; +import { EMBEDDING_MAX_INPUT_UTF8_BYTES, hasKnownEmbeddingBudget } from "@/lib/embedding-input-budget"; import { sourceSpanForText } from "@/lib/source-spans"; import { normalizeExtractedGlyphs, stripClassificationBanner } from "@/lib/source-text-sanitizer"; import type { ChunkInput, DocumentChunk } from "@/lib/types"; @@ -8,8 +9,12 @@ import type { ChunkInput, DocumentChunk } from "@/lib/types"; // re-index can tell which chunker produced a row, and so bumping the strategy deliberately // invalidates stale chunks. Bump this when chunk boundaries change (e.g. when cross-page // document-mode chunking is enabled by default). -export const CHUNKER_VERSION = "1.3.1-page"; -export const DOCUMENT_CHUNKER_VERSION = "1.3.1-document"; +export const CHUNKER_VERSION = "1.4.0-page"; +export const DOCUMENT_CHUNKER_VERSION = "1.4.0-document"; + +// Bytes are a conservative token upper bound, not an exact tokenizer count. +export const MAX_ATOMIC_TABLE_UTF8_BYTES = EMBEDDING_MAX_INPUT_UTF8_BYTES; +type TableOverflowObserver = (content: string) => void; const sentenceBoundary = /(?<=[.!?])\s+/; const sentenceBoundaryGlobal = new RegExp(sentenceBoundary.source, "g"); @@ -385,15 +390,29 @@ function adaptiveChunkProfile(text: string, sectionPath: string[]) { } export function chunkTextWithOverlap(text: string, chunkSize = env.CHUNK_SIZE, overlap = env.CHUNK_OVERLAP) { + return chunkTextWithOverlapInternal(text, chunkSize, overlap); +} + +function chunkTextWithOverlapInternal( + text: string, + chunkSize: number, + overlap: number, + onTableOverflow?: TableOverflowObserver, +) { const clean = removePageNoise(normalizeExtractedGlyphs(text)) .replace(trailingHorizontalWhitespaceBeforeNewlinePattern, "\n") .replace(horizontalWhitespaceRunPattern, " ") .trim(); if (!clean) return []; - if (clean.length <= chunkSize) return [clean]; + if ( + clean.length <= chunkSize && + (Buffer.byteLength(clean, "utf8") <= MAX_ATOMIC_TABLE_UTF8_BYTES || + !clean.split(lineBreakPattern).some((line) => tableRowPattern.test(line))) + ) + return [clean]; if (isTableBlock(clean)) { - return chunkTableBlock(clean, chunkSize); + return chunkTableBlock(clean, chunkSize, "", "", onTableOverflow); } const paragraphs = clean @@ -410,16 +429,20 @@ export function chunkTextWithOverlap(text: string, chunkSize = env.CHUNK_SIZE, o chunks.push(current.trim()); current = ""; } - chunks.push(...chunkTableBlock(paragraph, chunkSize)); + chunks.push(...chunkTableBlock(paragraph, chunkSize, "", "", onTableOverflow)); continue; } - if (paragraph.length > chunkSize) { + if ( + paragraph.length > chunkSize || + (Buffer.byteLength(paragraph, "utf8") > MAX_ATOMIC_TABLE_UTF8_BYTES && + paragraph.split(lineBreakPattern).some((line) => tableRowPattern.test(line))) + ) { if (current) { chunks.push(current.trim()); current = ""; } - chunks.push(...chunkOversizedParagraph(paragraph, chunkSize, overlap)); + chunks.push(...chunkOversizedParagraph(paragraph, chunkSize, overlap, onTableOverflow)); continue; } @@ -440,7 +463,7 @@ export function chunkTextWithOverlap(text: string, chunkSize = env.CHUNK_SIZE, o return chunks.filter(Boolean); } - return chunkOversizedParagraph(clean, chunkSize, overlap); + return chunkOversizedParagraph(clean, chunkSize, overlap, onTableOverflow); } // Audit 2026-10-20 ING-B-03: the PDF extractor joins a table's title, its @@ -448,7 +471,12 @@ export function chunkTextWithOverlap(text: string, chunkSize = env.CHUNK_SIZE, o // paragraph that isTableBlock rejects. Split such a paragraph into its prose // and table runs; each table run is chunked by rows with its header (and a // "Table ..." title line directly above it) repeated on every chunk. -function chunkOversizedParagraph(paragraph: string, chunkSize: number, overlap: number) { +function chunkOversizedParagraph( + paragraph: string, + chunkSize: number, + overlap: number, + onTableOverflow?: TableOverflowObserver, +) { const lines = paragraph.split(lineBreakPattern); const segments: Array<{ table: boolean; lines: string[] }> = []; for (const line of lines) { @@ -499,7 +527,7 @@ function chunkOversizedParagraph(paragraph: string, chunkSize: number, overlap: ) .join("\n") : ""; - chunks.push(...chunkTableBlock(segment.lines.join("\n"), chunkSize, caption, footnote)); + chunks.push(...chunkTableBlock(segment.lines.join("\n"), chunkSize, caption, footnote, onTableOverflow)); }); return chunks; } @@ -550,7 +578,7 @@ function isTableBlock(block: string) { return lines.every((line) => tableRowPattern.test(line)); } -// Refuse unrepresentable table evidence rather than truncate or separate cells +// Refuse table evidence beyond the hard byte ceiling rather than truncate or separate cells // and clinical qualifiers. Keep diagnostics content-free, including the message. export class OversizedTableRowError extends Error { readonly code = "TABLE_ROW_EXCEEDS_CHUNK_BUDGET"; @@ -561,38 +589,60 @@ export class OversizedTableRowError extends Error { } } -function chunkTableBlock(block: string, chunkSize: number, caption = "", footnote = "") { +function chunkTableBlock( + block: string, + chunkSize: number, + caption = "", + footnote = "", + onTableOverflow?: TableOverflowObserver, +) { const lines = block.split(lineBreakPattern).filter((line) => line.trim().length > 0); const captioned = [caption, block.trim(), footnote].filter(Boolean).join("\n"); - if (captioned.length <= chunkSize) return [captioned]; - // Reserve the same source qualification on every row group so retrieval cannot - // separate a dose/threshold from its immediately following warning. - const rowBudget = chunkSize - (footnote ? footnote.length + 1 : 0); + const fitsByteCeiling = (text: string) => Buffer.byteLength(text, "utf8") <= MAX_ATOMIC_TABLE_UTF8_BYTES; + if (captioned.length <= chunkSize && fitsByteCeiling(captioned)) return [captioned]; const withFootnote = (text: string) => (footnote ? `${text}\n${footnote}` : text); - const separatorIndex = lines.findIndex((line) => tableSeparatorRowPattern.test(line) && line.includes("-")); const headerLines = separatorIndex > 0 ? lines.slice(0, separatorIndex + 1) : lines.slice(0, 1); const bodyLines = lines.slice(headerLines.length); const header = [caption, ...headerLines].filter(Boolean).join("\n"); - // Cells and their qualifiers are indivisible evidence. Refuse the whole build - // before emitting any row group when that evidence cannot fit the character target. - if (header.length > rowBudget || bodyLines.some((row) => header.length + 1 + row.length > rowBudget)) { + const atomicFits = (text: string) => + fitsByteCeiling(text) && (text.length <= chunkSize || hasKnownEmbeddingBudget(env.OPENAI_EMBEDDING_MODEL)); + // Validate the entire table before emitting any group. A target overflow may + // preserve one indivisible row, but never an unbounded row or a guessed model. + if ( + bodyLines.length + ? bodyLines.some((row) => !atomicFits(withFootnote(`${header}\n${row}`))) + : !atomicFits(withFootnote(header)) + ) { throw new OversizedTableRowError(); } const chunks: string[] = []; - let current = header; + const emit = (text: string) => { + chunks.push(text); + if (text.length > chunkSize) onTableOverflow?.(text); + }; + let current = ""; for (const row of bodyLines) { - const candidate = `${current}\n${row}`; - if (candidate.length > rowBudget && current !== header) { - chunks.push(withFootnote(current.trim())); + const atomic = withFootnote(`${header}\n${row}`); + if (atomic.length > chunkSize) { + if (current) emit(withFootnote(current)); + current = ""; + emit(atomic); + continue; + } + const candidate = current ? `${current}\n${row}` : `${header}\n${row}`; + const qualified = withFootnote(candidate); + if (current && (qualified.length > chunkSize || !fitsByteCeiling(qualified))) { + emit(withFootnote(current)); current = `${header}\n${row}`; } else { current = candidate; } } - if (current.trim()) chunks.push(withFootnote(current.trim())); - return chunks.length > 0 ? chunks : [captioned]; + if (current) emit(withFootnote(current)); + if (chunks.length === 0) emit(withFootnote(header)); + return chunks; } const clinicalProhibitionPattern = @@ -968,6 +1018,7 @@ function emitChunk(args: { chunkProfile: ChunkProfile; pageChunkIndex: number; chunkerVersion: string; + tableOverflow: boolean; }) { const { content, sectionPath } = args; const contentLookup = normalizeLookupText(content); @@ -1034,6 +1085,16 @@ function emitChunk(args: { chunk_profile: args.chunkProfile.profile, adaptive_chunk_size: args.chunkProfile.chunkSize, adaptive_chunk_overlap: args.chunkProfile.overlap, + ...(args.tableOverflow + ? { + table_row_overflow: { + reason: "indivisible_table_evidence", + utf8_bytes: Buffer.byteLength(content, "utf8"), + max_utf8_bytes: MAX_ATOMIC_TABLE_UTF8_BYTES, + token_upper_bound: "utf8_bytes", + }, + } + : {}), page_start: args.pageStart, page_end: args.pageEnd, source_spans: args.spanPages.map((span) => { @@ -1078,7 +1139,10 @@ function buildPageModeChunks(inputs: ChunkInput[]) { const pageText = [cleanedPageText, imageContext].filter(Boolean).join("\n\n"); const pageLookupText = normalizeLookupText(normalizedPageText); const chunkProfile = adaptiveChunkProfile(cleanedPageText, sectionPath); - const pageChunks = chunkTextWithOverlap(pageText, chunkProfile.chunkSize, chunkProfile.overlap); + const tableOverflows = new Set(); + const pageChunks = chunkTextWithOverlapInternal(pageText, chunkProfile.chunkSize, chunkProfile.overlap, (content) => + tableOverflows.add(content), + ); pageChunks.forEach((content, pageChunkIndex) => { const fingerprint = dedupeChunkFingerprint(content); @@ -1100,6 +1164,7 @@ function buildPageModeChunks(inputs: ChunkInput[]) { chunkProfile, pageChunkIndex, chunkerVersion: CHUNKER_VERSION, + tableOverflow: tableOverflows.has(content), }); }); } @@ -1193,7 +1258,13 @@ function buildDocumentModeChunks(inputs: ChunkInput[]) { .join("\n\n"); const chunkProfile = adaptiveChunkProfile(combinedCleanText, section.sectionPath); const pageLookupText = normalizeLookupText(section.pages.map((page) => page.normalizedText).join(" ")); - const sectionChunks = chunkTextWithOverlap(combinedText, chunkProfile.chunkSize, chunkProfile.overlap); + const tableOverflows = new Set(); + const sectionChunks = chunkTextWithOverlapInternal( + combinedText, + chunkProfile.chunkSize, + chunkProfile.overlap, + (content) => tableOverflows.add(content), + ); sectionChunks.forEach((content, pageChunkIndex) => { const contentWords = new Set( @@ -1229,6 +1300,7 @@ function buildDocumentModeChunks(inputs: ChunkInput[]) { chunkProfile, pageChunkIndex, chunkerVersion: DOCUMENT_CHUNKER_VERSION, + tableOverflow: tableOverflows.has(content), }); }); } diff --git a/src/lib/embedding-input-budget.ts b/src/lib/embedding-input-budget.ts new file mode 100644 index 0000000000..e1c74c706e --- /dev/null +++ b/src/lib/embedding-input-budget.ts @@ -0,0 +1,49 @@ +// The installed OpenAI SDK's EmbeddingCreateParams documents 8,192 tokens per +// input and 300,000 per request. These models use byte-level BPE: UTF-8 bytes +// provide a conservative token upper bound, not an exact token count. +export const EMBEDDING_MAX_INPUT_UTF8_BYTES = 8_191; +export const EMBEDDING_MAX_BATCH_UTF8_BYTES = 300_000; + +const knownEmbeddingBudgetModels = new Set([ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002", +]); + +export function hasKnownEmbeddingBudget(model: string): boolean { + return knownEmbeddingBudgetModels.has(model); +} + +export class EmbeddingRequestBudgetError extends Error { + readonly code = "EMBEDDING_INPUT_EXCEEDS_REQUEST_BUDGET"; + + constructor() { + super("EMBEDDING_INPUT_EXCEEDS_REQUEST_BUDGET"); + this.name = "EmbeddingRequestBudgetError"; + } +} + +/** Preserve each input verbatim while bounding both request count and bytes. */ +export function batchEmbeddingInputs(texts: string[], maxInputs: number, model: string): string[][] { + if (!Number.isInteger(maxInputs) || maxInputs <= 0 || maxInputs > 2_048) { + throw new Error("Embedding batch size must be a positive integer at most 2048."); + } + const bounded = hasKnownEmbeddingBudget(model); + const batches: string[][] = []; + let current: string[] = []; + let currentBytes = 0; + for (const text of texts) { + const bytes = bounded ? Buffer.byteLength(text, "utf8") : 0; + // Validate the entire list before any caller can dispatch a partial build. + if (bytes > EMBEDDING_MAX_BATCH_UTF8_BYTES) throw new EmbeddingRequestBudgetError(); + if (current.length && (current.length >= maxInputs || currentBytes + bytes > EMBEDDING_MAX_BATCH_UTF8_BYTES)) { + batches.push(current); + current = []; + currentBytes = 0; + } + current.push(text); + currentBytes += bytes; + } + if (current.length) batches.push(current); + return batches; +} diff --git a/src/lib/openai.ts b/src/lib/openai.ts index 66986d36db..30a56b1ade 100644 --- a/src/lib/openai.ts +++ b/src/lib/openai.ts @@ -8,6 +8,7 @@ import { readBoundedLruEntry, writeBoundedLruEntry } from "@/lib/bounded-ttl-cac import { instrumentOpenAIClientForAgentMonitoring } from "@/lib/observability/agent-monitoring"; import { assessClinicalImageUse } from "@/lib/image-filtering"; import { PublicApiError } from "@/lib/http"; +import { batchEmbeddingInputs } from "@/lib/embedding-input-budget"; import { deterministicStructuredVisualProfile, normalizeStructuredVisualProfile, @@ -709,6 +710,10 @@ export async function embedTexts(texts: string[], options?: IngestionOpenAIOptio return nextIndex; }); + // Plan every request before creating a client. Intact table rows can exceed + // the chunk target; count-only batching could put their combined byte-BPE + // upper bound beyond the provider's per-request token budget. + const batches = batchEmbeddingInputs(uniqueTexts, env.OPENAI_EMBEDDING_BATCH_SIZE, env.OPENAI_EMBEDDING_MODEL); try { const client = createOpenAIClient(); // IDX-C3: a single embeddings request is capped at 2048 inputs / ~300k tokens, so a @@ -717,7 +722,6 @@ export async function embedTexts(texts: string[], options?: IngestionOpenAIOptio // uses the GLOBAL index (batchStart + item.index) so a chunk is never stored with the // embedding of an unrelated text, even across batch boundaries (extends IDX-C1). const byIndex = new Array(uniqueTexts.length); - const batches = chunkIntoBatches(uniqueTexts, env.OPENAI_EMBEDDING_BATCH_SIZE); // IDX-C3: a single embeddings request is capped at 2048 inputs / ~300k tokens, so a // full-corpus re-embed must be split. To maximize throughput while respecting rate diff --git a/tests/chunker-version-ratchet.test.ts b/tests/chunker-version-ratchet.test.ts index 6df0723206..26277b6a04 100644 --- a/tests/chunker-version-ratchet.test.ts +++ b/tests/chunker-version-ratchet.test.ts @@ -33,6 +33,7 @@ const recorded: Record = { "1.2.0-page|1.2.0-document": "e0eb0b337d010f3c4f078ef2df970b3b91d0d420dd2916b0f0f98a83600957e8", "1.3.0-page|1.3.0-document": "e0eb0b337d010f3c4f078ef2df970b3b91d0d420dd2916b0f0f98a83600957e8", "1.3.1-page|1.3.1-document": "e0eb0b337d010f3c4f078ef2df970b3b91d0d420dd2916b0f0f98a83600957e8", + "1.4.0-page|1.4.0-document": "e0eb0b337d010f3c4f078ef2df970b3b91d0d420dd2916b0f0f98a83600957e8", }; function fingerprint() { @@ -48,7 +49,44 @@ function fingerprint() { return hash.digest("hex"); } +// Keep the original corpus/history above intact. This additional ratchet covers +// the newly representable atomic row, repeated context, and neighboring rows. +const overflowCorpus = [ + [ + "Table 4. Synthetic monitoring", + "| Group | Action |", + "| --- | --- |", + "| GROUP-B | Repeat the measurement. |", + `| GROUP-A | Do not change treatment unless ${"the original condition remains applicable ".repeat(50).trim()}. |`, + "| GROUP-C | Review the result. |", + "Note: Do not change treatment", + "unless the original condition applies.", + ].join("\n"), +]; +const overflowRecorded: Record = { + "1.4.0-page|1.4.0-document": "bc6abc0a6a25d4491200d4b82646de78b9b92dc6811ed0d9e3ed4d143291b32e", +}; + +function overflowFingerprint() { + const hash = createHash("sha256"); + for (const text of overflowCorpus) { + for (const [size, overlap] of [ + [400, 80], + [1200, 200], + ] as const) { + hash.update(JSON.stringify(chunkTextWithOverlap(text, size, overlap))); + } + } + return hash.digest("hex"); +} + describe("chunker version ratchet", () => { + it("records the atomic table overflow fingerprint for the current chunker version", () => { + const key = `${CHUNKER_VERSION}|${DOCUMENT_CHUNKER_VERSION}`; + expect(overflowRecorded[key], `record the atomic overflow fingerprint for ${key}`).toBeDefined(); + expect(overflowFingerprint()).toBe(overflowRecorded[key]); + }); + it("records a fingerprint for the current chunker version", () => { const key = `${CHUNKER_VERSION}|${DOCUMENT_CHUNKER_VERSION}`; expect(recorded[key], `bump the chunker version and record a fingerprint for ${key}`).toBeDefined(); diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index 200c660342..fc24f0f129 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -3,6 +3,7 @@ import { CHUNKER_VERSION, DOCUMENT_CHUNKER_VERSION, OversizedTableRowError, + MAX_ATOMIC_TABLE_UTF8_BYTES, buildChunks, buildImageTag, chunkContentKey, @@ -832,17 +833,24 @@ describe("table-aware chunking with a title and footnote (ING-B-03)", () => { // Table cells stay atomic: a character target cannot justify splitting a // clinical condition from the assertion or another cell it qualifies. -describe("oversized table row refusal", () => { +describe("bounded atomic table row overflow", () => { const header = "| Group | Action |\n| --- | --- |"; - const row = `| GROUP-A | Do not change treatment unless ${"the original condition remains applicable ".repeat(20).trim()}. |`; + const row = `| GROUP-A | Do not change treatment unless ${"the original condition remains applicable ".repeat(50).trim()}. |`; + const hardOversizedRow = `| GROUP-A | ${"original condition ".repeat(500)} |`; afterEach(() => { vi.unstubAllEnvs(); vi.resetModules(); }); - it("refuses an indivisible long row with a content-free typed error", () => { + it("preserves an indivisible long row above the character target", () => { const text = `${header}\n${row}`; + expect(text.length).toBeGreaterThan(400); + expect(chunkTextWithOverlap(text, 400, 40)).toEqual([text]); + }); + + it("refuses an indivisible row beyond the hard ceiling with a content-free typed error", () => { + const text = `${header}\n${hardOversizedRow}`; expect(() => chunkTextWithOverlap(text, 400, 40)).toThrow(OversizedTableRowError); try { chunkTextWithOverlap(text, 400, 40); @@ -857,38 +865,142 @@ describe("oversized table row refusal", () => { } }); - it("accepts a row exactly at the character limit including title and footnote", () => { + it("keeps target-sized groups while emitting an overflowing row alone with its complete context", () => { + const title = "Table 4. Synthetic monitoring"; + const note = "Note: Do not change treatment\nunless the original condition applies."; + const before = "| GROUP-B | Repeat the measurement. |"; + const after = "| GROUP-C | Review the result. |"; + const text = [title, header, before, row, after, note].join("\n"); + expect(chunkTextWithOverlap(text, 250, 0)).toEqual( + [before, row, after].map((line) => [title, header, line, note].join("\n")), + ); + }); + + it("accepts a row exactly at the character target including title and footnote", () => { const title = "Table 4. Synthetic monitoring"; const note = "Note: Preserve the original clinical qualification."; const rows = ["| GROUP-A | Repeat the measurement. |", "| GROUP-B | Review the result. |"]; const atomic = [title, header, rows[0], note].join("\n"); const text = [title, header, ...rows, note].join("\n"); - const chunks = chunkTextWithOverlap(text, atomic.length, 0); - expect(chunks).toEqual(rows.map((line) => [title, header, line, note].join("\n"))); - for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(atomic.length); - expect(() => chunkTextWithOverlap(text, atomic.length - 1, 0)).toThrow(OversizedTableRowError); + const expected = rows.map((line) => [title, header, line, note].join("\n")); + expect(chunkTextWithOverlap(text, atomic.length, 0)).toEqual(expected); + expect(chunkTextWithOverlap(text, atomic.length - 1, 0)).toEqual(expected); }); - it("refuses when a header alone cannot fit the character budget", () => { + it("accepts exactly 8191 UTF-8 bytes including caption, header, row and wrapped note", () => { + const title = "Table 4. Synthetic monitoring"; + const note = "Note: Preserve this qualification\nunless the original condition changes."; + const prefix = [title, header, "| GROUP-A | "].join("\n"); + const suffix = ` |\n${note}`; + const text = prefix + "a".repeat(MAX_ATOMIC_TABLE_UTF8_BYTES - Buffer.byteLength(prefix + suffix)) + suffix; + expect(Buffer.byteLength(text)).toBe(8191); + expect(chunkTextWithOverlap(text, 400, 0)).toEqual([text]); + expect(() => chunkTextWithOverlap(text.replace("| GROUP-A | ", "| GROUP-A | a"), 400, 0)).toThrow( + OversizedTableRowError, + ); + }); + + it("uses UTF-8 bytes for multibyte evidence rather than character length", () => { + const prefix = `${header}\n| GROUP-A | `; + const suffix = " |"; + const remaining = MAX_ATOMIC_TABLE_UTF8_BYTES - Buffer.byteLength(prefix + suffix); + const text = prefix + "漢".repeat(Math.floor(remaining / 3)) + "a".repeat(remaining % 3) + suffix; + expect(text.length).toBeLessThan(MAX_ATOMIC_TABLE_UTF8_BYTES); + expect(Buffer.byteLength(text)).toBe(MAX_ATOMIC_TABLE_UTF8_BYTES); + expect(chunkTextWithOverlap(text, 400, 0)).toEqual([text]); + expect(() => chunkTextWithOverlap(text.replace("| GROUP-A | ", "| GROUP-A | 漢"), 400, 0)).toThrow( + OversizedTableRowError, + ); + }); + + it("keeps a large header intact within the hard ceiling and refuses one beyond it", () => { const text = `| Group | ${"Original column qualification ".repeat(20).trim()} |\n| --- | --- |`; - expect(() => chunkTextWithOverlap(text, 400, 0)).toThrow(OversizedTableRowError); + expect(chunkTextWithOverlap(text, 400, 0)).toEqual([text]); + const giant = text.replace("Original column qualification", "q".repeat(MAX_ATOMIC_TABLE_UTF8_BYTES)); + expect(() => chunkTextWithOverlap(giant, 400, 0)).toThrow(OversizedTableRowError); }); - it("refuses when the attached qualification alone exhausts the budget", () => { + it("preserves a large attached qualification and refuses one beyond the hard ceiling", () => { const note = `Note: ${"Preserve the original qualification. ".repeat(20).trim()}`; const text = [header, "| GROUP-A | Repeat the measurement. |", note].join("\n"); + expect(chunkTextWithOverlap(text, 400, 0)).toEqual([text]); + expect(() => chunkTextWithOverlap(`${text} ${"q".repeat(MAX_ATOMIC_TABLE_UTF8_BYTES)}`, 400, 0)).toThrow( + OversizedTableRowError, + ); + }); + + it("enforces the byte ceiling even when a configured character target is larger", () => { + expect(() => chunkTextWithOverlap(`${header}\n${hardOversizedRow}`, 20000, 0)).toThrow(OversizedTableRowError); + }); + + it("checks all rows before returning any earlier table group", () => { + const text = [header, "| GROUP-B | Repeat measurement. |", hardOversizedRow].join("\n"); expect(() => chunkTextWithOverlap(text, 400, 0)).toThrow(OversizedTableRowError); }); + it("refuses target overflow for a model with an unknown tokenizer budget", async () => { + vi.stubEnv("OPENAI_EMBEDDING_MODEL", "unknown-embedding-model"); + const mod = await import("../src/lib/chunking"); + expect(() => mod.chunkTextWithOverlap(`${header}\n${row}`, 400, 0)).toThrow(mod.OversizedTableRowError); + expect(mod.chunkTextWithOverlap(`${header}\n| GROUP-A | Review. |`, 400, 0)).toEqual([ + `${header}\n| GROUP-A | Review. |`, + ]); + }); + + it.each(["text-embedding-3-small", "text-embedding-3-large", "text-embedding-ada-002"])( + "permits bounded overflow for %s", + async (model) => { + vi.stubEnv("OPENAI_EMBEDDING_MODEL", model); + const mod = await import("../src/lib/chunking"); + expect(mod.chunkTextWithOverlap(`${header}\n${row}`, 400, 0)).toEqual([`${header}\n${row}`]); + }, + ); + + it.each(["page", "document"])( + "marks bounded atomic overflow and keeps image context separate in %s mode", + async (strategy) => { + vi.stubEnv("CHUNK_STRATEGY", strategy); + const mod = await import("../src/lib/chunking"); + const note = "Note: Do not change treatment\nunless the original condition applies."; + const text = ["Table 4. Monitoring", header, row, note].join("\n"); + const chunks = mod.buildChunks([ + { + documentId: "synthetic-document", + pageNumber: 2, + pageText: text, + images: [ + { id: "synthetic-image", pageNumber: 2, caption: "Synthetic chart context.", sourceKind: "embedded" }, + ], + }, + ]); + const atomic = chunks.find((chunk) => chunk.content.includes("GROUP-A")); + expect(atomic?.content).toBe(text); + expect(atomic?.metadata.table_row_overflow).toEqual({ + reason: "indivisible_table_evidence", + utf8_bytes: Buffer.byteLength(text), + max_utf8_bytes: 8191, + token_upper_bound: "utf8_bytes", + }); + expect(atomic?.content).not.toContain("IMAGE_DATA_START"); + expect(chunks.some((chunk) => chunk.content.includes("IMAGE_DATA_START"))).toBe(true); + for (const chunk of chunks.filter((chunk) => chunk !== atomic)) { + expect(chunk.metadata.table_row_overflow).toBeUndefined(); + } + }, + ); + it.each(["page", "document"])("propagates refusal without returning earlier chunks in %s mode", async (strategy) => { - vi.resetModules(); vi.stubEnv("CHUNK_STRATEGY", strategy); vi.stubEnv("CHUNK_SIZE", "400"); vi.stubEnv("CHUNK_OVERLAP", "40"); const mod = await import("../src/lib/chunking"); const inputs = [ { documentId: "synthetic-document", pageNumber: 1, pageText: "Earlier Section\n\nEarlier complete guidance." }, - { documentId: "synthetic-document", pageNumber: 2, pageText: `Table 4. Monitoring\n${header}\n${row}` }, + { + documentId: "synthetic-document", + pageNumber: 2, + pageText: `Table 4. Monitoring\n${header}\n${hardOversizedRow}`, + }, ]; expect(() => mod.buildChunks(inputs)).toThrow(mod.OversizedTableRowError); }); diff --git a/tests/embed-texts-integrity.test.ts b/tests/embed-texts-integrity.test.ts index 20005886b6..0254a517ed 100644 --- a/tests/embed-texts-integrity.test.ts +++ b/tests/embed-texts-integrity.test.ts @@ -136,6 +136,53 @@ describe("embedTexts integrity (IDX-C1, IDX-C2)", () => { }); describe("embedTexts batching (IDX-C3)", () => { + it("splits larger atomic inputs by request bytes and preserves reversed-response and duplicate mapping", async () => { + stubEmbeddingEnv(2); + vi.stubEnv("OPENAI_EMBEDDING_BATCH_SIZE", "256"); + const inputs = Array.from({ length: 40 }, (_, index) => `t${String(index).padStart(2, "0")}:${"界".repeat(2729)}`); + const calls: string[][] = []; + vi.doMock("openai", () => ({ + default: class MockOpenAI { + embeddings = { + create: vi.fn(async ({ input }: { input: string[] }) => { + calls.push(input); + return { + data: input.map((text, index) => ({ index, embedding: [Number(text.slice(1, 3)), 0] })).reverse(), + }; + }), + }; + responses = { create: vi.fn() }; + }, + })); + const { clearOpenAICaches, embedTexts } = await import("../src/lib/openai"); + clearOpenAICaches(); + const result = await embedTexts([...inputs, inputs[0]]); + expect(calls.map((batch) => batch.length)).toEqual([36, 4]); + for (const batch of calls) expect(Buffer.byteLength(batch.join(""), "utf8")).toBeLessThanOrEqual(300_000); + expect(calls.flat()).toEqual(inputs); + expect(result).toEqual([...inputs.map((_text, index) => [index, 0]), [0, 0]]); + }); + + it("rejects an unbatchable input before constructing a provider client", async () => { + stubEmbeddingEnv(2); + const constructed = vi.fn(); + vi.doMock("openai", () => ({ + default: class MockOpenAI { + constructor() { + constructed(); + } + embeddings = { create: vi.fn() }; + responses = { create: vi.fn() }; + }, + })); + const { clearOpenAICaches, embedTexts } = await import("../src/lib/openai"); + clearOpenAICaches(); + await expect(embedTexts(["earlier input", "a".repeat(300_001)])).rejects.toMatchObject({ + code: "EMBEDDING_INPUT_EXCEEDS_REQUEST_BUDGET", + }); + expect(constructed).not.toHaveBeenCalled(); + }); + function embeddingForText(text: string): number[] { // Encode the input's numeric suffix so a mis-mapped embedding is detectable. const n = Number(text.replace(/[^0-9]/g, "")); diff --git a/tests/embedding-input-budget.test.ts b/tests/embedding-input-budget.test.ts new file mode 100644 index 0000000000..546d969555 --- /dev/null +++ b/tests/embedding-input-budget.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { + batchEmbeddingInputs, + EmbeddingRequestBudgetError, + EMBEDDING_MAX_BATCH_UTF8_BYTES, + hasKnownEmbeddingBudget, +} from "../src/lib/embedding-input-budget"; + +describe("embedding request budgets", () => { + const model = "text-embedding-3-small"; + + it("preserves order and exact contents at the request-byte boundary", () => { + const inputs = ["a".repeat(100_000), "b".repeat(100_000), "c".repeat(100_000), "d"]; + expect(batchEmbeddingInputs(inputs, 256, model)).toEqual([inputs.slice(0, 3), inputs.slice(3)]); + }); + + it("counts UTF-8 bytes rather than characters", () => { + const inputs = ["界".repeat(50_000), "語".repeat(50_000), "é"]; + const batches = batchEmbeddingInputs(inputs, 256, model); + expect(batches).toEqual([inputs.slice(0, 2), inputs.slice(2)]); + expect(Buffer.byteLength(batches[0].join(""), "utf8")).toBe(EMBEDDING_MAX_BATCH_UTF8_BYTES); + }); + + it("also respects the configured count limit", () => { + expect(batchEmbeddingInputs(["a", "b", "c"], 2, model)).toEqual([["a", "b"], ["c"]]); + expect(batchEmbeddingInputs([], 2, model)).toEqual([]); + for (const size of [0, -1, 1.5, 2049]) expect(() => batchEmbeddingInputs(["a"], size, model)).toThrow(); + }); + + it("refuses an indivisible input beyond the whole request budget without exposing text", () => { + const input = "SYNTHETIC_PRIVATE_MARKER" + "a".repeat(EMBEDDING_MAX_BATCH_UTF8_BYTES); + expect(() => batchEmbeddingInputs(["safe earlier input", input], 256, model)).toThrow(EmbeddingRequestBudgetError); + try { + batchEmbeddingInputs([input], 256, model); + } catch (error) { + expect(String(error)).not.toContain("SYNTHETIC_PRIVATE_MARKER"); + } + }); + + it("keeps count-only behavior for an unknown model without claiming a byte-BPE bound", () => { + const inputs = ["a".repeat(300_001), "b"]; + expect(hasKnownEmbeddingBudget("custom-embedding-model")).toBe(false); + expect(batchEmbeddingInputs(inputs, 2, "custom-embedding-model")).toEqual([inputs]); + }); +}); From 3f3976b76a640c41b9a68d2fc16de894acd6358f Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 07:55:34 +0000 Subject: [PATCH 06/16] fix(extraction): resolve nested Form masks from bounded image headers --- worker/python/extract_pdf_assets.py | 82 ++++++++++++----- .../test_extract_pdf_assets_raster_tables.py | 2 +- ...test_extract_pdf_assets_resource_limits.py | 88 +++++++++++++++++-- 3 files changed, 142 insertions(+), 30 deletions(-) diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 72ba487518..602924c95e 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -28,7 +28,7 @@ # parser defect can be traced to the documents it touched. Bump the date and label whenever a # change alters the text, tables or images produced for the same PDF. 2026-09-16 is PR #2810, # which replaced line-major table text with table_aware_page_text (#GJHKYC). -EXTRACTOR_VERSION = "2026-10-21.image-decode-preflight-v11" +EXTRACTOR_VERSION = "2026-10-21.form-mask-headers-v12" TABLE_STRATEGY = "table_aware_page_text" # IDX-H6: caps on serialized table size. Raised from the previous 80/120-row and 8000-char @@ -182,6 +182,52 @@ def oversized_embedded_image_warning(image_info, max_pixels=MAX_EMBEDDED_IMAGE_P ) +def _displayed_masks_exceed_pixel_limit(page, displayed): + """Read displayed mask headers; None means native metadata is unverified. + + Public get_image_info omits mask dimensions and Form get_image_bbox hashes + pixels. This isolated adapter follows extractIMGINFO(hashes=False)'s own + native header access, using its block indices rather than resource names. + TextPage.this is a binding detail: validate against public metadata and fail + closed if an unsupported binding cannot supply matching image/mask headers. + No pixmap, pixel samples or compressed image bytes are requested. + """ + textpage = None + try: + textpage = page.get_textpage(flags=fitz.TEXT_PRESERVE_IMAGES) + headers = textpage.extractIMGINFO(hashes=False) + + def identity(header): + number, width, height = (header[key] for key in ("number", "width", "height")) + if any(type(value) is not int for value in (number, width, height)) or number < 0 or min(width, height) <= 0: + raise ValueError("invalid native image header") + if type(header["has-mask"]) is not bool: + raise ValueError("invalid native mask header") + return number, width, height, header["has-mask"] + + if [identity(header) for header in headers] != [identity(header) for header in displayed]: + return None + masked = {header["number"] for header in headers if header["has-mask"]} + observed = set() + unsafe = False + for number, block in enumerate(textpage.this): + if number not in masked: + continue + mask = block.i_image().mask() + width, height = mask.w(), mask.h() + if any(type(value) is not int or value <= 0 for value in (width, height)): + return None + observed.add(number) + unsafe = unsafe or width * height > MAX_EMBEDDED_IMAGE_PIXELS + return unsafe if observed == masked else None + except MemoryError: + raise + except Exception: + return None + finally: + del textpage + + def page_image_decode_preflight(page, image_infos): """Inspect image headers, including inline images and separate soft masks. @@ -209,26 +255,20 @@ def page_image_decode_preflight(page, image_infos): ) # get_images also lists dead resources: an unused oversized XObject must # still be skipped as an artifact, but cannot make a safe page render unsafe. - # Displayed image headers above cover the base pixels, including inline and - # Form images. Separate masks need their owning image's placement as well. - for image_info in image_infos: - if image_info[0] not in skipped or oversized_embedded_image_warning(image_info): - continue - if image_info[-1] != 0: - # get_image_bbox delegates Form referencers to hash-based placement. - # Unknown mask placement stays conservative without decoding it. - skipped[image_info[0]] += "; Form mask placement unverified without decoding" - unsafe = True - continue - try: - # Only top-level resources with transform=False use the metadata-only - # path; an unused resource returns an empty/infinite rectangle. - bbox = page.get_image_bbox(image_info) - unsafe = unsafe or not (bbox.is_empty or bbox.is_infinite) - except MemoryError: - raise - except Exception: - unsafe = True + # Displayed headers include inline and Form images. Inspect native mask + # headers lazily, only when a resource declares an oversized separate mask + # and the page actually displays a masked image. Dead Form resources and + # same-size but different visible masked images must not be conflated. + oversized_mask_xrefs = { + info[0] for info in image_infos + if info[0] in skipped and not oversized_embedded_image_warning(info) + } + if not unsafe and oversized_mask_xrefs and any(image.get("has-mask") for image in displayed): + masks_unsafe = _displayed_masks_exceed_pixel_limit(page, displayed) + if masks_unsafe is None: + for xref in oversized_mask_xrefs: + skipped[xref] += "; displayed mask placement unverified without decoding" + unsafe = unsafe or masks_unsafe is not False return skipped, unsafe diff --git a/worker/python/test_extract_pdf_assets_raster_tables.py b/worker/python/test_extract_pdf_assets_raster_tables.py index 57a8e05f6e..835458b033 100644 --- a/worker/python/test_extract_pdf_assets_raster_tables.py +++ b/worker/python/test_extract_pdf_assets_raster_tables.py @@ -555,7 +555,7 @@ def test_gated_merged_scan_keeps_exponent_cells_and_owning_rows(self): for cells, row_text in ((MERGED_ROWS[0], text[wbc:neutrophils]), (MERGED_ROWS[1], text[neutrophils:])): for lines in cells: self.assertIn(" ".join(lines), row_text) - self.assertEqual(result["extractor"]["version"], "2026-10-21.image-decode-preflight-v11") + self.assertEqual(result["extractor"]["version"], "2026-10-21.form-mask-headers-v12") provenance = result["pages"][0]["metadata"]["source_provenance"] self.assertEqual(provenance["ocr"]["status"], "unverified") self.assertIsNone(provenance["ocr"]["confidence"]) diff --git a/worker/python/test_extract_pdf_assets_resource_limits.py b/worker/python/test_extract_pdf_assets_resource_limits.py index 86dd66df84..f3dd7ed905 100644 --- a/worker/python/test_extract_pdf_assets_resource_limits.py +++ b/worker/python/test_extract_pdf_assets_resource_limits.py @@ -124,6 +124,23 @@ def write_indirect_decode_pdf(path, kind, side=9_000, text=PAGE_TEXT): document.save(path) +def write_form_mask_pdf(path, unused=False, visible_masked_sibling=False): + source = path + ".source.pdf" + write_indirect_decode_pdf(source, "soft_mask") + with fitz.open(source) as original, fitz.open() as document: + if unused: + # This synthetic fixture's final stream paints the oversized mask. + original.update_stream(original[0].get_contents()[-1], b"") + if visible_masked_sibling: + # Same base dimensions as the unsafe resource, different safe mask. + pixels = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 120, 90), True) + pixels.clear_with(100) + original[0].insert_image(fitz.Rect(200, 40, 320, 130), pixmap=pixels) + page = document.new_page(width=400, height=400) + page.show_pdf_page(page.rect, original, 0) + document.save(path) + + class TemporaryDirectoryTestCase(unittest.TestCase): def setUp(self): self._directory = tempfile.TemporaryDirectory() @@ -196,14 +213,8 @@ def test_small_image_with_oversized_soft_mask_is_skipped_before_any_decode(self) self.assertTrue(any("soft mask" in warning and "9000x9000" in warning for warning in result["warnings"])) def test_form_soft_mask_is_refused_without_hash_based_placement(self): - source = self.path("form-source.pdf") path = self.path("form-mask.pdf") - write_indirect_decode_pdf(source, "soft_mask") - with fitz.open(source) as original, fitz.open() as document: - page = document.new_page(width=400, height=400) - page.show_pdf_page(page.rect, original, 0) - self.assertTrue(all(info[-1] != 0 for info in page.get_images(full=True))) - document.save(path) + write_form_mask_pdf(path) with patch.object(fitz.Page, "get_image_bbox") as bbox, patch.object( fitz.Page, "get_image_rects" ) as rects, patch.object(fitz.Page, "get_pixmap") as render: @@ -213,7 +224,68 @@ def test_form_soft_mask_is_refused_without_hash_based_placement(self): render.assert_not_called() self.assertEqual(result["images"], []) self.assertIn("lithium level 0.6 to 0.8 mmol/L", result["pages"][0]["text"]) - self.assertTrue(any("Form mask placement unverified" in warning for warning in result["warnings"])) + self.assertTrue(any("soft mask" in warning and "9000x9000" in warning for warning in result["warnings"])) + + def test_unused_form_mask_does_not_block_same_size_visible_masked_sibling(self): + path = self.path("unused-form-mask.pdf") + write_form_mask_pdf(path, unused=True, visible_masked_sibling=True) + with fitz.open(path) as document: + page = document[0] + self.assertTrue(all(info[-1] != 0 for info in page.get_images(full=True))) + displayed = page.get_image_info(hashes=False, xrefs=False) + self.assertEqual([(info["width"], info["height"], info["has-mask"]) for info in displayed], + [(120, 90, True)]) + with patch.object(fitz.Page, "get_image_bbox") as bbox, patch.object( + fitz.Page, "get_image_rects" + ) as rects, patch.object(fitz.Page, "get_pixmap") as render: + skipped, unsafe = extractor.page_image_decode_preflight(page, page.get_images(full=True)) + bbox.assert_not_called() + rects.assert_not_called() + render.assert_not_called() + self.assertEqual(len(skipped), 1) + self.assertFalse(unsafe) + + def test_unused_form_mask_without_visible_masks_needs_no_native_adapter(self): + path = self.path("unused-form-no-mask.pdf") + write_form_mask_pdf(path, unused=True) + with fitz.open(path) as document: + page = document[0] + with patch.object(fitz.Page, "get_image_info", return_value=[]), patch.object( + fitz.Page, "get_textpage", side_effect=AssertionError("no masked image is displayed") + ) as native: + skipped, unsafe = extractor.page_image_decode_preflight(page, page.get_images(full=True)) + native.assert_not_called() + self.assertEqual(len(skipped), 1) + self.assertFalse(unsafe) + + def test_unknown_native_mask_adapter_keeps_refusal(self): + path = self.path("unknown-form-mask.pdf") + write_form_mask_pdf(path, unused=True, visible_masked_sibling=True) + with fitz.open(path) as document: + page = document[0] + displayed = page.get_image_info(hashes=False, xrefs=False) + with patch.object(fitz.Page, "get_image_info", return_value=displayed), patch.object( + fitz.Page, "get_textpage", return_value=object() + ): + skipped, unsafe = extractor.page_image_decode_preflight(page, page.get_images(full=True)) + self.assertTrue(unsafe) + self.assertTrue(any("mask placement unverified" in warning for warning in skipped.values())) + + def test_malformed_native_mask_headers_keep_refusal(self): + path = self.path("malformed-form-mask.pdf") + write_form_mask_pdf(path, unused=True, visible_masked_sibling=True) + with fitz.open(path) as document: + page = document[0] + displayed = page.get_image_info(hashes=False, xrefs=False) + from unittest.mock import Mock + native = Mock() + native.extractIMGINFO.return_value = [{**displayed[0], "number": "not-a-block-index"}] + with patch.object(fitz.Page, "get_image_info", return_value=displayed), patch.object( + fitz.Page, "get_textpage", return_value=native + ): + skipped, unsafe = extractor.page_image_decode_preflight(page, page.get_images(full=True)) + self.assertTrue(unsafe) + self.assertTrue(any("mask placement unverified" in warning for warning in skipped.values())) def test_safe_embedded_sibling_survives_without_hashing_unsafe_inline_image(self): source = self.path("inline-source.pdf") From fe2d7f2d8890c873ce942f0c47ec9f96b55ae798 Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 07:59:37 +0000 Subject: [PATCH 07/16] fix(chunking): preserve complete table qualification scope on overflow --- docs/reindex-runbook.md | 7 +- docs/worker-deploy-runbook.md | 10 + src/lib/chunking.ts | 30 +-- tests/chunker-version-ratchet.test.ts | 2 +- tests/chunking.test.ts | 33 +++- ..._extract_pdf_assets_coverage_acceptance.py | 184 ++++++++++++++++++ 6 files changed, 246 insertions(+), 20 deletions(-) create mode 100644 worker/python/test_extract_pdf_assets_coverage_acceptance.py diff --git a/docs/reindex-runbook.md b/docs/reindex-runbook.md index a3edea1fed..057544541d 100644 --- a/docs/reindex-runbook.md +++ b/docs/reindex-runbook.md @@ -79,12 +79,13 @@ recovery, including with `--apply`. If separate recoverable queue work also exis recovery may run, but the integrity alert remains until a subsequent probe confirms resolution. The configured character target controls ordinary table groups. A larger indivisible -row is supported as a standalone chunk with its full title, header and adjacent note, -up to 8,191 UTF-8 bytes for known OpenAI embedding models. This is a conservative +row is supported by retaining the complete table, including every qualification +band and adjacent note, as one chunk up to 8,191 UTF-8 bytes for known OpenAI embedding models. This is a conservative byte-BPE token upper bound, not an exact token count. The chunk records this overflow in metadata, and embedding requests are batched by both input count and total bytes. `TABLE_ROW_EXCEEDS_CHUNK_BUDGET` remains a permanent refusal above that hard ceiling -or for an unknown model's target overflow. The chunker returns no partial chunk array +or for an unknown model's target overflow. A qualification-bearing table whose +complete scope cannot fit is also refused. The chunker returns no partial chunk array and never splits cells or truncates evidence. Review the source structure before retrying; the refusal does not establish OCR accuracy. Atomic reindex failure preserves the existing index. diff --git a/docs/worker-deploy-runbook.md b/docs/worker-deploy-runbook.md index 70f70dcedd..8fc3ad2f23 100644 --- a/docs/worker-deploy-runbook.md +++ b/docs/worker-deploy-runbook.md @@ -33,6 +33,16 @@ no isolated memory limit inside the worker. Restore the Python runtime before retrying the document. The developer fallback retains its deadline and pixel limits; those limits do not establish a process memory boundary. +Offline OCR acceptance currently covers native merged qualifications and Unicode +symbols, scanned staggered columns, and the existing ruled-table cases. It does +not establish reliable scanned mathematical notation: the pinned Latin model's +vocabulary lacks `≤`, `≥` and `⁹`, and focused recognition trials lost equality +strokes, an exponent and a decimal. The strict scanned-symbol acceptance case is +recorded as an expected failure, separately from passing coverage. Keep these +OCR observations unverified; require source review or a demonstrated OCR-engine +improvement before relying on such numeric evidence. Changing image-size limits +does not address this model limitation. + --- ## 0. Pre-deploy gate — do not deploy before this is live diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index 9ef85bdb8a..576b604558 100644 --- a/src/lib/chunking.ts +++ b/src/lib/chunking.ts @@ -433,11 +433,11 @@ function chunkTextWithOverlapInternal( continue; } - if ( - paragraph.length > chunkSize || - (Buffer.byteLength(paragraph, "utf8") > MAX_ATOMIC_TABLE_UTF8_BYTES && - paragraph.split(lineBreakPattern).some((line) => tableRowPattern.test(line))) - ) { + const paragraphLines = paragraph.split(lineBreakPattern); + const containsTableRun = paragraphLines.some( + (line, index) => tableRowPattern.test(line) && tableRowPattern.test(paragraphLines[index + 1] ?? ""), + ); + if (paragraph.length > chunkSize || containsTableRun) { if (current) { chunks.push(current.trim()); current = ""; @@ -617,6 +617,19 @@ function chunkTableBlock( throw new OversizedTableRowError(); } + const overflowingRow = bodyLines.some((row) => withFootnote(`${header}\n${row}`).length > chunkSize); + const qualificationBand = bodyLines.some((row) => row.split("|").filter((cell) => cell.trim()).length === 1); + if (overflowingRow || qualificationBand || (!bodyLines.length && withFootnote(header).length > chunkSize)) { + // Flat markdown does not prove which earlier rows qualify a later action. + // Preserve the complete table when admitting a preferred-target overflow, + // including all merged bands; never invent row-to-condition relationships. + if (!fitsByteCeiling(captioned) || !hasKnownEmbeddingBudget(env.OPENAI_EMBEDDING_MODEL)) { + throw new OversizedTableRowError(); + } + if (captioned.length > chunkSize) onTableOverflow?.(captioned); + return [captioned]; + } + const chunks: string[] = []; const emit = (text: string) => { chunks.push(text); @@ -624,13 +637,6 @@ function chunkTableBlock( }; let current = ""; for (const row of bodyLines) { - const atomic = withFootnote(`${header}\n${row}`); - if (atomic.length > chunkSize) { - if (current) emit(withFootnote(current)); - current = ""; - emit(atomic); - continue; - } const candidate = current ? `${current}\n${row}` : `${header}\n${row}`; const qualified = withFootnote(candidate); if (current && (qualified.length > chunkSize || !fitsByteCeiling(qualified))) { diff --git a/tests/chunker-version-ratchet.test.ts b/tests/chunker-version-ratchet.test.ts index 26277b6a04..dec8efd807 100644 --- a/tests/chunker-version-ratchet.test.ts +++ b/tests/chunker-version-ratchet.test.ts @@ -64,7 +64,7 @@ const overflowCorpus = [ ].join("\n"), ]; const overflowRecorded: Record = { - "1.4.0-page|1.4.0-document": "bc6abc0a6a25d4491200d4b82646de78b9b92dc6811ed0d9e3ed4d143291b32e", + "1.4.0-page|1.4.0-document": "7a762b08fef8f1ba762591f3e645cf2c93a3f0877b8bed953c2ca0ba783ba502", }; function overflowFingerprint() { diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index fc24f0f129..10bdc6f795 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -865,15 +865,40 @@ describe("bounded atomic table row overflow", () => { } }); - it("keeps target-sized groups while emitting an overflowing row alone with its complete context", () => { + it("keeps the complete bounded table when a row exceeds the preferred target", () => { const title = "Table 4. Synthetic monitoring"; const note = "Note: Do not change treatment\nunless the original condition applies."; const before = "| GROUP-B | Repeat the measurement. |"; const after = "| GROUP-C | Review the result. |"; const text = [title, header, before, row, after, note].join("\n"); - expect(chunkTextWithOverlap(text, 250, 0)).toEqual( - [before, row, after].map((line) => [title, header, line, note].join("\n")), + expect(chunkTextWithOverlap(text, 250, 0)).toEqual([text]); + }); + + it("retains merged qualification bands with every newly admitted oversized action row", () => { + const initial = "| INITIAL PHASE ONLY: applies to both following rows | |"; + const maintenance = "| MAINTENANCE ONLY: applies to the following row | |"; + const text = [header, initial, row, "| GROUP-B | Review the result. |", maintenance, "| GROUP-C | Repeat. |"].join( + "\n", ); + expect(chunkTextWithOverlap(text, 400, 0)).toEqual([text]); + }); + + it("refuses a qualification-bearing table when its complete scope exceeds the hard ceiling", () => { + const band = "| INITIAL PHASE ONLY: applies to following rows | |"; + const rows = Array.from({ length: 3 }, (_, index) => `| GROUP-${index} | ${"Original condition. ".repeat(150)} |`); + const text = [header, band, ...rows].join("\n"); + expect(Buffer.byteLength(text)).toBeGreaterThan(MAX_ATOMIC_TABLE_UTF8_BYTES); + expect(() => chunkTextWithOverlap(text, 400, 0)).toThrow(OversizedTableRowError); + }); + + it("does not co-pack a captioned table with prose past the hard table ceiling", () => { + const table = ["Table 4. Monitoring", header, `| GROUP-A | ${"a".repeat(5900)} |`].join("\n"); + const prose = "Separate narrative. " + "b".repeat(4000); + const chunks = chunkTextWithOverlap(`${table}\n\n${prose}`, 20000, 0); + expect(chunks.join("\n")).toContain(prose); + const tables = chunks.filter((chunk) => chunk.includes("| GROUP-A |")); + expect(tables).toEqual([table]); + for (const chunk of tables) expect(Buffer.byteLength(chunk)).toBeLessThanOrEqual(MAX_ATOMIC_TABLE_UTF8_BYTES); }); it("accepts a row exactly at the character target including title and footnote", () => { @@ -884,7 +909,7 @@ describe("bounded atomic table row overflow", () => { const text = [title, header, ...rows, note].join("\n"); const expected = rows.map((line) => [title, header, line, note].join("\n")); expect(chunkTextWithOverlap(text, atomic.length, 0)).toEqual(expected); - expect(chunkTextWithOverlap(text, atomic.length - 1, 0)).toEqual(expected); + expect(chunkTextWithOverlap(text, atomic.length - 1, 0)).toEqual([text]); }); it("accepts exactly 8191 UTF-8 bytes including caption, header, row and wrapped note", () => { diff --git a/worker/python/test_extract_pdf_assets_coverage_acceptance.py b/worker/python/test_extract_pdf_assets_coverage_acceptance.py new file mode 100644 index 0000000000..37ddfdf66e --- /dev/null +++ b/worker/python/test_extract_pdf_assets_coverage_acceptance.py @@ -0,0 +1,184 @@ +"""Synthetic, bounded PDF acceptance cases; no clinical accuracy certification. + +The real-OCR cases use the existing pinned local model and page deadlines. A +readable-looking substitution for a comparator or exponent is a failure, even +though OCR provenance correctly remains unverified. +""" + +import importlib.util +import os +from pathlib import Path +import re +import shutil +import tempfile +import unittest +from unittest.mock import patch + +import extract_pdf_assets as extractor +from test_extract_pdf_assets_hard_tables import draw_table + + +fitz = extractor.fitz + + +def flat(text): + return re.sub(r"\s+", " ", text or "").strip() + + +def write_fixture(directory, build, scanned=False): + """Generate the same visible source as native text or a 300-dpi scan.""" + path = Path(directory) / "synthetic-coverage.pdf" + with fitz.open() as source: + page = source.new_page(width=595, height=842) + build(page) + if scanned: + pixels = page.get_pixmap(dpi=300, colorspace=fitz.csGRAY) + with fitz.open() as scan: + target = scan.new_page(width=595, height=842) + target.insert_image(target.rect, stream=pixels.tobytes("png")) + scan.save(path) + else: + source.save(path) + return path + + +def symbol_table(page): + """Unicode superscript and relation signs, rather than ASCII lookalikes.""" + xs, ys = (40, 220, 390, 555), (80, 112, 152, 192) + for x in xs: + page.draw_line((x, ys[0]), (x, ys[-1]), width=0.7) + for y in ys: + page.draw_line((xs[0], y), (xs[-1], y), width=0.7) + writer = fitz.TextWriter(page.rect) + font = fitz.Font("helv") + rows = ( + ("Count (x 10⁹/L)", "Threshold", "SYNTHETIC action"), + ("Example A", "≤ 1.5", "Withhold example A"), + ("Example B", "≥ 2.0", "Review example B"), + ) + for row_index, row in enumerate(rows): + for column, text in enumerate(row): + # TextWriter supplies MuPDF's bundled Unicode fallback for ⁹; + # no host font, downloaded asset or binary fixture is needed. + writer.append((xs[column] + 5, ys[row_index] + 21), text, font=font, fontsize=11) + writer.write_text(page) + + +def merged_qualifier_table(page): + xs = (40, 215, 390, 555) + ys = (80, 108, 136, 170, 204, 232, 266) + for y in ys: + page.draw_line((xs[0], y), (xs[-1], y), width=0.7) + for x in (xs[0], xs[-1]): + page.draw_line((x, ys[0]), (x, ys[-1]), width=0.7) + # The two qualification bands span all three columns mid-table. + for top, bottom in ((80, 108), (136, 204), (232, 266)): + for x in xs[1:-1]: + page.draw_line((x, top), (x, bottom), width=0.7) + rows = ( + ("SYNTHETIC phase", "Result", "Action"), + ("INITIAL PHASE ONLY: applies to both following rows",), + ("Example A", "Below 1.0", "Withhold example A"), + ("Example B", "Above 2.0", "Review example B"), + ("MAINTENANCE ONLY: applies to the following row",), + ("Example C", "Below 3.0", "Repeat example C"), + ) + for index, row in enumerate(rows): + for column, text in enumerate(row): + page.insert_text((xs[column] + 4, ys[index] + 18), text, fontsize=9) + + +def staggered_columns(page): + # Opposite insertion order and staggered baselines make PDF stream order + # and line-major order both discriminate against intact physical columns. + for x, top, lines in ( + (330, 60, ("RIGHT QUALIFIER", "Only after rest.", "Repeat before discharge.")), + (40, 45, ("LEFT QUALIFIER", "Only after fasting.", "Recheck before action.")), + ): + for offset, line in enumerate(lines): + page.insert_text((x, top + offset * 30), line, fontsize=11) + draw_table(page, [40, 215, 390, 555], 180, + ["SYNTHETIC result", "Action", "Timing"], [ + (["Example A"], ["Withhold example A"], ["Day 3"]), + (["Example B"], ["Review example B"], ["Day 5"]), + ]) + + +class NativeCoverageAcceptanceTests(unittest.TestCase): + def test_mid_table_merged_qualifiers_keep_their_row_groups(self): + with tempfile.TemporaryDirectory() as directory: + path = write_fixture(directory, merged_qualifier_table) + result = extractor.extract(str(path), str(Path(directory) / "out")) + text = flat(result["pages"][0]["text"]) + markers = ( + "INITIAL PHASE ONLY: applies to both following rows", + "Example A | Below 1.0 | Withhold example A", + "Example B | Above 2.0 | Review example B", + "MAINTENANCE ONLY: applies to the following row", + "Example C | Below 3.0 | Repeat example C", + ) + positions = [text.index(marker) for marker in markers] + self.assertEqual(positions, sorted(positions)) + for marker in markers: + self.assertEqual(text.count(marker), 1) + self.assertFalse(result["pages"][0]["ocrUsed"]) + + def test_native_unicode_exponent_and_inclusive_comparators_are_preserved(self): + with tempfile.TemporaryDirectory() as directory: + path = write_fixture(directory, symbol_table) + result = extractor.extract(str(path), str(Path(directory) / "out")) + text = flat(result["pages"][0]["text"]) + self.assertIn("Count (x 10⁹/L)", text) + self.assertIn("Example A | ≤ 1.5 | Withhold example A", text) + self.assertIn("Example B | ≥ 2.0 | Review example B", text) + self.assertFalse(result["pages"][0]["ocrUsed"]) + + +@unittest.skipUnless(shutil.which("tesseract") and importlib.util.find_spec("pytesseract") + and importlib.util.find_spec("PIL"), "Local OCR dependencies are required") +class RealOcrCoverageAcceptanceTests(unittest.TestCase): + def extract_scan(self, build): + with tempfile.TemporaryDirectory() as directory: + path = write_fixture(directory, build, scanned=True) + with patch.dict(os.environ, {"PDF_RASTER_TABLE_OCR": "1"}): + result = extractor.extract(str(path), str(Path(directory) / "out")) + page = result["pages"][0] + self.assertTrue(page["ocrUsed"]) + self.assertFalse(page["needsOcr"]) + provenance = page["metadata"]["source_provenance"] + self.assertEqual(provenance["ocr"]["status"], "unverified") + self.assertEqual(provenance["ocr"]["comparison"], "not_compared") + self.assertIsNone(provenance["ocr"]["confidence"]) + return flat(page["text"]) + + def test_staggered_outside_columns_keep_qualifiers_and_read_left_first(self): + text = self.extract_scan(staggered_columns) + left = "LEFT QUALIFIER Only after fasting. Recheck before action." + right = "RIGHT QUALIFIER Only after rest. Repeat before discharge." + self.assertIn(left, text) + self.assertIn(right, text) + self.assertLess(text.index(left), text.index(right)) + self.assertLess(text.index(right), text.index("Example A")) + self.assertIn("Example A | Withhold example A | Day 3", text) + self.assertIn("Example B | Review example B | Day 5", text) + + # Unresolved recognition defect: the pinned Latin vocabulary lacks these + # glyphs, and bounded crop/model/segmentation diagnostics did not recover + # equivalent semantics. Keep every strict assertion; report this expected + # failure separately. Unexpected success forces reassessment on an upgrade. + @unittest.expectedFailure + def test_scanned_unicode_exponent_and_inclusive_comparators_are_preserved(self): + text = self.extract_scan(symbol_table) + # Accept either unambiguous exponent spelling, never baseline 109 or + # a lost/reversed equality stroke. Source units must retain their power. + for name, pattern in ( + ("unit exponent", r"Count \(x 10(?:⁹|\^9)/L\)"), + ("inclusive upper threshold", r"Example A \| ≤ 1\.5 \| Withhold example A"), + ("inclusive lower threshold", r"Example B \| ≥ 2\.0 \| Review example B"), + ): + with self.subTest(source_semantics=name): + self.assertRegex(text, pattern) + + +if __name__ == "__main__": + unittest.main() From 96a65df7b47dd060f8cc4cb0215e197944408800 Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 08:05:43 +0000 Subject: [PATCH 08/16] docs(ingestion): track unresolved scanned mathematical notation --- .../f686046e-e9f4-451d-b225-68301dc7dd86.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docs/outstanding-issues-inbox/f686046e-e9f4-451d-b225-68301dc7dd86.json diff --git a/docs/outstanding-issues-inbox/f686046e-e9f4-451d-b225-68301dc7dd86.json b/docs/outstanding-issues-inbox/f686046e-e9f4-451d-b225-68301dc7dd86.json new file mode 100644 index 0000000000..9e4a196c65 --- /dev/null +++ b/docs/outstanding-issues-inbox/f686046e-e9f4-451d-b225-68301dc7dd86.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "id": "f686046e-e9f4-451d-b225-68301dc7dd86", + "createdOn": "2026-10-21", + "action": "add", + "payload": { + "pri": "P1", + "type": "issue", + "summary": "Pinned raster OCR loses inclusive comparators, exponent and decimal in scanned mathematical notation", + "detail": "Synthetic native source preserves Count (x 10⁹/L), ≤ 1.5 and ≥ 2.0; actual pinned Latin raster OCR yields 10?, <15 and >2.0. The 303-entry pinned Latin vocabulary lacks ≤, ≥ and ⁹. Twelve bounded crop trials across cached Latin/eng and PSM variants did not recover equivalent semantics. New strict acceptance case is explicitly expectedFailure, not passing accuracy evidence; OCR provenance remains unverified/not_compared. Resolve through a demonstrated OCR engine/model improvement and source acceptance, without clinical text substitution or relaxed assertions. No production reindex or provider inference was run.", + "source": "PR #3716; worker/python/test_extract_pdf_assets_coverage_acceptance.py::RealOcrCoverageAcceptanceTests.test_scanned_unicode_exponent_and_inclusive_comparators_are_preserved; worker/python/extract_pdf_assets.py; 2026-10-21 offline acceptance", + "issueUlid": "01M5EQWSKTMR9VJKFCS6GVZQK1" + } +} From 10ae1e41d8a424a220b7c74dfad4383ff648bec8 Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 08:56:06 +0000 Subject: [PATCH 09/16] fix(extraction): preserve observed mathematical glyph layout in bounded OCR --- src/lib/extractors/document.ts | 16 +- tests/extracted-document-validation.test.ts | 34 ++- worker/python/extract_pdf_assets.py | 37 ++- worker/python/raster_math_ocr.py | 213 ++++++++++++++++++ ..._extract_pdf_assets_coverage_acceptance.py | 201 ++++++++++++++--- ...est_extract_pdf_assets_math_integration.py | 88 ++++++++ .../test_extract_pdf_assets_raster_tables.py | 2 +- worker/python/test_raster_math_ocr.py | 158 +++++++++++++ 8 files changed, 715 insertions(+), 34 deletions(-) create mode 100644 worker/python/raster_math_ocr.py create mode 100644 worker/python/test_extract_pdf_assets_math_integration.py create mode 100644 worker/python/test_raster_math_ocr.py diff --git a/src/lib/extractors/document.ts b/src/lib/extractors/document.ts index a00f197c79..05fd076014 100644 --- a/src/lib/extractors/document.ts +++ b/src/lib/extractors/document.ts @@ -69,10 +69,24 @@ const rasterCropSchema = z pngSha256: rasterShaSchema, pixelsSha256: rasterShaSchema, textSha256: rasterShaSchema, + ocrInputTransform: z + .object({ + algorithm: z.literal("raster-math-topology-v1"), + kind: z.enum(["comparison", "superscript"]), + codeSha256: rasterShaSchema, + width: z.number().int().positive().max(400_000), + height: z.number().int().positive().max(400_000), + mode: z.enum(["L", "RGB"]), + pixelsSha256: rasterShaSchema, + }) + .strict() + .refine(({ width, height }) => width * height <= 400_000, "raster_math_input_pixel_limit") + .optional(), }) .strict() .refine(({ width, height }) => width * height <= 4_000_000, "raster_crop_pixel_limit") - .refine(({ kind, language }) => language === (kind === "cell" ? "Latin" : "eng"), "raster_crop_model_scope"); + .refine(({ kind, language }) => language === (kind === "cell" ? "Latin" : "eng"), "raster_crop_model_scope") + .refine(({ kind, ocrInputTransform }) => !ocrInputTransform || kind === "cell", "raster_math_cell_scope"); const rasterTableOcrSchema = z .object({ version: z.literal("raster-table-ocr-observations-v1"), diff --git a/tests/extracted-document-validation.test.ts b/tests/extracted-document-validation.test.ts index 30b931eb25..bbea9b124f 100644 --- a/tests/extracted-document-validation.test.ts +++ b/tests/extracted-document-validation.test.ts @@ -68,6 +68,38 @@ describe("Python extracted-document payload validation", () => { expect(parseExtractedDocumentPayload(packet(observations)).pages[0].metadata?.raster_table_ocr).toEqual( observations, ); + const transform = { + algorithm: "raster-math-topology-v1", + kind: "comparison", + codeSha256: "sha256:" + "c".repeat(64), + width: 100, + height: 50, + mode: "L", + pixelsSha256: "sha256:" + "b".repeat(64), + }; + const transformed = (value: unknown, crop = observations.crops[0]) => ({ + ...observations, + crops: [{ ...crop, ocrInputTransform: value }], + }); + for (const kind of ["comparison", "superscript"]) { + const value = transformed({ ...transform, kind }); + expect(parseExtractedDocumentPayload(packet(value)).pages[0].metadata?.raster_table_ocr).toEqual(value); + } + for (const invalidTransform of [ + { ...transform, algorithm: "unknown" }, + { ...transform, kind: "inferred_clinical_value" }, + { ...transform, width: 400_000 }, + { ...transform, mode: "CMYK" }, + { ...transform, pixelsSha256: "not-a-hash" }, + { ...transform, codeSha256: undefined }, + { ...transform, confidence: 1 }, + ]) + expect(() => parseExtractedDocumentPayload(packet(transformed(invalidTransform)))).toThrow(); + expect(() => + parseExtractedDocumentPayload( + packet(transformed(transform, { ...observations.crops[0], kind: "outside", language: "eng" })), + ), + ).toThrow(); for (const invalid of [ { ...observations, status: "verified" }, { ...observations, confidence: 1 }, @@ -114,7 +146,7 @@ describe("Python extracted-document payload validation", () => { describe("extraction provenance (audit F04)", () => { const provenance = { name: "extract_pdf_assets", - version: "2026-10-20.embedded-image-ceiling-v10", + version: "2026-10-21.raster-math-layout-v13", pymupdf: "1.26.0", tableStrategy: "table_aware_page_text", }; diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 845ec17425..8cc60526ce 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -28,7 +28,7 @@ # parser defect can be traced to the documents it touched. Bump the date and label whenever a # change alters the text, tables or images produced for the same PDF. 2026-09-16 is PR #2810, # which replaced line-major table text with table_aware_page_text (#GJHKYC). -EXTRACTOR_VERSION = "2026-10-21.form-mask-headers-v12" +EXTRACTOR_VERSION = "2026-10-21.raster-math-layout-v13" TABLE_STRATEGY = "table_aware_page_text" # IDX-H6: caps on serialized table size. Raised from the previous 80/120-row and 8000-char @@ -733,7 +733,14 @@ def maybe_raster_table_ocr(page, image, render_scale, budget, pytesseract, deadl except RasterLayoutAmbiguous: return None from PIL import Image + import raster_math_ocr + from raster_math_ocr import plan_raster_math_ocr model_directory = checked_raster_table_model(deadline) + if time.monotonic() >= deadline: + raise TimeoutError("raster helper identity exhausted page timeout") + math_code_sha256 = file_sha256(raster_math_ocr.__file__) + if time.monotonic() >= deadline: + raise TimeoutError("raster helper identity exhausted page timeout") record = {"version": "raster-table-ocr-observations-v1", "status": "unverified", "cellModel": {"language": "Latin", "sha256": "sha256:" + RASTER_TABLE_MODEL_SHA256, "bytes": RASTER_TABLE_MODEL_BYTES}, @@ -759,13 +766,31 @@ def read_region(rect, cell=False): if remaining <= 0: raise TimeoutError("raster cell OCR exhausted page timeout") config = f"--dpi {round(scale * 72)}" + math_plan = None if cell: mode = 7 if raster_ink_line_count(bitmap, deadline) == 1 else 6 + # Pixmaps round a physical clip outward. Analyze only pixels + # fully inside this cell, never infer that a corner mark is noise. + interior = ( + math.ceil(rect.x0 * scale) - pixels.x, + math.ceil(rect.y0 * scale) - pixels.y, + math.floor(rect.x1 * scale) - pixels.x, + math.floor(rect.y1 * scale) - pixels.y, + ) + if 0 <= interior[0] < interior[2] <= bitmap.width and 0 <= interior[1] < interior[3] <= bitmap.height: + cell_interior = bitmap.crop(interior) + if raster_ink_line_count(cell_interior, deadline) == 1: + math_plan = plan_raster_math_ocr(cell_interior, deadline, budget.limits["maxRenderPixels"]) + if math_plan is not None: + mode = 7 config = f"--psm {mode} --tessdata-dir {shlex.quote(model_directory)} " + config remaining = deadline - time.monotonic() if remaining <= 0: raise TimeoutError("cell layout exhausted page OCR timeout before launch") - text = pytesseract.image_to_string(bitmap, lang="Latin" if cell else "eng", config=config, timeout=remaining).strip() + ocr_bitmap = math_plan.bitmap if math_plan is not None else bitmap + text = pytesseract.image_to_string(ocr_bitmap, lang="Latin" if cell else "eng", config=config, timeout=remaining).strip() + if math_plan is not None: + text = math_plan.restore(text) if time.monotonic() >= deadline: raise TimeoutError("raster OCR completed after page deadline") if len(record["crops"]) >= 112: @@ -774,7 +799,13 @@ def read_region(rect, cell=False): "sourceRect": list(rect), "scale": scale, "width": pixels.width, "height": pixels.height, "pngSha256": "sha256:" + hashlib.sha256(png).hexdigest(), "pixelsSha256": "sha256:" + hashlib.sha256(pixels.samples).hexdigest(), - "textSha256": text_sha256(text)}) + "textSha256": text_sha256(text), + **({"ocrInputTransform": { + "algorithm": "raster-math-topology-v1", "kind": math_plan.kind, + "codeSha256": math_code_sha256, + "width": ocr_bitmap.width, "height": ocr_bitmap.height, "mode": ocr_bitmap.mode, + "pixelsSha256": "sha256:" + hashlib.sha256(ocr_bitmap.tobytes()).hexdigest(), + }} if math_plan is not None else {})}) if time.monotonic() >= deadline: raise TimeoutError("raster crop observation exhausted page deadline") return text diff --git a/worker/python/raster_math_ocr.py b/worker/python/raster_math_ocr.py new file mode 100644 index 0000000000..2baee2ffe9 --- /dev/null +++ b/worker/python/raster_math_ocr.py @@ -0,0 +1,213 @@ +"""Bounded pixel plans for isolated comparisons and raised digits; OCR stays unverified. + +No dictionary, units, expected values or source-specific templates are consulted. +Unsupported geometry keeps the original OCR path. A changed bitmap must always +pass restore(), otherwise the caller must discard the complete table candidate. +""" + +from dataclasses import dataclass +import re +from statistics import median +import time + + +MAX_MATH_PIXELS = 400_000 +SUPERSCRIPTS = str.maketrans("123456789", "¹²³⁴⁵⁶⁷⁸⁹") + + +def _check(deadline): + if time.monotonic() >= deadline: + raise TimeoutError("raster math analysis exhausted page deadline") + + +def _runs(values, deadline): + spans = [] + for index, value in enumerate(values): + if index % 64 == 0: + _check(deadline) + if value: + if spans and spans[-1][1] == index: + spans[-1][1] = index + 1 + else: + spans.append([index, index + 1]) + return spans + + +def _bar(ink, box): + x0, y0, x1, y1 = box + return ink.crop(box).histogram()[255] >= (x1 - x0) * (y1 - y0) * .65 + + +def _operator(ink, glyph, deadline): + """A chevron has two sloped arms; an inclusive sign has a separate bar below.""" + box, bands = glyph + x0, y0, x1, y1 = box + width = x1 - x0 + if len(bands) not in (1, 2): + return None + a, b = bands[0] + height = b - a + if height < 8 or not .35 <= width / height <= 1.7: + return None + centers = [] + for position in (.15, .30, .45, .55, .70, .85): + _check(deadline) + y = a + round((height - 1) * position) + spans = _runs(ink.crop((x0, y, x1, y + 1)).getprojection()[0], deadline) + if len(spans) != 1: + return None + centers.append(sum(spans[0]) / 2 / width) + slopes = [centers[1] - centers[0], centers[2] - centers[1], + centers[3] - centers[4], centers[4] - centers[5]] + if all(slope <= -.12 for slope in slopes) and max(centers[2:4]) < .35: + sign = "<" + elif all(slope >= .12 for slope in slopes) and min(centers[2:4]) > .65: + sign = ">" + else: + return None + if len(bands) == 2: + c, d = bands[1] + arm_bounds = ink.crop((x0, a, x1, b)).getbbox() + bar_bounds = ink.crop((x0, c, x1, d)).getbbox() + if not (0 < c - b < height * .5 and d - c <= height * .25 + and _bar(ink, (x0, c, x1, d)) + and max(abs(arm_bounds[edge] - bar_bounds[edge]) for edge in (0, 2)) <= 2): + return None + sign = "≤" if sign == "<" else "≥" + return sign + + +@dataclass +class MathOcrPlan: + bitmap: object + kind: str + glyphs: list + deadline: float + operator: str = "" + raised: tuple = () + body_height: float = 0 + baseline: float = 0 + + def restore(self, text): + _check(self.deadline) + if "\n" in text.strip() or "\r" in text.strip(): + raise ValueError("raster math text layout is ambiguous") + characters = [char for char in text if not char.isspace()] + if len(characters) != len(self.glyphs): + raise ValueError("raster math source alignment is ambiguous") + if self.kind == "comparison": + numeric = "".join(characters) + if not re.fullmatch(r"[0-9]+(?:\.[0-9]+)?", numeric): + raise ValueError("raster math numeric recognition is ambiguous") + for char, (box, _) in zip(characters, self.glyphs): + _check(self.deadline) + x0, y0, x1, y1 = box + if char == ".": + if not (max(x1 - x0, y1 - y0) <= self.body_height * .3 + and y0 >= self.baseline - self.body_height * .3 + and abs(y1 - self.baseline) <= self.body_height * .2): + raise ValueError("raster math decimal alignment is ambiguous") + elif y1 - y0 < self.body_height * .65: + raise ValueError("raster math digit alignment is ambiguous") + return self.operator + " " + numeric + # The digit's index is its observed, disjoint source x-interval. Never + # replace a text pattern such as 109, a question mark, or a known unit. + if any(characters[index] not in "123456789" for index in self.raised): + # Zero is intentionally unsupported: a normalized degree circle + # was actually recognized as 0 by the pinned model without a whitelist. + raise ValueError("raster math raised recognition is ambiguous") + result, index = [], 0 + for char in text: + _check(self.deadline) + if not char.isspace(): + if index in self.raised: + char = char.translate(SUPERSCRIPTS) + index += 1 + result.append(char) + return "".join(result) + + +def plan_raster_math_ocr(bitmap, deadline, max_pixels): + """Return one replacement OCR input plus a mandatory coordinate restoration. + + Only small, single-line, x-disjoint source components are supported. This + does not certify OCR accuracy or infer whether a raised digit is a power + or a footnote. Input/output pixels and all work share the caller's limits. + """ + _check(deadline) + cap = min(MAX_MATH_PIXELS, max_pixels) + if bitmap.width * bitmap.height > cap: + return None + ink = bitmap.convert("L").point(lambda value: 255 if value < 128 else 0) + _check(deadline) + glyphs = [] + for x0, x1 in _runs(ink.getprojection()[0], deadline): + bands = _runs(ink.crop((x0, 0, x1, ink.height)).getprojection()[1], deadline) + y0, y1 = bands[0][0], bands[-1][1] + glyphs.append(((x0, y0, x1, y1), bands)) + if len(glyphs) > 96: + return None + if len(glyphs) < 2: + return None + bands = _runs(ink.getprojection()[1], deadline) + gap = max(2, median(b - a for a, b in bands) * .35) + if any(right[0] - left[1] > gap for left, right in zip(bands, bands[1:])): + return None + heights = sorted(box[3] - box[1] for box, _ in glyphs) + height = median(heights[len(heights) // 2:]) + if height < 12: + return None + baseline = median(box[3] for box, _ in glyphs if box[3] - box[1] >= height * .8) + if any(abs(box[3] - baseline) > height * .35 for box, _ in glyphs + if box[3] - box[1] >= height * .8): + return None + sign = _operator(ink, glyphs[0], deadline) + if sign: + # ASCII <= and >= contain a second, side-by-side equals glyph. Keep + # that source spelling on the ordinary OCR path. + box, second_bands = glyphs[1] + if len(second_bands) == 2 and all( + _bar(ink, (box[0], a, box[2], b)) for a, b in second_bands): + return None + suffix = glyphs[1:] + left = suffix[0][0][0] + right = suffix[-1][0][2] + _check(deadline) + return MathOcrPlan(bitmap.crop((max(0, left - 4), 0, min(bitmap.width, right + 4), bitmap.height)), + "comparison", suffix, deadline, sign, body_height=height, baseline=baseline) + raised = [] + for index, (box, component_bands) in enumerate(glyphs): + _check(deadline) + x0, y0, x1, y1 = box + h = y1 - y0 + if .3 * height <= h <= .8 * height and y1 <= baseline - .25 * height: + # Near-circular degree signs, broad stars/letters, isolated marks + # and overlapping x-intervals do not become normalized digit inputs. + if (not index or len(component_bands) != 1 or not .18 <= (x1 - x0) / h <= .86 + or x0 - glyphs[index - 1][0][2] > height * .65): + return None + raised.append(index) + if not raised or len(raised) > 8: + return None + body = round(height) + widths = {i: round((glyphs[i][0][2] - glyphs[i][0][0]) * body / + (glyphs[i][0][3] - glyphs[i][0][1])) for i in raised} + extra = sum(widths[i] - (glyphs[i][0][2] - glyphs[i][0][0]) + 3 for i in raised) + if (bitmap.width + extra) * bitmap.height > cap or baseline < body: + return None + from PIL import Image + _check(deadline) + normalized = Image.new(bitmap.mode, (bitmap.width + extra, bitmap.height), "white") + source, shift = 0, 0 + for index in raised: + _check(deadline) + x0, y0, x1, y1 = glyphs[index][0] + normalized.paste(bitmap.crop((source, 0, x0, bitmap.height)), (source + shift, 0)) + patch = bitmap.crop((x0, y0, x1, y1)).resize((widths[index], body), Image.Resampling.LANCZOS) + normalized.paste(patch, (x0 + shift, round(baseline) - body)) + shift += widths[index] - (x1 - x0) + 3 + source = x1 + normalized.paste(bitmap.crop((source, 0, bitmap.width, bitmap.height)), (source + shift, 0)) + _check(deadline) + return MathOcrPlan(normalized, "superscript", glyphs, deadline, raised=tuple(raised), + body_height=height, baseline=baseline) diff --git a/worker/python/test_extract_pdf_assets_coverage_acceptance.py b/worker/python/test_extract_pdf_assets_coverage_acceptance.py index 37ddfdf66e..393c3ac153 100644 --- a/worker/python/test_extract_pdf_assets_coverage_acceptance.py +++ b/worker/python/test_extract_pdf_assets_coverage_acceptance.py @@ -6,6 +6,7 @@ """ import importlib.util +from functools import partial import os from pathlib import Path import re @@ -27,22 +28,28 @@ def flat(text): def write_fixture(directory, build, scanned=False): """Generate the same visible source as native text or a 300-dpi scan.""" + return write_paged_fixture(directory, (build,), scanned=scanned) + + +def write_paged_fixture(directory, builders, scanned=False): path = Path(directory) / "synthetic-coverage.pdf" with fitz.open() as source: - page = source.new_page(width=595, height=842) - build(page) + for build in builders: + page = source.new_page(width=595, height=842) + build(page) if scanned: - pixels = page.get_pixmap(dpi=300, colorspace=fitz.csGRAY) with fitz.open() as scan: - target = scan.new_page(width=595, height=842) - target.insert_image(target.rect, stream=pixels.tobytes("png")) + for page in source: + pixels = page.get_pixmap(dpi=300, colorspace=fitz.csGRAY) + target = scan.new_page(width=595, height=842) + target.insert_image(target.rect, stream=pixels.tobytes("png")) scan.save(path) else: source.save(path) return path -def symbol_table(page): +def symbol_table(page, font_name="helv", fontsize=11, exponent="⁹", thresholds=("1.5", "2.0")): """Unicode superscript and relation signs, rather than ASCII lookalikes.""" xs, ys = (40, 220, 390, 555), (80, 112, 152, 192) for x in xs: @@ -50,17 +57,17 @@ def symbol_table(page): for y in ys: page.draw_line((xs[0], y), (xs[-1], y), width=0.7) writer = fitz.TextWriter(page.rect) - font = fitz.Font("helv") + font = fitz.Font(font_name) rows = ( - ("Count (x 10⁹/L)", "Threshold", "SYNTHETIC action"), - ("Example A", "≤ 1.5", "Withhold example A"), - ("Example B", "≥ 2.0", "Review example B"), + (f"Count (x 10{exponent}/L)", "Threshold", "SYNTHETIC action"), + ("Example A", f"≤ {thresholds[0]}", "Withhold example A"), + ("Example B", f"≥ {thresholds[1]}", "Review example B"), ) for row_index, row in enumerate(rows): for column, text in enumerate(row): - # TextWriter supplies MuPDF's bundled Unicode fallback for ⁹; + # TextWriter supplies MuPDF's bundled Unicode fallback for powers; # no host font, downloaded asset or binary fixture is needed. - writer.append((xs[column] + 5, ys[row_index] + 21), text, font=font, fontsize=11) + writer.append((xs[column] + 5, ys[row_index] + 21), text, font=font, fontsize=fontsize) writer.write_text(page) @@ -104,7 +111,87 @@ def staggered_columns(page): ]) -class NativeCoverageAcceptanceTests(unittest.TestCase): +COMPARISON_ROWS = ( + ("Example A", "< 0.4", "Withhold example A"), + ("Example B", "<= 0.04", "Review example B"), + ("Example C", "> 1.2", "Repeat example C"), + ("Example D", ">= 12.0", "Recheck example D"), +) + + +def paired_comparison_table(page): + # These pairs deliberately differ in equality stroke, direction and decimal + # position. Inferring a familiar threshold or upgrading every < to <= fails. + draw_table(page, [40, 230, 390, 555], 80, + ["Count (x 10^9/L)", "Threshold", "SYNTHETIC action"], + [tuple([cell] for cell in row) for row in COMPARISON_ROWS], + merged_header="SYNTHETIC COMPARISON CONTROLS") + + +PAGED_NOTES = ( + ("INITIAL PHASE ONLY", "< 0.4", "Withhold example A", "Only after fasting.", "10^9"), + ("MAINTENANCE ONLY", "> 0.04", "Review example A", "Only after rest.", "10^6"), +) + + +def page_local_note_table(index): + def build(page): + phase, threshold, action, note, exponent = PAGED_NOTES[index] + draw_table(page, [40, 230, 390, 555], 80, + [f"Count (x {exponent}/L)", "Threshold", "SYNTHETIC action"], [ + (["Example A (a)"], [threshold], [action]), + (["Example B"], ["2.0"], ["Repeat example B"]), + ], merged_header=phase) + # The marker is intentionally reused on the next page with a different + # qualifier. A document-wide association must not join the two notes. + page.insert_text((40, 210), f"Note (a): {note}", fontsize=11) + return build + + +class CoverageAssertions: + def assert_comparison_cells(self, text): + self.assertIn("Count (x 10^9/L)", text) + positions = [] + for row in COMPARISON_ROWS: + marker = " | ".join(row) + self.assertEqual(text.count(marker), 1, marker) + positions.append(text.index(marker)) + self.assertEqual(positions, sorted(positions)) + + def assert_page_local_notes(self, pages): + self.assertEqual(len(pages), 2) + for index, page in enumerate(pages): + with self.subTest(physical_page=index + 1): + phase, threshold, action, note, exponent = PAGED_NOTES[index] + other_phase, other_threshold, other_action, other_note, other_exponent = PAGED_NOTES[1 - index] + text = flat(page["text"]) + row = f"Example A (a) | {threshold} | {action}" + footnote = f"Note (a): {note}" + for marker in (phase, f"Count (x {exponent}/L)", row, footnote): + self.assertEqual(text.count(marker), 1, marker) + self.assertLess(text.index(phase), text.index(row)) + self.assertLess(text.index(row), text.index(footnote)) + for foreign in (other_phase, other_threshold, other_action, other_note, other_exponent): + self.assertNotIn(foreign, text) + self.assertEqual(page["pageNumber"], index + 1) + self.assertEqual(page["metadata"]["source_provenance"]["physicalPageNumber"], index + 1) + + +class NativeCoverageAcceptanceTests(CoverageAssertions, unittest.TestCase): + def test_paired_comparators_keep_decimal_positions_and_owning_actions(self): + with tempfile.TemporaryDirectory() as directory: + path = write_fixture(directory, paired_comparison_table) + result = extractor.extract(str(path), str(Path(directory) / "out")) + self.assert_comparison_cells(flat(result["pages"][0]["text"])) + self.assertFalse(result["pages"][0]["ocrUsed"]) + + def test_repeated_headers_and_footnote_markers_keep_page_local_owners(self): + with tempfile.TemporaryDirectory() as directory: + path = write_paged_fixture(directory, tuple(page_local_note_table(index) for index in range(2))) + result = extractor.extract(str(path), str(Path(directory) / "out")) + self.assert_page_local_notes(result["pages"]) + self.assertTrue(all(not page["ocrUsed"] for page in result["pages"])) + def test_mid_table_merged_qualifiers_keep_their_row_groups(self): with tempfile.TemporaryDirectory() as directory: path = write_fixture(directory, merged_qualifier_table) @@ -136,20 +223,31 @@ def test_native_unicode_exponent_and_inclusive_comparators_are_preserved(self): @unittest.skipUnless(shutil.which("tesseract") and importlib.util.find_spec("pytesseract") and importlib.util.find_spec("PIL"), "Local OCR dependencies are required") -class RealOcrCoverageAcceptanceTests(unittest.TestCase): +class RealOcrCoverageAcceptanceTests(CoverageAssertions, unittest.TestCase): def extract_scan(self, build): + return flat(self.extract_scan_pages((build,))[0]["text"]) + + def extract_scan_pages(self, builders): with tempfile.TemporaryDirectory() as directory: - path = write_fixture(directory, build, scanned=True) + path = write_paged_fixture(directory, builders, scanned=True) with patch.dict(os.environ, {"PDF_RASTER_TABLE_OCR": "1"}): result = extractor.extract(str(path), str(Path(directory) / "out")) - page = result["pages"][0] - self.assertTrue(page["ocrUsed"]) - self.assertFalse(page["needsOcr"]) - provenance = page["metadata"]["source_provenance"] - self.assertEqual(provenance["ocr"]["status"], "unverified") - self.assertEqual(provenance["ocr"]["comparison"], "not_compared") - self.assertIsNone(provenance["ocr"]["confidence"]) - return flat(page["text"]) + self.assertEqual(len(result["pages"]), len(builders)) + for page in result["pages"]: + self.assertTrue(page["ocrUsed"]) + self.assertFalse(page["needsOcr"]) + provenance = page["metadata"]["source_provenance"] + self.assertEqual(provenance["ocr"]["status"], "unverified") + self.assertEqual(provenance["ocr"]["comparison"], "not_compared") + self.assertIsNone(provenance["ocr"]["confidence"]) + return result["pages"] + + def test_scanned_paired_comparators_keep_decimal_positions_and_owning_actions(self): + self.assert_comparison_cells(self.extract_scan(paired_comparison_table)) + + def test_scanned_repeated_headers_and_footnote_markers_keep_page_local_owners(self): + self.assert_page_local_notes(self.extract_scan_pages( + tuple(page_local_note_table(index) for index in range(2)))) def test_staggered_outside_columns_keep_qualifiers_and_read_left_first(self): text = self.extract_scan(staggered_columns) @@ -162,11 +260,6 @@ def test_staggered_outside_columns_keep_qualifiers_and_read_left_first(self): self.assertIn("Example A | Withhold example A | Day 3", text) self.assertIn("Example B | Review example B | Day 5", text) - # Unresolved recognition defect: the pinned Latin vocabulary lacks these - # glyphs, and bounded crop/model/segmentation diagnostics did not recover - # equivalent semantics. Keep every strict assertion; report this expected - # failure separately. Unexpected success forces reassessment on an upgrade. - @unittest.expectedFailure def test_scanned_unicode_exponent_and_inclusive_comparators_are_preserved(self): text = self.extract_scan(symbol_table) # Accept either unambiguous exponent spelling, never baseline 109 or @@ -179,6 +272,58 @@ def test_scanned_unicode_exponent_and_inclusive_comparators_are_preserved(self): with self.subTest(source_semantics=name): self.assertRegex(text, pattern) + def test_scanned_unicode_math_variants_keep_values_and_observe_input_transforms(self): + variants = ( + ("tiro", 12, "⁶", "6", ("0.4", "3.25")), + ("cour", 10, "³", "3", ("0.04", "12.0")), + ) + pages = self.extract_scan_pages(tuple( + partial(symbol_table, font_name=font, fontsize=size, exponent=exponent, thresholds=thresholds) + for font, size, exponent, _power, thresholds in variants + )) + for page, (font, size, exponent, power, thresholds) in zip(pages, variants): + with self.subTest(font=font, fontsize=size, power=power, thresholds=thresholds): + text = flat(page["text"]) + patterns = ( + rf"Count \(x 10(?:{re.escape(exponent)}|\^{power})/L\)", + rf"Example A \| ≤ {re.escape(thresholds[0])} \| Withhold example A", + rf"Example B \| ≥ {re.escape(thresholds[1])} \| Review example B", + ) + for pattern in patterns: + self.assertEqual(len(re.findall(pattern, text)), 1, pattern) + observations = page["metadata"]["raster_table_ocr"] + self.assertEqual(observations["status"], "unverified") + self.assertEqual(extractor.DEFAULT_BUDGET["maxRenderPixels"], 4_000_000) + self.assertEqual(extractor.MAX_OCR_RENDER_SCALE, 450 / 72) + self.assertEqual(observations["cellModel"], { + "language": "Latin", + "sha256": "sha256:6dbdaf8ecc6c40f025c2648bf3b3f3fbffe073e1fd2df2047fde2e2b2f020d53", + "bytes": 89_384_811, + }) + transformed = [crop for crop in observations["crops"] if "ocrInputTransform" in crop] + self.assertEqual(sorted(crop["ocrInputTransform"]["kind"] for crop in transformed), + ["comparison", "comparison", "superscript"]) + for crop in observations["crops"]: + self.assertGreater(crop["width"], 0) + self.assertGreater(crop["height"], 0) + self.assertLessEqual(crop["width"] * crop["height"], + extractor.DEFAULT_BUDGET["maxRenderPixels"]) + self.assertLessEqual(crop["scale"], extractor.MAX_OCR_RENDER_SCALE) + for crop in transformed: + transform = crop["ocrInputTransform"] + self.assertEqual(transform["algorithm"], "raster-math-topology-v1") + self.assertRegex(crop["pngSha256"], r"\Asha256:[0-9a-f]{64}\Z") + self.assertRegex(crop["pixelsSha256"], r"\Asha256:[0-9a-f]{64}\Z") + self.assertRegex(transform["pixelsSha256"], r"\Asha256:[0-9a-f]{64}\Z") + self.assertRegex(transform["codeSha256"], r"\Asha256:[0-9a-f]{64}\Z") + self.assertEqual(transform["codeSha256"], + extractor.file_sha256(Path(extractor.__file__).with_name("raster_math_ocr.py"))) + self.assertNotEqual(transform["pixelsSha256"], crop["pixelsSha256"]) + self.assertGreater(transform["width"], 0) + self.assertGreater(transform["height"], 0) + self.assertLessEqual(transform["width"] * transform["height"], + extractor.DEFAULT_BUDGET["maxRenderPixels"]) + if __name__ == "__main__": unittest.main() diff --git a/worker/python/test_extract_pdf_assets_math_integration.py b/worker/python/test_extract_pdf_assets_math_integration.py new file mode 100644 index 0000000000..1ea5c44a4e --- /dev/null +++ b/worker/python/test_extract_pdf_assets_math_integration.py @@ -0,0 +1,88 @@ +"""Mocked producer wiring: one OCR call, exact crop identities and fail-closed restoration.""" +import hashlib +import os +import time +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch +from PIL import Image +import extract_pdf_assets as extractor +import raster_math_ocr + + +class MathProducerIntegrationTests(unittest.TestCase): + def setUp(self): + self.document = extractor.fitz.open() + self.native_page = self.document.new_page(width=100, height=100) + self.native_page.insert_text((12, 40), "SYNTHETIC") + self.page = SimpleNamespace(rect=self.native_page.rect, rotation=0, + get_text=lambda *_: "", get_pixmap=self.native_page.get_pixmap, number=0) + self.input_bitmap = Image.new("RGB", (30, 20), "white") + self.input_bitmap.putpixel((10, 10), (0, 0, 0)) + self.plan = SimpleNamespace(bitmap=self.input_bitmap, kind="comparison", restore=Mock(return_value="≤ 1.5")) + self.engine = SimpleNamespace(image_to_string=Mock(return_value="1.5")) + + def tearDown(self): + self.document.close() + + def patches(self): + from contextlib import ExitStack + stack = ExitStack() + stack.enter_context(patch.dict(os.environ, {"PDF_RASTER_TABLE_OCR": "1"})) + stack.enter_context(patch.object(extractor, "checked_raster_table_model", return_value="/unused")) + stack.enter_context(patch.object(extractor, "observed_raster_runtime", return_value={})) + def grid(image, deadline=None, grid_bounds=None): + if grid_bounds is not None: + grid_bounds.update(top=0, bottom=image.height) + scale = image.width / 100 + return [[(scale, scale, 99 * scale, 99 * scale)]] + stack.enter_context(patch.object(extractor, "raster_ruled_cells", side_effect=grid)) + stack.enter_context(patch.object(extractor, "raster_ink_line_count", return_value=1)) + self.planner = stack.enter_context(patch.object(raster_math_ocr, "plan_raster_math_ocr", return_value=self.plan)) + return stack + + def test_one_ocr_call_keeps_source_crop_and_transformed_input_identities(self): + observations = {} + budget = extractor.ExtractionBudget() + with self.patches(): + text = extractor.maybe_raster_table_ocr(self.page, Image.new("RGB", (100, 100)), 1, + budget, self.engine, time.monotonic() + 30, observations) + self.assertEqual(text, "≤ 1.5") + self.engine.image_to_string.assert_called_once() + self.assertIs(self.engine.image_to_string.call_args.args[0], self.input_bitmap) + self.assertIn("--psm 7", self.engine.image_to_string.call_args.kwargs["config"]) + self.assertNotIn("whitelist", self.engine.image_to_string.call_args.kwargs["config"]) + rect = extractor.fitz.Rect(1, 1, 99, 99) + scale = extractor.bounded_clip_render_scale(rect, extractor.MAX_OCR_RENDER_SCALE, budget.limits["maxRenderPixels"]) + source = self.native_page.get_pixmap(matrix=extractor.fitz.Matrix(scale, scale), clip=rect, alpha=False) + crop = observations["crops"][0] + self.assertEqual(crop["pngSha256"], "sha256:" + hashlib.sha256(source.tobytes("png")).hexdigest()) + self.assertEqual(crop["pixelsSha256"], "sha256:" + hashlib.sha256(source.samples).hexdigest()) + transform = crop["ocrInputTransform"] + self.assertEqual(transform["pixelsSha256"], "sha256:" + hashlib.sha256(self.input_bitmap.tobytes()).hexdigest()) + self.assertEqual(transform["codeSha256"], extractor.file_sha256(raster_math_ocr.__file__)) + self.assertNotEqual(transform["pixelsSha256"], crop["pixelsSha256"]) + self.assertEqual((transform["width"], transform["height"], transform["mode"]), (30, 20, "RGB")) + self.assertEqual(observations["status"], "unverified") + # Outward-rounded pixels are excluded geometrically, without noise deletion. + analyzed = self.planner.call_args.args[0] + self.assertEqual(analyzed.size, (611, 611)) + + def test_failed_restoration_discards_candidate_and_keeps_original_page_fallback(self): + self.plan.restore.side_effect = ValueError("raster math source alignment is ambiguous") + self.engine.image_to_string.side_effect = ["1.5", "Original full-page source."] + observations = {"stale": "must be removed"} + with self.patches(), patch.dict("sys.modules", {"pytesseract": self.engine}): + text, warning = extractor.maybe_ocr_page(self.page, extractor.ExtractionBudget(), observations) + self.assertEqual(text, "Original full-page source.") + self.assertIn("raster_table_ocr_refused", warning) + self.assertEqual(observations, {}) + self.assertEqual(self.engine.image_to_string.call_count, 2) + calls = self.engine.image_to_string.call_args_list + self.assertIs(calls[0].args[0], self.input_bitmap) + self.assertNotIn("lang", calls[1].kwargs) + self.assertNotIn("--psm", calls[1].kwargs["config"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/worker/python/test_extract_pdf_assets_raster_tables.py b/worker/python/test_extract_pdf_assets_raster_tables.py index 835458b033..f5042132aa 100644 --- a/worker/python/test_extract_pdf_assets_raster_tables.py +++ b/worker/python/test_extract_pdf_assets_raster_tables.py @@ -555,7 +555,7 @@ def test_gated_merged_scan_keeps_exponent_cells_and_owning_rows(self): for cells, row_text in ((MERGED_ROWS[0], text[wbc:neutrophils]), (MERGED_ROWS[1], text[neutrophils:])): for lines in cells: self.assertIn(" ".join(lines), row_text) - self.assertEqual(result["extractor"]["version"], "2026-10-21.form-mask-headers-v12") + self.assertEqual(result["extractor"]["version"], "2026-10-21.raster-math-layout-v13") provenance = result["pages"][0]["metadata"]["source_provenance"] self.assertEqual(provenance["ocr"]["status"], "unverified") self.assertIsNone(provenance["ocr"]["confidence"]) diff --git a/worker/python/test_raster_math_ocr.py b/worker/python/test_raster_math_ocr.py new file mode 100644 index 0000000000..fd92afdbaa --- /dev/null +++ b/worker/python/test_raster_math_ocr.py @@ -0,0 +1,158 @@ +"""Source-pixel geometry controls with supplied OCR text, never OCR execution. + +These tests verify alignment and abstention contracts, not general recognition +accuracy. Independent fonts, values and units avoid a fixture-specific repair. +""" + +import time +import unittest +from unittest.mock import patch + +from PIL import Image, ImageDraw, ImageOps +import extract_pdf_assets as extractor +import raster_math_ocr as math_ocr + + +def render(text, font="helv", size=11): + with extractor.fitz.open() as document: + page = document.new_page(width=180, height=55) + writer = extractor.fitz.TextWriter(page.rect) + writer.append((5, 32), text, font=extractor.fitz.Font(font), fontsize=size) + writer.write_text(page) + pixels = page.get_pixmap(dpi=450, colorspace=extractor.fitz.csGRAY) + bitmap = Image.frombytes("L", (pixels.width, pixels.height), pixels.samples) + bounds = ImageOps.invert(bitmap).getbbox() + return ImageOps.expand(bitmap.crop(bounds), border=15, fill=255) + + +def plan(bitmap, cap=400_000): + return math_ocr.plan_raster_math_ocr(bitmap, time.monotonic() + 5, cap) + + +class RasterMathPlanTests(unittest.TestCase): + def test_comparisons_keep_exact_polarity_and_observed_decimal_positions(self): + for font, size in (("helv", 10), ("tiro", 13), ("cour", 11)): + for sign in ("≤", "≥", "<", ">"): + for numeric in ("7.25", "725", "0.08", "8.0"): + with self.subTest(font=font, size=size, sign=sign, numeric=numeric): + bitmap = render(sign + " " + numeric, font, size) + before = bitmap.tobytes() + candidate = plan(bitmap) + self.assertIsNotNone(candidate) + self.assertEqual(candidate.kind, "comparison") + self.assertEqual(candidate.restore(numeric), sign + " " + numeric) + self.assertEqual(bitmap.tobytes(), before) + self.assertLessEqual(candidate.bitmap.width * candidate.bitmap.height, 400_000) + + def test_decimal_addition_removal_and_relocation_are_refused(self): + for source, recognized in (("1.5", "15"), ("15", "1.5"), ("1.25", "12.5")): + with self.subTest(source=source, recognized=recognized): + candidate = plan(render("≤ " + source)) + self.assertIsNotNone(candidate) + with self.assertRaisesRegex(ValueError, "raster math"): + candidate.restore(recognized) + + def test_nonnumeric_suffix_is_never_forced_to_a_number(self): + candidate = plan(render("≤ A")) + if candidate is not None: + with self.assertRaises(ValueError): + candidate.restore("A") + + def test_ascii_equals_and_carets_keep_original_path(self): + for text in ("<= 0.04", ">= 12.0", "Density (x 10^6/U)", "N 10^9/Z"): + with self.subTest(text=text): + self.assertIsNone(plan(render(text))) + + def test_ordinary_letters_never_become_chevrons(self): + for text in ("Count", "Cell 105", "O 15", "C 1.5", "S 15", "= 3.1", "v 5.0", "_ 5.0"): + with self.subTest(text=text): + self.assertIsNone(plan(render(text))) + + def test_source_superscript_indices_preserve_other_digits_and_units(self): + superscripts = "¹²³⁴⁵⁶⁷⁸⁹" + for font, size in (("helv", 10), ("tiro", 12), ("cour", 14)): + for digit, superscript in zip("123456789", superscripts): + with self.subTest(font=font, size=size, digit=digit): + source = "N 10" + superscript + " / Z" + bitmap = render(source, font, size) + before = bitmap.tobytes() + candidate = plan(bitmap) + self.assertIsNotNone(candidate) + self.assertEqual(candidate.kind, "superscript") + self.assertEqual(candidate.restore("N 10" + digit + " / Z"), source) + self.assertEqual(bitmap.tobytes(), before) + + def test_baseline_digits_and_decimal_values_do_not_gain_superscripts(self): + for text in ("N 109/Z", "109", "1.5", "15", "10.9", "Footnote 9"): + with self.subTest(text=text): + self.assertIsNone(plan(render(text))) + + def test_footnote_digit_retains_raised_glyph_without_inferred_meaning(self): + candidate = plan(render("Note⁹", "tiro", 12)) + self.assertIsNotNone(candidate) + self.assertEqual(candidate.restore("Note9"), "Note⁹") + + def test_circle_star_letter_and_blank_are_not_digit_inputs(self): + for text in ("N 10°/Z", "N 10*/Z", "N 10ª/Z"): + with self.subTest(text=text): + self.assertIsNone(plan(render(text, "helv", 12))) + self.assertIsNone(plan(Image.new("L", (80, 50), 255))) + + def test_zero_and_unknown_recognition_are_refused_after_normalization(self): + candidate = plan(render("N 10⁰/Z", "helv", 12)) + self.assertIsNotNone(candidate) + with self.assertRaises(ValueError): + candidate.restore("N 100/Z") + candidate = plan(render("N 10⁹/Z", "helv", 12)) + for text in ("N 10?/Z", "N 10*/Z", "N 10a/Z", "N 100/Z", "N 1099/Z", "N 10/Z"): + with self.subTest(text=text): + with self.assertRaises(ValueError): + candidate.restore(text) + + def test_multiple_raised_source_positions_restore_independently(self): + candidate = plan(render("N 10⁹/Z 10⁷/R", "helv", 11)) + self.assertIsNotNone(candidate) + self.assertEqual(candidate.restore("N 109/Z 107/R"), "N 10⁹/Z 10⁷/R") + + def test_internal_newlines_cannot_change_source_index_alignment(self): + for source, recognized in (("≤ 1.5", "1.\n5"), ("N 10⁹/Z", "N 10\n9/Z")): + candidate = plan(render(source)) + self.assertIsNotNone(candidate) + with self.assertRaisesRegex(ValueError, "layout"): + candidate.restore(recognized) + + def test_multiline_overlapping_and_noisy_geometry_abstains(self): + bitmap = render("N 10⁹/Z") + double = Image.new("L", (bitmap.width, bitmap.height * 2 + 20), 255) + double.paste(bitmap, (0, 0)) + double.paste(bitmap, (0, bitmap.height + 20)) + self.assertIsNone(plan(double)) + noisy = bitmap.copy() + noisy.putpixel((noisy.width - 1, noisy.height - 1), 0) + self.assertIsNone(plan(noisy)) + + def test_detached_offset_minus_is_not_an_inclusive_sign(self): + bitmap = render("< 7.25") + # A separate, offset annotation below the chevron does not own its width. + draw = ImageDraw.Draw(bitmap) + draw.line((3, bitmap.height - 5, 24, bitmap.height - 5), fill=0, width=3) + self.assertIsNone(plan(bitmap)) + + def test_allocation_and_deadline_limits_precede_work(self): + bitmap = render("N 10⁹/Z") + self.assertIsNone(plan(bitmap, cap=bitmap.width * bitmap.height - 1)) + self.assertIsNone(plan(Image.new("L", (1000, 401), 255))) + candidate = plan(bitmap) + self.assertIsNotNone(candidate) + # A cap which admits the original but not the normalized image abstains. + self.assertIsNone(plan(bitmap, cap=bitmap.width * bitmap.height)) + with patch.object(math_ocr.time, "monotonic", return_value=100): + with self.assertRaises(TimeoutError): + math_ocr.plan_raster_math_ocr(bitmap, 99, 400_000) + candidate.deadline = 99 + with self.assertRaises(TimeoutError): + candidate.restore("N 109/Z") + + +if __name__ == "__main__": + unittest.main() From 56b69dca2487c6f7e6123f7d39e0a520d7946fe6 Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 09:14:06 +0000 Subject: [PATCH 10/16] fix(ocr): condition small raised-digit lines without changing source values --- .../80b7674b-a3ee-42f9-9d2d-a937f65e168a.json | 10 ++++++ .../ca05be88-08ed-4bb1-99f0-fac733f69a98.json | 11 +++++++ .../d5159483-c5d8-404a-8191-5d4c163f97c6.json | 11 +++++++ docs/worker-deploy-runbook.md | 31 ++++++++++++----- worker/python/raster_math_ocr.py | 18 ++++++++-- worker/python/test_raster_math_ocr.py | 33 +++++++++++++++++++ 6 files changed, 102 insertions(+), 12 deletions(-) create mode 100644 docs/outstanding-issues-inbox/80b7674b-a3ee-42f9-9d2d-a937f65e168a.json create mode 100644 docs/outstanding-issues-inbox/ca05be88-08ed-4bb1-99f0-fac733f69a98.json create mode 100644 docs/outstanding-issues-inbox/d5159483-c5d8-404a-8191-5d4c163f97c6.json diff --git a/docs/outstanding-issues-inbox/80b7674b-a3ee-42f9-9d2d-a937f65e168a.json b/docs/outstanding-issues-inbox/80b7674b-a3ee-42f9-9d2d-a937f65e168a.json new file mode 100644 index 0000000000..f759acf557 --- /dev/null +++ b/docs/outstanding-issues-inbox/80b7674b-a3ee-42f9-9d2d-a937f65e168a.json @@ -0,0 +1,10 @@ +{ + "version": 2, + "id": "80b7674b-a3ee-42f9-9d2d-a937f65e168a", + "createdOn": "2026-10-21", + "action": "cancel", + "payload": { + "requestId": "d5159483-c5d8-404a-8191-5d4c163f97c6", + "reason": "Completion was queued before final independent-font acceptance settled. Courier subtest failed; preserve the strict case and diagnose before queuing any replacement completion." + } +} diff --git a/docs/outstanding-issues-inbox/ca05be88-08ed-4bb1-99f0-fac733f69a98.json b/docs/outstanding-issues-inbox/ca05be88-08ed-4bb1-99f0-fac733f69a98.json new file mode 100644 index 0000000000..baa1b0d4d1 --- /dev/null +++ b/docs/outstanding-issues-inbox/ca05be88-08ed-4bb1-99f0-fac733f69a98.json @@ -0,0 +1,11 @@ +{ + "version": 2, + "id": "ca05be88-08ed-4bb1-99f0-fac733f69a98", + "createdOn": "2026-10-21", + "action": "done", + "payload": { + "id": "#MR9VJK", + "outcome": "Resolved the captured default-off ruled-table OCR regression through bounded source-pixel comparison isolation and raised-position normalization, with source-preserving whole-line conditioning for small-font unit fidelity. The original Unicode assertions are unchanged and now ordinary passing tests. Final targeted correction: 21 tests passed, 94.194 seconds, zero skips/expected failures, including serif/Courier values and powers. Independent actual Courier controls retain both baseline 1 and L. The preceding domain run supplied 117 unchanged passing cases; its Courier failure was corrected, not waived. Strict source alignment, original/input/helper hashes, one OCR call, deadlines, caps and fallback cleanup remain. General clinical-corpus accuracy, raised zero/degree and uncovered/noisy layouts are not certified. No live ingestion/reindex or paid inference was performed.", + "targetUlid": "01M5EQWSKTMR9VJKFCS6GVZQK1" + } +} diff --git a/docs/outstanding-issues-inbox/d5159483-c5d8-404a-8191-5d4c163f97c6.json b/docs/outstanding-issues-inbox/d5159483-c5d8-404a-8191-5d4c163f97c6.json new file mode 100644 index 0000000000..6198c5a4af --- /dev/null +++ b/docs/outstanding-issues-inbox/d5159483-c5d8-404a-8191-5d4c163f97c6.json @@ -0,0 +1,11 @@ +{ + "version": 2, + "id": "d5159483-c5d8-404a-8191-5d4c163f97c6", + "createdOn": "2026-10-21", + "action": "done", + "payload": { + "id": "#MR9VJK", + "outcome": "The reported ruled-cell Unicode reproduction now passes unchanged strict assertions as an ordinary test, with two further fonts, distinct thresholds and powers. Producer v13 isolates source comparison strokes and normalizes source-positioned raised digits using the same pinned model and existing OCR call; decimal/glyph alignment, helper/input hashes, shared deadlines and fallback cleanup are tested. This closes the captured default-off raster-table regression, not general OCR accuracy or clinical-corpus acceptance. Raised zero/degree and ambiguous or uncovered layouts remain unsupported; OCR provenance remains unverified/not_compared. No production ingestion, reindex or paid inference was run.", + "targetUlid": "01M5EQWSKTMR9VJKFCS6GVZQK1" + } +} diff --git a/docs/worker-deploy-runbook.md b/docs/worker-deploy-runbook.md index 8fc3ad2f23..6fc9d3f886 100644 --- a/docs/worker-deploy-runbook.md +++ b/docs/worker-deploy-runbook.md @@ -33,15 +33,28 @@ no isolated memory limit inside the worker. Restore the Python runtime before retrying the document. The developer fallback retains its deadline and pixel limits; those limits do not establish a process memory boundary. -Offline OCR acceptance currently covers native merged qualifications and Unicode -symbols, scanned staggered columns, and the existing ruled-table cases. It does -not establish reliable scanned mathematical notation: the pinned Latin model's -vocabulary lacks `≤`, `≥` and `⁹`, and focused recognition trials lost equality -strokes, an exponent and a decimal. The strict scanned-symbol acceptance case is -recorded as an expected failure, separately from passing coverage. Keep these -OCR observations unverified; require source review or a demonstrated OCR-engine -improvement before relying on such numeric evidence. Changing image-size limits -does not address this model limitation. +Offline OCR acceptance covers native merged qualifications, scanned staggered +columns, opposing comparison signs and decimal positions, and repeated headers +with page-local footnotes. The original scanned `≤`, `≥`, exponent and decimal +regression is now an ordinary strict acceptance case, with additional font and +value variants. These are synthetic source-retention checks, not clinical-corpus +certification. + +The optional raster-table reader can isolate an observed comparison sign before +numeric OCR and normalize an observed raised digit before restoring its source +position. It never guesses a threshold from a medicine or unit. Only small, +single-line, disjoint glyphs are supported; decimal positions and glyph counts +must agree with the source. Ambiguous alignment refuses the candidate and keeps +the original full-page fallback. Raised zero is unsupported because a degree +sign was misread as zero during testing. Unknown layouts, noisy scans and +uncovered fonts retain their existing limitations. + +This uses the existing OCR call, pinned model, page deadline and resource caps. +Source crop hashes remain unchanged; `ocrInputTransform` separately records the +algorithm, helper-code hash and exact transformed input identity. All OCR +observations remain `unverified` / `not_compared`, without confidence or clinical +approval. `PDF_RASTER_TABLE_OCR` remains default-off. Review source accuracy before +clinical use or any mass reindex; increasing image-size limits is not acceptance. --- diff --git a/worker/python/raster_math_ocr.py b/worker/python/raster_math_ocr.py index 2baee2ffe9..b96a3248e2 100644 --- a/worker/python/raster_math_ocr.py +++ b/worker/python/raster_math_ocr.py @@ -136,7 +136,7 @@ def plan_raster_math_ocr(bitmap, deadline, max_pixels): """ _check(deadline) cap = min(MAX_MATH_PIXELS, max_pixels) - if bitmap.width * bitmap.height > cap: + if bitmap.mode not in ("L", "RGB") or bitmap.width * bitmap.height > cap: return None ink = bitmap.convert("L").point(lambda value: 255 if value < 128 else 0) _check(deadline) @@ -195,7 +195,7 @@ def plan_raster_math_ocr(bitmap, deadline, max_pixels): extra = sum(widths[i] - (glyphs[i][0][2] - glyphs[i][0][0]) + 3 for i in raised) if (bitmap.width + extra) * bitmap.height > cap or baseline < body: return None - from PIL import Image + from PIL import Image, ImageChops, ImageOps _check(deadline) normalized = Image.new(bitmap.mode, (bitmap.width + extra, bitmap.height), "white") source, shift = 0, 0 @@ -209,5 +209,17 @@ def plan_raster_math_ocr(bitmap, deadline, max_pixels): source = x1 normalized.paste(bitmap.crop((source, 0, bitmap.width, bitmap.height)), (source + shift, 0)) _check(deadline) - return MathOcrPlan(normalized, "superscript", glyphs, deadline, raised=tuple(raised), + # Small source fonts can confuse ordinary body glyphs after the raised + # component is enlarged. Condition the entire line equally: remove only + # exactly white margins, double resolution, then add a 40-pixel white border + # (20 source pixels). No letter or digit is classified or substituted here. + bounds = ImageChops.difference(normalized, Image.new(bitmap.mode, normalized.size, "white")).getbbox() + ink_width, ink_height = bounds[2] - bounds[0], bounds[3] - bounds[1] + if (ink_width + 40) * (ink_height + 40) * 4 > cap: + return None + _check(deadline) + conditioned = ImageOps.expand(normalized.crop(bounds).resize( + (ink_width * 2, ink_height * 2), Image.Resampling.LANCZOS), border=40, fill="white") + _check(deadline) + return MathOcrPlan(conditioned, "superscript", glyphs, deadline, raised=tuple(raised), body_height=height, baseline=baseline) diff --git a/worker/python/test_raster_math_ocr.py b/worker/python/test_raster_math_ocr.py index fd92afdbaa..eb0ee0b0e5 100644 --- a/worker/python/test_raster_math_ocr.py +++ b/worker/python/test_raster_math_ocr.py @@ -153,6 +153,39 @@ def test_allocation_and_deadline_limits_precede_work(self): with self.assertRaises(TimeoutError): candidate.restore("N 109/Z") + def test_superscript_conditioning_enlarges_full_line_and_preserves_source(self): + source = render("Count (x 10³/L)", "cour", 10) + before = source.tobytes() + candidate = plan(source) + self.assertIsNotNone(candidate) + bounds = ImageOps.invert(candidate.bitmap).getbbox() + self.assertGreaterEqual(bounds[0], 40) + self.assertGreaterEqual(bounds[1], 40) + self.assertLessEqual(bounds[2], candidate.bitmap.width - 40) + self.assertLessEqual(bounds[3], candidate.bitmap.height - 40) + self.assertGreaterEqual(bounds[3] - bounds[1], 2 * candidate.body_height) + self.assertLessEqual(candidate.bitmap.width * candidate.bitmap.height, 400_000) + self.assertEqual(source.tobytes(), before) + # Conditioning changes pixels submitted to OCR, never letter/value text. + self.assertEqual(candidate.restore("Count (x 103/L)"), "Count (x 10³/L)") + self.assertEqual(candidate.restore("Count (x 103/1)"), "Count (x 10³/1)") + + def test_conditioning_cap_is_checked_before_full_line_resizing(self): + source = render("Count (x 10³/L)", "cour", 10) + cap = source.width * source.height + 5000 + actual_resize, calls = Image.Image.resize, [] + + def bounded_resize(image, size, *args, **kwargs): + calls.append(size) + self.assertLessEqual(size[0] * size[1], cap) + return actual_resize(image, size, *args, **kwargs) + + with patch.object(Image.Image, "resize", bounded_resize): + self.assertIsNone(plan(source, cap=cap)) + # The raised component fits; the subsequent full-line doubling must + # abstain before making a resize allocation larger than the caller cap. + self.assertTrue(calls) + if __name__ == "__main__": unittest.main() From faa9f00c643483f13ce2d1f168cb37b44a160a83 Mon Sep 17 00:00:00 2001 From: Joshua Simpson Date: Wed, 21 Oct 2026 10:34:19 +0000 Subject: [PATCH 11/16] docs(ocr): record paired clinical table comparison and activation limits --- docs/audit/2026-10-21-ocr-comparison.json | 465 ++++++++++++++++++++++ docs/audit/2026-10-21-ocr-comparison.md | 162 ++++++++ 2 files changed, 627 insertions(+) create mode 100644 docs/audit/2026-10-21-ocr-comparison.json create mode 100644 docs/audit/2026-10-21-ocr-comparison.md diff --git a/docs/audit/2026-10-21-ocr-comparison.json b/docs/audit/2026-10-21-ocr-comparison.json new file mode 100644 index 0000000000..efdf46ece4 --- /dev/null +++ b/docs/audit/2026-10-21-ocr-comparison.json @@ -0,0 +1,465 @@ +{ + "kind": "offline_paired_ocr_comparison", + "date": "2026-10-21", + "tested_product_head": "56b69dca2487c6f7e6123f7d39e0a520d7946fe6", + "recommendation": "Keep PDF_RASTER_TABLE_OCR default-off; limited real engagement and no demonstrated clinical-number superiority", + "cohort": { + "clinical_full_pages": 10, + "clinical_publication_families": 6, + "mixed_domain_benchmark_table_crops": 20, + "total_cases": 30, + "source_families": 26, + "authentic_field_scans": 0 + }, + "bounds": { + "ocr_page_seconds": 60, + "max_render_pixels": 4000000, + "outer_extraction_seconds": 100, + "max_producer_concurrency": 2 + }, + "actual_run": { + "unique_paired_cli_runs": 60, + "completed": 60, + "wall_seconds": 527.479, + "byte_identical_pairs": 29, + "accepted_enhanced_cases": 1, + "explicit_candidate_runtime_error_fallbacks": 3 + }, + "frozen_input_hashes": { + "manifest_sha256": "010ef92a7efb5098241a5ccc57a44c0f504b40a308846067f0b105fd0da9e015", + "truth_sha256": "9eb9fb53a7f35dba03fbf43b9d0c7effbf2dabf8dcd8757d27b53ef0045f25c6" + }, + "primary_score": { + "actual_engagement_pages": 1, + "all_checks_correct_pages": { + "baseline": 0, + "enhanced": 1 + }, + "all_roster_checks": { + "baseline_passed": 39, + "both_fail": 140, + "both_pass": 39, + "checks": 180, + "enhanced_losses": 0, + "enhanced_passed": 40, + "enhanced_wins": 1 + }, + "by_engagement": { + "False": { + "baseline_passed": 34, + "both_fail": 140, + "both_pass": 34, + "checks": 174, + "enhanced_losses": 0, + "enhanced_passed": 34, + "enhanced_wins": 0 + }, + "True": { + "baseline_passed": 5, + "both_fail": 0, + "both_pass": 5, + "checks": 6, + "enhanced_losses": 0, + "enhanced_passed": 6, + "enhanced_wins": 1 + } + }, + "by_stratum": { + "benchmark_crop": { + "baseline_passed": 11, + "both_fail": 109, + "both_pass": 11, + "checks": 120, + "enhanced_losses": 0, + "enhanced_passed": 11, + "enhanced_wins": 0 + }, + "clinical_fullpage": { + "baseline_passed": 28, + "both_fail": 31, + "both_pass": 28, + "checks": 60, + "enhanced_losses": 0, + "enhanced_passed": 29, + "enhanced_wins": 1 + } + }, + "by_type": { + "numeric": { + "baseline_passed": 8, + "both_fail": 2, + "both_pass": 8, + "checks": 10, + "enhanced_losses": 0, + "enhanced_passed": 8, + "enhanced_wins": 0 + }, + "qualifier": { + "baseline_passed": 10, + "both_fail": 10, + "both_pass": 10, + "checks": 20, + "enhanced_losses": 0, + "enhanced_passed": 10, + "enhanced_wins": 0 + }, + "row_relation": { + "baseline_passed": 16, + "both_fail": 123, + "both_pass": 16, + "checks": 140, + "enhanced_losses": 0, + "enhanced_passed": 17, + "enhanced_wins": 1 + }, + "symbol_unit": { + "baseline_passed": 5, + "both_fail": 5, + "both_pass": 5, + "checks": 10, + "enhanced_losses": 0, + "enhanced_passed": 5, + "enhanced_wins": 0 + } + }, + "freeze_verified": true, + "interpretation": "Bounded offline sample; inspect exact regressions and source/layout/engagement evidence; no clinical sign-off", + "manifest_sha256": "010ef92a7efb5098241a5ccc57a44c0f504b40a308846067f0b105fd0da9e015", + "observations_sha256": "75d600ea7f9dbeb1bdcd663c707bf717360a8a9bf9691626fa733a33140f971e", + "pages": [ + { + "baseline": 4, + "engaged": false, + "enhanced": 4, + "page_id": "who-htn-p40", + "paired": "tie", + "source_family": "57f6376d5c9bc4ea6c44873625547c8a5443d5a6ad8c8c422733681563db05fb", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 3, + "engaged": false, + "enhanced": 3, + "page_id": "iadpsg-p3", + "paired": "tie", + "source_family": "53ab7af43f9e9f5c90f064f9cca7777a1cdd13bde8a1186f51e1929a0408764b", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 2, + "engaged": false, + "enhanced": 2, + "page_id": "iadpsg-p5", + "paired": "tie", + "source_family": "53ab7af43f9e9f5c90f064f9cca7777a1cdd13bde8a1186f51e1929a0408764b", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 2, + "engaged": false, + "enhanced": 2, + "page_id": "nice-ng133-p16", + "paired": "tie", + "source_family": "e943ba2bdc31c772bb3b1332dfbcd36887e2f4308ac0f4163745f2c2fe6b78ec", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 2, + "engaged": false, + "enhanced": 2, + "page_id": "nice-ng133-p21", + "paired": "tie", + "source_family": "e943ba2bdc31c772bb3b1332dfbcd36887e2f4308ac0f4163745f2c2fe6b78ec", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 5, + "engaged": true, + "enhanced": 6, + "page_id": "nice-ng133-p24", + "paired": "win", + "source_family": "e943ba2bdc31c772bb3b1332dfbcd36887e2f4308ac0f4163745f2c2fe6b78ec", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "cdc-gbs-p16", + "paired": "tie", + "source_family": "8819eb6a23b4bffb949b27199cc26518cf7a2f4cf68a9f7642a2ec0d62d7f067", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "cdc-gbs-p23", + "paired": "tie", + "source_family": "8819eb6a23b4bffb949b27199cc26518cf7a2f4cf68a9f7642a2ec0d62d7f067", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 3, + "engaged": false, + "enhanced": 3, + "page_id": "uspstf-p2", + "paired": "tie", + "source_family": "57b0d389fc692ecbee12edf341aae361c0875eda14fce997401faa4f58a2336d", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 5, + "engaged": false, + "enhanced": 5, + "page_id": "who-acs-p10", + "paired": "tie", + "source_family": "5005890cdc3ec2531d62aefaada7e5f2a9ac8adf5d75ec94ffe35fe133fe7f20", + "stratum": "clinical_fullpage", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "PMC1626454", + "paired": "tie", + "source_family": "PMC1626454", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "PMC2753619", + "paired": "tie", + "source_family": "PMC2753619", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "PMC2759935", + "paired": "tie", + "source_family": "PMC2759935", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "PMC2838834", + "paired": "tie", + "source_family": "PMC2838834", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC3519711", + "paired": "tie", + "source_family": "PMC3519711", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC3826085", + "paired": "tie", + "source_family": "PMC3826085", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "PMC3907710", + "paired": "tie", + "source_family": "PMC3907710", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 3, + "engaged": false, + "enhanced": 3, + "page_id": "PMC4003957", + "paired": "tie", + "source_family": "PMC4003957", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC4172848", + "paired": "tie", + "source_family": "PMC4172848", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC4517499", + "paired": "tie", + "source_family": "PMC4517499", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC4682394", + "paired": "tie", + "source_family": "PMC4682394", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "PMC4776821", + "paired": "tie", + "source_family": "PMC4776821", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC4840965", + "paired": "tie", + "source_family": "PMC4840965", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "PMC5134617", + "paired": "tie", + "source_family": "PMC5134617", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC5198506", + "paired": "tie", + "source_family": "PMC5198506", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC5332562", + "paired": "tie", + "source_family": "PMC5332562", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC5402779", + "paired": "tie", + "source_family": "PMC5402779", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC5577841", + "paired": "tie", + "source_family": "PMC5577841", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 1, + "engaged": false, + "enhanced": 1, + "page_id": "PMC5679144", + "paired": "tie", + "source_family": "PMC5679144", + "stratum": "benchmark_crop", + "total": 6 + }, + { + "baseline": 0, + "engaged": false, + "enhanced": 0, + "page_id": "PMC5897438", + "paired": "tie", + "source_family": "PMC5897438", + "stratum": "benchmark_crop", + "total": 6 + } + ], + "paired_pages": { + "tie": 29, + "win": 1 + }, + "regressions": [], + "roster_pages": 30, + "source_families": 26, + "status_counts": { + "baseline": { + "completed": 30 + }, + "enhanced": { + "completed": 30 + } + }, + "truth_sha256": "9eb9fb53a7f35dba03fbf43b9d0c7effbf2dabf8dcd8757d27b53ef0045f25c6", + "warnings": [] + }, + "reference_limitations": { + "known_checks": [ + "who-acs-p10-c6", + "cdc-gbs-p23-c5", + "cdc-gbs-p23-c6" + ], + "detail": "WHO expected within rather than visible in; CDC references dehyphenate source line wraps. Primary truth and denominator preserved; these are reference/presentation limitations, not established semantic OCR failures." + }, + "posthoc_single_reference_wording_correction": { + "baseline_passed": 40, + "enhanced_passed": 41, + "checks": 180, + "note": "WHO wording correction only; exploratory sensitivity, not primary score" + }, + "evidence_scope": "Selected exact text and adjudicated relationships; not whole-document accuracy or clinical signoff", + "publication_scope": "Report and aggregate counts only; originals, scans, full text, per-check adjudications and scratch tools retained privately" +} diff --git a/docs/audit/2026-10-21-ocr-comparison.md b/docs/audit/2026-10-21-ocr-comparison.md new file mode 100644 index 0000000000..7bfe81e75a --- /dev/null +++ b/docs/audit/2026-10-21-ocr-comparison.md @@ -0,0 +1,162 @@ +# Standard and optional table OCR comparison — 21 October 2026 + +Knowledge and records owns this report. It records an experiment on product +commit `56b69dca2487c6f7e6123f7d39e0a520d7946fe6` for PR #3716. +This publication adds documentation and aggregate results; it changes no product +code, resource limit, extraction default or clinical approval. + +**Recommendation: keep `PDF_RASTER_TABLE_OCR` default-off.** The optional reader +demonstrated a small table transcription benefit, but this sample did not +establish better clinical-number or symbol accuracy across real documents. + +## Cohort and execution + +Thirty cases were processed under both flag settings: 60 full Python producer +CLI runs, with 180 predeclared checks assessed under each setting. + +- Ten full pages from six clinical publications: WHO hypertension, IADPSG + gestational diabetes, NICE NG133, CDC group B streptococcus prevention, USPSTF + aspirin and WHO antenatal corticosteroids. Eight pages contain clinical tables + and two contain dose algorithms. A seventh acquired PDF, NICE NG28, was not + selected because its relevant material was narrative. +- All 20 official PubTabNet example table crops from 20 distinct PMC article IDs. + They include mixed scientific domains and low-resolution text. Footnote bodies + outside the crops are unavailable. They are a separate benchmark stratum, not + 20 clinical guidelines or full pages. +- There were 26 source families. The experiment did not reach a cohort of 30 + full clinical pages from 10 clinical publications and included no authentic + field or camera scans. Full PDF pages were rasterized at 300 dpi. Unmodified + crop pixels were wrapped in PDFs at a declared 72 dpi because their original + effective DPI is unknown; upscaling cannot restore missing source detail. + +The source roster, source-derived checks and input hashes were frozen before +OCR. Coding agents inspected source images before candidate outputs. Agents +adjudicated mode-masked text; an independent agent audited hashes, denominators, +exact observations and the changed clinical page. Output formatting can reveal +the intervention, so label masking is not formal clinical-study blinding. + +Both settings received identical image-only PDF bytes and used the same runtime, +model and limits: 4 million OCR-render pixels and 60 seconds of OCR time per page, +a 100-second outer extraction watchdog and at most two producer subprocesses. +Pair submission order alternated; concurrent execution order was not guaranteed. +All 60 scored runs completed. No case was dropped or selectively retried. Paired +execution took 527.479 seconds; sourcing, reference preparation and adjudication +were additional work. Software/model identities and freeze hashes are retained +in the private evidence; aggregate results are in +[the accompanying JSON](2026-10-21-ocr-comparison.json). + +## Results + +| Selected checks meeting exact-text and required association criteria | Standard | Optional setting | +| --- | ---: | ---: | +| Clinical full pages | 28/60 | 29/60 | +| Benchmark table crops | 11/120 | 11/120 | +| Combined frozen checks | 39/180 | 40/180 | + +There was one check improvement, zero check regressions and 179 tied checks. +At case level there was one improvement and 29 ties. The 29 tied pairs returned +byte-identical text, verified against raw JSON, text files and SHA256 receipts. + +These are selected-check verification counts, not whole-document OCR accuracy +percentages. Nonpasses include actual digit/symbol corruption, strict typography +differences, unverifiable association and known reference limitations. They must +not all be described as clinical errors. Low-resolution mixed-domain crop scores +do not estimate production clinical error rates. Seven exact clinical matches +per mode and six exact benchmark matches per mode were correctly withheld from +passes because their required associations could not be verified. + +The optional reader actually engaged on only one case: NICE NG133 physical +page 24, producing eight cell crops and two outside-prose crops for one table. +Its six checks improved from 5/6 to 6/6: `37 weeks lonwards` became +`37 weeks onwards`, with explicit delimited rows. Independent source and output +reviewers agreed. The selected improvement did not change a clinical number, +unit, inequality sign or superscript. An exploratory review outside the six +checks found an additional `NICE quideline` to `NICE guideline` correction and +no additional enhanced numeric/unit/qualifier regression; this is not a +whole-page accuracy certificate. + +The other 29 optional-setting cases returned standard full-page OCR fallback. +Three receipts explicitly record a candidate `RuntimeError`; the precise +attempt/detector reason is not exposed for the other 26 nonengaged cases. +Nonengagement must not be counted as a completed successful candidate attempt. + +## Remaining source-fidelity failures + +Shared fallback outputs still corrupted important source-visible information: + +| Source/check | Source text | Both returned outputs | +| --- | --- | --- | +| WHO hypertension p40 c2 | `≥20/10 mmHg` | `20/10 mmHg` | +| IADPSG p3 c2 | `≥7.0 mmol/l (126 mg/dl)` | `=7.0 mmol/l (126 mg/dl)` | +| IADPSG p3 c6 threshold fragment | `≥11.1 mmol/l (200 mg/dl)` | `211.1 mmol/l (200 mg/dl)` | +| CDC p16 c2 | `≥18 hours` | `218 hours` | + +These are extraction observations, not clinical recommendations. OCR provenance +remained unverified. Successful process exit and readable prose do not verify +clinical figures. Leaving the current default unchanged does not endorse these +fallback outputs as clinically verified. + +## Reference limitations and audit corrections + +The frozen source truth and denominator remain unchanged. The audit found that +WHO ACS p10 c6 expected `within the next 7 days.` while the source and both +outputs say `in the next 7 days.` An initial adjudication also repeated that +incorrect expected wording as an observation. The superseded adjudication was +retained, the actual observation corrected, and both primary scores reduced by +one. This is a reference transcription mistake, not an OCR defect. A separately +documented posthoc correction of that single reference wording yields 40/180 and +41/180; the paired improvement remains one. + +Two CDC p23 checks use dehyphenated reference words despite visible source/output +line-wrap hyphenation. Their strict failures are reference/presentation +limitations, not established clinical semantic changes. None of these three +checks was silently removed. Three ambiguous benchmark range-symbol checks were +excluded and replaced using source-only inspection before OCR, with an amendment +preserving the originals; two suspected image/HTML discrepancies were not scored. +All 20 benchmark cases remained in the roster. + +Only Unicode NFC and whitespace runs were normalized. Every final claimed exact +match was additionally verified against returned text with numeric boundaries +and ordered fragments: `7.1` cannot match `17.1`, and `5` cannot match `15`, +`0.5` or `5.0`. Twelve literal-boundary controls passed. The scorer passed eight +normalization and three scoring smoke controls. Independent final accounting +found no unresolved discrepancy. Association judgments remain agent review, +not an automated proof of layout or clinical-owner sign-off. + +## Controls and runtime limits + +Fresh separate controls passed: paired native text stayed byte-identical without +OCR; a password-locked PDF was refused with exit 4; five existing mocked producer +tests covered missing-model refusal, transient candidate cleanup, late candidate +rejection, late full-page rejection and failed math-restoration cleanup. These +controls are outside the 30-case accuracy denominator. A frozen manifest note +named four focused tests; five were actually run, adding coverage without +rewriting the experiment's manifest. + +Standard CLI median/max wall times were 11.139/48.117 seconds; optional-setting +median/max were 11.267/64.713 seconds. They include non-OCR artifact work, so +whole-CLI time can exceed the OCR-only 60-second limit. These concurrently +scheduled single attempts are descriptive timings, not a controlled speed +benchmark. Peak RSS was not measured. Time/pixel bounds are execution controls, +not accuracy claims. + +## Follow-up and publication boundary + +The next useful work is to diagnose the three candidate refusals, retain the +source-grounded number/comparator failures as regression evidence, and test +more genuinely eligible clinical tables with complete row/column/qualifier +truth. Broader acceptance still needs more clinical publications, authentic +degraded scans and full footnote context. A later implementation comparison +needs a new frozen run; tuning against this sample does not create a holdout. + +Originals came from pinned public GitHub mirrors and the official PubTabNet +examples. Canonical-publication byte parity and each benchmark article's license +were not independently established. Original PDFs, scans, full extracted text, +per-check evidence, amendments and scratch tools are retained privately in the +task's saved execution artifact rather than redistributed in this commit. + +No production ingestion, reindexing, database operation, provider inference, +deployment, extraction-default change or clinical sign-off occurred. This +publication-only addition did not repeat formatting, tests, lint, typecheck, +builds or hosted-CI observation; prior exact-product-head evidence remains +historical evidence rather than a new gate verdict. From cf2b6e13ee5df3f3ea4dd34b3716bd4f31fe3c8c Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 19:42:47 +0800 Subject: [PATCH 12/16] fix(ocr,chunking): address #3716 review threads under chunker 1.3.1 - raster_math_ocr: a comparison whose suffix is not a bare number (units or text, e.g. "< 0.5 mmol/L", "> 2 weeks") raises ComparisonSuffixUnsupported; the extractor then discards the transformed OCR and re-reads the unchanged source bitmap on the ordinary path instead of failing the page's table OCR. Numeric misalignment still discards the candidate as before. - tests: unit-suffix comparisons fall back; a one-filled-cell row is not a band forcing one chunk; a single oversized row is split with the header repeated. --- tests/chunking.test.ts | 32 +++++++++++++++++++ worker/python/extract_pdf_assets.py | 14 +++++++- worker/python/raster_math_ocr.py | 16 ++++++++-- ...est_extract_pdf_assets_math_integration.py | 15 +++++++++ worker/python/test_raster_math_ocr.py | 18 +++++++++++ 5 files changed, 92 insertions(+), 3 deletions(-) diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index bb03be2cb2..5fa5799fbb 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -945,3 +945,35 @@ describe("table context, footnotes, referrals and oversized rows (chunker 1.3.1) expect(content.match(/Lifeline 13 11 14 www\.lifeline\.org\.au/g)).toHaveLength(2); }); }); + +describe("table row groups under chunker 1.3.1 (PR #3716 review follow-ups)", () => { + const header = ["| Drug | Monitoring |", "| --- | --- |"]; + + it("does not treat a row with one filled cell as a merged band that forces one chunk", () => { + const rows = Array.from({ length: 12 }, (_, i) => `| Agent ${i} | Check level ${i} at review |`); + rows.splice(4, 0, "| Lithium | |"); + const table = [...header, ...rows].join("\n"); + const chunks = chunkTextWithOverlap(table, 200, 0); + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) { + expect(chunk.startsWith(header.join("\n"))).toBe(true); + expect(chunk.length).toBeLessThanOrEqual(200); + } + expect(chunks.some((chunk) => chunk.includes("| Lithium | |"))).toBe(true); + }); + + it("splits a single oversized row instead of refusing or keeping the whole table", () => { + const longRow = `| Agent L | ${"review the plasma level and renal function ".repeat(12).trim()} |`; + const rows = ["| Agent A | Short |", longRow, "| Agent B | Short |"]; + const table = [...header, ...rows].join("\n"); + const chunks = chunkTextWithOverlap(table, 240, 0); + expect(chunks.length).toBeGreaterThan(2); + for (const chunk of chunks) { + expect(chunk.startsWith(header.join("\n"))).toBe(true); + expect(chunk.length).toBeLessThanOrEqual(240); + } + const joined = chunks.join("\n"); + expect(joined).toContain("| Agent A | Short |"); + expect(joined).toContain("| Agent B | Short |"); + }); +}); diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 8cc60526ce..848758e5d0 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -781,6 +781,7 @@ def read_region(rect, cell=False): cell_interior = bitmap.crop(interior) if raster_ink_line_count(cell_interior, deadline) == 1: math_plan = plan_raster_math_ocr(cell_interior, deadline, budget.limits["maxRenderPixels"]) + base_mode = mode if math_plan is not None: mode = 7 config = f"--psm {mode} --tessdata-dir {shlex.quote(model_directory)} " + config @@ -790,7 +791,18 @@ def read_region(rect, cell=False): ocr_bitmap = math_plan.bitmap if math_plan is not None else bitmap text = pytesseract.image_to_string(ocr_bitmap, lang="Latin" if cell else "eng", config=config, timeout=remaining).strip() if math_plan is not None: - text = math_plan.restore(text) + try: + text = math_plan.restore(text) + except raster_math_ocr.ComparisonSuffixUnsupported: + # Units or text after a comparison: discard the transformed OCR + # and read the unchanged source bitmap on the ordinary path. + math_plan = None + config = (f"--psm {base_mode} --tessdata-dir {shlex.quote(model_directory)} " + f"--dpi {round(scale * 72)}") + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("raster math fallback exhausted page OCR timeout") + text = pytesseract.image_to_string(bitmap, lang="Latin", config=config, timeout=remaining).strip() if time.monotonic() >= deadline: raise TimeoutError("raster OCR completed after page deadline") if len(record["crops"]) >= 112: diff --git a/worker/python/raster_math_ocr.py b/worker/python/raster_math_ocr.py index b96a3248e2..cddb4c8970 100644 --- a/worker/python/raster_math_ocr.py +++ b/worker/python/raster_math_ocr.py @@ -1,8 +1,10 @@ """Bounded pixel plans for isolated comparisons and raised digits; OCR stays unverified. No dictionary, units, expected values or source-specific templates are consulted. -Unsupported geometry keeps the original OCR path. A changed bitmap must always -pass restore(), otherwise the caller must discard the complete table candidate. +Unsupported geometry keeps the original OCR path. A comparison whose suffix is +not a bare number (units or text) also returns to the original OCR path on the +unchanged bitmap. Any other restore() failure means the caller must discard the +complete table candidate. """ from dataclasses import dataclass @@ -77,6 +79,14 @@ def _operator(ink, glyph, deadline): return sign +class ComparisonSuffixUnsupported(ValueError): + """The comparison suffix is not a bare number (e.g. a unit or text follows). + + The caller must discard the transformed input and OCR the unchanged source + bitmap on the ordinary path; nothing from the transformed OCR is kept. + """ + + @dataclass class MathOcrPlan: bitmap: object @@ -93,6 +103,8 @@ def restore(self, text): if "\n" in text.strip() or "\r" in text.strip(): raise ValueError("raster math text layout is ambiguous") characters = [char for char in text if not char.isspace()] + if self.kind == "comparison" and any(char not in "0123456789." for char in characters): + raise ComparisonSuffixUnsupported("raster math comparison suffix is not a bare number") if len(characters) != len(self.glyphs): raise ValueError("raster math source alignment is ambiguous") if self.kind == "comparison": diff --git a/worker/python/test_extract_pdf_assets_math_integration.py b/worker/python/test_extract_pdf_assets_math_integration.py index 1ea5c44a4e..930bcde454 100644 --- a/worker/python/test_extract_pdf_assets_math_integration.py +++ b/worker/python/test_extract_pdf_assets_math_integration.py @@ -68,6 +68,21 @@ def test_one_ocr_call_keeps_source_crop_and_transformed_input_identities(self): analyzed = self.planner.call_args.args[0] self.assertEqual(analyzed.size, (611, 611)) + def test_comparison_with_unit_suffix_rereads_unchanged_source_without_transform(self): + self.plan.restore.side_effect = raster_math_ocr.ComparisonSuffixUnsupported("unit suffix") + self.engine.image_to_string.side_effect = ["0.5mmol/L", "< 0.5 mmol/L"] + observations = {} + with self.patches(): + text = extractor.maybe_raster_table_ocr(self.page, Image.new("RGB", (100, 100)), 1, + extractor.ExtractionBudget(), self.engine, time.monotonic() + 30, observations) + self.assertEqual(text, "< 0.5 mmol/L") + calls = self.engine.image_to_string.call_args_list + self.assertEqual(len(calls), 2) + self.assertIs(calls[0].args[0], self.input_bitmap) + self.assertIsNot(calls[1].args[0], self.input_bitmap) + self.assertEqual(calls[1].kwargs["lang"], "Latin") + self.assertNotIn("ocrInputTransform", observations["crops"][0]) + def test_failed_restoration_discards_candidate_and_keeps_original_page_fallback(self): self.plan.restore.side_effect = ValueError("raster math source alignment is ambiguous") self.engine.image_to_string.side_effect = ["1.5", "Original full-page source."] diff --git a/worker/python/test_raster_math_ocr.py b/worker/python/test_raster_math_ocr.py index eb0ee0b0e5..31fc6dad35 100644 --- a/worker/python/test_raster_math_ocr.py +++ b/worker/python/test_raster_math_ocr.py @@ -58,6 +58,24 @@ def test_nonnumeric_suffix_is_never_forced_to_a_number(self): with self.assertRaises(ValueError): candidate.restore("A") + def test_comparison_with_units_or_text_returns_to_original_path(self): + for source, recognized in (("< 0.5 mmol/L", "0.5 mmol/L"), ("> 2 weeks", "2 weeks"), ("≤ 7.25 mg", "7.25mg")): + with self.subTest(source=source): + candidate = plan(render(source)) + self.assertIsNotNone(candidate) + self.assertEqual(candidate.kind, "comparison") + with self.assertRaises(math_ocr.ComparisonSuffixUnsupported): + candidate.restore(recognized) + + def test_unsupported_suffix_is_distinct_from_numeric_misalignment(self): + candidate = plan(render("≤ 1.5")) + self.assertIsNotNone(candidate) + with self.assertRaises(ValueError) as caught: + candidate.restore("15") + self.assertNotIsInstance(caught.exception, math_ocr.ComparisonSuffixUnsupported) + with self.assertRaises(math_ocr.ComparisonSuffixUnsupported): + candidate.restore("1.5 mg") + def test_ascii_equals_and_carets_keep_original_path(self): for text in ("<= 0.04", ">= 12.0", "Density (x 10^6/U)", "N 10^9/Z"): with self.subTest(text=text): From 65f0cfded42b2f03f810ef8db353fc64ebfe2327 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 19:56:34 +0800 Subject: [PATCH 13/16] style(docs): prettier format OCR comparison audit record (whitespace only) --- docs/audit/2026-10-21-ocr-comparison.json | 6 +----- docs/audit/2026-10-21-ocr-comparison.md | 18 +++++++++--------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/audit/2026-10-21-ocr-comparison.json b/docs/audit/2026-10-21-ocr-comparison.json index efdf46ece4..84b5ef953c 100644 --- a/docs/audit/2026-10-21-ocr-comparison.json +++ b/docs/audit/2026-10-21-ocr-comparison.json @@ -447,11 +447,7 @@ "warnings": [] }, "reference_limitations": { - "known_checks": [ - "who-acs-p10-c6", - "cdc-gbs-p23-c5", - "cdc-gbs-p23-c6" - ], + "known_checks": ["who-acs-p10-c6", "cdc-gbs-p23-c5", "cdc-gbs-p23-c6"], "detail": "WHO expected within rather than visible in; CDC references dehyphenate source line wraps. Primary truth and denominator preserved; these are reference/presentation limitations, not established semantic OCR failures." }, "posthoc_single_reference_wording_correction": { diff --git a/docs/audit/2026-10-21-ocr-comparison.md b/docs/audit/2026-10-21-ocr-comparison.md index 7bfe81e75a..c25f0ac148 100644 --- a/docs/audit/2026-10-21-ocr-comparison.md +++ b/docs/audit/2026-10-21-ocr-comparison.md @@ -48,10 +48,10 @@ in the private evidence; aggregate results are in ## Results | Selected checks meeting exact-text and required association criteria | Standard | Optional setting | -| --- | ---: | ---: | -| Clinical full pages | 28/60 | 29/60 | -| Benchmark table crops | 11/120 | 11/120 | -| Combined frozen checks | 39/180 | 40/180 | +| -------------------------------------------------------------------- | -------: | ---------------: | +| Clinical full pages | 28/60 | 29/60 | +| Benchmark table crops | 11/120 | 11/120 | +| Combined frozen checks | 39/180 | 40/180 | There was one check improvement, zero check regressions and 179 tied checks. At case level there was one improvement and 29 ties. The 29 tied pairs returned @@ -84,12 +84,12 @@ Nonengagement must not be counted as a completed successful candidate attempt. Shared fallback outputs still corrupted important source-visible information: -| Source/check | Source text | Both returned outputs | -| --- | --- | --- | -| WHO hypertension p40 c2 | `≥20/10 mmHg` | `20/10 mmHg` | -| IADPSG p3 c2 | `≥7.0 mmol/l (126 mg/dl)` | `=7.0 mmol/l (126 mg/dl)` | +| Source/check | Source text | Both returned outputs | +| ------------------------------- | -------------------------- | -------------------------- | +| WHO hypertension p40 c2 | `≥20/10 mmHg` | `20/10 mmHg` | +| IADPSG p3 c2 | `≥7.0 mmol/l (126 mg/dl)` | `=7.0 mmol/l (126 mg/dl)` | | IADPSG p3 c6 threshold fragment | `≥11.1 mmol/l (200 mg/dl)` | `211.1 mmol/l (200 mg/dl)` | -| CDC p16 c2 | `≥18 hours` | `218 hours` | +| CDC p16 c2 | `≥18 hours` | `218 hours` | These are extraction observations, not clinical recommendations. OCR provenance remained unverified. Successful process exit and readable prose do not verify From 1deb4155a7fd49899915750bbb43d011b0561abb Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 20:01:09 +0800 Subject: [PATCH 14/16] fix(extraction): use main's extractor label merged-cell-provenance-v11 (owner ruling 20:00 AWST) --- tests/extracted-document-validation.test.ts | 2 +- worker/python/extract_pdf_assets.py | 2 +- worker/python/test_extract_pdf_assets_raster_tables.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/extracted-document-validation.test.ts b/tests/extracted-document-validation.test.ts index bbea9b124f..4f5e150b39 100644 --- a/tests/extracted-document-validation.test.ts +++ b/tests/extracted-document-validation.test.ts @@ -146,7 +146,7 @@ describe("Python extracted-document payload validation", () => { describe("extraction provenance (audit F04)", () => { const provenance = { name: "extract_pdf_assets", - version: "2026-10-21.raster-math-layout-v13", + version: "2026-10-21.merged-cell-provenance-v11", pymupdf: "1.26.0", tableStrategy: "table_aware_page_text", }; diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 848758e5d0..b492d99dc0 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -28,7 +28,7 @@ # parser defect can be traced to the documents it touched. Bump the date and label whenever a # change alters the text, tables or images produced for the same PDF. 2026-09-16 is PR #2810, # which replaced line-major table text with table_aware_page_text (#GJHKYC). -EXTRACTOR_VERSION = "2026-10-21.raster-math-layout-v13" +EXTRACTOR_VERSION = "2026-10-21.merged-cell-provenance-v11" TABLE_STRATEGY = "table_aware_page_text" # IDX-H6: caps on serialized table size. Raised from the previous 80/120-row and 8000-char diff --git a/worker/python/test_extract_pdf_assets_raster_tables.py b/worker/python/test_extract_pdf_assets_raster_tables.py index f5042132aa..1b35b80bf1 100644 --- a/worker/python/test_extract_pdf_assets_raster_tables.py +++ b/worker/python/test_extract_pdf_assets_raster_tables.py @@ -555,7 +555,7 @@ def test_gated_merged_scan_keeps_exponent_cells_and_owning_rows(self): for cells, row_text in ((MERGED_ROWS[0], text[wbc:neutrophils]), (MERGED_ROWS[1], text[neutrophils:])): for lines in cells: self.assertIn(" ".join(lines), row_text) - self.assertEqual(result["extractor"]["version"], "2026-10-21.raster-math-layout-v13") + self.assertEqual(result["extractor"]["version"], "2026-10-21.merged-cell-provenance-v11") provenance = result["pages"][0]["metadata"]["source_provenance"] self.assertEqual(provenance["ocr"]["status"], "unverified") self.assertIsNone(provenance["ocr"]["confidence"]) From 379fdac4b5384168c875a59647aa47f9387a0971 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 20:09:11 +0800 Subject: [PATCH 15/16] chore(snapshot): regenerate repo-awareness snapshot for new audit docs --- data/repo-awareness-snapshot.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data/repo-awareness-snapshot.json b/data/repo-awareness-snapshot.json index 105d167828..d30b08b58c 100644 --- a/data/repo-awareness-snapshot.json +++ b/data/repo-awareness-snapshot.json @@ -2918,6 +2918,11 @@ "section": "superpowers", "catalogued": true }, + { + "path": "docs/audit/2026-10-21-ocr-comparison.md", + "section": "audit", + "catalogued": true + }, { "path": "docs/superpowers/rag-upgrade/canonical/START-HERE.cloud.md", "section": "superpowers", From 0ad1dbf5039e262516c65ee7a83c1171ccb1050b Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Wed, 21 Oct 2026 20:14:08 +0800 Subject: [PATCH 16/16] fix(extraction): restore main's merged-cell provenance lost in merge 8519926b2 The second merge resolved extract_pdf_assets.py with the branch side wholesale, dropping main's merged_cells / table_merged_cells hunks (#3715). Re-run the three-way merge for that file; only the version label conflicted. --- worker/python/extract_pdf_assets.py | 38 ++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index b492d99dc0..1f992907bc 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -1568,12 +1568,21 @@ def text_grid_rows(text): def extract_table_payload(table): try: rows = table.extract() - cleaned = [[clean_cell(cell) for cell in row] for row in rows if any(clean_cell(cell) for cell in row)] + cleaned = [] + merged_cells = [] + for row in rows: + if not any(clean_cell(cell) for cell in row): + continue + # PyMuPDF returns None (not "") for a cell covered by a merged span, so + # these coordinates are confirmed merge provenance, not merely blank cells. + merged_cells.extend([len(cleaned), column] for column, cell in enumerate(row) if cell is None) + cleaned.append([clean_cell(cell) for cell in row]) column_count = max((len(row) for row in cleaned), default=0) column_ranges = _extract_table_column_ranges(table, column_count) return { "text": table_rows_to_markdown(rows), "rows": cleaned[:MAX_TABLE_ROWS], + "merged_cells": [cell for cell in merged_cells if cell[0] < MAX_TABLE_ROWS], "columns": table_columns_from_rows(rows)[:24], "accessible_markdown": table_rows_to_markdown(rows), "row_count": len(cleaned), @@ -1586,6 +1595,7 @@ def extract_table_payload(table): return { "text": "", "rows": [], + "merged_cells": [], "columns": [], "accessible_markdown": "", "row_count": 0, @@ -1865,6 +1875,7 @@ def likely_table_candidates(page, rejected_rects=None): "table_role": role, "table_confidence": table_candidate_confidence(label, title, table_text, row_count, column_count), "table_rows": payload.get("rows", []), + "table_merged_cells": payload.get("merged_cells", []), "table_columns": payload.get("columns", []), "accessible_table_markdown": payload.get("accessible_markdown") or table_text, "row_count": row_count, @@ -2000,7 +2011,14 @@ def merge_related_table_candidates(candidates, page_rect): rect = merge_rects([item["rect"] for item in group], page_rect) merged_text = "\n\n".join(item.get("table_text") or "" for item in group).strip() merged_rows = [] + merged_cells = [] + row_breaks = [] for item in group: + offset = len(merged_rows) + if offset: + # Each grouped table starts a new section: label carry-down must not cross it. + row_breaks.append(offset) + merged_cells.extend([row + offset, column] for row, column in item.get("table_merged_cells") or []) merged_rows.extend(item.get("table_rows") or []) candidate = { **candidate, @@ -2013,6 +2031,8 @@ def merge_related_table_candidates(candidates, page_rect): ), "table_confidence": max(item.get("table_confidence") or 0 for item in group), "table_rows": merged_rows[:MAX_TABLE_ROWS], + "table_merged_cells": [cell for cell in merged_cells if cell[0] < MAX_TABLE_ROWS], + "table_row_breaks": [row for row in row_breaks if row < MAX_TABLE_ROWS], "table_columns": (merged_rows[0] if merged_rows else candidate.get("table_columns") or [])[:24], "accessible_table_markdown": table_rows_to_markdown(merged_rows) if merged_rows else merged_text, "row_count": sum(item.get("row_count") or 0 for item in group), @@ -2508,6 +2528,8 @@ def _extract_document(document, output_dir, budget, identity): detected_rect = table_candidate["rect"] edge_rect, edge_info = extend_table_rect_for_edge_content(page, detected_rect) table_rows = list(table_candidate.get("table_rows") or []) + table_merged_cells = [list(cell) for cell in table_candidate.get("table_merged_cells") or []] + table_row_breaks = list(table_candidate.get("table_row_breaks") or []) column_count = int(table_candidate.get("column_count") or (len(table_rows[0]) if table_rows else 0) or 0) recovered_rows = [] unassigned_edge_text = [] @@ -2525,6 +2547,18 @@ def _extract_document(document, output_dir, budget, identity): recovered_rows = top_rows + bottom_rows if recovered_rows: combined_rows = top_rows + table_rows + bottom_rows + table_merged_cells = [ + [row + len(top_rows), column] + for row, column in table_merged_cells + if row + len(top_rows) < MAX_TABLE_ROWS + ] + # Rows recovered past the detected edge (e.g. across a page break) start new segments. + table_row_breaks = sorted( + {row + len(top_rows) for row in table_row_breaks} + | ({len(top_rows)} if top_rows else set()) + | ({len(top_rows) + len(table_rows)} if bottom_rows else set()) + ) + table_row_breaks = [row for row in table_row_breaks if row < MAX_TABLE_ROWS] recovered_truncated = len(combined_rows) > MAX_TABLE_ROWS table_rows = combined_rows[:MAX_TABLE_ROWS] rows_truncated = ( @@ -2555,6 +2589,8 @@ def _extract_document(document, output_dir, budget, identity): "table_role": table_candidate.get("table_role"), "table_confidence": table_candidate.get("table_confidence"), "table_rows": table_rows, + "table_merged_cells": table_merged_cells, + "table_row_breaks": table_row_breaks, "table_columns": table_candidate.get("table_columns") or [], "accessible_table_markdown": accessible, "row_count": len(table_rows) if table_rows else table_candidate.get("row_count"),