From fa3811f5904abc024213dba196e5051d53a6838e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 02:58:09 +0000 Subject: [PATCH 1/5] Stop a note under a recovered table row being glued into that row A lone line beneath a row recovered from a table's edge strip was merged into the row's cell whenever it sat within 16pt. A short note such as 'See note' under 'Clozapine' became 'Clozapine See note'. Now the line joins the row only when the cell above actually wrapped: its text must reach closer to the column's right edge than the width of the next line's first word. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01M6gRtN6r4vZDgDfzMtjs95 --- .../99b5370e-305b-47f4-b0b5-5c81c37f39d3.json | 13 ++++++ tests/pdf-table-extraction.test.ts | 43 +++++++++++++++++++ worker/python/extract_pdf_assets.py | 32 ++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 docs/outstanding-issues-inbox/99b5370e-305b-47f4-b0b5-5c81c37f39d3.json diff --git a/docs/outstanding-issues-inbox/99b5370e-305b-47f4-b0b5-5c81c37f39d3.json b/docs/outstanding-issues-inbox/99b5370e-305b-47f4-b0b5-5c81c37f39d3.json new file mode 100644 index 0000000000..203a47ca6c --- /dev/null +++ b/docs/outstanding-issues-inbox/99b5370e-305b-47f4-b0b5-5c81c37f39d3.json @@ -0,0 +1,13 @@ +{ + "version": 2, + "id": "99b5370e-305b-47f4-b0b5-5c81c37f39d3", + "createdOn": "2026-10-17", + "action": "update", + "payload": { + "id": "#ZWJCK4", + "pri": "P2", + "detail": "Rechecked 17 Oct 2026 against main 99507ebca. The original fault is fixed: PR #3571 made worker/python/extract_pdf_assets.py _recover_table_rows_split admit a recovered line only when each piece fits one column, keep top and bottom strips in order, and tests/pdf-table-extraction.test.ts pins full-width prose being rejected. A leftover was fixed on 17 Oct: a short note under a recovered row (for example 'See note') was glued into that row's first cell; a lone line now joins the row above only when that cell actually wrapped (_continues_wrapped_cell), with tests both ways. Still open, low risk: text laid out in the table's own columns directly under a table (an abbreviation key, or a two-column page) becomes an extra row, and only when a drawing below the table extends the crop over it. Geometry alone cannot tell such a key from a real row. Changes reach search only after a re-index.", + "baseRowFingerprint": "4167a93e573dfb9618b0f1ccf98de7b816ead5366c7370fe5dddd6919e264eae", + "targetUlid": "01M4T67MV6ZWJCK4234NWY7VKF" + } +} diff --git a/tests/pdf-table-extraction.test.ts b/tests/pdf-table-extraction.test.ts index eb5c5b7bbf..6e291570fc 100644 --- a/tests/pdf-table-extraction.test.ts +++ b/tests/pdf-table-extraction.test.ts @@ -94,6 +94,49 @@ describe.runIf(hasPyMuPDF)("PDF table edge-strip row recovery (3344a3a9)", () => ]); }); + function recoverBottomRow(lines: string[]): string[][] { + const script = [ + "import json, sys", + "import pymupdf as fitz", + `sys.path.insert(0, ${JSON.stringify(extractorDir)})`, + "import extract_pdf_assets as e", + "doc = fitz.open()", + "page = doc.new_page(width=595, height=842)", + "original_rect = fitz.Rect(50, 120, 560, 200)", + "extended_rect = fitz.Rect(50, 120, 560, 250)", + "existing_rows = [['0', 'Asleep', 'Monitor vitals']]", + "column_ranges = [(50.0, 120.0), (120.0, 310.0), (310.0, 560.0)]", + "page.insert_text((54, 218), '5', fontsize=9)", + "page.insert_text((124, 218), 'Highly aroused', fontsize=9)", + ...lines, + "recovered = e.recover_table_rows_from_edge_strip(", + " page, original_rect, extended_rect, existing_rows, column_count=3, column_ranges=column_ranges", + ")", + "print(json.dumps(recovered))", + ].join("\n"); + return JSON.parse(execFileSync(pythonBin, ["-c", script], { cwd: process.cwd(), encoding: "utf8" }).trim()); + } + + it("keeps a short note under a recovered row out of that row's cells", () => { + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)", + "page.insert_text((54, 229), 'See note', fontsize=9)", + ]), + ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); + }); + + it("still joins a cell whose text wrapped onto the next line", () => { + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Give oral medication first, then call the duty registrar,', fontsize=9)", + "page.insert_text((314, 229), 'security and nurse in charge', fontsize=9)", + ]), + ).toEqual([ + ["5", "Highly aroused", "Give oral medication first, then call the duty registrar, security and nurse in charge"], + ]); + }); + it("extracts column ranges cleanly when a table row contains (None, None, None, None) merged/empty cells", () => { const script = [ "import json, sys", diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index abc1a2db6d..9e0ddd39e9 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -738,6 +738,22 @@ def _match_segment_to_column(sx0, sx1, effective_ranges): return col_idx +def _continues_wrapped_cell(prev_line_end, segment, column_range, slack=12.0): + """True when a lone line under a row reads as the wrapped remainder of that row's cell. + + Text wraps because its next word did not fit, so the line above must end + closer to the column's right edge than the width of this line's first word. + """ + if prev_line_end is None: + return False + sx0, sx1, text = segment + words = clean_cell(text).split(" ") + if not words or not words[0]: + return False + first_word_width = (sx1 - sx0) * (len(words[0]) / max(1, len(clean_cell(text)))) + return (column_range[1] - prev_line_end) <= first_word_width + slack + + def _recover_table_rows_split(page, original_rect, extended_rect, existing_rows, column_count=0, column_ranges=None): existing_keys = { tuple(clean_cell(cell) for cell in row) @@ -789,11 +805,16 @@ def collect_rows(strip_segments): rows_out = [] prev_band_y1 = None + # Right edge of the last line of text in each column of the latest row, + # used to tell a wrapped cell from a note that merely sits below it. + prev_row_line_ends = {} for band in bands: band.sort(key=lambda s: s[0]) band_y0 = min(s[1] for s in band) band_y1 = max(s[3] for s in band) row_cells = [""] * num_cols + line_ends = {} + first_segment = {} valid_band = True for sx0, _sy0, sx1, _sy1, text in band: col_idx = _match_segment_to_column(sx0, sx1, effective_ranges) @@ -801,11 +822,21 @@ def collect_rows(strip_segments): valid_band = False break row_cells[col_idx] = clean_cell(f"{row_cells[col_idx]} {text}") if row_cells[col_idx] else clean_cell(text) + line_ends[col_idx] = max(line_ends.get(col_idx, sx1), sx1) + first_segment.setdefault(col_idx, (sx0, sx1, text)) if not valid_band: prev_band_y1 = None continue populated_cols = sum(1 for cell in row_cells if clean_cell(cell)) if populated_cols == 1 and rows_out and prev_band_y1 is not None and (band_y0 - prev_band_y1) <= 16.0: + col_idx = next(iter(first_segment)) + if not _continues_wrapped_cell( + prev_row_line_ends.get(col_idx), first_segment[col_idx], effective_ranges[col_idx] + ): + # A short note or caption under the table, not the rest of a wrapped cell. + prev_band_y1 = None + continue + prev_row_line_ends[col_idx] = line_ends[col_idx] prev_row = rows_out[-1] existing_keys.discard(tuple(prev_row)) for idx, val in enumerate(row_cells): @@ -822,6 +853,7 @@ def collect_rows(strip_segments): continue existing_keys.add(key) rows_out.append(row_cells) + prev_row_line_ends = line_ends prev_band_y1 = band_y1 return rows_out From 5543e94d4e6ef74c4cc8bb8a33da684cd6b87e1d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 03:01:22 +0000 Subject: [PATCH 2/5] Keep deliberate line breaks inside a recovered table cell A dose written under a drug name, or a bulleted step, does not reach the column's right edge, so the wrap check dropped it. A line that starts at the cell's own left edge with a bullet, a lower-case word or a number now joins the cell too. A capitalised note such as 'See note' still stays out. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01M6gRtN6r4vZDgDfzMtjs95 --- tests/pdf-table-extraction.test.ts | 10 ++++++++++ worker/python/extract_pdf_assets.py | 23 ++++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/pdf-table-extraction.test.ts b/tests/pdf-table-extraction.test.ts index 6e291570fc..b10260c8c9 100644 --- a/tests/pdf-table-extraction.test.ts +++ b/tests/pdf-table-extraction.test.ts @@ -137,6 +137,16 @@ describe.runIf(hasPyMuPDF)("PDF table edge-strip row recovery (3344a3a9)", () => ]); }); + it("still joins a deliberate line break inside a cell, such as a dose under a drug name", () => { + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Lorazepam', fontsize=9)", + "page.insert_text((314, 229), '1 to 2 mg orally', fontsize=9)", + "page.insert_text((314, 240), '- repeat once after 1 hour', fontsize=9)", + ]), + ).toEqual([["5", "Highly aroused", "Lorazepam 1 to 2 mg orally - repeat once after 1 hour"]]); + }); + it("extracts column ranges cleanly when a table row contains (None, None, None, None) merged/empty cells", () => { const script = [ "import json, sys", diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 9e0ddd39e9..aedf8a4a05 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -738,19 +738,27 @@ def _match_segment_to_column(sx0, sx1, effective_ranges): return col_idx -def _continues_wrapped_cell(prev_line_end, segment, column_range, slack=12.0): - """True when a lone line under a row reads as the wrapped remainder of that row's cell. +def _continues_wrapped_cell(prev_line, segment, column_range, slack=12.0): + """True when a lone line under a row reads as the rest of that row's cell. Text wraps because its next word did not fit, so the line above must end closer to the column's right edge than the width of this line's first word. + A deliberate break also counts: a line starting at the cell's own left edge + with a bullet, a lower-case word or a number (a dose under a drug name). """ - if prev_line_end is None: + if prev_line is None: return False + prev_line_start, prev_line_end = prev_line sx0, sx1, text = segment - words = clean_cell(text).split(" ") + cleaned = clean_cell(text) + words = cleaned.split(" ") if not words or not words[0]: return False - first_word_width = (sx1 - sx0) * (len(words[0]) / max(1, len(clean_cell(text)))) + if abs(sx0 - prev_line_start) <= 3.0 and ( + cleaned[0] in "•·-–*" or cleaned[0].islower() or cleaned[0].isdigit() + ): + return True + first_word_width = (sx1 - sx0) * (len(words[0]) / max(1, len(cleaned))) return (column_range[1] - prev_line_end) <= first_word_width + slack @@ -805,7 +813,7 @@ def collect_rows(strip_segments): rows_out = [] prev_band_y1 = None - # Right edge of the last line of text in each column of the latest row, + # Left and right edges of the last line of text in each column of the latest row, # used to tell a wrapped cell from a note that merely sits below it. prev_row_line_ends = {} for band in bands: @@ -822,7 +830,8 @@ def collect_rows(strip_segments): valid_band = False break row_cells[col_idx] = clean_cell(f"{row_cells[col_idx]} {text}") if row_cells[col_idx] else clean_cell(text) - line_ends[col_idx] = max(line_ends.get(col_idx, sx1), sx1) + start, end = line_ends.get(col_idx, (sx0, sx1)) + line_ends[col_idx] = (min(start, sx0), max(end, sx1)) first_segment.setdefault(col_idx, (sx0, sx1, text)) if not valid_band: prev_band_y1 = None From 44657fe61d53b6b6aeedc1c1e54e2af0041f87fd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 03:03:54 +0000 Subject: [PATCH 3/5] Keep star and numbered footnotes out of a recovered table row A footnote flush with the first cell, such as '* Off-label use' or '1 Not PBS listed', matched the deliberate-break rule and was glued into the row. '*' no longer counts as a continuation marker, and a leading number counts only when the next word is not capitalised, so doses like '1 to 2 mg' still join. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01M6gRtN6r4vZDgDfzMtjs95 --- tests/pdf-table-extraction.test.ts | 15 +++++++++++++++ worker/python/extract_pdf_assets.py | 5 ++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/pdf-table-extraction.test.ts b/tests/pdf-table-extraction.test.ts index b10260c8c9..b625fd188f 100644 --- a/tests/pdf-table-extraction.test.ts +++ b/tests/pdf-table-extraction.test.ts @@ -147,6 +147,21 @@ describe.runIf(hasPyMuPDF)("PDF table edge-strip row recovery (3344a3a9)", () => ).toEqual([["5", "Highly aroused", "Lorazepam 1 to 2 mg orally - repeat once after 1 hour"]]); }); + it("keeps footnotes flush with the first cell out of a recovered row", () => { + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)", + "page.insert_text((54, 229), '* Off-label use', fontsize=9)", + ]), + ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)", + "page.insert_text((54, 229), '1 Not PBS listed', fontsize=9)", + ]), + ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); + }); + it("extracts column ranges cleanly when a table row contains (None, None, None, None) merged/empty cells", () => { const script = [ "import json, sys", diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index aedf8a4a05..7575edfa34 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -745,6 +745,8 @@ def _continues_wrapped_cell(prev_line, segment, column_range, slack=12.0): closer to the column's right edge than the width of this line's first word. A deliberate break also counts: a line starting at the cell's own left edge with a bullet, a lower-case word or a number (a dose under a drug name). + Footnotes look alike, so '*' never counts and a number counts only when + the word after it is not capitalised ('1 Not PBS listed' stays out). """ if prev_line is None: return False @@ -754,8 +756,9 @@ def _continues_wrapped_cell(prev_line, segment, column_range, slack=12.0): words = cleaned.split(" ") if not words or not words[0]: return False + numbered_footnote = cleaned[0].isdigit() and len(words) > 1 and words[1][:1].isupper() if abs(sx0 - prev_line_start) <= 3.0 and ( - cleaned[0] in "•·-–*" or cleaned[0].islower() or cleaned[0].isdigit() + cleaned[0] in "•·-–" or cleaned[0].islower() or (cleaned[0].isdigit() and not numbered_footnote) ): return True first_word_width = (sx1 - sx0) * (len(words[0]) / max(1, len(cleaned))) From acc233883ef7261806a34a252aa4d6a0f99301c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 03:28:39 +0000 Subject: [PATCH 4/5] Count free space on both sides of a cell when judging a wrap A short right-aligned value (a score of 5) ends near the column's right edge without having wrapped, so a note under it was still joined. The wrap test now counts the free space on either side of the line above. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01M6gRtN6r4vZDgDfzMtjs95 --- tests/pdf-table-extraction.test.ts | 16 ++++++++++++++-- worker/python/extract_pdf_assets.py | 8 +++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/pdf-table-extraction.test.ts b/tests/pdf-table-extraction.test.ts index b625fd188f..495a8c91f6 100644 --- a/tests/pdf-table-extraction.test.ts +++ b/tests/pdf-table-extraction.test.ts @@ -94,7 +94,7 @@ describe.runIf(hasPyMuPDF)("PDF table edge-strip row recovery (3344a3a9)", () => ]); }); - function recoverBottomRow(lines: string[]): string[][] { + function recoverBottomRow(lines: string[], scoreX = 54): string[][] { const script = [ "import json, sys", "import pymupdf as fitz", @@ -106,7 +106,7 @@ describe.runIf(hasPyMuPDF)("PDF table edge-strip row recovery (3344a3a9)", () => "extended_rect = fitz.Rect(50, 120, 560, 250)", "existing_rows = [['0', 'Asleep', 'Monitor vitals']]", "column_ranges = [(50.0, 120.0), (120.0, 310.0), (310.0, 560.0)]", - "page.insert_text((54, 218), '5', fontsize=9)", + `page.insert_text((${scoreX}, 218), '5', fontsize=9)`, "page.insert_text((124, 218), 'Highly aroused', fontsize=9)", ...lines, "recovered = e.recover_table_rows_from_edge_strip(", @@ -126,6 +126,18 @@ describe.runIf(hasPyMuPDF)("PDF table edge-strip row recovery (3344a3a9)", () => ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); }); + it("keeps a note out of the row when the cell above it is a short right-aligned value", () => { + expect( + recoverBottomRow( + [ + "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)", + "page.insert_text((54, 229), 'See note', fontsize=9)", + ], + 111, + ), + ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); + }); + it("still joins a cell whose text wrapped onto the next line", () => { expect( recoverBottomRow([ diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 7575edfa34..9f5b8a9aa6 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -741,8 +741,9 @@ def _match_segment_to_column(sx0, sx1, effective_ranges): def _continues_wrapped_cell(prev_line, segment, column_range, slack=12.0): """True when a lone line under a row reads as the rest of that row's cell. - Text wraps because its next word did not fit, so the line above must end - closer to the column's right edge than the width of this line's first word. + Text wraps because its next word did not fit, so the free space left in + the column beside the line above (on either side, which covers right-aligned + and centred cells) must be narrower than this line's first word. A deliberate break also counts: a line starting at the cell's own left edge with a bullet, a lower-case word or a number (a dose under a drug name). Footnotes look alike, so '*' never counts and a number counts only when @@ -762,7 +763,8 @@ def _continues_wrapped_cell(prev_line, segment, column_range, slack=12.0): ): return True first_word_width = (sx1 - sx0) * (len(words[0]) / max(1, len(cleaned))) - return (column_range[1] - prev_line_end) <= first_word_width + slack + free_space = (column_range[1] - prev_line_end) + max(0.0, prev_line_start - column_range[0]) + return free_space <= first_word_width + slack def _recover_table_rows_split(page, original_rect, extended_rect, existing_rows, column_count=0, column_ranges=None): From d5dfe961f4bc5973ec86374a11e89fe1061f02b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 03:32:42 +0000 Subject: [PATCH 5/5] Let a lower-starting cell still fill an empty cell of the recovered row A row whose cells start on staggered baselines had its later cell dropped by the new note check, because that column had no line above it to compare against. A lone line now fills a still-empty cell as it did before; the note check applies only to cells that already hold text. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01M6gRtN6r4vZDgDfzMtjs95 --- tests/pdf-table-extraction.test.ts | 6 ++++++ worker/python/extract_pdf_assets.py | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/pdf-table-extraction.test.ts b/tests/pdf-table-extraction.test.ts index 495a8c91f6..f043fb12e9 100644 --- a/tests/pdf-table-extraction.test.ts +++ b/tests/pdf-table-extraction.test.ts @@ -138,6 +138,12 @@ describe.runIf(hasPyMuPDF)("PDF table edge-strip row recovery (3344a3a9)", () => ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); }); + it("still fills an empty cell whose text starts a line lower than the rest of the row", () => { + expect(recoverBottomRow(["page.insert_text((314, 229), 'Call the duty registrar', fontsize=9)"])).toEqual([ + ["5", "Highly aroused", "Call the duty registrar"], + ]); + }); + it("still joins a cell whose text wrapped onto the next line", () => { expect( recoverBottomRow([ diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index 9f5b8a9aa6..d9c18b2108 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -844,7 +844,10 @@ def collect_rows(strip_segments): populated_cols = sum(1 for cell in row_cells if clean_cell(cell)) if populated_cols == 1 and rows_out and prev_band_y1 is not None and (band_y0 - prev_band_y1) <= 16.0: col_idx = next(iter(first_segment)) - if not _continues_wrapped_cell( + # A cell that starts lower than its neighbours (staggered or vertically + # centred text) fills a still-empty cell of the row above. + fills_empty_cell = col_idx not in prev_row_line_ends and not rows_out[-1][col_idx] + if not fills_empty_cell and not _continues_wrapped_cell( prev_row_line_ends.get(col_idx), first_segment[col_idx], effective_ranges[col_idx] ): # A short note or caption under the table, not the rest of a wrapped cell.