diff --git a/docs/outstanding-issues-inbox/99b5370e-305b-47f4-b0b5-5c81c37f39d3.json b/docs/outstanding-issues-inbox/99b5370e-305b-47f4-b0b5-5c81c37f39d3.json new file mode 100644 index 0000000000..203a47ca6c --- /dev/null +++ b/docs/outstanding-issues-inbox/99b5370e-305b-47f4-b0b5-5c81c37f39d3.json @@ -0,0 +1,13 @@ +{ + "version": 2, + "id": "99b5370e-305b-47f4-b0b5-5c81c37f39d3", + "createdOn": "2026-10-17", + "action": "update", + "payload": { + "id": "#ZWJCK4", + "pri": "P2", + "detail": "Rechecked 17 Oct 2026 against main 99507ebca. The original fault is fixed: PR #3571 made worker/python/extract_pdf_assets.py _recover_table_rows_split admit a recovered line only when each piece fits one column, keep top and bottom strips in order, and tests/pdf-table-extraction.test.ts pins full-width prose being rejected. A leftover was fixed on 17 Oct: a short note under a recovered row (for example 'See note') was glued into that row's first cell; a lone line now joins the row above only when that cell actually wrapped (_continues_wrapped_cell), with tests both ways. Still open, low risk: text laid out in the table's own columns directly under a table (an abbreviation key, or a two-column page) becomes an extra row, and only when a drawing below the table extends the crop over it. Geometry alone cannot tell such a key from a real row. Changes reach search only after a re-index.", + "baseRowFingerprint": "4167a93e573dfb9618b0f1ccf98de7b816ead5366c7370fe5dddd6919e264eae", + "targetUlid": "01M4T67MV6ZWJCK4234NWY7VKF" + } +} diff --git a/tests/pdf-table-extraction.test.ts b/tests/pdf-table-extraction.test.ts index eb5c5b7bbf..f043fb12e9 100644 --- a/tests/pdf-table-extraction.test.ts +++ b/tests/pdf-table-extraction.test.ts @@ -94,6 +94,92 @@ describe.runIf(hasPyMuPDF)("PDF table edge-strip row recovery (3344a3a9)", () => ]); }); + function recoverBottomRow(lines: string[], scoreX = 54): string[][] { + const script = [ + "import json, sys", + "import pymupdf as fitz", + `sys.path.insert(0, ${JSON.stringify(extractorDir)})`, + "import extract_pdf_assets as e", + "doc = fitz.open()", + "page = doc.new_page(width=595, height=842)", + "original_rect = fitz.Rect(50, 120, 560, 200)", + "extended_rect = fitz.Rect(50, 120, 560, 250)", + "existing_rows = [['0', 'Asleep', 'Monitor vitals']]", + "column_ranges = [(50.0, 120.0), (120.0, 310.0), (310.0, 560.0)]", + `page.insert_text((${scoreX}, 218), '5', fontsize=9)`, + "page.insert_text((124, 218), 'Highly aroused', fontsize=9)", + ...lines, + "recovered = e.recover_table_rows_from_edge_strip(", + " page, original_rect, extended_rect, existing_rows, column_count=3, column_ranges=column_ranges", + ")", + "print(json.dumps(recovered))", + ].join("\n"); + return JSON.parse(execFileSync(pythonBin, ["-c", script], { cwd: process.cwd(), encoding: "utf8" }).trim()); + } + + it("keeps a short note under a recovered row out of that row's cells", () => { + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)", + "page.insert_text((54, 229), 'See note', fontsize=9)", + ]), + ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); + }); + + it("keeps a note out of the row when the cell above it is a short right-aligned value", () => { + expect( + recoverBottomRow( + [ + "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)", + "page.insert_text((54, 229), 'See note', fontsize=9)", + ], + 111, + ), + ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); + }); + + it("still fills an empty cell whose text starts a line lower than the rest of the row", () => { + expect(recoverBottomRow(["page.insert_text((314, 229), 'Call the duty registrar', fontsize=9)"])).toEqual([ + ["5", "Highly aroused", "Call the duty registrar"], + ]); + }); + + it("still joins a cell whose text wrapped onto the next line", () => { + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Give oral medication first, then call the duty registrar,', fontsize=9)", + "page.insert_text((314, 229), 'security and nurse in charge', fontsize=9)", + ]), + ).toEqual([ + ["5", "Highly aroused", "Give oral medication first, then call the duty registrar, security and nurse in charge"], + ]); + }); + + it("still joins a deliberate line break inside a cell, such as a dose under a drug name", () => { + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Lorazepam', fontsize=9)", + "page.insert_text((314, 229), '1 to 2 mg orally', fontsize=9)", + "page.insert_text((314, 240), '- repeat once after 1 hour', fontsize=9)", + ]), + ).toEqual([["5", "Highly aroused", "Lorazepam 1 to 2 mg orally - repeat once after 1 hour"]]); + }); + + it("keeps footnotes flush with the first cell out of a recovered row", () => { + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)", + "page.insert_text((54, 229), '* Off-label use', fontsize=9)", + ]), + ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); + expect( + recoverBottomRow([ + "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)", + "page.insert_text((54, 229), '1 Not PBS listed', fontsize=9)", + ]), + ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]); + }); + it("extracts column ranges cleanly when a table row contains (None, None, None, None) merged/empty cells", () => { const script = [ "import json, sys", diff --git a/worker/python/extract_pdf_assets.py b/worker/python/extract_pdf_assets.py index abc1a2db6d..d9c18b2108 100644 --- a/worker/python/extract_pdf_assets.py +++ b/worker/python/extract_pdf_assets.py @@ -738,6 +738,35 @@ def _match_segment_to_column(sx0, sx1, effective_ranges): return col_idx +def _continues_wrapped_cell(prev_line, segment, column_range, slack=12.0): + """True when a lone line under a row reads as the rest of that row's cell. + + Text wraps because its next word did not fit, so the free space left in + the column beside the line above (on either side, which covers right-aligned + and centred cells) must be narrower than this line's first word. + A deliberate break also counts: a line starting at the cell's own left edge + with a bullet, a lower-case word or a number (a dose under a drug name). + Footnotes look alike, so '*' never counts and a number counts only when + the word after it is not capitalised ('1 Not PBS listed' stays out). + """ + if prev_line is None: + return False + prev_line_start, prev_line_end = prev_line + sx0, sx1, text = segment + cleaned = clean_cell(text) + words = cleaned.split(" ") + if not words or not words[0]: + return False + numbered_footnote = cleaned[0].isdigit() and len(words) > 1 and words[1][:1].isupper() + if abs(sx0 - prev_line_start) <= 3.0 and ( + cleaned[0] in "•·-–" or cleaned[0].islower() or (cleaned[0].isdigit() and not numbered_footnote) + ): + return True + first_word_width = (sx1 - sx0) * (len(words[0]) / max(1, len(cleaned))) + free_space = (column_range[1] - prev_line_end) + max(0.0, prev_line_start - column_range[0]) + return free_space <= first_word_width + slack + + def _recover_table_rows_split(page, original_rect, extended_rect, existing_rows, column_count=0, column_ranges=None): existing_keys = { tuple(clean_cell(cell) for cell in row) @@ -789,11 +818,16 @@ def collect_rows(strip_segments): rows_out = [] prev_band_y1 = None + # Left and right edges of the last line of text in each column of the latest row, + # used to tell a wrapped cell from a note that merely sits below it. + prev_row_line_ends = {} for band in bands: band.sort(key=lambda s: s[0]) band_y0 = min(s[1] for s in band) band_y1 = max(s[3] for s in band) row_cells = [""] * num_cols + line_ends = {} + first_segment = {} valid_band = True for sx0, _sy0, sx1, _sy1, text in band: col_idx = _match_segment_to_column(sx0, sx1, effective_ranges) @@ -801,11 +835,25 @@ def collect_rows(strip_segments): valid_band = False break row_cells[col_idx] = clean_cell(f"{row_cells[col_idx]} {text}") if row_cells[col_idx] else clean_cell(text) + start, end = line_ends.get(col_idx, (sx0, sx1)) + line_ends[col_idx] = (min(start, sx0), max(end, sx1)) + first_segment.setdefault(col_idx, (sx0, sx1, text)) if not valid_band: prev_band_y1 = None continue populated_cols = sum(1 for cell in row_cells if clean_cell(cell)) if populated_cols == 1 and rows_out and prev_band_y1 is not None and (band_y0 - prev_band_y1) <= 16.0: + col_idx = next(iter(first_segment)) + # A cell that starts lower than its neighbours (staggered or vertically + # centred text) fills a still-empty cell of the row above. + fills_empty_cell = col_idx not in prev_row_line_ends and not rows_out[-1][col_idx] + if not fills_empty_cell and not _continues_wrapped_cell( + prev_row_line_ends.get(col_idx), first_segment[col_idx], effective_ranges[col_idx] + ): + # A short note or caption under the table, not the rest of a wrapped cell. + prev_band_y1 = None + continue + prev_row_line_ends[col_idx] = line_ends[col_idx] prev_row = rows_out[-1] existing_keys.discard(tuple(prev_row)) for idx, val in enumerate(row_cells): @@ -822,6 +870,7 @@ def collect_rows(strip_segments): continue existing_keys.add(key) rows_out.append(row_cells) + prev_row_line_ends = line_ends prev_band_y1 = band_y1 return rows_out