Stop a note under a recovered table row being glued into that row

docs/outstanding-issues-inbox/99b5370e-305b-47f4-b0b5-5c81c37f39d3.json

@@ -0,0 +1,13 @@
+{
+  "version": 2,
+  "id": "99b5370e-305b-47f4-b0b5-5c81c37f39d3",
+  "createdOn": "2026-10-17",
+  "action": "update",
+  "payload": {
+    "id": "#ZWJCK4",
+    "pri": "P2",
+    "detail": "Rechecked 17 Oct 2026 against main 99507ebca. The original fault is fixed: PR #3571 made worker/python/extract_pdf_assets.py _recover_table_rows_split admit a recovered line only when each piece fits one column, keep top and bottom strips in order, and tests/pdf-table-extraction.test.ts pins full-width prose being rejected. A leftover was fixed on 17 Oct: a short note under a recovered row (for example 'See note') was glued into that row's first cell; a lone line now joins the row above only when that cell actually wrapped (_continues_wrapped_cell), with tests both ways. Still open, low risk: text laid out in the table's own columns directly under a table (an abbreviation key, or a two-column page) becomes an extra row, and only when a drawing below the table extends the crop over it. Geometry alone cannot tell such a key from a real row. Changes reach search only after a re-index.",
+    "baseRowFingerprint": "4167a93e573dfb9618b0f1ccf98de7b816ead5366c7370fe5dddd6919e264eae",
+    "targetUlid": "01M4T67MV6ZWJCK4234NWY7VKF"
+  }
+}

tests/pdf-table-extraction.test.ts

@@ -94,6 +94,92 @@ describe.runIf(hasPyMuPDF)("PDF table edge-strip row recovery (3344a3a9)", () =>
     ]);
   });
 
+  function recoverBottomRow(lines: string[], scoreX = 54): string[][] {
+    const script = [
+      "import json, sys",
+      "import pymupdf as fitz",
+      `sys.path.insert(0, ${JSON.stringify(extractorDir)})`,
+      "import extract_pdf_assets as e",
+      "doc = fitz.open()",
+      "page = doc.new_page(width=595, height=842)",
+      "original_rect = fitz.Rect(50, 120, 560, 200)",
+      "extended_rect = fitz.Rect(50, 120, 560, 250)",
+      "existing_rows = [['0', 'Asleep', 'Monitor vitals']]",
+      "column_ranges = [(50.0, 120.0), (120.0, 310.0), (310.0, 560.0)]",
+      `page.insert_text((${scoreX}, 218), '5', fontsize=9)`,
+      "page.insert_text((124, 218), 'Highly aroused', fontsize=9)",
+      ...lines,
+      "recovered = e.recover_table_rows_from_edge_strip(",
+      "    page, original_rect, extended_rect, existing_rows, column_count=3, column_ranges=column_ranges",
+      ")",
+      "print(json.dumps(recovered))",
+    ].join("\n");
+    return JSON.parse(execFileSync(pythonBin, ["-c", script], { cwd: process.cwd(), encoding: "utf8" }).trim());
+  }
+
+  it("keeps a short note under a recovered row out of that row's cells", () => {
+    expect(
+      recoverBottomRow([
+        "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)",
+        "page.insert_text((54, 229), 'See note', fontsize=9)",
+      ]),
+    ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]);
+  });
+
+  it("keeps a note out of the row when the cell above it is a short right-aligned value", () => {
+    expect(
+      recoverBottomRow(
+        [
+          "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)",
+          "page.insert_text((54, 229), 'See note', fontsize=9)",
+        ],
+        111,
+      ),
+    ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]);
+  });
+
+  it("still fills an empty cell whose text starts a line lower than the rest of the row", () => {
+    expect(recoverBottomRow(["page.insert_text((314, 229), 'Call the duty registrar', fontsize=9)"])).toEqual([
+      ["5", "Highly aroused", "Call the duty registrar"],
+    ]);
+  });
+
+  it("still joins a cell whose text wrapped onto the next line", () => {
+    expect(
+      recoverBottomRow([
+        "page.insert_text((314, 218), 'Give oral medication first, then call the duty registrar,', fontsize=9)",
+        "page.insert_text((314, 229), 'security and nurse in charge', fontsize=9)",
+      ]),
+    ).toEqual([
+      ["5", "Highly aroused", "Give oral medication first, then call the duty registrar, security and nurse in charge"],
+    ]);
+  });
+
+  it("still joins a deliberate line break inside a cell, such as a dose under a drug name", () => {
+    expect(
+      recoverBottomRow([
+        "page.insert_text((314, 218), 'Lorazepam', fontsize=9)",
+        "page.insert_text((314, 229), '1 to 2 mg orally', fontsize=9)",
+        "page.insert_text((314, 240), '- repeat once after 1 hour', fontsize=9)",
+      ]),
+    ).toEqual([["5", "Highly aroused", "Lorazepam 1 to 2 mg orally - repeat once after 1 hour"]]);
+  });
+
+  it("keeps footnotes flush with the first cell out of a recovered row", () => {
+    expect(
+      recoverBottomRow([
+        "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)",
+        "page.insert_text((54, 229), '* Off-label use', fontsize=9)",
+      ]),
+    ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]);
+    expect(
+      recoverBottomRow([
+        "page.insert_text((314, 218), 'Call the duty registrar', fontsize=9)",
+        "page.insert_text((54, 229), '1 Not PBS listed', fontsize=9)",
+      ]),
+    ).toEqual([["5", "Highly aroused", "Call the duty registrar"]]);
+  });
+
   it("extracts column ranges cleanly when a table row contains (None, None, None, None) merged/empty cells", () => {
     const script = [
       "import json, sys",

worker/python/extract_pdf_assets.py

@@ -738,6 +738,35 @@ def _match_segment_to_column(sx0, sx1, effective_ranges):
     return col_idx
 
 
+def _continues_wrapped_cell(prev_line, segment, column_range, slack=12.0):
+    """True when a lone line under a row reads as the rest of that row's cell.
+
+    Text wraps because its next word did not fit, so the free space left in
+    the column beside the line above (on either side, which covers right-aligned
+    and centred cells) must be narrower than this line's first word.
+    A deliberate break also counts: a line starting at the cell's own left edge
+    with a bullet, a lower-case word or a number (a dose under a drug name).
+    Footnotes look alike, so '*' never counts and a number counts only when
+    the word after it is not capitalised ('1 Not PBS listed' stays out).
+    """
+    if prev_line is None:
+        return False
+    prev_line_start, prev_line_end = prev_line
+    sx0, sx1, text = segment
+    cleaned = clean_cell(text)
+    words = cleaned.split(" ")
+    if not words or not words[0]:
+        return False
+    numbered_footnote = cleaned[0].isdigit() and len(words) > 1 and words[1][:1].isupper()
+    if abs(sx0 - prev_line_start) <= 3.0 and (
+        cleaned[0] in "•·-–" or cleaned[0].islower() or (cleaned[0].isdigit() and not numbered_footnote)
+    ):
+        return True
+    first_word_width = (sx1 - sx0) * (len(words[0]) / max(1, len(cleaned)))
+    free_space = (column_range[1] - prev_line_end) + max(0.0, prev_line_start - column_range[0])
+    return free_space <= first_word_width + slack
+
+
 def _recover_table_rows_split(page, original_rect, extended_rect, existing_rows, column_count=0, column_ranges=None):
     existing_keys = {
         tuple(clean_cell(cell) for cell in row)
@@ -789,23 +818,42 @@ def collect_rows(strip_segments):
 
         rows_out = []
         prev_band_y1 = None
+        # Left and right edges of the last line of text in each column of the latest row,
+        # used to tell a wrapped cell from a note that merely sits below it.
+        prev_row_line_ends = {}
         for band in bands:
             band.sort(key=lambda s: s[0])
             band_y0 = min(s[1] for s in band)
             band_y1 = max(s[3] for s in band)
             row_cells = [""] * num_cols
+            line_ends = {}
+            first_segment = {}
             valid_band = True
             for sx0, _sy0, sx1, _sy1, text in band:
                 col_idx = _match_segment_to_column(sx0, sx1, effective_ranges)
                 if col_idx is None:
                     valid_band = False
                     break
                 row_cells[col_idx] = clean_cell(f"{row_cells[col_idx]} {text}") if row_cells[col_idx] else clean_cell(text)
+                start, end = line_ends.get(col_idx, (sx0, sx1))
+                line_ends[col_idx] = (min(start, sx0), max(end, sx1))
+                first_segment.setdefault(col_idx, (sx0, sx1, text))
             if not valid_band:
                 prev_band_y1 = None
                 continue
             populated_cols = sum(1 for cell in row_cells if clean_cell(cell))
             if populated_cols == 1 and rows_out and prev_band_y1 is not None and (band_y0 - prev_band_y1) <= 16.0:
+                col_idx = next(iter(first_segment))
+                # A cell that starts lower than its neighbours (staggered or vertically
+                # centred text) fills a still-empty cell of the row above.
+                fills_empty_cell = col_idx not in prev_row_line_ends and not rows_out[-1][col_idx]
+                if not fills_empty_cell and not _continues_wrapped_cell(
+                    prev_row_line_ends.get(col_idx), first_segment[col_idx], effective_ranges[col_idx]
+                ):
+                    # A short note or caption under the table, not the rest of a wrapped cell.
+                    prev_band_y1 = None
+                    continue
+                prev_row_line_ends[col_idx] = line_ends[col_idx]
                 prev_row = rows_out[-1]
                 existing_keys.discard(tuple(prev_row))
                 for idx, val in enumerate(row_cells):
@@ -822,6 +870,7 @@ def collect_rows(strip_segments):
                 continue
             existing_keys.add(key)
             rows_out.append(row_cells)
+            prev_row_line_ends = line_ends
             prev_band_y1 = band_y1
         return rows_out