diff --git a/Cargo.toml b/Cargo.toml index 2a334b72..4a485d16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ version = "0.2.4" edition = "2024" # Edition 2024 needs 1.85; zip raises it to 1.88. rust-version = "1.88" -description = "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" +description = "Convert documents (doc, docx, odt, rtf, epub, pdf, ofd, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" license = "MIT" repository = "https://github.com/firecrawl/anydoc" readme = "README.md" diff --git a/README.md b/README.md index 2a920ec8..8504508a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![skills.sh](https://skills.sh/b/firecrawl/anydoc)](https://skills.sh/firecrawl/anydoc) -Fast Rust library that converts documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF) into clean GitHub-Flavored Markdown. Includes bindings for [Node.js](node/README.md), [Python](python/README.md), and the [browser](wasm/README.md) (WebAssembly). +Fast Rust library that converts documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, PDF, and OFD) into clean GitHub-Flavored Markdown. Includes bindings for [Node.js](node/README.md), [Python](python/README.md), and the [browser](wasm/README.md) (WebAssembly). Built by [Firecrawl](https://firecrawl.dev) to turn any office document into LLM-ready Markdown in single-digit milliseconds, with one consistent output no matter which format goes in. It powers [Firecrawl Parse](https://firecrawl.dev/parse), so if you'd rather not run it yourself, the hosted API gives you the same conversion plus our OCR models for the scanned pages anydoc can't read on its own. @@ -134,7 +134,7 @@ let document = anydoc::to_document(&bytes, None)?; ## OCR -anydoc reads text-based PDFs locally but does no OCR, so a PDF with scanned or image-only pages fails with `NeedsOcr`. Opt in and those documents go to [Firecrawl Parse](https://firecrawl.dev/parse), which OCRs them and returns the same Markdown. No signup needed; set `FIRECRAWL_API_KEY` for higher limits. +anydoc reads text-based PDFs and OFDs locally but does no OCR, so scanned or image-only pages fail with `NeedsOcr`. For PDFs, opt in and those documents go to [Firecrawl Parse](https://firecrawl.dev/parse), which OCRs them and returns the same Markdown. The hosted PDF parser does not accept OFD; scanned OFD pages still return `NeedsOcr`. No signup needed; set `FIRECRAWL_API_KEY` for higher limits. | | Opt in | Key, else `FIRECRAWL_API_KEY` | | ------ | ---------------------------------------------- | ----------------------------- | @@ -168,6 +168,9 @@ Only documents that need OCR leave the machine, and the whole document goes, sin | EPUB | `.epub` | | CSV | `.csv` | | PDF | `.pdf` | +| OFD | `.ofd` (text objects) | + +OFD conversion preserves outline headings, large-font headings, bold and italic text, URI links, printed lists, and complete bordered table grids. Tables and large-font headings are inferred from fixed-page geometry; pages without those cues remain paragraphs. Image-only pages return `NeedsOcr`; images, vector drawings, and non-link annotations are not included in Markdown. ## Benchmark @@ -242,7 +245,7 @@ match anydoc::to_markdown(path) { | Variant | Meaning | | --------------- | ------------------------------------------------------------------- | | `Unsupported` | Unknown format, or one that cannot be converted | -| `NeedsOcr` | Scanned or image-only pages of a PDF, listed in `pages` | +| `NeedsOcr` | Scanned or image-only PDF/OFD pages, listed in `pages` | | `Malformed` | Structurally unusable: no meaningful content could be extracted | | `Encrypted` | Encrypted or password-protected | | `ResourceLimit` | Crossed a fixed safety limit (decompression, nesting, node count) | @@ -251,7 +254,7 @@ match anydoc::to_markdown(path) { Node and wasm publish the variant name on `error.code`; Python raises one `anydoc.ConvertError` subclass per variant, or `OSError` when the file cannot be read. -anydoc converts locally and does not do OCR, so a PDF with scanned pages fails with `NeedsOcr`. Opt in with `ocr: 'hosted'` in Node, `ocr="hosted"` in Python or `--ocr hosted` on the CLI to send that document to [Firecrawl Parse](https://firecrawl.dev/parse). No signup needed. Set `FIRECRAWL_API_KEY` for higher limits. +anydoc converts locally and does not do OCR, so a PDF or OFD with scanned pages fails with `NeedsOcr`. For PDFs, opt in with `ocr: 'hosted'` in Node, `ocr="hosted"` in Python or `--ocr hosted` on the CLI to send that document to [Firecrawl Parse](https://firecrawl.dev/parse). OFD pages continue to return `NeedsOcr` because this hosted mode uses a PDF parser. No signup needed. Set `FIRECRAWL_API_KEY` for higher limits. ## How it works diff --git a/node/README.md b/node/README.md index 12ee0ce4..bb1ee4fd 100644 --- a/node/README.md +++ b/node/README.md @@ -3,7 +3,7 @@ [![npm](https://img.shields.io/npm/v/@firecrawl/anydoc.svg)](https://www.npmjs.com/package/@firecrawl/anydoc) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/firecrawl/anydoc/blob/main/LICENSE) -Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF files into clean GitHub-Flavored Markdown. Node.js bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. +Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, PDF, and OFD files into clean GitHub-Flavored Markdown. Node.js bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. Every format parses into one shared document model and renders through a single Markdown serializer, so headings, tables, lists, and footnotes come out the same no matter which format goes in. Conversion runs on the libuv thread pool and never blocks the event loop. TypeScript types ship with the package. @@ -23,6 +23,9 @@ npm install @firecrawl/anydoc | EPUB | `.epub` | | CSV | `.csv` | | PDF | `.pdf` | +| OFD | `.ofd` (text objects) | + +OFD conversion preserves outline headings, large-font headings, bold and italic text, URI links, printed lists, and complete bordered table grids. Tables and large-font headings are inferred from fixed-page geometry; pages without those cues remain paragraphs. Image-only pages return `needsOcr`; images, vector drawings, and non-link annotations are not included in Markdown. ## CLI @@ -57,7 +60,7 @@ const document = await toDocument(bytes); ## Scanned pages -anydoc converts locally and does not do OCR, so a PDF with scanned or image-only pages rejects with `needsOcr`. Opt in with `ocr: 'hosted'` to send that document to [Firecrawl Parse](https://firecrawl.dev/parse). No signup needed. Set `apiKey` or `FIRECRAWL_API_KEY` for higher limits. +anydoc converts locally and does not do OCR, so a PDF or OFD with scanned or image-only pages rejects with `needsOcr`. For PDFs, opt in with `ocr: 'hosted'` to send that document to [Firecrawl Parse](https://firecrawl.dev/parse). This hosted PDF parser does not accept OFD; OFD pages still reject with `needsOcr` even when hosted OCR is requested. No signup needed. Set `apiKey` or `FIRECRAWL_API_KEY` for higher limits. ```js const markdown = await toMarkdown('scan.pdf', { ocr: 'hosted' }); @@ -91,7 +94,7 @@ try { | `code` | Meaning | | --------------- | ------------------------------------------------------------------- | | `unsupported` | Unknown format, or one that cannot be converted | -| `needsOcr` | Scanned or image-only pages of a PDF, listed in `pages` | +| `needsOcr` | Scanned or image-only PDF/OFD pages, listed in `pages` | | `malformed` | Structurally unusable: no meaningful content could be extracted | | `encrypted` | Encrypted or password-protected | | `resourceLimit` | Crossed a fixed safety limit (decompression, nesting, node count) | diff --git a/node/anydoc.js b/node/anydoc.js index af620c75..bbba5ff2 100644 --- a/node/anydoc.js +++ b/node/anydoc.js @@ -22,7 +22,9 @@ async function toMarkdown(path, options) { return await native.toMarkdown(path) } catch (error) { if (!sendsToHosted(error, options)) throw error - return parseHosted(await readFile(path), basename(path), options) + const bytes = await readFile(path) + if ((native.formatFromBytes(bytes) ?? native.formatFromPath(path)) !== 'pdf') throw error + return parseHosted(bytes, basename(path), options) } } @@ -33,6 +35,7 @@ async function toMarkdownBytes(bytes, format, options) { return await native.toMarkdownBytes(bytes, format) } catch (error) { if (!sendsToHosted(error, options)) throw error + if ((format ?? native.formatFromBytes(bytes)) !== 'pdf') throw error return parseHosted(bytes, 'document.pdf', options) } } diff --git a/node/cli.js b/node/cli.js index 44fcd9dd..5eea3a60 100644 --- a/node/cli.js +++ b/node/cli.js @@ -3,7 +3,7 @@ const { readFile, writeFile } = require('node:fs/promises') -const FORMATS = 'doc, docx, odt, pdf, ppt, pptx, rtf, epub, xlsx, ods, odp, csv' +const FORMATS = 'doc, docx, odt, pdf, ofd, ppt, pptx, rtf, epub, xlsx, ods, odp, csv' const HELP = `anydoc: convert documents to GitHub-Flavored Markdown @@ -37,14 +37,14 @@ Options: The format is detected from the file content; the file extension is the fallback for signature-less formats (CSV). stdin has no extension, so CSV input from stdin needs --format csv. Scanned or image-only pages need OCR, -which anydoc does not do: the document exits 3, or goes to Firecrawl Parse -with --ocr hosted. +which anydoc does not do: the document exits 3. PDF can go to Firecrawl +Parse with --ocr hosted; OFD still exits 3. Exit codes: 0 success 1 the document could not be read or converted 2 usage error: unknown option, missing input, or invalid --format - 3 pages of a PDF need OCR + 3 PDF or OFD pages need OCR Examples: anydoc report.docx diff --git a/node/dts-header.d.ts b/node/dts-header.d.ts index 5efbf637..f032dd1d 100644 --- a/node/dts-header.d.ts +++ b/node/dts-header.d.ts @@ -9,7 +9,7 @@ export type ConvertErrorCode = /** Unknown format, or one that cannot be converted. */ | 'unsupported' /** - * Pages of a PDF are scanned or image-only and need OCR, which anydoc does + * PDF or OFD pages are scanned or image-only and need OCR, which anydoc does * not do. The error is a `NeedsOcrError` naming them. */ | 'needsOcr' diff --git a/node/index.d.ts b/node/index.d.ts index e3e4fd72..07ca5258 100644 --- a/node/index.d.ts +++ b/node/index.d.ts @@ -9,7 +9,7 @@ export type ConvertErrorCode = /** Unknown format, or one that cannot be converted. */ | 'unsupported' /** - * Pages of a PDF are scanned or image-only and need OCR, which anydoc does + * PDF or OFD pages are scanned or image-only and need OCR, which anydoc does * not do. The error is a `NeedsOcrError` naming them. */ | 'needsOcr' @@ -124,6 +124,7 @@ export declare const enum Format { * `needsOcr` naming them. */ pdf = 'pdf', + ofd = 'ofd', ppt = 'ppt', pptx = 'pptx', rtf = 'rtf', diff --git a/node/package.json b/node/package.json index fae9b054..bc33cce3 100644 --- a/node/package.json +++ b/node/package.json @@ -1,7 +1,7 @@ { "name": "@firecrawl/anydoc", "version": "0.2.4", - "description": "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown", + "description": "Convert documents (doc, docx, odt, rtf, epub, pdf, ofd, presentations, spreadsheets, csv) to GitHub-Flavored Markdown", "license": "MIT", "homepage": "https://github.com/firecrawl/anydoc#readme", "repository": { diff --git a/node/src/lib.rs b/node/src/lib.rs index c32bd79e..4e2e9bde 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -22,6 +22,7 @@ pub enum Format { /// need OCR, which anydoc does not do: the document rejects with /// `needsOcr` naming them. pdf, + ofd, ppt, pptx, rtf, @@ -39,6 +40,7 @@ impl From for anydoc::Format { Format::docx => anydoc::Format::Docx, Format::odt => anydoc::Format::Odt, Format::pdf => anydoc::Format::Pdf, + Format::ofd => anydoc::Format::Ofd, Format::ppt => anydoc::Format::Ppt, Format::pptx => anydoc::Format::Pptx, Format::rtf => anydoc::Format::Rtf, @@ -58,6 +60,7 @@ impl From for Format { anydoc::Format::Docx => Format::docx, anydoc::Format::Odt => Format::odt, anydoc::Format::Pdf => Format::pdf, + anydoc::Format::Ofd => Format::ofd, anydoc::Format::Ppt => Format::ppt, anydoc::Format::Pptx => Format::pptx, anydoc::Format::Rtf => Format::rtf, diff --git a/node/test.mjs b/node/test.mjs index 3636206e..1ef86956 100644 --- a/node/test.mjs +++ b/node/test.mjs @@ -1,7 +1,7 @@ // Smoke test: the bindings load and every entry point round-trips a fixture. import assert from 'node:assert/strict' import { execFile } from 'node:child_process' -import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { createServer } from 'node:http' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -25,6 +25,7 @@ const RICH = fixture('docx/handmade-rich.docx') const CSV = fixture('csv/sheet.csv') const ENCRYPTED = fixture('malformed/encrypted--errors.odt') const MIXED = fixture('pdf/handmade-mixed.pdf') +const SCANNED_OFD = fixture('ofd/image-only.ofd') test('toMarkdown detects the format from the file content', async () => { const markdown = await toMarkdown(OUTLINE) @@ -67,6 +68,7 @@ test('format detection reads content, extension, and path', async () => { assert.equal(formatFromBytes(await readFile(CSV)), null) assert.equal(formatFromExtension('.pptm'), 'pptx') assert.equal(formatFromExtension('xls'), 'xlsx') + assert.equal(formatFromExtension('ofd'), 'ofd') assert.equal(formatFromPath('/tmp/report.odt'), 'odt') assert.equal(formatFromPath('/tmp/report.unknown'), null) }) @@ -149,6 +151,39 @@ test("ocr: 'hosted' sends a pdf with scanned pages to Firecrawl Parse, and nothi }) }) +test("ocr: 'hosted' leaves image-only OFD as needsOcr", async () => { + await withHostedStub(HOSTED, async (hits) => { + for (const convert of [ + () => toMarkdown(SCANNED_OFD, { ocr: 'hosted' }), + async () => toMarkdownBytes(await readFile(SCANNED_OFD), 'ofd', { ocr: 'hosted' }), + async () => toMarkdownBytes(await readFile(SCANNED_OFD), undefined, { ocr: 'hosted' }), + ]) { + await assert.rejects(convert(), (error) => error.code === 'needsOcr') + } + assert.deepEqual(hits, []) + }) +}) + +test('hosted fallback uses file content before a misleading extension', async () => { + const dir = await mkdtemp(join(tmpdir(), 'anydoc-hosted-')) + try { + const ofdNamedPdf = join(dir, 'scan.pdf') + const pdfNamedOfd = join(dir, 'scan.ofd') + await Promise.all([ + writeFile(ofdNamedPdf, await readFile(SCANNED_OFD)), + writeFile(pdfNamedOfd, await readFile(MIXED)), + ]) + await withHostedStub(HOSTED, async (hits) => { + await assert.rejects(toMarkdown(ofdNamedPdf, { ocr: 'hosted' }), (error) => error.code === 'needsOcr') + assert.deepEqual(hits, []) + assert.equal(await toMarkdown(pdfNamedOfd, { ocr: 'hosted' }), HOSTED.body.data.markdown) + assert.deepEqual(hits, [['/v2/parse', true]]) + }) + } finally { + await rm(dir, { recursive: true, force: true }) + } +}) + test("parseMode picks the Parse pdf mode, 'auto' by default", async () => { await withHostedStub(HOSTED, async (hits, url, sent) => { await toMarkdown(MIXED, { ocr: 'hosted' }) @@ -241,6 +276,15 @@ test('cli --ocr hosted converts a pdf with scanned pages through Firecrawl Parse }) }) +test('cli --ocr hosted keeps image-only OFD at needs OCR exit code', async () => { + await withHostedStub(HOSTED, async (hits, url) => { + const { code, stderr } = await runCli([SCANNED_OFD, '--ocr', 'hosted', '--api-url', url]) + assert.equal(code, 3) + assert.match(stderr, /needs OCR/) + assert.deepEqual(hits, []) + }) +}) + test('cli --parse-mode picks the Parse pdf mode', async () => { await withHostedStub(HOSTED, async (hits, url, sent) => { const { code } = await runCli([MIXED, '--ocr', 'hosted', '--parse-mode', 'ocr', '--api-url', url]) diff --git a/python/README.md b/python/README.md index 9680b568..326156d3 100644 --- a/python/README.md +++ b/python/README.md @@ -3,7 +3,7 @@ [![PyPI](https://img.shields.io/pypi/v/firecrawl-anydoc.svg)](https://pypi.org/project/firecrawl-anydoc/) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/firecrawl/anydoc/blob/main/LICENSE) -Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF files into clean GitHub-Flavored Markdown. Python bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. +Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, PDF, and OFD files into clean GitHub-Flavored Markdown. Python bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. Every format parses into one shared document model and renders through a single Markdown serializer, so headings, tables, lists, and footnotes come out the same no matter which format goes in. Conversion releases the GIL, so other threads keep running. Type stubs ship with the package. @@ -25,6 +25,9 @@ The package installs as `firecrawl-anydoc` and imports as `anydoc`. | EPUB | `.epub` | | CSV | `.csv` | | PDF | `.pdf` | +| OFD | `.ofd` (text objects) | + +OFD conversion preserves outline headings, large-font headings, bold and italic text, URI links, printed lists, and complete bordered table grids. Tables and large-font headings are inferred from fixed-page geometry; pages without those cues remain paragraphs. Image-only pages raise `NeedsOcrError`; images, vector drawings, and non-link annotations are not included in Markdown. ## Usage @@ -46,7 +49,7 @@ document = anydoc.to_document(data) ## Scanned pages -anydoc converts locally and does not do OCR, so a PDF with scanned or image-only pages raises `NeedsOcrError`. Opt in with `ocr="hosted"` to send that document to [Firecrawl Parse](https://firecrawl.dev/parse). No signup needed. Set `api_key` or `FIRECRAWL_API_KEY` for higher limits. +anydoc converts locally and does not do OCR, so a PDF or OFD with scanned or image-only pages raises `NeedsOcrError`. For PDFs, opt in with `ocr="hosted"` to send that document to [Firecrawl Parse](https://firecrawl.dev/parse). This hosted PDF parser does not accept OFD; OFD pages still raise `NeedsOcrError` even when hosted OCR is requested. No signup needed. Set `api_key` or `FIRECRAWL_API_KEY` for higher limits. ```python markdown = anydoc.to_markdown("scan.pdf", ocr="hosted") @@ -74,7 +77,7 @@ except (anydoc.EncryptedError, anydoc.UnsupportedError) as error: | Exception | Raised when | | -------------------- | ------------------------------------------------------------------- | | `UnsupportedError` | Unknown format, or one that cannot be converted | -| `NeedsOcrError` | Scanned or image-only pages of a PDF, listed in `pages` | +| `NeedsOcrError` | Scanned or image-only PDF/OFD pages, listed in `pages` | | `MalformedError` | Structurally unusable: no meaningful content could be extracted | | `EncryptedError` | Encrypted or password-protected | | `ResourceLimitError` | Crossed a fixed safety limit (decompression, nesting, node count) | diff --git a/python/anydoc/__init__.py b/python/anydoc/__init__.py index 1cd64e73..90002970 100644 --- a/python/anydoc/__init__.py +++ b/python/anydoc/__init__.py @@ -39,17 +39,17 @@ from anydoc._anydoc import to_markdown_bytes as _to_markdown_bytes Format = Literal[ - "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv" + "doc", "docx", "odt", "pdf", "ofd", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv" ] """Input format, named after the extension that identifies it. Container variants that share a parser (`.docm`, `.xlsm`, `.ppsx`, ...) map onto these via `format_from_bytes` or `format_from_extension`.""" Ocr = Literal["reject", "hosted"] -"""What happens to a PDF whose pages need OCR. `reject` (the default) raises -`NeedsOcrError` naming the pages. `hosted` sends the whole document to -Firecrawl Parse instead, keyless unless a key is given. Documents anydoc -converts itself never leave the machine.""" +"""What happens when PDF or OFD pages need OCR. `reject` (the default) +raises `NeedsOcrError` naming the pages. `hosted` sends PDFs to Firecrawl +Parse, keyless unless a key is given. OFD still raises `NeedsOcrError`. +Documents anydoc converts itself never leave the machine.""" ParseMode = Literal["auto", "fast", "ocr"] """Firecrawl Parse PDF mode for `ocr="hosted"`. `auto` (the default) extracts @@ -83,8 +83,11 @@ def to_markdown( except NeedsOcrError: if ocr != "hosted": raise - path = Path(path) - return _parse_hosted(path.read_bytes(), path.name, parse_mode, api_key, api_url) + path = Path(path) + data = path.read_bytes() + if (format_from_bytes(data) or format_from_path(path)) != "pdf": + raise + return _parse_hosted(data, path.name, parse_mode, api_key, api_url) def to_markdown_bytes( @@ -104,7 +107,7 @@ def to_markdown_bytes( try: return _to_markdown_bytes(data, format) except NeedsOcrError: - if ocr != "hosted": + if ocr != "hosted" or (format or format_from_bytes(bytes(data))) != "pdf": raise return _parse_hosted(bytes(data), "document.pdf", parse_mode, api_key, api_url) diff --git a/python/anydoc/_anydoc.pyi b/python/anydoc/_anydoc.pyi index ca45a408..6c2591ef 100644 --- a/python/anydoc/_anydoc.pyi +++ b/python/anydoc/_anydoc.pyi @@ -4,7 +4,7 @@ import os from typing import Literal, final Format = Literal[ - "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv" + "doc", "docx", "odt", "pdf", "ofd", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv" ] class ConvertError(Exception): @@ -16,7 +16,7 @@ class UnsupportedError(ConvertError): """The format is unknown, or cannot be converted at all.""" class NeedsOcrError(ConvertError): - """Pages of a PDF are scanned or image-only and need OCR, which anydoc + """PDF or OFD pages are scanned or image-only and need OCR, which anydoc does not do.""" pages: list[int] diff --git a/python/pyproject.toml b/python/pyproject.toml index 81885146..255e8fa7 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "maturin" # PyPI is held by an unrelated package) and imports as `anydoc`. [project] name = "firecrawl-anydoc" -description = "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" +description = "Convert documents (doc, docx, odt, rtf, epub, pdf, ofd, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" readme = "README.md" license = "MIT" requires-python = ">=3.10" diff --git a/python/src/lib.rs b/python/src/lib.rs index a38a09de..44193cc6 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -28,7 +28,7 @@ create_exception!( anydoc, NeedsOcrError, ConvertError, - "Pages of a PDF are scanned or image-only and need OCR, which anydoc does \ + "PDF or OFD pages are scanned or image-only and need OCR, which anydoc does \ not do. `pages` lists them (1-indexed) and `page_count` is the length of \ the document." ); @@ -67,11 +67,12 @@ create_exception!( /// Format names, as the extension that identifies each format. Container /// variants that share a parser (`.docm`, `.xlsm`, `.ppsx`, ...) map onto /// these via `format_from_bytes` or `format_from_extension`. -const FORMATS: [(&str, anydoc::Format); 12] = [ +const FORMATS: [(&str, anydoc::Format); 13] = [ ("doc", anydoc::Format::Doc), ("docx", anydoc::Format::Docx), ("odt", anydoc::Format::Odt), ("pdf", anydoc::Format::Pdf), + ("ofd", anydoc::Format::Ofd), ("ppt", anydoc::Format::Ppt), ("pptx", anydoc::Format::Pptx), ("rtf", anydoc::Format::Rtf), diff --git a/python/tests/test_anydoc.py b/python/tests/test_anydoc.py index 32a42ad6..987b17a7 100644 --- a/python/tests/test_anydoc.py +++ b/python/tests/test_anydoc.py @@ -4,6 +4,7 @@ import io import json import os +import tempfile import threading import unittest import zipfile @@ -20,6 +21,7 @@ ENCRYPTED = FIXTURES / "malformed" / "encrypted--errors.odt" ZIPBOMB = FIXTURES / "abuse" / "zipbomb--errors.docx" MIXED = FIXTURES / "pdf" / "handmade-mixed.pdf" +SCANNED_OFD = FIXTURES / "ofd" / "image-only.ofd" HOSTED_MARKDOWN = "# Read by the hosted parser\n" @@ -107,6 +109,7 @@ def test_format_detection_reads_content_extension_and_path(self): self.assertIsNone(anydoc.format_from_bytes(CSV.read_bytes())) self.assertEqual(anydoc.format_from_extension(".pptm"), "pptx") self.assertEqual(anydoc.format_from_extension("xls"), "xlsx") + self.assertEqual(anydoc.format_from_extension("ofd"), "ofd") self.assertEqual(anydoc.format_from_path("report.odt"), "odt") self.assertIsNone(anydoc.format_from_path("report.unknown")) @@ -149,6 +152,32 @@ def test_ocr_hosted_sends_a_pdf_with_scanned_pages_to_firecrawl_parse_and_nothin self.assertRegex(anydoc.to_markdown(OUTLINE, ocr="hosted"), r"(?m)^# ") self.assertEqual(hits, [("/v2/parse", True)]) + def test_ocr_hosted_keeps_image_only_ofd_as_needs_ocr(self): + reply = {"success": True, "data": {"markdown": HOSTED_MARKDOWN}} + with hosted_stub(200, reply) as hits: + for convert in ( + lambda: anydoc.to_markdown(SCANNED_OFD, ocr="hosted"), + lambda: anydoc.to_markdown_bytes(SCANNED_OFD.read_bytes(), "ofd", ocr="hosted"), + lambda: anydoc.to_markdown_bytes(SCANNED_OFD.read_bytes(), ocr="hosted"), + ): + with self.assertRaises(anydoc.NeedsOcrError): + convert() + self.assertEqual(hits, []) + + def test_hosted_fallback_uses_content_before_a_misleading_extension(self): + reply = {"success": True, "data": {"markdown": HOSTED_MARKDOWN}} + with tempfile.TemporaryDirectory() as directory: + ofd_named_pdf = Path(directory) / "scan.pdf" + pdf_named_ofd = Path(directory) / "scan.ofd" + ofd_named_pdf.write_bytes(SCANNED_OFD.read_bytes()) + pdf_named_ofd.write_bytes(MIXED.read_bytes()) + with hosted_stub(200, reply) as hits: + with self.assertRaises(anydoc.NeedsOcrError): + anydoc.to_markdown(ofd_named_pdf, ocr="hosted") + self.assertEqual(hits, []) + self.assertEqual(anydoc.to_markdown(pdf_named_ofd, ocr="hosted"), HOSTED_MARKDOWN) + self.assertEqual(hits, [("/v2/parse", True)]) + def test_parse_mode_picks_the_parse_pdf_mode_auto_by_default(self): reply = {"success": True, "data": {"markdown": HOSTED_MARKDOWN}} sent = [] diff --git a/src/error.rs b/src/error.rs index dddd99b7..cfe80600 100644 --- a/src/error.rs +++ b/src/error.rs @@ -14,7 +14,7 @@ use std::fmt; pub enum ConvertError { /// The format is unknown or cannot be converted at all. Unsupported(String), - /// Some pages of a PDF are scanned or image-only and need OCR, which + /// Some PDF or OFD pages are scanned or image-only and need OCR, which /// anydoc does not do. Nothing is returned for the document, so output /// missing those pages never passes as complete. NeedsOcr { diff --git a/src/formats/detect.rs b/src/formats/detect.rs index 727a69f6..6f35ec65 100644 --- a/src/formats/detect.rs +++ b/src/formats/detect.rs @@ -74,6 +74,27 @@ fn detect_ole(bytes: &[u8]) -> Option { fn detect_zip(bytes: &[u8]) -> Option { let mut pkg = Package::open(bytes).ok()?; + // OFD's root descriptor is the package identity. Inspect its root to + // avoid classifying an arbitrary ZIP that happens to contain OFD.xml. + if let Ok(Some(tree)) = pkg.optional_xml_part("OFD.xml") + && tree + .child_elems() + .next() + .is_some_and(|root| root.is("http://www.ofdspec.org/2016", "OFD")) + { + return Some(Format::Ofd); + } + // Encrypted OFD replaces OFD.xml with encrypted entries; its encryption + // descriptor remains in clear text. + if let Ok(Some(tree)) = pkg.optional_xml_part("Encryptions.xml") + && tree + .child_elems() + .next() + .is_some_and(|root| root.is("http://www.ofdspec.org/2016", "Encryptions")) + { + return Some(Format::Ofd); + } + // ODF and EPUB designate the `mimetype` part as the package identity. if let Ok(Some(mime)) = pkg.optional_part("mimetype") { let mime = std::str::from_utf8(&mime).ok()?.trim(); diff --git a/src/formats/mod.rs b/src/formats/mod.rs index d570b264..df5399db 100644 --- a/src/formats/mod.rs +++ b/src/formats/mod.rs @@ -6,6 +6,7 @@ mod doc; mod docx; mod epub; mod odf; +mod ofd; pub mod pdf; mod ppt; mod pptx; @@ -24,6 +25,7 @@ pub fn parse(bytes: &[u8], format: Format) -> Result { Format::Odt | Format::Ods | Format::Odp => odf::parse(bytes), Format::Pptx => pptx::parse(bytes), Format::Epub => epub::parse(bytes), + Format::Ofd => ofd::parse(bytes), Format::Rtf => rtf::parse(bytes), // RTF files wearing a .doc extension are common in the wild. Format::Doc if bytes.starts_with(b"{\\rtf") => rtf::parse(bytes), diff --git a/src/formats/ofd.rs b/src/formats/ofd.rs new file mode 100644 index 00000000..c5f67da5 --- /dev/null +++ b/src/formats/ofd.rs @@ -0,0 +1,930 @@ +//! OFD fixed-page text extraction. The package, XML and path handling use +//! anydoc's existing bounded readers; no ofdrw runtime dependency is needed. + +use crate::error::ConvertError; +use crate::model::{ + Block, Cell, Document, GridBuilder, Inline, LinkTarget, List, ListItem, MarkerKind, Style, + Table, TableKind, +}; +use crate::package::xml::Element; +use crate::package::{Package, path}; +use crate::shared::header::resolve_header_rows; +use std::collections::HashMap; + +const OFD: &str = "http://www.ofdspec.org/2016"; + +pub fn parse(bytes: &[u8]) -> Result { + let mut pkg = Package::open(bytes)?; + if pkg.has_part("Encryptions.xml") { + return Err(ConvertError::Encrypted); + } + let tree = pkg.required_xml_part("OFD.xml")?; + let root = tree + .child_elems() + .next() + .ok_or_else(|| ConvertError::malformed_part("OFD.xml", "expected OFD root element"))?; + if !root.is(OFD, "OFD") { + return Err(ConvertError::malformed_part("OFD.xml", "expected OFD root element")); + } + + let mut doc = Document::default(); + let mut bodies = 0; + let mut page_count = 0u32; + let mut ocr_pages = Vec::new(); + for body in root.find_all(OFD, "DocBody") { + bodies += 1; + let root_loc = body + .find(OFD, "DocRoot") + .map(Element::text) + .ok_or_else(|| ConvertError::malformed_part("OFD.xml", "DocBody has no DocRoot"))?; + let root_path = resolve("OFD.xml", root_loc.trim())?; + let document_tree = pkg.required_xml_part(&root_path)?; + let document = document_tree.child_elems().next().ok_or_else(|| { + ConvertError::malformed_part(&root_path, "expected Document root element") + })?; + if !document.is(OFD, "Document") { + return Err(ConvertError::malformed_part(&root_path, "expected Document root element")); + } + let pages = document.find(OFD, "Pages").ok_or_else(|| { + ConvertError::malformed_part(&root_path, "Document has no Pages element") + })?; + let templates: HashMap<&str, String> = document + .find(OFD, "CommonData") + .into_iter() + .flat_map(|data| data.find_all(OFD, "TemplatePage")) + .filter_map(|template| { + Some((template.attr_any("ID")?, template.attr_any("BaseLoc")?.to_string())) + }) + .collect(); + let mut annotation_pages = HashMap::new(); + if let Some(loc) = document.find(OFD, "Annotations").map(Element::text) { + let list_path = resolve(&root_path, loc.trim())?; + if let Some(tree) = pkg.optional_xml_part(&list_path)? { + for item in tree.descendants(OFD, "Page") { + if let (Some(id), Some(file)) = + (item.attr_any("PageID"), item.find(OFD, "FileLoc")) + { + annotation_pages + .insert(id.to_string(), resolve(&list_path, file.text().trim())?); + } + } + } + } + let mut headings = Vec::new(); + if let Some(outlines) = document.find(OFD, "Outlines") { + collect_outlines(outlines, 0, &mut headings); + } + for page in pages.find_all(OFD, "Page") { + page_count += 1; + let Some(loc) = page.attr_any("BaseLoc") else { + return Err(ConvertError::malformed_part(&root_path, "Page has no BaseLoc")); + }; + let page_path = resolve(&root_path, loc)?; + let page_tree = pkg.required_xml_part(&page_path)?; + let page_xml = page_tree.child_elems().next().ok_or_else(|| { + ConvertError::malformed_part(&page_path, "expected Page root element") + })?; + if !page_xml.is(OFD, "Page") { + return Err(ConvertError::malformed_part(&page_path, "expected Page root element")); + } + let links = if let Some(annotation_path) = + page.attr_any("ID").and_then(|id| annotation_pages.get(id)) + { + read_links(&mut pkg, annotation_path)? + } else { + Vec::new() + }; + let mut template_stats = ContentStats::default(); + for template in page_xml.find_all(OFD, "Template") { + if template.attr_any("ZOrder") != Some("Foreground") { + template_stats.merge(append_template( + &mut pkg, &templates, &root_path, template, &mut doc, + )?); + } + } + let mut page_stats = ContentStats::default(); + if let Some(content) = page_xml.find(OFD, "Content") { + page_stats = append_content(content, &page_path, &mut doc, &mut headings, &links)?; + } + for template in page_xml.find_all(OFD, "Template") { + if template.attr_any("ZOrder") == Some("Foreground") { + template_stats.merge(append_template( + &mut pkg, &templates, &root_path, template, &mut doc, + )?); + } + } + if (page_stats.image && !page_stats.text) + || (template_stats.image && !template_stats.text && !page_stats.text) + { + ocr_pages.push(page_count); + } + } + } + if bodies == 0 { + return Err(ConvertError::malformed_part("OFD.xml", "no DocBody elements")); + } + if !ocr_pages.is_empty() { + return Err(ConvertError::NeedsOcr { pages: ocr_pages, page_count }); + } + Ok(doc) +} + +fn append_template( + pkg: &mut Package<'_>, + templates: &HashMap<&str, String>, + document_path: &str, + template: &Element, + doc: &mut Document, +) -> Result { + let Some(loc) = template.attr_any("TemplateID").and_then(|id| templates.get(id)) else { + return Ok(ContentStats::default()); + }; + let template_path = resolve(document_path, loc)?; + let tree = pkg.required_xml_part(&template_path)?; + let Some(page) = tree.child_elems().next() else { + return Err(ConvertError::malformed_part(&template_path, "empty template page")); + }; + if !page.is(OFD, "Page") { + return Err(ConvertError::malformed_part(&template_path, "expected Page root element")); + } + if let Some(content) = page.find(OFD, "Content") { + return append_content(content, &template_path, doc, &mut Vec::new(), &[]); + } + Ok(ContentStats::default()) +} + +#[derive(Default)] +struct ContentStats { + image: bool, + text: bool, +} + +impl ContentStats { + fn merge(&mut self, other: Self) { + self.image |= other.image; + self.text |= other.text; + } +} + +fn read_links(pkg: &mut Package<'_>, part: &str) -> Result, ConvertError> { + let Some(tree) = pkg.optional_xml_part(part)? else { return Ok(Vec::new()) }; + let mut links = Vec::new(); + for annot in tree.descendants(OFD, "Annot") { + if annot.attr_any("Type") != Some("Link") { + continue; + } + let Some(appearance) = annot.find(OFD, "Appearance") else { continue }; + let Some(bounds) = appearance.attr_any("Boundary").and_then(Bounds::parse) else { + continue; + }; + let Some(uri) = appearance + .descendants(OFD, "URI") + .next() + .and_then(|uri| uri.attr_any("URI")) + .filter(|uri| !uri.is_empty()) + else { + continue; + }; + links.push((bounds, uri.to_string())); + } + Ok(links) +} + +#[derive(Clone, Copy)] +struct Bounds { + x: f64, + y: f64, + w: f64, + h: f64, +} + +impl Bounds { + fn parse(value: &str) -> Option { + let values = + value.split_whitespace().map(str::parse::).collect::, _>>().ok()?; + let [x, y, w, h] = values.as_slice() else { return None }; + if !values.iter().all(|v| v.is_finite()) || *w <= 0.0 || *h <= 0.0 { + return None; + } + Some(Self { x: *x, y: *y, w: *w, h: *h }) + } + + fn contains_center(self, other: Self) -> bool { + let x = other.x + other.w / 2.0; + let y = other.y + other.h / 2.0; + x >= self.x - 0.1 + && x <= self.x + self.w + 0.1 + && y >= self.y - 0.1 + && y <= self.y + self.h + 0.1 + } +} + +struct TextObject { + text: String, + style: Style, + size: Option, + bounds: Option, + link: Option, +} + +fn text_inline(object: &TextObject) -> Inline { + let text = Inline::Text { text: object.text.trim().to_string(), style: object.style }; + match &object.link { + Some(uri) => { + Inline::Link { content: vec![text], target: LinkTarget::External(uri.clone()) } + } + None => text, + } +} + +fn collect_outlines(parent: &Element, depth: u8, headings: &mut Vec<(String, u8)>) { + for item in parent.find_all(OFD, "OutlineElem") { + if let Some(title) = item.attr_any("Title") { + headings.push((title.trim().to_string(), depth.saturating_add(1).min(6))); + } + collect_outlines(item, depth.saturating_add(1), headings); + } +} + +fn append_content( + content: &Element, + part: &str, + doc: &mut Document, + headings: &mut Vec<(String, u8)>, + links: &[(Bounds, String)], +) -> Result { + let start = doc.blocks.len(); + let mut objects = Vec::new(); + let mut stats = ContentStats { + image: content.descendants(OFD, "ImageObject").next().is_some(), + text: false, + }; + for object in content.descendants(OFD, "TextObject") { + let mut text = String::new(); + for code in object.find_all(OFD, "TextCode") { + text.push_str(&decode_text_code(&code.text(), part)?); + } + if text.trim().is_empty() { + continue; + } + stats.text = true; + let weight = object.attr_any("Weight").and_then(|v| v.parse::().ok()).unwrap_or(400); + let style = Style { + bold: weight >= 600, + italic: object.attr_any("Italic").is_some_and(|v| v.eq_ignore_ascii_case("true")), + ..Style::PLAIN + }; + let bounds = object.attr_any("Boundary").and_then(Bounds::parse); + objects.push(TextObject { + text, + style, + size: object + .attr_any("Size") + .and_then(|v| v.parse::().ok()) + .filter(|v| *v > 0.0 && v.is_finite()), + bounds, + link: object + .descendants(OFD, "URI") + .next() + .and_then(|uri| uri.attr_any("URI")) + .filter(|uri| !uri.is_empty()) + .map(str::to_string) + .or_else(|| { + bounds.and_then(|bounds| { + links + .iter() + .find(|(region, _)| region.contains_center(bounds)) + .map(|(_, uri)| uri.clone()) + }) + }), + }); + } + + let rects: Vec = + content.descendants(OFD, "PathObject").filter_map(rectangle_border).collect(); + let tables = detect_tables(&rects, &objects)?; + let mut used = vec![false; objects.len()]; + let mut table_at = Vec::new(); + for (table, indexes) in tables { + if indexes.iter().any(|index| used[*index]) { + continue; + } + if let Some(at) = indexes.iter().copied().min() { + table_at.push((at, table)); + } + for index in indexes { + used[index] = true; + } + } + table_at.sort_by_key(|(at, _)| *at); + let mut table_at = table_at.into_iter().peekable(); + let mut sizes: Vec = + objects.iter().enumerate().filter(|(i, _)| !used[*i]).filter_map(|(_, o)| o.size).collect(); + sizes.sort_by(f64::total_cmp); + let body_size = sizes.get(sizes.len().saturating_sub(1) / 2).copied(); + for (index, object) in objects.into_iter().enumerate() { + if table_at.peek().is_some_and(|(at, _)| *at == index) { + let (_, table) = table_at.next().expect("table anchor exists"); + doc.blocks.push(Block::Table(table)); + } + if used[index] { + continue; + } + let text = object.text.trim().to_string(); + let inline = text_inline(&object); + if let Some(pos) = headings.iter().position(|(title, _)| title == &text) { + let (_, level) = headings.remove(pos); + doc.blocks.push(Block::heading(level, vec![inline])); + } else if let (Some(size), Some(body)) = (object.size, body_size) + && size >= body * 1.4 + && text.chars().count() <= 120 + { + let level = if size >= body * 1.8 { + 1 + } else if size >= body * 1.6 { + 2 + } else { + 3 + }; + doc.blocks.push(Block::heading(level, vec![inline])); + } else { + doc.blocks.push(Block::Paragraph(vec![inline])); + } + } + fold_lists(&mut doc.blocks, start); + Ok(stats) +} + +fn rectangle_border(path: &Element) -> Option { + if !path.attr_any("Stroke")?.eq_ignore_ascii_case("true") { + return None; + } + let rect = Bounds::parse(path.attr_any("Boundary")?)?; + let data = path.find(OFD, "AbbreviatedData")?.text(); + let parts: Vec<&str> = data.split_whitespace().collect(); + let ["M", x0, y0, "L", x1, y1, "L", x2, y2, "L", x3, y3, "C"] = parts.as_slice() else { + return None; + }; + let points = [x0, y0, x1, y1, x2, y2, x3, y3] + .map(|v| v.parse::().ok()) + .into_iter() + .collect::>>()?; + let expected = [0.0, 0.0, rect.w, 0.0, rect.w, rect.h, 0.0, rect.h]; + if points.iter().zip(expected).all(|(actual, want)| (*actual - want).abs() < 0.1) { + Some(rect) + } else { + None + } +} + +fn list_marker(text: &str) -> Option<(MarkerKind, u64, &str)> { + let text = text.trim_start(); + if let Some(body) = text.strip_prefix(['•', '●', '-', '*']) + && body.starts_with(char::is_whitespace) + { + return Some((MarkerKind::Bullet, 1, body.trim_start())); + } + let digits = text.bytes().take_while(u8::is_ascii_digit).count(); + if digits == 0 || digits > 9 { + return None; + } + let number = text[..digits].parse().ok()?; + let rest = text.get(digits..)?; + let body = rest.strip_prefix(['.', ')'])?; + if body.starts_with(char::is_whitespace) { + Some((MarkerKind::Decimal, number, body.trim_start())) + } else { + None + } +} + +fn fold_lists(blocks: &mut Vec, start: usize) { + let tail = blocks.split_off(start); + let mut output = Vec::with_capacity(tail.len()); + let mut iter = tail.into_iter().peekable(); + while let Some(block) = iter.next() { + let Block::Paragraph(inlines) = &block else { + output.push(block); + continue; + }; + let [Inline::Text { text, .. }] = inlines.as_slice() else { + output.push(block); + continue; + }; + let Some((kind, number, _)) = list_marker(text) else { + output.push(block); + continue; + }; + let mut run = vec![block]; + while let Some(Block::Paragraph(next)) = iter.peek() { + let [Inline::Text { text, .. }] = next.as_slice() else { break }; + let Some((next_kind, next_number, _)) = list_marker(text) else { break }; + if next_kind != kind + || (kind == MarkerKind::Decimal && next_number != number + run.len() as u64) + { + break; + } + run.push(iter.next().expect("peeked block exists")); + } + if run.len() < 2 { + output.extend(run); + continue; + } + let items = run + .into_iter() + .filter_map(|block| { + let Block::Paragraph(mut inlines) = block else { return None }; + let Inline::Text { text, .. } = &mut inlines[0] else { return None }; + let (_, _, body) = list_marker(text)?; + *text = body.to_string(); + Some(ListItem { blocks: vec![Block::Paragraph(inlines)], marker_label: None }) + }) + .collect(); + output.push(Block::List(List { marker: kind, start: number, items })); + } + blocks.extend(output); +} + +fn detect_tables( + rects: &[Bounds], + objects: &[TextObject], +) -> Result)>, ConvertError> { + if rects.len() < 4 || rects.len() > 2000 || objects.len() > 5000 { + return Ok(Vec::new()); + } + let mut seen = vec![false; rects.len()]; + let mut tables = Vec::new(); + for first in 0..rects.len() { + if seen[first] { + continue; + } + seen[first] = true; + let mut group = vec![rects[first]]; + let mut pending = vec![first]; + while let Some(index) = pending.pop() { + for other in 0..rects.len() { + if !seen[other] && borders_touch(rects[index], rects[other]) { + seen[other] = true; + group.push(rects[other]); + pending.push(other); + } + } + } + if let Some(table) = detect_table(&group, objects)? { + tables.push(table); + } + } + Ok(tables) +} + +fn borders_touch(a: Bounds, b: Bounds) -> bool { + let overlap = |a0: f64, a1: f64, b0: f64, b1: f64| a1.min(b1) - a0.max(b0) > 0.1; + (((a.x + a.w - b.x).abs() < 0.1 || (b.x + b.w - a.x).abs() < 0.1) + && overlap(a.y, a.y + a.h, b.y, b.y + b.h)) + || (((a.y + a.h - b.y).abs() < 0.1 || (b.y + b.h - a.y).abs() < 0.1) + && overlap(a.x, a.x + a.w, b.x, b.x + b.w)) +} + +fn detect_table( + rects: &[Bounds], + objects: &[TextObject], +) -> Result)>, ConvertError> { + // A border-only grid is strong evidence for a table. Without a complete + // grid, leave the text as paragraphs rather than inventing cell order. + if rects.len() < 4 || rects.len() > 2000 { + return Ok(None); + } + let mut xs = Vec::new(); + let mut ys = Vec::new(); + for rect in rects { + xs.extend([rect.x, rect.x + rect.w]); + ys.extend([rect.y, rect.y + rect.h]); + } + xs.sort_by(f64::total_cmp); + ys.sort_by(f64::total_cmp); + xs.dedup_by(|a, b| (*a - *b).abs() < 0.1); + ys.dedup_by(|a, b| (*a - *b).abs() < 0.1); + let columns = xs.len().saturating_sub(1); + let rows = ys.len().saturating_sub(1); + if columns < 2 || rows < 2 || columns.saturating_mul(rows) > 2000 { + return Ok(None); + } + let mut slots = vec![false; rows * columns]; + let mut origins = Vec::new(); + for rect in rects { + let edge = |edges: &[f64], value: f64| edges.iter().position(|v| (*v - value).abs() < 0.1); + let (Some(c0), Some(c1), Some(r0), Some(r1)) = ( + edge(&xs, rect.x), + edge(&xs, rect.x + rect.w), + edge(&ys, rect.y), + edge(&ys, rect.y + rect.h), + ) else { + return Ok(None); + }; + if c0 >= c1 || r0 >= r1 { + return Ok(None); + } + for row in r0..r1 { + for column in c0..c1 { + let slot = &mut slots[row * columns + column]; + if *slot { + return Ok(None); + } + *slot = true; + } + } + origins.push((r0, c0, r1 - r0, c1 - c0, *rect)); + } + if slots.iter().any(|slot| !slot) { + return Ok(None); + } + origins.sort_by_key(|(row, col, ..)| (*row, *col)); + let mut builder = GridBuilder::new(); + let mut used = Vec::new(); + let mut first_row_bold = true; + let mut first_row_has_text = false; + let mut later_bold = false; + for row in 0..rows { + builder.next_row(); + for (_, _, row_span, col_span, rect) in origins.iter().filter(|(r, ..)| *r == row) { + let mut content = Vec::new(); + for (index, object) in objects.iter().enumerate() { + if let Some(bounds) = object.bounds + && rect.contains_center(bounds) + && !used.contains(&index) + { + content.push(text_inline(object)); + used.push(index); + if row == 0 { + first_row_has_text = true; + first_row_bold &= object.style.bold; + } else { + later_bold |= object.style.bold; + } + } + } + let blocks = + if content.is_empty() { Vec::new() } else { vec![Block::Paragraph(content)] }; + builder.place(Cell::spanning(blocks, *col_span as u32, *row_span as u32))?; + } + } + if used.len() < 2 { + return Ok(None); + } + let mut table = builder.finish(TableKind::Data); + table.header_rows = resolve_header_rows( + &table, + usize::from(first_row_has_text && first_row_bold && !later_bold), + ); + Ok(Some((table, used))) +} + +fn resolve(base: &str, loc: &str) -> Result { + if loc.is_empty() { + return Err(ConvertError::malformed_part(base, "empty OFD part location")); + } + let target = path::resolve(base, loc)?; + if target.path.is_empty() { + return Err(ConvertError::malformed_part(base, "empty OFD part location")); + } + Ok(target.path) +} + +// OFD TextCode uses backslash followed by four hex digits for UTF-16 units. +// Surrogate pairs are decoded together; literal backslashes remain literal. +fn decode_text_code(raw: &str, part: &str) -> Result { + let bytes = raw.as_bytes(); + let mut out = String::with_capacity(raw.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\\' + && bytes.get(i + 1..i + 5).is_some_and(|s| s.iter().all(u8::is_ascii_hexdigit)) + { + let unit = u16::from_str_radix(&raw[i + 1..i + 5], 16) + .map_err(|_| ConvertError::malformed_part(part, "invalid TextCode escape"))?; + i += 5; + let scalar = if (0xD800..=0xDBFF).contains(&unit) { + if bytes.get(i) != Some(&b'\\') + || !bytes.get(i + 1..i + 5).is_some_and(|s| s.iter().all(u8::is_ascii_hexdigit)) + { + return Err(ConvertError::malformed_part(part, "unpaired TextCode surrogate")); + } + let low = u16::from_str_radix(&raw[i + 1..i + 5], 16) + .map_err(|_| ConvertError::malformed_part(part, "invalid TextCode escape"))?; + if !(0xDC00..=0xDFFF).contains(&low) { + return Err(ConvertError::malformed_part(part, "unpaired TextCode surrogate")); + } + i += 5; + 0x10000 + (u32::from(unit - 0xD800) << 10) + u32::from(low - 0xDC00) + } else { + u32::from(unit) + }; + let ch = char::from_u32(scalar) + .ok_or_else(|| ConvertError::malformed_part(part, "unpaired TextCode surrogate"))?; + out.push(ch); + } else { + let ch = raw[i..] + .chars() + .next() + .ok_or_else(|| ConvertError::malformed_part(part, "invalid TextCode"))?; + out.push(ch); + i += ch.len_utf8(); + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Cursor, Write}; + + fn package(parts: &[(&str, &str)]) -> Vec { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (name, xml) in parts { + writer.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + writer.write_all(xml.as_bytes()).unwrap(); + } + writer.finish().unwrap().into_inner() + } + + #[test] + fn text_from_pages_and_bodies_in_declared_order() { + let bytes = package(&[ + ( + "OFD.xml", + r#"Doc_0/Document.xmlDoc_1/Document.xml"#, + ), + ( + "Doc_0/Document.xml", + r#""#, + ), + ( + "Doc_0/Pages/p2.xml", + r#"