diff --git a/CHANGELOG.md b/CHANGELOG.md index f63e8bb..163ce3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add a pluggable post-download `ContentProcessor` registry and built-in `PdfProcessor` that extracts Markdown from text-based PDFs with local `pdf-inspector` processing and explicit OCR guidance for unsupported pages. +- Add a built-in `OfficeProcessor` that converts Word (DOCX), PowerPoint (PPTX) + and spreadsheet (XLSX, XLSM, XLS, ODS) documents to Markdown locally, with + zip-bomb limits checked before any part is inflated. - Discover agent-facing resources from HTTP and HTML declarations plus bounded conventional probes, including `llms.txt`, `auth.md`, OAuth metadata, MCP server cards, A2A agent cards, and Agent Skills indexes. Markdown responses @@ -27,6 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Markdown requests now ask for `text/markdown` first in the `Accept` header, so + sites that serve Markdown on request return it instead of HTML. - `ContentProcessorInput` now includes the requested output format and HTML content focus. Custom processors constructing this public input must populate the new fields. diff --git a/Cargo.lock b/Cargo.lock index 1944d41..a24c9f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,13 +132,23 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2 1.0.106", "quote 1.0.46", - "syn 2.0.118", + "syn 3.0.9", +] + +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", ] [[package]] @@ -266,6 +276,23 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "calamine" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +dependencies = [ + "atoi_simd", + "byteorder", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip", +] + [[package]] name = "cbc" version = "0.1.2" @@ -382,6 +409,15 @@ dependencies = [ "cc", ] +[[package]] +name = "codepage" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdff162541cd8b79de82e2edcc7eff3a8c2a6dc3d75152636028f96d93de3b26" +dependencies = [ + "encoding_rs", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -483,9 +519,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] @@ -602,6 +638,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + [[package]] name = "defmt" version = "1.1.1" @@ -846,6 +888,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fast-float2" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e8948ce679d00a02a94739ea185595dca7118ed04feb991127e443bd3d761f" + [[package]] name = "fastrand" version = "2.4.1" @@ -859,12 +907,14 @@ dependencies = [ "async-trait", "base64", "bytes", + "calamine", "ed25519-dalek", "futures", "libc", "mira-eval", "pdf-inspector", "percent-encoding", + "quick-xml", "rakers", "rand 0.10.2", "reqwest", @@ -881,6 +931,7 @@ dependencies = [ "tracing-subscriber", "url", "wiremock", + "zip", ] [[package]] @@ -925,6 +976,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1626,9 +1678,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lopdf" @@ -2137,6 +2189,16 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quinn" version = "0.11.11" @@ -3000,6 +3062,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2 1.0.106", + "quote 1.0.46", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -3376,6 +3449,12 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -3965,8 +4044,40 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 581c34e..28986c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,9 @@ tower = { version = "0.5", features = ["util"] } # Binary document extraction pdf-inspector = "0.1.7" +calamine = { version = "0.36", default-features = false } +zip = { version = "8", default-features = false, features = ["deflate"] } +quick-xml = "0.41" # Platform filesystem flags libc = "0.2" diff --git a/README.md b/README.md index c729c03..60cc502 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ AI-friendly web content fetching tool designed for LLM consumption. Rust library - **Agent content focus** - Optional low-noise extraction mode for AI agents - **Crawl discovery** - Optional bounded same-origin page discovery for AI agents - **HTML-to-Text** - Plain text extraction with clean formatting -- **Content processors** - Post-download extraction for text PDFs, with an extensible registry +- **Content processors** - Post-download extraction for text PDFs and Office documents, with an extensible registry - **Binary detection** - Returns metadata only for unsupported binary formats - **Timeout handling** - 1s first-byte, 30s body with partial content on timeout - **Safety limits** - 10 MB default decompressed body cap with truncation @@ -49,6 +49,8 @@ requests. - `PdfProcessor` - text-based PDF classification and Markdown extraction via [`pdf-inspector`](https://github.com/firecrawl/pdf-inspector); scanned or image-only pages are reported as requiring OCR +- `OfficeProcessor` - Word (DOCX), PowerPoint (PPTX) and spreadsheet (XLSX, XLS, + ODS) documents to Markdown, parsed locally with zip-bomb limits Custom processors implement `ContentProcessor` and can be registered in a `ContentProcessorRegistry`. Pass that registry to @@ -339,10 +341,14 @@ body limit and converted to Markdown. PDF parsing runs locally; no OCR service o additional network request is used. Scanned/image-only PDFs report `use_ocr` as the suggested next action. +Office documents are converted the same way, locally: Word headings, lists and +tables, one section per PowerPoint slide, and one Markdown table per spreadsheet +sheet (DOCX, PPTX, XLSX, XLS, ODS). Legacy DOC and PPT files are not supported. + Other binary content returns metadata only: - Images, audio, video, fonts - Archives (zip, tar, rar, 7z) -- Office documents +- Legacy Word and PowerPoint files (DOC, PPT) ### HTML Conversion diff --git a/crates/fetchkit/Cargo.toml b/crates/fetchkit/Cargo.toml index 9b231e6..5489dc9 100644 --- a/crates/fetchkit/Cargo.toml +++ b/crates/fetchkit/Cargo.toml @@ -31,6 +31,9 @@ bytes = { workspace = true } async-trait = { workspace = true } tower = { workspace = true } pdf-inspector = { workspace = true } +calamine = { workspace = true } +zip = { workspace = true } +quick-xml = { workspace = true } libc = { workspace = true } rakers = { workspace = true, optional = true } diff --git a/crates/fetchkit/src/content.rs b/crates/fetchkit/src/content.rs index 7e704f2..6d693ed 100644 --- a/crates/fetchkit/src/content.rs +++ b/crates/fetchkit/src/content.rs @@ -16,11 +16,13 @@ use crate::convert::{ extract_headings, extract_metadata, extract_readable_content, filter_excessive_newlines, html_to_text, strip_boilerplate, BuiltinHtmlToMarkdownConverter, HtmlToMarkdownConverter, }; +use crate::office::OfficeProcessor; use crate::{PageMetadata, PageQuality}; -// pdf-inspector uses Rayon internally; bound concurrent documents so batch fetches -// cannot multiply parser thread pools without limit. -static PDF_PROCESSING_LIMIT: LazyLock> = +// pdf-inspector uses Rayon internally, and every document parser is CPU-bound; bound +// concurrent documents (PDF and Office alike) so batch fetches cannot multiply parser +// work without limit. +pub(crate) static DOCUMENT_PROCESSING_LIMIT: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(2))); /// Bounded response bytes and metadata passed to a [`ContentProcessor`]. @@ -147,6 +149,7 @@ impl ContentProcessorRegistry { pub fn with_defaults() -> Self { let mut registry = Self::new(); registry.register(Box::new(PdfProcessor)); + registry.register(Box::new(OfficeProcessor)); registry.register(Box::new(HtmlProcessor::default())); registry } @@ -271,15 +274,17 @@ impl ContentProcessor for HtmlProcessor { /// Extracts native text PDFs as Markdown using `pdf-inspector`. pub struct PdfProcessor; -async fn run_pdf_task(limit: Arc, task: F) -> Result +pub(crate) async fn run_document_task( + limit: Arc, + task: F, +) -> Result where T: Send + 'static, F: FnOnce() -> T + Send + 'static, { - let permit = limit - .acquire_owned() - .await - .map_err(|error| ContentProcessorError(format!("PDF processing unavailable: {error}")))?; + let permit = limit.acquire_owned().await.map_err(|error| { + ContentProcessorError(format!("document processing unavailable: {error}")) + })?; tokio::task::spawn_blocking(move || { // Keep the permit in the blocking task: cancelling its async caller must not // admit another parse while this non-cancellable work is still running. @@ -287,7 +292,7 @@ where task() }) .await - .map_err(|error| ContentProcessorError(format!("PDF processor task failed: {error}"))) + .map_err(|error| ContentProcessorError(format!("document processor task failed: {error}"))) } #[async_trait] @@ -315,7 +320,7 @@ impl ContentProcessor for PdfProcessor { &self, input: ContentProcessorInput, ) -> Result { - let result = run_pdf_task(Arc::clone(&PDF_PROCESSING_LIMIT), move || { + let result = run_document_task(Arc::clone(&DOCUMENT_PROCESSING_LIMIT), move || { process_pdf_mem(&input.body) }) .await? @@ -445,7 +450,7 @@ mod tests { let limit = Arc::new(Semaphore::new(1)); let (started_tx, started_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel(); - let handle = tokio::spawn(run_pdf_task(Arc::clone(&limit), move || { + let handle = tokio::spawn(run_document_task(Arc::clone(&limit), move || { started_tx.send(()).unwrap(); release_rx.recv().unwrap(); })); diff --git a/crates/fetchkit/src/fetchers/default.rs b/crates/fetchkit/src/fetchers/default.rs index cad0a02..35db5b4 100644 --- a/crates/fetchkit/src/fetchers/default.rs +++ b/crates/fetchkit/src/fetchers/default.rs @@ -85,6 +85,7 @@ const BINARY_PREFIXES: &[&str] = &[ "application/x-7z", "application/vnd.ms-", "application/vnd.openxmlformats", + "application/vnd.oasis.opendocument", "font/", ]; @@ -461,7 +462,9 @@ impl Fetcher for DefaultFetcher { let max_body_size = options.max_body_size.unwrap_or(DEFAULT_MAX_BODY_SIZE); let accept = if wants_markdown { - "text/html, text/markdown, text/plain, */*;q=0.8" + // Markdown first: sites that serve it on request (Markdown for Agents) skip + // HTML conversion entirely. + "text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.7" } else if wants_text { "text/html, text/plain, */*;q=0.8" } else { diff --git a/crates/fetchkit/src/lib.rs b/crates/fetchkit/src/lib.rs index d6ba4b2..e33288a 100644 --- a/crates/fetchkit/src/lib.rs +++ b/crates/fetchkit/src/lib.rs @@ -85,6 +85,7 @@ mod dns; mod error; pub mod fetchers; pub mod file_saver; +mod office; mod tool; pub mod transport; mod types; @@ -107,6 +108,7 @@ pub use fetchers::{ RSSFeedFetcher, StackOverflowFetcher, TwitterFetcher, WikipediaFetcher, YouTubeFetcher, }; pub use file_saver::{FileSaveError, FileSaver, LocalFileSaver, SaveResult}; +pub use office::OfficeProcessor; pub use tool::{ Tool, ToolBuilder, ToolExecution, ToolImage, ToolOutput, ToolOutputMetadata, ToolService, ToolStatus, @@ -128,7 +130,7 @@ pub const DEFAULT_USER_AGENT: &str = "Everruns Fetchkit/1.0"; /// Backward-compatible full description string with file-saving enabled. pub const TOOL_DESCRIPTION: &str = - "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses or save bytes to file."; + "Fetch URL content as text or markdown, including text-based PDFs and Office documents; return metadata for unsupported binary responses or save bytes to file."; /// Backward-compatible help document with file-saving enabled. pub static TOOL_LLMTXT: std::sync::LazyLock = diff --git a/crates/fetchkit/src/office.rs b/crates/fetchkit/src/office.rs new file mode 100644 index 0000000..aa22f69 --- /dev/null +++ b/crates/fetchkit/src/office.rs @@ -0,0 +1,973 @@ +//! Office document extraction: Word (DOCX), PowerPoint (PPTX) and spreadsheets +//! (XLSX, XLS, ODS) to Markdown. +//! +//! Decisions: +//! - Everything is parsed locally from the bounded response bytes. Documents never +//! leave the host, and no account or network call is involved, so the processor +//! behaves the same in every embedding (CLI, MCP, library, hosted runtimes). +//! - DOCX and PPTX text is read straight from their WordprocessingML / DrawingML XML +//! parts with `quick-xml`; full fidelity (styles, images, footnotes) is not a goal, +//! readable text with headings, lists and tables is. Spreadsheets go through +//! `calamine`, which also covers legacy binary XLS. +//! - THREAT[TM-DOS-008]: OOXML and ODS files are zip archives. Before any part is +//! inflated, the central directory is checked against an entry-count and a total +//! uncompressed-size budget, and every part fetchkit reads itself is read through a +//! size-capped reader, so a small zip bomb within `max_body_size` cannot expand +//! without limit. Spreadsheet output is capped per sheet (rows and columns) and the +//! caller caps the final Markdown at `max_body_size`. Parsing runs on a blocking +//! task under the same two-document concurrency cap as PDFs. + +use std::borrow::Cow; +use std::io::{Cursor, Read}; + +use async_trait::async_trait; +use calamine::{open_workbook_auto_from_rs, Data, Reader}; +use quick_xml::events::{BytesStart, Event}; +use quick_xml::Reader as XmlReader; +use url::Url; + +use crate::content::{ + run_document_task, ContentProcessor, ContentProcessorError, ContentProcessorInput, + ProcessedContent, DOCUMENT_PROCESSING_LIMIT, +}; +use crate::{PageMetadata, PageQuality}; + +/// Maximum number of entries in an Office zip archive. +const MAX_ARCHIVE_ENTRIES: usize = 10_000; +/// Maximum total uncompressed size of an Office zip archive. +const MAX_ARCHIVE_UNCOMPRESSED_BYTES: u64 = 200 * 1024 * 1024; +/// Maximum uncompressed size of one XML part fetchkit reads itself. +const MAX_PART_BYTES: u64 = 64 * 1024 * 1024; +/// Maximum slides read from one presentation. +const MAX_SLIDES: usize = 1_000; +/// Maximum rows rendered per spreadsheet sheet. +const MAX_SHEET_ROWS: usize = 1_000; +/// Maximum columns rendered per spreadsheet sheet. +const MAX_SHEET_COLUMNS: usize = 50; + +/// Supported Office document kinds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum OfficeKind { + Docx, + Pptx, + Spreadsheet, +} + +impl OfficeKind { + fn from_media_type(media_type: &str) -> Option { + let media_type = media_type.to_ascii_lowercase(); + match media_type.as_str() { + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => { + Some(Self::Docx) + } + "application/vnd.openxmlformats-officedocument.presentationml.presentation" => { + Some(Self::Pptx) + } + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + | "application/vnd.ms-excel" + | "application/vnd.ms-excel.sheet.macroenabled.12" + | "application/vnd.oasis.opendocument.spreadsheet" => Some(Self::Spreadsheet), + _ => None, + } + } + + fn from_path(path: &str) -> Option { + let path = path.to_ascii_lowercase(); + let extension = path.rsplit_once('.')?.1; + match extension { + "docx" => Some(Self::Docx), + "pptx" => Some(Self::Pptx), + "xlsx" | "xlsm" | "xls" | "ods" => Some(Self::Spreadsheet), + _ => None, + } + } + + fn detect(url: &Url, content_type: Option<&str>) -> Option { + let media_type = content_type + .and_then(|value| value.split(';').next()) + .map(str::trim) + .unwrap_or_default(); + if let Some(kind) = Self::from_media_type(media_type) { + return Some(kind); + } + // Download endpoints often serve documents as generic binary; trust the + // extension only then, never over a specific media type. + let generic = media_type.is_empty() + || media_type.eq_ignore_ascii_case("application/octet-stream") + || media_type.eq_ignore_ascii_case("application/zip"); + if generic { + Self::from_path(url.path()) + } else { + None + } + } + + fn extraction_method(self) -> &'static str { + match self { + Self::Docx => "office_docx", + Self::Pptx => "office_pptx", + Self::Spreadsheet => "office_spreadsheet", + } + } +} + +/// Extracts Word, PowerPoint and spreadsheet documents as Markdown. +pub struct OfficeProcessor; + +#[async_trait] +impl ContentProcessor for OfficeProcessor { + fn name(&self) -> &'static str { + "office" + } + + fn matches(&self, url: &Url, content_type: Option<&str>) -> bool { + OfficeKind::detect(url, content_type).is_some() + } + + async fn process( + &self, + input: ContentProcessorInput, + ) -> Result { + let kind = OfficeKind::detect(&input.url, input.content_type.as_deref()) + .ok_or_else(|| ContentProcessorError("unsupported Office document".to_string()))?; + let body = input.body; + let extracted = run_document_task(DOCUMENT_PROCESSING_LIMIT.clone(), move || { + extract(kind, &body) + }) + .await??; + + let content = Some(extracted.markdown).filter(|value| !value.trim().is_empty()); + let error = content + .is_none() + .then(|| "Document contains no extractable text".to_string()); + let metadata = PageMetadata { + title: extracted.title, + extraction_method: Some(kind.extraction_method().to_string()), + ..Default::default() + }; + let quality = PageQuality { + score: if content.is_none() { + 0.0 + } else if extracted.warnings.is_empty() { + 1.0 + } else { + 0.8 + }, + warnings: extracted.warnings, + extraction_method: Some(kind.extraction_method().to_string()), + ..Default::default() + }; + + Ok(ProcessedContent { + format: content.as_ref().map(|_| "markdown".to_string()), + content, + metadata: Some(metadata), + quality: Some(quality), + error, + }) + } +} + +#[derive(Debug, Default)] +struct Extracted { + markdown: String, + title: Option, + warnings: Vec, +} + +fn extract(kind: OfficeKind, body: &[u8]) -> Result { + match kind { + OfficeKind::Docx => { + let mut archive = open_archive(body)?; + let document = read_part(&mut archive, "word/document.xml")? + .ok_or_else(|| ContentProcessorError("DOCX has no word/document.xml".into()))?; + Ok(Extracted { + markdown: docx_to_markdown(&document)?, + title: core_title(&mut archive), + warnings: Vec::new(), + }) + } + OfficeKind::Pptx => { + let mut archive = open_archive(body)?; + pptx_to_markdown(&mut archive) + } + OfficeKind::Spreadsheet => { + // XLS is a compound binary file, not a zip; only zip containers get the + // archive budget check. + if body.starts_with(b"PK") { + let mut archive = open_archive(body)?; + let title = core_title(&mut archive); + let mut extracted = spreadsheet_to_markdown(body)?; + extracted.title = title; + Ok(extracted) + } else { + spreadsheet_to_markdown(body) + } + } + } +} + +type Archive<'a> = zip::ZipArchive>; + +fn open_archive(body: &[u8]) -> Result, ContentProcessorError> { + open_archive_within(body, MAX_ARCHIVE_UNCOMPRESSED_BYTES) +} + +fn open_archive_within(body: &[u8], budget: u64) -> Result, ContentProcessorError> { + let mut archive = zip::ZipArchive::new(Cursor::new(body)) + .map_err(|error| ContentProcessorError(format!("invalid Office archive: {error}")))?; + if archive.len() > MAX_ARCHIVE_ENTRIES { + return Err(ContentProcessorError(format!( + "Office archive has too many entries ({})", + archive.len() + ))); + } + let mut total: u64 = 0; + for index in 0..archive.len() { + let entry = archive + .by_index_raw(index) + .map_err(|error| ContentProcessorError(format!("invalid Office archive: {error}")))?; + total = total.saturating_add(entry.size()); + } + if total > budget { + return Err(ContentProcessorError(format!( + "Office archive expands to {total} bytes, over the {budget} byte limit" + ))); + } + Ok(archive) +} + +/// Read one archive part as UTF-8, or `None` when the part does not exist. +fn read_part( + archive: &mut Archive<'_>, + name: &str, +) -> Result, ContentProcessorError> { + let entry = match archive.by_name(name) { + Ok(entry) => entry, + Err(zip::result::ZipError::FileNotFound) => return Ok(None), + Err(error) => { + return Err(ContentProcessorError(format!( + "cannot read Office part {name}: {error}" + ))) + } + }; + // The declared size was budgeted in open_archive, but declared sizes can lie; + // cap what is actually inflated. + let mut bytes = Vec::new(); + entry + .take(MAX_PART_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|error| { + ContentProcessorError(format!("cannot read Office part {name}: {error}")) + })?; + if bytes.len() as u64 > MAX_PART_BYTES { + return Err(ContentProcessorError(format!( + "Office part {name} is larger than {MAX_PART_BYTES} bytes" + ))); + } + Ok(Some(String::from_utf8_lossy(&bytes).into_owned())) +} + +/// `dc:title` from `docProps/core.xml`, when present. +fn core_title(archive: &mut Archive<'_>) -> Option { + let core = read_part(archive, "docProps/core.xml").ok()??; + let mut reader = XmlReader::from_str(&core); + let mut in_title = false; + let mut title = String::new(); + loop { + match reader.read_event().ok()? { + Event::Start(start) if start.local_name().as_ref() == b"title" => in_title = true, + Event::End(end) if end.local_name().as_ref() == b"title" => break, + Event::Text(text) if in_title => title.push_str(&text.xml10_content().ok()?), + Event::GeneralRef(reference) if in_title => push_reference(&mut title, &reference), + Event::Eof => break, + _ => {} + } + } + let title = title.trim(); + (!title.is_empty()).then(|| title.to_string()) +} + +fn push_reference(out: &mut String, reference: &quick_xml::events::BytesRef<'_>) { + if let Ok(Some(character)) = reference.resolve_char_ref() { + out.push(character); + return; + } + let name = reference.decode().unwrap_or(Cow::Borrowed("")); + out.push_str(match name.as_ref() { + "amp" => "&", + "lt" => "<", + "gt" => ">", + "quot" => "\"", + "apos" => "'", + _ => "", + }); +} + +fn attribute(start: &BytesStart<'_>, local: &[u8]) -> Option { + start.attributes().flatten().find_map(|attr| { + (attr.key.local_name().as_ref() == local) + .then(|| { + attr.normalized_value(quick_xml::XmlVersion::Implicit1_0) + .ok() + .map(Cow::into_owned) + }) + .flatten() + }) +} + +fn xml_error(error: quick_xml::Error) -> ContentProcessorError { + ContentProcessorError(format!("invalid Office XML: {error}")) +} + +/// Paragraph being assembled from WordprocessingML runs. +#[derive(Default)] +struct DocxParagraph { + text: String, + heading: Option, + list_level: Option, +} + +impl DocxParagraph { + fn render(self) -> Option { + let text = self.text.trim(); + if text.is_empty() { + return None; + } + Some(match (self.heading, self.list_level) { + (Some(level), _) => format!("{} {text}", "#".repeat(level)), + (None, Some(level)) => format!("{}- {text}", " ".repeat(level)), + (None, None) => text.to_string(), + }) + } +} + +/// Heading level for a WordprocessingML paragraph style id. +fn heading_level(style: &str) -> Option { + let style = style.to_ascii_lowercase(); + if style == "title" { + return Some(1); + } + let level = style + .strip_prefix("heading")? + .trim_start_matches(['_', ' ', '-']); + level + .parse::() + .ok() + .filter(|level| (1..=6).contains(level)) +} + +fn docx_to_markdown(xml: &str) -> Result { + let mut reader = XmlReader::from_str(xml); + let mut blocks: Vec = Vec::new(); + let mut paragraph: Option = None; + let mut in_text = false; + // Tables: rows of cells, each cell a list of paragraph texts. Nested tables are + // flattened into their parent cell. + let mut table_depth = 0usize; + let mut rows: Vec> = Vec::new(); + let mut cell: Option> = None; + + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(start) => match start.local_name().as_ref() { + b"p" => paragraph = Some(DocxParagraph::default()), + b"t" => in_text = true, + b"tbl" => { + table_depth += 1; + if table_depth == 1 { + rows.clear(); + } + } + b"tr" if table_depth == 1 => rows.push(Vec::new()), + b"tc" if table_depth == 1 => cell = Some(Vec::new()), + _ => docx_property(&start, &mut paragraph), + }, + // Self-closing elements never open a container: an empty `` or + // `` has no matching End event. + Event::Empty(start) => docx_property(&start, &mut paragraph), + Event::Text(text) if in_text => { + let text = text.xml10_content().map_err(|e| xml_error(e.into()))?; + push_paragraph_text(&mut paragraph, &text); + } + Event::GeneralRef(reference) if in_text => { + if let Some(p) = &mut paragraph { + push_reference(&mut p.text, &reference); + } + } + Event::End(end) => match end.local_name().as_ref() { + b"t" => in_text = false, + b"p" => { + if let Some(rendered) = paragraph.take().and_then(DocxParagraph::render) { + match &mut cell { + Some(cell) => cell.push(rendered), + None => blocks.push(rendered), + } + } + } + b"tc" if table_depth == 1 => { + if let (Some(done), Some(row)) = (cell.take(), rows.last_mut()) { + row.push(done.join("
")); + } + } + b"tbl" => { + table_depth = table_depth.saturating_sub(1); + if table_depth == 0 { + if let Some(table) = markdown_table(&std::mem::take(&mut rows)) { + blocks.push(table); + } + } + } + _ => {} + }, + Event::Eof => break, + _ => {} + } + } + Ok(blocks.join("\n\n")) +} + +/// Paragraph properties and inline breaks, which may be self-closing. +fn docx_property(start: &BytesStart<'_>, paragraph: &mut Option) { + let Some(p) = paragraph else { + return; + }; + match start.local_name().as_ref() { + b"pStyle" => { + if let Some(style) = attribute(start, b"val") { + p.heading = heading_level(&style); + } + } + b"numPr" => { + p.list_level.get_or_insert(0); + } + b"ilvl" => { + let level = attribute(start, b"val") + .and_then(|value| value.parse().ok()) + .unwrap_or(0usize); + p.list_level = Some(level.min(8)); + } + b"tab" => p.text.push('\t'), + b"br" | b"cr" => p.text.push('\n'), + _ => {} + } +} + +fn push_paragraph_text(paragraph: &mut Option, text: &str) { + if let Some(p) = paragraph { + p.text.push_str(text); + } +} + +/// Slide part names in presentation order, read from `ppt/presentation.xml` +/// and its relationships; falls back to numeric file order when either is missing. +fn slide_parts(archive: &mut Archive<'_>) -> Result, ContentProcessorError> { + let ordered = (|| -> Result>, ContentProcessorError> { + let Some(presentation) = read_part(archive, "ppt/presentation.xml")? else { + return Ok(None); + }; + let Some(rels) = read_part(archive, "ppt/_rels/presentation.xml.rels")? else { + return Ok(None); + }; + let mut targets = std::collections::HashMap::new(); + let mut reader = XmlReader::from_str(&rels); + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(start) | Event::Empty(start) + if start.local_name().as_ref() == b"Relationship" => + { + if let (Some(id), Some(target)) = + (attribute(&start, b"Id"), attribute(&start, b"Target")) + { + targets.insert(id, target); + } + } + Event::Eof => break, + _ => {} + } + } + let mut slides = Vec::new(); + let mut reader = XmlReader::from_str(&presentation); + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(start) | Event::Empty(start) + if start.local_name().as_ref() == b"sldId" => + { + if let Some(target) = relationship_id(&start).and_then(|id| targets.get(&id)) { + let target = target.trim_start_matches('/'); + let part = if target.starts_with("ppt/") { + target.to_string() + } else { + format!("ppt/{target}") + }; + slides.push(part); + } + } + Event::Eof => break, + _ => {} + } + } + Ok((!slides.is_empty()).then_some(slides)) + })()?; + + if let Some(slides) = ordered { + return Ok(slides); + } + let mut numbered: Vec<(u32, String)> = archive + .file_names() + .filter_map(|name| { + let number = name + .strip_prefix("ppt/slides/slide")? + .strip_suffix(".xml")? + .parse() + .ok()?; + Some((number, name.to_string())) + }) + .collect(); + numbered.sort(); + Ok(numbered.into_iter().map(|(_, name)| name).collect()) +} + +/// The `r:id` attribute of a `p:sldId` element. Its local name is `id`, the same +/// as the numeric slide id, so the namespace prefix tells them apart. +fn relationship_id(start: &BytesStart<'_>) -> Option { + start.attributes().flatten().find_map(|attr| { + let key = attr.key.as_ref(); + (key.ends_with(b":id") && key != b"id") + .then(|| { + attr.normalized_value(quick_xml::XmlVersion::Implicit1_0) + .ok() + .map(Cow::into_owned) + }) + .flatten() + }) +} + +fn pptx_to_markdown(archive: &mut Archive<'_>) -> Result { + let mut parts = slide_parts(archive)?; + let mut warnings = Vec::new(); + if parts.len() > MAX_SLIDES { + parts.truncate(MAX_SLIDES); + warnings.push("slides_truncated".to_string()); + } + let mut sections = Vec::new(); + for (index, part) in parts.iter().enumerate() { + let Some(xml) = read_part(archive, part)? else { + continue; + }; + let paragraphs = slide_paragraphs(&xml)?; + let mut section = format!("## Slide {}", index + 1); + let mut body = paragraphs.into_iter(); + if let Some(title) = body.next() { + section = format!("{section}: {title}"); + } + let rest: Vec = body.collect(); + if !rest.is_empty() { + section.push_str("\n\n"); + section.push_str(&rest.join("\n")); + } + sections.push(section); + } + Ok(Extracted { + markdown: sections.join("\n\n"), + title: core_title(archive), + warnings, + }) +} + +/// Non-empty `a:p` paragraphs of one slide, in document order. +fn slide_paragraphs(xml: &str) -> Result, ContentProcessorError> { + let mut reader = XmlReader::from_str(xml); + let mut paragraphs = Vec::new(); + let mut current: Option = None; + let mut in_text = false; + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(start) => match start.local_name().as_ref() { + b"p" => current = Some(String::new()), + b"t" => in_text = true, + _ => {} + }, + Event::Empty(start) if start.local_name().as_ref() == b"br" => { + if let Some(text) = &mut current { + text.push(' '); + } + } + Event::Text(text) if in_text => { + if let Some(current) = &mut current { + current.push_str(&text.xml10_content().map_err(|e| xml_error(e.into()))?); + } + } + Event::GeneralRef(reference) if in_text => { + if let Some(current) = &mut current { + push_reference(current, &reference); + } + } + Event::End(end) => match end.local_name().as_ref() { + b"t" => in_text = false, + b"p" => { + if let Some(text) = current.take() { + let text = text.trim(); + if !text.is_empty() { + paragraphs.push(text.to_string()); + } + } + } + _ => {} + }, + Event::Eof => break, + _ => {} + } + } + Ok(paragraphs) +} + +fn spreadsheet_to_markdown(body: &[u8]) -> Result { + let mut workbook = open_workbook_auto_from_rs(Cursor::new(body)) + .map_err(|error| ContentProcessorError(format!("cannot open spreadsheet: {error}")))?; + let mut warnings = Vec::new(); + let mut sections = Vec::new(); + for name in workbook.sheet_names() { + let range = match workbook.worksheet_range(&name) { + Ok(range) => range, + Err(error) => { + warnings.push("sheet_unreadable".to_string()); + sections.push(format!("## {name}\n\n_Sheet could not be read: {error}_")); + continue; + } + }; + let total_rows = range.height(); + let total_columns = range.width(); + let rows: Vec> = range + .rows() + .take(MAX_SHEET_ROWS) + .map(|row| row.iter().take(MAX_SHEET_COLUMNS).map(cell_text).collect()) + .collect(); + let Some(table) = markdown_table(&rows) else { + continue; + }; + let mut section = format!("## {name}\n\n{table}"); + if total_rows > MAX_SHEET_ROWS || total_columns > MAX_SHEET_COLUMNS { + push_unique(&mut warnings, "sheet_truncated"); + section.push_str(&format!( + "\n\n_Showing {} of {total_rows} rows and {} of {total_columns} columns._", + total_rows.min(MAX_SHEET_ROWS), + total_columns.min(MAX_SHEET_COLUMNS) + )); + } + sections.push(section); + } + Ok(Extracted { + markdown: sections.join("\n\n"), + title: None, + warnings, + }) +} + +fn push_unique(warnings: &mut Vec, warning: &str) { + if !warnings.iter().any(|existing| existing == warning) { + warnings.push(warning.to_string()); + } +} + +fn cell_text(cell: &Data) -> String { + match cell { + Data::Empty => String::new(), + // Excel stores dates as day serials; show them as ISO 8601 rather than + // numbers such as 41640. Durations keep their numeric form. + Data::DateTime(value) if value.is_datetime() => { + let (year, month, day, hour, minute, second, _) = value.to_ymd_hms_milli(); + if (hour, minute, second) == (0, 0, 0) { + format!("{year:04}-{month:02}-{day:02}") + } else { + format!("{year:04}-{month:02}-{day:02} {hour:02}:{minute:02}:{second:02}") + } + } + other => other.to_string(), + } +} + +/// Render rows as a Markdown table, the first row as the header. Rows are padded +/// to the widest row; fully empty rows and trailing empty columns are dropped. +fn markdown_table(rows: &[Vec]) -> Option { + let rows: Vec<&Vec> = rows + .iter() + .filter(|row| row.iter().any(|cell| !cell.trim().is_empty())) + .collect(); + let width = rows + .iter() + .map(|row| { + row.iter() + .rposition(|cell| !cell.trim().is_empty()) + .map_or(0, |last| last + 1) + }) + .max() + .filter(|width| *width > 0)?; + let render_row = |row: &Vec| { + let cells: Vec = (0..width) + .map(|index| escape_cell(row.get(index).map(String::as_str).unwrap_or(""))) + .collect(); + format!("| {} |", cells.join(" | ")) + }; + let mut lines = vec![render_row(rows[0])]; + lines.push(format!("|{}", " --- |".repeat(width))); + lines.extend(rows[1..].iter().map(|row| render_row(row))); + Some(lines.join("\n")) +} + +fn escape_cell(cell: &str) -> String { + cell.trim() + .replace('\\', "\\\\") + .replace('|', "\\|") + .replace("\r\n", "
") + .replace(['\n', '\r'], "
") +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use bytes::Bytes; + + use super::*; + use crate::content::{ContentFocus, ContentOutputFormat}; + + const DOCX: &str = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + const PPTX: &str = "application/vnd.openxmlformats-officedocument.presentationml.presentation"; + + fn zip_of(parts: &[(&str, &str)]) -> Bytes { + let mut buffer = Cursor::new(Vec::new()); + { + let mut writer = zip::ZipWriter::new(&mut buffer); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated); + for (name, content) in parts { + writer.start_file(*name, options).unwrap(); + writer.write_all(content.as_bytes()).unwrap(); + } + writer.finish().unwrap(); + } + Bytes::from(buffer.into_inner()) + } + + async fn process(url: &str, content_type: &str, body: Bytes) -> ProcessedContent { + OfficeProcessor + .process(ContentProcessorInput { + url: Url::parse(url).unwrap(), + content_type: Some(content_type.to_string()), + body, + output_format: ContentOutputFormat::Markdown, + content_focus: ContentFocus::Full, + }) + .await + .unwrap() + } + + const DOCUMENT_XML: &str = r#" + + + Quarterly report + Revenue grew 12% & costs fell. + First point + Nested point + + RegionSales + EU|West42 + + + +"#; + + const CORE_XML: &str = r#" +Q3 & outlook"#; + + #[tokio::test] + async fn docx_becomes_markdown_with_headings_lists_and_tables() { + let body = zip_of(&[ + ("word/document.xml", DOCUMENT_XML), + ("docProps/core.xml", CORE_XML), + ]); + let processed = process("https://example.com/report", DOCX, body).await; + + assert_eq!(processed.format.as_deref(), Some("markdown")); + assert_eq!( + processed.content.as_deref(), + Some( + "# Quarterly report\n\nRevenue grew 12% & costs fell.\n\n- First point\n\n - Nested point\n\n| Region | Sales |\n| --- | --- |\n| EU\\|West | 42 |" + ) + ); + let metadata = processed.metadata.unwrap(); + assert_eq!(metadata.title.as_deref(), Some("Q3 & outlook")); + assert_eq!(metadata.extraction_method.as_deref(), Some("office_docx")); + assert!(processed.error.is_none()); + } + + #[tokio::test] + async fn pptx_slides_follow_presentation_order() { + let slide = |title: &str, body: &str| { + format!( + r#" +{title} +{body} +"# + ) + }; + let first = slide("Intro", "Why we are here"); + let second = slide("Plan", "Ship it"); + let body = zip_of(&[ + ( + "ppt/presentation.xml", + r#""#, + ), + ( + "ppt/_rels/presentation.xml.rels", + r#""#, + ), + ("ppt/slides/slide1.xml", &second), + ("ppt/slides/slide2.xml", &first), + ]); + let processed = process("https://example.com/deck.pptx", PPTX, body).await; + + assert_eq!( + processed.content.as_deref(), + Some("## Slide 1: Intro\n\nWhy we are here\n\n## Slide 2: Plan\n\nShip it") + ); + } + + #[test] + fn matches_media_types_and_generic_downloads_by_extension() { + let processor = OfficeProcessor; + let page = Url::parse("https://example.com/download").unwrap(); + let xlsx = Url::parse("https://example.com/data.XLSX").unwrap(); + assert!(processor.matches(&page, Some(DOCX))); + assert!(processor.matches(&page, Some("application/vnd.ms-excel"))); + assert!(processor.matches( + &page, + Some("application/vnd.oasis.opendocument.spreadsheet") + )); + assert!(processor.matches(&xlsx, Some("application/octet-stream"))); + assert!(processor.matches(&xlsx, None)); + assert!(!processor.matches(&xlsx, Some("text/html"))); + assert!(!processor.matches(&page, Some("application/octet-stream"))); + assert!(!processor.matches(&page, Some("application/vnd.ms-powerpoint"))); + } + + #[tokio::test] + async fn spreadsheet_from_ods_becomes_tables() { + let content = r#" + + + +ItemPrice +Tea3.5 + +"#; + let body = zip_of(&[ + ("mimetype", "application/vnd.oasis.opendocument.spreadsheet"), + ( + "META-INF/manifest.xml", + r#""#, + ), + ("content.xml", content), + ]); + let processed = process( + "https://example.com/prices", + "application/vnd.oasis.opendocument.spreadsheet", + body, + ) + .await; + + assert_eq!( + processed.content.as_deref(), + Some("## Prices\n\n| Item | Price |\n| --- | --- |\n| Tea | 3.5 |") + ); + assert_eq!( + processed.metadata.unwrap().extraction_method.as_deref(), + Some("office_spreadsheet") + ); + } + + #[test] + fn archive_budget_rejects_zip_bombs() { + // 4 MB of zeros deflates to a few KB; a budget of 1 MB must reject it from + // the central directory alone, before inflating anything. + let zeros = "\0".repeat(4 * 1024 * 1024); + let body = zip_of(&[("word/document.xml", &zeros)]); + assert!(body.len() < 64 * 1024); + + let error = open_archive_within(&body, 1024 * 1024).unwrap_err(); + assert!(error.0.contains("over the 1048576 byte limit"), "{error}"); + assert!(open_archive_within(&body, 8 * 1024 * 1024).is_ok()); + } + + #[tokio::test] + async fn xlsx_becomes_tables() { + let body = zip_of(&[ + ( + "[Content_Types].xml", + r#""#, + ), + ( + "_rels/.rels", + r#""#, + ), + ( + "xl/workbook.xml", + r#""#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#""#, + ), + ( + "xl/worksheets/sheet1.xml", + r#"TeamSpendInfra1200"#, + ), + ]); + let processed = process( + "https://example.com/budget.xlsx", + "application/octet-stream", + body, + ) + .await; + + assert_eq!( + processed.content.as_deref(), + Some("## Budget\n\n| Team | Spend |\n| --- | --- |\n| Infra | 1200 |") + ); + } + + #[test] + fn corrupt_document_is_an_error_not_garbage() { + let error = extract(OfficeKind::Docx, b"not a zip").unwrap_err(); + assert!(error.0.contains("invalid Office archive"), "{error}"); + } + + #[test] + fn spreadsheet_dates_render_as_iso() { + use calamine::{ExcelDateTime, ExcelDateTimeType}; + + let date = Data::DateTime(ExcelDateTime::new( + 41640.0, + ExcelDateTimeType::DateTime, + false, + )); + let stamp = Data::DateTime(ExcelDateTime::new( + 45943.5, + ExcelDateTimeType::DateTime, + false, + )); + assert_eq!(cell_text(&date), "2014-01-01"); + assert_eq!(cell_text(&stamp), "2025-10-13 12:00:00"); + assert_eq!(cell_text(&Data::Float(3.5)), "3.5"); + } + + #[test] + fn heading_levels_from_style_ids() { + assert_eq!(heading_level("Heading1"), Some(1)); + assert_eq!(heading_level("heading 3"), Some(3)); + assert_eq!(heading_level("Title"), Some(1)); + assert_eq!(heading_level("Heading9"), None); + assert_eq!(heading_level("Normal"), None); + } +} diff --git a/crates/fetchkit/src/tool.rs b/crates/fetchkit/src/tool.rs index ed94984..c073a0e 100644 --- a/crates/fetchkit/src/tool.rs +++ b/crates/fetchkit/src/tool.rs @@ -969,15 +969,15 @@ fn display_name(locale: &str) -> &'static str { fn description(locale: &str, enable_save_to_file: bool) -> String { if is_ukrainian(locale) { if enable_save_to_file { - "Завантажити URL як текст або markdown, включно з текстовими PDF; повернути метадані для непідтримуваного бінарного вмісту або зберегти байти через save_to_file.".to_string() + "Завантажити URL як текст або markdown, включно з текстовими PDF і документами Office; повернути метадані для непідтримуваного бінарного вмісту або зберегти байти через save_to_file.".to_string() } else { - "Завантажити URL як текст або markdown, включно з текстовими PDF; повернути метадані для непідтримуваного бінарного вмісту." + "Завантажити URL як текст або markdown, включно з текстовими PDF і документами Office; повернути метадані для непідтримуваного бінарного вмісту." .to_string() } } else if enable_save_to_file { - "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses or save bytes with save_to_file.".to_string() + "Fetch URL content as text or markdown, including text-based PDFs and Office documents; return metadata for unsupported binary responses or save bytes with save_to_file.".to_string() } else { - "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses.".to_string() + "Fetch URL content as text or markdown, including text-based PDFs and Office documents; return metadata for unsupported binary responses.".to_string() } } diff --git a/crates/fetchkit/tests/integration.rs b/crates/fetchkit/tests/integration.rs index 05d23c5..31ab0a8 100644 --- a/crates/fetchkit/tests/integration.rs +++ b/crates/fetchkit/tests/integration.rs @@ -9,7 +9,7 @@ use serde_json::json; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tower::Service; -use wiremock::matchers::{method, path}; +use wiremock::matchers::{header_regex, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; /// Helper: create FetchOptions that permit loopback (for wiremock tests) @@ -465,6 +465,78 @@ async fn test_pdf_content_processor_returns_markdown() { assert!(resp.error.is_none()); } +fn docx_with_paragraph(text: &str) -> Vec { + use std::io::Write; + + let document = format!( + r#"Minutes{text}"# + ); + let mut buffer = std::io::Cursor::new(Vec::new()); + let mut writer = zip::ZipWriter::new(&mut buffer); + writer + .start_file( + "word/document.xml", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + writer.write_all(document.as_bytes()).unwrap(); + writer.finish().unwrap(); + buffer.into_inner() +} + +#[tokio::test] +async fn test_docx_content_processor_returns_markdown() { + let mock_server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/files/minutes")) + .respond_with( + ResponseTemplate::new(200) + .set_body_bytes(docx_with_paragraph("Ship the release on Friday.")) + .insert_header( + "content-type", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ), + ) + .mount(&mock_server) + .await; + + let req = FetchRequest::new(format!("{}/files/minutes", mock_server.uri())).as_markdown(); + let resp = fetch_with_options(req, test_options()).await.unwrap(); + + assert_eq!(resp.format.as_deref(), Some("markdown")); + assert_eq!( + resp.content.as_deref(), + Some("## Minutes\n\nShip the release on Friday.") + ); + assert_eq!( + resp.metadata.unwrap().extraction_method.as_deref(), + Some("office_docx") + ); + assert!(resp.error.is_none()); +} + +#[tokio::test] +async fn test_markdown_request_prefers_markdown_representation() { + let mock_server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/page")) + .and(header_regex("accept", r"^text/markdown(,|$)")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw("# Served as markdown", "text/markdown; charset=utf-8"), + ) + .mount(&mock_server) + .await; + + let req = FetchRequest::new(format!("{}/page", mock_server.uri())).as_markdown(); + let resp = fetch_with_options(req, test_options()).await.unwrap(); + + assert_eq!(resp.status_code, 200); + assert!(resp.content.unwrap().contains("# Served as markdown")); +} + #[tokio::test] async fn test_custom_content_processor_registry_can_disable_pdf_processing() { let mock_server = MockServer::start().await; diff --git a/knowledge/foundations/fetchers.md b/knowledge/foundations/fetchers.md index ed118bf..f794cf0 100644 --- a/knowledge/foundations/fetchers.md +++ b/knowledge/foundations/fetchers.md @@ -89,6 +89,19 @@ allowing callers to replace the default content processor registry. - Extracted output is capped by the same configured `max_body_size`; partial PDF input is not passed to the parser. +#### OfficeProcessor + +- Matches Word (DOCX), PowerPoint (PPTX) and spreadsheet (XLSX, XLSM, XLS, ODS) media + types. The file extension is a fallback only for missing Content-Type, + `application/octet-stream` or `application/zip`. +- Parses locally: DOCX and PPTX XML parts with `quick-xml`, spreadsheets with + `calamine`. Documents never leave the process; there is no hosted converter. +- Zip containers are checked against an entry-count and total uncompressed-size + budget from the central directory before any part is inflated, and each part it + reads itself is read through a size cap (TM-DOS-008). +- Shares the PDF processor's blocking-task path and two-document concurrency cap. +- Output caps: 1,000 slides, 1,000 rows and 50 columns per sheet, then `max_body_size`. + ### Built-in Fetchers #### DefaultFetcher (lowest priority) @@ -392,6 +405,7 @@ Both built-in fetchers integrate resolve-then-check DNS validation: ``` crates/fetchkit/src/ ├── content.rs # ContentProcessor trait, registry, PDF processor +├── office.rs # Office (DOCX, PPTX, XLSX, XLS, ODS) processor ├── dns.rs # DnsPolicy - SSRF prevention via resolve-then-check ├── file_saver.rs # FileSaver trait, LocalFileSaver, SaveResult, FileSaveError ├── fetchers/ diff --git a/knowledge/foundations/tool-contract.md b/knowledge/foundations/tool-contract.md index 858bb7c..eb6f26e 100644 --- a/knowledge/foundations/tool-contract.md +++ b/knowledge/foundations/tool-contract.md @@ -216,7 +216,7 @@ Provide a builder to configure tool options, including: - Expose a single `web_fetch` tool over MCP. - Input schema: derived from `FetchRequest` via tool builder (disabled options omitted). - Output: Markdown with YAML frontmatter (same format as CLI `--output md`). -- Tool description: "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses or save bytes to file." +- Tool description: "Fetch URL content as text or markdown, including text-based PDFs and Office documents; return metadata for unsupported binary responses or save bytes to file." ### Python Bindings @@ -270,7 +270,8 @@ By default, Fetchkit blocks connections to private/reserved IP ranges: - Ambient proxy environment variables are ignored by default. - Opt in via `ToolBuilder::respect_proxy_env(true)` or CLI `--allow-env-proxy`. - Accept header: - - Markdown: `text/html, text/markdown, text/plain, */*;q=0.8` + - Markdown: `text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.7`, so sites + that serve Markdown on request (Markdown for Agents) return it instead of HTML - Text: `text/html, text/plain, */*;q=0.8` - Raw: `*/*` - HEAD requests use HTTP HEAD method, return metadata only. @@ -320,7 +321,8 @@ By default, Fetchkit blocks connections to private/reserved IP ranges: - Detect binary by Content-Type prefix: - `image/`, `audio/`, `video/`, `application/octet-stream`, `application/pdf`, `application/x-pdf`, `application/zip`, `application/gzip`, `application/x-tar`, `application/x-rar`, - `application/x-7z`, `application/vnd.ms-`, `application/vnd.openxmlformats`, `font/`. + `application/x-7z`, `application/vnd.ms-`, `application/vnd.openxmlformats`, + `application/vnd.oasis.opendocument`, `font/`. - When `as_markdown` is requested, a matching registered content processor runs on the bounded response body before the generic binary response is returned. - The built-in PDF processor: @@ -332,6 +334,20 @@ By default, Fetchkit blocks connections to private/reserved IP ranges: `pdf_requires_ocr` quality warnings and suggest `use_ocr`. - Runs on a bounded blocking pool path, and both PDF input and extracted output honor `max_body_size`. Partial PDF inputs are not parsed. +- The built-in Office processor: + - Matches DOCX, PPTX, XLSX/XLSM, XLS and ODS media types, or those file + extensions when served as `application/octet-stream`, `application/zip` or + without Content-Type. + - Returns `format: "markdown"`: Word headings, lists and tables; one + `## Slide N: title` section per slide in presentation order; one `## sheet` + Markdown table per spreadsheet sheet (first row as header, at most 1,000 rows + and 50 columns, with a `sheet_truncated` warning beyond that). + - Reports `office_docx`, `office_pptx` or `office_spreadsheet` as the extraction + method and `dc:title` from the document properties as the title. + - Performs no network requests. Zip containers are checked against an entry and + uncompressed-size budget before inflating (TM-DOS-008); shares the PDF + processor's blocking pool and two-document concurrency cap. Partial inputs are + not parsed. - For binary without a matching processor: - Return metadata (`content_type`, `size`, `filename`, `last_modified`) - Include `error: "Binary content is not supported. Only textual content (HTML, text, JSON, etc.) can be fetched."` diff --git a/knowledge/log.md b/knowledge/log.md index f109374..8c2d65e 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -1,5 +1,9 @@ # Fetchkit Knowledge Update Log +## 2026-10-12 + +* **Office documents**: Added the local Office processor (DOCX, PPTX, XLSX, XLS, ODS) to the fetchers and tool contract, its zip-bomb threat TM-DOS-008, and the Markdown-first `Accept` header. + ## 2026-08-08 * **HTML processing**: Routed HTML extraction and Markdown/text conversion through the content processor registry, added a replaceable native converter, expanded structural fidelity for code, tables, footnotes, math, callouts, and related rich content, and bounded nested conversion depth. diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 8283e36..6428670 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -298,6 +298,7 @@ to protect against changes between validation and write. | TM-DOS-005 | DNS resolution delay | Low | DNS resolution uses system resolver; no explicit timeout on DNS lookup | **ACCEPTED** | | TM-DOS-006 | Memory exhaustion from large HTML conversion | Medium | Conversion input bounded by `max_body_size` (10 MB default) | MITIGATED | | TM-DOS-007 | CPU/memory exhaustion from adversarial PDF parsing | Medium | Bounded input/output, blocking-task isolation, and a cancellation-safe two-document concurrency cap; parser-internal expansion remains possible | **ACCEPTED** | +| TM-DOS-008 | Zip bomb or adversarial Office document | Medium | Central-directory entry and uncompressed-size budget before inflating, size-capped part reads, per-sheet output caps, shared two-document blocking cap | MITIGATED | ### Mitigation Details @@ -329,6 +330,16 @@ Operators handling hostile, high-volume PDFs should isolate the process and appl request CPU/memory limits. Accepted because the in-process Rust parser has no reliable cancellation boundary once a blocking parse begins. +**TM-DOS-008 — Zip bomb or adversarial Office document (MITIGATED):** +Office documents are zip archives (except legacy XLS), so a body within `max_body_size` +can declare gigabytes of content. Before any part is inflated, the Office processor +rejects archives with more than 10,000 entries or more than 200 MB of declared +uncompressed content, and every XML part it reads itself is read through a 64 MB cap, +since declared sizes can lie. Spreadsheet output is capped at 1,000 rows and 50 columns +per sheet before the `max_body_size` output cap. Parsing shares the PDF processor's +blocking path and two-document concurrency cap. `calamine` inflates sheet parts itself; +the declared-size budget bounds those as well. + ## 5. Information Leakage (TM-LEAK) | ID | Threat | Severity | Mitigation | Status |