From 397a68bdcd3e143b2b2dc6adb49d91aefc9ce350 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 12 Oct 2026 17:39:06 +0000 Subject: [PATCH 1/2] feat(content): convert Office documents to Markdown locally Add OfficeProcessor for DOCX, PPTX, XLSX/XLSM, XLS and ODS. Word headings, lists and tables, one section per slide in presentation order, one table per sheet. Zip containers are checked against an entry and uncompressed size budget before inflating (TM-DOS-008). Markdown requests now put text/markdown first in Accept so sites that serve Markdown on request return it instead of HTML. --- CHANGELOG.md | 5 + Cargo.lock | 108 ++- Cargo.toml | 3 + README.md | 10 +- crates/fetchkit/Cargo.toml | 3 + crates/fetchkit/src/content.rs | 27 +- crates/fetchkit/src/fetchers/default.rs | 5 +- crates/fetchkit/src/lib.rs | 4 +- crates/fetchkit/src/office.rs | 973 ++++++++++++++++++++++++ crates/fetchkit/src/tool.rs | 8 +- crates/fetchkit/tests/integration.rs | 74 +- knowledge/foundations/fetchers.md | 14 + knowledge/foundations/tool-contract.md | 22 +- knowledge/log.md | 4 + knowledge/security/threat-model.md | 11 + 15 files changed, 1244 insertions(+), 27 deletions(-) create mode 100644 crates/fetchkit/src/office.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f63e8bb..163ce3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add a pluggable post-download `ContentProcessor` registry and built-in `PdfProcessor` that extracts Markdown from text-based PDFs with local `pdf-inspector` processing and explicit OCR guidance for unsupported pages. +- Add a built-in `OfficeProcessor` that converts Word (DOCX), PowerPoint (PPTX) + and spreadsheet (XLSX, XLSM, XLS, ODS) documents to Markdown locally, with + zip-bomb limits checked before any part is inflated. - Discover agent-facing resources from HTTP and HTML declarations plus bounded conventional probes, including `llms.txt`, `auth.md`, OAuth metadata, MCP server cards, A2A agent cards, and Agent Skills indexes. Markdown responses @@ -27,6 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Markdown requests now ask for `text/markdown` first in the `Accept` header, so + sites that serve Markdown on request return it instead of HTML. - `ContentProcessorInput` now includes the requested output format and HTML content focus. Custom processors constructing this public input must populate the new fields. diff --git a/Cargo.lock b/Cargo.lock index 1944d41..4a15fc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,6 +141,16 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -266,6 +276,23 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "calamine" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +dependencies = [ + "atoi_simd", + "byteorder", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip", +] + [[package]] name = "cbc" version = "0.1.2" @@ -382,6 +409,15 @@ dependencies = [ "cc", ] +[[package]] +name = "codepage" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdff162541cd8b79de82e2edcc7eff3a8c2a6dc3d75152636028f96d93de3b26" +dependencies = [ + "encoding_rs", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -483,9 +519,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] @@ -602,6 +638,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + [[package]] name = "defmt" version = "1.1.1" @@ -846,6 +888,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fast-float2" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e8948ce679d00a02a94739ea185595dca7118ed04feb991127e443bd3d761f" + [[package]] name = "fastrand" version = "2.4.1" @@ -859,12 +907,14 @@ dependencies = [ "async-trait", "base64", "bytes", + "calamine", "ed25519-dalek", "futures", "libc", "mira-eval", "pdf-inspector", "percent-encoding", + "quick-xml", "rakers", "rand 0.10.2", "reqwest", @@ -881,6 +931,7 @@ dependencies = [ "tracing-subscriber", "url", "wiremock", + "zip", ] [[package]] @@ -925,6 +976,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1626,9 +1678,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lopdf" @@ -2137,6 +2189,16 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quinn" version = "0.11.11" @@ -3376,6 +3438,12 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -3965,8 +4033,40 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 581c34e..28986c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,9 @@ tower = { version = "0.5", features = ["util"] } # Binary document extraction pdf-inspector = "0.1.7" +calamine = { version = "0.36", default-features = false } +zip = { version = "8", default-features = false, features = ["deflate"] } +quick-xml = "0.41" # Platform filesystem flags libc = "0.2" diff --git a/README.md b/README.md index c729c03..60cc502 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ AI-friendly web content fetching tool designed for LLM consumption. Rust library - **Agent content focus** - Optional low-noise extraction mode for AI agents - **Crawl discovery** - Optional bounded same-origin page discovery for AI agents - **HTML-to-Text** - Plain text extraction with clean formatting -- **Content processors** - Post-download extraction for text PDFs, with an extensible registry +- **Content processors** - Post-download extraction for text PDFs and Office documents, with an extensible registry - **Binary detection** - Returns metadata only for unsupported binary formats - **Timeout handling** - 1s first-byte, 30s body with partial content on timeout - **Safety limits** - 10 MB default decompressed body cap with truncation @@ -49,6 +49,8 @@ requests. - `PdfProcessor` - text-based PDF classification and Markdown extraction via [`pdf-inspector`](https://github.com/firecrawl/pdf-inspector); scanned or image-only pages are reported as requiring OCR +- `OfficeProcessor` - Word (DOCX), PowerPoint (PPTX) and spreadsheet (XLSX, XLS, + ODS) documents to Markdown, parsed locally with zip-bomb limits Custom processors implement `ContentProcessor` and can be registered in a `ContentProcessorRegistry`. Pass that registry to @@ -339,10 +341,14 @@ body limit and converted to Markdown. PDF parsing runs locally; no OCR service o additional network request is used. Scanned/image-only PDFs report `use_ocr` as the suggested next action. +Office documents are converted the same way, locally: Word headings, lists and +tables, one section per PowerPoint slide, and one Markdown table per spreadsheet +sheet (DOCX, PPTX, XLSX, XLS, ODS). Legacy DOC and PPT files are not supported. + Other binary content returns metadata only: - Images, audio, video, fonts - Archives (zip, tar, rar, 7z) -- Office documents +- Legacy Word and PowerPoint files (DOC, PPT) ### HTML Conversion diff --git a/crates/fetchkit/Cargo.toml b/crates/fetchkit/Cargo.toml index 9b231e6..5489dc9 100644 --- a/crates/fetchkit/Cargo.toml +++ b/crates/fetchkit/Cargo.toml @@ -31,6 +31,9 @@ bytes = { workspace = true } async-trait = { workspace = true } tower = { workspace = true } pdf-inspector = { workspace = true } +calamine = { workspace = true } +zip = { workspace = true } +quick-xml = { workspace = true } libc = { workspace = true } rakers = { workspace = true, optional = true } diff --git a/crates/fetchkit/src/content.rs b/crates/fetchkit/src/content.rs index 7e704f2..6d693ed 100644 --- a/crates/fetchkit/src/content.rs +++ b/crates/fetchkit/src/content.rs @@ -16,11 +16,13 @@ use crate::convert::{ extract_headings, extract_metadata, extract_readable_content, filter_excessive_newlines, html_to_text, strip_boilerplate, BuiltinHtmlToMarkdownConverter, HtmlToMarkdownConverter, }; +use crate::office::OfficeProcessor; use crate::{PageMetadata, PageQuality}; -// pdf-inspector uses Rayon internally; bound concurrent documents so batch fetches -// cannot multiply parser thread pools without limit. -static PDF_PROCESSING_LIMIT: LazyLock> = +// pdf-inspector uses Rayon internally, and every document parser is CPU-bound; bound +// concurrent documents (PDF and Office alike) so batch fetches cannot multiply parser +// work without limit. +pub(crate) static DOCUMENT_PROCESSING_LIMIT: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(2))); /// Bounded response bytes and metadata passed to a [`ContentProcessor`]. @@ -147,6 +149,7 @@ impl ContentProcessorRegistry { pub fn with_defaults() -> Self { let mut registry = Self::new(); registry.register(Box::new(PdfProcessor)); + registry.register(Box::new(OfficeProcessor)); registry.register(Box::new(HtmlProcessor::default())); registry } @@ -271,15 +274,17 @@ impl ContentProcessor for HtmlProcessor { /// Extracts native text PDFs as Markdown using `pdf-inspector`. pub struct PdfProcessor; -async fn run_pdf_task(limit: Arc, task: F) -> Result +pub(crate) async fn run_document_task( + limit: Arc, + task: F, +) -> Result where T: Send + 'static, F: FnOnce() -> T + Send + 'static, { - let permit = limit - .acquire_owned() - .await - .map_err(|error| ContentProcessorError(format!("PDF processing unavailable: {error}")))?; + let permit = limit.acquire_owned().await.map_err(|error| { + ContentProcessorError(format!("document processing unavailable: {error}")) + })?; tokio::task::spawn_blocking(move || { // Keep the permit in the blocking task: cancelling its async caller must not // admit another parse while this non-cancellable work is still running. @@ -287,7 +292,7 @@ where task() }) .await - .map_err(|error| ContentProcessorError(format!("PDF processor task failed: {error}"))) + .map_err(|error| ContentProcessorError(format!("document processor task failed: {error}"))) } #[async_trait] @@ -315,7 +320,7 @@ impl ContentProcessor for PdfProcessor { &self, input: ContentProcessorInput, ) -> Result { - let result = run_pdf_task(Arc::clone(&PDF_PROCESSING_LIMIT), move || { + let result = run_document_task(Arc::clone(&DOCUMENT_PROCESSING_LIMIT), move || { process_pdf_mem(&input.body) }) .await? @@ -445,7 +450,7 @@ mod tests { let limit = Arc::new(Semaphore::new(1)); let (started_tx, started_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel(); - let handle = tokio::spawn(run_pdf_task(Arc::clone(&limit), move || { + let handle = tokio::spawn(run_document_task(Arc::clone(&limit), move || { started_tx.send(()).unwrap(); release_rx.recv().unwrap(); })); diff --git a/crates/fetchkit/src/fetchers/default.rs b/crates/fetchkit/src/fetchers/default.rs index cad0a02..35db5b4 100644 --- a/crates/fetchkit/src/fetchers/default.rs +++ b/crates/fetchkit/src/fetchers/default.rs @@ -85,6 +85,7 @@ const BINARY_PREFIXES: &[&str] = &[ "application/x-7z", "application/vnd.ms-", "application/vnd.openxmlformats", + "application/vnd.oasis.opendocument", "font/", ]; @@ -461,7 +462,9 @@ impl Fetcher for DefaultFetcher { let max_body_size = options.max_body_size.unwrap_or(DEFAULT_MAX_BODY_SIZE); let accept = if wants_markdown { - "text/html, text/markdown, text/plain, */*;q=0.8" + // Markdown first: sites that serve it on request (Markdown for Agents) skip + // HTML conversion entirely. + "text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.7" } else if wants_text { "text/html, text/plain, */*;q=0.8" } else { diff --git a/crates/fetchkit/src/lib.rs b/crates/fetchkit/src/lib.rs index d6ba4b2..e33288a 100644 --- a/crates/fetchkit/src/lib.rs +++ b/crates/fetchkit/src/lib.rs @@ -85,6 +85,7 @@ mod dns; mod error; pub mod fetchers; pub mod file_saver; +mod office; mod tool; pub mod transport; mod types; @@ -107,6 +108,7 @@ pub use fetchers::{ RSSFeedFetcher, StackOverflowFetcher, TwitterFetcher, WikipediaFetcher, YouTubeFetcher, }; pub use file_saver::{FileSaveError, FileSaver, LocalFileSaver, SaveResult}; +pub use office::OfficeProcessor; pub use tool::{ Tool, ToolBuilder, ToolExecution, ToolImage, ToolOutput, ToolOutputMetadata, ToolService, ToolStatus, @@ -128,7 +130,7 @@ pub const DEFAULT_USER_AGENT: &str = "Everruns Fetchkit/1.0"; /// Backward-compatible full description string with file-saving enabled. pub const TOOL_DESCRIPTION: &str = - "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses or save bytes to file."; + "Fetch URL content as text or markdown, including text-based PDFs and Office documents; return metadata for unsupported binary responses or save bytes to file."; /// Backward-compatible help document with file-saving enabled. pub static TOOL_LLMTXT: std::sync::LazyLock = diff --git a/crates/fetchkit/src/office.rs b/crates/fetchkit/src/office.rs new file mode 100644 index 0000000..aa22f69 --- /dev/null +++ b/crates/fetchkit/src/office.rs @@ -0,0 +1,973 @@ +//! Office document extraction: Word (DOCX), PowerPoint (PPTX) and spreadsheets +//! (XLSX, XLS, ODS) to Markdown. +//! +//! Decisions: +//! - Everything is parsed locally from the bounded response bytes. Documents never +//! leave the host, and no account or network call is involved, so the processor +//! behaves the same in every embedding (CLI, MCP, library, hosted runtimes). +//! - DOCX and PPTX text is read straight from their WordprocessingML / DrawingML XML +//! parts with `quick-xml`; full fidelity (styles, images, footnotes) is not a goal, +//! readable text with headings, lists and tables is. Spreadsheets go through +//! `calamine`, which also covers legacy binary XLS. +//! - THREAT[TM-DOS-008]: OOXML and ODS files are zip archives. Before any part is +//! inflated, the central directory is checked against an entry-count and a total +//! uncompressed-size budget, and every part fetchkit reads itself is read through a +//! size-capped reader, so a small zip bomb within `max_body_size` cannot expand +//! without limit. Spreadsheet output is capped per sheet (rows and columns) and the +//! caller caps the final Markdown at `max_body_size`. Parsing runs on a blocking +//! task under the same two-document concurrency cap as PDFs. + +use std::borrow::Cow; +use std::io::{Cursor, Read}; + +use async_trait::async_trait; +use calamine::{open_workbook_auto_from_rs, Data, Reader}; +use quick_xml::events::{BytesStart, Event}; +use quick_xml::Reader as XmlReader; +use url::Url; + +use crate::content::{ + run_document_task, ContentProcessor, ContentProcessorError, ContentProcessorInput, + ProcessedContent, DOCUMENT_PROCESSING_LIMIT, +}; +use crate::{PageMetadata, PageQuality}; + +/// Maximum number of entries in an Office zip archive. +const MAX_ARCHIVE_ENTRIES: usize = 10_000; +/// Maximum total uncompressed size of an Office zip archive. +const MAX_ARCHIVE_UNCOMPRESSED_BYTES: u64 = 200 * 1024 * 1024; +/// Maximum uncompressed size of one XML part fetchkit reads itself. +const MAX_PART_BYTES: u64 = 64 * 1024 * 1024; +/// Maximum slides read from one presentation. +const MAX_SLIDES: usize = 1_000; +/// Maximum rows rendered per spreadsheet sheet. +const MAX_SHEET_ROWS: usize = 1_000; +/// Maximum columns rendered per spreadsheet sheet. +const MAX_SHEET_COLUMNS: usize = 50; + +/// Supported Office document kinds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum OfficeKind { + Docx, + Pptx, + Spreadsheet, +} + +impl OfficeKind { + fn from_media_type(media_type: &str) -> Option { + let media_type = media_type.to_ascii_lowercase(); + match media_type.as_str() { + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => { + Some(Self::Docx) + } + "application/vnd.openxmlformats-officedocument.presentationml.presentation" => { + Some(Self::Pptx) + } + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + | "application/vnd.ms-excel" + | "application/vnd.ms-excel.sheet.macroenabled.12" + | "application/vnd.oasis.opendocument.spreadsheet" => Some(Self::Spreadsheet), + _ => None, + } + } + + fn from_path(path: &str) -> Option { + let path = path.to_ascii_lowercase(); + let extension = path.rsplit_once('.')?.1; + match extension { + "docx" => Some(Self::Docx), + "pptx" => Some(Self::Pptx), + "xlsx" | "xlsm" | "xls" | "ods" => Some(Self::Spreadsheet), + _ => None, + } + } + + fn detect(url: &Url, content_type: Option<&str>) -> Option { + let media_type = content_type + .and_then(|value| value.split(';').next()) + .map(str::trim) + .unwrap_or_default(); + if let Some(kind) = Self::from_media_type(media_type) { + return Some(kind); + } + // Download endpoints often serve documents as generic binary; trust the + // extension only then, never over a specific media type. + let generic = media_type.is_empty() + || media_type.eq_ignore_ascii_case("application/octet-stream") + || media_type.eq_ignore_ascii_case("application/zip"); + if generic { + Self::from_path(url.path()) + } else { + None + } + } + + fn extraction_method(self) -> &'static str { + match self { + Self::Docx => "office_docx", + Self::Pptx => "office_pptx", + Self::Spreadsheet => "office_spreadsheet", + } + } +} + +/// Extracts Word, PowerPoint and spreadsheet documents as Markdown. +pub struct OfficeProcessor; + +#[async_trait] +impl ContentProcessor for OfficeProcessor { + fn name(&self) -> &'static str { + "office" + } + + fn matches(&self, url: &Url, content_type: Option<&str>) -> bool { + OfficeKind::detect(url, content_type).is_some() + } + + async fn process( + &self, + input: ContentProcessorInput, + ) -> Result { + let kind = OfficeKind::detect(&input.url, input.content_type.as_deref()) + .ok_or_else(|| ContentProcessorError("unsupported Office document".to_string()))?; + let body = input.body; + let extracted = run_document_task(DOCUMENT_PROCESSING_LIMIT.clone(), move || { + extract(kind, &body) + }) + .await??; + + let content = Some(extracted.markdown).filter(|value| !value.trim().is_empty()); + let error = content + .is_none() + .then(|| "Document contains no extractable text".to_string()); + let metadata = PageMetadata { + title: extracted.title, + extraction_method: Some(kind.extraction_method().to_string()), + ..Default::default() + }; + let quality = PageQuality { + score: if content.is_none() { + 0.0 + } else if extracted.warnings.is_empty() { + 1.0 + } else { + 0.8 + }, + warnings: extracted.warnings, + extraction_method: Some(kind.extraction_method().to_string()), + ..Default::default() + }; + + Ok(ProcessedContent { + format: content.as_ref().map(|_| "markdown".to_string()), + content, + metadata: Some(metadata), + quality: Some(quality), + error, + }) + } +} + +#[derive(Debug, Default)] +struct Extracted { + markdown: String, + title: Option, + warnings: Vec, +} + +fn extract(kind: OfficeKind, body: &[u8]) -> Result { + match kind { + OfficeKind::Docx => { + let mut archive = open_archive(body)?; + let document = read_part(&mut archive, "word/document.xml")? + .ok_or_else(|| ContentProcessorError("DOCX has no word/document.xml".into()))?; + Ok(Extracted { + markdown: docx_to_markdown(&document)?, + title: core_title(&mut archive), + warnings: Vec::new(), + }) + } + OfficeKind::Pptx => { + let mut archive = open_archive(body)?; + pptx_to_markdown(&mut archive) + } + OfficeKind::Spreadsheet => { + // XLS is a compound binary file, not a zip; only zip containers get the + // archive budget check. + if body.starts_with(b"PK") { + let mut archive = open_archive(body)?; + let title = core_title(&mut archive); + let mut extracted = spreadsheet_to_markdown(body)?; + extracted.title = title; + Ok(extracted) + } else { + spreadsheet_to_markdown(body) + } + } + } +} + +type Archive<'a> = zip::ZipArchive>; + +fn open_archive(body: &[u8]) -> Result, ContentProcessorError> { + open_archive_within(body, MAX_ARCHIVE_UNCOMPRESSED_BYTES) +} + +fn open_archive_within(body: &[u8], budget: u64) -> Result, ContentProcessorError> { + let mut archive = zip::ZipArchive::new(Cursor::new(body)) + .map_err(|error| ContentProcessorError(format!("invalid Office archive: {error}")))?; + if archive.len() > MAX_ARCHIVE_ENTRIES { + return Err(ContentProcessorError(format!( + "Office archive has too many entries ({})", + archive.len() + ))); + } + let mut total: u64 = 0; + for index in 0..archive.len() { + let entry = archive + .by_index_raw(index) + .map_err(|error| ContentProcessorError(format!("invalid Office archive: {error}")))?; + total = total.saturating_add(entry.size()); + } + if total > budget { + return Err(ContentProcessorError(format!( + "Office archive expands to {total} bytes, over the {budget} byte limit" + ))); + } + Ok(archive) +} + +/// Read one archive part as UTF-8, or `None` when the part does not exist. +fn read_part( + archive: &mut Archive<'_>, + name: &str, +) -> Result, ContentProcessorError> { + let entry = match archive.by_name(name) { + Ok(entry) => entry, + Err(zip::result::ZipError::FileNotFound) => return Ok(None), + Err(error) => { + return Err(ContentProcessorError(format!( + "cannot read Office part {name}: {error}" + ))) + } + }; + // The declared size was budgeted in open_archive, but declared sizes can lie; + // cap what is actually inflated. + let mut bytes = Vec::new(); + entry + .take(MAX_PART_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|error| { + ContentProcessorError(format!("cannot read Office part {name}: {error}")) + })?; + if bytes.len() as u64 > MAX_PART_BYTES { + return Err(ContentProcessorError(format!( + "Office part {name} is larger than {MAX_PART_BYTES} bytes" + ))); + } + Ok(Some(String::from_utf8_lossy(&bytes).into_owned())) +} + +/// `dc:title` from `docProps/core.xml`, when present. +fn core_title(archive: &mut Archive<'_>) -> Option { + let core = read_part(archive, "docProps/core.xml").ok()??; + let mut reader = XmlReader::from_str(&core); + let mut in_title = false; + let mut title = String::new(); + loop { + match reader.read_event().ok()? { + Event::Start(start) if start.local_name().as_ref() == b"title" => in_title = true, + Event::End(end) if end.local_name().as_ref() == b"title" => break, + Event::Text(text) if in_title => title.push_str(&text.xml10_content().ok()?), + Event::GeneralRef(reference) if in_title => push_reference(&mut title, &reference), + Event::Eof => break, + _ => {} + } + } + let title = title.trim(); + (!title.is_empty()).then(|| title.to_string()) +} + +fn push_reference(out: &mut String, reference: &quick_xml::events::BytesRef<'_>) { + if let Ok(Some(character)) = reference.resolve_char_ref() { + out.push(character); + return; + } + let name = reference.decode().unwrap_or(Cow::Borrowed("")); + out.push_str(match name.as_ref() { + "amp" => "&", + "lt" => "<", + "gt" => ">", + "quot" => "\"", + "apos" => "'", + _ => "", + }); +} + +fn attribute(start: &BytesStart<'_>, local: &[u8]) -> Option { + start.attributes().flatten().find_map(|attr| { + (attr.key.local_name().as_ref() == local) + .then(|| { + attr.normalized_value(quick_xml::XmlVersion::Implicit1_0) + .ok() + .map(Cow::into_owned) + }) + .flatten() + }) +} + +fn xml_error(error: quick_xml::Error) -> ContentProcessorError { + ContentProcessorError(format!("invalid Office XML: {error}")) +} + +/// Paragraph being assembled from WordprocessingML runs. +#[derive(Default)] +struct DocxParagraph { + text: String, + heading: Option, + list_level: Option, +} + +impl DocxParagraph { + fn render(self) -> Option { + let text = self.text.trim(); + if text.is_empty() { + return None; + } + Some(match (self.heading, self.list_level) { + (Some(level), _) => format!("{} {text}", "#".repeat(level)), + (None, Some(level)) => format!("{}- {text}", " ".repeat(level)), + (None, None) => text.to_string(), + }) + } +} + +/// Heading level for a WordprocessingML paragraph style id. +fn heading_level(style: &str) -> Option { + let style = style.to_ascii_lowercase(); + if style == "title" { + return Some(1); + } + let level = style + .strip_prefix("heading")? + .trim_start_matches(['_', ' ', '-']); + level + .parse::() + .ok() + .filter(|level| (1..=6).contains(level)) +} + +fn docx_to_markdown(xml: &str) -> Result { + let mut reader = XmlReader::from_str(xml); + let mut blocks: Vec = Vec::new(); + let mut paragraph: Option = None; + let mut in_text = false; + // Tables: rows of cells, each cell a list of paragraph texts. Nested tables are + // flattened into their parent cell. + let mut table_depth = 0usize; + let mut rows: Vec> = Vec::new(); + let mut cell: Option> = None; + + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(start) => match start.local_name().as_ref() { + b"p" => paragraph = Some(DocxParagraph::default()), + b"t" => in_text = true, + b"tbl" => { + table_depth += 1; + if table_depth == 1 { + rows.clear(); + } + } + b"tr" if table_depth == 1 => rows.push(Vec::new()), + b"tc" if table_depth == 1 => cell = Some(Vec::new()), + _ => docx_property(&start, &mut paragraph), + }, + // Self-closing elements never open a container: an empty `` or + // `` has no matching End event. + Event::Empty(start) => docx_property(&start, &mut paragraph), + Event::Text(text) if in_text => { + let text = text.xml10_content().map_err(|e| xml_error(e.into()))?; + push_paragraph_text(&mut paragraph, &text); + } + Event::GeneralRef(reference) if in_text => { + if let Some(p) = &mut paragraph { + push_reference(&mut p.text, &reference); + } + } + Event::End(end) => match end.local_name().as_ref() { + b"t" => in_text = false, + b"p" => { + if let Some(rendered) = paragraph.take().and_then(DocxParagraph::render) { + match &mut cell { + Some(cell) => cell.push(rendered), + None => blocks.push(rendered), + } + } + } + b"tc" if table_depth == 1 => { + if let (Some(done), Some(row)) = (cell.take(), rows.last_mut()) { + row.push(done.join("
")); + } + } + b"tbl" => { + table_depth = table_depth.saturating_sub(1); + if table_depth == 0 { + if let Some(table) = markdown_table(&std::mem::take(&mut rows)) { + blocks.push(table); + } + } + } + _ => {} + }, + Event::Eof => break, + _ => {} + } + } + Ok(blocks.join("\n\n")) +} + +/// Paragraph properties and inline breaks, which may be self-closing. +fn docx_property(start: &BytesStart<'_>, paragraph: &mut Option) { + let Some(p) = paragraph else { + return; + }; + match start.local_name().as_ref() { + b"pStyle" => { + if let Some(style) = attribute(start, b"val") { + p.heading = heading_level(&style); + } + } + b"numPr" => { + p.list_level.get_or_insert(0); + } + b"ilvl" => { + let level = attribute(start, b"val") + .and_then(|value| value.parse().ok()) + .unwrap_or(0usize); + p.list_level = Some(level.min(8)); + } + b"tab" => p.text.push('\t'), + b"br" | b"cr" => p.text.push('\n'), + _ => {} + } +} + +fn push_paragraph_text(paragraph: &mut Option, text: &str) { + if let Some(p) = paragraph { + p.text.push_str(text); + } +} + +/// Slide part names in presentation order, read from `ppt/presentation.xml` +/// and its relationships; falls back to numeric file order when either is missing. +fn slide_parts(archive: &mut Archive<'_>) -> Result, ContentProcessorError> { + let ordered = (|| -> Result>, ContentProcessorError> { + let Some(presentation) = read_part(archive, "ppt/presentation.xml")? else { + return Ok(None); + }; + let Some(rels) = read_part(archive, "ppt/_rels/presentation.xml.rels")? else { + return Ok(None); + }; + let mut targets = std::collections::HashMap::new(); + let mut reader = XmlReader::from_str(&rels); + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(start) | Event::Empty(start) + if start.local_name().as_ref() == b"Relationship" => + { + if let (Some(id), Some(target)) = + (attribute(&start, b"Id"), attribute(&start, b"Target")) + { + targets.insert(id, target); + } + } + Event::Eof => break, + _ => {} + } + } + let mut slides = Vec::new(); + let mut reader = XmlReader::from_str(&presentation); + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(start) | Event::Empty(start) + if start.local_name().as_ref() == b"sldId" => + { + if let Some(target) = relationship_id(&start).and_then(|id| targets.get(&id)) { + let target = target.trim_start_matches('/'); + let part = if target.starts_with("ppt/") { + target.to_string() + } else { + format!("ppt/{target}") + }; + slides.push(part); + } + } + Event::Eof => break, + _ => {} + } + } + Ok((!slides.is_empty()).then_some(slides)) + })()?; + + if let Some(slides) = ordered { + return Ok(slides); + } + let mut numbered: Vec<(u32, String)> = archive + .file_names() + .filter_map(|name| { + let number = name + .strip_prefix("ppt/slides/slide")? + .strip_suffix(".xml")? + .parse() + .ok()?; + Some((number, name.to_string())) + }) + .collect(); + numbered.sort(); + Ok(numbered.into_iter().map(|(_, name)| name).collect()) +} + +/// The `r:id` attribute of a `p:sldId` element. Its local name is `id`, the same +/// as the numeric slide id, so the namespace prefix tells them apart. +fn relationship_id(start: &BytesStart<'_>) -> Option { + start.attributes().flatten().find_map(|attr| { + let key = attr.key.as_ref(); + (key.ends_with(b":id") && key != b"id") + .then(|| { + attr.normalized_value(quick_xml::XmlVersion::Implicit1_0) + .ok() + .map(Cow::into_owned) + }) + .flatten() + }) +} + +fn pptx_to_markdown(archive: &mut Archive<'_>) -> Result { + let mut parts = slide_parts(archive)?; + let mut warnings = Vec::new(); + if parts.len() > MAX_SLIDES { + parts.truncate(MAX_SLIDES); + warnings.push("slides_truncated".to_string()); + } + let mut sections = Vec::new(); + for (index, part) in parts.iter().enumerate() { + let Some(xml) = read_part(archive, part)? else { + continue; + }; + let paragraphs = slide_paragraphs(&xml)?; + let mut section = format!("## Slide {}", index + 1); + let mut body = paragraphs.into_iter(); + if let Some(title) = body.next() { + section = format!("{section}: {title}"); + } + let rest: Vec = body.collect(); + if !rest.is_empty() { + section.push_str("\n\n"); + section.push_str(&rest.join("\n")); + } + sections.push(section); + } + Ok(Extracted { + markdown: sections.join("\n\n"), + title: core_title(archive), + warnings, + }) +} + +/// Non-empty `a:p` paragraphs of one slide, in document order. +fn slide_paragraphs(xml: &str) -> Result, ContentProcessorError> { + let mut reader = XmlReader::from_str(xml); + let mut paragraphs = Vec::new(); + let mut current: Option = None; + let mut in_text = false; + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(start) => match start.local_name().as_ref() { + b"p" => current = Some(String::new()), + b"t" => in_text = true, + _ => {} + }, + Event::Empty(start) if start.local_name().as_ref() == b"br" => { + if let Some(text) = &mut current { + text.push(' '); + } + } + Event::Text(text) if in_text => { + if let Some(current) = &mut current { + current.push_str(&text.xml10_content().map_err(|e| xml_error(e.into()))?); + } + } + Event::GeneralRef(reference) if in_text => { + if let Some(current) = &mut current { + push_reference(current, &reference); + } + } + Event::End(end) => match end.local_name().as_ref() { + b"t" => in_text = false, + b"p" => { + if let Some(text) = current.take() { + let text = text.trim(); + if !text.is_empty() { + paragraphs.push(text.to_string()); + } + } + } + _ => {} + }, + Event::Eof => break, + _ => {} + } + } + Ok(paragraphs) +} + +fn spreadsheet_to_markdown(body: &[u8]) -> Result { + let mut workbook = open_workbook_auto_from_rs(Cursor::new(body)) + .map_err(|error| ContentProcessorError(format!("cannot open spreadsheet: {error}")))?; + let mut warnings = Vec::new(); + let mut sections = Vec::new(); + for name in workbook.sheet_names() { + let range = match workbook.worksheet_range(&name) { + Ok(range) => range, + Err(error) => { + warnings.push("sheet_unreadable".to_string()); + sections.push(format!("## {name}\n\n_Sheet could not be read: {error}_")); + continue; + } + }; + let total_rows = range.height(); + let total_columns = range.width(); + let rows: Vec> = range + .rows() + .take(MAX_SHEET_ROWS) + .map(|row| row.iter().take(MAX_SHEET_COLUMNS).map(cell_text).collect()) + .collect(); + let Some(table) = markdown_table(&rows) else { + continue; + }; + let mut section = format!("## {name}\n\n{table}"); + if total_rows > MAX_SHEET_ROWS || total_columns > MAX_SHEET_COLUMNS { + push_unique(&mut warnings, "sheet_truncated"); + section.push_str(&format!( + "\n\n_Showing {} of {total_rows} rows and {} of {total_columns} columns._", + total_rows.min(MAX_SHEET_ROWS), + total_columns.min(MAX_SHEET_COLUMNS) + )); + } + sections.push(section); + } + Ok(Extracted { + markdown: sections.join("\n\n"), + title: None, + warnings, + }) +} + +fn push_unique(warnings: &mut Vec, warning: &str) { + if !warnings.iter().any(|existing| existing == warning) { + warnings.push(warning.to_string()); + } +} + +fn cell_text(cell: &Data) -> String { + match cell { + Data::Empty => String::new(), + // Excel stores dates as day serials; show them as ISO 8601 rather than + // numbers such as 41640. Durations keep their numeric form. + Data::DateTime(value) if value.is_datetime() => { + let (year, month, day, hour, minute, second, _) = value.to_ymd_hms_milli(); + if (hour, minute, second) == (0, 0, 0) { + format!("{year:04}-{month:02}-{day:02}") + } else { + format!("{year:04}-{month:02}-{day:02} {hour:02}:{minute:02}:{second:02}") + } + } + other => other.to_string(), + } +} + +/// Render rows as a Markdown table, the first row as the header. Rows are padded +/// to the widest row; fully empty rows and trailing empty columns are dropped. +fn markdown_table(rows: &[Vec]) -> Option { + let rows: Vec<&Vec> = rows + .iter() + .filter(|row| row.iter().any(|cell| !cell.trim().is_empty())) + .collect(); + let width = rows + .iter() + .map(|row| { + row.iter() + .rposition(|cell| !cell.trim().is_empty()) + .map_or(0, |last| last + 1) + }) + .max() + .filter(|width| *width > 0)?; + let render_row = |row: &Vec| { + let cells: Vec = (0..width) + .map(|index| escape_cell(row.get(index).map(String::as_str).unwrap_or(""))) + .collect(); + format!("| {} |", cells.join(" | ")) + }; + let mut lines = vec![render_row(rows[0])]; + lines.push(format!("|{}", " --- |".repeat(width))); + lines.extend(rows[1..].iter().map(|row| render_row(row))); + Some(lines.join("\n")) +} + +fn escape_cell(cell: &str) -> String { + cell.trim() + .replace('\\', "\\\\") + .replace('|', "\\|") + .replace("\r\n", "
") + .replace(['\n', '\r'], "
") +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use bytes::Bytes; + + use super::*; + use crate::content::{ContentFocus, ContentOutputFormat}; + + const DOCX: &str = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + const PPTX: &str = "application/vnd.openxmlformats-officedocument.presentationml.presentation"; + + fn zip_of(parts: &[(&str, &str)]) -> Bytes { + let mut buffer = Cursor::new(Vec::new()); + { + let mut writer = zip::ZipWriter::new(&mut buffer); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated); + for (name, content) in parts { + writer.start_file(*name, options).unwrap(); + writer.write_all(content.as_bytes()).unwrap(); + } + writer.finish().unwrap(); + } + Bytes::from(buffer.into_inner()) + } + + async fn process(url: &str, content_type: &str, body: Bytes) -> ProcessedContent { + OfficeProcessor + .process(ContentProcessorInput { + url: Url::parse(url).unwrap(), + content_type: Some(content_type.to_string()), + body, + output_format: ContentOutputFormat::Markdown, + content_focus: ContentFocus::Full, + }) + .await + .unwrap() + } + + const DOCUMENT_XML: &str = r#" + + + Quarterly report + Revenue grew 12% & costs fell. + First point + Nested point + + RegionSales + EU|West42 + + + +"#; + + const CORE_XML: &str = r#" +Q3 & outlook"#; + + #[tokio::test] + async fn docx_becomes_markdown_with_headings_lists_and_tables() { + let body = zip_of(&[ + ("word/document.xml", DOCUMENT_XML), + ("docProps/core.xml", CORE_XML), + ]); + let processed = process("https://example.com/report", DOCX, body).await; + + assert_eq!(processed.format.as_deref(), Some("markdown")); + assert_eq!( + processed.content.as_deref(), + Some( + "# Quarterly report\n\nRevenue grew 12% & costs fell.\n\n- First point\n\n - Nested point\n\n| Region | Sales |\n| --- | --- |\n| EU\\|West | 42 |" + ) + ); + let metadata = processed.metadata.unwrap(); + assert_eq!(metadata.title.as_deref(), Some("Q3 & outlook")); + assert_eq!(metadata.extraction_method.as_deref(), Some("office_docx")); + assert!(processed.error.is_none()); + } + + #[tokio::test] + async fn pptx_slides_follow_presentation_order() { + let slide = |title: &str, body: &str| { + format!( + r#" +{title} +{body} +"# + ) + }; + let first = slide("Intro", "Why we are here"); + let second = slide("Plan", "Ship it"); + let body = zip_of(&[ + ( + "ppt/presentation.xml", + r#""#, + ), + ( + "ppt/_rels/presentation.xml.rels", + r#""#, + ), + ("ppt/slides/slide1.xml", &second), + ("ppt/slides/slide2.xml", &first), + ]); + let processed = process("https://example.com/deck.pptx", PPTX, body).await; + + assert_eq!( + processed.content.as_deref(), + Some("## Slide 1: Intro\n\nWhy we are here\n\n## Slide 2: Plan\n\nShip it") + ); + } + + #[test] + fn matches_media_types_and_generic_downloads_by_extension() { + let processor = OfficeProcessor; + let page = Url::parse("https://example.com/download").unwrap(); + let xlsx = Url::parse("https://example.com/data.XLSX").unwrap(); + assert!(processor.matches(&page, Some(DOCX))); + assert!(processor.matches(&page, Some("application/vnd.ms-excel"))); + assert!(processor.matches( + &page, + Some("application/vnd.oasis.opendocument.spreadsheet") + )); + assert!(processor.matches(&xlsx, Some("application/octet-stream"))); + assert!(processor.matches(&xlsx, None)); + assert!(!processor.matches(&xlsx, Some("text/html"))); + assert!(!processor.matches(&page, Some("application/octet-stream"))); + assert!(!processor.matches(&page, Some("application/vnd.ms-powerpoint"))); + } + + #[tokio::test] + async fn spreadsheet_from_ods_becomes_tables() { + let content = r#" + + + +ItemPrice +Tea3.5 + +"#; + let body = zip_of(&[ + ("mimetype", "application/vnd.oasis.opendocument.spreadsheet"), + ( + "META-INF/manifest.xml", + r#""#, + ), + ("content.xml", content), + ]); + let processed = process( + "https://example.com/prices", + "application/vnd.oasis.opendocument.spreadsheet", + body, + ) + .await; + + assert_eq!( + processed.content.as_deref(), + Some("## Prices\n\n| Item | Price |\n| --- | --- |\n| Tea | 3.5 |") + ); + assert_eq!( + processed.metadata.unwrap().extraction_method.as_deref(), + Some("office_spreadsheet") + ); + } + + #[test] + fn archive_budget_rejects_zip_bombs() { + // 4 MB of zeros deflates to a few KB; a budget of 1 MB must reject it from + // the central directory alone, before inflating anything. + let zeros = "\0".repeat(4 * 1024 * 1024); + let body = zip_of(&[("word/document.xml", &zeros)]); + assert!(body.len() < 64 * 1024); + + let error = open_archive_within(&body, 1024 * 1024).unwrap_err(); + assert!(error.0.contains("over the 1048576 byte limit"), "{error}"); + assert!(open_archive_within(&body, 8 * 1024 * 1024).is_ok()); + } + + #[tokio::test] + async fn xlsx_becomes_tables() { + let body = zip_of(&[ + ( + "[Content_Types].xml", + r#""#, + ), + ( + "_rels/.rels", + r#""#, + ), + ( + "xl/workbook.xml", + r#""#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#""#, + ), + ( + "xl/worksheets/sheet1.xml", + r#"TeamSpendInfra1200"#, + ), + ]); + let processed = process( + "https://example.com/budget.xlsx", + "application/octet-stream", + body, + ) + .await; + + assert_eq!( + processed.content.as_deref(), + Some("## Budget\n\n| Team | Spend |\n| --- | --- |\n| Infra | 1200 |") + ); + } + + #[test] + fn corrupt_document_is_an_error_not_garbage() { + let error = extract(OfficeKind::Docx, b"not a zip").unwrap_err(); + assert!(error.0.contains("invalid Office archive"), "{error}"); + } + + #[test] + fn spreadsheet_dates_render_as_iso() { + use calamine::{ExcelDateTime, ExcelDateTimeType}; + + let date = Data::DateTime(ExcelDateTime::new( + 41640.0, + ExcelDateTimeType::DateTime, + false, + )); + let stamp = Data::DateTime(ExcelDateTime::new( + 45943.5, + ExcelDateTimeType::DateTime, + false, + )); + assert_eq!(cell_text(&date), "2014-01-01"); + assert_eq!(cell_text(&stamp), "2025-10-13 12:00:00"); + assert_eq!(cell_text(&Data::Float(3.5)), "3.5"); + } + + #[test] + fn heading_levels_from_style_ids() { + assert_eq!(heading_level("Heading1"), Some(1)); + assert_eq!(heading_level("heading 3"), Some(3)); + assert_eq!(heading_level("Title"), Some(1)); + assert_eq!(heading_level("Heading9"), None); + assert_eq!(heading_level("Normal"), None); + } +} diff --git a/crates/fetchkit/src/tool.rs b/crates/fetchkit/src/tool.rs index ed94984..c073a0e 100644 --- a/crates/fetchkit/src/tool.rs +++ b/crates/fetchkit/src/tool.rs @@ -969,15 +969,15 @@ fn display_name(locale: &str) -> &'static str { fn description(locale: &str, enable_save_to_file: bool) -> String { if is_ukrainian(locale) { if enable_save_to_file { - "Завантажити URL як текст або markdown, включно з текстовими PDF; повернути метадані для непідтримуваного бінарного вмісту або зберегти байти через save_to_file.".to_string() + "Завантажити URL як текст або markdown, включно з текстовими PDF і документами Office; повернути метадані для непідтримуваного бінарного вмісту або зберегти байти через save_to_file.".to_string() } else { - "Завантажити URL як текст або markdown, включно з текстовими PDF; повернути метадані для непідтримуваного бінарного вмісту." + "Завантажити URL як текст або markdown, включно з текстовими PDF і документами Office; повернути метадані для непідтримуваного бінарного вмісту." .to_string() } } else if enable_save_to_file { - "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses or save bytes with save_to_file.".to_string() + "Fetch URL content as text or markdown, including text-based PDFs and Office documents; return metadata for unsupported binary responses or save bytes with save_to_file.".to_string() } else { - "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses.".to_string() + "Fetch URL content as text or markdown, including text-based PDFs and Office documents; return metadata for unsupported binary responses.".to_string() } } diff --git a/crates/fetchkit/tests/integration.rs b/crates/fetchkit/tests/integration.rs index 05d23c5..31ab0a8 100644 --- a/crates/fetchkit/tests/integration.rs +++ b/crates/fetchkit/tests/integration.rs @@ -9,7 +9,7 @@ use serde_json::json; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tower::Service; -use wiremock::matchers::{method, path}; +use wiremock::matchers::{header_regex, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; /// Helper: create FetchOptions that permit loopback (for wiremock tests) @@ -465,6 +465,78 @@ async fn test_pdf_content_processor_returns_markdown() { assert!(resp.error.is_none()); } +fn docx_with_paragraph(text: &str) -> Vec { + use std::io::Write; + + let document = format!( + r#"Minutes{text}"# + ); + let mut buffer = std::io::Cursor::new(Vec::new()); + let mut writer = zip::ZipWriter::new(&mut buffer); + writer + .start_file( + "word/document.xml", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + writer.write_all(document.as_bytes()).unwrap(); + writer.finish().unwrap(); + buffer.into_inner() +} + +#[tokio::test] +async fn test_docx_content_processor_returns_markdown() { + let mock_server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/files/minutes")) + .respond_with( + ResponseTemplate::new(200) + .set_body_bytes(docx_with_paragraph("Ship the release on Friday.")) + .insert_header( + "content-type", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ), + ) + .mount(&mock_server) + .await; + + let req = FetchRequest::new(format!("{}/files/minutes", mock_server.uri())).as_markdown(); + let resp = fetch_with_options(req, test_options()).await.unwrap(); + + assert_eq!(resp.format.as_deref(), Some("markdown")); + assert_eq!( + resp.content.as_deref(), + Some("## Minutes\n\nShip the release on Friday.") + ); + assert_eq!( + resp.metadata.unwrap().extraction_method.as_deref(), + Some("office_docx") + ); + assert!(resp.error.is_none()); +} + +#[tokio::test] +async fn test_markdown_request_prefers_markdown_representation() { + let mock_server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/page")) + .and(header_regex("accept", r"^text/markdown(,|$)")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw("# Served as markdown", "text/markdown; charset=utf-8"), + ) + .mount(&mock_server) + .await; + + let req = FetchRequest::new(format!("{}/page", mock_server.uri())).as_markdown(); + let resp = fetch_with_options(req, test_options()).await.unwrap(); + + assert_eq!(resp.status_code, 200); + assert!(resp.content.unwrap().contains("# Served as markdown")); +} + #[tokio::test] async fn test_custom_content_processor_registry_can_disable_pdf_processing() { let mock_server = MockServer::start().await; diff --git a/knowledge/foundations/fetchers.md b/knowledge/foundations/fetchers.md index ed118bf..f794cf0 100644 --- a/knowledge/foundations/fetchers.md +++ b/knowledge/foundations/fetchers.md @@ -89,6 +89,19 @@ allowing callers to replace the default content processor registry. - Extracted output is capped by the same configured `max_body_size`; partial PDF input is not passed to the parser. +#### OfficeProcessor + +- Matches Word (DOCX), PowerPoint (PPTX) and spreadsheet (XLSX, XLSM, XLS, ODS) media + types. The file extension is a fallback only for missing Content-Type, + `application/octet-stream` or `application/zip`. +- Parses locally: DOCX and PPTX XML parts with `quick-xml`, spreadsheets with + `calamine`. Documents never leave the process; there is no hosted converter. +- Zip containers are checked against an entry-count and total uncompressed-size + budget from the central directory before any part is inflated, and each part it + reads itself is read through a size cap (TM-DOS-008). +- Shares the PDF processor's blocking-task path and two-document concurrency cap. +- Output caps: 1,000 slides, 1,000 rows and 50 columns per sheet, then `max_body_size`. + ### Built-in Fetchers #### DefaultFetcher (lowest priority) @@ -392,6 +405,7 @@ Both built-in fetchers integrate resolve-then-check DNS validation: ``` crates/fetchkit/src/ ├── content.rs # ContentProcessor trait, registry, PDF processor +├── office.rs # Office (DOCX, PPTX, XLSX, XLS, ODS) processor ├── dns.rs # DnsPolicy - SSRF prevention via resolve-then-check ├── file_saver.rs # FileSaver trait, LocalFileSaver, SaveResult, FileSaveError ├── fetchers/ diff --git a/knowledge/foundations/tool-contract.md b/knowledge/foundations/tool-contract.md index 858bb7c..eb6f26e 100644 --- a/knowledge/foundations/tool-contract.md +++ b/knowledge/foundations/tool-contract.md @@ -216,7 +216,7 @@ Provide a builder to configure tool options, including: - Expose a single `web_fetch` tool over MCP. - Input schema: derived from `FetchRequest` via tool builder (disabled options omitted). - Output: Markdown with YAML frontmatter (same format as CLI `--output md`). -- Tool description: "Fetch URL content as text or markdown, including text-based PDFs; return metadata for unsupported binary responses or save bytes to file." +- Tool description: "Fetch URL content as text or markdown, including text-based PDFs and Office documents; return metadata for unsupported binary responses or save bytes to file." ### Python Bindings @@ -270,7 +270,8 @@ By default, Fetchkit blocks connections to private/reserved IP ranges: - Ambient proxy environment variables are ignored by default. - Opt in via `ToolBuilder::respect_proxy_env(true)` or CLI `--allow-env-proxy`. - Accept header: - - Markdown: `text/html, text/markdown, text/plain, */*;q=0.8` + - Markdown: `text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.7`, so sites + that serve Markdown on request (Markdown for Agents) return it instead of HTML - Text: `text/html, text/plain, */*;q=0.8` - Raw: `*/*` - HEAD requests use HTTP HEAD method, return metadata only. @@ -320,7 +321,8 @@ By default, Fetchkit blocks connections to private/reserved IP ranges: - Detect binary by Content-Type prefix: - `image/`, `audio/`, `video/`, `application/octet-stream`, `application/pdf`, `application/x-pdf`, `application/zip`, `application/gzip`, `application/x-tar`, `application/x-rar`, - `application/x-7z`, `application/vnd.ms-`, `application/vnd.openxmlformats`, `font/`. + `application/x-7z`, `application/vnd.ms-`, `application/vnd.openxmlformats`, + `application/vnd.oasis.opendocument`, `font/`. - When `as_markdown` is requested, a matching registered content processor runs on the bounded response body before the generic binary response is returned. - The built-in PDF processor: @@ -332,6 +334,20 @@ By default, Fetchkit blocks connections to private/reserved IP ranges: `pdf_requires_ocr` quality warnings and suggest `use_ocr`. - Runs on a bounded blocking pool path, and both PDF input and extracted output honor `max_body_size`. Partial PDF inputs are not parsed. +- The built-in Office processor: + - Matches DOCX, PPTX, XLSX/XLSM, XLS and ODS media types, or those file + extensions when served as `application/octet-stream`, `application/zip` or + without Content-Type. + - Returns `format: "markdown"`: Word headings, lists and tables; one + `## Slide N: title` section per slide in presentation order; one `## sheet` + Markdown table per spreadsheet sheet (first row as header, at most 1,000 rows + and 50 columns, with a `sheet_truncated` warning beyond that). + - Reports `office_docx`, `office_pptx` or `office_spreadsheet` as the extraction + method and `dc:title` from the document properties as the title. + - Performs no network requests. Zip containers are checked against an entry and + uncompressed-size budget before inflating (TM-DOS-008); shares the PDF + processor's blocking pool and two-document concurrency cap. Partial inputs are + not parsed. - For binary without a matching processor: - Return metadata (`content_type`, `size`, `filename`, `last_modified`) - Include `error: "Binary content is not supported. Only textual content (HTML, text, JSON, etc.) can be fetched."` diff --git a/knowledge/log.md b/knowledge/log.md index f109374..8c2d65e 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -1,5 +1,9 @@ # Fetchkit Knowledge Update Log +## 2026-10-12 + +* **Office documents**: Added the local Office processor (DOCX, PPTX, XLSX, XLS, ODS) to the fetchers and tool contract, its zip-bomb threat TM-DOS-008, and the Markdown-first `Accept` header. + ## 2026-08-08 * **HTML processing**: Routed HTML extraction and Markdown/text conversion through the content processor registry, added a replaceable native converter, expanded structural fidelity for code, tables, footnotes, math, callouts, and related rich content, and bounded nested conversion depth. diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 8283e36..6428670 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -298,6 +298,7 @@ to protect against changes between validation and write. | TM-DOS-005 | DNS resolution delay | Low | DNS resolution uses system resolver; no explicit timeout on DNS lookup | **ACCEPTED** | | TM-DOS-006 | Memory exhaustion from large HTML conversion | Medium | Conversion input bounded by `max_body_size` (10 MB default) | MITIGATED | | TM-DOS-007 | CPU/memory exhaustion from adversarial PDF parsing | Medium | Bounded input/output, blocking-task isolation, and a cancellation-safe two-document concurrency cap; parser-internal expansion remains possible | **ACCEPTED** | +| TM-DOS-008 | Zip bomb or adversarial Office document | Medium | Central-directory entry and uncompressed-size budget before inflating, size-capped part reads, per-sheet output caps, shared two-document blocking cap | MITIGATED | ### Mitigation Details @@ -329,6 +330,16 @@ Operators handling hostile, high-volume PDFs should isolate the process and appl request CPU/memory limits. Accepted because the in-process Rust parser has no reliable cancellation boundary once a blocking parse begins. +**TM-DOS-008 — Zip bomb or adversarial Office document (MITIGATED):** +Office documents are zip archives (except legacy XLS), so a body within `max_body_size` +can declare gigabytes of content. Before any part is inflated, the Office processor +rejects archives with more than 10,000 entries or more than 200 MB of declared +uncompressed content, and every XML part it reads itself is read through a 64 MB cap, +since declared sizes can lie. Spreadsheet output is capped at 1,000 rows and 50 columns +per sheet before the `max_body_size` output cap. Parsing shares the PDF processor's +blocking path and two-document concurrency cap. `calamine` inflates sheet parts itself; +the declared-size budget bounds those as well. + ## 5. Information Leakage (TM-LEAK) | ID | Threat | Severity | Mitigation | Status | From 1ab54370cc68dc106dff687e4ee268b6b13ef312 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 12 Oct 2026 17:44:55 +0000 Subject: [PATCH 2/2] chore(deps): bump async-trait to 0.1.92 for Rust 1.99 clippy Clippy 1.99's double_must_use lint fires on code generated by async-trait 0.1.89 for every async trait, failing Lint on main too. 0.1.92 no longer emits the redundant must_use. --- Cargo.lock | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a15fc2..a24c9f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,13 +132,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2 1.0.106", "quote 1.0.46", - "syn 2.0.118", + "syn 3.0.9", ] [[package]] @@ -3062,6 +3062,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2 1.0.106", + "quote 1.0.46", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2"