diff --git a/Sources/DigUpKit/Extractors.swift b/Sources/DigUpKit/Extractors.swift index 425407f0..a2050c17 100644 --- a/Sources/DigUpKit/Extractors.swift +++ b/Sources/DigUpKit/Extractors.swift @@ -56,6 +56,9 @@ enum DocExtractor { case "docx", "doc", "odt", "rtf": return try richText(of: url, type: ext) case "xlsx", "pptx", "ods", "odp": return try OfficeText.text(of: url, type: ext) case "html", "htm": return plainText(fromHTML: try readText(url)) + case "emlx": // Apple Mail's message files: header lines + text bodies (issue #146); nil = the raw text + let mail = try readText(url) + return MailMessage.text(from: Data(mail.utf8)) ?? mail default: return try readText(url) } } diff --git a/Sources/DigUpKit/MailMessage.swift b/Sources/DigUpKit/MailMessage.swift new file mode 100644 index 00000000..128dbec6 --- /dev/null +++ b/Sources/DigUpKit/MailMessage.swift @@ -0,0 +1,199 @@ +import Foundation + +/// Apple Mail's messages (`.emlx`) read for search: the sender, recipient, subject and +/// date header lines first, then the body — `text/*` parts as they are, `text/html` +/// stripped, quoted-printable and base64 transfer encodings decoded, attachments left +/// out (the files beside them are indexed by their own readers; a message whose only +/// body is an attachment keeps its header lines, which is the searchable part of it). +/// +/// Deliberately small: RFC 822 is wide, and a file this reader can't make sense of +/// answers `nil` — `DocExtractor` then returns the raw text, which for mail is mostly +/// readable words and never less than what people get today by treating the file as +/// text. It is *not* a mail client: attachments are not unpacked, DKIM and MIME quirks +/// are not parsed, nested multiparts go eight levels deep at most, charsets are read as +/// UTF-8 (what mail uses today), and Mail.app's own database is not read — issue #146's +/// full Mail-source idea (Envelope Index, `message://` opening, a Settings source +/// toggle) is its own feature; this is the file-reading slice, the one its "the same way +/// it searches PDFs and documents" asks for. +enum MailMessage { + /// Search text for one message's bytes, nil when its shape is not understood. + static func text(from data: Data) -> String? { + guard data.count <= 8_000_000 else { return nil } // a base64 vault isn't a mail to read + return one(String(decoding: data, as: UTF8.self), depth: 0) + } + + private static func one(_ raw: String, depth: Int) -> String? { + guard depth <= 8, let split = raw.firstRange(of: "\r\n\r\n") ?? raw.firstRange(of: "\n\n") else { return nil } + let header = String(raw[raw.startIndex.. [String] { + let normalized = header.replacingOccurrences(of: "\r\n", with: "\n").replacingOccurrences(of: "\r", with: "\n") + var out: [String] = [] + var current = "" + var collecting = false + for line in normalized.components(separatedBy: "\n") { + if line.isEmpty { continue } + if line.first == " " || line.first == "\t" { + if collecting { current += " " + line.trimmingCharacters(in: .whitespaces) } + continue + } + if collecting, !current.isEmpty { out.append(decodedHeader(current)) } + collecting = false + guard let colon = line.firstIndex(of: ":") else { continue } + guard line[line.startIndex.. String { + var out = "" + var rest = Substring(value) + while let start = rest.firstRange(of: "=?") { + out += rest[rest.startIndex.. String? { + switch encoding.uppercased() { + case "Q": + var bytes: [UInt8] = [] + var rest = Substring(text) + // the earlier of the next `_` (mail's space) and `=` (hex escape) — an + // `=`-escape before a later `_` must not stop at the `_` with the raw + // `=C3=A9` left in the text. + while let at = [rest.firstIndex(of: "_"), rest.firstIndex(of: "=")].compactMap({ $0 }).min() { + bytes.append(contentsOf: rest[rest.startIndex..= 2, let byte = UInt8(after.prefix(2), radix: 16) else { return nil } + bytes.append(byte) + rest = after.dropFirst(2) + } + bytes.append(contentsOf: rest.utf8) + return String(decoding: bytes, as: UTF8.self) + case "B": + var word = text + while word.count % 4 != 0 { word += "=" } // RFC 2047 sends 7-bit words unpadded + guard let data = Data(base64Encoded: word, options: [.ignoreUnknownCharacters]) else { return nil } + return String(decoding: data, as: UTF8.self) + default: + return nil + } + } + + /// The quoted string or bare token of a MIME parameter: `boundary="x;1"` → `x;1`. + /// The case-insensitive search runs on the value *itself*: indices from + /// `value.lowercased()` do not align when lowering changes a string's UTF-8 length + /// (`İ` is 2 bytes, its lower case 3), and slicing the original with them is a + /// fatal trap the callers cannot catch — mail headers are untrusted input. + static func parameter(_ name: String, in value: String?) -> String? { + guard let value else { return nil } + guard let at = value.range(of: name + "=", options: [.caseInsensitive]) else { return nil } + var tail = value[at.upperBound...] + if tail.first == "\"" { + tail = tail.dropFirst() + guard let close = tail.firstIndex(of: "\"") else { return String(tail) } + return String(tail[.. String { + var text = body + switch transfer.lowercased() { + case "quoted-printable", "quoted_printable": text = quotedPrintable(body) + case "base64": + var clean = body.filter { !$0.isNewline } + while clean.count % 4 != 0 { clean += "=" } + text = Data(base64Encoded: clean, options: [.ignoreUnknownCharacters]) + .map { String(decoding: $0, as: UTF8.self) } ?? body + default: break + } + if contentType.contains("html") { text = DocExtractor.plainText(fromHTML: text) } + return text + } + + /// `=3D` hex escapes and `=\r\n` soft line breaks (RFC 2045) — collected as bytes and + /// read back as UTF-8 at the end, so `=C3=A9` lands as "é" and not as two letters. + /// Plain text (the run between escapes, and a hex-looking word after an unescaped + /// `=`) passes through as written. The walk stays on views of the same string: + /// rebuilding the rest per escape made it quadratic (measured at the review: + /// 128k escapes, 76 s on this box) — a body of dense escapes at the 8 MB cap + /// would take hours of one worker. + static func quotedPrintable(_ text: String) -> String { + var bytes: [UInt8] = [] + var rest = Substring(text) + while let at = rest.firstIndex(of: "=") { + bytes.append(contentsOf: rest[rest.startIndex.. = [ "txt", "text", "md", "markdown", "typ", "rtf", "docx", "doc", "odt", "html", "htm", "xlsx", "pptx", "ods", "odp", + "emlx", // Apple Mail messages, read by `MailMessage` (issue #146) ] /// FairPlay-protected music (Apple Music downloads, iTunes Store songs bought before 2009): only the Music app can diff --git a/Tests/DigUpKitTests/MailMessageTests.swift b/Tests/DigUpKitTests/MailMessageTests.swift new file mode 100644 index 00000000..9801d304 --- /dev/null +++ b/Tests/DigUpKitTests/MailMessageTests.swift @@ -0,0 +1,126 @@ +import Foundation +import Testing +@testable import DigUpKit + +/// Issue #146, the file-reading slice: Apple Mail's `.emlx` as searchable text — +/// header lines, quoted-printable and base64 bodies, HTML stripped, RFC 2047 encoded +/// words, multipart nesting, attachments left out. Built from bytes here (no fixtures, +/// no Mail.app): the shapes real .emlx files carry, as Apple's Mail writes them. +@Suite struct MailMessageTests { + func bytes(_ s: String) -> Data { Data(s.utf8) } + + @Test func plainMessageGivesItsHeaderLinesThenItsBody() throws { + let mail = "From: Ana \r\n" + + "To: Bo \r\n" + + "Subject: The bridge survey\r\n" + + "Date: Tue, 4 Nov 2025 09:12:00 +0100\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + + "\r\n" + + "The ferry timetables change with the tide.\r\n" + + "Bring the survey flags on Saturday.\r\n" + let text = try #require(MailMessage.text(from: bytes(mail))) + #expect(text.contains("Subject: The bridge survey")) + #expect(text.contains("From: Ana ")) + #expect(text.contains("ferry timetables")) + #expect(text.contains("survey flags")) + } + + @Test func quotedPrintableAndSoftBreaksAreDecoded() throws { + let mail = "Subject: =?utf-8?Q?Copied=5FPath?=\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "A =3D sign, a =C3=A9 accent, and a soft=\r\n" + + "broken line.\r\n" + let text = try #require(MailMessage.text(from: bytes(mail))) + #expect(text.contains("Copied_Path")) + #expect(text.contains("A = sign")) + #expect(text.contains("é accent")) + #expect(text.contains("a softbroken line.") || text.contains("a soft broken line.")) + } + + @Test func htmlBodiesAreStripped() throws { + let mail = "Subject: Notes\r\nContent-Type: text/html; charset=utf-8\r\n\r\n" + + "

The millrace freezes at the edges.

" + + "\r\n" + let text = try #require(MailMessage.text(from: bytes(mail))) + #expect(text.contains("The millrace freezes at the edges.")) + #expect(!text.contains("")) + #expect(!text.contains("var x")) + } + + @Test func base64BodyDecodes() throws { + let body = Data("The lighthouse keeper climbed the stairs.".utf8).base64EncodedString() + let mail = "Subject: Late note\r\nContent-Type: text/plain\r\n" + + "Content-Transfer-Encoding: base64\r\n\r\n" + body + "\r\n" + let text = try #require(MailMessage.text(from: bytes(mail))) + #expect(text.contains("lighthouse keeper climbed")) + } + + @Test func attachmentsStayOutAndTheHeaderStaysSearchable() throws { + let png = Data([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]).base64EncodedString() + let mail = "From: Ana \r\nSubject: The violin varnish\r\n" + + "Content-Type: multipart/mixed; boundary=\"b1\"\r\n\r\n" + + "--b1\r\nContent-Type: text/plain\r\n\r\nThe varnish coat dries by afternoon.\r\n" + + "--b1\r\nContent-Type: image/png; name=coat.png\r\n" + + "Content-Transfer-Encoding: base64\r\n\r\n\(png)\r\n--b1--\r\n" + let text = try #require(MailMessage.text(from: bytes(mail))) + #expect(text.contains("Subject: The violin varnish")) + #expect(text.contains("varnish coat dries")) + #expect(!text.contains(png)) // the attachment's bytes are not the message's words + } + + @Test func attachmentOnlyMessagesKeepTheirHeaderLines() throws { + let png = Data([0x89, 0x50, 0x4E, 0x47]).base64EncodedString() + let mail = "From: Ana \r\nSubject: The scan\r\n" + + "Content-Type: image/png; name=scan.png\r\nContent-Transfer-Encoding: base64\r\n\r\n\(png)\r\n" + let text = try #require(MailMessage.text(from: bytes(mail))) + #expect(text.contains("Subject: The scan")) + #expect(!text.contains(png)) + } + + @Test func nestedMultipartNests() throws { + let mail = "Subject: Layers\r\nContent-Type: multipart/alternative; boundary=\"outer\"\r\n\r\n" + + "--outer\r\nContent-Type: text/plain\r\n\r\nPlain words: the compost and straw.\r\n" + + "--outer\r\nContent-Type: multipart/related; boundary=\"inner\"\r\n\r\n" + + "--inner\r\nContent-Type: text/html\r\n\r\n

Rich words: the patient gardener.

\r\n" + + "--inner--\r\n--outer--\r\n" + let text = try #require(MailMessage.text(from: bytes(mail))) + #expect(text.contains("compost and straw")) + #expect(text.contains("patient gardener")) + } + + @Test func encodedWordsInHeadersDecode() { + #expect(MailMessage.decodedHeader("=?utf-8?B?QnJpZQ?= =?utf-8?Q?and?= jam") == "Brie and jam") + #expect(MailMessage.decodedHeader("plain, no words") == "plain, no words") + // hex escapes and `_` spaces in one word — the earlier marker wins, and the + // word decodes fully: Apple Mail writes subjects like this one. + #expect(MailMessage.decodedHeader("=?utf-8?Q?R=C3=A9_Erde?=") == "Ré Erde") + } + + @Test func charsetNamesThatShrinkWhenLoweredDoNotTrap() throws { + // Slicing the value with indices from value.lowercased() traps (fatal, not a + // catchable Error — the whole helper would die) when lowering changes UTF-8 + // length: `İ` is 2 bytes, its lower case 3. Mail headers are untrusted input. + let mail = "Content-Type: text/plain; charset=İso-8859-1\r\n\r\nWords about the quays.\r\n" + let text = MailMessage.text(from: bytes(mail)) + #expect(text == nil || text!.contains("Words about the quays")) + #expect(MailMessage.parameter("boundary", in: "multipart/mixed; boundary=\"İzmir\"") == "İzmir") + #expect(MailMessage.parameter("CHARSET", in: "text/plain; Charset=UTF-8") == "UTF-8") + } + + @Test func quotedBoundariesAreHonored() { + #expect(MailMessage.parameter("boundary", in: "multipart/mixed; boundary=\"x;1\"") == "x;1") + #expect(MailMessage.parameter("boundary", in: "multipart/mixed; boundary=abc123") == "abc123") + #expect(MailMessage.parameter("charset", in: "text/plain; charset=utf-8") == "utf-8") + } + + @Test func shapesTheReaderDoesNotUnderstandFallBackToRawText() throws { + // No header/body split at all: the caller (`DocExtractor`) keeps the raw text. + #expect(MailMessage.text(from: bytes("just one line, no blank line")) == nil) + } + + @Test func emlxIsADocExtension() { + #expect(Rules.docExtensions.contains("emlx")) + } +}