Read Apple Mail .emlx Files: Decode Quoted-Printable Bodies, Skip Base64 Attachments, and Handle Mixed-Case Headers

Sources/DigUpKit/Extractors.swift

@@ -56,6 +56,9 @@ enum DocExtractor {
         case "docx", "doc", "odt", "rtf": return try richText(of: url, type: ext)
         case "xlsx", "pptx", "ods", "odp": return try OfficeText.text(of: url, type: ext)
         case "html", "htm": return plainText(fromHTML: try readText(url))
+        case "emlx":   // Apple Mail's message files: header lines + text bodies (issue #146); nil = the raw text
+            let mail = try readText(url)
+            return MailMessage.text(from: Data(mail.utf8)) ?? mail
         default: return try readText(url)
         }
     }

Sources/DigUpKit/MailMessage.swift

@@ -0,0 +1,199 @@
+import Foundation
+
+/// Apple Mail's messages (`.emlx`) read for search: the sender, recipient, subject and
+/// date header lines first, then the body — `text/*` parts as they are, `text/html`
+/// stripped, quoted-printable and base64 transfer encodings decoded, attachments left
+/// out (the files beside them are indexed by their own readers; a message whose only
+/// body is an attachment keeps its header lines, which is the searchable part of it).
+///
+/// Deliberately small: RFC 822 is wide, and a file this reader can't make sense of
+/// answers `nil` — `DocExtractor` then returns the raw text, which for mail is mostly
+/// readable words and never less than what people get today by treating the file as
+/// text. It is *not* a mail client: attachments are not unpacked, DKIM and MIME quirks
+/// are not parsed, nested multiparts go eight levels deep at most, charsets are read as
+/// UTF-8 (what mail uses today), and Mail.app's own database is not read — issue #146's
+/// full Mail-source idea (Envelope Index, `message://` opening, a Settings source
+/// toggle) is its own feature; this is the file-reading slice, the one its "the same way
+/// it searches PDFs and documents" asks for.
+enum MailMessage {
+    /// Search text for one message's bytes, nil when its shape is not understood.
+    static func text(from data: Data) -> String? {
+        guard data.count <= 8_000_000 else { return nil }   // a base64 vault isn't a mail to read
+        return one(String(decoding: data, as: UTF8.self), depth: 0)
+    }
+
+    private static func one(_ raw: String, depth: Int) -> String? {
+        guard depth <= 8, let split = raw.firstRange(of: "\r\n\r\n") ?? raw.firstRange(of: "\n\n") else { return nil }
+        let header = String(raw[raw.startIndex..<split.lowerBound])
+        let body = String(raw[split.upperBound...])
+        var lines: [String] = []
+        for field in ["From", "To", "Cc", "Subject", "Date"] {
+            for value in headerField(field, in: header) { lines.append("\(field): \(value)") }
+        }
+        let content = headerField("Content-Type", in: header).first?.lowercased() ?? "text/plain"
+        let transfer = headerField("Content-Transfer-Encoding", in: header).first ?? "7bit"
+        var out: [String] = [lines.joined(separator: "\n")]
+        if content.contains("multipart") {
+            guard let boundary = parameter("boundary", in: content) else { return nil }
+            for chunk in body.components(separatedBy: "--\(boundary)") {
+                guard let part = one(chunk, depth: depth + 1) else { continue }
+                out.append(part)
+            }
+        } else if content.hasPrefix("text/") {
+            out.append(decode(body, transfer, content))
+        }   // an attachment's bytes: the header lines above are all it offers to a text index
+        let text = out.filter { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }
+            .joined(separator: "\n\n")
+        return text.isEmpty ? nil : text
+    }
+
+    // MARK: Headers
+
+    /// Values of a header field: continuation lines (the ones starting with a space) joined,
+    /// RFC 2047 encoded-words decoded — the shape mail has used since 1997.
+    static func headerField(_ name: String, in header: String) -> [String] {
+        let normalized = header.replacingOccurrences(of: "\r\n", with: "\n").replacingOccurrences(of: "\r", with: "\n")
+        var out: [String] = []
+        var current = ""
+        var collecting = false
+        for line in normalized.components(separatedBy: "\n") {
+            if line.isEmpty { continue }
+            if line.first == " " || line.first == "\t" {
+                if collecting { current += " " + line.trimmingCharacters(in: .whitespaces) }
+                continue
+            }
+            if collecting, !current.isEmpty { out.append(decodedHeader(current)) }
+            collecting = false
+            guard let colon = line.firstIndex(of: ":") else { continue }
+            guard line[line.startIndex..<colon].trimmingCharacters(in: .whitespaces)
+                    .caseInsensitiveCompare(name) == .orderedSame
+            else { continue }
+            collecting = true
+            current = String(line[line.index(after: colon)...].trimmingCharacters(in: .whitespaces))
+        }
+        if collecting, !current.isEmpty { out.append(decodedHeader(current)) }
+        return out
+    }
+
+    /// "=?utf-8?Q?Copied=5FPath?=" → "Copied_Path"-style words, `B` base64 and `Q`
+    /// quoted-printable (where `_` is the space). An undecodable word stays as
+    /// written: mail readers are generous, and a literal is better search text than
+    /// dropping the line.
+    static func decodedHeader(_ value: String) -> String {
+        var out = ""
+        var rest = Substring(value)
+        while let start = rest.firstRange(of: "=?") {
+            out += rest[rest.startIndex..<start.lowerBound]
+            let tail = rest[start.upperBound...]
+            guard let end = tail.firstRange(of: "?=") else { out += rest[start.lowerBound...]; return out }
+            let words = String(tail[tail.startIndex..<end.lowerBound])
+            rest = tail[end.upperBound...]
+            let parts = words.split(separator: "?", omittingEmptySubsequences: false).map(String.init)
+            guard parts.count == 3, let decoded = decodedHeaderWord(encoding: parts[1], text: parts[2]) else {
+                out += "=\(words)?="
+                continue
+            }
+            out += decoded
+        }
+        return out + String(rest)
+    }
+
+    private static func decodedHeaderWord(encoding: String, text: String) -> String? {
+        switch encoding.uppercased() {
+        case "Q":
+            var bytes: [UInt8] = []
+            var rest = Substring(text)
+            // the earlier of the next `_` (mail's space) and `=` (hex escape) — an
+            // `=`-escape before a later `_` must not stop at the `_` with the raw
+            // `=C3=A9` left in the text.
+            while let at = [rest.firstIndex(of: "_"), rest.firstIndex(of: "=")].compactMap({ $0 }).min() {
+                bytes.append(contentsOf: rest[rest.startIndex..<at].utf8)
+                if rest[at] == "_" {
+                    bytes.append(0x20)
+                    rest = rest[rest.index(after: at)...]
+                    continue
+                }
+                let after = rest[rest.index(after: at)...]
+                guard after.count >= 2, let byte = UInt8(after.prefix(2), radix: 16) else { return nil }
+                bytes.append(byte)
+                rest = after.dropFirst(2)
+            }
+            bytes.append(contentsOf: rest.utf8)
+            return String(decoding: bytes, as: UTF8.self)
+        case "B":
+            var word = text
+            while word.count % 4 != 0 { word += "=" }   // RFC 2047 sends 7-bit words unpadded
+            guard let data = Data(base64Encoded: word, options: [.ignoreUnknownCharacters]) else { return nil }
+            return String(decoding: data, as: UTF8.self)
+        default:
+            return nil
+        }
+    }
+
+    /// The quoted string or bare token of a MIME parameter: `boundary="x;1"` → `x;1`.
+    /// The case-insensitive search runs on the value *itself*: indices from
+    /// `value.lowercased()` do not align when lowering changes a string's UTF-8 length
+    /// (`İ` is 2 bytes, its lower case 3), and slicing the original with them is a
+    /// fatal trap the callers cannot catch — mail headers are untrusted input.
+    static func parameter(_ name: String, in value: String?) -> String? {
+        guard let value else { return nil }
+        guard let at = value.range(of: name + "=", options: [.caseInsensitive]) else { return nil }
+        var tail = value[at.upperBound...]
+        if tail.first == "\"" {
+            tail = tail.dropFirst()
+            guard let close = tail.firstIndex(of: "\"") else { return String(tail) }
+            return String(tail[..<close])
+        }
+        return String(tail.prefix { !$0.isWhitespace && $0 != ";" && $0 != "," })
+    }
+
+    // MARK: Bodies
+
+    private static func decode(_ body: String, _ transfer: String, _ contentType: String) -> String {
+        var text = body
+        switch transfer.lowercased() {
+        case "quoted-printable", "quoted_printable": text = quotedPrintable(body)
+        case "base64":
+            var clean = body.filter { !$0.isNewline }
+            while clean.count % 4 != 0 { clean += "=" }
+            text = Data(base64Encoded: clean, options: [.ignoreUnknownCharacters])
+                .map { String(decoding: $0, as: UTF8.self) } ?? body
+        default: break
+        }
+        if contentType.contains("html") { text = DocExtractor.plainText(fromHTML: text) }
+        return text
+    }
+
+    /// `=3D` hex escapes and `=\r\n` soft line breaks (RFC 2045) — collected as bytes and
+    /// read back as UTF-8 at the end, so `=C3=A9` lands as "é" and not as two letters.
+    /// Plain text (the run between escapes, and a hex-looking word after an unescaped
+    /// `=`) passes through as written. The walk stays on views of the same string:
+    /// rebuilding the rest per escape made it quadratic (measured at the review:
+    /// 128k escapes, 76 s on this box) — a body of dense escapes at the 8 MB cap
+    /// would take hours of one worker.
+    static func quotedPrintable(_ text: String) -> String {
+        var bytes: [UInt8] = []
+        var rest = Substring(text)
+        while let at = rest.firstIndex(of: "=") {
+            bytes.append(contentsOf: rest[rest.startIndex..<at].utf8)
+            let after = rest[rest.index(after: at)...]
+            if after.isEmpty { bytes.append(0x3D); return String(decoding: bytes, as: UTF8.self) }
+            if after.first == "\n" || after.first == "\r" || after.first == "\r\n" {
+                // Soft break: mail wraps long lines by ending them with "=". A CRLF pair is
+                // one Character in Swift (grapheme), so all three shapes drop as one.
+                rest = after.dropFirst(1)
+                continue
+            }
+            let hex = after.prefix(2)
+            if hex.count == 2, let byte = UInt8(hex, radix: 16) {   // not `after.count >= 2`:
+                bytes.append(byte)                                    // Substring.count scans
+                rest = after.dropFirst(2)                             // the whole rest — quadratic
+                continue
+            }
+            bytes.append(0x3D)   // a stray "=" is mail being generous: keep it
+            rest = after
+        }
+        bytes.append(contentsOf: rest.utf8)
+        return String(decoding: bytes, as: UTF8.self)
+    }
+}

Sources/DigUpKit/Rules.swift

@@ -23,6 +23,7 @@ public enum Rules {
 
     static let docExtensions: Set<String> = [
         "txt", "text", "md", "markdown", "typ", "rtf", "docx", "doc", "odt", "html", "htm", "xlsx", "pptx", "ods", "odp",
+        "emlx",   // Apple Mail messages, read by `MailMessage` (issue #146)
     ]
 
     /// FairPlay-protected music (Apple Music downloads, iTunes Store songs bought before 2009): only the Music app can

Tests/DigUpKitTests/MailMessageTests.swift

@@ -0,0 +1,126 @@
+import Foundation
+import Testing
+@testable import DigUpKit
+
+/// Issue #146, the file-reading slice: Apple Mail's `.emlx` as searchable text —
+/// header lines, quoted-printable and base64 bodies, HTML stripped, RFC 2047 encoded
+/// words, multipart nesting, attachments left out. Built from bytes here (no fixtures,
+/// no Mail.app): the shapes real .emlx files carry, as Apple's Mail writes them.
+@Suite struct MailMessageTests {
+    func bytes(_ s: String) -> Data { Data(s.utf8) }
+
+    @Test func plainMessageGivesItsHeaderLinesThenItsBody() throws {
+        let mail = "From: Ana <ana@example.org>\r\n"
+            + "To: Bo <bo@example.org>\r\n"
+            + "Subject: The bridge survey\r\n"
+            + "Date: Tue, 4 Nov 2025 09:12:00 +0100\r\n"
+            + "Content-Type: text/plain; charset=utf-8\r\n"
+            + "\r\n"
+            + "The ferry timetables change with the tide.\r\n"
+            + "Bring the survey flags on Saturday.\r\n"
+        let text = try #require(MailMessage.text(from: bytes(mail)))
+        #expect(text.contains("Subject: The bridge survey"))
+        #expect(text.contains("From: Ana <ana@example.org>"))
+        #expect(text.contains("ferry timetables"))
+        #expect(text.contains("survey flags"))
+    }
+
+    @Test func quotedPrintableAndSoftBreaksAreDecoded() throws {
+        let mail = "Subject: =?utf-8?Q?Copied=5FPath?=\r\n"
+            + "Content-Type: text/plain; charset=utf-8\r\n"
+            + "Content-Transfer-Encoding: quoted-printable\r\n"
+            + "\r\n"
+            + "A =3D sign, a =C3=A9 accent, and a soft=\r\n"
+            + "broken line.\r\n"
+        let text = try #require(MailMessage.text(from: bytes(mail)))
+        #expect(text.contains("Copied_Path"))
+        #expect(text.contains("A = sign"))
+        #expect(text.contains("é accent"))
+        #expect(text.contains("a softbroken line.") || text.contains("a soft broken line."))
+    }
+
+    @Test func htmlBodiesAreStripped() throws {
+        let mail = "Subject: Notes\r\nContent-Type: text/html; charset=utf-8\r\n\r\n"
+            + "<html><body><style>p{}</style><p>The millrace <b>freezes</b> at the edges.</p>"
+            + "<script>var x=1;</script></body></html>\r\n"
+        let text = try #require(MailMessage.text(from: bytes(mail)))
+        #expect(text.contains("The millrace freezes at the edges."))
+        #expect(!text.contains("<b>"))
+        #expect(!text.contains("var x"))
+    }
+
+    @Test func base64BodyDecodes() throws {
+        let body = Data("The lighthouse keeper climbed the stairs.".utf8).base64EncodedString()
+        let mail = "Subject: Late note\r\nContent-Type: text/plain\r\n"
+            + "Content-Transfer-Encoding: base64\r\n\r\n" + body + "\r\n"
+        let text = try #require(MailMessage.text(from: bytes(mail)))
+        #expect(text.contains("lighthouse keeper climbed"))
+    }
+
+    @Test func attachmentsStayOutAndTheHeaderStaysSearchable() throws {
+        let png = Data([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]).base64EncodedString()
+        let mail = "From: Ana <ana@example.org>\r\nSubject: The violin varnish\r\n"
+            + "Content-Type: multipart/mixed; boundary=\"b1\"\r\n\r\n"
+            + "--b1\r\nContent-Type: text/plain\r\n\r\nThe varnish coat dries by afternoon.\r\n"
+            + "--b1\r\nContent-Type: image/png; name=coat.png\r\n"
+            + "Content-Transfer-Encoding: base64\r\n\r\n\(png)\r\n--b1--\r\n"
+        let text = try #require(MailMessage.text(from: bytes(mail)))
+        #expect(text.contains("Subject: The violin varnish"))
+        #expect(text.contains("varnish coat dries"))
+        #expect(!text.contains(png))   // the attachment's bytes are not the message's words
+    }
+
+    @Test func attachmentOnlyMessagesKeepTheirHeaderLines() throws {
+        let png = Data([0x89, 0x50, 0x4E, 0x47]).base64EncodedString()
+        let mail = "From: Ana <ana@example.org>\r\nSubject: The scan\r\n"
+            + "Content-Type: image/png; name=scan.png\r\nContent-Transfer-Encoding: base64\r\n\r\n\(png)\r\n"
+        let text = try #require(MailMessage.text(from: bytes(mail)))
+        #expect(text.contains("Subject: The scan"))
+        #expect(!text.contains(png))
+    }
+
+    @Test func nestedMultipartNests() throws {
+        let mail = "Subject: Layers\r\nContent-Type: multipart/alternative; boundary=\"outer\"\r\n\r\n"
+            + "--outer\r\nContent-Type: text/plain\r\n\r\nPlain words: the compost and straw.\r\n"
+            + "--outer\r\nContent-Type: multipart/related; boundary=\"inner\"\r\n\r\n"
+            + "--inner\r\nContent-Type: text/html\r\n\r\n<p>Rich words: the patient gardener.</p>\r\n"
+            + "--inner--\r\n--outer--\r\n"
+        let text = try #require(MailMessage.text(from: bytes(mail)))
+        #expect(text.contains("compost and straw"))
+        #expect(text.contains("patient gardener"))
+    }
+
+    @Test func encodedWordsInHeadersDecode() {
+        #expect(MailMessage.decodedHeader("=?utf-8?B?QnJpZQ?= =?utf-8?Q?and?= jam") == "Brie and jam")
+        #expect(MailMessage.decodedHeader("plain, no words") == "plain, no words")
+        // hex escapes and `_` spaces in one word — the earlier marker wins, and the
+        // word decodes fully: Apple Mail writes subjects like this one.
+        #expect(MailMessage.decodedHeader("=?utf-8?Q?R=C3=A9_Erde?=") == "Ré Erde")
+    }
+
+    @Test func charsetNamesThatShrinkWhenLoweredDoNotTrap() throws {
+        // Slicing the value with indices from value.lowercased() traps (fatal, not a
+        // catchable Error — the whole helper would die) when lowering changes UTF-8
+        // length: `İ` is 2 bytes, its lower case 3. Mail headers are untrusted input.
+        let mail = "Content-Type: text/plain; charset=İso-8859-1\r\n\r\nWords about the quays.\r\n"
+        let text = MailMessage.text(from: bytes(mail))
+        #expect(text == nil || text!.contains("Words about the quays"))
+        #expect(MailMessage.parameter("boundary", in: "multipart/mixed; boundary=\"İzmir\"") == "İzmir")
+        #expect(MailMessage.parameter("CHARSET", in: "text/plain; Charset=UTF-8") == "UTF-8")
+    }
+
+    @Test func quotedBoundariesAreHonored() {
+        #expect(MailMessage.parameter("boundary", in: "multipart/mixed; boundary=\"x;1\"") == "x;1")
+        #expect(MailMessage.parameter("boundary", in: "multipart/mixed; boundary=abc123") == "abc123")
+        #expect(MailMessage.parameter("charset", in: "text/plain; charset=utf-8") == "utf-8")
+    }
+
+    @Test func shapesTheReaderDoesNotUnderstandFallBackToRawText() throws {
+        // No header/body split at all: the caller (`DocExtractor`) keeps the raw text.
+        #expect(MailMessage.text(from: bytes("just one line, no blank line")) == nil)
+    }
+
+    @Test func emlxIsADocExtension() {
+        #expect(Rules.docExtensions.contains("emlx"))
+    }
+}