diff --git a/CHANGELOG.md b/CHANGELOG.md index 123213f78..4e9678c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,12 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. `code_required` (below). A server and a computer must be upgraded together: an older computer has no `/sign-in/code`. +### An HTML-only email reaches its trigger as the sender wrote it + +An email trigger reads an HTML-only mail by stripping its tags, and the character references stayed: +the Bot read `AT&T`, `we’re` and `Q3 numbers`. Numeric references, `&`, `<`, +`>`, `"`, `'` and ` ` are now decoded, once, so `&lt;` reads as `<`. + ### The Bot's screen opens only when there is something to watch Asking a Bot a question it answered in words could still open its computer beside the conversation, diff --git a/server/src/responsibilities/email.ts b/server/src/responsibilities/email.ts index 50b139494..e65d674dd 100644 --- a/server/src/responsibilities/email.ts +++ b/server/src/responsibilities/email.ts @@ -172,6 +172,36 @@ function decodeBody(body: string, encoding: string | undefined) { ).toString("utf8"); return body; } +/** + * An HTML-only mail writes `AT&T` and `we’re`, and the Bot should read what the sender + * wrote. Numeric references, the five XML names and ` ` are decoded in one pass after the tags + * are stripped, so `&lt;` reads as `<` rather than as a tag, and any other name is left as + * written. + */ +const CHARACTER_REFERENCE = + /&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|(amp|lt|gt|quot|apos|nbsp));/gi; +const NAMED_CHARACTERS: Record = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", +}; +function decodeCharacterReference( + reference: string, + decimal: string | undefined, + hex: string | undefined, + name: string | undefined, +) { + if (name) return NAMED_CHARACTERS[name.toLowerCase()] ?? reference; + const code = decimal + ? Number.parseInt(decimal, 10) + : Number.parseInt(hex ?? "", 16); + return code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff) + ? String.fromCodePoint(code) + : reference; +} /** Minimal MIME walk: the first text/plain part, else text/html with tags stripped. */ export function extractEmailText(raw: string, depth = 0): string { const split = /\r?\n\r?\n/.exec(raw); @@ -204,6 +234,7 @@ export function extractEmailText(raw: string, depth = 0): string { ? decoded .replace(/|/gi, "") .replace(/<[^>]+>/g, " ") + .replace(CHARACTER_REFERENCE, decodeCharacterReference) .replace(/\s+/g, " ") .trim() : decoded.trim(); diff --git a/server/tests/responsibility-triggers.test.ts b/server/tests/responsibility-triggers.test.ts index 12a6f491c..a36232a52 100644 --- a/server/tests/responsibility-triggers.test.ts +++ b/server/tests/responsibility-triggers.test.ts @@ -696,6 +696,16 @@ describe("inbound email via SES -> SNS", () => { test("MIME text extraction prefers text/plain and decodes quoted-printable", () => { expect(extractEmailText(mime)).toBe("The disk on db-1 is at 99%."); }); + test("MIME text extraction decodes character references in an HTML-only part", () => { + const html = [ + "Content-Type: text/html; charset=utf-8", + "", + "

Q3 numbers for AT&T: margin < 5%, we’re "on track" — &lt; stays

", + ].join("\r\n"); + expect(extractEmailText(html)).toBe( + 'Q3 numbers for AT&T: margin < 5%, we’re "on track" — < stays', + ); + }); test("signing certificate URLs must be HTTPS on an sns..amazonaws.com host", () => { expect(isSnsUrl(CERT_URL, true)).toBe(true); expect(