Decode an inbound email part in the charset it names, not always as UTF-8

CHANGELOG.md

@@ -38,6 +38,12 @@ An email trigger reads an HTML-only mail by stripping its tags, and the characte
 the Bot read `AT&T`, `we’re` and `Q3 numbers`. Numeric references, `&`, `<`,
 `>`, `"`, `'` and ` ` are now decoded, once, so `<` reads as `<`.
 
+### An email in another charset reaches its trigger as the sender wrote it
+
+A base64 or quoted-printable email part was always read as UTF-8, whatever charset its Content-Type
+named, so an `iso-8859-1` mail reached the Bot with every accented letter turned into `�`. Each part
+is now decoded in the charset it names, and a charset the runtime does not know still reads as UTF-8.
+
 ### The Bot's screen opens only when there is something to watch
 
 Asking a Bot a question it answered in words could still open its computer beside the conversation,

server/src/responsibilities/email.ts

@@ -157,19 +157,46 @@ function unfoldHeaders(block: string) {
   }
   return headers;
 }
-function decodeBody(body: string, encoding: string | undefined) {
+/**
+ * The text a part's transfer-encoded bytes spell, in the charset its Content-Type names.
+ *
+ * Base64 and quoted-printable carry bytes, not text: a `charset=iso-8859-1` part saying "caf=E9" is
+ * "café", and reading those bytes as UTF-8 turned every accented letter into U+FFFD. A charset the
+ * runtime does not know falls back to UTF-8, as before.
+ */
+function decodeBytes(bytes: Buffer, charset: string | undefined) {
+  if (charset) {
+    try {
+      return new TextDecoder(charset).decode(bytes);
+    } catch {
+      // unknown label: read as UTF-8
+    }
+  }
+  return bytes.toString("utf8");
+}
+function decodeBody(
+  body: string,
+  encoding: string | undefined,
+  charset: string | undefined,
+) {
   const kind = encoding?.toLowerCase();
   if (kind === "base64")
-    return Buffer.from(body.replace(/\s+/g, ""), "base64").toString("utf8");
+    return decodeBytes(
+      Buffer.from(body.replace(/\s+/g, ""), "base64"),
+      charset,
+    );
   if (kind === "quoted-printable")
-    return Buffer.from(
-      body
-        .replace(/=\r?\n/g, "")
-        .replace(/=([0-9A-F]{2})/gi, (_, hex: string) =>
-          String.fromCharCode(Number.parseInt(hex, 16)),
-        ),
-      "latin1",
-    ).toString("utf8");
+    return decodeBytes(
+      Buffer.from(
+        body
+          .replace(/=\r?\n/g, "")
+          .replace(/=([0-9A-F]{2})/gi, (_, hex: string) =>
+            String.fromCharCode(Number.parseInt(hex, 16)),
+          ),
+        "latin1",
+      ),
+      charset,
+    );
   return body;
 }
 /**
@@ -229,7 +256,11 @@ export function extractEmailText(raw: string, depth = 0): string {
       plain ?? texts.find((part) => /^text\/html/i.test(part.type));
     return chosen ? extractEmailText(chosen.part, depth + 1) : "";
   }
-  const decoded = decodeBody(body, headers.get("content-transfer-encoding"));
+  const decoded = decodeBody(
+    body,
+    headers.get("content-transfer-encoding"),
+    /charset="?([^";\s]+)"?/i.exec(type)?.[1],
+  );
   return /^text\/html/i.test(type)
     ? decoded
         .replace(/<style[\s\S]*?<\/style>|<script[\s\S]*?<\/script>/gi, "")

server/tests/responsibility-triggers.test.ts

@@ -706,6 +706,29 @@ describe("inbound email via SES -> SNS", () => {
       'Q3 numbers for AT&T: margin < 5%, we’re "on track" — &lt; stays',
     );
   });
+  test("MIME text extraction decodes in the charset the part names", () => {
+    const latin1 = [
+      "Content-Type: text/plain; charset=iso-8859-1",
+      "Content-Transfer-Encoding: quoted-printable",
+      "",
+      "Caf=E9 ferm=E9 =E0 minuit.",
+    ].join("\r\n");
+    expect(extractEmailText(latin1)).toBe("Café fermé à minuit.");
+    const base64 = [
+      'Content-Type: text/plain; charset="windows-1252"',
+      "Content-Transfer-Encoding: base64",
+      "",
+      Buffer.from("Na\xefve r\xe9sum\xe9", "latin1").toString("base64"),
+    ].join("\r\n");
+    expect(extractEmailText(base64)).toBe("Naïve résumé");
+    const utf8 = [
+      "Content-Type: text/plain; charset=utf-8",
+      "Content-Transfer-Encoding: quoted-printable",
+      "",
+      "Caf=C3=A9",
+    ].join("\r\n");
+    expect(extractEmailText(utf8)).toBe("Café");
+  });
   test("signing certificate URLs must be HTTPS on an sns.<region>.amazonaws.com host", () => {
     expect(isSnsUrl(CERT_URL, true)).toBe(true);
     expect(