Skip to content

Commit bbec24b

Browse files
matmenclaude
andcommitted
use TextEncoder/TextDecoder for UTF-8
Every supported runtime has them. The hand-written decoder accepted overlong sequences and encoded surrogates, which TextDecoder replaces with U+FFFD as the encoding standard specifies, and it's faster on large SVGs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ljnxm1Qjrocq1izGajLx8J
1 parent 165d534 commit bbec24b

1 file changed

Lines changed: 8 additions & 51 deletions

File tree

‎src/util/utf8.ts‎

Lines changed: 8 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,17 @@
1-
// UTF-8 helpers, so the library doesn't depend on TextEncoder/TextDecoder being available
1+
// TextEncoder and TextDecoder are web APIs, not part of the es2022 lib: declare the parts used here
2+
declare const TextEncoder: { new(): { encode(text: string): Uint8Array } };
3+
declare const TextDecoder: { new(): { decode(bytes: Uint8Array): string } };
24

3-
export function encode(text: string): Uint8Array {
4-
const out: number[] = [];
5-
6-
for (const char of text) {
7-
const c = char.codePointAt(0) as number;
8-
if (c < 0x80) out.push(c);
9-
else if (c < 0x800) out.push(0xc0 | c >> 6, 0x80 | c & 63);
10-
else if (c < 0x10000) out.push(0xe0 | c >> 12, 0x80 | c >> 6 & 63, 0x80 | c & 63);
11-
else out.push(0xf0 | c >> 18, 0x80 | c >> 12 & 63, 0x80 | c >> 6 & 63, 0x80 | c & 63);
12-
}
5+
const encoder = new TextEncoder();
6+
const decoder = new TextDecoder();
137

14-
return Uint8Array.from(out);
8+
export function encode(text: string): Uint8Array {
9+
return encoder.encode(text);
1510
}
1611

1712
/**
1813
* Decodes UTF-8, replacing invalid sequences with U+FFFD and skipping a leading byte order mark
1914
*/
2015
export function decode(bytes: Uint8Array): string {
21-
let out = '';
22-
let i = bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf ? 3 : 0;
23-
24-
while (i < bytes.length) {
25-
const b = bytes[i++];
26-
let c: number, extra: number;
27-
28-
if (b < 0x80) {
29-
c = b;
30-
extra = 0;
31-
} else if (b >= 0xc2 && b < 0xe0) {
32-
c = b & 31;
33-
extra = 1;
34-
} else if (b >= 0xe0 && b < 0xf0) {
35-
c = b & 15;
36-
extra = 2;
37-
} else if (b >= 0xf0 && b < 0xf5) {
38-
c = b & 7;
39-
extra = 3;
40-
} else {
41-
out += '�';
42-
continue;
43-
}
44-
45-
let valid = true;
46-
for (let k = 0; k < extra; k++) {
47-
const next = bytes[i];
48-
if (next === undefined || (next & 0xc0) !== 0x80) {
49-
valid = false;
50-
break;
51-
}
52-
c = c << 6 | next & 63;
53-
i++;
54-
}
55-
56-
out += valid && c <= 0x10ffff ? String.fromCodePoint(c) : '�';
57-
}
58-
59-
return out;
16+
return decoder.decode(bytes);
6017
}

0 commit comments

Comments
 (0)