From 0b8d19b39c3763440fff46a5eb9e57e1f6f50a49 Mon Sep 17 00:00:00 2001 From: Domenic Denicola Date: Sun, 4 Oct 2026 08:19:01 +0000 Subject: [PATCH] Update for Unicode 18.0.0 Update IDNA processing to Unicode 18.0.0 so newly assigned characters, including Small Seal characters, are accepted. Load the Unicode data package's ESM default exports when generating validation regexes. Use the new Unicode test data URL and reject failed HTTP downloads instead of allowing error pages to silently replace the Unicode test cases. Refresh the WPT fixtures to include web-platform-tests/wpt#63170. Exclude ASCII domains from these URL-derived tests because their expectations now reflect the URL Standard's bypass of IDNA processing. Co-Authored-By: codex --- package-lock.json | 12 ++++++------ package.json | 4 ++-- scripts/generateRegexes.js | 24 ++++++++++++------------ scripts/getLatestTests.js | 13 +++++++++++-- test/toascii.js | 10 ++++++++++ 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/package-lock.json b/package-lock.json index bc944c0..295745e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,13 +13,13 @@ }, "devDependencies": { "@domenic/eslint-config": "^4.0.1", - "@unicode/unicode-17.0.0": "^1.6.12", + "@unicode/unicode-18.0.0": "^2.0.7", "eslint": "^9.35.0", "globals": "^16.4.0", "regenerate": "^1.4.2" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/@domenic/eslint-config": { @@ -252,10 +252,10 @@ "dev": true, "license": "MIT" }, - "node_modules/@unicode/unicode-17.0.0": { - "version": "1.6.12", - "resolved": "https://registry.npmjs.org/@unicode/unicode-17.0.0/-/unicode-17.0.0-1.6.12.tgz", - "integrity": "sha512-4IyVjMPIjHThNGZ3PSv7CTtkHrTF8YlkA2HuNAMrqnjIiN2ghCtnwUwEqjfpUT+dfBWjiZJGkiInlqF/3fPiZQ==", + "node_modules/@unicode/unicode-18.0.0": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@unicode/unicode-18.0.0/-/unicode-18.0.0-2.0.7.tgz", + "integrity": "sha512-sxWcUPwfbWVcSt+RPixHI468bs97A2+ftrC9b6wsGEkUTp2eod0qp7J59A+ngd4mFRISElQhPkYClhtXwNuLQw==", "dev": true, "license": "MIT" }, diff --git a/package.json b/package.json index 7a5142b..9841b69 100644 --- a/package.json +++ b/package.json @@ -38,10 +38,10 @@ }, "devDependencies": { "@domenic/eslint-config": "^4.0.1", - "@unicode/unicode-17.0.0": "^1.6.12", + "@unicode/unicode-18.0.0": "^2.0.7", "eslint": "^9.35.0", "globals": "^16.4.0", "regenerate": "^1.4.2" }, - "unicodeVersion": "17.0.0" + "unicodeVersion": "18.0.0" } diff --git a/scripts/generateRegexes.js b/scripts/generateRegexes.js index 0c6c9fd..d524110 100644 --- a/scripts/generateRegexes.js +++ b/scripts/generateRegexes.js @@ -12,22 +12,22 @@ main().catch(e => { async function main() { const cp = { - Mark: require(`@unicode/unicode-${unicodeVersion}/General_Category/Mark/code-points.js`), + Mark: (await import(`@unicode/unicode-${unicodeVersion}/General_Category/Mark/code-points.mjs`)).default, JT: await generateUnicodeCodePoints(`https://unicode.org/Public/${unicodeVersion}/ucd/extracted/DerivedJoiningType.txt`, ["L", "R", "D", "T"]), CombiningClassVirama: (await generateUnicodeCodePoints(`https://unicode.org/Public/${unicodeVersion}/ucd/extracted/DerivedCombiningClass.txt`, ["9"]))["9"], // https://tools.ietf.org/html/rfc5893#section-1.4 - L: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Left_To_Right/code-points.js`), - R: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Right_To_Left/code-points.js`), - AL: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Arabic_Letter/code-points.js`), - EN: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/European_Number/code-points.js`), - ES: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/European_Separator/code-points.js`), - ET: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/European_Terminator/code-points.js`), - AN: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Arabic_Number/code-points.js`), - CS: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Common_Separator/code-points.js`), - NSM: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Nonspacing_Mark/code-points.js`), - BN: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Boundary_Neutral/code-points.js`), - ON: require(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Other_Neutral/code-points.js`) + L: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Left_To_Right/code-points.mjs`)).default, + R: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Right_To_Left/code-points.mjs`)).default, + AL: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Arabic_Letter/code-points.mjs`)).default, + EN: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/European_Number/code-points.mjs`)).default, + ES: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/European_Separator/code-points.mjs`)).default, + ET: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/European_Terminator/code-points.mjs`)).default, + AN: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Arabic_Number/code-points.mjs`)).default, + CS: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Common_Separator/code-points.mjs`)).default, + NSM: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Nonspacing_Mark/code-points.mjs`)).default, + BN: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Boundary_Neutral/code-points.mjs`)).default, + ON: (await import(`@unicode/unicode-${unicodeVersion}/Bidi_Class/Other_Neutral/code-points.mjs`)).default }; function r(strings, ...regs) { diff --git a/scripts/getLatestTests.js b/scripts/getLatestTests.js index ee388c7..2d7308e 100644 --- a/scripts/getLatestTests.js +++ b/scripts/getLatestTests.js @@ -10,7 +10,7 @@ const { unicodeVersion } = require("../package.json"); // Update this by going to https://github.com/web-platform-tests/wpt/tree/master/url/resources and pressing "y" on the // keyboard. -const wptSHA = "40fc257a28faf7c378f59185235685ea8684e8f4"; +const wptSHA = "13748c36fdf9dd77cd0b34dd6b244256b365ea60"; main().catch(e => { console.error(e); @@ -21,17 +21,26 @@ async function main() { await Promise.all([ (async () => { const target = path.resolve(__dirname, "../test/fixtures/IdnaTestV2.txt"); - const response = await fetch(`https://unicode.org/Public/idna/${unicodeVersion}/IdnaTestV2.txt`); + const response = await fetch(`https://unicode.org/Public/${unicodeVersion}/idna/IdnaTestV2.txt`); + if (!response.ok) { + throw new Error(`Failed to fetch ${response.url}: ${response.status}`); + } await fs.writeFile(target, response.body); })(), (async () => { const target = path.resolve(__dirname, "../test/fixtures/toascii.json"); const response = await fetch(`https://github.com/web-platform-tests/wpt/raw/${wptSHA}/url/resources/toascii.json`); + if (!response.ok) { + throw new Error(`Failed to fetch ${response.url}: ${response.status}`); + } await fs.writeFile(target, response.body); })(), (async () => { const target = path.resolve(__dirname, "../test/fixtures/IdnaTestV2ToASCII.json"); const response = await fetch(`https://github.com/web-platform-tests/wpt/raw/${wptSHA}/url/resources/IdnaTestV2.json`); + if (!response.ok) { + throw new Error(`Failed to fetch ${response.url}: ${response.status}`); + } await fs.writeFile(target, response.body); })() ]); diff --git a/test/toascii.js b/test/toascii.js index df013ec..9f8cca8 100644 --- a/test/toascii.js +++ b/test/toascii.js @@ -32,6 +32,11 @@ describe("ToASCII", () => { continue; } + // The URL Standard bypasses IDNA processing for ASCII domains. + if (/^[\x00-\x7F]*$/u.test(testCase.input)) { + continue; + } + let description = testCase.input; if (testCase.comment) { description += ` (${testCase.comment})`; @@ -54,6 +59,11 @@ describe("ToASCII via IdnaTestV2.json in wpt", () => { continue; } + // The URL Standard bypasses IDNA processing for ASCII domains. + if (/^[\x00-\x7F]*$/u.test(testCase.input)) { + continue; + } + let description = testCase.input; if (testCase.comment) { description += ` (${testCase.comment})`;