From 898ced313779e4350209c0e83e55efc91862956a Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Wed, 8 Jul 2026 23:39:02 +0200 Subject: [PATCH 1/5] HTML API: Add ASCII case helpers, fix locale-dependent attribute prefix matching. WP_HTML_Decoder::attribute_starts_with() promises ASCII-only case folding in its 'ascii-case-insensitive' mode but relied on two locale-sensitive comparisons: per-byte strtolower() (locale-dependent before PHP 8.2) for plain characters, and substr_compare() with its case-insensitivity flag (locale-dependent on every PHP version) for decoded character references. Under locales whose C library folds non-ASCII bytes (e.g. C.UTF-8 on macOS, de_DE.ISO8859-1), non-ASCII bytes could spuriously match as case variants of each other, and the same decoded value could produce different answers depending on whether it appeared raw or encoded in the attribute value. Introduce locale-independent ASCII case helpers on WP_HTML_Decoder and use them for both comparison paths so they can no longer diverge. Found via differential fuzzing against a decode_attribute() + str_starts_with() oracle. See #65372. Claude-Session: https://claude.ai/code/session_01K3vwt69YyVNbHAwu1Ue1Nf --- .../html-api/class-wp-html-decoder.php | 117 +++++++++++++++++- .../phpunit/tests/html-api/wpHtmlDecoder.php | 67 ++++++++++ 2 files changed, 182 insertions(+), 2 deletions(-) diff --git a/src/wp-includes/html-api/class-wp-html-decoder.php b/src/wp-includes/html-api/class-wp-html-decoder.php index 6c375b8512946..f3b8b0a38354a 100644 --- a/src/wp-includes/html-api/class-wp-html-decoder.php +++ b/src/wp-includes/html-api/class-wp-html-decoder.php @@ -10,6 +10,116 @@ * @since 6.6.0 */ class WP_HTML_Decoder { + /** + * Indicates if a span of text matches a given string, ignoring ASCII case. + * + * Matching is performed byte-by-byte and folds only the ASCII uppercase letters + * A-Z onto their lowercase counterparts, regardless of the process locale. + * + * This exists because PHP's built-in case-insensitive comparisons are locale + * sensitive: `substr_compare()` with its case-insensitivity flag and (before + * PHP 8.2) `strcasecmp()`, `strtolower()`, and friends fold bytes according + * to the current process locale, which may treat non-ASCII bytes as case + * variants of each other or of ASCII letters. HTML requires ASCII + * case insensitivity regardless of locale. + * + * The span begins at the given byte offset into the text and runs for the + * byte length of the search string. A span extending past the end of the + * text never matches, even if the available bytes match the search string. + * + * Example: + * + * true === WP_HTML_Decoder::matches_ascii_case_insensitively( 'DIV', 'div' ); + * true === WP_HTML_Decoder::matches_ascii_case_insensitively( '', 'HTML', 10 ); + * false === WP_HTML_Decoder::matches_ascii_case_insensitively( "\xCC", "\xEC" ); + * false === WP_HTML_Decoder::matches_ascii_case_insensitively( 'DIVERT', 'div', 3 ); + * + * @since 7.1.0 + * + * @access private + * + * @see https://infra.spec.whatwg.org/#ascii-case-insensitive + * + * @param string $text Text possibly containing the search string. + * @param string $search Search string; its byte length determines the span length. + * @param int $offset Optional. Byte offset into the text where the span begins. Default 0. + * @return bool Whether the span is an ASCII case-insensitive match for the search string. + */ + public static function matches_ascii_case_insensitively( string $text, string $search, int $offset = 0 ): bool { + $length = strlen( $search ); + if ( $offset < 0 || $offset + $length > strlen( $text ) ) { + return false; + } + + for ( $at = 0; $at < $length; $at++ ) { + $text_byte = $text[ $offset + $at ]; + $search_byte = $search[ $at ]; + + if ( $text_byte === $search_byte ) { + continue; + } + + $text_ord = ord( $text_byte ); + $search_ord = ord( $search_byte ); + + // Fold uppercase ASCII letters onto their lowercase counterparts; no other bytes fold. + if ( $text_ord >= 0x41 && $text_ord <= 0x5A ) { + $text_ord += 0x20; + } + if ( $search_ord >= 0x41 && $search_ord <= 0x5A ) { + $search_ord += 0x20; + } + + if ( $text_ord !== $search_ord ) { + return false; + } + } + + return true; + } + + /** + * Returns the ASCII lowercase version of a given string. + * + * Only the ASCII uppercase letters A-Z are folded onto their lowercase + * counterparts; all other bytes are left unchanged. This differs from + * `strtolower()`, which before PHP 8.2 folds bytes according to the + * current process locale. + * + * @since 7.1.0 + * + * @access private + * + * @see https://infra.spec.whatwg.org/#ascii-lowercase + * + * @param string $text Text to fold. + * @return string ASCII-lowercase version of the given text. + */ + public static function ascii_lowercase( string $text ): string { + return strtr( $text, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz' ); + } + + /** + * Returns the ASCII uppercase version of a given string. + * + * Only the ASCII lowercase letters a-z are folded onto their uppercase + * counterparts; all other bytes are left unchanged. This differs from + * `strtoupper()`, which before PHP 8.2 folds bytes according to the + * current process locale. + * + * @since 7.1.0 + * + * @access private + * + * @see https://infra.spec.whatwg.org/#ascii-uppercase + * + * @param string $text Text to fold. + * @return string ASCII-uppercase version of the given text. + */ + public static function ascii_uppercase( string $text ): string { + return strtr( $text, 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' ); + } + /** * Indicates if an attribute value starts with a given raw string value. * @@ -40,7 +150,7 @@ public static function attribute_starts_with( $haystack, $search_text, $case_sen while ( $search_at < $search_length && $haystack_at < $haystack_end ) { $chars_match = $loose_case - ? strtolower( $haystack[ $haystack_at ] ) === strtolower( $search_text[ $search_at ] ) + ? self::matches_ascii_case_insensitively( $haystack, $search_text[ $search_at ], $haystack_at ) : $haystack[ $haystack_at ] === $search_text[ $search_at ]; $is_introducer = '&' === $haystack[ $haystack_at ]; @@ -61,7 +171,10 @@ public static function attribute_starts_with( $haystack, $search_text, $case_sen } // If there is a character reference, then the decoded value must exactly match what follows in the search string. - if ( 0 !== substr_compare( $search_text, $next_chunk, $search_at, strlen( $next_chunk ), $loose_case ) ) { + $chunk_matches = $loose_case + ? self::matches_ascii_case_insensitively( $search_text, $next_chunk, $search_at ) + : 0 === substr_compare( $search_text, $next_chunk, $search_at, strlen( $next_chunk ) ); + if ( ! $chunk_matches ) { return false; } diff --git a/tests/phpunit/tests/html-api/wpHtmlDecoder.php b/tests/phpunit/tests/html-api/wpHtmlDecoder.php index 7fe39a63d1f3b..6f88292a5a3e0 100644 --- a/tests/phpunit/tests/html-api/wpHtmlDecoder.php +++ b/tests/phpunit/tests/html-api/wpHtmlDecoder.php @@ -389,6 +389,73 @@ public static function data_attributes_with_prefix_and_case_sensitive_match() { array( 'http://wordpress.org', 'Http', 'ascii-case-insensitive', true ), array( 'http://wordpress.org', 'https', 'case-sensitive', false ), array( 'http://wordpress.org', 'https', 'ascii-case-insensitive', false ), + + /* + * ASCII case insensitivity must not extend to non-ASCII bytes, no matter + * the process locale. These byte pairs are case variants of each other in + * common single-byte charmaps (0xCC/0xEC are Ì/ì in Latin-1) and serve as + * canaries: they will match if a locale-sensitive comparison sneaks in. + * The same decoded value must produce the same answer whether it appears + * raw or as a character reference ("\xCC\xB8" is U+0338 in UTF-8). + */ + 'Raw non-ASCII byte does not case-fold' => array( "\xCC\xB8", "\xEC\xB8", 'ascii-case-insensitive', false ), + 'Encoded non-ASCII byte does not case-fold' => array( '̸', "\xEC\xB8", 'ascii-case-insensitive', false ), + 'Encoded non-ASCII matches its exact bytes' => array( '̸', "\xCC\xB8", 'ascii-case-insensitive', true ), + 'Encoded non-ASCII exact bytes are sensible' => array( '̸', "\xCC\xB8", 'case-sensitive', true ), + 'Raw Ä does not loosely match raw ä' => array( "\xC4", "\xE4", 'ascii-case-insensitive', false ), + 'Encoded Ä matches its exact UTF-8 bytes' => array( 'Ä', "\xC3\x84", 'ascii-case-insensitive', true ), + 'Encoded Ä does not loosely match ä bytes' => array( 'Ä', "\xC3\xA4", 'ascii-case-insensitive', false ), + 'ASCII case folds around encoded spans' => array( 'JAVASCRIPT:', 'javascript:', 'ascii-case-insensitive', true ), + 'Search text ending inside an encoded span' => array( '…', "\xE2\x80", 'ascii-case-insensitive', false ), ); } + + /** + * Ensures ASCII case-insensitive matching ignores the process locale. + * + * Locale-sensitive case comparisons can treat non-ASCII bytes as case + * variants of each other, e.g. 0xCC/0xEC (Ì/ì in Latin-1). This test + * switches to a locale where the C library folds those bytes and + * asserts that matching remains byte-exact outside of ASCII. + * + * @ticket 65372 + */ + public function test_attribute_starts_with_ignores_process_locale() { + $folding_locale = null; + foreach ( array( 'de_DE.ISO8859-1', 'de_DE.iso88591', 'de_DE', 'C.UTF-8', 'C.utf8', 'en_US.UTF-8' ) as $locale ) { + // Detect a locale whose C-library case folding maps Ä (0xC4) onto ä (0xE4). + if ( false !== setlocale( LC_CTYPE, $locale ) && 0 === substr_compare( "\xC4", "\xE4", 0, 1, true ) ) { + $folding_locale = $locale; + break; + } + } + + if ( self::$original_lc_ctype ) { + setlocale( LC_CTYPE, self::$original_lc_ctype ); + } + + if ( null === $folding_locale ) { + $this->markTestSkipped( 'No locale with non-ASCII case folding is available.' ); + } + + setlocale( LC_CTYPE, $folding_locale ); + try { + $this->assertFalse( + WP_HTML_Decoder::attribute_starts_with( "\xC4hnlich", "\xE4hnlich", 'ascii-case-insensitive' ), + 'Should not have case-folded a raw non-ASCII byte.' + ); + $this->assertFalse( + WP_HTML_Decoder::attribute_starts_with( '̸', "\xEC\xB8", 'ascii-case-insensitive' ), + 'Should not have case-folded a decoded non-ASCII byte.' + ); + $this->assertTrue( + WP_HTML_Decoder::attribute_starts_with( 'JAVASCRIPT:alert(1)', 'javascript:', 'ascii-case-insensitive' ), + 'Should have case-folded ASCII letters.' + ); + } finally { + if ( self::$original_lc_ctype ) { + setlocale( LC_CTYPE, self::$original_lc_ctype ); + } + } + } } From 562cda6bfb6f9ef869cf244e3167b96d84056ae3 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Wed, 8 Jul 2026 23:45:09 +0200 Subject: [PATCH 2/5] HTML API: Use locale-independent ASCII case comparisons in the Tag Processor. Tag, attribute, and class name matching in WP_HTML_Tag_Processor promises ASCII case insensitivity but relied on locale-sensitive comparisons: substr_compare() with its case-insensitivity flag (locale-dependent on every PHP version) when matching sought tag names and quirks-mode class names; and strtolower()/strtoupper()/stripos() (locale-dependent before PHP 8.2) when normalizing names for comparison. Under locales whose C library case-folds non-ASCII bytes, tag and class names containing non-ASCII bytes could spuriously match each other. Replace these with the locale-independent ASCII case helpers. See #65372. Claude-Session: https://claude.ai/code/session_01K3vwt69YyVNbHAwu1Ue1Nf --- .../html-api/class-wp-html-tag-processor.php | 50 +++++++-------- .../tests/html-api/wpHtmlProcessor.php | 64 +++++++++++++++++++ .../tests/html-api/wpHtmlTagProcessor.php | 26 ++++++++ 3 files changed, 115 insertions(+), 25 deletions(-) diff --git a/src/wp-includes/html-api/class-wp-html-tag-processor.php b/src/wp-includes/html-api/class-wp-html-tag-processor.php index a3dac33fc55ca..f51e9d2d70a3a 100644 --- a/src/wp-includes/html-api/class-wp-html-tag-processor.php +++ b/src/wp-includes/html-api/class-wp-html-tag-processor.php @@ -1219,7 +1219,7 @@ public function class_list() { $name = substr( $class, $at, $length ); if ( $is_quirks ) { - $name = strtolower( $name ); + $name = WP_HTML_Decoder::ascii_lowercase( $name ); } $at += $length; @@ -1257,7 +1257,9 @@ public function has_class( $wanted_class ): ?bool { foreach ( $this->class_list() as $class_name ) { if ( strlen( $class_name ) === $wanted_length && - 0 === substr_compare( $class_name, $wanted_class, 0, strlen( $wanted_class ), $case_insensitive ) + ( $case_insensitive + ? WP_HTML_Decoder::matches_ascii_case_insensitively( $class_name, $wanted_class ) + : $class_name === $wanted_class ) ) { return true; } @@ -1448,10 +1450,7 @@ private function skip_rcdata( string $tag_name ): bool { * normalization could not be part of a tag name. */ for ( $i = 0; $i < $tag_length; $i++ ) { - $tag_char = $tag_name[ $i ]; - $html_char = $html[ $at + $i ]; - - if ( $html_char !== $tag_char && strtoupper( $html_char ) !== $tag_char ) { + if ( ! WP_HTML_Decoder::matches_ascii_case_insensitively( $html, $tag_name[ $i ], $at + $i ) ) { $at += $i; continue 2; } @@ -2267,7 +2266,7 @@ private function parse_next_attribute(): bool { * * @see https://html.spec.whatwg.org/#attribute-name-state */ - $comparable_name = strtolower( str_replace( "\x00", "\u{FFFD}", $attribute_name ) ); + $comparable_name = WP_HTML_Decoder::ascii_lowercase( str_replace( "\x00", "\u{FFFD}", $attribute_name ) ); // If an attribute is listed many times, only use the first declaration and ignore the rest. if ( ! isset( $this->attributes[ $comparable_name ] ) ) { @@ -2446,7 +2445,7 @@ private function class_name_updates_to_attributes_updates(): void { if ( $is_quirks ) { foreach ( $this->classname_updates as $updated_name => $action ) { if ( self::REMOVE_CLASS === $action ) { - $to_remove[] = strtolower( $updated_name ); + $to_remove[] = WP_HTML_Decoder::ascii_lowercase( $updated_name ); } } } else { @@ -2473,7 +2472,7 @@ private function class_name_updates_to_attributes_updates(): void { } $name = substr( $existing_class, $at, $name_length ); - $comparable_class_name = $is_quirks ? strtolower( $name ) : $name; + $comparable_class_name = $is_quirks ? WP_HTML_Decoder::ascii_lowercase( $name ) : $name; $at += $name_length; // If this class is marked for removal, remove it and move on to the next one. @@ -2509,7 +2508,7 @@ private function class_name_updates_to_attributes_updates(): void { // Add new classes by appending those which haven't already been seen. foreach ( $this->classname_updates as $name => $operation ) { - $comparable_name = $is_quirks ? strtolower( $name ) : $name; + $comparable_name = $is_quirks ? WP_HTML_Decoder::ascii_lowercase( $name ) : $name; if ( self::ADD_CLASS === $operation && ! in_array( $comparable_name, $seen, true ) ) { $modified = true; @@ -2809,7 +2808,7 @@ public function get_attribute( $name ) { return null; } - $comparable = strtolower( $name ); + $comparable = WP_HTML_Decoder::ascii_lowercase( $name ); /* * For every attribute other than `class` it's possible to perform a quick check if @@ -2912,7 +2911,7 @@ public function get_attribute_names_with_prefix( $prefix ): ?array { return null; } - $comparable = strtolower( $prefix ); + $comparable = WP_HTML_Decoder::ascii_lowercase( $prefix ); $matches = array(); foreach ( array_keys( $this->attributes ) as $attr_name ) { @@ -2958,7 +2957,7 @@ public function get_tag(): ?string { $tag_name = str_replace( "\x00", "\u{FFFD}", substr( $this->html, $this->tag_name_starts_at, $this->tag_name_length ) ); if ( self::STATE_MATCHED_TAG === $this->parser_state ) { - return strtoupper( $tag_name ); + return WP_HTML_Decoder::ascii_uppercase( $tag_name ); } if ( @@ -2989,7 +2988,7 @@ public function get_qualified_tag_name(): ?string { return $tag_name; } - $lower_tag_name = strtolower( $tag_name ); + $lower_tag_name = WP_HTML_Decoder::ascii_lowercase( $tag_name ); if ( 'math' === $this->get_namespace() ) { return $lower_tag_name; } @@ -3138,7 +3137,7 @@ public function get_qualified_attribute_name( $attribute_name ): ?string { } $namespace = $this->get_namespace(); - $lower_name = strtolower( $attribute_name ); + $lower_name = WP_HTML_Decoder::ascii_lowercase( $attribute_name ); if ( 'math' === $namespace && 'definitionurl' === $lower_name ) { return 'definitionURL'; @@ -3911,9 +3910,10 @@ public function set_modifiable_text( string $plaintext_content ): bool { * HTML structure is rejected here. It’s the responsibility of calling code to * perform whatever semantic escaping is necessary to avoid problematic strings. */ + $comparable_content = WP_HTML_Decoder::ascii_lowercase( $plaintext_content ); if ( - false !== stripos( $plaintext_content, ' If the script block's type string is a JavaScript MIME type essence match, then @@ -4300,7 +4300,7 @@ private static function escape_javascript_script_contents( string $sourcecode ): $has_closing_slash = $tag_name_at < $end && '/' === $sourcecode[ $tag_name_at ]; $tag_name_at += $has_closing_slash ? 1 : 0; - if ( 0 !== substr_compare( $sourcecode, 'script', $tag_name_at, 6, true ) ) { + if ( ! WP_HTML_Decoder::matches_ascii_case_insensitively( $sourcecode, 'script', $tag_name_at ) ) { $at = $tag_at + 1; continue; } @@ -4415,7 +4415,7 @@ public function set_attribute( $name, $value ): bool { if ( true === $value ) { $updated_attribute = $name; } else { - $comparable_name = strtolower( $name ); + $comparable_name = WP_HTML_Decoder::ascii_lowercase( $name ); /** * Escape attribute values appropriately. @@ -4451,7 +4451,7 @@ public function set_attribute( $name, $value ): bool { * * @see https://html.spec.whatwg.org/multipage/syntax.html#attributes-2:ascii-case-insensitive */ - $comparable_name = strtolower( $name ); + $comparable_name = WP_HTML_Decoder::ascii_lowercase( $name ); if ( isset( $this->attributes[ $comparable_name ] ) ) { /* @@ -4527,7 +4527,7 @@ public function remove_attribute( $name ): bool { * * @see https://html.spec.whatwg.org/multipage/syntax.html#attributes-2:ascii-case-insensitive */ - $name = strtolower( $name ); + $name = WP_HTML_Decoder::ascii_lowercase( $name ); /* * Any calls to update the `class` attribute directly should wipe out any @@ -4612,7 +4612,7 @@ public function add_class( $class_name ): bool { foreach ( $this->classname_updates as $updated_name => $action ) { if ( strlen( $updated_name ) === $class_name_length && - 0 === substr_compare( $updated_name, $class_name, 0, $class_name_length, true ) + WP_HTML_Decoder::matches_ascii_case_insensitively( $updated_name, $class_name ) ) { $this->classname_updates[ $updated_name ] = self::ADD_CLASS; return true; @@ -4654,7 +4654,7 @@ public function remove_class( $class_name ): bool { foreach ( $this->classname_updates as $updated_name => $action ) { if ( strlen( $updated_name ) === $class_name_length && - 0 === substr_compare( $updated_name, $class_name, 0, $class_name_length, true ) + WP_HTML_Decoder::matches_ascii_case_insensitively( $updated_name, $class_name ) ) { $this->classname_updates[ $updated_name ] = self::REMOVE_CLASS; return true; @@ -4821,7 +4821,7 @@ private function matches(): bool { $tag_name = $this->get_tag(); if ( strlen( $this->sought_tag_name ) !== strlen( $tag_name ) || - 0 !== substr_compare( $tag_name, $this->sought_tag_name, 0, null, true ) + ! WP_HTML_Decoder::matches_ascii_case_insensitively( $tag_name, $this->sought_tag_name ) ) { return false; } diff --git a/tests/phpunit/tests/html-api/wpHtmlProcessor.php b/tests/phpunit/tests/html-api/wpHtmlProcessor.php index a978422c20098..dade2f3d27ca5 100644 --- a/tests/phpunit/tests/html-api/wpHtmlProcessor.php +++ b/tests/phpunit/tests/html-api/wpHtmlProcessor.php @@ -836,6 +836,70 @@ public function test_class_list_quirks_mode() { ); } + /** + * Ensures quirks-mode class matching folds ASCII letters only, regardless of locale. + * + * The byte pair 0xCC/0xEC (Ì/ì in ISO-8859-1) is a case pair in common single-byte + * charmaps: a locale-sensitive comparison would treat the class names below as + * ASCII case-insensitive matches for each other. + * + * @ticket 65372 + * + * @covers ::has_class + */ + public function test_has_class_quirks_mode_folds_ascii_case_only() { + $processor = WP_HTML_Processor::create_full_parser( "" ); + $processor->next_tag( 'SPAN' ); + $this->assertTrue( + $processor->has_class( "gr\xCCn" ), + 'Should have matched the class name with ASCII letters case-folded.' + ); + $this->assertFalse( + $processor->has_class( "gr\xECn" ), + 'Should not have case-folded non-ASCII bytes in class names.' + ); + } + + /** + * Ensures quirks-mode class updates fold ASCII letters only, regardless of locale. + * + * @ticket 65372 + * + * @covers ::add_class + * @covers ::remove_class + */ + public function test_add_class_quirks_mode_folds_ascii_case_only() { + $processor = WP_HTML_Processor::create_full_parser( '' ); + $processor->next_tag( 'SPAN' ); + $processor->add_class( "GR\xCCN" ); + $processor->add_class( "gr\xCCn" ); + $this->assertSame( + "", + $processor->get_updated_html(), + 'Should have deduplicated ASCII case variants of the same class name.' + ); + + $processor = WP_HTML_Processor::create_full_parser( '' ); + $processor->next_tag( 'SPAN' ); + $processor->add_class( "GR\xCCN" ); + $processor->add_class( "gr\xECn" ); + $this->assertSame( + "", + $processor->get_updated_html(), + 'Should have added both class names: non-ASCII bytes are not case variants.' + ); + + $processor = WP_HTML_Processor::create_full_parser( '' ); + $processor->next_tag( 'SPAN' ); + $processor->add_class( "GR\xCCN" ); + $processor->remove_class( "gr\xECn" ); + $this->assertSame( + "", + $processor->get_updated_html(), + 'Should not have cancelled a pending class addition differing in non-ASCII bytes.' + ); + } + /** * Ensures that the processor correctly adjusts the namespace * for elements inside HTML integration points. diff --git a/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php b/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php index 88762ddbb60c4..cee983c1f09c7 100644 --- a/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php +++ b/tests/phpunit/tests/html-api/wpHtmlTagProcessor.php @@ -3247,6 +3247,32 @@ public function test_recognizes_uppercase_tag_name( string $char ) { ); } + /** + * Ensures tag-name matching folds ASCII letters only, regardless of locale. + * + * Tag names may contain non-ASCII bytes, which must match byte-for-byte. + * The byte pair 0xCC/0xEC (Ì/ì in ISO-8859-1) is a case pair in common + * single-byte charmaps: a locale-sensitive comparison would treat the tag + * names below as ASCII case-insensitive matches for each other. + * + * @ticket 65372 + * + * @covers ::next_tag + */ + public function test_next_tag_matches_tag_names_with_ascii_case_folding_only() { + $processor = new WP_HTML_Tag_Processor( "" ); + $this->assertFalse( + $processor->next_tag( array( 'tag_name' => "d\xECta" ) ), + 'Should not have matched a tag name differing in non-ASCII bytes.' + ); + + $processor = new WP_HTML_Tag_Processor( "" ); + $this->assertTrue( + $processor->next_tag( array( 'tag_name' => "D\xCCTA" ) ), + 'Should have matched the tag name with ASCII letters case-folded.' + ); + } + /** * Data provider. * From bab55b0c76907b840018a5bbd21aacfa374f42c9 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Wed, 8 Jul 2026 23:47:15 +0200 Subject: [PATCH 3/5] HTML API: Use locale-independent ASCII case comparisons in Doctype Info. DOCTYPE parsing matches the ' The system identifier and public identifier strings must be compared... * > in an ASCII case-insensitive manner. */ - $public_identifier = null === $public_identifier ? '' : strtolower( $public_identifier ); - $system_identifier = null === $system_identifier ? '' : strtolower( $system_identifier ); + $public_identifier = null === $public_identifier ? '' : WP_HTML_Decoder::ascii_lowercase( $public_identifier ); + $system_identifier = null === $system_identifier ? '' : WP_HTML_Decoder::ascii_lowercase( $system_identifier ); /* * > The public identifier is set to… @@ -436,7 +436,7 @@ public static function from_doctype_token( string $doctype_html ): ?self { */ if ( $end < 9 || - 0 !== substr_compare( $doctype_html, ' case-insensitive match for the word "PUBLIC", then consume those characters * > and switch to the after DOCTYPE public keyword state. */ - if ( 0 === substr_compare( $doctype_html, 'PUBLIC', $at, 6, true ) ) { + if ( WP_HTML_Decoder::matches_ascii_case_insensitively( $doctype_html, 'PUBLIC', $at ) ) { $at += 6; $at += strspn( $doctype_html, " \t\n\f\r", $at, $end - $at ); if ( $at >= $end ) { @@ -528,7 +528,7 @@ public static function from_doctype_token( string $doctype_html ): ?self { * > case-insensitive match for the word "SYSTEM", then consume those characters and switch * > to the after DOCTYPE system keyword state. */ - if ( 0 === substr_compare( $doctype_html, 'SYSTEM', $at, 6, true ) ) { + if ( WP_HTML_Decoder::matches_ascii_case_insensitively( $doctype_html, 'SYSTEM', $at ) ) { $at += 6; $at += strspn( $doctype_html, " \t\n\f\r", $at, $end - $at ); if ( $at >= $end ) { diff --git a/tests/phpunit/tests/html-api/wpHtmlDoctypeInfo.php b/tests/phpunit/tests/html-api/wpHtmlDoctypeInfo.php index 2abc1cdc1ca5a..338799a4d018e 100644 --- a/tests/phpunit/tests/html-api/wpHtmlDoctypeInfo.php +++ b/tests/phpunit/tests/html-api/wpHtmlDoctypeInfo.php @@ -64,31 +64,44 @@ public function test_doctype_doc_info( */ public static function data_parseable_raw_doctypes(): array { return array( - 'Missing doctype name' => array( '', 'quirks' ), - 'HTML5 doctype' => array( '', 'no-quirks', 'html' ), - 'HTML5 doctype no whitespace before name' => array( '', 'no-quirks', 'html' ), - 'XHTML doctype' => array( '', 'no-quirks', 'html', '-//W3C//DTD HTML 4.01//EN', 'http://www.w3.org/TR/html4/strict.dtd' ), - 'SVG doctype' => array( '', 'quirks', 'svg', '-//W3C//DTD SVG 1.1//EN', 'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd' ), - 'MathML doctype' => array( '', 'quirks', 'math', '-//W3C//DTD MathML 2.0//EN', 'http://www.w3.org/Math/DTD/mathml2/mathml2.dtd' ), - 'Doctype with null byte replacement' => array( "", 'quirks', "null-\u{FFFD}", "\u{FFFD}", "\u{FFFD}\u{FFFD}" ), - 'Uppercase doctype' => array( '', 'quirks', 'uppercase' ), - 'Lowercase doctype' => array( '', 'quirks', 'lowercase' ), - 'Doctype with whitespace' => array( "", 'no-quirks', 'html', '', '' ), - 'Doctype trailing characters' => array( "", 'no-quirks', 'html', '', '' ), - 'An ugly no-quirks doctype' => array( "", 'no-quirks', 'html', 'pub-id', 'sysid' ), - 'Missing public ID' => array( '', 'quirks', 'html' ), - 'Missing system ID' => array( '', 'quirks', 'html' ), - 'Missing close quote public ID' => array( "", 'quirks', 'html', 'xyz' ), - 'Missing close quote system ID' => array( "", 'quirks', 'html', null, 'xyz' ), - 'Missing close quote system ID with public' => array( "", 'quirks', 'html', 'abc', 'xyz' ), - 'Bogus characters instead of system/public' => array( '', 'quirks', 'html' ), - 'Bogus characters instead of PUBLIC quote' => array( "", 'quirks', 'html' ), - 'Bogus characters instead of SYSTEM quote ' => array( "", 'quirks', 'html' ), - 'Emoji' => array( '', 'quirks', "\u{1F3F4}\u{E0067}\u{E0062}\u{E0065}\u{E006E}\u{E0067}\u{E007F}", '🔥', '😈' ), + 'Missing doctype name' => array( '', 'quirks' ), + 'HTML5 doctype' => array( '', 'no-quirks', 'html' ), + 'HTML5 doctype no whitespace before name' => array( '', 'no-quirks', 'html' ), + 'XHTML doctype' => array( '', 'no-quirks', 'html', '-//W3C//DTD HTML 4.01//EN', 'http://www.w3.org/TR/html4/strict.dtd' ), + 'SVG doctype' => array( '', 'quirks', 'svg', '-//W3C//DTD SVG 1.1//EN', 'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd' ), + 'MathML doctype' => array( '', 'quirks', 'math', '-//W3C//DTD MathML 2.0//EN', 'http://www.w3.org/Math/DTD/mathml2/mathml2.dtd' ), + 'Doctype with null byte replacement' => array( "", 'quirks', "null-\u{FFFD}", "\u{FFFD}", "\u{FFFD}\u{FFFD}" ), + 'Uppercase doctype' => array( '', 'quirks', 'uppercase' ), + 'Lowercase doctype' => array( '', 'quirks', 'lowercase' ), + 'Doctype with whitespace' => array( "", 'no-quirks', 'html', '', '' ), + 'Doctype trailing characters' => array( "", 'no-quirks', 'html', '', '' ), + 'An ugly no-quirks doctype' => array( "", 'no-quirks', 'html', 'pub-id', 'sysid' ), + 'Missing public ID' => array( '', 'quirks', 'html' ), + 'Missing system ID' => array( '', 'quirks', 'html' ), + 'Missing close quote public ID' => array( "", 'quirks', 'html', 'xyz' ), + 'Missing close quote system ID' => array( "", 'quirks', 'html', null, 'xyz' ), + 'Missing close quote system ID with public' => array( "", 'quirks', 'html', 'abc', 'xyz' ), + 'Bogus characters instead of system/public' => array( '', 'quirks', 'html' ), + 'Bogus characters instead of PUBLIC quote' => array( "", 'quirks', 'html' ), + 'Bogus characters instead of SYSTEM quote ' => array( "", 'quirks', 'html' ), + 'Emoji' => array( '', 'quirks', "\u{1F3F4}\u{E0067}\u{E0062}\u{E0065}\u{E006E}\u{E0067}\u{E007F}", '🔥', '😈' ), 'Bogus characters instead of SYSTEM quote after public' => array( "", 'quirks', 'html', '' ), - 'Special quirks mode if system unset' => array( '', 'quirks', 'html', '-//W3C//DTD HTML 4.01 Frameset//' ), - 'Special quirks mode if system empty' => array( '', 'quirks', 'html', '-//W3C//DTD HTML 4.01 Frameset//', '' ), + 'Special quirks mode if system unset' => array( '', 'quirks', 'html', '-//W3C//DTD HTML 4.01 Frameset//' ), + 'Special quirks mode if system empty' => array( '', 'quirks', 'html', '-//W3C//DTD HTML 4.01 Frameset//', '' ), 'Special limited-quirks mode if system is non-empty' => array( '', 'limited-quirks', 'html', '-//W3C//DTD HTML 4.01 Frameset//', 'non-empty' ), + 'Mixed-case system keyword' => array( "", 'no-quirks', 'html', null, 'about:legacy-compat' ), + + /* + * Keyword and identifier matching must fold ASCII letters only, regardless + * of the process locale. The non-ASCII bytes below are letters in common + * single-byte charmaps (0xC4 is Ä in ISO-8859-1), so a locale-sensitive + * comparison could fold them. + */ + 'Non-ASCII byte does not match PUBLIC keyword' => array( "", 'quirks', 'html' ), + 'Non-ASCII byte does not match SYSTEM keyword' => array( "", 'quirks', 'html' ), + 'Non-ASCII byte in DOCTYPE name is preserved' => array( "", 'quirks', "html\xC4" ), + 'Quirky public ID matches ASCII-insensitively' => array( '', 'quirks', 'html', '-//W3O//DTD W3 HTML STRICT 3.0//EN//' ), + 'Quirky public ID with non-ASCII byte does not match' => array( "", 'no-quirks', 'html', "-//W3O//DTD W3 HTML STR\xDDCT 3.0//EN//" ), ); } @@ -108,12 +121,14 @@ public function test_invalid_inputs_return_null( string $html ) { */ public static function invalid_inputs(): array { return array( - 'Empty string' => array( '' ), - 'Other HTML' => array( '
' ), - 'DOCTYPE after HTML' => array( 'x' ), - 'DOCTYPE before HTML' => array( 'x' ), - 'Incomplete DOCTYPE' => array( '"' => array( '">' ), + 'Empty string' => array( '' ), + 'Other HTML' => array( '
' ), + 'DOCTYPE after HTML' => array( 'x' ), + 'DOCTYPE before HTML' => array( 'x' ), + 'Incomplete DOCTYPE' => array( '"' => array( '">' ), + // 0xD6 is Ö in ISO-8859-1: it must not fold onto the ASCII letter O. + 'Non-ASCII byte in DOCTYPE keyword' => array( "" ), ); } } From 907e47741fa6a4a372c1fe1ff2fe613551b05479 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Wed, 8 Jul 2026 23:51:27 +0200 Subject: [PATCH 4/5] HTML API: Use locale-independent ASCII case comparisons in the HTML Processor. The HTML Processor folds tag names, breadcrumbs, attribute values, and encoding labels for ASCII case-insensitive comparison using strcasecmp(), strtoupper(), and strtolower(), all of which are locale-dependent before PHP 8.2. WP_HTML_Open_Elements::current_node_is() classified node names with ctype_upper(), which is locale-dependent on every PHP version. Under locales whose C library case-folds non-ASCII bytes, foreign-content end tags and tag queries containing non-ASCII bytes could spuriously match each other, and under locales which fold across the ASCII boundary, ASCII values such as the 'hidden' input type could fail to match. Replace these with the locale-independent ASCII case helpers and a locale-independent uppercase check. See #65372. Claude-Session: https://claude.ai/code/session_01K3vwt69YyVNbHAwu1Ue1Nf --- .../html-api/class-wp-html-open-elements.php | 7 +- .../html-api/class-wp-html-processor.php | 40 ++++----- .../tests/html-api/wpHtmlProcessor.php | 81 +++++++++++++++++++ 3 files changed, 108 insertions(+), 20 deletions(-) diff --git a/src/wp-includes/html-api/class-wp-html-open-elements.php b/src/wp-includes/html-api/class-wp-html-open-elements.php index 5c99db6d5eb4e..c13e24fd1c1f1 100644 --- a/src/wp-includes/html-api/class-wp-html-open-elements.php +++ b/src/wp-includes/html-api/class-wp-html-open-elements.php @@ -226,7 +226,12 @@ public function current_node_is( string $identity ): bool { return ( $current_node_name === $identity || ( '#doctype' === $identity && 'html' === $current_node_name ) || - ( '#tag' === $identity && ctype_upper( $current_node_name ) ) + /* + * Elements are identified by their uppercase ASCII names. This check + * avoids `ctype_upper()`, whose classification of bytes outside of + * ASCII depends on the process locale. + */ + ( '#tag' === $identity && strlen( $current_node_name ) === strspn( $current_node_name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' ) ) ); } diff --git a/src/wp-includes/html-api/class-wp-html-processor.php b/src/wp-includes/html-api/class-wp-html-processor.php index 6513db35c1243..995c3513ddc5b 100644 --- a/src/wp-includes/html-api/class-wp-html-processor.php +++ b/src/wp-includes/html-api/class-wp-html-processor.php @@ -715,7 +715,7 @@ public function next_tag( $query = null ): bool { } if ( isset( $query['tag_name'] ) ) { - $query['tag_name'] = strtoupper( $query['tag_name'] ); + $query['tag_name'] = WP_HTML_Decoder::ascii_uppercase( $query['tag_name'] ); } $needs_class = ( isset( $query['class_name'] ) && is_string( $query['class_name'] ) ) @@ -934,13 +934,13 @@ public function matches_breadcrumbs( $breadcrumbs ): bool { // Start at the last crumb. $crumb = end( $breadcrumbs ); - if ( '*' !== $crumb && $this->get_tag() !== strtoupper( $crumb ) ) { + if ( '*' !== $crumb && $this->get_tag() !== WP_HTML_Decoder::ascii_uppercase( $crumb ) ) { return false; } for ( $i = count( $this->breadcrumbs ) - 1; $i >= 0; $i-- ) { $node = $this->breadcrumbs[ $i ]; - $crumb = strtoupper( current( $breadcrumbs ) ); + $crumb = WP_HTML_Decoder::ascii_uppercase( current( $breadcrumbs ) ); if ( '*' !== $crumb && $node !== $crumb ) { return false; @@ -1413,7 +1413,7 @@ public function serialize_token(): string { $tag_name = str_replace( "\x00", "\u{FFFD}", $this->get_tag() ); $in_html = 'html' === $this->get_namespace(); - $qualified_name = $in_html ? strtolower( $tag_name ) : $this->get_qualified_tag_name(); + $qualified_name = $in_html ? WP_HTML_Decoder::ascii_lowercase( $tag_name ) : $this->get_qualified_tag_name(); $qualified_name = str_replace( "\x00", "\u{FFFD}", $qualified_name ); if ( $this->is_tag_closer() ) { @@ -1897,7 +1897,7 @@ private function step_in_head(): bool { if ( is_string( $http_equiv ) && is_string( $content ) && - 0 === strcasecmp( $http_equiv, 'Content-Type' ) + 'content-type' === WP_HTML_Decoder::ascii_lowercase( $http_equiv ) ) { $this->bail( 'Cannot yet process META tags with http-equiv Content-Type to determine encoding.' ); } @@ -3015,7 +3015,7 @@ private function step_in_body(): bool { * > string "hidden", then: set the frameset-ok flag to "not ok". */ $type_attribute = $this->get_attribute( 'type' ); - if ( ! is_string( $type_attribute ) || 'hidden' !== strtolower( $type_attribute ) ) { + if ( ! is_string( $type_attribute ) || 'hidden' !== WP_HTML_Decoder::ascii_lowercase( $type_attribute ) ) { $this->state->frameset_ok = false; } @@ -3539,7 +3539,7 @@ private function step_in_table(): bool { */ case '+INPUT': $type_attribute = $this->get_attribute( 'type' ); - if ( ! is_string( $type_attribute ) || 'hidden' !== strtolower( $type_attribute ) ) { + if ( ! is_string( $type_attribute ) || 'hidden' !== WP_HTML_Decoder::ascii_lowercase( $type_attribute ) ) { goto anything_else; } // @todo Indicate a parse error once it's possible. @@ -5133,7 +5133,10 @@ private function step_in_foreign_content(): bool { * > of the token, pop elements from the stack of open elements until node has * > been popped from the stack, and then return. */ - if ( 0 === strcasecmp( $node->node_name, $tag_name ) ) { + if ( + strlen( $node->node_name ) === strlen( $tag_name ) && + WP_HTML_Decoder::matches_ascii_case_insensitively( $node->node_name, $tag_name ) + ) { foreach ( $this->state->stack_of_open_elements->walk_up() as $item ) { $this->state->stack_of_open_elements->pop(); if ( $node === $item ) { @@ -6534,14 +6537,13 @@ private function is_html_integration_point(): bool { } $encoding = $this->get_attribute( 'encoding' ); + if ( ! is_string( $encoding ) ) { + return false; + } - return ( - is_string( $encoding ) && - ( - 0 === strcasecmp( $encoding, 'application/xhtml+xml' ) || - 0 === strcasecmp( $encoding, 'text/html' ) - ) - ); + $encoding = WP_HTML_Decoder::ascii_lowercase( $encoding ); + + return 'application/xhtml+xml' === $encoding || 'text/html' === $encoding; } $this->bail( 'Should not have reached end of HTML Integration Point detection: check HTML API code.' ); @@ -6561,10 +6563,10 @@ private function is_html_integration_point(): bool { */ public static function is_special( $tag_name ): bool { if ( is_string( $tag_name ) ) { - $tag_name = strtoupper( $tag_name ); + $tag_name = WP_HTML_Decoder::ascii_uppercase( $tag_name ); } else { $tag_name = 'html' === $tag_name->namespace - ? strtoupper( $tag_name->node_name ) + ? WP_HTML_Decoder::ascii_uppercase( $tag_name->node_name ) : "{$tag_name->namespace} {$tag_name->node_name}"; } @@ -6681,7 +6683,7 @@ public static function is_special( $tag_name ): bool { * @return bool Whether the given tag is an HTML Void Element. */ public static function is_void( $tag_name ): bool { - $tag_name = strtoupper( $tag_name ); + $tag_name = WP_HTML_Decoder::ascii_uppercase( $tag_name ); return ( 'AREA' === $tag_name || @@ -6738,7 +6740,7 @@ protected static function get_encoding( string $label ): ?string { * > If label is an ASCII case-insensitive match for any of the labels listed in the * > table below, then return the corresponding encoding; otherwise return failure. */ - switch ( strtolower( $label ) ) { + switch ( WP_HTML_Decoder::ascii_lowercase( $label ) ) { case 'unicode-1-1-utf-8': case 'unicode11utf8': case 'unicode20utf8': diff --git a/tests/phpunit/tests/html-api/wpHtmlProcessor.php b/tests/phpunit/tests/html-api/wpHtmlProcessor.php index dade2f3d27ca5..f100603549882 100644 --- a/tests/phpunit/tests/html-api/wpHtmlProcessor.php +++ b/tests/phpunit/tests/html-api/wpHtmlProcessor.php @@ -900,6 +900,87 @@ public function test_add_class_quirks_mode_folds_ascii_case_only() { ); } + /** + * Ensures tag queries and breadcrumbs fold ASCII letters only, regardless of locale. + * + * Tag names may contain non-ASCII bytes, which must match byte-for-byte. + * The byte pair 0xC4/0xE4 (Ä/ä in ISO-8859-1) is a case pair in common + * single-byte charmaps: a locale-sensitive comparison would treat the tag + * names below as ASCII case-insensitive matches for each other. + * + * @ticket 65372 + * + * @covers ::next_tag + * @covers ::matches_breadcrumbs + */ + public function test_tag_queries_fold_ascii_case_only() { + $processor = WP_HTML_Processor::create_fragment( "" ); + $this->assertFalse( + $processor->next_tag( array( 'tag_name' => "d\xE4ta" ) ), + 'Should not have matched a tag name differing in non-ASCII bytes.' + ); + + $processor = WP_HTML_Processor::create_fragment( "" ); + $this->assertTrue( + $processor->next_tag( array( 'tag_name' => "D\xC4TA" ) ), + 'Should have matched the tag name with ASCII letters case-folded.' + ); + $this->assertTrue( + $processor->matches_breadcrumbs( array( 'body', "d\xC4ta" ) ), + 'Should have matched breadcrumbs with ASCII letters case-folded.' + ); + $this->assertFalse( + $processor->matches_breadcrumbs( array( 'body', "d\xE4ta" ) ), + 'Should not have matched breadcrumbs differing in non-ASCII bytes.' + ); + } + + /** + * Ensures foreign-content end tag matching folds ASCII letters only, regardless of locale. + * + * @ticket 65372 + */ + public function test_foreign_content_end_tags_fold_ascii_case_only() { + $processor = WP_HTML_Processor::create_fragment( "" ); + $processor->next_tag( 'RECT' ); + $this->assertSame( + array( 'HTML', 'BODY', 'SVG', "F\xC4OO", 'RECT' ), + $processor->get_breadcrumbs(), + 'Should not have closed a foreign element on an end tag differing in non-ASCII bytes.' + ); + + $processor = WP_HTML_Processor::create_fragment( "" ); + $processor->next_tag( 'RECT' ); + $this->assertSame( + array( 'HTML', 'BODY', 'SVG', 'RECT' ), + $processor->get_breadcrumbs(), + 'Should have closed the foreign element on its exact end tag.' + ); + } + + /** + * Ensures MathML annotation-xml encoding checks fold ASCII letters only, regardless of locale. + * + * @ticket 65372 + */ + public function test_annotation_xml_encoding_folds_ascii_case_only() { + $processor = WP_HTML_Processor::create_fragment( '

' ); + $processor->next_tag( 'P' ); + $this->assertSame( + array( 'HTML', 'BODY', 'MATH', 'ANNOTATION-XML', 'P' ), + $processor->get_breadcrumbs(), + 'Should have recognized the HTML integration point with ASCII letters case-folded.' + ); + + $processor = WP_HTML_Processor::create_fragment( "

" ); + $processor->next_tag( 'P' ); + $this->assertSame( + array( 'HTML', 'BODY', 'P' ), + $processor->get_breadcrumbs(), + 'Should not have recognized an HTML integration point with an encoding differing in non-ASCII bytes.' + ); + } + /** * Ensures that the processor correctly adjusts the namespace * for elements inside HTML integration points. From ba42636edd426a2bab7bcae9a2843e7b139777b4 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Wed, 8 Jul 2026 23:55:50 +0200 Subject: [PATCH 5/5] Code Quality: Make WP_Token_Map ASCII case-insensitive lookups locale-independent. The 'ascii-case-insensitive' lookup mode relied on locale-sensitive comparisons: substr_compare() with its case-insensitivity flag (locale-dependent on every PHP version) and stripos()/strtoupper() (locale-dependent before PHP 8.2). Under locales whose C library folds non-ASCII bytes, words differing in non-ASCII bytes could spuriously match each other. Additionally, a misplaced parenthesis in read_small_token() uppercased the result of a comparison instead of the compared byte, so ignore-case lookups of small words failed to fold ASCII case beyond the first byte: read_token( 'AB', ..., 'ascii-case-insensitive' ) failed to find 'ab' even though contains( 'AB', 'ascii-case-insensitive' ) reported it present. Replace the comparisons with locale-independent ASCII case folding, keeping WP_Token_Map free of dependencies on other classes. See #65372. Claude-Session: https://claude.ai/code/session_01K3vwt69YyVNbHAwu1Ue1Nf --- src/wp-includes/class-wp-token-map.php | 101 ++++++++++++++++-- .../phpunit/tests/wp-token-map/wpTokenMap.php | 81 ++++++++++++++ 2 files changed, 174 insertions(+), 8 deletions(-) diff --git a/src/wp-includes/class-wp-token-map.php b/src/wp-includes/class-wp-token-map.php index fc223b187f8c5..fefdeb2854645 100644 --- a/src/wp-includes/class-wp-token-map.php +++ b/src/wp-includes/class-wp-token-map.php @@ -452,7 +452,9 @@ public function contains( string $word, string $case_sensitivity = 'case-sensiti } $term = str_pad( $word, $this->key_length + 1, "\x00", STR_PAD_RIGHT ); - $word_at = $ignore_case ? stripos( $this->small_words, $term ) : strpos( $this->small_words, $term ); + $word_at = $ignore_case + ? strpos( self::ascii_uppercase( $this->small_words ), self::ascii_uppercase( $term ) ) + : strpos( $this->small_words, $term ); if ( false === $word_at ) { return false; } @@ -461,7 +463,9 @@ public function contains( string $word, string $case_sensitivity = 'case-sensiti } $group_key = substr( $word, 0, $this->key_length ); - $group_at = $ignore_case ? stripos( $this->groups, $group_key ) : strpos( $this->groups, $group_key ); + $group_at = $ignore_case + ? strpos( self::ascii_uppercase( $this->groups ), self::ascii_uppercase( $group_key ) ) + : strpos( $this->groups, $group_key ); if ( false === $group_at ) { return false; } @@ -478,7 +482,12 @@ public function contains( string $word, string $case_sensitivity = 'case-sensiti $mapping_length = unpack( 'C', $group[ $at++ ] )[1]; $mapping_at = $at; - if ( $token_length === $length && 0 === substr_compare( $group, $slug, $token_at, $token_length, $ignore_case ) ) { + if ( + $token_length === $length && + ( $ignore_case + ? self::matches_ascii_case_insensitively( $group, $slug, $token_at ) + : 0 === substr_compare( $group, $slug, $token_at, $token_length ) ) + ) { return true; } @@ -547,7 +556,9 @@ public function read_token( string $text, int $offset = 0, &$matched_token_byte_ } $group_key = substr( $text, $offset, $this->key_length ); - $group_at = $ignore_case ? stripos( $this->groups, $group_key ) : strpos( $this->groups, $group_key ); + $group_at = $ignore_case + ? strpos( self::ascii_uppercase( $this->groups ), self::ascii_uppercase( $group_key ) ) + : strpos( $this->groups, $group_key ); if ( false === $group_at ) { // Perhaps a short word then. return strlen( $this->small_words ) > 0 @@ -565,7 +576,11 @@ public function read_token( string $text, int $offset = 0, &$matched_token_byte_ $mapping_length = unpack( 'C', $group[ $at++ ] )[1]; $mapping_at = $at; - if ( 0 === substr_compare( $text, $token, $offset + $this->key_length, $token_length, $ignore_case ) ) { + $token_matches = $ignore_case + ? self::matches_ascii_case_insensitively( $text, $token, $offset + $this->key_length ) + : 0 === substr_compare( $text, $token, $offset + $this->key_length, $token_length ); + + if ( $token_matches ) { $matched_token_byte_length = $this->key_length + $token_length; return substr( $group, $mapping_at, $mapping_length ); } @@ -597,7 +612,7 @@ private function read_small_token( string $text, int $offset = 0, &$matched_toke $small_length = strlen( $this->small_words ); $search_text = substr( $text, $offset, $this->key_length ); if ( $ignore_case ) { - $search_text = strtoupper( $search_text ); + $search_text = self::ascii_uppercase( $search_text ); } $starting_char = $search_text[0]; @@ -605,7 +620,7 @@ private function read_small_token( string $text, int $offset = 0, &$matched_toke while ( $at < $small_length ) { if ( $starting_char !== $this->small_words[ $at ] && - ( ! $ignore_case || strtoupper( $this->small_words[ $at ] ) !== $starting_char ) + ( ! $ignore_case || self::ascii_uppercase( $this->small_words[ $at ] ) !== $starting_char ) ) { $at += $this->key_length + 1; continue; @@ -619,7 +634,7 @@ private function read_small_token( string $text, int $offset = 0, &$matched_toke if ( $search_text[ $adjust ] !== $this->small_words[ $at + $adjust ] && - ( ! $ignore_case || strtoupper( $this->small_words[ $at + $adjust ] !== $search_text[ $adjust ] ) ) + ( ! $ignore_case || self::ascii_uppercase( $this->small_words[ $at + $adjust ] ) !== $search_text[ $adjust ] ) ) { $at += $this->key_length + 1; continue 2; @@ -633,6 +648,76 @@ private function read_small_token( string $text, int $offset = 0, &$matched_toke return null; } + /** + * Returns the ASCII uppercase version of a given string. + * + * Only the ASCII lowercase letters a-z are folded onto their uppercase + * counterparts; all other bytes are left unchanged. This differs from + * `strtoupper()`, which before PHP 8.2 folds bytes according to the + * current process locale. Lookup must be locale-independent. + * + * @since 7.1.0 + * + * @param string $text Text to fold. + * @return string ASCII-uppercase version of the given text. + */ + private static function ascii_uppercase( string $text ): string { + return strtr( $text, 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' ); + } + + /** + * Indicates if a span of text matches a given string, ignoring ASCII case. + * + * Matching is performed byte-by-byte and folds only the ASCII letters A-Z + * and a-z onto each other, regardless of the process locale. This differs + * from `substr_compare()` with its case-insensitivity flag, which folds + * bytes according to the current process locale and may treat non-ASCII + * bytes as case variants of each other. + * + * The span begins at the given byte offset into the text and runs for the + * byte length of the search string. A span extending past the end of the + * text never matches. + * + * @since 7.1.0 + * + * @param string $text Text possibly containing the search string. + * @param string $search Search string; its byte length determines the span length. + * @param int $offset Byte offset into the text where the span begins. + * @return bool Whether the span is an ASCII case-insensitive match for the search string. + */ + private static function matches_ascii_case_insensitively( string $text, string $search, int $offset ): bool { + $length = strlen( $search ); + if ( $offset < 0 || $offset + $length > strlen( $text ) ) { + return false; + } + + for ( $at = 0; $at < $length; $at++ ) { + $text_byte = $text[ $offset + $at ]; + $search_byte = $search[ $at ]; + + if ( $text_byte === $search_byte ) { + continue; + } + + $text_ord = ord( $text_byte ); + $search_ord = ord( $search_byte ); + + // Fold uppercase ASCII letters onto their lowercase counterparts; no other bytes fold. + if ( $text_ord >= 0x41 && $text_ord <= 0x5A ) { + $text_ord += 0x20; + } + if ( $search_ord >= 0x41 && $search_ord <= 0x5A ) { + $search_ord += 0x20; + } + + if ( $text_ord !== $search_ord ) { + return false; + } + } + + return true; + } + /** * Exports the token map into an associate array of key/value pairs. * diff --git a/tests/phpunit/tests/wp-token-map/wpTokenMap.php b/tests/phpunit/tests/wp-token-map/wpTokenMap.php index 23edc5de80b94..aa0ef8ef57a0d 100644 --- a/tests/phpunit/tests/wp-token-map/wpTokenMap.php +++ b/tests/phpunit/tests/wp-token-map/wpTokenMap.php @@ -381,6 +381,87 @@ public function data_html5_test_dataset() { } } + /** + * Ensures ASCII case-insensitive lookup folds ASCII letters only, regardless of locale. + * + * The byte pair 0xCC/0xEC (Ì/ì in ISO-8859-1) is a case pair in common single-byte + * charmaps: a locale-sensitive comparison would treat the words below as ASCII + * case-insensitive matches for each other. + * + * @ticket 65372 + * + * @covers ::contains + * @covers ::read_token + */ + public function test_ascii_case_insensitive_lookup_folds_ascii_case_only() { + // Words longer than the group key length are stored in word groups. + $map = WP_Token_Map::from_array( array( "ab\xCC" => 'replacement' ) ); + + $this->assertFalse( + $map->contains( "ab\xEC", 'ascii-case-insensitive' ), + 'Should not have found a word differing in non-ASCII bytes.' + ); + $this->assertNull( + $map->read_token( "ab\xEC", 0, $token_length, 'ascii-case-insensitive' ), + 'Should not have read a token differing in non-ASCII bytes.' + ); + $this->assertTrue( + $map->contains( "AB\xCC", 'ascii-case-insensitive' ), + 'Should have found the word with ASCII letters case-folded.' + ); + $this->assertSame( + 'replacement', + $map->read_token( "AB\xCC", 0, $token_length, 'ascii-case-insensitive' ), + 'Should have read the token with ASCII letters case-folded.' + ); + + // Words not longer than the group key length are stored as small words. + $small_map = WP_Token_Map::from_array( array( "k\xCC" => 'replacement' ) ); + + $this->assertFalse( + $small_map->contains( "k\xEC", 'ascii-case-insensitive' ), + 'Should not have found a small word differing in non-ASCII bytes.' + ); + $this->assertNull( + $small_map->read_token( "k\xEC", 0, $token_length, 'ascii-case-insensitive' ), + 'Should not have read a small-word token differing in non-ASCII bytes.' + ); + $this->assertTrue( + $small_map->contains( "K\xCC", 'ascii-case-insensitive' ), + 'Should have found the small word with ASCII letters case-folded.' + ); + $this->assertSame( + 'replacement', + $small_map->read_token( "K\xCC", 0, $token_length, 'ascii-case-insensitive' ), + 'Should have read the small-word token with ASCII letters case-folded.' + ); + } + + /** + * Ensures that contains() and read_token() agree on ASCII case-insensitive lookups. + * + * Previously, read_token() failed to fold ASCII case beyond the first byte of + * small words, disagreeing with contains() for the same word and map. + * + * @ticket 65372 + * + * @covers ::contains + * @covers ::read_token + */ + public function test_reads_small_word_tokens_ascii_case_insensitively() { + $map = WP_Token_Map::from_array( array( 'ab' => 'replacement' ) ); + + $this->assertTrue( + $map->contains( 'AB', 'ascii-case-insensitive' ), + 'Should have found the word with ASCII letters case-folded.' + ); + $this->assertSame( + 'replacement', + $map->read_token( 'AB', 0, $token_length, 'ascii-case-insensitive' ), + 'Should have read the token with ASCII letters case-folded.' + ); + } + /** * Returns a static copy of the Token Map for HTML5. * This is a test performance optimization.