Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion phpstan.neon.dist
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ includes:
- tests/phpstan/baselines/parameter.defaultValue.neon
- tests/phpstan/baselines/parameter.notFound.neon
- tests/phpstan/baselines/parameterByRef.type.neon
- tests/phpstan/baselines/parameterByRef.unusedType.neon
- tests/phpstan/baselines/property.defaultValue.neon
- tests/phpstan/baselines/property.nonObject.neon
- tests/phpstan/baselines/property.notFound.neon
Expand Down
12 changes: 7 additions & 5 deletions src/wp-includes/compat-utf8.php
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
* @param int|null $max_bytes Stop scanning after this many bytes have been seen.
* @param int|null $max_code_points Stop scanning after this many code points have been seen.
* @param bool|null $has_noncharacters Set to indicate if scanned string contained noncharacters.
* @param-out bool $has_noncharacters

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please verify that this doesn’t break the docs-generation. before we start changing @param declarations, we should ensure that the published docs page isn’t broken.

you can hopefully lean on dmsnell#29 to figure out how to preview the generation.

I see that this is an additional annotation beyond the @param, but it would be ideal to double-check first.

* @return int How many code points were successfully scanned.
*/
function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max_bytes = null, ?int $max_code_points = null, ?bool &$has_noncharacters = null ): int {
Expand Down Expand Up @@ -151,7 +152,7 @@ function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max

// Covers the range U+FDD0–U+FDEF, U+FFFE, U+FFFF.
if ( 0xEF === $b1 ) {
$has_noncharacters |= (
$has_noncharacters = $has_noncharacters || (
( 0xB7 === $b2 && $b3 >= 0x90 && $b3 <= 0xAF ) ||
( 0xBF === $b2 && ( 0xBE === $b3 || 0xBF === $b3 ) )
);
Expand All @@ -176,7 +177,7 @@ function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max
$i += 3;

// Covers U+1FFFE, U+1FFFF, U+2FFFE, U+2FFFF, …, U+10FFFE, U+10FFFF.
$has_noncharacters |= (
$has_noncharacters = $has_noncharacters || (
( 0x0F === ( $b2 & 0x0F ) ) &&
0xBF === $b3 &&
( 0xBE === $b4 || 0xBF === $b4 )
Expand Down Expand Up @@ -373,9 +374,10 @@ function _wp_utf8_codepoint_count( string $text, ?int $byte_offset = 0, ?int $ma
* @param int $byte_offset Start counting at this byte offset.
* @param int $max_code_points Stop counting after this many code points have been seen,
* or at the end of the string.
* @param ?int $found_code_points Optional. Will be set to number of found code points in
* span, as this might be smaller than the maximum count if
* the string is not long enough.
* @param int|null $found_code_points Optional. Will be set to number of found code points in
* span, as this might be smaller than the maximum count if
* the string is not long enough. Default 0.
* @param-out int $found_code_points
* @return int Number of bytes spanned by the code points.
*/
function _wp_utf8_codepoint_span( string $text, int $byte_offset, int $max_code_points, ?int &$found_code_points = 0 ): int {
Expand Down
5 changes: 0 additions & 5 deletions tests/phpstan/baselines/parameterByRef.type.neon
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,6 @@ parameters:
identifier: parameterByRef.type
count: 1
path: ../../../src/wp-includes/class-wp-token-map.php
-
message: '#^Parameter &\$has_noncharacters by\-ref type of function _wp_scan_utf8\(\) expects bool\|null, int given\.$#'
identifier: parameterByRef.type
count: 2
path: ../../../src/wp-includes/compat-utf8.php
-
message: '#^Parameter &\$result by\-ref type of function _page_traverse_name\(\) expects array\<string\>, array given\.$#'
identifier: parameterByRef.type
Expand Down
30 changes: 0 additions & 30 deletions tests/phpstan/baselines/parameterByRef.unusedType.neon

This file was deleted.

80 changes: 80 additions & 0 deletions tests/phpunit/tests/compat/wpScanUtf8.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
<?php
/**
* Unit tests covering _wp_scan_utf8().
*
* @package WordPress
* @subpackage Charset
*
* @group compat
*
* @covers ::_wp_scan_utf8
*/
class Tests_Compat_wpScanUtf8 extends WP_UnitTestCase {

/**
* Tests scanning a valid ASCII string.
*
* @ticket 65817
*/
public function test_scan_ascii_string() {
$text = 'Hello, World!';
$at = 0;
$invalid_length = 0;
$has_noncharacters = null;

$scanned = _wp_scan_utf8( $text, $at, $invalid_length, null, null, $has_noncharacters );

$this->assertSame( 13, $scanned );
$this->assertSame( 0, $invalid_length );
$this->assertFalse( $has_noncharacters );
}

/**
* Tests scanning UTF-8 string containing noncharacters.
*
* @ticket 65817
*
* @dataProvider data_strings_with_noncharacters
*
* @param string $text UTF-8 string containing a noncharacter.
*/
public function test_scan_string_with_noncharacters( string $text ) {
$at = 0;
$invalid_length = 0;
$has_noncharacters = null;

_wp_scan_utf8( $text, $at, $invalid_length, null, null, $has_noncharacters );

$this->assertTrue( $has_noncharacters );
}

/**
* Data provider for noncharacters.
*
* @return array<string, array{0: string}>
*/
public static function data_strings_with_noncharacters() {
return array(
'U+FFFF (3-byte noncharacter)' => array( "test \xEF\xBF\xBF end" ),
'U+FFFE (3-byte noncharacter)' => array( "test \xEF\xBF\xBE end" ),
'U+FDD0 (3-byte noncharacter)' => array( "test \xEF\xB7\x90 end" ),
'U+1FFFF (4-byte noncharacter)' => array( "test \xF0\x9F\xBF\xBF end" ),
);
}

/**
* Tests scanning UTF-8 string without noncharacters.
*
* @ticket 65817
*/
public function test_scan_string_without_noncharacters() {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a misleading title. it would be clearer to show variant cases of multi-byte UTF-8 code points and extended grapheme clusters, which would clarify the behavior of the function.

let’s use a data provider and show interesting cases, and then use mb_strlen( $text, 'UTF-8' ) inside an assertSame() to verify the counts.

this way we can cover two-, three-, and four-byte sequences, spans of invalid bytes (covering the maximal subpart behavior), ZWJ sequences, and grapheme clusters.

$text = 'WordPress: 日本語 and Piña!';
$at = 0;
$invalid_length = 0;
$has_noncharacters = null;

_wp_scan_utf8( $text, $at, $invalid_length, null, null, $has_noncharacters );

$this->assertFalse( $has_noncharacters );
}
}
Loading