Fix #131: Deprecate the .. range syntax in the characters parameter of Trim, LeftTrim and RightTrim attributes, add multibyte and encoding parameters to Trim, LeftTrim, RightTrim and ToArrayOfStrings attributes and their resolvers

.github/workflows/mutation.yml

@@ -30,6 +30,6 @@ jobs:
       os: >-
         ['ubuntu-latest']
       php: >-
-        ['8.3']
+        ['8.5']
     secrets:
       STRYKER_DASHBOARD_API_KEY: ${{ secrets.STRYKER_DASHBOARD_API_KEY }}

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## 1.6.4 under development
 
+- Chg #131: Deprecate the `..` range syntax in the `characters` parameter of `Trim`, `LeftTrim` and `RightTrim`
+  attributes (@vjik)
+- Enh #131: Add `multibyte` and `encoding` parameters to `Trim`, `LeftTrim`, `RightTrim` and `ToArrayOfStrings`
+  attributes and their resolvers (@vjik)
 - Enh #117: Explicitly import functions and constants in "use" section (@mspirkov)
 - Enh #132: Exclude development files from the distribution archive (@vjik)
 

composer-dependency-analyser.php

@@ -5,10 +5,22 @@
 use ShipMonk\ComposerDependencyAnalyser\Config\Configuration;
 use ShipMonk\ComposerDependencyAnalyser\Config\ErrorType;
 
-return (new Configuration())
+$config = (new Configuration())
     ->disableComposerAutoloadPathScan()
     ->setFileExtensions(['php'])
     ->addPathToScan(__DIR__ . '/config', isDev: false)
     ->addPathToScan(__DIR__ . '/src', isDev: false)
     ->addPathToScan(__DIR__ . '/tests', isDev: true)
     ->ignoreErrorsOnExtension('ext-intl', [ErrorType::SHADOW_DEPENDENCY]);
+
+// Multibyte trim support (`Trim`, `LeftTrim`, `RightTrim`, `ToArrayOfStrings` attributes and `TrimCharacters`
+// helper) uses `mb_str_split()`, `mb_ord()`, `mb_chr()` and, since PHP 8.4, `mb_trim()`/`mb_ltrim()`/`mb_rtrim()`.
+// All of them come either from the "mbstring" extension, or, for the trim functions on PHP older than 8.4, from
+// the "symfony/polyfill-mbstring" package. The extension is only suggested to users, and the package is required
+// for dev only, to run tests on PHP older than 8.4, so neither of them is intentionally required in production.
+$config->ignoreErrorsOnExtension('ext-mbstring', [ErrorType::SHADOW_DEPENDENCY]);
+if (PHP_VERSION_ID < 80400) {
+    $config->ignoreErrorsOnPackage('symfony/polyfill-mbstring', [ErrorType::DEV_DEPENDENCY_IN_PROD]);
+}
+
+return $config;

composer.json

@@ -38,14 +38,17 @@
         "rector/rector": "^2.6.7",
         "shipmonk/composer-dependency-analyser": "^1.8",
         "spatie/phpunit-watcher": "^1.24",
+        "symfony/polyfill-mbstring": "^1.31",
         "vimeo/psalm": "^6.19.0",
         "yiisoft/code-style": "^1.1",
         "yiisoft/di": "^1.4",
         "yiisoft/dummy-provider": "^1.1.0",
         "yiisoft/test-support": "^3.0.2"
     },
     "suggest": {
-        "ext-intl": "Allows using `ToDateTime` parameter attribute"
+        "ext-intl": "Allows using `ToDateTime` parameter attribute",
+        "ext-mbstring": "Allows using multibyte mode of `Trim`, `LeftTrim`, `RightTrim` and `ToArrayOfStrings` attributes on PHP 8.4 or later",
+        "symfony/polyfill-mbstring": "Allows using multibyte mode of `Trim`, `LeftTrim`, `RightTrim` and `ToArrayOfStrings` attributes on PHP versions earlier than 8.4"
     },
     "autoload": {
         "psr-4": {

docs/guide/en/typecasting.md

@@ -163,6 +163,33 @@ class Person
 $person = $hydrator->create(Person::class, ['name' => '  John ']);
 ```
 
+By default, these attributes are not multibyte-aware, so Unicode whitespace characters, such as `U+00A0` (no-break
+space) or `U+2003` (em space), are kept. To strip them as well, enable the `multibyte` parameter:
+
+```php
+use Yiisoft\Hydrator\Attribute\Parameter\Trim;
+
+class Person
+{
+    public function __construct(
+        #[Trim(multibyte: true)] // "\u{A0}John\u{2003}" → 'John'
+        private ?string $name = null,
+    ) {}
+}
+
+$person = $hydrator->create(Person::class, ['name' => "\u{A0}John\u{2003}"]);
+```
+
+Multibyte mode uses `mb_trim()`, `mb_ltrim()` and `mb_rtrim()` functions that are provided by `mbstring` PHP extension
+since PHP 8.4. To use it with an earlier PHP version, install
+[symfony/polyfill-mbstring](https://github.com/symfony/polyfill-mbstring) package. The `encoding` parameter selects
+the encoding used in multibyte mode; `null` (default) means using `mb_internal_encoding()`.
+
+With `..` you can specify a range of characters in the `characters` parameter, for example, `a..z`. It works both in
+the default and in the multibyte mode, but in the default mode ranges are byte-based, so a range of non-ASCII
+characters, such as `а..я`, works correctly only in the multibyte mode. This syntax is deprecated and will be removed
+in the next major version, so avoid it in new code.
+
 ### `ToDatetime`
 
 To cast a value to `DateTimeImmutable` or `DateTime` object explicitly, you can use `ToDateTime` attribute:
@@ -239,4 +266,10 @@ Attribute parameters:
 - `removeEmpty` — remove empty strings from array (boolean, default `false`);
 - `splitResolvedValue` — split resolved value by separator (boolean, default `true`);
 - `separator` — the boundary string (default, `\R`), it's a part of regular expression so should be taken into account 
-  or properly escaped with `preg_quote()`.
+  or properly escaped with `preg_quote()`;
+- `multibyte` — whether to use multibyte-aware trimming when `trim` is enabled (nullable boolean, default `null`
+  meaning the resolver default is used); requires the `mb_trim()` function provided by `mbstring` PHP extension
+  since PHP 8.4, or by [symfony/polyfill-mbstring](https://github.com/symfony/polyfill-mbstring) package on earlier
+  versions;
+- `encoding` — the encoding to use in multibyte mode (nullable string, default `null` meaning the resolver default
+  is used).

infection.json.dist

@@ -1,4 +1,5 @@
 {
+    "bootstrap": "./vendor/autoload.php",
     "source": {
         "directories": [
             "src"

src/Attribute/Parameter/LeftTrim.php

@@ -9,18 +9,30 @@
 /**
  * Strip whitespace (or other characters) from the beginning of a resolved string value.
  *
+ * In multibyte mode, Unicode whitespace characters, such as `U+00A0` (no-break space), are stripped as well.
+ * It requires the `mb_ltrim()` function provided by the `mbstring` PHP extension since PHP 8.4, or by the
+ * `symfony/polyfill-mbstring` package on earlier versions.
+ *
  * @see https://www.php.net/manual/function.ltrim.php
+ * @see https://www.php.net/manual/function.mb-ltrim.php
  */
 #[Attribute(Attribute::TARGET_PROPERTY | Attribute::TARGET_PARAMETER | Attribute::IS_REPEATABLE)]
 final class LeftTrim implements ParameterAttributeInterface
 {
     /**
-     * @param string|null $characters The list all characters that you want to be stripped. With `..` you can specify
-     * a range of characters.
+     * @param string|null $characters The list of all characters that you want to be stripped. With `..` you can
+     * specify a range of characters, both in the default and in the multibyte mode. This syntax is deprecated and
+     * will be removed in the next major version.
+     * @param bool|null $multibyte Whether to use multibyte-aware trimming. `null` means using the resolver default.
+     * @param string|null $encoding The encoding to use in multibyte mode. `null` means using the resolver default.
      */
     public function __construct(
         public readonly ?string $characters = null,
-    ) {}
+        public readonly ?bool $multibyte = null,
+        public readonly ?string $encoding = null,
+    ) {
+        TrimCharacters::checkMultibyteFunctionsExist($multibyte);
+    }
 
     public function getResolver(): string
     {

src/Attribute/Parameter/LeftTrimResolver.php

@@ -10,11 +10,25 @@
 
 use function is_string;
 
+/**
+ * Resolver for {@see LeftTrim} attribute.
+ */
 final class LeftTrimResolver implements ParameterAttributeResolverInterface
 {
+    /**
+     * @param string|null $characters The list of characters to strip when it is not specified in the attribute.
+     * With `..` you can specify a range of characters, both in the default and in the multibyte mode. This syntax
+     * is deprecated and will be removed in the next major version.
+     * @param bool $multibyte Whether to use multibyte-aware trimming when it is not specified in the attribute.
+     * @param string|null $encoding The encoding to use in multibyte mode when it is not specified in the attribute.
+     */
     public function __construct(
         private readonly ?string $characters = null,
-    ) {}
+        private readonly bool $multibyte = false,
+        private readonly ?string $encoding = null,
+    ) {
+        TrimCharacters::checkMultibyteFunctionsExist($multibyte);
+    }
 
     public function getParameterValue(
         ParameterAttributeInterface $attribute,
@@ -25,7 +39,7 @@
         }
 
         if (!$context->isResolved()) {
             return Result::fail();
         }
 
         $resolvedValue = $context->getResolvedValue();
@@ -34,9 +48,23 @@
         }
 
         $characters = $attribute->characters ?? $this->characters;
+        $multibyte = $attribute->multibyte ?? $this->multibyte;
+        $encoding = $attribute->encoding ?? $this->encoding;
+
+        TrimCharacters::checkDeprecatedRanges($characters, $multibyte, $encoding);
+
+        if (!$multibyte) {
+            return Result::success(
+                $characters === null ? ltrim($resolvedValue) : ltrim($resolvedValue, $characters),
+            );
+        }
 
         return Result::success(
-            $characters === null ? ltrim($resolvedValue) : ltrim($resolvedValue, $characters),
+            mb_ltrim(
+                $resolvedValue,
+                $characters === null ? null : TrimCharacters::expandRanges($characters, $encoding),
+                $encoding,
+            ),
         );
     }
 }

src/Attribute/Parameter/RightTrim.php

@@ -9,18 +9,30 @@
 /**
  * Strip whitespace (or other characters) from the end of a resolved string value.
  *
+ * In multibyte mode, Unicode whitespace characters, such as `U+00A0` (no-break space), are stripped as well.
+ * It requires the `mb_rtrim()` function provided by the `mbstring` PHP extension since PHP 8.4, or by the
+ * `symfony/polyfill-mbstring` package on earlier versions.
+ *
  * @see https://www.php.net/manual/function.rtrim.php
+ * @see https://www.php.net/manual/function.mb-rtrim.php
  */
 #[Attribute(Attribute::TARGET_PROPERTY | Attribute::TARGET_PARAMETER | Attribute::IS_REPEATABLE)]
 final class RightTrim implements ParameterAttributeInterface
 {
     /**
-     * @param string|null $characters The list all characters that you want to be stripped. With `..` you can specify
-     * a range of characters.
+     * @param string|null $characters The list of all characters that you want to be stripped. With `..` you can
+     * specify a range of characters, both in the default and in the multibyte mode. This syntax is deprecated and
+     * will be removed in the next major version.
+     * @param bool|null $multibyte Whether to use multibyte-aware trimming. `null` means using the resolver default.
+     * @param string|null $encoding The encoding to use in multibyte mode. `null` means using the resolver default.
      */
     public function __construct(
         public readonly ?string $characters = null,
-    ) {}
+        public readonly ?bool $multibyte = null,
+        public readonly ?string $encoding = null,
+    ) {
+        TrimCharacters::checkMultibyteFunctionsExist($multibyte);
+    }
 
     public function getResolver(): string
     {

src/Attribute/Parameter/RightTrimResolver.php

@@ -10,11 +10,25 @@
 
 use function is_string;
 
+/**
+ * Resolver for {@see RightTrim} attribute.
+ */
 final class RightTrimResolver implements ParameterAttributeResolverInterface
 {
+    /**
+     * @param string|null $characters The list of characters to strip when it is not specified in the attribute.
+     * With `..` you can specify a range of characters, both in the default and in the multibyte mode. This syntax
+     * is deprecated and will be removed in the next major version.
+     * @param bool $multibyte Whether to use multibyte-aware trimming when it is not specified in the attribute.
+     * @param string|null $encoding The encoding to use in multibyte mode when it is not specified in the attribute.
+     */
     public function __construct(
         private readonly ?string $characters = null,
-    ) {}
+        private readonly bool $multibyte = false,
+        private readonly ?string $encoding = null,
+    ) {
+        TrimCharacters::checkMultibyteFunctionsExist($multibyte);
+    }
 
     public function getParameterValue(
         ParameterAttributeInterface $attribute,
@@ -25,7 +39,7 @@
         }
 
         if (!$context->isResolved()) {
             return Result::fail();
         }
 
         $resolvedValue = $context->getResolvedValue();
@@ -34,9 +48,23 @@
         }
 
         $characters = $attribute->characters ?? $this->characters;
+        $multibyte = $attribute->multibyte ?? $this->multibyte;
+        $encoding = $attribute->encoding ?? $this->encoding;
+
+        TrimCharacters::checkDeprecatedRanges($characters, $multibyte, $encoding);
+
+        if (!$multibyte) {
+            return Result::success(
+                $characters === null ? rtrim($resolvedValue) : rtrim($resolvedValue, $characters),
+            );
+        }
 
         return Result::success(
-            $characters === null ? rtrim($resolvedValue) : rtrim($resolvedValue, $characters),
+            mb_rtrim(
+                $resolvedValue,
+                $characters === null ? null : TrimCharacters::expandRanges($characters, $encoding),
+                $encoding,
+            ),
         );
     }
 }

src/Attribute/Parameter/ToArrayOfStrings.php

@@ -8,6 +8,12 @@
 
 /**
  * Casts the resolved value to array of strings.
+ *
+ * In multibyte mode, trimming strips Unicode whitespace characters, such as `U+00A0` (no-break space), as well.
+ * It requires the `mb_trim()` function provided by the `mbstring` PHP extension since PHP 8.4, or by the
+ * `symfony/polyfill-mbstring` package on earlier versions.
+ *
+ * @see https://www.php.net/manual/function.mb-trim.php
  */
 #[Attribute(Attribute::TARGET_PROPERTY | Attribute::TARGET_PARAMETER | Attribute::IS_REPEATABLE)]
 final class ToArrayOfStrings implements ParameterAttributeInterface
@@ -18,13 +24,20 @@
      * @param bool $splitResolvedValue Split non-array resolved value to array of strings by {@see $separator}.
      * @param string $separator The boundary string. It is a part of regular expression
      * so should be taken into account or properly escaped with {@see preg_quote()}.
+     * @param bool|null $multibyte Whether to use multibyte-aware trimming when {@see $trim} is enabled. `null`
+     * means using the resolver default.
+     * @param string|null $encoding The encoding to use in multibyte mode. `null` means using the resolver default.
      */
     public function __construct(
         public readonly bool $trim = false,
         public readonly bool $removeEmpty = false,
         public readonly bool $splitResolvedValue = true,
         public readonly string $separator = '\R',
-    ) {}
+        public readonly ?bool $multibyte = null,
+        public readonly ?string $encoding = null,
+    ) {
+        TrimCharacters::checkMultibyteFunctionsExist($multibyte);
+    }
 
     public function getResolver(): string
     {

src/Attribute/Parameter/ToArrayOfStringsResolver.php

@@ -12,8 +12,24 @@
 
 use function is_scalar;
 
+/**
+ * Resolver for {@see ToArrayOfStrings} attribute.
+ */
 final class ToArrayOfStringsResolver implements ParameterAttributeResolverInterface
 {
+    /**
+     * @param bool $multibyte Whether to use multibyte-aware trimming that strips Unicode whitespace characters
+     * such as `U+00A0` (no-break space) as well, when it is not specified in the attribute. Requires PHP 8.4 or
+     * later with `mbstring` extension, or `symfony/polyfill-mbstring` package.
+     * @param string|null $encoding The encoding to use in multibyte mode when it is not specified in the attribute.
+     */
+    public function __construct(
+        private readonly bool $multibyte = false,
+        private readonly ?string $encoding = null,
+    ) {
+        TrimCharacters::checkMultibyteFunctionsExist($multibyte);
+    }
+
     public function getParameterValue(
         ParameterAttributeInterface $attribute,
         ParameterAttributeResolveContext $context,
@@ -44,7 +60,13 @@
         }
 
         if ($attribute->trim) {
-            $array = array_map(trim(...), $array);
+            $multibyte = $attribute->multibyte ?? $this->multibyte;
+            $encoding = $attribute->encoding ?? $this->encoding;
+
+            $array = array_map(
+                $multibyte ? static fn(string $value): string => mb_trim($value, null, $encoding) : trim(...),
+                $array,
+            );
         }
 
         if ($attribute->removeEmpty) {

src/Attribute/Parameter/Trim.php

@@ -9,18 +9,30 @@
 /**
  * Strip whitespace (or other characters) from the beginning and end of a resolved string value.
  *
+ * In multibyte mode, Unicode whitespace characters, such as `U+00A0` (no-break space), are stripped as well.
+ * It requires the `mb_trim()` function provided by the `mbstring` PHP extension since PHP 8.4, or by the
+ * `symfony/polyfill-mbstring` package on earlier versions.
+ *
  * @see https://www.php.net/manual/function.trim.php
+ * @see https://www.php.net/manual/function.mb-trim.php
  */
 #[Attribute(Attribute::TARGET_PROPERTY | Attribute::TARGET_PARAMETER | Attribute::IS_REPEATABLE)]
 final class Trim implements ParameterAttributeInterface
 {
     /**
-     * @param string|null $characters The list all characters that you want to be stripped. With `..` you can specify
-     * a range of characters.
+     * @param string|null $characters The list of all characters that you want to be stripped. With `..` you can
+     * specify a range of characters, both in the default and in the multibyte mode. This syntax is deprecated and
+     * will be removed in the next major version.
+     * @param bool|null $multibyte Whether to use multibyte-aware trimming. `null` means using the resolver default.
+     * @param string|null $encoding The encoding to use in multibyte mode. `null` means using the resolver default.
      */
     public function __construct(
         public readonly ?string $characters = null,
-    ) {}
+        public readonly ?bool $multibyte = null,
+        public readonly ?string $encoding = null,
+    ) {
+        TrimCharacters::checkMultibyteFunctionsExist($multibyte);
+    }
 
     public function getResolver(): string
     {